Phase 3a-1: lazy fault-in of raster keyframe pixels

Raster keyframes are no longer eagerly decoded at load — `raw_pixels` stays empty
and is paged in on demand from the project container, so a big paint project opens
instantly and only touched frames hit RAM.

- core: `read_packed_media_readonly` (fresh read-only connection, can't conflict
  with an in-place save) + `RasterStore` (holds the container path; `load_pixels`
  reads+decodes a keyframe's PNG by id). `load_beam_sqlite` stops eager-decoding and
  instead marks every raster keyframe `needs_fault_in` (recursively, incl. nested);
  a freshly-created keyframe stays false (blank-resident, nothing to page). Added
  `Document::all_layers_mut`.
- editor: the canvas records a fault-in request when it needs a paged-out keyframe
  (empty pixels && needs_fault_in); the App drains the sink at the top of update(),
  pages the pixels in via the store, clears the flag, and repaints. Store path is set
  on load and after save. Export faults in synchronously per frame.

Cold-scrub still shows a 1-frame gap and the page-in is synchronous; the image proxy
(3a-2) and async load (3a-3) remove those next.

lightningbeam-ui/lightningbeam-core/src/beam_archive.rs

@@ -766,6 +766,32 @@ fn set_meta_conn(conn: &Connection, key: &str, value: &str) -> Result<(), String
     Ok(())
 }
 
+/// Read a packed media item's full bytes via a fresh **read-only** connection,
+/// without an open [`BeamArchive`] handle. Returns `Ok(None)` if the item has no
+/// chunk rows. Used for on-demand raster fault-in: a read-only connection can't
+/// conflict with an in-place save and needs no long-lived handle.
+pub fn read_packed_media_readonly(db_path: &Path, id: Uuid) -> Result<Option<Vec<u8>>, String> {
+    let conn = Connection::open_with_flags(
+        db_path,
+        OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX,
+    )
+    .map_err(map_sql)?;
+    let id_bytes = id.as_bytes().to_vec();
+    let mut stmt = conn
+        .prepare("SELECT bytes FROM media_chunk WHERE media_id = ?1 ORDER BY chunk_index")
+        .map_err(map_sql)?;
+    let rows = stmt
+        .query_map([&id_bytes], |r| r.get::<_, Vec<u8>>(0))
+        .map_err(map_sql)?;
+    let mut out = Vec::new();
+    let mut any = false;
+    for row in rows {
+        out.extend_from_slice(&row.map_err(map_sql)?);
+        any = true;
+    }
+    Ok(if any { Some(out) } else { None })
+}
+
 fn map_sql(e: rusqlite::Error) -> String {
     format!("SQLite error: {}", e)
 }

lightningbeam-ui/lightningbeam-core/src/document.rs

@@ -650,6 +650,29 @@ impl Document {
         layers
     }
 
+    /// Get mutable references to all layers across the entire document
+    /// (root + nested groups + inside all vector clips). Mirrors [`all_layers`].
+    pub fn all_layers_mut(&mut self) -> Vec<&mut AnyLayer> {
+        let mut layers: Vec<&mut AnyLayer> = Vec::new();
+        // Iterative walk with an explicit stack of child slices. Group layers are
+        // descended into but not themselves collected (they hold no keyframes).
+        let mut stack: Vec<&mut [AnyLayer]> = vec![&mut self.root.children];
+        while let Some(list) = stack.pop() {
+            for layer in list {
+                match layer {
+                    AnyLayer::Group(g) => {
+                        stack.push(&mut g.children);
+                    }
+                    other => layers.push(other),
+                }
+            }
+        }
+        for clip in self.vector_clips.values_mut() {
+            layers.extend(clip.layers.root_data_mut());
+        }
+        layers
+    }
+
     // === CLIP LIBRARY METHODS ===
 
     /// Add a vector clip to the library

lightningbeam-ui/lightningbeam-core/src/file_io.rs

@@ -561,24 +561,19 @@ fn load_beam_sqlite(path: &Path) -> Result<LoadedProject, String> {
         restored_entries.push(e);
     }
 
-    // Raster keyframes: load PNG bytes from media rows into raw_pixels.
+    // Raster keyframes are NOT eagerly decoded (Phase 3 paging): `raw_pixels` stays
+    // empty and is faulted in on demand from the container's `Raster` rows via the
+    // editor's `RasterStore` (keyed by `kf.id`). Loading a big paint project is now
+    // instant and only the resident window lives in RAM. Mark every keyframe
+    // `needs_fault_in` (recursively, incl. nested layers) so the renderer requests a
+    // page-in; a freshly-created keyframe stays `false` (blank-resident, nothing to load).
     let mut raster_load_count = 0usize;
-    for layer in document.root.children.iter_mut() {
+    for layer in document.all_layers_mut() {
         if let crate::layer::AnyLayer::Raster(rl) = layer {
             for kf in &mut rl.keyframes {
-                if let Ok(Some(_)) = archive.media_info(kf.id) {
+                kf.needs_fault_in = true;
-                    match archive.read_media_full(kf.id) {
-                        Ok(png_bytes) => match crate::brush_engine::decode_png(&png_bytes) {
-                            Ok(rgba) => {
-                                kf.raw_pixels = rgba.into_raw();
                 raster_load_count += 1;
             }
-                            Err(e) => eprintln!("⚠️ [LOAD_BEAM] Failed to decode raster {}: {}", kf.id, e),
-                        },
-                        Err(e) => eprintln!("⚠️ [LOAD_BEAM] Failed to read raster {}: {}", kf.id, e),
-                    }
-                }
-            }
         }
     }
 

lightningbeam-ui/lightningbeam-core/src/lib.rs

@@ -54,6 +54,7 @@ pub mod svg_export;
 pub mod snap;
 pub mod webcam;
 pub mod raster_layer;
+pub mod raster_store;
 pub mod brush_settings;
 pub mod brush_engine;
 pub mod raster_draw;

lightningbeam-ui/lightningbeam-core/src/raster_layer.rs

@@ -117,6 +117,12 @@ pub struct RasterKeyframe {
     /// Always `true` after load; cleared by the renderer after uploading.
     #[serde(skip, default = "default_true")]
     pub texture_dirty: bool,
+    /// Phase 3 paging: the keyframe's pixels live in the container and must be
+    /// faulted in (`raw_pixels` empty *and* this true ⇒ page in from the store).
+    /// A *new* keyframe is `false` (intentionally blank/resident, nothing to load);
+    /// set true on load and again when evicted. Never serialized.
+    #[serde(skip)]
+    pub needs_fault_in: bool,
 }
 
 fn default_true() -> bool { true }
@@ -140,6 +146,7 @@ impl RasterKeyframe {
             tween_after: TweenType::Hold,
             raw_pixels: Vec::new(),
             texture_dirty: true,
+            needs_fault_in: false,
         }
     }
 }

lightningbeam-ui/lightningbeam-core/src/raster_store.rs

@@ -0,0 +1,56 @@
+//! On-demand loader for raster keyframe pixels backed by the project `.beam`
+//! container (Phase 3 paging).
+//!
+//! Raster keyframes are no longer eagerly decoded at load; `raw_pixels` stays
+//! empty until something needs the frame, then it is faulted in from the
+//! container's `Raster` media row (keyed by the keyframe id). The store holds only
+//! the container path and reads through a fresh **read-only** connection per call,
+//! so it never conflicts with an in-place save and keeps no long-lived handle.
+//! `None` path = an unsaved document (nothing to fault in).
+
+use std::path::PathBuf;
+use uuid::Uuid;
+
+/// Faults in raster keyframe pixels from the project container on demand.
+#[derive(Default, Clone)]
+pub struct RasterStore {
+    path: Option<PathBuf>,
+}
+
+impl RasterStore {
+    pub fn new(path: Option<PathBuf>) -> Self {
+        Self { path }
+    }
+
+    /// Point the store at a (possibly new) container path, or `None` for an
+    /// unsaved document. Call on load and on save-as.
+    pub fn set_path(&mut self, path: Option<PathBuf>) {
+        self.path = path;
+    }
+
+    pub fn has_path(&self) -> bool {
+        self.path.is_some()
+    }
+
+    /// Decode the keyframe's full RGBA pixels from the container, or `None` if the
+    /// container has no row for it (or decoding fails). The returned buffer is the
+    /// working `raw_pixels` representation (`width*height*4` sRGB-premultiplied RGBA).
+    pub fn load_pixels(&self, kf_id: Uuid) -> Option<Vec<u8>> {
+        let path = self.path.as_ref()?;
+        let png = match crate::beam_archive::read_packed_media_readonly(path, kf_id) {
+            Ok(Some(bytes)) => bytes,
+            Ok(None) => return None,
+            Err(e) => {
+                eprintln!("[RasterStore] read {} failed: {}", kf_id, e);
+                return None;
+            }
+        };
+        match crate::brush_engine::decode_png(&png) {
+            Ok(img) => Some(img.into_raw()),
+            Err(e) => {
+                eprintln!("[RasterStore] decode {} failed: {}", kf_id, e);
+                None
+            }
+        }
+    }
+}

lightningbeam-ui/lightningbeam-editor/src/export/mod.rs

@@ -528,6 +528,7 @@ impl ExportOrchestrator {
         image_cache: &mut ImageCache,
         video_manager: &Arc<std::sync::Mutex<VideoManager>>,
         floating_selection: Option<&lightningbeam_core::selection::RasterFloatingSelection>,
+        raster_store: Option<&lightningbeam_core::raster_store::RasterStore>,
     ) -> Result<bool, String> {
         if self.cancel_flag.load(Ordering::Relaxed) {
             self.image_state = None;
@@ -575,6 +576,7 @@ impl ExportOrchestrator {
                 output_view,
                 floating_selection,
                 state.settings.allow_transparency,
+                raster_store,
             )?;
             queue.submit(Some(encoder.finish()));
 
@@ -1029,6 +1031,7 @@ impl ExportOrchestrator {
         renderer: &mut vello::Renderer,
         image_cache: &mut ImageCache,
         video_manager: &Arc<std::sync::Mutex<VideoManager>>,
+        raster_store: Option<&lightningbeam_core::raster_store::RasterStore>,
     ) -> Result<bool, String> {
         use std::time::Instant;
 
@@ -1126,6 +1129,7 @@ impl ExportOrchestrator {
                     gpu_resources, &acquired.rgba_texture_view,
                     None,  // No floating selection during video export
                     false, // Video export is never transparent
+                    raster_store,
                 )?;
                 let render_end = Instant::now();
 

lightningbeam-ui/lightningbeam-editor/src/export/video_exporter.rs

@@ -1179,6 +1179,39 @@ pub fn render_frame_to_rgba_hdr(
 ///
 /// # Returns
 /// Command encoder ready for submission (caller submits via ReadbackPipeline)
+/// Fault in raster keyframe pixels needed to composite the document at its current
+/// time, decoding them from the project `.beam` container via `raster_store`.
+///
+/// Mutates the document in place: for every raster layer's active keyframe whose
+/// `raw_pixels` are empty, loads + sets them (and marks `texture_dirty`). A no-op
+/// when `raster_store` is `None`/unsaved or everything is already resident.
+fn fault_in_raster_for_frame(
+    document: &mut Document,
+    raster_store: Option<&lightningbeam_core::raster_store::RasterStore>,
+) {
+    let store = match raster_store {
+        Some(s) if s.has_path() => s,
+        _ => return,
+    };
+    let now = document.current_time;
+    for layer in document.all_layers_mut() {
+        if let lightningbeam_core::layer::AnyLayer::Raster(rl) = layer {
+            // Resolve the active keyframe id at the current time, then fault it in.
+            let kf_id = match rl.keyframe_at(now) {
+                Some(kf) if kf.raw_pixels.is_empty() && kf.needs_fault_in => kf.id,
+                _ => continue,
+            };
+            if let Some(kf) = rl.keyframes.iter_mut().find(|kf| kf.id == kf_id) {
+                if let Some(pixels) = store.load_pixels(kf_id) {
+                    kf.raw_pixels = pixels;
+                    kf.texture_dirty = true;
+                }
+                kf.needs_fault_in = false;
+            }
+        }
+    }
+}
+
 pub fn render_frame_to_gpu_rgba(
     document: &mut Document,
     timestamp: f64,
@@ -1193,12 +1226,19 @@ pub fn render_frame_to_gpu_rgba(
     rgba_texture_view: &wgpu::TextureView,
     floating_selection: Option<&lightningbeam_core::selection::RasterFloatingSelection>,
     allow_transparency: bool,
+    raster_store: Option<&lightningbeam_core::raster_store::RasterStore>,
 ) -> Result<wgpu::CommandEncoder, String> {
     use vello::kurbo::Affine;
 
     // Set document time to the frame timestamp
     document.current_time = timestamp;
 
+    // Fault in raster keyframe pixels for this frame (Phase 3 paging). Offline
+    // export renders synchronously with no "next frame", so unlike the live canvas
+    // we must page the pixels in here, before compositing. Cheap no-op when every
+    // keyframe is already resident or when the document is unsaved (no store path).
+    fault_in_raster_for_frame(document, raster_store);
+
     // Scale the document to the export resolution. The core renderer bakes this
     // base transform into every layer (vector scenes, raster and video layer
     // transforms), so the whole stage scales up/down to fill the output. When the

lightningbeam-ui/lightningbeam-editor/src/main.rs

@@ -1026,6 +1026,12 @@ struct EditorApp {
     recording_mirror_rx: Option<rtrb::Consumer<f32>>,
     /// Current file path (None if not yet saved)
     current_file_path: Option<std::path::PathBuf>,
+    /// On-demand loader for raster keyframe pixels from the project `.beam` container.
+    raster_store: lightningbeam_core::raster_store::RasterStore,
+    /// Miss-sink: raster keyframe ids the canvas wanted but whose pixels weren't
+    /// resident. Drained and faulted in at the top of the next `update()`.
+    raster_fault_requests:
+        std::sync::Arc<std::sync::Mutex<std::collections::HashSet<uuid::Uuid>>>,
     /// Application configuration (recent files, etc.)
     config: AppConfig,
     /// Remappable keyboard shortcut manager
@@ -1289,6 +1295,8 @@ impl EditorApp {
             waveform_result_tx,
             recording_mirror_rx,
             current_file_path: None, // No file loaded initially
+            raster_store: lightningbeam_core::raster_store::RasterStore::new(None),
+            raster_fault_requests: Default::default(),
             keymap: KeymapManager::new(&config.keybindings),
             config,
             file_command_tx,
@@ -4200,6 +4208,8 @@ impl EditorApp {
         self.playback_time = 0.0;
         self.is_playing = false;
         self.current_file_path = Some(path.clone());
+        // Point the raster paging store at the loaded container so faulting works.
+        self.raster_store.set_path(self.current_file_path.clone());
 
         // Add to recent files
         self.config.add_recent_file(path.clone());
@@ -5133,6 +5143,46 @@ impl eframe::App for EditorApp {
     fn update(&mut self, ctx: &egui::Context, frame: &mut eframe::Frame) {
         let _frame_start = std::time::Instant::now();
 
+        // === Raster fault-in (Phase 3 paging) ===
+        // The canvas records raster keyframe ids whose `raw_pixels` weren't resident
+        // (it can't mutate the document while rendering). Drain that sink here, BEFORE
+        // any rendering, and fault the pixels in from the project container. A drain of
+        // an empty set is ~free, so this is cheap when everything is already resident.
+        {
+            let wanted: Vec<uuid::Uuid> = {
+                let mut s = self.raster_fault_requests.lock().unwrap();
+                s.drain().collect()
+            };
+            if !wanted.is_empty() && self.raster_store.has_path() {
+                let mut any_loaded = false;
+                let doc = self.action_executor.document_mut();
+                for kf_id in wanted {
+                    // Find the keyframe by id across every raster layer in the document.
+                    let kf = doc.all_layers_mut().into_iter().find_map(|l| match l {
+                        lightningbeam_core::layer::AnyLayer::Raster(rl) => {
+                            rl.keyframes.iter_mut().find(|kf| kf.id == kf_id)
+                        }
+                        _ => None,
+                    });
+                    if let Some(kf) = kf {
+                        if kf.raw_pixels.is_empty() && kf.needs_fault_in {
+                            if let Some(pixels) = self.raster_store.load_pixels(kf_id) {
+                                kf.raw_pixels = pixels;
+                                kf.texture_dirty = true;
+                                any_loaded = true;
+                            }
+                            // Resolved (loaded, or genuinely absent/blank) — clear the
+                            // flag so we don't re-request a page-in every frame.
+                            kf.needs_fault_in = false;
+                        }
+                    }
+                }
+                if any_loaded {
+                    ctx.request_repaint();
+                }
+            }
+        }
+
         // Consume any pending tool switch from the tablet (eraser in/out).
         if let Some(tool) = self.tablet.pending_tool_switch.take() {
             self.selected_tool = tool;
@@ -5274,6 +5324,9 @@ impl eframe::App for EditorApp {
                             FileProgress::Done => {
                                 println!("✅ Save complete!");
                                 self.current_file_path = Some(path.clone());
+                                // Container path may be new (Save As); update the
+                                // raster paging store so future faults read the right file.
+                                self.raster_store.set_path(self.current_file_path.clone());
 
                                 // Add to recent files
                                 self.config.add_recent_file(path.clone());
@@ -6002,6 +6055,7 @@ impl eframe::App for EditorApp {
                             renderer,
                             &mut temp_image_cache,
                             &self.video_manager,
+                            Some(&self.raster_store),
                         ) {
                             if has_more {
                                 ctx.request_repaint();
@@ -6017,6 +6071,7 @@ impl eframe::App for EditorApp {
                             &mut temp_image_cache,
                             &self.video_manager,
                             self.selection.raster_floating.as_ref(),
+                            Some(&self.raster_store),
                         ) {
                             Ok(false) => { ctx.request_repaint(); } // readback pending
                             Ok(true)  => {}                          // done or cancelled
@@ -6287,6 +6342,7 @@ impl eframe::App for EditorApp {
                     raw_audio_cache: &self.raw_audio_cache,
                     waveform_gpu_dirty: &mut self.waveform_gpu_dirty,
                     waveform_minmax_pools: &self.waveform_minmax_pools,
+                    raster_fault_requests: &self.raster_fault_requests,
                     effect_to_load: &mut self.effect_to_load,
                     effect_thumbnail_requests: &mut effect_thumbnail_requests,
                     effect_thumbnail_cache: self.effect_thumbnail_generator.as_ref()

lightningbeam-ui/lightningbeam-editor/src/panes/mod.rs

@@ -246,6 +246,12 @@ pub struct SharedPaneState<'a> {
     /// raw samples (pool_index -> `B`, floor frames-per-texel). Drives the GPU
     /// min/max upload path and the floor's effective rate `sr/B` in the renderer.
     pub waveform_minmax_pools: &'a std::collections::HashMap<usize, u32>,
+    /// Miss-sink for on-demand raster keyframe pixel faulting (Phase 3 paging).
+    /// The canvas inserts the id of any raster keyframe it wants to upload whose
+    /// `raw_pixels` aren't resident; the App drains this at the top of the next
+    /// `update()` and faults the pixels in from the project container.
+    pub raster_fault_requests:
+        &'a std::sync::Arc<std::sync::Mutex<std::collections::HashSet<uuid::Uuid>>>,
     /// Effect ID to load into shader editor (set by asset library, consumed by shader editor)
     pub effect_to_load: &'a mut Option<Uuid>,
     /// Queue for effect thumbnail requests (effect IDs to generate thumbnails for)

lightningbeam-ui/lightningbeam-editor/src/panes/stage.rs

@@ -523,6 +523,12 @@ struct VelloRenderContext {
     /// When `Some`, readback this B-canvas into `RASTER_READBACK_RESULTS` after
     /// dispatching GPU tool work.  Set on mouseup by the unified raster tool commit path.
     pending_tool_readback_b: Option<uuid::Uuid>,
+    /// Miss-sink for on-demand raster keyframe pixel faulting (Phase 3 paging).
+    /// When the compositor wants to upload an idle raster keyframe whose `raw_pixels`
+    /// aren't resident, it inserts the keyframe id here; the App drains this at the
+    /// top of the next `update()` and faults the pixels in from the project container.
+    raster_fault_requests:
+        std::sync::Arc<std::sync::Mutex<std::collections::HashSet<uuid::Uuid>>>,
 }
 
 /// Callback for Vello rendering within egui
@@ -1206,6 +1212,15 @@ impl egui_wgpu::CallbackTrait for VelloCallback {
                                     );
                                     Some(kf_id)
                                 } else {
+                                    // Empty pixels: if the frame is paged out (lives in the
+                                    // container), record a fault-in request so the App pages it
+                                    // in at the top of the next frame. A new blank keyframe
+                                    // (needs_fault_in == false) has nothing to load — skip it.
+                                    if kf.needs_fault_in {
+                                        if let Ok(mut reqs) = self.ctx.raster_fault_requests.lock() {
+                                            reqs.insert(kf_id);
+                                        }
+                                    }
                                     None
                                 }
                             } else {
@@ -11902,6 +11917,7 @@ impl PaneRenderer for StagePane {
                 .and_then(|(tool, _)| tool.take_pending_gpu_work()),
             pending_layer_cache_removals: std::mem::take(&mut self.pending_layer_cache_removals),
             pending_tool_readback_b: self.pending_tool_readback_b.take(),
+            raster_fault_requests: std::sync::Arc::clone(shared.raster_fault_requests),
         }};
 
         let cb = egui_wgpu::Callback::new_paint_callback(