Phase 3d: dirty-rect diffs for raster undo (bound undo-history RAM)

`RasterStrokeAction`/`RasterFillAction` stored the whole before+after RGBA frame
(~16 MB/action at 1080p → up to ~1.6 GB at the 100-action cap). They now store a
`RasterDiff` — only the changed bounding box's pixels before and after — computed
once in `new()` from the full buffers, which are then dropped. A brush dab shrinks
from ~16 MB to tens of KB; a full-canvas fill is unchanged (its bbox is the frame).

Paging interaction: a diff overwrites just the bbox, so the keyframe's pixels must
be resident when undo/redo applies. A clean evicted frame's container bytes equal
its current logical state, so the editor faults the target frame in (synchronously)
before undo/redo via a new `Action::raster_resident_hint` + `peek_undo/redo_raster_hint`.
Dirty frames are never evicted, so they're already resident. If a base is somehow
not resident the apply is skipped (logged), never resized-and-corrupted.

Unit tests cover exact before/after round-trip, blank-first-stroke, no-op, and the
non-resident-base skip.

lightningbeam-ui/lightningbeam-core/src/action.rs

@@ -71,6 +71,15 @@ pub trait Action: Send {
     /// Get a human-readable description of this action (for UI display)
     fn description(&self) -> String;
 
+    /// For raster actions that store dirty-rect diffs: the `(layer_id, time)` of the
+    /// keyframe whose full pixels must be resident before `execute`/`rollback` can
+    /// apply the diff. The editor faults the frame in (synchronously) before undo/redo
+    /// so a paged-out clean frame is restored to its container state first. Non-raster
+    /// actions (and full-buffer ones) return `None`.
+    fn raster_resident_hint(&self) -> Option<(Uuid, f64)> {
+        None
+    }
+
     /// Execute backend operations after document changes
     ///
     /// Called AFTER execute() succeeds. If this returns an error, execute()
@@ -290,6 +299,18 @@ impl ActionExecutor {
         self.undo_stack.last().map(|a| a.description())
     }
 
+    /// `(layer_id, time)` of the raster keyframe the next undo needs resident, if any.
+    /// The editor faults it in before calling `undo()` so a paged-out clean frame is
+    /// restored to its container state, giving the diff a correct base to apply onto.
+    pub fn peek_undo_raster_hint(&self) -> Option<(Uuid, f64)> {
+        self.undo_stack.last().and_then(|a| a.raster_resident_hint())
+    }
+
+    /// `(layer_id, time)` of the raster keyframe the next redo needs resident, if any.
+    pub fn peek_redo_raster_hint(&self) -> Option<(Uuid, f64)> {
+        self.redo_stack.last().and_then(|a| a.raster_resident_hint())
+    }
+
     /// Get MIDI cache data from the last action on the undo stack (after redo).
     /// Returns the notes reflecting execute state.
     pub fn last_undo_midi_notes(&self) -> Option<(u32, &[(f64, u8, u8, f64)])> {

lightningbeam-ui/lightningbeam-core/src/actions/mod.rs

@@ -34,6 +34,7 @@ pub mod convert_to_movie_clip;
 pub mod region_split;
 pub mod toggle_group_expansion;
 pub mod group_layers;
+pub mod raster_diff;
 pub mod raster_stroke;
 pub mod raster_fill;
 pub mod move_layer;

lightningbeam-ui/lightningbeam-core/src/actions/raster_diff.rs

@@ -0,0 +1,195 @@
+//! Dirty-rect diff for raster undo/redo.
+//!
+//! Brush strokes and fills used to store the *entire* before/after RGBA frame in the
+//! undo stack (~8 MB each at 1080p → up to ~1.6 GB at the 100-action cap). A
+//! [`RasterDiff`] instead stores only the changed bounding box's pixels before and
+//! after, which for a typical brush dab is a few tens of KB.
+//!
+//! Applying a diff overwrites just the bbox of the keyframe's `raw_pixels`, so the
+//! buffer **must be resident** (full length `w*h*4`) when `apply_*` runs. The editor
+//! guarantees this by faulting the target frame in before undo/redo (a clean evicted
+//! frame's container bytes equal its current logical state, so the restored base is
+//! correct). If the base is somehow not resident we skip rather than corrupt.
+
+/// Normalize a buffer to full length `n`; an empty/short buffer becomes transparent.
+fn normalize(buf: &[u8], n: usize) -> std::borrow::Cow<[u8]> {
+    if buf.len() == n {
+        std::borrow::Cow::Borrowed(buf)
+    } else {
+        std::borrow::Cow::Owned(vec![0u8; n])
+    }
+}
+
+/// A minimal before/after record of the region a raster edit changed.
+#[derive(Clone, Debug)]
+pub struct RasterDiff {
+    full_width: u32,
+    full_height: u32,
+    /// Changed bounding box `(x, y, w, h)`; `None` when before == after (no-op).
+    bbox: Option<(u32, u32, u32, u32)>,
+    /// bbox-sized RGBA (`w*h*4`) of the region before the edit.
+    before_region: Vec<u8>,
+    /// bbox-sized RGBA (`w*h*4`) of the region after the edit.
+    after_region: Vec<u8>,
+}
+
+impl RasterDiff {
+    /// Build a diff from full before/after buffers. `after` is expected to be the
+    /// resident post-edit buffer (`width*height*4`); `before` may be empty (a blank
+    /// keyframe's first stroke), treated as fully transparent.
+    pub fn compute(before: &[u8], after: &[u8], width: u32, height: u32) -> Self {
+        let n = width as usize * height as usize * 4;
+        // Normalize both sides to full length; empty/short ⇒ transparent.
+        let before_full = normalize(before, n);
+        let after_full = normalize(after, n);
+
+        // Find the tight bbox of differing pixels (compare 4-byte texels).
+        let (w, h) = (width as usize, height as usize);
+        let (mut min_x, mut min_y, mut max_x, mut max_y) = (usize::MAX, usize::MAX, 0usize, 0usize);
+        let mut any = false;
+        for y in 0..h {
+            let row = y * w * 4;
+            for x in 0..w {
+                let i = row + x * 4;
+                if before_full[i..i + 4] != after_full[i..i + 4] {
+                    any = true;
+                    if x < min_x { min_x = x; }
+                    if x > max_x { max_x = x; }
+                    if y < min_y { min_y = y; }
+                    if y > max_y { max_y = y; }
+                }
+            }
+        }
+
+        if !any {
+            return Self { full_width: width, full_height: height, bbox: None,
+                          before_region: Vec::new(), after_region: Vec::new() };
+        }
+
+        let bw = max_x - min_x + 1;
+        let bh = max_y - min_y + 1;
+        let crop = |full: &[u8]| -> Vec<u8> {
+            let mut out = Vec::with_capacity(bw * bh * 4);
+            for row in 0..bh {
+                let src = ((min_y + row) * w + min_x) * 4;
+                out.extend_from_slice(&full[src..src + bw * 4]);
+            }
+            out
+        };
+
+        Self {
+            full_width: width,
+            full_height: height,
+            bbox: Some((min_x as u32, min_y as u32, bw as u32, bh as u32)),
+            before_region: crop(&before_full),
+            after_region: crop(&after_full),
+        }
+    }
+
+    /// Approximate retained size in bytes (the two cropped regions).
+    pub fn byte_size(&self) -> usize {
+        self.before_region.len() + self.after_region.len()
+    }
+
+    /// Restore the pre-edit pixels into `raw` (undo).
+    pub fn apply_before(&self, raw: &mut Vec<u8>) {
+        self.apply(&self.before_region, raw);
+    }
+
+    /// Re-apply the post-edit pixels into `raw` (redo).
+    pub fn apply_after(&self, raw: &mut Vec<u8>) {
+        self.apply(&self.after_region, raw);
+    }
+
+    fn apply(&self, region: &[u8], raw: &mut Vec<u8>) {
+        let n = self.full_width as usize * self.full_height as usize * 4;
+        let (x, y, bw, bh) = match self.bbox {
+            Some(b) => b,
+            None => return, // no change
+        };
+        if raw.len() != n {
+            // Base not resident: the editor should have faulted it in. Skip rather
+            // than resize-and-corrupt — the frame will re-page to its container state.
+            eprintln!(
+                "⚠️ [RASTER_DIFF] base not resident ({} != {}); skipping undo/redo apply",
+                raw.len(), n
+            );
+            return;
+        }
+        let (x, y, bw, bh) = (x as usize, y as usize, bw as usize, bh as usize);
+        let fw = self.full_width as usize;
+        for row in 0..bh {
+            let dst = ((y + row) * fw + x) * 4;
+            let src = row * bw * 4;
+            raw[dst..dst + bw * 4].copy_from_slice(&region[src..src + bw * 4]);
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn solid(w: u32, h: u32, px: [u8; 4]) -> Vec<u8> {
+        px.iter().copied().cycle().take((w * h * 4) as usize).collect()
+    }
+
+    #[test]
+    fn roundtrip_reproduces_buffers_exactly() {
+        let (w, h) = (8, 6);
+        let before = solid(w, h, [10, 20, 30, 255]);
+        let mut after = before.clone();
+        // Change a 2x2 region at (3,2).
+        for (dy, dx) in [(0, 0), (0, 1), (1, 0), (1, 1)] {
+            let i = (((2 + dy) * w + (3 + dx)) * 4) as usize;
+            after[i..i + 4].copy_from_slice(&[200, 100, 50, 255]);
+        }
+        let diff = RasterDiff::compute(&before, &after, w, h);
+        assert_eq!(diff.bbox, Some((3, 2, 2, 2)));
+
+        let mut buf = after.clone();
+        diff.apply_before(&mut buf);
+        assert_eq!(buf, before, "undo must reproduce the pre-edit buffer exactly");
+        diff.apply_after(&mut buf);
+        assert_eq!(buf, after, "redo must reproduce the post-edit buffer exactly");
+    }
+
+    #[test]
+    fn blank_before_first_stroke() {
+        let (w, h) = (4, 4);
+        let before: Vec<u8> = Vec::new(); // blank keyframe
+        let mut after = vec![0u8; (w * h * 4) as usize];
+        let i = ((1 * w + 1) * 4) as usize;
+        after[i..i + 4].copy_from_slice(&[255, 0, 0, 255]);
+        let diff = RasterDiff::compute(&before, &after, w, h);
+        assert_eq!(diff.bbox, Some((1, 1, 1, 1)));
+
+        // Undo onto the resident post-stroke buffer → fully transparent (blank-equiv).
+        let mut buf = after.clone();
+        diff.apply_before(&mut buf);
+        assert_eq!(buf, vec![0u8; (w * h * 4) as usize]);
+    }
+
+    #[test]
+    fn no_change_is_noop() {
+        let (w, h) = (4, 4);
+        let buf = solid(w, h, [1, 2, 3, 4]);
+        let diff = RasterDiff::compute(&buf, &buf, w, h);
+        assert_eq!(diff.bbox, None);
+        assert_eq!(diff.byte_size(), 0);
+        let mut b = buf.clone();
+        diff.apply_before(&mut b);
+        assert_eq!(b, buf);
+    }
+
+    #[test]
+    fn not_resident_base_is_skipped_not_corrupted() {
+        let (w, h) = (4, 4);
+        let before = solid(w, h, [9, 9, 9, 255]);
+        let after = solid(w, h, [1, 2, 3, 255]);
+        let diff = RasterDiff::compute(&before, &after, w, h);
+        let mut empty: Vec<u8> = Vec::new();
+        diff.apply_before(&mut empty); // base not resident
+        assert!(empty.is_empty(), "must not resize/corrupt a non-resident base");
+    }
+}

lightningbeam-ui/lightningbeam-core/src/actions/raster_fill.rs

@@ -1,6 +1,7 @@
 //! Raster flood-fill action — records and undoes a paint bucket fill on a RasterLayer.
 
 use crate::action::Action;
+use crate::actions::raster_diff::RasterDiff;
 use crate::document::Document;
 use crate::layer::AnyLayer;
 use uuid::Uuid;
@@ -8,11 +9,10 @@ use uuid::Uuid;
 pub struct RasterFillAction {
     layer_id: Uuid,
     time: f64,
-    buffer_before: Vec<u8>,
-    buffer_after: Vec<u8>,
     width: u32,
     height: u32,
     name: String,
+    diff: RasterDiff,
 }
 
 impl RasterFillAction {
@@ -24,7 +24,8 @@ impl RasterFillAction {
         width: u32,
         height: u32,
     ) -> Self {
-        Self { layer_id, time, buffer_before, buffer_after, width, height, name: "Flood fill".to_string() }
+        let diff = RasterDiff::compute(&buffer_before, &buffer_after, width, height);
+        Self { layer_id, time, width, height, name: "Flood fill".to_string(), diff }
     }
 
     pub fn with_description(mut self, name: &str) -> Self {
@@ -42,7 +43,7 @@ impl Action for RasterFillAction {
             _ => return Err("Not a raster layer".to_string()),
         };
         let kf = raster.ensure_keyframe_at(self.time, self.width, self.height);
-        kf.raw_pixels = self.buffer_after.clone();
+        self.diff.apply_after(&mut kf.raw_pixels);
         kf.texture_dirty = true;
         kf.dirty = true;
         Ok(())
@@ -56,7 +57,7 @@ impl Action for RasterFillAction {
             _ => return Err("Not a raster layer".to_string()),
         };
         let kf = raster.ensure_keyframe_at(self.time, self.width, self.height);
-        kf.raw_pixels = self.buffer_before.clone();
+        self.diff.apply_before(&mut kf.raw_pixels);
         kf.texture_dirty = true;
         kf.dirty = true;
         Ok(())
@@ -65,4 +66,8 @@ impl Action for RasterFillAction {
     fn description(&self) -> String {
         self.name.clone()
     }
+
+    fn raster_resident_hint(&self) -> Option<(Uuid, f64)> {
+        Some((self.layer_id, self.time))
+    }
 }

lightningbeam-ui/lightningbeam-core/src/actions/raster_stroke.rs

@@ -9,6 +9,7 @@
 //! `rollback` → swap in `buffer_before`
 
 use crate::action::Action;
+use crate::actions::raster_diff::RasterDiff;
 use crate::document::Document;
 use crate::layer::AnyLayer;
 use uuid::Uuid;
@@ -16,16 +17,14 @@ use uuid::Uuid;
 /// Action that records a single brush stroke for undo/redo.
 ///
 /// The stroke must already be painted into the document's `raw_pixels` before
-/// this action is executed for the first time.
+/// this action is executed for the first time. Only the changed bounding box is
+/// retained (see [`RasterDiff`]) rather than two full frame buffers.
 pub struct RasterStrokeAction {
     layer_id: Uuid,
     time: f64,
-    /// Raw RGBA pixels *before* the stroke (for rollback / undo)
-    buffer_before: Vec<u8>,
-    /// Raw RGBA pixels *after* the stroke (for execute / redo)
-    buffer_after: Vec<u8>,
     width: u32,
     height: u32,
+    diff: RasterDiff,
 }
 
 impl RasterStrokeAction {
@@ -33,6 +32,8 @@ impl RasterStrokeAction {
     ///
     /// * `buffer_before` – raw RGBA pixels captured just before the stroke began.
     /// * `buffer_after`  – raw RGBA pixels captured just after the stroke finished.
+    ///
+    /// The full buffers are diffed down to the changed bbox here and then dropped.
     pub fn new(
         layer_id: Uuid,
         time: f64,
@@ -41,14 +42,15 @@ impl RasterStrokeAction {
         width: u32,
         height: u32,
     ) -> Self {
-        Self { layer_id, time, buffer_before, buffer_after, width, height }
+        let diff = RasterDiff::compute(&buffer_before, &buffer_after, width, height);
+        Self { layer_id, time, width, height, diff }
     }
 }
 
 impl Action for RasterStrokeAction {
     fn execute(&mut self, document: &mut Document) -> Result<(), String> {
         let kf = get_keyframe_mut(document, &self.layer_id, self.time, self.width, self.height)?;
-        kf.raw_pixels = self.buffer_after.clone();
+        self.diff.apply_after(&mut kf.raw_pixels);
         kf.texture_dirty = true;
         kf.dirty = true;
         Ok(())
@@ -56,7 +58,7 @@ impl Action for RasterStrokeAction {
 
     fn rollback(&mut self, document: &mut Document) -> Result<(), String> {
         let kf = get_keyframe_mut(document, &self.layer_id, self.time, self.width, self.height)?;
-        kf.raw_pixels = self.buffer_before.clone();
+        self.diff.apply_before(&mut kf.raw_pixels);
         kf.texture_dirty = true;
         kf.dirty = true;
         Ok(())
@@ -65,6 +67,10 @@ impl Action for RasterStrokeAction {
     fn description(&self) -> String {
         "Paint stroke".to_string()
     }
+
+    fn raster_resident_hint(&self) -> Option<(Uuid, f64)> {
+        Some((self.layer_id, self.time))
+    }
 }
 
 fn get_keyframe_mut<'a>(

lightningbeam-ui/lightningbeam-editor/src/main.rs

@@ -2324,6 +2324,39 @@ impl EditorApp {
 
     /// Cancel a floating raster selection: restore the canvas from the
     /// pre-cut/paste snapshot.  No undo entry is created.
+    /// Fault in the raster keyframe that a pending undo/redo needs resident, so its
+    /// dirty-rect diff applies onto the correct full base. A clean evicted frame's
+    /// container bytes equal its current logical state, so this restores the right
+    /// base. Synchronous — undo/redo is a discrete user action, not a hot path.
+    fn ensure_raster_resident_for_undo(&mut self, hint: Option<(uuid::Uuid, f64)>) {
+        use lightningbeam_core::layer::AnyLayer;
+        let Some((layer_id, time)) = hint else { return };
+        if !self.raster_store.has_path() { return; }
+        // Identify the (paged-out) keyframe the action will touch.
+        let target = {
+            let doc = self.action_executor.document();
+            match doc.get_layer(&layer_id) {
+                Some(AnyLayer::Raster(rl)) => rl
+                    .keyframe_at(time)
+                    .filter(|kf| kf.raw_pixels.is_empty() && kf.needs_fault_in)
+                    .map(|kf| kf.id),
+                _ => None,
+            }
+        };
+        if let Some(kf_id) = target {
+            if let Some(pixels) = self.raster_store.load_pixels(kf_id) {
+                let doc = self.action_executor.document_mut();
+                if let Some(AnyLayer::Raster(rl)) = doc.get_layer_mut(&layer_id) {
+                    if let Some(kf) = rl.keyframes.iter_mut().find(|k| k.id == kf_id) {
+                        kf.raw_pixels = pixels;
+                        kf.texture_dirty = true;
+                        kf.needs_fault_in = false;
+                    }
+                }
+            }
+        }
+    }
+
     fn cancel_raster_floating(&mut self) {
         use lightningbeam_core::layer::AnyLayer;
 
@@ -3282,6 +3315,10 @@ impl EditorApp {
                     self.cancel_raster_floating();
                     return;
                 }
+                // Page in the target raster frame first so its dirty-rect diff has a
+                // resident base to apply onto.
+                let hint = self.action_executor.peek_undo_raster_hint();
+                self.ensure_raster_resident_for_undo(hint);
                 let undo_succeeded = if let Some(ref controller_arc) = self.audio_controller {
                     let mut controller = controller_arc.lock().unwrap();
                     let mut backend_context = lightningbeam_core::action::BackendContext {
@@ -3331,6 +3368,8 @@ impl EditorApp {
                 }
             }
             MenuAction::Redo => {
+                let hint = self.action_executor.peek_redo_raster_hint();
+                self.ensure_raster_resident_for_undo(hint);
                 let redo_succeeded = if let Some(ref controller_arc) = self.audio_controller {
                     let mut controller = controller_arc.lock().unwrap();
                     let mut backend_context = lightningbeam_core::action::BackendContext {