use channel and jpeg compression to speed up playback

src-tauri/Cargo.lock

@@ -635,6 +635,12 @@ dependencies = [
  "objc",
 ]
 
+[[package]]
+name = "color_quant"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b"
+
 [[package]]
 name = "combine"
 version = "4.6.7"
@@ -2197,6 +2203,19 @@ dependencies = [
  "icu_properties",
 ]
 
+[[package]]
+name = "image"
+version = "0.24.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5690139d2f55868e080017335e4b94cb7414274c74f1669c84fb5feba2c9f69d"
+dependencies = [
+ "bytemuck",
+ "byteorder",
+ "color_quant",
+ "jpeg-decoder",
+ "num-traits",
+]
+
 [[package]]
 name = "indexmap"
 version = "1.9.3"
@@ -2337,6 +2356,12 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "jpeg-decoder"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "00810f1d8b74be64b13dbf3db89ac67740615d6c891f0e7b6179326533011a07"
+
 [[package]]
 name = "js-sys"
 version = "0.3.77"
@@ -2467,6 +2492,7 @@ dependencies = [
  "cpal",
  "daw-backend",
  "ffmpeg-next",
+ "image",
  "log",
  "lru",
  "rtrb",

src-tauri/Cargo.toml

@@ -39,6 +39,7 @@ rtrb = "0.3"
 # Video decoding
 ffmpeg-next = "7.0"
 lru = "0.12"
+image = { version = "0.24", default-features = false, features = ["jpeg"] }
 
 [profile.dev]
 opt-level = 1  # Enable basic optimizations in debug mode for audio decoding performance

src-tauri/src/video.rs

@@ -1,8 +1,10 @@
 use std::sync::{Arc, Mutex};
 use std::num::NonZeroUsize;
+use std::io::Cursor;
 use ffmpeg_next as ffmpeg;
 use lru::LruCache;
 use daw_backend::WaveformPeak;
+use image::{RgbaImage, ImageEncoder};
 
 #[derive(serde::Serialize, Clone)]
 pub struct VideoFileMetadata {
@@ -453,16 +455,16 @@ fn generate_waveform(audio_data: &[f32], channels: u32, target_peaks: usize) ->
     waveform
 }
 
-// Use a custom serializer wrapper for efficient binary transfer
-#[derive(serde::Serialize)]
-struct BinaryFrame(#[serde(with = "serde_bytes")] Vec<u8>);
-
 #[tauri::command]
 pub async fn video_get_frame(
     state: tauri::State<'_, Arc<Mutex<VideoState>>>,
     pool_index: usize,
     timestamp: f64,
-) -> Result<Vec<u8>, String> {
+    use_jpeg: bool,
+    channel: tauri::ipc::Channel,
+) -> Result<(), String> {
+    use std::time::Instant;
+
     let video_state = state.lock().unwrap();
 
     let decoder = video_state.pool.get(pool_index)
@@ -472,7 +474,51 @@ pub async fn video_get_frame(
     drop(video_state);
 
     let mut decoder = decoder.lock().unwrap();
-    decoder.get_frame(timestamp)
+    let frame_data = decoder.get_frame(timestamp)?;
+
+    let data_to_send = if use_jpeg {
+        let t_compress_start = Instant::now();
+
+        // Get frame dimensions from decoder
+        let width = decoder.output_width;
+        let height = decoder.output_height;
+
+        // Create image from raw RGBA data
+        let img = RgbaImage::from_raw(width, height, frame_data)
+            .ok_or("Failed to create image from frame data")?;
+
+        // Convert RGBA to RGB (JPEG doesn't support alpha)
+        let rgb_img = image::DynamicImage::ImageRgba8(img).to_rgb8();
+
+        // Encode to JPEG with quality 85 (good balance of size/quality)
+        let mut jpeg_data = Vec::new();
+        let mut encoder = image::codecs::jpeg::JpegEncoder::new_with_quality(&mut jpeg_data, 85);
+        encoder.encode(
+            rgb_img.as_raw(),
+            rgb_img.width(),
+            rgb_img.height(),
+            image::ColorType::Rgb8
+        ).map_err(|e| format!("JPEG encoding failed: {}", e))?;
+
+        let compress_time = t_compress_start.elapsed().as_millis();
+        let original_size = width as usize * height as usize * 4;
+        let compressed_size = jpeg_data.len();
+        let ratio = original_size as f32 / compressed_size as f32;
+
+        eprintln!("[Video JPEG] Compressed {}KB -> {}KB ({}x) in {}ms",
+            original_size / 1024, compressed_size / 1024, ratio, compress_time);
+
+        jpeg_data
+    } else {
+        frame_data
+    };
+
+    // Send binary data through channel (bypasses JSON serialization)
+    // InvokeResponseBody::Raw sends raw binary data without JSON encoding
+    channel.send(tauri::ipc::InvokeResponseBody::Raw(data_to_send))
+        .map_err(|e| format!("Channel send error: {}", e))?;
+
+    Ok(())
 }
 
 #[tauri::command]

src/models/layer.js

@@ -18,7 +18,7 @@ import {
 const Tone = window.Tone;
 
 // Tauri API
-const { invoke } = window.__TAURI__.core;
+const { invoke, Channel } = window.__TAURI__.core;
 
 // Helper function for UUID generation
 function uuidv4() {
@@ -1280,6 +1280,9 @@ class VideoLayer extends Widget {
     // Associated audio track (if video has audio)
     this.linkedAudioTrack = null;  // Reference to AudioTrack
 
+    // Performance settings
+    this.useJpegCompression = true;  // Enable JPEG compression for faster transfer (default: true)
+
     // Timeline display
     this.collapsed = false;
     this.curvesMode = 'segment';
@@ -1343,34 +1346,76 @@ class VideoLayer extends Widget {
         clip.lastFetchedTimestamp = videoTimestamp;
 
         try {
-          // Request frame from Rust backend
+          // Request frame from Rust backend using IPC Channel for efficient binary transfer
           const t_start = performance.now();
-          let frameData = await invoke('video_get_frame', {
+
-            poolIndex: clip.poolIndex,
+          // Create a promise that resolves when channel receives data
-            timestamp: videoTimestamp
+          const frameDataPromise = new Promise((resolve, reject) => {
+            const channel = new Channel();
+
+            channel.onmessage = (data) => {
+              resolve(data);
+            };
+
+            // Invoke command with channel
+            invoke('video_get_frame', {
+              poolIndex: clip.poolIndex,
+              timestamp: videoTimestamp,
+              useJpeg: this.useJpegCompression,
+              channel: channel
+            }).catch(reject);
           });
+
+          // Wait for the frame data
+          let frameData = await frameDataPromise;
           const t_after_ipc = performance.now();
 
-          // Handle different formats that Tauri might return
+          // Ensure data is Uint8Array
-          // ByteBuf from Rust can come as Uint8Array or Array depending on serialization
           if (!(frameData instanceof Uint8Array)) {
             frameData = new Uint8Array(frameData);
           }
 
-          // Validate frame data size
+          let imageData;
-          const expectedSize = clip.width * clip.height * 4; // RGBA = 4 bytes per pixel
+          const t_before_conversion = performance.now();
 
-          if (frameData.length !== expectedSize) {
+          if (this.useJpegCompression) {
-            throw new Error(`Invalid frame data size: got ${frameData.length}, expected ${expectedSize}`);
+            // Decode JPEG data
+            const blob = new Blob([frameData], { type: 'image/jpeg' });
+            const imageUrl = URL.createObjectURL(blob);
+
+            // Load and decode JPEG
+            const img = new Image();
+            await new Promise((resolve, reject) => {
+              img.onload = resolve;
+              img.onerror = reject;
+              img.src = imageUrl;
+            });
+
+            // Create temporary canvas to extract ImageData
+            const tempCanvas = document.createElement('canvas');
+            tempCanvas.width = clip.width;
+            tempCanvas.height = clip.height;
+            const tempCtx = tempCanvas.getContext('2d');
+            tempCtx.drawImage(img, 0, 0);
+            imageData = tempCtx.getImageData(0, 0, clip.width, clip.height);
+
+            // Cleanup
+            URL.revokeObjectURL(imageUrl);
+          } else {
+            // Raw RGBA data
+            const expectedSize = clip.width * clip.height * 4; // RGBA = 4 bytes per pixel
+
+            if (frameData.length !== expectedSize) {
+              throw new Error(`Invalid frame data size: got ${frameData.length}, expected ${expectedSize}`);
+            }
+
+            imageData = new ImageData(
+              new Uint8ClampedArray(frameData),
+              clip.width,
+              clip.height
+            );
           }
 
-          // Convert to ImageData
-          const t_before_conversion = performance.now();
-          const imageData = new ImageData(
-            new Uint8ClampedArray(frameData),
-            clip.width,
-            clip.height
-          );
           const t_after_conversion = performance.now();
 
           // Create or reuse temp canvas
@@ -1392,8 +1437,9 @@ class VideoLayer extends Widget {
           const ipc_time = t_after_ipc - t_start;
           const conversion_time = t_after_conversion - t_before_conversion;
           const putimage_time = t_after_putimage - t_before_putimage;
+          const compression_mode = this.useJpegCompression ? 'JPEG' : 'RAW';
 
-          console.log(`[JS Video Timing] ts=${videoTimestamp.toFixed(3)}s | Total: ${total_time.toFixed(1)}ms | IPC: ${ipc_time.toFixed(1)}ms (${(ipc_time/total_time*100).toFixed(0)}%) | Convert: ${conversion_time.toFixed(1)}ms | PutImage: ${putimage_time.toFixed(1)}ms | Size: ${(frameData.length/1024/1024).toFixed(2)}MB`);
+          console.log(`[JS Video Timing ${compression_mode}] ts=${videoTimestamp.toFixed(3)}s | Total: ${total_time.toFixed(1)}ms | IPC: ${ipc_time.toFixed(1)}ms (${(ipc_time/total_time*100).toFixed(0)}%) | Convert: ${conversion_time.toFixed(1)}ms | PutImage: ${putimage_time.toFixed(1)}ms | Size: ${(frameData.length/1024/1024).toFixed(2)}MB`);
         } catch (error) {
           console.error('Failed to get video frame:', error);
           clip.currentFrame = null;
@@ -1479,6 +1525,11 @@ class VideoLayer extends Widget {
     videoLayer.visible = json.visible;
     videoLayer.audible = json.audible;
 
+    // Restore compression setting (default to true if not specified for backward compatibility)
+    if (json.useJpegCompression !== undefined) {
+      videoLayer.useJpegCompression = json.useJpegCompression;
+    }
+
     return videoLayer;
   }
 
@@ -1491,7 +1542,8 @@ class VideoLayer extends Widget {
       audible: this.audible,
       animationData: this.animationData.toJSON(),
       clips: this.clips,
-      linkedAudioTrack: this.linkedAudioTrack?.idx
+      linkedAudioTrack: this.linkedAudioTrack?.idx,
+      useJpegCompression: this.useJpegCompression
     };
   }