diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index ed43b02..e40e7f6 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -635,6 +635,12 @@ dependencies = [ "objc", ] +[[package]] +name = "color_quant" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" + [[package]] name = "combine" version = "4.6.7" @@ -2197,6 +2203,19 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "image" +version = "0.24.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5690139d2f55868e080017335e4b94cb7414274c74f1669c84fb5feba2c9f69d" +dependencies = [ + "bytemuck", + "byteorder", + "color_quant", + "jpeg-decoder", + "num-traits", +] + [[package]] name = "indexmap" version = "1.9.3" @@ -2337,6 +2356,12 @@ dependencies = [ "libc", ] +[[package]] +name = "jpeg-decoder" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00810f1d8b74be64b13dbf3db89ac67740615d6c891f0e7b6179326533011a07" + [[package]] name = "js-sys" version = "0.3.77" @@ -2467,6 +2492,7 @@ dependencies = [ "cpal", "daw-backend", "ffmpeg-next", + "image", "log", "lru", "rtrb", diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index 9e1b91c..3ca25d7 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -39,6 +39,7 @@ rtrb = "0.3" # Video decoding ffmpeg-next = "7.0" lru = "0.12" +image = { version = "0.24", default-features = false, features = ["jpeg"] } [profile.dev] opt-level = 1 # Enable basic optimizations in debug mode for audio decoding performance diff --git a/src-tauri/src/video.rs b/src-tauri/src/video.rs index 38feeb7..b58f035 100644 --- a/src-tauri/src/video.rs +++ b/src-tauri/src/video.rs @@ -1,8 +1,10 @@ use std::sync::{Arc, Mutex}; use std::num::NonZeroUsize; +use std::io::Cursor; use ffmpeg_next as ffmpeg; use lru::LruCache; use daw_backend::WaveformPeak; +use image::{RgbaImage, ImageEncoder}; #[derive(serde::Serialize, Clone)] pub struct VideoFileMetadata { @@ -453,16 +455,16 @@ fn generate_waveform(audio_data: &[f32], channels: u32, target_peaks: usize) -> waveform } -// Use a custom serializer wrapper for efficient binary transfer -#[derive(serde::Serialize)] -struct BinaryFrame(#[serde(with = "serde_bytes")] Vec); - #[tauri::command] pub async fn video_get_frame( state: tauri::State<'_, Arc>>, pool_index: usize, timestamp: f64, -) -> Result, String> { + use_jpeg: bool, + channel: tauri::ipc::Channel, +) -> Result<(), String> { + use std::time::Instant; + let video_state = state.lock().unwrap(); let decoder = video_state.pool.get(pool_index) @@ -472,7 +474,51 @@ pub async fn video_get_frame( drop(video_state); let mut decoder = decoder.lock().unwrap(); - decoder.get_frame(timestamp) + let frame_data = decoder.get_frame(timestamp)?; + + let data_to_send = if use_jpeg { + let t_compress_start = Instant::now(); + + // Get frame dimensions from decoder + let width = decoder.output_width; + let height = decoder.output_height; + + // Create image from raw RGBA data + let img = RgbaImage::from_raw(width, height, frame_data) + .ok_or("Failed to create image from frame data")?; + + // Convert RGBA to RGB (JPEG doesn't support alpha) + let rgb_img = image::DynamicImage::ImageRgba8(img).to_rgb8(); + + // Encode to JPEG with quality 85 (good balance of size/quality) + let mut jpeg_data = Vec::new(); + let mut encoder = image::codecs::jpeg::JpegEncoder::new_with_quality(&mut jpeg_data, 85); + encoder.encode( + rgb_img.as_raw(), + rgb_img.width(), + rgb_img.height(), + image::ColorType::Rgb8 + ).map_err(|e| format!("JPEG encoding failed: {}", e))?; + + let compress_time = t_compress_start.elapsed().as_millis(); + let original_size = width as usize * height as usize * 4; + let compressed_size = jpeg_data.len(); + let ratio = original_size as f32 / compressed_size as f32; + + eprintln!("[Video JPEG] Compressed {}KB -> {}KB ({}x) in {}ms", + original_size / 1024, compressed_size / 1024, ratio, compress_time); + + jpeg_data + } else { + frame_data + }; + + // Send binary data through channel (bypasses JSON serialization) + // InvokeResponseBody::Raw sends raw binary data without JSON encoding + channel.send(tauri::ipc::InvokeResponseBody::Raw(data_to_send)) + .map_err(|e| format!("Channel send error: {}", e))?; + + Ok(()) } #[tauri::command] diff --git a/src/models/layer.js b/src/models/layer.js index b6a9ef3..31f06b7 100644 --- a/src/models/layer.js +++ b/src/models/layer.js @@ -18,7 +18,7 @@ import { const Tone = window.Tone; // Tauri API -const { invoke } = window.__TAURI__.core; +const { invoke, Channel } = window.__TAURI__.core; // Helper function for UUID generation function uuidv4() { @@ -1280,6 +1280,9 @@ class VideoLayer extends Widget { // Associated audio track (if video has audio) this.linkedAudioTrack = null; // Reference to AudioTrack + // Performance settings + this.useJpegCompression = true; // Enable JPEG compression for faster transfer (default: true) + // Timeline display this.collapsed = false; this.curvesMode = 'segment'; @@ -1343,34 +1346,76 @@ class VideoLayer extends Widget { clip.lastFetchedTimestamp = videoTimestamp; try { - // Request frame from Rust backend + // Request frame from Rust backend using IPC Channel for efficient binary transfer const t_start = performance.now(); - let frameData = await invoke('video_get_frame', { - poolIndex: clip.poolIndex, - timestamp: videoTimestamp + + // Create a promise that resolves when channel receives data + const frameDataPromise = new Promise((resolve, reject) => { + const channel = new Channel(); + + channel.onmessage = (data) => { + resolve(data); + }; + + // Invoke command with channel + invoke('video_get_frame', { + poolIndex: clip.poolIndex, + timestamp: videoTimestamp, + useJpeg: this.useJpegCompression, + channel: channel + }).catch(reject); }); + + // Wait for the frame data + let frameData = await frameDataPromise; const t_after_ipc = performance.now(); - // Handle different formats that Tauri might return - // ByteBuf from Rust can come as Uint8Array or Array depending on serialization + // Ensure data is Uint8Array if (!(frameData instanceof Uint8Array)) { frameData = new Uint8Array(frameData); } - // Validate frame data size - const expectedSize = clip.width * clip.height * 4; // RGBA = 4 bytes per pixel + let imageData; + const t_before_conversion = performance.now(); - if (frameData.length !== expectedSize) { - throw new Error(`Invalid frame data size: got ${frameData.length}, expected ${expectedSize}`); + if (this.useJpegCompression) { + // Decode JPEG data + const blob = new Blob([frameData], { type: 'image/jpeg' }); + const imageUrl = URL.createObjectURL(blob); + + // Load and decode JPEG + const img = new Image(); + await new Promise((resolve, reject) => { + img.onload = resolve; + img.onerror = reject; + img.src = imageUrl; + }); + + // Create temporary canvas to extract ImageData + const tempCanvas = document.createElement('canvas'); + tempCanvas.width = clip.width; + tempCanvas.height = clip.height; + const tempCtx = tempCanvas.getContext('2d'); + tempCtx.drawImage(img, 0, 0); + imageData = tempCtx.getImageData(0, 0, clip.width, clip.height); + + // Cleanup + URL.revokeObjectURL(imageUrl); + } else { + // Raw RGBA data + const expectedSize = clip.width * clip.height * 4; // RGBA = 4 bytes per pixel + + if (frameData.length !== expectedSize) { + throw new Error(`Invalid frame data size: got ${frameData.length}, expected ${expectedSize}`); + } + + imageData = new ImageData( + new Uint8ClampedArray(frameData), + clip.width, + clip.height + ); } - // Convert to ImageData - const t_before_conversion = performance.now(); - const imageData = new ImageData( - new Uint8ClampedArray(frameData), - clip.width, - clip.height - ); const t_after_conversion = performance.now(); // Create or reuse temp canvas @@ -1392,8 +1437,9 @@ class VideoLayer extends Widget { const ipc_time = t_after_ipc - t_start; const conversion_time = t_after_conversion - t_before_conversion; const putimage_time = t_after_putimage - t_before_putimage; + const compression_mode = this.useJpegCompression ? 'JPEG' : 'RAW'; - console.log(`[JS Video Timing] ts=${videoTimestamp.toFixed(3)}s | Total: ${total_time.toFixed(1)}ms | IPC: ${ipc_time.toFixed(1)}ms (${(ipc_time/total_time*100).toFixed(0)}%) | Convert: ${conversion_time.toFixed(1)}ms | PutImage: ${putimage_time.toFixed(1)}ms | Size: ${(frameData.length/1024/1024).toFixed(2)}MB`); + console.log(`[JS Video Timing ${compression_mode}] ts=${videoTimestamp.toFixed(3)}s | Total: ${total_time.toFixed(1)}ms | IPC: ${ipc_time.toFixed(1)}ms (${(ipc_time/total_time*100).toFixed(0)}%) | Convert: ${conversion_time.toFixed(1)}ms | PutImage: ${putimage_time.toFixed(1)}ms | Size: ${(frameData.length/1024/1024).toFixed(2)}MB`); } catch (error) { console.error('Failed to get video frame:', error); clip.currentFrame = null; @@ -1479,6 +1525,11 @@ class VideoLayer extends Widget { videoLayer.visible = json.visible; videoLayer.audible = json.audible; + // Restore compression setting (default to true if not specified for backward compatibility) + if (json.useJpegCompression !== undefined) { + videoLayer.useJpegCompression = json.useJpegCompression; + } + return videoLayer; } @@ -1491,7 +1542,8 @@ class VideoLayer extends Widget { audible: this.audible, animationData: this.animationData.toJSON(), clips: this.clips, - linkedAudioTrack: this.linkedAudioTrack?.idx + linkedAudioTrack: this.linkedAudioTrack?.idx, + useJpegCompression: this.useJpegCompression }; }