skyler
/
Lightningbeam
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

use channel and jpeg compression to speed up playback

This commit is contained in:
Skyler Lehmkuhl 2025-11-06 06:42:12 -05:00
parent 3c5a24e0b6
commit 09426e21f4
4 changed files with 151 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
src-tauri/Cargo.lock generated
View File

@ -635,6 +635,12 @@ dependencies = [
"objc",
]
[[package]]
name = "color_quant"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b"
[[package]]
name = "combine"
version = "4.6.7"
@ -2197,6 +2203,19 @@ dependencies = [
"icu_properties",
]
[[package]]
name = "image"
version = "0.24.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5690139d2f55868e080017335e4b94cb7414274c74f1669c84fb5feba2c9f69d"
dependencies = [
"bytemuck",
"byteorder",
"color_quant",
"jpeg-decoder",
"num-traits",
]
[[package]]
name = "indexmap"
version = "1.9.3"
@ -2337,6 +2356,12 @@ dependencies = [
"libc",
]
[[package]]
name = "jpeg-decoder"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "00810f1d8b74be64b13dbf3db89ac67740615d6c891f0e7b6179326533011a07"
[[package]]
name = "js-sys"
version = "0.3.77"
@ -2467,6 +2492,7 @@ dependencies = [
"cpal",
"daw-backend",
"ffmpeg-next",
"image",
"log",
"lru",
"rtrb",

1
src-tauri/Cargo.toml
View File

@ -39,6 +39,7 @@ rtrb = "0.3"
# Video decoding
ffmpeg-next = "7.0"
lru = "0.12"
image = { version = "0.24", default-features = false, features = ["jpeg"] }
[profile.dev]
opt-level = 1 # Enable basic optimizations in debug mode for audio decoding performance

58
src-tauri/src/video.rs
View File

@ -1,8 +1,10 @@
use std::sync::{Arc, Mutex};
use std::num::NonZeroUsize;
use std::io::Cursor;
use ffmpeg_next as ffmpeg;
use lru::LruCache;
use daw_backend::WaveformPeak;
use image::{RgbaImage, ImageEncoder};
#[derive(serde::Serialize, Clone)]
pub struct VideoFileMetadata {
@ -453,16 +455,16 @@ fn generate_waveform(audio_data: &[f32], channels: u32, target_peaks: usize) ->
waveform
}
// Use a custom serializer wrapper for efficient binary transfer
#[derive(serde::Serialize)]
struct BinaryFrame(#[serde(with = "serde_bytes")] Vec<u8>);
#[tauri::command]
pub async fn video_get_frame(
state: tauri::State<'_, Arc<Mutex<VideoState>>>,
pool_index: usize,
timestamp: f64,
) -> Result<Vec<u8>, String> {
use_jpeg: bool,
channel: tauri::ipc::Channel,
) -> Result<(), String> {
use std::time::Instant;
let video_state = state.lock().unwrap();
let decoder = video_state.pool.get(pool_index)
@ -472,7 +474,51 @@ pub async fn video_get_frame(
drop(video_state);
let mut decoder = decoder.lock().unwrap();
decoder.get_frame(timestamp)
let frame_data = decoder.get_frame(timestamp)?;
let data_to_send = if use_jpeg {
let t_compress_start = Instant::now();
// Get frame dimensions from decoder
let width = decoder.output_width;
let height = decoder.output_height;
// Create image from raw RGBA data
let img = RgbaImage::from_raw(width, height, frame_data)
.ok_or("Failed to create image from frame data")?;
// Convert RGBA to RGB (JPEG doesn't support alpha)
let rgb_img = image::DynamicImage::ImageRgba8(img).to_rgb8();
// Encode to JPEG with quality 85 (good balance of size/quality)
let mut jpeg_data = Vec::new();
let mut encoder = image::codecs::jpeg::JpegEncoder::new_with_quality(&mut jpeg_data, 85);
encoder.encode(
rgb_img.as_raw(),
rgb_img.width(),
rgb_img.height(),
image::ColorType::Rgb8
).map_err(|e| format!("JPEG encoding failed: {}", e))?;
let compress_time = t_compress_start.elapsed().as_millis();
let original_size = width as usize * height as usize * 4;
let compressed_size = jpeg_data.len();
let ratio = original_size as f32 / compressed_size as f32;
eprintln!("[Video JPEG] Compressed {}KB -> {}KB ({}x) in {}ms",
original_size / 1024, compressed_size / 1024, ratio, compress_time);
jpeg_data
} else {
frame_data
};
// Send binary data through channel (bypasses JSON serialization)
// InvokeResponseBody::Raw sends raw binary data without JSON encoding
channel.send(tauri::ipc::InvokeResponseBody::Raw(data_to_send))
.map_err(|e| format!("Channel send error: {}", e))?;
Ok(())
}
#[tauri::command]

76
src/models/layer.js
View File

@ -18,7 +18,7 @@ import {
const Tone = window.Tone;
// Tauri API
const { invoke } = window.__TAURI__.core;
const { invoke, Channel } = window.__TAURI__.core;
// Helper function for UUID generation
function uuidv4() {
@ -1280,6 +1280,9 @@ class VideoLayer extends Widget {
// Associated audio track (if video has audio)
this.linkedAudioTrack = null; // Reference to AudioTrack
// Performance settings
this.useJpegCompression = true; // Enable JPEG compression for faster transfer (default: true)
// Timeline display
this.collapsed = false;
this.curvesMode = 'segment';
@ -1343,34 +1346,76 @@ class VideoLayer extends Widget {
clip.lastFetchedTimestamp = videoTimestamp;
try {
// Request frame from Rust backend
// Request frame from Rust backend using IPC Channel for efficient binary transfer
const t_start = performance.now();
let frameData = await invoke('video_get_frame', {
// Create a promise that resolves when channel receives data
const frameDataPromise = new Promise((resolve, reject) => {
const channel = new Channel();
channel.onmessage = (data) => {
resolve(data);
};
// Invoke command with channel
invoke('video_get_frame', {
poolIndex: clip.poolIndex,
timestamp: videoTimestamp
timestamp: videoTimestamp,
useJpeg: this.useJpegCompression,
channel: channel
}).catch(reject);
});
// Wait for the frame data
let frameData = await frameDataPromise;
const t_after_ipc = performance.now();
// Handle different formats that Tauri might return
// ByteBuf from Rust can come as Uint8Array or Array depending on serialization
// Ensure data is Uint8Array
if (!(frameData instanceof Uint8Array)) {
frameData = new Uint8Array(frameData);
}
// Validate frame data size
let imageData;
const t_before_conversion = performance.now();
if (this.useJpegCompression) {
// Decode JPEG data
const blob = new Blob([frameData], { type: 'image/jpeg' });
const imageUrl = URL.createObjectURL(blob);
// Load and decode JPEG
const img = new Image();
await new Promise((resolve, reject) => {
img.onload = resolve;
img.onerror = reject;
img.src = imageUrl;
});
// Create temporary canvas to extract ImageData
const tempCanvas = document.createElement('canvas');
tempCanvas.width = clip.width;
tempCanvas.height = clip.height;
const tempCtx = tempCanvas.getContext('2d');
tempCtx.drawImage(img, 0, 0);
imageData = tempCtx.getImageData(0, 0, clip.width, clip.height);
// Cleanup
URL.revokeObjectURL(imageUrl);
} else {
// Raw RGBA data
const expectedSize = clip.width * clip.height * 4; // RGBA = 4 bytes per pixel
if (frameData.length !== expectedSize) {
throw new Error(`Invalid frame data size: got ${frameData.length}, expected ${expectedSize}`);
}
// Convert to ImageData
const t_before_conversion = performance.now();
const imageData = new ImageData(
imageData = new ImageData(
new Uint8ClampedArray(frameData),
clip.width,
clip.height
);
}
const t_after_conversion = performance.now();
// Create or reuse temp canvas
@ -1392,8 +1437,9 @@ class VideoLayer extends Widget {
const ipc_time = t_after_ipc - t_start;
const conversion_time = t_after_conversion - t_before_conversion;
const putimage_time = t_after_putimage - t_before_putimage;
const compression_mode = this.useJpegCompression ? 'JPEG' : 'RAW';
console.log(`[JS Video Timing] ts=${videoTimestamp.toFixed(3)}s | Total: ${total_time.toFixed(1)}ms | IPC: ${ipc_time.toFixed(1)}ms (${(ipc_time/total_time*100).toFixed(0)}%) | Convert: ${conversion_time.toFixed(1)}ms | PutImage: ${putimage_time.toFixed(1)}ms | Size: ${(frameData.length/1024/1024).toFixed(2)}MB`);
console.log(`[JS Video Timing ${compression_mode}] ts=${videoTimestamp.toFixed(3)}s | Total: ${total_time.toFixed(1)}ms | IPC: ${ipc_time.toFixed(1)}ms (${(ipc_time/total_time*100).toFixed(0)}%) | Convert: ${conversion_time.toFixed(1)}ms | PutImage: ${putimage_time.toFixed(1)}ms | Size: ${(frameData.length/1024/1024).toFixed(2)}MB`);
} catch (error) {
console.error('Failed to get video frame:', error);
clip.currentFrame = null;
@ -1479,6 +1525,11 @@ class VideoLayer extends Widget {
videoLayer.visible = json.visible;
videoLayer.audible = json.audible;
// Restore compression setting (default to true if not specified for backward compatibility)
if (json.useJpegCompression !== undefined) {
videoLayer.useJpegCompression = json.useJpegCompression;
}
return videoLayer;
}
@ -1491,7 +1542,8 @@ class VideoLayer extends Widget {
audible: this.audible,
animationData: this.animationData.toJSON(),
clips: this.clips,
linkedAudioTrack: this.linkedAudioTrack?.idx
linkedAudioTrack: this.linkedAudioTrack?.idx,
useJpegCompression: this.useJpegCompression
};
}