From 07dc7efbe41ba3ecf15193ee57e7919c8955b1e8 Mon Sep 17 00:00:00 2001 From: Skyler Lehmkuhl Date: Wed, 5 Nov 2025 19:18:11 -0500 Subject: [PATCH 1/5] Rename Layer to VectorLayer --- src/main.js | 4 ++-- src/models/graphics-object.js | 6 +++--- src/models/layer.js | 14 +++++++------- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/main.js b/src/main.js index 34fe61a..f87a3d9 100644 --- a/src/main.js +++ b/src/main.js @@ -93,7 +93,7 @@ import { AnimationData } from "./models/animation.js"; import { - Layer, + VectorLayer, AudioTrack, initializeLayerDependencies } from "./models/layer.js"; @@ -638,7 +638,7 @@ function redo() { // ============================================================================ // ============================================================================ -// Layer system classes (Layer, AudioTrack) +// Layer system classes (VectorLayer, AudioTrack, VideoLayer) // have been moved to src/models/layer.js and are imported at the top of this file // ============================================================================ diff --git a/src/models/graphics-object.js b/src/models/graphics-object.js index 4f8044b..07adc4d 100644 --- a/src/models/graphics-object.js +++ b/src/models/graphics-object.js @@ -1,7 +1,7 @@ // GraphicsObject model: Main container for layers and animation import { context, config, pointerList, startProps } from '../state.js'; -import { Layer, AudioTrack } from './layer.js'; +import { VectorLayer, AudioTrack } from './layer.js'; import { TempShape } from './shapes.js'; import { AnimationCurve, Keyframe } from './animation.js'; import { Widget } from '../widgets.js'; @@ -54,7 +54,7 @@ class GraphicsObject extends Widget { this.audioTracks = []; if (initialChildType === 'layer') { - this.children = [new Layer(uuid + "-L1", this)]; + this.children = [new VectorLayer(uuid + "-L1", this)]; this.currentLayer = 0; // Set first layer as active } else if (initialChildType === 'midi') { const midiTrack = new AudioTrack(uuid + "-M1", "MIDI 1", 'midi'); @@ -103,7 +103,7 @@ class GraphicsObject extends Widget { graphicsObject.parent = pointerList[json.parent] } for (let layer of json.layers) { - graphicsObject.layers.push(Layer.fromJSON(layer, graphicsObject)); + graphicsObject.layers.push(VectorLayer.fromJSON(layer, graphicsObject)); } // Handle audioTracks (may not exist in older files) if (json.audioTracks) { diff --git a/src/models/layer.js b/src/models/layer.js index 463e286..55a66de 100644 --- a/src/models/layer.js +++ b/src/models/layer.js @@ -1,4 +1,4 @@ -// Layer models: Layer and AudioLayer classes +// Layer models: VectorLayer, AudioTrack, and VideoLayer classes import { context, config, pointerList } from '../state.js'; import { Frame, AnimationData, Keyframe, tempFrame } from './animation.js'; @@ -65,7 +65,7 @@ export function initializeLayerDependencies(deps) { actions = deps.actions; } -class Layer extends Widget { +class VectorLayer extends Widget { constructor(uuid, parentObject = null) { super(0,0) if (!uuid) { @@ -73,7 +73,7 @@ class Layer extends Widget { } else { this.idx = uuid; } - this.name = "Layer"; + this.name = "VectorLayer"; // LEGACY: Keep frames array for backwards compatibility during migration this.frames = [new Frame("keyframe", this.idx + "-F1")]; this.animationData = new AnimationData(this); @@ -86,7 +86,7 @@ class Layer extends Widget { this.shapes = [] } static fromJSON(json, parentObject = null) { - const layer = new Layer(json.idx, parentObject); + const layer = new VectorLayer(json.idx, parentObject); for (let i in json.children) { const child = json.children[i]; const childObject = GraphicsObject.fromJSON(child); @@ -136,7 +136,7 @@ class Layer extends Widget { } toJSON(randomizeUuid = false) { const json = {}; - json.type = "Layer"; + json.type = "VectorLayer"; if (randomizeUuid) { json.idx = uuidv4(); json.name = this.name + " copy"; @@ -468,7 +468,7 @@ class Layer extends Widget { } } copy(idx) { - let newLayer = new Layer(idx.slice(0, 8) + this.idx.slice(8)); + let newLayer = new VectorLayer(idx.slice(0, 8) + this.idx.slice(8)); let idxMapping = {}; for (let child of this.children) { let newChild = child.copy(idx); @@ -1245,4 +1245,4 @@ class AudioTrack { } } -export { Layer, AudioTrack }; +export { VectorLayer, AudioTrack }; From 3c5a24e0b65e9a90013f2f7818d7f9b63a8ac5ac Mon Sep 17 00:00:00 2001 From: Skyler Lehmkuhl Date: Thu, 6 Nov 2025 06:04:39 -0500 Subject: [PATCH 2/5] video backend --- daw-backend/src/audio/engine.rs | 22 + daw-backend/src/command/types.rs | 2 + src-tauri/Cargo.lock | 97 +++- src-tauri/Cargo.toml | 4 + src-tauri/examples/video_inspect.rs | 104 ++++ src-tauri/src/audio.rs | 34 +- src-tauri/src/lib.rs | 7 + src-tauri/src/video.rs | 503 +++++++++++++++++++ src/actions/index.js | 157 +++++- src/main.js | 156 +++++- src/models/graphics-object.js | 43 +- src/models/layer.js | 267 +++++++++- src/state.js | 1 + src/timeline.js | 5 +- src/widgets.js | 736 ++++++++++++++++++++++++---- 15 files changed, 2011 insertions(+), 127 deletions(-) create mode 100644 src-tauri/examples/video_inspect.rs create mode 100644 src-tauri/src/video.rs diff --git a/daw-backend/src/audio/engine.rs b/daw-backend/src/audio/engine.rs index 1d42291..2133750 100644 --- a/daw-backend/src/audio/engine.rs +++ b/daw-backend/src/audio/engine.rs @@ -384,6 +384,24 @@ impl Engine { _ => {} } } + Command::TrimClip(track_id, clip_id, new_start_time, new_duration, new_offset) => { + match self.project.get_track_mut(track_id) { + Some(crate::audio::track::TrackNode::Audio(track)) => { + if let Some(clip) = track.clips.iter_mut().find(|c| c.id == clip_id) { + clip.start_time = new_start_time; + clip.duration = new_duration; + clip.offset = new_offset; + } + } + Some(crate::audio::track::TrackNode::Midi(track)) => { + if let Some(clip) = track.clips.iter_mut().find(|c| c.id == clip_id) { + clip.start_time = new_start_time; + clip.duration = new_duration; + } + } + _ => {} + } + } Command::CreateMetatrack(name) => { let track_id = self.project.add_group_track(name.clone(), None); // Notify UI about the new metatrack @@ -1893,6 +1911,10 @@ impl EngineController { let _ = self.command_tx.push(Command::MoveClip(track_id, clip_id, new_start_time)); } + pub fn trim_clip(&mut self, track_id: TrackId, clip_id: ClipId, new_start_time: f64, new_duration: f64, new_offset: f64) { + let _ = self.command_tx.push(Command::TrimClip(track_id, clip_id, new_start_time, new_duration, new_offset)); + } + /// Send a generic command to the audio thread pub fn send_command(&mut self, command: Command) { let _ = self.command_tx.push(command); diff --git a/daw-backend/src/command/types.rs b/daw-backend/src/command/types.rs index 099bdad..7846c98 100644 --- a/daw-backend/src/command/types.rs +++ b/daw-backend/src/command/types.rs @@ -29,6 +29,8 @@ pub enum Command { // Clip management commands /// Move a clip to a new timeline position MoveClip(TrackId, ClipId, f64), + /// Trim a clip (track_id, clip_id, new_start_time, new_duration, new_offset) + TrimClip(TrackId, ClipId, f64, f64, f64), // Metatrack management commands /// Create a new metatrack with a name diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index 4952019..ed43b02 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -247,6 +247,24 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "bindgen" +version = "0.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f49d8fed880d473ea71efb9bf597651e77201bdd4893efe54c9e5d65ae04ce6f" +dependencies = [ + "bitflags 2.8.0", + "cexpr", + "clang-sys", + "itertools 0.13.0", + "proc-macro2", + "quote", + "regex", + "rustc-hash 1.1.0", + "shlex", + "syn 2.0.96", +] + [[package]] name = "bindgen" version = "0.72.1" @@ -260,7 +278,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "rustc-hash", + "rustc-hash 2.1.1", "shlex", "syn 2.0.96", ] @@ -732,7 +750,7 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ceec7a6067e62d6f931a2baf6f3a751f4a892595bcec1461a3c94ef9949864b6" dependencies = [ - "bindgen", + "bindgen 0.72.1", ] [[package]] @@ -1372,6 +1390,31 @@ dependencies = [ "log", ] +[[package]] +name = "ffmpeg-next" +version = "7.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da02698288e0275e442a47fc12ca26d50daf0d48b15398ba5906f20ac2e2a9f9" +dependencies = [ + "bitflags 2.8.0", + "ffmpeg-sys-next", + "libc", +] + +[[package]] +name = "ffmpeg-sys-next" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9e9c75ebd4463de9d8998fb134ba26347fe5faee62fabf0a4b4d41bd500b4ad" +dependencies = [ + "bindgen 0.70.1", + "cc", + "libc", + "num_cpus", + "pkg-config", + "vcpkg", +] + [[package]] name = "field-offset" version = "0.3.6" @@ -1866,6 +1909,12 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + [[package]] name = "hex" version = "0.4.3" @@ -2417,7 +2466,9 @@ dependencies = [ "chrono", "cpal", "daw-backend", + "ffmpeg-next", "log", + "lru", "rtrb", "serde", "serde_json", @@ -2764,6 +2815,16 @@ dependencies = [ "autocfg", ] +[[package]] +name = "num_cpus" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" +dependencies = [ + "hermit-abi", + "libc", +] + [[package]] name = "num_enum" version = "0.7.3" @@ -3835,6 +3896,12 @@ version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + [[package]] name = "rustc-hash" version = "2.1.1" @@ -3960,10 +4027,11 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.217" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" dependencies = [ + "serde_core", "serde_derive", ] @@ -3979,10 +4047,19 @@ dependencies = [ ] [[package]] -name = "serde_derive" -version = "1.0.217" +name = "serde_core" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", @@ -5491,6 +5568,12 @@ version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ef4c4aa54d5d05a279399bfa921ec387b7aba77caf7a682ae8d86785b8fdad2" +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version-compare" version = "0.2.0" diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index d7199da..9e1b91c 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -36,6 +36,10 @@ daw-backend = { path = "../daw-backend" } cpal = "0.15" rtrb = "0.3" +# Video decoding +ffmpeg-next = "7.0" +lru = "0.12" + [profile.dev] opt-level = 1 # Enable basic optimizations in debug mode for audio decoding performance diff --git a/src-tauri/examples/video_inspect.rs b/src-tauri/examples/video_inspect.rs new file mode 100644 index 0000000..9950fd3 --- /dev/null +++ b/src-tauri/examples/video_inspect.rs @@ -0,0 +1,104 @@ +extern crate ffmpeg_next as ffmpeg; + +use std::env; + +fn main() { + ffmpeg::init().unwrap(); + + let args: Vec = env::args().collect(); + if args.len() < 2 { + eprintln!("Usage: {} ", args[0]); + std::process::exit(1); + } + + let path = &args[1]; + let input = ffmpeg::format::input(path).expect("Failed to open video"); + + println!("=== VIDEO FILE INFORMATION ==="); + println!("File: {}", path); + println!("Format: {}", input.format().name()); + println!("Duration: {:.2}s", input.duration() as f64 / f64::from(ffmpeg::ffi::AV_TIME_BASE)); + println!(); + + let video_stream = input.streams() + .best(ffmpeg::media::Type::Video) + .expect("No video stream found"); + + let stream_index = video_stream.index(); + let time_base = f64::from(video_stream.time_base()); + let duration = video_stream.duration() as f64 * time_base; + let fps = f64::from(video_stream.avg_frame_rate()); + + println!("=== VIDEO STREAM ==="); + println!("Stream index: {}", stream_index); + println!("Time base: {} ({:.10})", video_stream.time_base(), time_base); + println!("Duration: {:.2}s", duration); + println!("FPS: {:.2}", fps); + println!("Frames: {}", video_stream.frames()); + + let context = ffmpeg::codec::context::Context::from_parameters(video_stream.parameters()) + .expect("Failed to create context"); + let decoder = context.decoder().video().expect("Failed to create decoder"); + + println!("Codec: {:?}", decoder.id()); + println!("Resolution: {}x{}", decoder.width(), decoder.height()); + println!("Pixel format: {:?}", decoder.format()); + println!(); + + println!("=== SCANNING FRAMES ==="); + println!("Timestamp (ts) | Time (s) | Key | Type"); + println!("---------------|----------|-----|-----"); + + let mut input = ffmpeg::format::input(path).expect("Failed to reopen video"); + let context = ffmpeg::codec::context::Context::from_parameters( + input.streams().best(ffmpeg::media::Type::Video).unwrap().parameters() + ).expect("Failed to create context"); + let mut decoder = context.decoder().video().expect("Failed to create decoder"); + + let mut frame_count = 0; + let mut keyframe_count = 0; + + for (stream, packet) in input.packets() { + if stream.index() == stream_index { + let packet_pts = packet.pts().unwrap_or(0); + let packet_time = packet_pts as f64 * time_base; + let is_key = packet.is_key(); + + if is_key { + keyframe_count += 1; + } + + // Print first 50 packets and all keyframes + if frame_count < 50 || is_key { + println!("{:14} | {:8.2} | {:3} | {:?}", + packet_pts, + packet_time, + if is_key { "KEY" } else { " " }, + if is_key { "I-frame" } else { "P/B-frame" } + ); + } + + decoder.send_packet(&packet).ok(); + let mut frame = ffmpeg::util::frame::Video::empty(); + while decoder.receive_frame(&mut frame).is_ok() { + frame_count += 1; + } + } + } + + // Flush decoder + decoder.send_eof().ok(); + let mut frame = ffmpeg::util::frame::Video::empty(); + while decoder.receive_frame(&mut frame).is_ok() { + frame_count += 1; + } + + println!(); + println!("=== SUMMARY ==="); + println!("Total frames decoded: {}", frame_count); + println!("Total keyframes: {}", keyframe_count); + if keyframe_count > 0 { + println!("Average keyframe interval: {:.2} frames", frame_count as f64 / keyframe_count as f64); + println!("Average keyframe interval: {:.2}s", duration / keyframe_count as f64); + } +} diff --git a/src-tauri/src/audio.rs b/src-tauri/src/audio.rs index 9578d37..c34d49a 100644 --- a/src-tauri/src/audio.rs +++ b/src-tauri/src/audio.rs @@ -30,15 +30,15 @@ pub struct MidiFileMetadata { } pub struct AudioState { - controller: Option, - sample_rate: u32, - channels: u32, - buffer_size: u32, - next_track_id: u32, - next_pool_index: usize, - next_graph_node_id: u32, + pub(crate) controller: Option, + pub(crate) sample_rate: u32, + pub(crate) channels: u32, + pub(crate) buffer_size: u32, + pub(crate) next_track_id: u32, + pub(crate) next_pool_index: usize, + pub(crate) next_graph_node_id: u32, // Track next node ID for each VoiceAllocator template (VoiceAllocator backend ID -> next template node ID) - template_node_counters: HashMap, + pub(crate) template_node_counters: HashMap, } impl Default for AudioState { @@ -367,6 +367,24 @@ pub async fn audio_move_clip( } } +#[tauri::command] +pub async fn audio_trim_clip( + state: tauri::State<'_, Arc>>, + track_id: u32, + clip_id: u32, + new_start_time: f64, + new_duration: f64, + new_offset: f64, +) -> Result<(), String> { + let mut audio_state = state.lock().unwrap(); + if let Some(controller) = &mut audio_state.controller { + controller.trim_clip(track_id, clip_id, new_start_time, new_duration, new_offset); + Ok(()) + } else { + Err("Audio not initialized".to_string()) + } +} + #[tauri::command] pub async fn audio_start_recording( state: tauri::State<'_, Arc>>, diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 29902ff..6aca3dd 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -7,6 +7,7 @@ use chrono::Local; use tauri::{AppHandle, Manager, Url, WebviewUrl, WebviewWindowBuilder}; mod audio; +mod video; #[derive(Default)] @@ -130,6 +131,7 @@ pub fn run() { tauri::Builder::default() .manage(Mutex::new(AppState::default())) .manage(Arc::new(Mutex::new(audio::AudioState::default()))) + .manage(Arc::new(Mutex::new(video::VideoState::default()))) .setup(|app| { #[cfg(any(windows, target_os = "linux"))] // Windows/Linux needs different handling from macOS { @@ -204,6 +206,7 @@ pub fn run() { audio::audio_load_file, audio::audio_add_clip, audio::audio_move_clip, + audio::audio_trim_clip, audio::audio_start_recording, audio::audio_stop_recording, audio::audio_pause_recording, @@ -250,6 +253,10 @@ pub fn run() { audio::audio_resolve_missing_file, audio::audio_serialize_track_graph, audio::audio_load_track_graph, + video::video_load_file, + video::video_get_frame, + video::video_set_cache_size, + video::video_get_pool_info, ]) // .manage(window_counter) .build(tauri::generate_context!()) diff --git a/src-tauri/src/video.rs b/src-tauri/src/video.rs new file mode 100644 index 0000000..38feeb7 --- /dev/null +++ b/src-tauri/src/video.rs @@ -0,0 +1,503 @@ +use std::sync::{Arc, Mutex}; +use std::num::NonZeroUsize; +use ffmpeg_next as ffmpeg; +use lru::LruCache; +use daw_backend::WaveformPeak; + +#[derive(serde::Serialize, Clone)] +pub struct VideoFileMetadata { + pub pool_index: usize, + pub width: u32, + pub height: u32, + pub fps: f64, + pub duration: f64, + pub has_audio: bool, + pub audio_pool_index: Option, + pub audio_duration: Option, + pub audio_sample_rate: Option, + pub audio_channels: Option, + pub audio_waveform: Option>, +} + +struct VideoDecoder { + path: String, + width: u32, // Original video width + height: u32, // Original video height + output_width: u32, // Scaled output width + output_height: u32, // Scaled output height + fps: f64, + duration: f64, + time_base: f64, + stream_index: usize, + frame_cache: LruCache>, // timestamp -> RGBA data + input: Option, + decoder: Option, + last_decoded_ts: i64, // Track the last decoded frame timestamp +} + +impl VideoDecoder { + fn new(path: String, cache_size: usize, max_width: Option, max_height: Option) -> Result { + ffmpeg::init().map_err(|e| e.to_string())?; + + let input = ffmpeg::format::input(&path) + .map_err(|e| format!("Failed to open video: {}", e))?; + + let video_stream = input.streams() + .best(ffmpeg::media::Type::Video) + .ok_or("No video stream found")?; + + let stream_index = video_stream.index(); + + let context_decoder = ffmpeg::codec::context::Context::from_parameters( + video_stream.parameters() + ).map_err(|e| e.to_string())?; + + let decoder = context_decoder.decoder().video() + .map_err(|e| e.to_string())?; + + let width = decoder.width(); + let height = decoder.height(); + let time_base = f64::from(video_stream.time_base()); + + // Calculate output dimensions (scale down if larger than max) + let (output_width, output_height) = if let (Some(max_w), Some(max_h)) = (max_width, max_height) { + // Calculate scale to fit within max dimensions while preserving aspect ratio + let scale = (max_w as f32 / width as f32).min(max_h as f32 / height as f32).min(1.0); + ((width as f32 * scale) as u32, (height as f32 * scale) as u32) + } else { + (width, height) + }; + + // Try to get duration from stream, fallback to container + let duration = if video_stream.duration() > 0 { + video_stream.duration() as f64 * time_base + } else if input.duration() > 0 { + input.duration() as f64 / f64::from(ffmpeg::ffi::AV_TIME_BASE) + } else { + // If no duration available, estimate from frame count and fps + let fps = f64::from(video_stream.avg_frame_rate()); + if video_stream.frames() > 0 && fps > 0.0 { + video_stream.frames() as f64 / fps + } else { + 0.0 // Unknown duration + } + }; + + let fps = f64::from(video_stream.avg_frame_rate()); + + Ok(Self { + path, + width, + height, + output_width, + output_height, + fps, + duration, + time_base, + stream_index, + frame_cache: LruCache::new( + NonZeroUsize::new(cache_size).unwrap() + ), + input: None, + decoder: None, + last_decoded_ts: -1, + }) + } + + fn get_frame(&mut self, timestamp: f64) -> Result, String> { + use std::time::Instant; + let t_start = Instant::now(); + + // Convert timestamp to frame timestamp + let frame_ts = (timestamp / self.time_base) as i64; + + // Check cache + if let Some(cached_frame) = self.frame_cache.get(&frame_ts) { + eprintln!("[Video Timing] Cache hit for ts={:.3}s ({}ms)", timestamp, t_start.elapsed().as_millis()); + return Ok(cached_frame.clone()); + } + + let t_after_cache = Instant::now(); + + // Determine if we need to seek + // Seek if: no decoder open, going backwards, or jumping forward more than 2 seconds + let need_seek = self.decoder.is_none() + || frame_ts < self.last_decoded_ts + || frame_ts > self.last_decoded_ts + (2.0 / self.time_base) as i64; + + if need_seek { + let t_seek_start = Instant::now(); + + // Reopen input + let mut input = ffmpeg::format::input(&self.path) + .map_err(|e| format!("Failed to reopen video: {}", e))?; + + // Seek to timestamp + input.seek(frame_ts, ..frame_ts) + .map_err(|e| format!("Seek failed: {}", e))?; + + let context_decoder = ffmpeg::codec::context::Context::from_parameters( + input.streams().best(ffmpeg::media::Type::Video).unwrap().parameters() + ).map_err(|e| e.to_string())?; + + let decoder = context_decoder.decoder().video() + .map_err(|e| e.to_string())?; + + self.input = Some(input); + self.decoder = Some(decoder); + self.last_decoded_ts = -1; // Reset since we seeked + + eprintln!("[Video Timing] Seek took {}ms", t_seek_start.elapsed().as_millis()); + } + + let input = self.input.as_mut().unwrap(); + let decoder = self.decoder.as_mut().unwrap(); + + // Decode frames until we find the one closest to our target timestamp + let mut best_frame_data: Option> = None; + let mut best_frame_ts: Option = None; + let t_decode_start = Instant::now(); + let mut decode_count = 0; + let mut scale_time_ms = 0u128; + + for (stream, packet) in input.packets() { + if stream.index() == self.stream_index { + decoder.send_packet(&packet) + .map_err(|e| e.to_string())?; + + let mut frame = ffmpeg::util::frame::Video::empty(); + while decoder.receive_frame(&mut frame).is_ok() { + decode_count += 1; + let current_frame_ts = frame.timestamp().unwrap_or(0); + self.last_decoded_ts = current_frame_ts; // Update last decoded position + + // Check if this frame is closer to our target than the previous best + let is_better = match best_frame_ts { + None => true, + Some(best_ts) => { + (current_frame_ts - frame_ts).abs() < (best_ts - frame_ts).abs() + } + }; + + if is_better { + let t_scale_start = Instant::now(); + + // Convert to RGBA and scale to output size + let mut scaler = ffmpeg::software::scaling::context::Context::get( + frame.format(), + frame.width(), + frame.height(), + ffmpeg::format::Pixel::RGBA, + self.output_width, + self.output_height, + ffmpeg::software::scaling::flag::Flags::BILINEAR, + ).map_err(|e| e.to_string())?; + + let mut rgb_frame = ffmpeg::util::frame::Video::empty(); + scaler.run(&frame, &mut rgb_frame) + .map_err(|e| e.to_string())?; + + // Remove stride padding to create tightly packed RGBA data + let width = self.output_width as usize; + let height = self.output_height as usize; + let stride = rgb_frame.stride(0); + let row_size = width * 4; // RGBA = 4 bytes per pixel + let source_data = rgb_frame.data(0); + + let mut packed_data = Vec::with_capacity(row_size * height); + for y in 0..height { + let row_start = y * stride; + let row_end = row_start + row_size; + packed_data.extend_from_slice(&source_data[row_start..row_end]); + } + + scale_time_ms += t_scale_start.elapsed().as_millis(); + best_frame_data = Some(packed_data); + best_frame_ts = Some(current_frame_ts); + } + + // If we've reached or passed the target timestamp, we can stop + if current_frame_ts >= frame_ts { + // Found our frame, cache and return it + if let Some(data) = best_frame_data { + let total_time = t_start.elapsed().as_millis(); + let decode_time = t_decode_start.elapsed().as_millis(); + eprintln!("[Video Timing] ts={:.3}s | Decoded {} frames in {}ms | Scale: {}ms | Total: {}ms", + timestamp, decode_count, decode_time, scale_time_ms, total_time); + self.frame_cache.put(frame_ts, data.clone()); + return Ok(data); + } + break; + } + } + } + } + + eprintln!("[Video Decoder] ERROR: Failed to decode frame for timestamp {}", timestamp); + Err("Failed to decode frame".to_string()) + } +} + +pub struct VideoState { + pool: Vec>>, + next_pool_index: usize, + cache_size: usize, +} + +impl Default for VideoState { + fn default() -> Self { + Self { + pool: Vec::new(), + next_pool_index: 0, + cache_size: 20, // Default cache size + } + } +} + +#[tauri::command] +pub async fn video_load_file( + video_state: tauri::State<'_, Arc>>, + audio_state: tauri::State<'_, Arc>>, + path: String, +) -> Result { + ffmpeg::init().map_err(|e| e.to_string())?; + + // Open input to check for audio stream + let mut input = ffmpeg::format::input(&path) + .map_err(|e| format!("Failed to open video: {}", e))?; + + let audio_stream_opt = input.streams() + .best(ffmpeg::media::Type::Audio); + + let has_audio = audio_stream_opt.is_some(); + + // Extract audio if present + let (audio_pool_index, audio_duration, audio_sample_rate, audio_channels, audio_waveform) = if has_audio { + let audio_stream = audio_stream_opt.unwrap(); + let audio_index = audio_stream.index(); + + // Get audio properties + let context_decoder = ffmpeg::codec::context::Context::from_parameters( + audio_stream.parameters() + ).map_err(|e| e.to_string())?; + + let mut audio_decoder = context_decoder.decoder().audio() + .map_err(|e| e.to_string())?; + + let sample_rate = audio_decoder.rate(); + let channels = audio_decoder.channels() as u32; + + // Decode all audio frames + let mut audio_samples: Vec = Vec::new(); + + for (stream, packet) in input.packets() { + if stream.index() == audio_index { + audio_decoder.send_packet(&packet) + .map_err(|e| e.to_string())?; + + let mut audio_frame = ffmpeg::util::frame::Audio::empty(); + while audio_decoder.receive_frame(&mut audio_frame).is_ok() { + // Convert audio to f32 planar format + let format = audio_frame.format(); + let frame_channels = audio_frame.channels() as usize; + + // Create resampler to convert to f32 planar + let mut resampler = ffmpeg::software::resampling::context::Context::get( + format, + audio_frame.channel_layout(), + sample_rate, + ffmpeg::format::Sample::F32(ffmpeg::format::sample::Type::Packed), + audio_frame.channel_layout(), + sample_rate, + ).map_err(|e| e.to_string())?; + + let mut resampled_frame = ffmpeg::util::frame::Audio::empty(); + resampler.run(&audio_frame, &mut resampled_frame) + .map_err(|e| e.to_string())?; + + // Extract f32 samples (interleaved format) + let data_ptr = resampled_frame.data(0).as_ptr() as *const f32; + let total_samples = resampled_frame.samples() * frame_channels; + let samples_slice = unsafe { + std::slice::from_raw_parts(data_ptr, total_samples) + }; + + audio_samples.extend_from_slice(samples_slice); + } + } + } + + // Flush audio decoder + audio_decoder.send_eof().map_err(|e| e.to_string())?; + let mut audio_frame = ffmpeg::util::frame::Audio::empty(); + while audio_decoder.receive_frame(&mut audio_frame).is_ok() { + let format = audio_frame.format(); + let frame_channels = audio_frame.channels() as usize; + + let mut resampler = ffmpeg::software::resampling::context::Context::get( + format, + audio_frame.channel_layout(), + sample_rate, + ffmpeg::format::Sample::F32(ffmpeg::format::sample::Type::Packed), + audio_frame.channel_layout(), + sample_rate, + ).map_err(|e| e.to_string())?; + + let mut resampled_frame = ffmpeg::util::frame::Audio::empty(); + resampler.run(&audio_frame, &mut resampled_frame) + .map_err(|e| e.to_string())?; + + let data_ptr = resampled_frame.data(0).as_ptr() as *const f32; + let total_samples = resampled_frame.samples() * frame_channels; + let samples_slice = unsafe { + std::slice::from_raw_parts(data_ptr, total_samples) + }; + + audio_samples.extend_from_slice(samples_slice); + } + + // Calculate audio duration + let total_samples_per_channel = audio_samples.len() / channels as usize; + let audio_duration = total_samples_per_channel as f64 / sample_rate as f64; + + // Generate waveform + let target_peaks = ((audio_duration * 300.0) as usize).clamp(1000, 20000); + let waveform = generate_waveform(&audio_samples, channels, target_peaks); + + // Send audio to DAW backend + let mut audio_state_guard = audio_state.lock().unwrap(); + let audio_pool_index = audio_state_guard.next_pool_index; + audio_state_guard.next_pool_index += 1; + + if let Some(controller) = &mut audio_state_guard.controller { + controller.add_audio_file( + path.clone(), + audio_samples, + channels, + sample_rate, + ); + } + drop(audio_state_guard); + + (Some(audio_pool_index), Some(audio_duration), Some(sample_rate), Some(channels), Some(waveform)) + } else { + (None, None, None, None, None) + }; + + // Create video decoder with max dimensions for playback (800x600) + // This scales down high-res videos to reduce data transfer + let mut video_state_guard = video_state.lock().unwrap(); + let pool_index = video_state_guard.next_pool_index; + video_state_guard.next_pool_index += 1; + + let decoder = VideoDecoder::new(path, video_state_guard.cache_size, Some(800), Some(600))?; + + let metadata = VideoFileMetadata { + pool_index, + width: decoder.output_width, // Return scaled dimensions to JS + height: decoder.output_height, + fps: decoder.fps, + duration: decoder.duration, + has_audio, + audio_pool_index, + audio_duration, + audio_sample_rate, + audio_channels, + audio_waveform, + }; + + video_state_guard.pool.push(Arc::new(Mutex::new(decoder))); + + Ok(metadata) +} + +fn generate_waveform(audio_data: &[f32], channels: u32, target_peaks: usize) -> Vec { + let total_samples = audio_data.len(); + let samples_per_channel = total_samples / channels as usize; + let samples_per_peak = (samples_per_channel / target_peaks).max(1); + + let mut waveform = Vec::new(); + + for peak_idx in 0..target_peaks { + let start_sample = peak_idx * samples_per_peak; + let end_sample = ((peak_idx + 1) * samples_per_peak).min(samples_per_channel); + + if start_sample >= samples_per_channel { + break; + } + + let mut min_val = 0.0f32; + let mut max_val = 0.0f32; + + for sample_idx in start_sample..end_sample { + // Average across channels + let mut channel_sum = 0.0f32; + for ch in 0..channels as usize { + let idx = sample_idx * channels as usize + ch; + if idx < total_samples { + channel_sum += audio_data[idx]; + } + } + let avg_sample = channel_sum / channels as f32; + + min_val = min_val.min(avg_sample); + max_val = max_val.max(avg_sample); + } + + waveform.push(WaveformPeak { + min: min_val, + max: max_val, + }); + } + + waveform +} + +// Use a custom serializer wrapper for efficient binary transfer +#[derive(serde::Serialize)] +struct BinaryFrame(#[serde(with = "serde_bytes")] Vec); + +#[tauri::command] +pub async fn video_get_frame( + state: tauri::State<'_, Arc>>, + pool_index: usize, + timestamp: f64, +) -> Result, String> { + let video_state = state.lock().unwrap(); + + let decoder = video_state.pool.get(pool_index) + .ok_or("Invalid pool index")? + .clone(); + + drop(video_state); + + let mut decoder = decoder.lock().unwrap(); + decoder.get_frame(timestamp) +} + +#[tauri::command] +pub async fn video_set_cache_size( + state: tauri::State<'_, Arc>>, + cache_size: usize, +) -> Result<(), String> { + let mut video_state = state.lock().unwrap(); + video_state.cache_size = cache_size; + Ok(()) +} + +#[tauri::command] +pub async fn video_get_pool_info( + state: tauri::State<'_, Arc>>, + pool_index: usize, +) -> Result<(u32, u32, f64), String> { + let video_state = state.lock().unwrap(); + let decoder = video_state.pool.get(pool_index) + .ok_or("Invalid pool index")? + .lock().unwrap(); + + Ok(( + decoder.output_width, // Return scaled dimensions + decoder.output_height, + decoder.fps + )) +} diff --git a/src/actions/index.js b/src/actions/index.js index d1d7000..ab5f79d 100644 --- a/src/actions/index.js +++ b/src/actions/index.js @@ -12,7 +12,7 @@ import { Frame } from '../models/animation.js'; import { GraphicsObject } from '../models/graphics-object.js'; -import { Layer, AudioTrack } from '../models/layer.js'; +import { VectorLayer, AudioTrack, VideoLayer } from '../models/layer.js'; import { arraysAreEqual, lerp, @@ -161,6 +161,7 @@ let redoStack = null; let updateMenu = null; let updateLayers = null; let updateUI = null; +let updateVideoFrames = null; let updateInfopanel = null; let invoke = null; let config = null; @@ -186,6 +187,7 @@ export function initializeActions(deps) { updateMenu = deps.updateMenu; updateLayers = deps.updateLayers; updateUI = deps.updateUI; + updateVideoFrames = deps.updateVideoFrames; updateInfopanel = deps.updateInfopanel; invoke = deps.invoke; config = deps.config; @@ -587,6 +589,147 @@ export const actions = { } }, }, + addVideo: { + create: (filePath, object, videoname) => { + redoStack.length = 0; + let action = { + filePath: filePath, + videoname: videoname, + layeruuid: uuidv4(), + object: object.idx, + }; + undoStack.push({ name: "addVideo", action: action }); + actions.addVideo.execute(action); + updateMenu(); + }, + execute: async (action) => { + // Create new VideoLayer + let newVideoLayer = new VideoLayer(action.layeruuid, action.videoname); + let object = pointerList[action.object]; + + // Add layer to object + object.layers.push(newVideoLayer); + + // Update UI + updateLayers(); + if (context.timelineWidget) { + context.timelineWidget.requestRedraw(); + } + + // Load video asynchronously + try { + const metadata = await invoke('video_load_file', { + path: action.filePath + }); + + // Add clip to video layer + await newVideoLayer.addClip( + metadata.pool_index, + 0, // startTime + metadata.duration, + 0, // offset + action.videoname, + metadata.duration // sourceDuration + ); + + // If video has audio, create linked AudioTrack + if (metadata.has_audio && metadata.audio_pool_index !== null) { + const audioTrackUuid = uuidv4(); + const audioTrackName = `${action.videoname} (Audio)`; + const newAudioTrack = new AudioTrack(audioTrackUuid, audioTrackName); + + // Initialize track in backend + await newAudioTrack.initializeTrack(); + + // Add audio clip using the extracted audio + const audioClipId = newAudioTrack.clips.length; + await invoke('audio_add_clip', { + trackId: newAudioTrack.audioTrackId, + poolIndex: metadata.audio_pool_index, + startTime: 0, + duration: metadata.audio_duration, + offset: 0 + }); + + const audioClip = { + clipId: audioClipId, + poolIndex: metadata.audio_pool_index, + name: audioTrackName, + startTime: 0, + duration: metadata.audio_duration, + offset: 0, + waveform: metadata.audio_waveform, + sourceDuration: metadata.audio_duration + }; + newAudioTrack.clips.push(audioClip); + + // Link the clips to each other + const videoClip = newVideoLayer.clips[0]; // The video clip we just added + if (videoClip) { + videoClip.linkedAudioClip = audioClip; + audioClip.linkedVideoClip = videoClip; + } + + // Also keep track-level references for convenience + newVideoLayer.linkedAudioTrack = newAudioTrack; + newAudioTrack.linkedVideoLayer = newVideoLayer; + + // Add audio track to object + object.audioTracks.push(newAudioTrack); + + // Store reference for rollback + action.audioTrackUuid = audioTrackUuid; + + console.log(`Video audio extracted: ${metadata.audio_duration}s, ${metadata.audio_sample_rate}Hz, ${metadata.audio_channels}ch`); + } + + // Update UI with real clip data + updateLayers(); + if (context.timelineWidget) { + context.timelineWidget.requestRedraw(); + } + + // Make this the active layer + if (context.activeObject) { + context.activeObject.activeLayer = newVideoLayer; + updateLayers(); + } + + // Fetch first frame + if (updateVideoFrames) { + await updateVideoFrames(context.activeObject.currentTime || 0); + } + + // Trigger redraw to show the first frame + updateUI(); + + console.log(`Video loaded: ${action.videoname}, ${metadata.width}x${metadata.height}, ${metadata.duration}s`); + } catch (error) { + console.error('Failed to load video:', error); + } + }, + rollback: (action) => { + let object = pointerList[action.object]; + let layer = pointerList[action.layeruuid]; + object.layers.splice(object.layers.indexOf(layer), 1); + + // Remove linked audio track if it was created + if (action.audioTrackUuid) { + let audioTrack = pointerList[action.audioTrackUuid]; + if (audioTrack) { + const index = object.audioTracks.indexOf(audioTrack); + if (index !== -1) { + object.audioTracks.splice(index, 1); + } + } + } + + updateLayers(); + if (context.timelineWidget) { + context.timelineWidget.requestRedraw(); + } + }, + }, addMIDI: { create: (filePath, object, midiname) => { redoStack.length = 0; @@ -832,8 +975,8 @@ export const actions = { }, execute: (action) => { let object = pointerList[action.object]; - let layer = new Layer(action.uuid); - layer.name = `Layer ${object.layers.length + 1}`; + let layer = new VectorLayer(action.uuid); + layer.name = `VectorLayer ${object.layers.length + 1}`; object.layers.push(layer); object.currentLayer = object.layers.indexOf(layer); updateLayers(); @@ -854,7 +997,7 @@ export const actions = { redoStack.length = 0; // Don't allow deleting the only layer if (context.activeObject.layers.length == 1) return; - if (!(layer instanceof Layer)) { + if (!(layer instanceof VectorLayer)) { layer = context.activeObject.activeLayer; } let action = { @@ -929,8 +1072,8 @@ export const actions = { let object = GraphicsObject.fromJSON(action.object); activeObject.addObject(object); break; - case "Layer": - let layer = Layer.fromJSON(action.object); + case "VectorLayer": + let layer = VectorLayer.fromJSON(action.object); activeObject.addLayer(layer); } updateUI(); @@ -943,7 +1086,7 @@ export const actions = { let object = pointerList[action.object.idx]; activeObject.removeChild(object); break; - case "Layer": + case "VectorLayer": let layer = pointerList[action.object.idx]; activeObject.removeLayer(layer); } diff --git a/src/main.js b/src/main.js index f87a3d9..e47be2d 100644 --- a/src/main.js +++ b/src/main.js @@ -95,6 +95,7 @@ import { import { VectorLayer, AudioTrack, + VideoLayer, initializeLayerDependencies } from "./models/layer.js"; import { @@ -135,6 +136,7 @@ const { getVersion } = window.__TAURI__.app; // Supported file extensions const imageExtensions = ["png", "gif", "avif", "jpg", "jpeg"]; const audioExtensions = ["mp3", "wav", "aiff", "ogg", "flac"]; +const videoExtensions = ["mp4", "mov", "avi", "mkv", "webm", "m4v"]; const midiExtensions = ["mid", "midi"]; const beamExtensions = ["beam"]; @@ -343,6 +345,7 @@ let mouseEvent; window.context = context; window.actions = actions; window.addKeyframeAtPlayhead = addKeyframeAtPlayhead; +window.updateVideoFrames = null; // Will be set after function is defined function uuidv4() { return "10000000-1000-4000-8000-100000000000".replace(/[018]/g, (c) => @@ -1009,6 +1012,38 @@ function playbackLoop() { } } +// Update video frames for all VideoLayers in the scene +async function updateVideoFrames(currentTime) { + // Recursively find all VideoLayers in the scene + function findVideoLayers(obj) { + const videoLayers = []; + if (obj.layers) { + for (let layer of obj.layers) { + if (layer.type === 'video') { + videoLayers.push(layer); + } + } + } + // Recursively check children (GraphicsObjects can contain other GraphicsObjects) + if (obj.children) { + for (let child of obj.children) { + videoLayers.push(...findVideoLayers(child)); + } + } + return videoLayers; + } + + const videoLayers = findVideoLayers(context.activeObject); + + // Update all video layers in parallel + await Promise.all(videoLayers.map(layer => layer.updateFrame(currentTime))); + + // Note: No updateUI() call here - renderUI() will draw after awaiting this function +} + +// Expose updateVideoFrames globally +window.updateVideoFrames = updateVideoFrames; + // Single-step forward by one frame/second function advance() { if (context.timelineWidget?.timelineState?.timeFormat === "frames") { @@ -1025,6 +1060,9 @@ function advance() { // Sync DAW backend invoke('audio_seek', { seconds: context.activeObject.currentTime }); + // Update video frames + updateVideoFrames(context.activeObject.currentTime); + updateLayers(); updateMenu(); updateUI(); @@ -1108,6 +1146,10 @@ async function handleAudioEvent(event) { if (context.timelineWidget?.timelineState) { context.timelineWidget.timelineState.currentTime = quantizedTime; } + + // Update video frames + updateVideoFrames(quantizedTime); + // Update time display if (context.updateTimeDisplay) { context.updateTimeDisplay(); @@ -2329,6 +2371,10 @@ async function importFile() { name: "Audio files", extensions: audioExtensions, }, + { + name: "Video files", + extensions: videoExtensions, + }, { name: "MIDI files", extensions: midiExtensions, @@ -2384,10 +2430,12 @@ async function importFile() { let usedFilterIndex = 0; if (audioExtensions.includes(ext)) { usedFilterIndex = 1; // Audio + } else if (videoExtensions.includes(ext)) { + usedFilterIndex = 2; // Video } else if (midiExtensions.includes(ext)) { - usedFilterIndex = 2; // MIDI + usedFilterIndex = 3; // MIDI } else if (beamExtensions.includes(ext)) { - usedFilterIndex = 3; // Lightningbeam + usedFilterIndex = 4; // Lightningbeam } else { usedFilterIndex = 0; // Image (default) } @@ -2454,6 +2502,9 @@ async function importFile() { } else if (audioExtensions.includes(ext)) { // Handle audio files - pass file path directly to backend actions.addAudio.create(path, context.activeObject, filename); + } else if (videoExtensions.includes(ext)) { + // Handle video files + actions.addVideo.create(path, context.activeObject, filename); } else if (midiExtensions.includes(ext)) { // Handle MIDI files actions.addMIDI.create(path, context.activeObject, filename); @@ -4804,6 +4855,12 @@ function timelineV2() { timelineWidget.lastDragEvent = e; timelineWidget.handleMouseEvent("mousemove", x, y); + + // Update cursor based on widget's cursor property + if (timelineWidget.cursor) { + canvas.style.cursor = timelineWidget.cursor; + } + updateCanvasSize(); // Redraw after interaction }); @@ -5520,7 +5577,12 @@ function updateUI() { context.updateUI = updateUI; context.updateMenu = updateMenu; -function renderUI() { +async function renderUI() { + // Update video frames BEFORE drawing + if (context.activeObject) { + await updateVideoFrames(context.activeObject.currentTime); + } + for (let canvas of canvases) { let ctx = canvas.getContext("2d"); ctx.resetTransform(); @@ -6548,6 +6610,11 @@ async function renderMenu() { action: actions.addLayer.create, accelerator: getShortcut("addLayer"), }, + { + text: "Add Video Layer", + enabled: true, + action: addVideoLayer, + }, { text: "Add Audio Track", enabled: true, @@ -10787,10 +10854,33 @@ function getMimeType(filePath) { } -function renderAll() { +let renderInProgress = false; +let rafScheduled = false; + +// FPS tracking +let lastFpsLogTime = 0; +let frameCount = 0; +let fpsHistory = []; + +async function renderAll() { + rafScheduled = false; + + // Skip if a render is already in progress (prevent stacking async calls) + if (renderInProgress) { + // Schedule another attempt if not already scheduled + if (!rafScheduled) { + rafScheduled = true; + requestAnimationFrame(renderAll); + } + return; + } + + renderInProgress = true; + const renderStartTime = performance.now(); + try { if (uiDirty) { - renderUI(); + await renderUI(); uiDirty = false; } if (layersDirty) { @@ -10823,7 +10913,33 @@ function renderAll() { repeatCount = 2; } } finally { - requestAnimationFrame(renderAll); + renderInProgress = false; + + // FPS logging (only when playing) + if (playing) { + frameCount++; + const now = performance.now(); + const renderTime = now - renderStartTime; + + if (now - lastFpsLogTime >= 1000) { + const fps = frameCount / ((now - lastFpsLogTime) / 1000); + fpsHistory.push({ fps, renderTime }); + console.log(`[FPS] ${fps.toFixed(1)} fps | Render time: ${renderTime.toFixed(1)}ms`); + frameCount = 0; + lastFpsLogTime = now; + + // Keep only last 10 samples + if (fpsHistory.length > 10) { + fpsHistory.shift(); + } + } + } + + // Schedule next frame if not already scheduled + if (!rafScheduled) { + rafScheduled = true; + requestAnimationFrame(renderAll); + } } } @@ -10834,6 +10950,7 @@ initializeActions({ updateMenu, updateLayers, updateUI, + updateVideoFrames, updateInfopanel, invoke, config @@ -10923,6 +11040,33 @@ async function addEmptyMIDITrack() { } } +async function addVideoLayer() { + console.log('[addVideoLayer] Creating new video layer'); + const layerName = `Video ${context.activeObject.layers.filter(l => l.type === 'video').length + 1}`; + const layerUuid = uuidv4(); + + try { + // Create new VideoLayer + const newVideoLayer = new VideoLayer(layerUuid, layerName); + + // Add layer to active object + context.activeObject.layers.push(newVideoLayer); + + // Select the newly created layer + context.activeObject.activeLayer = newVideoLayer; + + // Update UI + updateLayers(); + if (context.timelineWidget) { + context.timelineWidget.requestRedraw(); + } + + console.log('Empty video layer created:', layerName); + } catch (error) { + console.error('Failed to create video layer:', error); + } +} + // MIDI Command Wrappers // Note: getAvailableInstruments() removed - now using node-based instruments diff --git a/src/models/graphics-object.js b/src/models/graphics-object.js index 07adc4d..f17538a 100644 --- a/src/models/graphics-object.js +++ b/src/models/graphics-object.js @@ -1,7 +1,7 @@ // GraphicsObject model: Main container for layers and animation import { context, config, pointerList, startProps } from '../state.js'; -import { VectorLayer, AudioTrack } from './layer.js'; +import { VectorLayer, AudioTrack, VideoLayer } from './layer.js'; import { TempShape } from './shapes.js'; import { AnimationCurve, Keyframe } from './animation.js'; import { Widget } from '../widgets.js'; @@ -45,8 +45,20 @@ class GraphicsObject extends Widget { this.name = this.idx; this.currentFrameNum = 0; // LEGACY: kept for backwards compatibility - this.currentTime = 0; // New: continuous time for AnimationData curves + this._currentTime = 0; // Internal storage for currentTime this.currentLayer = 0; + + // Make currentTime a getter/setter property + Object.defineProperty(this, 'currentTime', { + get: function() { + return this._currentTime; + }, + set: function(value) { + this._currentTime = value; + }, + enumerable: true, + configurable: true + }); this._activeAudioTrack = null; // Reference to active audio track (if any) // Initialize children and audioTracks based on initialChildType @@ -56,6 +68,9 @@ class GraphicsObject extends Widget { if (initialChildType === 'layer') { this.children = [new VectorLayer(uuid + "-L1", this)]; this.currentLayer = 0; // Set first layer as active + } else if (initialChildType === 'video') { + this.children = [new VideoLayer(uuid + "-V1", "Video 1")]; + this.currentLayer = 0; // Set first video layer as active } else if (initialChildType === 'midi') { const midiTrack = new AudioTrack(uuid + "-M1", "MIDI 1", 'midi'); this.audioTracks.push(midiTrack); @@ -103,7 +118,12 @@ class GraphicsObject extends Widget { graphicsObject.parent = pointerList[json.parent] } for (let layer of json.layers) { - graphicsObject.layers.push(VectorLayer.fromJSON(layer, graphicsObject)); + if (layer.type === 'VideoLayer') { + graphicsObject.layers.push(VideoLayer.fromJSON(layer)); + } else { + // Default to VectorLayer + graphicsObject.layers.push(VectorLayer.fromJSON(layer, graphicsObject)); + } } // Handle audioTracks (may not exist in older files) if (json.audioTracks) { @@ -177,9 +197,20 @@ class GraphicsObject extends Widget { // Check visual layers for (let layer of this.layers) { + // Check animation data duration if (layer.animationData && layer.animationData.duration > maxDuration) { maxDuration = layer.animationData.duration; } + + // Check video layer clips (VideoLayer has clips like AudioTrack) + if (layer.type === 'video' && layer.clips) { + for (let clip of layer.clips) { + const clipEnd = clip.startTime + clip.duration; + if (clipEnd > maxDuration) { + maxDuration = clipEnd; + } + } + } } // Check audio tracks @@ -300,6 +331,12 @@ class GraphicsObject extends Widget { for (let layer of this.layers) { if (context.activeObject == this && !layer.visible) continue; + // Handle VideoLayer differently - call its draw method + if (layer.type === 'video') { + layer.draw(context); + continue; + } + // Draw activeShape (shape being drawn in progress) for active layer only if (layer === context.activeLayer && layer.activeShape) { let cxt = {...context}; diff --git a/src/models/layer.js b/src/models/layer.js index 55a66de..b6a9ef3 100644 --- a/src/models/layer.js +++ b/src/models/layer.js @@ -1245,4 +1245,269 @@ class AudioTrack { } } -export { VectorLayer, AudioTrack }; +class VideoLayer extends Widget { + constructor(uuid, name) { + super(0, 0); + if (!uuid) { + this.idx = uuidv4(); + } else { + this.idx = uuid; + } + this.name = name || "Video"; + this.type = 'video'; + this.visible = true; + this.audible = true; + this.animationData = new AnimationData(this); + + // Empty arrays for layer compatibility + Object.defineProperty(this, 'shapes', { + value: Object.freeze([]), + writable: false, + enumerable: true, + configurable: false + }); + Object.defineProperty(this, 'children', { + value: Object.freeze([]), + writable: false, + enumerable: true, + configurable: false + }); + + // Video clips on this layer + // { clipId, poolIndex, name, startTime, duration, offset, width, height } + this.clips = []; + + // Associated audio track (if video has audio) + this.linkedAudioTrack = null; // Reference to AudioTrack + + // Timeline display + this.collapsed = false; + this.curvesMode = 'segment'; + this.curvesHeight = 150; + + pointerList[this.idx] = this; + } + + async addClip(poolIndex, startTime, duration, offset = 0.0, name = '', sourceDuration = null) { + const poolInfo = await invoke('video_get_pool_info', { poolIndex }); + // poolInfo is [width, height, fps] tuple from Rust + const [width, height, fps] = poolInfo; + + this.clips.push({ + clipId: this.clips.length, + poolIndex, + name: name || `Video ${this.clips.length + 1}`, + startTime, + duration, + offset, + width, + height, + sourceDuration: sourceDuration || duration // Store original file duration + }); + + console.log(`Video clip added: ${name}, ${width}x${height}, duration: ${duration}s`); + } + + // Pre-fetch frames for current time (call before draw) + async updateFrame(currentTime) { + // Prevent concurrent calls - if already updating, skip + if (this.updateInProgress) { + return; + } + this.updateInProgress = true; + + try { + for (let clip of this.clips) { + // Check if clip is active at current time + if (currentTime < clip.startTime || + currentTime >= clip.startTime + clip.duration) { + clip.currentFrame = null; + continue; + } + + // Calculate video timestamp from clip time + const clipTime = currentTime - clip.startTime; + const videoTimestamp = clip.offset + clipTime; + + // Only fetch if timestamp changed + if (clip.lastFetchedTimestamp === videoTimestamp && clip.currentFrame) { + continue; + } + + // Skip if already fetching this frame + if (clip.fetchInProgress === videoTimestamp) { + continue; + } + + clip.fetchInProgress = videoTimestamp; + clip.lastFetchedTimestamp = videoTimestamp; + + try { + // Request frame from Rust backend + const t_start = performance.now(); + let frameData = await invoke('video_get_frame', { + poolIndex: clip.poolIndex, + timestamp: videoTimestamp + }); + const t_after_ipc = performance.now(); + + // Handle different formats that Tauri might return + // ByteBuf from Rust can come as Uint8Array or Array depending on serialization + if (!(frameData instanceof Uint8Array)) { + frameData = new Uint8Array(frameData); + } + + // Validate frame data size + const expectedSize = clip.width * clip.height * 4; // RGBA = 4 bytes per pixel + + if (frameData.length !== expectedSize) { + throw new Error(`Invalid frame data size: got ${frameData.length}, expected ${expectedSize}`); + } + + // Convert to ImageData + const t_before_conversion = performance.now(); + const imageData = new ImageData( + new Uint8ClampedArray(frameData), + clip.width, + clip.height + ); + const t_after_conversion = performance.now(); + + // Create or reuse temp canvas + if (!clip.frameCanvas) { + clip.frameCanvas = document.createElement('canvas'); + clip.frameCanvas.width = clip.width; + clip.frameCanvas.height = clip.height; + } + + const tempCtx = clip.frameCanvas.getContext('2d'); + const t_before_putimage = performance.now(); + tempCtx.putImageData(imageData, 0, 0); + const t_after_putimage = performance.now(); + + clip.currentFrame = clip.frameCanvas; + + // Log detailed timing breakdown + const total_time = t_after_putimage - t_start; + const ipc_time = t_after_ipc - t_start; + const conversion_time = t_after_conversion - t_before_conversion; + const putimage_time = t_after_putimage - t_before_putimage; + + console.log(`[JS Video Timing] ts=${videoTimestamp.toFixed(3)}s | Total: ${total_time.toFixed(1)}ms | IPC: ${ipc_time.toFixed(1)}ms (${(ipc_time/total_time*100).toFixed(0)}%) | Convert: ${conversion_time.toFixed(1)}ms | PutImage: ${putimage_time.toFixed(1)}ms | Size: ${(frameData.length/1024/1024).toFixed(2)}MB`); + } catch (error) { + console.error('Failed to get video frame:', error); + clip.currentFrame = null; + } finally { + clip.fetchInProgress = null; + } + } + } finally { + this.updateInProgress = false; + } + } + + // Draw cached frames (synchronous) + draw(cxt, currentTime) { + if (!this.visible) { + return; + } + + const ctx = cxt.ctx || cxt; + + // Use currentTime from context if not provided + if (currentTime === undefined) { + currentTime = cxt.activeObject?.currentTime || 0; + } + + for (let clip of this.clips) { + // Check if clip is active at current time + if (currentTime < clip.startTime || + currentTime >= clip.startTime + clip.duration) { + continue; + } + + // Draw cached frame if available + if (clip.currentFrame) { + try { + // Scale to fit canvas while maintaining aspect ratio + const canvasWidth = config.fileWidth; + const canvasHeight = config.fileHeight; + const scale = Math.min( + canvasWidth / clip.width, + canvasHeight / clip.height + ); + const scaledWidth = clip.width * scale; + const scaledHeight = clip.height * scale; + const x = (canvasWidth - scaledWidth) / 2; + const y = (canvasHeight - scaledHeight) / 2; + + ctx.drawImage(clip.currentFrame, x, y, scaledWidth, scaledHeight); + } catch (error) { + console.error('Failed to draw video frame:', error); + } + } else { + // Draw placeholder if frame not loaded yet + ctx.save(); + ctx.fillStyle = '#333333'; + ctx.fillRect(0, 0, config.fileWidth, config.fileHeight); + ctx.fillStyle = '#ffffff'; + ctx.font = '24px sans-serif'; + ctx.textAlign = 'center'; + ctx.textBaseline = 'middle'; + ctx.fillText('Loading...', config.fileWidth / 2, config.fileHeight / 2); + ctx.restore(); + } + } + } + + static fromJSON(json) { + const videoLayer = new VideoLayer(json.idx, json.name); + + if (json.animationData) { + videoLayer.animationData = AnimationData.fromJSON(json.animationData, videoLayer); + } + + if (json.clips) { + videoLayer.clips = json.clips; + } + + if (json.linkedAudioTrack) { + // Will be resolved after all objects are loaded + videoLayer.linkedAudioTrack = json.linkedAudioTrack; + } + + videoLayer.visible = json.visible; + videoLayer.audible = json.audible; + + return videoLayer; + } + + toJSON(randomizeUuid = false) { + return { + type: "VideoLayer", + idx: randomizeUuid ? uuidv4() : this.idx, + name: randomizeUuid ? this.name + " copy" : this.name, + visible: this.visible, + audible: this.audible, + animationData: this.animationData.toJSON(), + clips: this.clips, + linkedAudioTrack: this.linkedAudioTrack?.idx + }; + } + + copy(idx) { + const json = this.toJSON(true); + json.idx = idx.slice(0, 8) + this.idx.slice(8); + return VideoLayer.fromJSON(json); + } + + // Compatibility methods for layer interface + bbox() { + return { + x: { min: 0, max: config.fileWidth }, + y: { min: 0, max: config.fileHeight } + }; + } +} + +export { VectorLayer, AudioTrack, VideoLayer }; diff --git a/src/state.js b/src/state.js index f287cc2..674f842 100644 --- a/src/state.js +++ b/src/state.js @@ -85,6 +85,7 @@ export let config = { reopenLastSession: false, lastImportFilterIndex: 0, // Index of last used filter in import dialog (0=Image, 1=Audio, 2=Lightningbeam) audioBufferSize: 256, // Audio buffer size in frames (128, 256, 512, 1024, etc. - requires restart) + minClipDuration: 0.1, // Minimum clip duration in seconds when trimming // Layout settings currentLayout: "animation", // Current active layout key defaultLayout: "animation", // Default layout for new files diff --git a/src/timeline.js b/src/timeline.js index 1b81101..81eae63 100644 --- a/src/timeline.js +++ b/src/timeline.js @@ -571,9 +571,12 @@ class TrackHierarchy { // Iterate through layers (GraphicsObject.children are Layers) for (let layer of graphicsObject.children) { + // Determine layer type - check if it's a VideoLayer + const layerType = layer.type === 'video' ? 'video' : 'layer' + // Add layer track const layerTrack = { - type: 'layer', + type: layerType, object: layer, name: layer.name || 'Layer', indent: 0, diff --git a/src/widgets.js b/src/widgets.js index ae2afeb..5640a4e 100644 --- a/src/widgets.js +++ b/src/widgets.js @@ -947,6 +947,112 @@ class TimelineWindowV2 extends Widget { /** * Draw track backgrounds in timeline area (Phase 2) */ + // Create a cached pattern for the timeline grid + createTimelinePattern(trackHeight) { + const cacheKey = `${this.timelineState.timeFormat}_${this.timelineState.pixelsPerSecond}_${this.timelineState.framerate}_${this.timelineState.bpm}_${trackHeight}` + + // Return cached pattern if available + if (this.cachedPattern && this.cachedPatternKey === cacheKey) { + return this.cachedPattern + } + + let patternWidth, patternHeight = trackHeight + + if (this.timelineState.timeFormat === 'frames') { + // Pattern for 5 frames + const frameDuration = 1 / this.timelineState.framerate + const frameWidth = frameDuration * this.timelineState.pixelsPerSecond + patternWidth = frameWidth * 5 + } else if (this.timelineState.timeFormat === 'measures') { + // Pattern for one measure + const beatsPerSecond = this.timelineState.bpm / 60 + const beatsPerMeasure = this.timelineState.timeSignature.numerator + const beatWidth = (1 / beatsPerSecond) * this.timelineState.pixelsPerSecond + patternWidth = beatWidth * beatsPerMeasure + } else { + // Pattern for seconds - use 10 second intervals + patternWidth = this.timelineState.pixelsPerSecond * 10 + } + + // Create pattern canvas + const patternCanvas = document.createElement('canvas') + patternCanvas.width = Math.ceil(patternWidth) + patternCanvas.height = patternHeight + const pctx = patternCanvas.getContext('2d') + + // Fill background + pctx.fillStyle = shade + pctx.fillRect(0, 0, patternWidth, patternHeight) + + if (this.timelineState.timeFormat === 'frames') { + const frameDuration = 1 / this.timelineState.framerate + const frameWidth = frameDuration * this.timelineState.pixelsPerSecond + + for (let i = 0; i < 5; i++) { + const x = i * frameWidth + if (i === 0) { + // First frame in pattern (every 5th): shade it + pctx.fillStyle = shadow + pctx.fillRect(x, 0, frameWidth, patternHeight) + } else { + // Regular frame: draw edge line + pctx.strokeStyle = shadow + pctx.lineWidth = 1 + pctx.beginPath() + pctx.moveTo(x, 0) + pctx.lineTo(x, patternHeight) + pctx.stroke() + } + } + } else if (this.timelineState.timeFormat === 'measures') { + const beatsPerSecond = this.timelineState.bpm / 60 + const beatsPerMeasure = this.timelineState.timeSignature.numerator + const beatWidth = (1 / beatsPerSecond) * this.timelineState.pixelsPerSecond + + for (let i = 0; i < beatsPerMeasure; i++) { + const x = i * beatWidth + const isMeasureBoundary = i === 0 + const isEvenBeat = (i % 2) === 0 + + pctx.save() + if (isMeasureBoundary) { + pctx.globalAlpha = 1.0 + } else if (isEvenBeat) { + pctx.globalAlpha = 0.5 + } else { + pctx.globalAlpha = 0.25 + } + + pctx.strokeStyle = shadow + pctx.lineWidth = 1 + pctx.beginPath() + pctx.moveTo(x, 0) + pctx.lineTo(x, patternHeight) + pctx.stroke() + pctx.restore() + } + } else { + // Seconds mode: draw lines every second for 10 seconds + const secondWidth = this.timelineState.pixelsPerSecond + + for (let i = 0; i < 10; i++) { + const x = i * secondWidth + pctx.strokeStyle = shadow + pctx.lineWidth = 1 + pctx.beginPath() + pctx.moveTo(x, 0) + pctx.lineTo(x, patternHeight) + pctx.stroke() + } + } + + // Cache the pattern + this.cachedPatternKey = cacheKey + this.cachedPattern = pctx.createPattern(patternCanvas, 'repeat') + + return this.cachedPattern + } + drawTracks(ctx) { ctx.save() ctx.translate(this.trackHeaderWidth, this.ruler.height) // Start after headers, below ruler @@ -966,96 +1072,18 @@ class TimelineWindowV2 extends Widget { const y = this.trackHierarchy.getTrackY(i) const trackHeight = this.trackHierarchy.getTrackHeight(track) - // Draw track background (same color for all tracks) - ctx.fillStyle = shade - ctx.fillRect(0, y, trackAreaWidth, trackHeight) + // Create and apply pattern for this track + const pattern = this.createTimelinePattern(trackHeight) - // Draw interval markings + // Calculate pattern offset based on viewport start time const visibleStartTime = this.timelineState.viewportStartTime - const visibleEndTime = visibleStartTime + (trackAreaWidth / this.timelineState.pixelsPerSecond) + const patternOffsetX = -this.timelineState.timeToPixel(visibleStartTime) - if (this.timelineState.timeFormat === 'frames') { - // Frames mode: mark every frame edge, with every 5th frame shaded - const frameDuration = 1 / this.timelineState.framerate - const startFrame = Math.floor(visibleStartTime / frameDuration) - const endFrame = Math.ceil(visibleEndTime / frameDuration) - - for (let frame = startFrame; frame <= endFrame; frame++) { - const time = frame * frameDuration - const x = this.timelineState.timeToPixel(time) - const nextX = this.timelineState.timeToPixel((frame + 1) * frameDuration) - - if (x >= 0 && x <= trackAreaWidth) { - if (frame % 5 === 0) { - // Every 5th frame: shade the entire frame width - ctx.fillStyle = shadow - ctx.fillRect(x, y, nextX - x, trackHeight) - } else { - // Regular frame: draw edge line - ctx.strokeStyle = shadow - ctx.lineWidth = 1 - ctx.beginPath() - ctx.moveTo(x, y) - ctx.lineTo(x, y + trackHeight) - ctx.stroke() - } - } - } - } else if (this.timelineState.timeFormat === 'measures') { - // Measures mode: draw beats with varying opacity - const beatsPerSecond = this.timelineState.bpm / 60 - const beatsPerMeasure = this.timelineState.timeSignature.numerator - const startBeat = Math.floor(visibleStartTime * beatsPerSecond) - const endBeat = Math.ceil(visibleEndTime * beatsPerSecond) - - for (let beat = startBeat; beat <= endBeat; beat++) { - const time = beat / beatsPerSecond - const x = this.timelineState.timeToPixel(time) - - if (x >= 0 && x <= trackAreaWidth) { - // Determine position within the measure - const beatInMeasure = beat % beatsPerMeasure - const isMeasureBoundary = beatInMeasure === 0 - const isEvenBeatInMeasure = (beatInMeasure % 2) === 0 - - // Set opacity based on position - ctx.save() - if (isMeasureBoundary) { - ctx.globalAlpha = 1.0 // Full opacity for measure boundaries - } else if (isEvenBeatInMeasure) { - ctx.globalAlpha = 0.5 // Half opacity for even beats - } else { - ctx.globalAlpha = 0.25 // Quarter opacity for odd beats - } - - ctx.strokeStyle = shadow - ctx.lineWidth = 1 - ctx.beginPath() - ctx.moveTo(x, y) - ctx.lineTo(x, y + trackHeight) - ctx.stroke() - ctx.restore() - } - } - } else { - // Seconds mode: mark every second edge - const startSecond = Math.floor(visibleStartTime) - const endSecond = Math.ceil(visibleEndTime) - - ctx.strokeStyle = shadow - ctx.lineWidth = 1 - - for (let second = startSecond; second <= endSecond; second++) { - const x = this.timelineState.timeToPixel(second) - - if (x >= 0 && x <= trackAreaWidth) { - ctx.beginPath() - ctx.moveTo(x, y) - ctx.lineTo(x, y + trackHeight) - ctx.stroke() - } - } - } + ctx.save() + ctx.translate(patternOffsetX, y) + ctx.fillStyle = pattern + ctx.fillRect(-patternOffsetX, 0, trackAreaWidth, trackHeight) + ctx.restore() // Draw track border ctx.strokeStyle = shadow @@ -1425,19 +1453,25 @@ class TimelineWindowV2 extends Widget { const waveformHeight = trackHeight - 14 // Leave padding at top/bottom const waveformData = clip.waveform - // Calculate how many pixels each waveform peak represents - const pixelsPerPeak = clipWidth / waveformData.length + // Calculate the full source audio duration and pixels per peak based on that + const sourceDuration = clip.sourceDuration || clip.duration + const pixelsPerSecond = this.timelineState.pixelsPerSecond + const fullSourceWidth = sourceDuration * pixelsPerSecond + const pixelsPerPeak = fullSourceWidth / waveformData.length - // Calculate the range of visible peaks - const firstVisiblePeak = Math.max(0, Math.floor((visibleStart - startX) / pixelsPerPeak)) - const lastVisiblePeak = Math.min(waveformData.length - 1, Math.ceil((visibleEnd - startX) / pixelsPerPeak)) + // Calculate which peak corresponds to the clip's offset (trimmed left edge) + const offsetPeakIndex = Math.floor((clip.offset / sourceDuration) * waveformData.length) + + // Calculate the range of visible peaks, accounting for offset + const firstVisiblePeak = Math.max(offsetPeakIndex, Math.floor((visibleStart - startX) / pixelsPerPeak) + offsetPeakIndex) + const lastVisiblePeak = Math.min(waveformData.length - 1, Math.ceil((visibleEnd - startX) / pixelsPerPeak) + offsetPeakIndex) // Draw waveform as a filled path ctx.beginPath() // Trace along the max values (left to right) for (let i = firstVisiblePeak; i <= lastVisiblePeak; i++) { - const peakX = startX + (i * pixelsPerPeak) + const peakX = startX + ((i - offsetPeakIndex) * pixelsPerPeak) const peak = waveformData[i] const maxY = centerY + (peak.max * waveformHeight * 0.5) @@ -1450,7 +1484,7 @@ class TimelineWindowV2 extends Widget { // Trace back along the min values (right to left) for (let i = lastVisiblePeak; i >= firstVisiblePeak; i--) { - const peakX = startX + (i * pixelsPerPeak) + const peakX = startX + ((i - offsetPeakIndex) * pixelsPerPeak) const peak = waveformData[i] const minY = centerY + (peak.min * waveformHeight * 0.5) ctx.lineTo(peakX, minY) @@ -1462,6 +1496,58 @@ class TimelineWindowV2 extends Widget { } } } + } else if (track.type === 'video') { + // Draw video clips for VideoLayer + const videoLayer = track.object + const y = this.trackHierarchy.getTrackY(i) + const trackHeight = this.trackHierarchy.trackHeight // Use base height for clips + + // Draw each clip + for (let clip of videoLayer.clips) { + const startX = this.timelineState.timeToPixel(clip.startTime) + const endX = this.timelineState.timeToPixel(clip.startTime + clip.duration) + const clipWidth = endX - startX + + // Video clips use purple/magenta color + const clipColor = '#9b59b6' // Purple for video clips + + // Draw clip rectangle + ctx.fillStyle = clipColor + ctx.fillRect( + startX, + y + 5, + clipWidth, + trackHeight - 10 + ) + + // Draw border + ctx.strokeStyle = shadow + ctx.lineWidth = 1 + ctx.strokeRect( + startX, + y + 5, + clipWidth, + trackHeight - 10 + ) + + // Draw clip name if there's enough space + const minWidthForLabel = 40 + if (clipWidth >= minWidthForLabel) { + ctx.fillStyle = labelColor + ctx.font = '11px sans-serif' + ctx.textAlign = 'left' + ctx.textBaseline = 'middle' + + // Clip text to clip bounds + ctx.save() + ctx.beginPath() + ctx.rect(startX + 2, y + 5, clipWidth - 4, trackHeight - 10) + ctx.clip() + + ctx.fillText(clip.name, startX + 4, y + trackHeight / 2) + ctx.restore() + } + } } } @@ -2101,6 +2187,39 @@ class TimelineWindowV2 extends Widget { return true } + // Check if clicking on audio clip edge to start trimming + const audioEdgeInfo = this.getAudioClipEdgeAtPoint(track, adjustedX, adjustedY) + if (audioEdgeInfo) { + // Skip if right-clicking (button 2) + if (this.lastClickEvent?.button === 2) { + return false + } + + // Select the track + this.selectTrack(track) + + // Start audio clip edge dragging + this.draggingAudioClipEdge = { + track: track, + edge: audioEdgeInfo.edge, + clip: audioEdgeInfo.clip, + clipIndex: audioEdgeInfo.clipIndex, + audioTrack: audioEdgeInfo.audioTrack, + initialClipStart: audioEdgeInfo.clip.startTime, + initialClipDuration: audioEdgeInfo.clip.duration, + initialClipOffset: audioEdgeInfo.clip.offset, + initialLinkedVideoOffset: audioEdgeInfo.clip.linkedVideoClip?.offset || 0 + } + + // Enable global mouse events for dragging + this._globalEvents.add("mousemove") + this._globalEvents.add("mouseup") + + console.log('Started dragging audio clip', audioEdgeInfo.edge, 'edge') + if (this.requestRedraw) this.requestRedraw() + return true + } + // Check if clicking on audio clip to start dragging const audioClipInfo = this.getAudioClipAtPoint(track, adjustedX, adjustedY) if (audioClipInfo) { @@ -2132,6 +2251,70 @@ class TimelineWindowV2 extends Widget { return true } + // Check if clicking on video clip edge to start trimming + const videoEdgeInfo = this.getVideoClipEdgeAtPoint(track, adjustedX, adjustedY) + if (videoEdgeInfo) { + // Skip if right-clicking (button 2) + if (this.lastClickEvent?.button === 2) { + return false + } + + // Select the track + this.selectTrack(track) + + // Start video clip edge dragging + this.draggingVideoClipEdge = { + track: track, + edge: videoEdgeInfo.edge, + clip: videoEdgeInfo.clip, + clipIndex: videoEdgeInfo.clipIndex, + videoLayer: videoEdgeInfo.videoLayer, + initialClipStart: videoEdgeInfo.clip.startTime, + initialClipDuration: videoEdgeInfo.clip.duration, + initialClipOffset: videoEdgeInfo.clip.offset, + initialLinkedAudioOffset: videoEdgeInfo.clip.linkedAudioClip?.offset || 0 + } + + // Enable global mouse events for dragging + this._globalEvents.add("mousemove") + this._globalEvents.add("mouseup") + + console.log('Started dragging video clip', videoEdgeInfo.edge, 'edge') + if (this.requestRedraw) this.requestRedraw() + return true + } + + // Check if clicking on video clip to start dragging + const videoClipInfo = this.getVideoClipAtPoint(track, adjustedX, adjustedY) + if (videoClipInfo) { + // Skip drag if right-clicking (button 2) + if (this.lastClickEvent?.button === 2) { + return false + } + + // Select the track + this.selectTrack(track) + + // Start video clip dragging + const clickTime = this.timelineState.pixelToTime(adjustedX) + this.draggingVideoClip = { + track: track, + clip: videoClipInfo.clip, + clipIndex: videoClipInfo.clipIndex, + videoLayer: videoClipInfo.videoLayer, + initialMouseTime: clickTime, + initialClipStartTime: videoClipInfo.clip.startTime + } + + // Enable global mouse events for dragging + this._globalEvents.add("mousemove") + this._globalEvents.add("mouseup") + + console.log('Started dragging video clip at time', videoClipInfo.clip.startTime) + if (this.requestRedraw) this.requestRedraw() + return true + } + // Phase 6: Check if clicking on segment to start dragging const segmentInfo = this.getSegmentAtPoint(track, adjustedX, adjustedY) if (segmentInfo) { @@ -2657,6 +2840,115 @@ class TimelineWindowV2 extends Widget { return null } + getAudioClipEdgeAtPoint(track, x, y) { + const clipInfo = this.getAudioClipAtPoint(track, x, y) + if (!clipInfo) return null + + const clickTime = this.timelineState.pixelToTime(x) + const edgeThreshold = 8 / this.timelineState.pixelsPerSecond // 8 pixels in time units + + const clipStart = clipInfo.clip.startTime + const clipEnd = clipInfo.clip.startTime + clipInfo.clip.duration + + // Check if near left edge + if (Math.abs(clickTime - clipStart) <= edgeThreshold) { + return { + edge: 'left', + clip: clipInfo.clip, + clipIndex: clipInfo.clipIndex, + audioTrack: clipInfo.audioTrack, + clipStart: clipStart, + clipEnd: clipEnd + } + } + + // Check if near right edge + if (Math.abs(clickTime - clipEnd) <= edgeThreshold) { + return { + edge: 'right', + clip: clipInfo.clip, + clipIndex: clipInfo.clipIndex, + audioTrack: clipInfo.audioTrack, + clipStart: clipStart, + clipEnd: clipEnd + } + } + + return null + } + + getVideoClipAtPoint(track, x, y) { + if (track.type !== 'video') return null + + const trackIndex = this.trackHierarchy.tracks.indexOf(track) + if (trackIndex === -1) return null + + const trackY = this.trackHierarchy.getTrackY(trackIndex) + const trackHeight = this.trackHierarchy.trackHeight + const clipTop = trackY + 5 + const clipBottom = trackY + trackHeight - 5 + + // Check if y is within clip bounds + if (y < clipTop || y > clipBottom) return null + + const clickTime = this.timelineState.pixelToTime(x) + const videoLayer = track.object + + // Check each clip + for (let i = 0; i < videoLayer.clips.length; i++) { + const clip = videoLayer.clips[i] + const clipStart = clip.startTime + const clipEnd = clip.startTime + clip.duration + + if (clickTime >= clipStart && clickTime <= clipEnd) { + return { + clip: clip, + clipIndex: i, + videoLayer: videoLayer + } + } + } + + return null + } + + getVideoClipEdgeAtPoint(track, x, y) { + const clipInfo = this.getVideoClipAtPoint(track, x, y) + if (!clipInfo) return null + + const clickTime = this.timelineState.pixelToTime(x) + const edgeThreshold = 8 / this.timelineState.pixelsPerSecond // 8 pixels in time units + + const clipStart = clipInfo.clip.startTime + const clipEnd = clipInfo.clip.startTime + clipInfo.clip.duration + + // Check if near left edge + if (Math.abs(clickTime - clipStart) <= edgeThreshold) { + return { + edge: 'left', + clip: clipInfo.clip, + clipIndex: clipInfo.clipIndex, + videoLayer: clipInfo.videoLayer, + clipStart: clipStart, + clipEnd: clipEnd + } + } + + // Check if near right edge + if (Math.abs(clickTime - clipEnd) <= edgeThreshold) { + return { + edge: 'right', + clip: clipInfo.clip, + clipIndex: clipInfo.clipIndex, + videoLayer: clipInfo.videoLayer, + clipStart: clipStart, + clipEnd: clipEnd + } + } + + return null + } + /** * Get segment edge at a point (Phase 6) * Returns {edge: 'left'|'right', startTime, endTime, keyframe, animationData, curveName} if near an edge @@ -3496,6 +3788,54 @@ class TimelineWindowV2 extends Widget { return true } + // Handle audio clip edge dragging (trimming) + if (this.draggingAudioClipEdge) { + const adjustedX = x - this.trackHeaderWidth + const newTime = this.timelineState.pixelToTime(adjustedX) + const minClipDuration = this.context.config.minClipDuration + + if (this.draggingAudioClipEdge.edge === 'left') { + // Dragging left edge - adjust startTime and offset + const initialEnd = this.draggingAudioClipEdge.initialClipStart + this.draggingAudioClipEdge.initialClipDuration + const maxStartTime = initialEnd - minClipDuration + const newStartTime = Math.max(0, Math.min(newTime, maxStartTime)) + const startTimeDelta = newStartTime - this.draggingAudioClipEdge.initialClipStart + + this.draggingAudioClipEdge.clip.startTime = newStartTime + this.draggingAudioClipEdge.clip.offset = this.draggingAudioClipEdge.initialClipOffset + startTimeDelta + this.draggingAudioClipEdge.clip.duration = this.draggingAudioClipEdge.initialClipDuration - startTimeDelta + + // Also trim linked video clip if it exists + if (this.draggingAudioClipEdge.clip.linkedVideoClip) { + const videoClip = this.draggingAudioClipEdge.clip.linkedVideoClip + videoClip.startTime = newStartTime + videoClip.offset = (this.draggingAudioClipEdge.initialLinkedVideoOffset || 0) + startTimeDelta + videoClip.duration = this.draggingAudioClipEdge.initialClipDuration - startTimeDelta + } + } else { + // Dragging right edge - adjust duration + const minEndTime = this.draggingAudioClipEdge.initialClipStart + minClipDuration + const newEndTime = Math.max(minEndTime, newTime) + let newDuration = newEndTime - this.draggingAudioClipEdge.clip.startTime + + // Constrain duration to not exceed source file duration minus offset + const maxAvailableDuration = this.draggingAudioClipEdge.clip.sourceDuration - this.draggingAudioClipEdge.clip.offset + newDuration = Math.min(newDuration, maxAvailableDuration) + + this.draggingAudioClipEdge.clip.duration = newDuration + + // Also trim linked video clip if it exists + if (this.draggingAudioClipEdge.clip.linkedVideoClip) { + const linkedMaxDuration = this.draggingAudioClipEdge.clip.linkedVideoClip.sourceDuration - this.draggingAudioClipEdge.clip.linkedVideoClip.offset + this.draggingAudioClipEdge.clip.linkedVideoClip.duration = Math.min(newDuration, linkedMaxDuration) + } + } + + // Trigger timeline redraw + if (this.requestRedraw) this.requestRedraw() + return true + } + // Handle audio clip dragging if (this.draggingAudioClip) { // Adjust coordinates to timeline area @@ -3510,6 +3850,83 @@ class TimelineWindowV2 extends Widget { // Update clip's start time (ensure it doesn't go negative) this.draggingAudioClip.clip.startTime = Math.max(0, this.draggingAudioClip.initialClipStartTime + timeDelta) + // Also move linked video clip if it exists + if (this.draggingAudioClip.clip.linkedVideoClip) { + this.draggingAudioClip.clip.linkedVideoClip.startTime = this.draggingAudioClip.clip.startTime + } + + // Trigger timeline redraw + if (this.requestRedraw) this.requestRedraw() + return true + } + + // Handle video clip edge dragging (trimming) + if (this.draggingVideoClipEdge) { + const adjustedX = x - this.trackHeaderWidth + const newTime = this.timelineState.pixelToTime(adjustedX) + const minClipDuration = this.context.config.minClipDuration + + if (this.draggingVideoClipEdge.edge === 'left') { + // Dragging left edge - adjust startTime and offset + const initialEnd = this.draggingVideoClipEdge.initialClipStart + this.draggingVideoClipEdge.initialClipDuration + const maxStartTime = initialEnd - minClipDuration + const newStartTime = Math.max(0, Math.min(newTime, maxStartTime)) + const startTimeDelta = newStartTime - this.draggingVideoClipEdge.initialClipStart + + this.draggingVideoClipEdge.clip.startTime = newStartTime + this.draggingVideoClipEdge.clip.offset = this.draggingVideoClipEdge.initialClipOffset + startTimeDelta + this.draggingVideoClipEdge.clip.duration = this.draggingVideoClipEdge.initialClipDuration - startTimeDelta + + // Also trim linked audio clip if it exists + if (this.draggingVideoClipEdge.clip.linkedAudioClip) { + const audioClip = this.draggingVideoClipEdge.clip.linkedAudioClip + audioClip.startTime = newStartTime + audioClip.offset = (this.draggingVideoClipEdge.initialLinkedAudioOffset || 0) + startTimeDelta + audioClip.duration = this.draggingVideoClipEdge.initialClipDuration - startTimeDelta + } + } else { + // Dragging right edge - adjust duration + const minEndTime = this.draggingVideoClipEdge.initialClipStart + minClipDuration + const newEndTime = Math.max(minEndTime, newTime) + let newDuration = newEndTime - this.draggingVideoClipEdge.clip.startTime + + // Constrain duration to not exceed source file duration minus offset + const maxAvailableDuration = this.draggingVideoClipEdge.clip.sourceDuration - this.draggingVideoClipEdge.clip.offset + newDuration = Math.min(newDuration, maxAvailableDuration) + + this.draggingVideoClipEdge.clip.duration = newDuration + + // Also trim linked audio clip if it exists + if (this.draggingVideoClipEdge.clip.linkedAudioClip) { + const linkedMaxDuration = this.draggingVideoClipEdge.clip.linkedAudioClip.sourceDuration - this.draggingVideoClipEdge.clip.linkedAudioClip.offset + this.draggingVideoClipEdge.clip.linkedAudioClip.duration = Math.min(newDuration, linkedMaxDuration) + } + } + + // Trigger timeline redraw + if (this.requestRedraw) this.requestRedraw() + return true + } + + // Handle video clip dragging + if (this.draggingVideoClip) { + // Adjust coordinates to timeline area + const adjustedX = x - this.trackHeaderWidth + + // Convert mouse position to time + const newTime = this.timelineState.pixelToTime(adjustedX) + + // Calculate time delta + const timeDelta = newTime - this.draggingVideoClip.initialMouseTime + + // Update clip's start time (ensure it doesn't go negative) + this.draggingVideoClip.clip.startTime = Math.max(0, this.draggingVideoClip.initialClipStartTime + timeDelta) + + // Also move linked audio clip if it exists + if (this.draggingVideoClip.clip.linkedAudioClip) { + this.draggingVideoClip.clip.linkedAudioClip.startTime = this.draggingVideoClip.clip.startTime + } + // Trigger timeline redraw if (this.requestRedraw) this.requestRedraw() return true @@ -3569,6 +3986,39 @@ class TimelineWindowV2 extends Widget { return true } + // Update cursor based on hover position (when not dragging) + if (!this.draggingAudioClip && !this.draggingVideoClip && + !this.draggingAudioClipEdge && !this.draggingVideoClipEdge && + !this.draggingKeyframe && !this.draggingPlayhead && !this.draggingSegment) { + const trackY = y - this.ruler.height + if (trackY >= 0 && x >= this.trackHeaderWidth) { + const adjustedY = trackY - this.trackScrollOffset + const adjustedX = x - this.trackHeaderWidth + const track = this.trackHierarchy.getTrackAtY(adjustedY) + + if (track) { + // Check for audio clip edge + if (track.type === 'audio') { + const audioEdgeInfo = this.getAudioClipEdgeAtPoint(track, adjustedX, adjustedY) + if (audioEdgeInfo) { + this.cursor = audioEdgeInfo.edge === 'left' ? 'w-resize' : 'e-resize' + return false + } + } + // Check for video clip edge + else if (track.type === 'video') { + const videoEdgeInfo = this.getVideoClipEdgeAtPoint(track, adjustedX, adjustedY) + if (videoEdgeInfo) { + this.cursor = videoEdgeInfo.edge === 'left' ? 'w-resize' : 'e-resize' + return false + } + } + } + } + // Reset cursor if not over an edge + this.cursor = 'default' + } + return false } @@ -3631,6 +4081,67 @@ class TimelineWindowV2 extends Widget { return true } + // Complete audio clip edge dragging (trimming) + if (this.draggingAudioClipEdge) { + console.log('Finished trimming audio clip edge') + + // Update backend with new clip trim + invoke('audio_trim_clip', { + trackId: this.draggingAudioClipEdge.audioTrack.audioTrackId, + clipId: this.draggingAudioClipEdge.clip.clipId, + newStartTime: this.draggingAudioClipEdge.clip.startTime, + newDuration: this.draggingAudioClipEdge.clip.duration, + newOffset: this.draggingAudioClipEdge.clip.offset + }).catch(error => { + console.error('Failed to trim audio clip in backend:', error) + }) + + // Also update linked video clip if it exists + if (this.draggingAudioClipEdge.clip.linkedVideoClip) { + console.log('Linked video clip also trimmed') + } + + // Clean up dragging state + this.draggingAudioClipEdge = null + this._globalEvents.delete("mousemove") + this._globalEvents.delete("mouseup") + + // Final redraw + if (this.requestRedraw) this.requestRedraw() + return true + } + + // Complete video clip edge dragging (trimming) + if (this.draggingVideoClipEdge) { + console.log('Finished trimming video clip edge') + + // Update linked audio clip in backend if it exists + if (this.draggingVideoClipEdge.clip.linkedAudioClip) { + const linkedAudioClip = this.draggingVideoClipEdge.clip.linkedAudioClip + const audioTrack = this.draggingVideoClipEdge.videoLayer.linkedAudioTrack + if (audioTrack) { + invoke('audio_trim_clip', { + trackId: audioTrack.audioTrackId, + clipId: linkedAudioClip.clipId, + newStartTime: linkedAudioClip.startTime, + newDuration: linkedAudioClip.duration, + newOffset: linkedAudioClip.offset + }).catch(error => { + console.error('Failed to trim linked audio clip in backend:', error) + }) + } + } + + // Clean up dragging state + this.draggingVideoClipEdge = null + this._globalEvents.delete("mousemove") + this._globalEvents.delete("mouseup") + + // Final redraw + if (this.requestRedraw) this.requestRedraw() + return true + } + // Complete audio clip dragging if (this.draggingAudioClip) { console.log('Finished dragging audio clip') @@ -3644,6 +4155,12 @@ class TimelineWindowV2 extends Widget { console.error('Failed to move clip in backend:', error) }) + // Also update linked video clip in backend if it exists + if (this.draggingAudioClip.clip.linkedVideoClip) { + // Video clips don't have a backend move command yet, so just log for now + console.log('Linked video clip also moved to time', this.draggingAudioClip.clip.startTime) + } + // Clean up dragging state this.draggingAudioClip = null this._globalEvents.delete("mousemove") @@ -3654,6 +4171,37 @@ class TimelineWindowV2 extends Widget { return true } + // Complete video clip dragging + if (this.draggingVideoClip) { + console.log('Finished dragging video clip') + + // Video clips don't have a backend position yet (they're just visual) + // But we need to update the linked audio clip in the backend + if (this.draggingVideoClip.clip.linkedAudioClip) { + const linkedAudioClip = this.draggingVideoClip.clip.linkedAudioClip + // Find the audio track that contains this clip + const audioTrack = this.draggingVideoClip.videoLayer.linkedAudioTrack + if (audioTrack) { + invoke('audio_move_clip', { + trackId: audioTrack.audioTrackId, + clipId: linkedAudioClip.clipId, + newStartTime: linkedAudioClip.startTime + }).catch(error => { + console.error('Failed to move linked audio clip in backend:', error) + }) + } + } + + // Clean up dragging state + this.draggingVideoClip = null + this._globalEvents.delete("mousemove") + this._globalEvents.delete("mouseup") + + // Final redraw + if (this.requestRedraw) this.requestRedraw() + return true + } + // Phase 6: Complete segment dragging if (this.draggingSegment) { console.log('Finished dragging segment') From 09426e21f4a505c4cf29b25619a7d224460ef60a Mon Sep 17 00:00:00 2001 From: Skyler Lehmkuhl Date: Thu, 6 Nov 2025 06:42:12 -0500 Subject: [PATCH 3/5] use channel and jpeg compression to speed up playback --- src-tauri/Cargo.lock | 26 ++++++++++++ src-tauri/Cargo.toml | 1 + src-tauri/src/video.rs | 58 +++++++++++++++++++++++--- src/models/layer.js | 92 +++++++++++++++++++++++++++++++++--------- 4 files changed, 151 insertions(+), 26 deletions(-) diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index ed43b02..e40e7f6 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -635,6 +635,12 @@ dependencies = [ "objc", ] +[[package]] +name = "color_quant" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" + [[package]] name = "combine" version = "4.6.7" @@ -2197,6 +2203,19 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "image" +version = "0.24.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5690139d2f55868e080017335e4b94cb7414274c74f1669c84fb5feba2c9f69d" +dependencies = [ + "bytemuck", + "byteorder", + "color_quant", + "jpeg-decoder", + "num-traits", +] + [[package]] name = "indexmap" version = "1.9.3" @@ -2337,6 +2356,12 @@ dependencies = [ "libc", ] +[[package]] +name = "jpeg-decoder" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00810f1d8b74be64b13dbf3db89ac67740615d6c891f0e7b6179326533011a07" + [[package]] name = "js-sys" version = "0.3.77" @@ -2467,6 +2492,7 @@ dependencies = [ "cpal", "daw-backend", "ffmpeg-next", + "image", "log", "lru", "rtrb", diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index 9e1b91c..3ca25d7 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -39,6 +39,7 @@ rtrb = "0.3" # Video decoding ffmpeg-next = "7.0" lru = "0.12" +image = { version = "0.24", default-features = false, features = ["jpeg"] } [profile.dev] opt-level = 1 # Enable basic optimizations in debug mode for audio decoding performance diff --git a/src-tauri/src/video.rs b/src-tauri/src/video.rs index 38feeb7..b58f035 100644 --- a/src-tauri/src/video.rs +++ b/src-tauri/src/video.rs @@ -1,8 +1,10 @@ use std::sync::{Arc, Mutex}; use std::num::NonZeroUsize; +use std::io::Cursor; use ffmpeg_next as ffmpeg; use lru::LruCache; use daw_backend::WaveformPeak; +use image::{RgbaImage, ImageEncoder}; #[derive(serde::Serialize, Clone)] pub struct VideoFileMetadata { @@ -453,16 +455,16 @@ fn generate_waveform(audio_data: &[f32], channels: u32, target_peaks: usize) -> waveform } -// Use a custom serializer wrapper for efficient binary transfer -#[derive(serde::Serialize)] -struct BinaryFrame(#[serde(with = "serde_bytes")] Vec); - #[tauri::command] pub async fn video_get_frame( state: tauri::State<'_, Arc>>, pool_index: usize, timestamp: f64, -) -> Result, String> { + use_jpeg: bool, + channel: tauri::ipc::Channel, +) -> Result<(), String> { + use std::time::Instant; + let video_state = state.lock().unwrap(); let decoder = video_state.pool.get(pool_index) @@ -472,7 +474,51 @@ pub async fn video_get_frame( drop(video_state); let mut decoder = decoder.lock().unwrap(); - decoder.get_frame(timestamp) + let frame_data = decoder.get_frame(timestamp)?; + + let data_to_send = if use_jpeg { + let t_compress_start = Instant::now(); + + // Get frame dimensions from decoder + let width = decoder.output_width; + let height = decoder.output_height; + + // Create image from raw RGBA data + let img = RgbaImage::from_raw(width, height, frame_data) + .ok_or("Failed to create image from frame data")?; + + // Convert RGBA to RGB (JPEG doesn't support alpha) + let rgb_img = image::DynamicImage::ImageRgba8(img).to_rgb8(); + + // Encode to JPEG with quality 85 (good balance of size/quality) + let mut jpeg_data = Vec::new(); + let mut encoder = image::codecs::jpeg::JpegEncoder::new_with_quality(&mut jpeg_data, 85); + encoder.encode( + rgb_img.as_raw(), + rgb_img.width(), + rgb_img.height(), + image::ColorType::Rgb8 + ).map_err(|e| format!("JPEG encoding failed: {}", e))?; + + let compress_time = t_compress_start.elapsed().as_millis(); + let original_size = width as usize * height as usize * 4; + let compressed_size = jpeg_data.len(); + let ratio = original_size as f32 / compressed_size as f32; + + eprintln!("[Video JPEG] Compressed {}KB -> {}KB ({}x) in {}ms", + original_size / 1024, compressed_size / 1024, ratio, compress_time); + + jpeg_data + } else { + frame_data + }; + + // Send binary data through channel (bypasses JSON serialization) + // InvokeResponseBody::Raw sends raw binary data without JSON encoding + channel.send(tauri::ipc::InvokeResponseBody::Raw(data_to_send)) + .map_err(|e| format!("Channel send error: {}", e))?; + + Ok(()) } #[tauri::command] diff --git a/src/models/layer.js b/src/models/layer.js index b6a9ef3..31f06b7 100644 --- a/src/models/layer.js +++ b/src/models/layer.js @@ -18,7 +18,7 @@ import { const Tone = window.Tone; // Tauri API -const { invoke } = window.__TAURI__.core; +const { invoke, Channel } = window.__TAURI__.core; // Helper function for UUID generation function uuidv4() { @@ -1280,6 +1280,9 @@ class VideoLayer extends Widget { // Associated audio track (if video has audio) this.linkedAudioTrack = null; // Reference to AudioTrack + // Performance settings + this.useJpegCompression = true; // Enable JPEG compression for faster transfer (default: true) + // Timeline display this.collapsed = false; this.curvesMode = 'segment'; @@ -1343,34 +1346,76 @@ class VideoLayer extends Widget { clip.lastFetchedTimestamp = videoTimestamp; try { - // Request frame from Rust backend + // Request frame from Rust backend using IPC Channel for efficient binary transfer const t_start = performance.now(); - let frameData = await invoke('video_get_frame', { - poolIndex: clip.poolIndex, - timestamp: videoTimestamp + + // Create a promise that resolves when channel receives data + const frameDataPromise = new Promise((resolve, reject) => { + const channel = new Channel(); + + channel.onmessage = (data) => { + resolve(data); + }; + + // Invoke command with channel + invoke('video_get_frame', { + poolIndex: clip.poolIndex, + timestamp: videoTimestamp, + useJpeg: this.useJpegCompression, + channel: channel + }).catch(reject); }); + + // Wait for the frame data + let frameData = await frameDataPromise; const t_after_ipc = performance.now(); - // Handle different formats that Tauri might return - // ByteBuf from Rust can come as Uint8Array or Array depending on serialization + // Ensure data is Uint8Array if (!(frameData instanceof Uint8Array)) { frameData = new Uint8Array(frameData); } - // Validate frame data size - const expectedSize = clip.width * clip.height * 4; // RGBA = 4 bytes per pixel + let imageData; + const t_before_conversion = performance.now(); - if (frameData.length !== expectedSize) { - throw new Error(`Invalid frame data size: got ${frameData.length}, expected ${expectedSize}`); + if (this.useJpegCompression) { + // Decode JPEG data + const blob = new Blob([frameData], { type: 'image/jpeg' }); + const imageUrl = URL.createObjectURL(blob); + + // Load and decode JPEG + const img = new Image(); + await new Promise((resolve, reject) => { + img.onload = resolve; + img.onerror = reject; + img.src = imageUrl; + }); + + // Create temporary canvas to extract ImageData + const tempCanvas = document.createElement('canvas'); + tempCanvas.width = clip.width; + tempCanvas.height = clip.height; + const tempCtx = tempCanvas.getContext('2d'); + tempCtx.drawImage(img, 0, 0); + imageData = tempCtx.getImageData(0, 0, clip.width, clip.height); + + // Cleanup + URL.revokeObjectURL(imageUrl); + } else { + // Raw RGBA data + const expectedSize = clip.width * clip.height * 4; // RGBA = 4 bytes per pixel + + if (frameData.length !== expectedSize) { + throw new Error(`Invalid frame data size: got ${frameData.length}, expected ${expectedSize}`); + } + + imageData = new ImageData( + new Uint8ClampedArray(frameData), + clip.width, + clip.height + ); } - // Convert to ImageData - const t_before_conversion = performance.now(); - const imageData = new ImageData( - new Uint8ClampedArray(frameData), - clip.width, - clip.height - ); const t_after_conversion = performance.now(); // Create or reuse temp canvas @@ -1392,8 +1437,9 @@ class VideoLayer extends Widget { const ipc_time = t_after_ipc - t_start; const conversion_time = t_after_conversion - t_before_conversion; const putimage_time = t_after_putimage - t_before_putimage; + const compression_mode = this.useJpegCompression ? 'JPEG' : 'RAW'; - console.log(`[JS Video Timing] ts=${videoTimestamp.toFixed(3)}s | Total: ${total_time.toFixed(1)}ms | IPC: ${ipc_time.toFixed(1)}ms (${(ipc_time/total_time*100).toFixed(0)}%) | Convert: ${conversion_time.toFixed(1)}ms | PutImage: ${putimage_time.toFixed(1)}ms | Size: ${(frameData.length/1024/1024).toFixed(2)}MB`); + console.log(`[JS Video Timing ${compression_mode}] ts=${videoTimestamp.toFixed(3)}s | Total: ${total_time.toFixed(1)}ms | IPC: ${ipc_time.toFixed(1)}ms (${(ipc_time/total_time*100).toFixed(0)}%) | Convert: ${conversion_time.toFixed(1)}ms | PutImage: ${putimage_time.toFixed(1)}ms | Size: ${(frameData.length/1024/1024).toFixed(2)}MB`); } catch (error) { console.error('Failed to get video frame:', error); clip.currentFrame = null; @@ -1479,6 +1525,11 @@ class VideoLayer extends Widget { videoLayer.visible = json.visible; videoLayer.audible = json.audible; + // Restore compression setting (default to true if not specified for backward compatibility) + if (json.useJpegCompression !== undefined) { + videoLayer.useJpegCompression = json.useJpegCompression; + } + return videoLayer; } @@ -1491,7 +1542,8 @@ class VideoLayer extends Widget { audible: this.audible, animationData: this.animationData.toJSON(), clips: this.clips, - linkedAudioTrack: this.linkedAudioTrack?.idx + linkedAudioTrack: this.linkedAudioTrack?.idx, + useJpegCompression: this.useJpegCompression }; } From 430ecb0ae6ab1a820b5140f5b5e2aea2e471340a Mon Sep 17 00:00:00 2001 From: Skyler Lehmkuhl Date: Thu, 6 Nov 2025 11:36:56 -0500 Subject: [PATCH 4/5] use native player to speed up playback --- src-tauri/Cargo.lock | 31 +++ src-tauri/Cargo.toml | 3 + src-tauri/src/lib.rs | 11 + src-tauri/src/video.rs | 359 +++++++++++++++++++++++++-- src-tauri/src/video_server.rs | 203 +++++++++++++++ src/actions/index.js | 3 +- src/main.js | 86 +++++-- src/models/layer.js | 447 ++++++++++++++++++++++++++++------ src/state.js | 3 +- 9 files changed, 1039 insertions(+), 107 deletions(-) create mode 100644 src-tauri/src/video_server.rs diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index e40e7f6..fb1f00e 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -136,6 +136,12 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" +[[package]] +name = "ascii" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d92bec98840b8f03a5ff5413de5293bfcd8bf96467cf5452609f939ec6f5de16" + [[package]] name = "ashpd" version = "0.10.2" @@ -594,6 +600,12 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "chunked_transfer" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e4de3bc4ea267985becf712dc6d9eed8b04c953b3fcfb339ebc87acd9804901" + [[package]] name = "clang-sys" version = "1.8.1" @@ -1987,6 +1999,12 @@ version = "1.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d71d3574edd2771538b901e6549113b4006ece66150fb69c0fb6d9a2adae946" +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + [[package]] name = "hyper" version = "1.5.2" @@ -2504,6 +2522,7 @@ dependencies = [ "tauri-plugin-fs", "tauri-plugin-log", "tauri-plugin-shell", + "tiny_http", "tracing", "tracing-subscriber", ] @@ -5191,6 +5210,18 @@ dependencies = [ "time-core", ] +[[package]] +name = "tiny_http" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389915df6413a2e74fb181895f933386023c71110878cd0825588928e64cdc82" +dependencies = [ + "ascii", + "chunked_transfer", + "httpdate", + "log", +] + [[package]] name = "tinystr" version = "0.7.6" diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index 3ca25d7..248155a 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -41,6 +41,9 @@ ffmpeg-next = "7.0" lru = "0.12" image = { version = "0.24", default-features = false, features = ["jpeg"] } +# HTTP server for video streaming +tiny_http = "0.12" + [profile.dev] opt-level = 1 # Enable basic optimizations in debug mode for audio decoding performance diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 6aca3dd..3d91f68 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -8,6 +8,7 @@ use tauri::{AppHandle, Manager, Url, WebviewUrl, WebviewWindowBuilder}; mod audio; mod video; +mod video_server; #[derive(Default)] @@ -128,10 +129,16 @@ fn handle_file_associations(app: AppHandle, files: Vec) { #[cfg_attr(mobile, tauri::mobile_entry_point)] pub fn run() { let pkg_name = env!("CARGO_PKG_NAME").to_string(); + // Initialize video HTTP server + let video_server = video_server::VideoServer::new() + .expect("Failed to start video server"); + eprintln!("[App] Video server started on port {}", video_server.port()); + tauri::Builder::default() .manage(Mutex::new(AppState::default())) .manage(Arc::new(Mutex::new(audio::AudioState::default()))) .manage(Arc::new(Mutex::new(video::VideoState::default()))) + .manage(Arc::new(Mutex::new(video_server))) .setup(|app| { #[cfg(any(windows, target_os = "linux"))] // Windows/Linux needs different handling from macOS { @@ -255,8 +262,12 @@ pub fn run() { audio::audio_load_track_graph, video::video_load_file, video::video_get_frame, + video::video_get_frames_batch, video::video_set_cache_size, video::video_get_pool_info, + video::video_ipc_benchmark, + video::video_get_transcode_status, + video::video_allow_asset, ]) // .manage(window_counter) .build(tauri::generate_context!()) diff --git a/src-tauri/src/video.rs b/src-tauri/src/video.rs index b58f035..9b2490a 100644 --- a/src-tauri/src/video.rs +++ b/src-tauri/src/video.rs @@ -1,10 +1,10 @@ use std::sync::{Arc, Mutex}; use std::num::NonZeroUsize; -use std::io::Cursor; use ffmpeg_next as ffmpeg; use lru::LruCache; use daw_backend::WaveformPeak; -use image::{RgbaImage, ImageEncoder}; +use image::RgbaImage; +use tauri::Manager; #[derive(serde::Serialize, Clone)] pub struct VideoFileMetadata { @@ -19,6 +19,10 @@ pub struct VideoFileMetadata { pub audio_sample_rate: Option, pub audio_channels: Option, pub audio_waveform: Option>, + pub codec_name: String, + pub is_browser_compatible: bool, + pub http_url: Option, // HTTP URL to stream video (if compatible or transcode complete) + pub transcoding: bool, // True if currently transcoding } struct VideoDecoder { @@ -119,7 +123,7 @@ impl VideoDecoder { return Ok(cached_frame.clone()); } - let t_after_cache = Instant::now(); + let _t_after_cache = Instant::now(); // Determine if we need to seek // Seek if: no decoder open, going backwards, or jumping forward more than 2 seconds @@ -240,10 +244,24 @@ impl VideoDecoder { } } +use std::collections::HashMap; +use std::path::PathBuf; + +#[derive(Clone)] +pub struct TranscodeJob { + pub pool_index: usize, + pub input_path: String, + pub output_path: String, + pub http_url: Option, // HTTP URL when transcode completes + pub progress: f32, // 0.0 to 1.0 + pub completed: bool, +} + pub struct VideoState { pool: Vec>>, next_pool_index: usize, cache_size: usize, + transcode_jobs: Arc>>, // pool_index -> job } impl Default for VideoState { @@ -252,6 +270,7 @@ impl Default for VideoState { pool: Vec::new(), next_pool_index: 0, cache_size: 20, // Default cache size + transcode_jobs: Arc::new(Mutex::new(HashMap::new())), } } } @@ -260,8 +279,11 @@ impl Default for VideoState { pub async fn video_load_file( video_state: tauri::State<'_, Arc>>, audio_state: tauri::State<'_, Arc>>, + video_server: tauri::State<'_, Arc>>, path: String, ) -> Result { + eprintln!("[Video] Loading file: {}", path); + ffmpeg::init().map_err(|e| e.to_string())?; // Open input to check for audio stream @@ -386,13 +408,45 @@ pub async fn video_load_file( (None, None, None, None, None) }; + // Detect video codec + let video_stream = input.streams() + .best(ffmpeg::media::Type::Video) + .ok_or("No video stream found")?; + + let codec_id = video_stream.parameters().id(); + let codec_name = ffmpeg::codec::Id::name(&codec_id).to_string(); + + // Check if codec is browser-compatible (can play directly) + // Browsers support: H.264/AVC, VP8, VP9, AV1 (limited) + let is_browser_compatible = matches!( + codec_id, + ffmpeg::codec::Id::H264 | + ffmpeg::codec::Id::VP8 | + ffmpeg::codec::Id::VP9 | + ffmpeg::codec::Id::AV1 + ); + + eprintln!("[Video Codec] {} - Browser compatible: {}", codec_name, is_browser_compatible); + // Create video decoder with max dimensions for playback (800x600) // This scales down high-res videos to reduce data transfer let mut video_state_guard = video_state.lock().unwrap(); let pool_index = video_state_guard.next_pool_index; video_state_guard.next_pool_index += 1; - let decoder = VideoDecoder::new(path, video_state_guard.cache_size, Some(800), Some(600))?; + let decoder = VideoDecoder::new(path.clone(), video_state_guard.cache_size, Some(800), Some(600))?; + + // Add file to HTTP server if browser-compatible + let http_url = if is_browser_compatible { + let server = video_server.lock().unwrap(); + let url_path = format!("/video/{}", pool_index); + server.add_file(url_path.clone(), PathBuf::from(&path)); + let http_url = server.get_url(&url_path); + eprintln!("[Video] Browser-compatible, serving at: {}", http_url); + Some(http_url) + } else { + None + }; let metadata = VideoFileMetadata { pool_index, @@ -406,13 +460,152 @@ pub async fn video_load_file( audio_sample_rate, audio_channels, audio_waveform, + codec_name, + is_browser_compatible, + http_url, + transcoding: !is_browser_compatible, }; video_state_guard.pool.push(Arc::new(Mutex::new(decoder))); + // Start background transcoding if not browser-compatible + if !is_browser_compatible { + eprintln!("[Video Transcode] Starting background transcode for pool_index {}", pool_index); + let jobs = video_state_guard.transcode_jobs.clone(); + let input_path = path.clone(); + let pool_idx = pool_index; + let server = video_server.inner().clone(); + + tauri::async_runtime::spawn(async move { + if let Err(e) = start_transcode(jobs, pool_idx, input_path, server).await { + eprintln!("[Video Transcode] Failed: {}", e); + } + }); + } + Ok(metadata) } +// Background transcode to WebM/VP9 for browser compatibility +async fn start_transcode( + jobs: Arc>>, + pool_index: usize, + input_path: String, + video_server: Arc>, +) -> Result<(), String> { + use std::process::Command; + + // Generate output path in system cache directory + let cache_dir = std::env::temp_dir().join("lightningbeam_transcoded"); + std::fs::create_dir_all(&cache_dir).map_err(|e| e.to_string())?; + + let input_file = PathBuf::from(&input_path); + let file_stem = input_file.file_stem() + .ok_or("Invalid input path")? + .to_string_lossy(); + let output_path = cache_dir.join(format!("{}_{}.webm", file_stem, pool_index)); + + // Create job entry + { + let mut jobs_guard = jobs.lock().unwrap(); + jobs_guard.insert(pool_index, TranscodeJob { + pool_index, + input_path: input_path.clone(), + output_path: output_path.to_string_lossy().to_string(), + http_url: None, + progress: 0.0, + completed: false, + }); + } + + eprintln!("[Video Transcode] Output: {}", output_path.display()); + + // Run FFmpeg transcode command + // Using VP9 codec with CRF 30 (good quality/size balance) and fast encoding + let output = Command::new("ffmpeg") + .args(&[ + "-i", &input_path, + "-c:v", "libvpx-vp9", // VP9 video codec + "-crf", "30", // Quality (lower = better, 23-32 recommended) + "-b:v", "0", // Use CRF mode + "-threads", "4", // Use 4 threads + "-row-mt", "1", // Enable row-based multithreading + "-speed", "4", // Encoding speed (0=slowest/best, 4=good balance) + "-c:a", "libopus", // Opus audio codec (best for WebM) + "-b:a", "128k", // Audio bitrate + "-y", // Overwrite output + output_path.to_str().ok_or("Invalid output path")?, + ]) + .output() + .map_err(|e| format!("Failed to spawn ffmpeg: {}", e))?; + + if output.status.success() { + eprintln!("[Video Transcode] Completed: {}", output_path.display()); + + // Add transcoded file to HTTP server + let server = video_server.lock().unwrap(); + let url_path = format!("/video/{}", pool_index); + server.add_file(url_path.clone(), output_path.clone()); + let http_url = server.get_url(&url_path); + eprintln!("[Video Transcode] Serving at: {}", http_url); + drop(server); + + // Mark as completed and store HTTP URL + let mut jobs_guard = jobs.lock().unwrap(); + if let Some(job) = jobs_guard.get_mut(&pool_index) { + job.progress = 1.0; + job.completed = true; + job.http_url = Some(http_url); + } + eprintln!("[Video Transcode] Job completed for pool_index {}", pool_index); + Ok(()) + } else { + let stderr = String::from_utf8_lossy(&output.stderr); + eprintln!("[Video Transcode] FFmpeg error: {}", stderr); + Err(format!("FFmpeg failed: {}", stderr)) + } +} + +// Get transcode status for a pool index +#[tauri::command] +pub async fn video_get_transcode_status( + video_state: tauri::State<'_, Arc>>, + pool_index: usize, +) -> Result)>, String> { + let state = video_state.lock().unwrap(); + let jobs = state.transcode_jobs.lock().unwrap(); + + if let Some(job) = jobs.get(&pool_index) { + Ok(Some((job.output_path.clone(), job.progress, job.completed, job.http_url.clone()))) + } else { + Ok(None) + } +} + +// Add a video file to asset protocol scope so browser can access it +#[tauri::command] +pub async fn video_allow_asset( + app: tauri::AppHandle, + path: String, +) -> Result<(), String> { + use tauri_plugin_fs::FsExt; + + let file_path = PathBuf::from(&path); + + // Add to FS scope + let fs_scope = app.fs_scope(); + fs_scope.allow_file(&file_path) + .map_err(|e| format!("Failed to allow file in fs scope: {}", e))?; + + // Add to asset protocol scope + let asset_scope = app.asset_protocol_scope(); + asset_scope.allow_file(&file_path) + .map_err(|e| format!("Failed to allow file in asset scope: {}", e))?; + + eprintln!("[Video] Added to asset scope: {}", path); + Ok(()) +} + fn generate_waveform(audio_data: &[f32], channels: u32, target_peaks: usize) -> Vec { let total_samples = audio_data.len(); let samples_per_channel = total_samples / channels as usize; @@ -465,6 +658,9 @@ pub async fn video_get_frame( ) -> Result<(), String> { use std::time::Instant; + let t_total_start = Instant::now(); + + let t_lock_start = Instant::now(); let video_state = state.lock().unwrap(); let decoder = video_state.pool.get(pool_index) @@ -474,11 +670,14 @@ pub async fn video_get_frame( drop(video_state); let mut decoder = decoder.lock().unwrap(); + let t_lock_end = Instant::now(); + + let t_decode_start = Instant::now(); let frame_data = decoder.get_frame(timestamp)?; + let t_decode_end = Instant::now(); + let t_compress_start = Instant::now(); let data_to_send = if use_jpeg { - let t_compress_start = Instant::now(); - // Get frame dimensions from decoder let width = decoder.output_width; let height = decoder.output_height; @@ -500,23 +699,36 @@ pub async fn video_get_frame( image::ColorType::Rgb8 ).map_err(|e| format!("JPEG encoding failed: {}", e))?; - let compress_time = t_compress_start.elapsed().as_millis(); - let original_size = width as usize * height as usize * 4; - let compressed_size = jpeg_data.len(); - let ratio = original_size as f32 / compressed_size as f32; - - eprintln!("[Video JPEG] Compressed {}KB -> {}KB ({}x) in {}ms", - original_size / 1024, compressed_size / 1024, ratio, compress_time); - jpeg_data } else { frame_data }; + let t_compress_end = Instant::now(); + // Drop decoder lock before sending to avoid blocking + drop(decoder); + + let t_send_start = Instant::now(); // Send binary data through channel (bypasses JSON serialization) // InvokeResponseBody::Raw sends raw binary data without JSON encoding - channel.send(tauri::ipc::InvokeResponseBody::Raw(data_to_send)) + channel.send(tauri::ipc::InvokeResponseBody::Raw(data_to_send.clone())) .map_err(|e| format!("Channel send error: {}", e))?; + let t_send_end = Instant::now(); + + let t_total_end = Instant::now(); + + // Detailed profiling + let lock_time = t_lock_end.duration_since(t_lock_start).as_micros(); + let decode_time = t_decode_end.duration_since(t_decode_start).as_micros(); + let compress_time = t_compress_end.duration_since(t_compress_start).as_micros(); + let send_time = t_send_end.duration_since(t_send_start).as_micros(); + let total_time = t_total_end.duration_since(t_total_start).as_micros(); + + let size_kb = data_to_send.len() / 1024; + let mode = if use_jpeg { "JPEG" } else { "RAW" }; + + eprintln!("[Video Profile {}] Size: {}KB | Lock: {}μs | Decode: {}μs | Compress: {}μs | Send: {}μs | Total: {}μs", + mode, size_kb, lock_time, decode_time, compress_time, send_time, total_time); Ok(()) } @@ -547,3 +759,120 @@ pub async fn video_get_pool_info( decoder.fps )) } + +// Benchmark command to test IPC performance with various payload sizes +#[tauri::command] +pub async fn video_ipc_benchmark( + size_bytes: usize, + channel: tauri::ipc::Channel, +) -> Result<(), String> { + use std::time::Instant; + + let t_start = Instant::now(); + + // Create dummy data of requested size + let data = vec![0u8; size_bytes]; + + let t_after_alloc = Instant::now(); + + // Send through channel + channel.send(tauri::ipc::InvokeResponseBody::Raw(data)) + .map_err(|e| format!("Channel send error: {}", e))?; + + let t_after_send = Instant::now(); + + let alloc_time = t_after_alloc.duration_since(t_start).as_micros(); + let send_time = t_after_send.duration_since(t_after_alloc).as_micros(); + let total_time = t_after_send.duration_since(t_start).as_micros(); + + eprintln!("[IPC Benchmark Rust] Size: {}KB | Alloc: {}μs | Send: {}μs | Total: {}μs", + size_bytes / 1024, alloc_time, send_time, total_time); + + Ok(()) +} + +// Batch frame request - get multiple frames in one IPC call +#[tauri::command] +pub async fn video_get_frames_batch( + state: tauri::State<'_, Arc>>, + pool_index: usize, + timestamps: Vec, + use_jpeg: bool, + channel: tauri::ipc::Channel, +) -> Result<(), String> { + use std::time::Instant; + + let t_total_start = Instant::now(); + + let video_state = state.lock().unwrap(); + let decoder = video_state.pool.get(pool_index) + .ok_or("Invalid pool index")? + .clone(); + drop(video_state); + + let mut decoder = decoder.lock().unwrap(); + + // Decode all frames + let mut all_frames = Vec::new(); + let mut total_decode_time = 0u128; + let mut total_compress_time = 0u128; + + for timestamp in ×tamps { + let t_decode_start = Instant::now(); + let frame_data = decoder.get_frame(*timestamp)?; + let t_decode_end = Instant::now(); + total_decode_time += t_decode_end.duration_since(t_decode_start).as_micros(); + + let t_compress_start = Instant::now(); + let data = if use_jpeg { + let width = decoder.output_width; + let height = decoder.output_height; + let img = RgbaImage::from_raw(width, height, frame_data) + .ok_or("Failed to create image from frame data")?; + let rgb_img = image::DynamicImage::ImageRgba8(img).to_rgb8(); + let mut jpeg_data = Vec::new(); + let mut encoder = image::codecs::jpeg::JpegEncoder::new_with_quality(&mut jpeg_data, 85); + encoder.encode( + rgb_img.as_raw(), + rgb_img.width(), + rgb_img.height(), + image::ColorType::Rgb8 + ).map_err(|e| format!("JPEG encoding failed: {}", e))?; + jpeg_data + } else { + frame_data + }; + let t_compress_end = Instant::now(); + total_compress_time += t_compress_end.duration_since(t_compress_start).as_micros(); + + all_frames.push(data); + } + + drop(decoder); + + // Pack all frames into one buffer with metadata + // Format: [frame_count: u32][frame1_size: u32][frame1_data...][frame2_size: u32][frame2_data...] + let mut packed_data = Vec::new(); + packed_data.extend_from_slice(&(all_frames.len() as u32).to_le_bytes()); + + for frame in &all_frames { + packed_data.extend_from_slice(&(frame.len() as u32).to_le_bytes()); + packed_data.extend_from_slice(frame); + } + + let total_size_kb = packed_data.len() / 1024; + + let t_send_start = Instant::now(); + channel.send(tauri::ipc::InvokeResponseBody::Raw(packed_data)) + .map_err(|e| format!("Channel send error: {}", e))?; + let t_send_end = Instant::now(); + + let send_time = t_send_end.duration_since(t_send_start).as_micros(); + let total_time = t_send_end.duration_since(t_total_start).as_micros(); + + let mode = if use_jpeg { "JPEG" } else { "RAW" }; + eprintln!("[Video Batch {}] Frames: {} | Size: {}KB | Decode: {}μs | Compress: {}μs | Send: {}μs | Total: {}μs", + mode, timestamps.len(), total_size_kb, total_decode_time, total_compress_time, send_time, total_time); + + Ok(()) +} diff --git a/src-tauri/src/video_server.rs b/src-tauri/src/video_server.rs new file mode 100644 index 0000000..7258221 --- /dev/null +++ b/src-tauri/src/video_server.rs @@ -0,0 +1,203 @@ +use std::sync::{Arc, Mutex}; +use std::collections::HashMap; +use std::fs::File; +use std::io::{Read, Seek, SeekFrom}; +use std::path::PathBuf; +use tiny_http::{Server, Response, Request, Header, StatusCode}; + +pub struct VideoServer { + port: u16, + allowed_files: Arc>>, // URL path -> file path +} + +impl VideoServer { + pub fn new() -> Result { + // Bind to localhost on a random available port + let server = Server::http("127.0.0.1:0") + .map_err(|e| format!("Failed to create HTTP server: {}", e))?; + + let port = match server.server_addr() { + tiny_http::ListenAddr::IP(addr) => addr.port(), + _ => return Err("Unexpected server address type".to_string()), + }; + let allowed_files = Arc::new(Mutex::new(HashMap::new())); + + eprintln!("[Video Server] Started on port {}", port); + + // Spawn server thread + let files = allowed_files.clone(); + std::thread::spawn(move || { + for request in server.incoming_requests() { + handle_request(request, &files); + } + }); + + Ok(Self { + port, + allowed_files, + }) + } + + pub fn port(&self) -> u16 { + self.port + } + + pub fn add_file(&self, url_path: String, file_path: PathBuf) { + eprintln!("[Video Server] Adding file: {} -> {:?}", url_path, file_path); + let mut files = self.allowed_files.lock().unwrap(); + files.insert(url_path, file_path); + } + + pub fn get_url(&self, url_path: &str) -> String { + format!("http://127.0.0.1:{}{}", self.port, url_path) + } +} + +fn handle_request(request: Request, allowed_files: &Arc>>) { + let url = request.url(); + let method = request.method(); + + eprintln!("[Video Server] {} {}", method, url); + + // Get file path + let files = allowed_files.lock().unwrap(); + let file_path = match files.get(url) { + Some(path) => path.clone(), + None => { + eprintln!("[Video Server] File not found: {}", url); + let response = Response::from_string("Not Found") + .with_status_code(StatusCode(404)); + let _ = request.respond(response); + return; + } + }; + drop(files); + + // Open file + let mut file = match File::open(&file_path) { + Ok(f) => f, + Err(e) => { + eprintln!("[Video Server] Failed to open file: {}", e); + let response = Response::from_string("Internal Server Error") + .with_status_code(StatusCode(500)); + let _ = request.respond(response); + return; + } + }; + + // Get file size + let file_size = match file.metadata() { + Ok(meta) => meta.len(), + Err(e) => { + eprintln!("[Video Server] Failed to get file metadata: {}", e); + let response = Response::from_string("Internal Server Error") + .with_status_code(StatusCode(500)); + let _ = request.respond(response); + return; + } + }; + + // Check for Range header - convert to owned String to avoid borrow issues + let range_header = request.headers().iter() + .find(|h| h.field.equiv("Range")) + .map(|h| h.value.as_str().to_string()); + + match range_header { + Some(range) if range.starts_with("bytes=") => { + // Parse range request + eprintln!("[Video Server] Range request: {}", range); + handle_range_request(request, file, file_size, &range, &file_path); + } + _ => { + // Serve entire file + eprintln!("[Video Server] Full file request"); + handle_full_request(request, file, file_size, &file_path); + } + } +} + +fn handle_range_request( + request: Request, + mut file: File, + file_size: u64, + range: &str, + file_path: &PathBuf, +) { + // Parse "bytes=start-end" + let range = range.trim_start_matches("bytes="); + let parts: Vec<&str> = range.split('-').collect(); + + let start = parts[0].parse::().unwrap_or(0); + let end = if parts.len() > 1 && !parts[1].is_empty() { + parts[1].parse::().unwrap_or(file_size - 1) + } else { + file_size - 1 + }; + + let length = end - start + 1; + + // Seek to start position + if let Err(e) = file.seek(SeekFrom::Start(start)) { + eprintln!("[Video Server] Failed to seek: {}", e); + let response = Response::from_string("Internal Server Error") + .with_status_code(StatusCode(500)); + let _ = request.respond(response); + return; + } + + // Read the requested range + let mut buffer = vec![0u8; length as usize]; + if let Err(e) = file.read_exact(&mut buffer) { + eprintln!("[Video Server] Failed to read range: {}", e); + let response = Response::from_string("Internal Server Error") + .with_status_code(StatusCode(500)); + let _ = request.respond(response); + return; + } + + // Determine content type + let content_type = get_content_type(file_path); + + // Send 206 Partial Content response + let content_range = format!("bytes {}-{}/{}", start, end, file_size); + let response = Response::from_data(buffer) + .with_status_code(StatusCode(206)) + .with_header(Header::from_bytes(&b"Content-Type"[..], content_type.as_bytes()).unwrap()) + .with_header(Header::from_bytes(&b"Content-Length"[..], length.to_string().as_bytes()).unwrap()) + .with_header(Header::from_bytes(&b"Content-Range"[..], content_range.as_bytes()).unwrap()) + .with_header(Header::from_bytes(&b"Accept-Ranges"[..], &b"bytes"[..]).unwrap()) + .with_header(Header::from_bytes(&b"Access-Control-Allow-Origin"[..], &b"*"[..]).unwrap()); + + let _ = request.respond(response); +} + +fn handle_full_request( + request: Request, + file: File, + file_size: u64, + file_path: &PathBuf, +) { + // Determine content type + let content_type = get_content_type(file_path); + + // Send 200 OK response using from_file to avoid chunked encoding + let response = Response::from_file(file) + .with_status_code(StatusCode(200)) + .with_chunked_threshold(usize::MAX) // Force Content-Length instead of chunked + .with_header(Header::from_bytes(&b"Content-Type"[..], content_type.as_bytes()).unwrap()) + .with_header(Header::from_bytes(&b"Accept-Ranges"[..], &b"bytes"[..]).unwrap()) + .with_header(Header::from_bytes(&b"Access-Control-Allow-Origin"[..], &b"*"[..]).unwrap()); + + let _ = request.respond(response); +} + +fn get_content_type(path: &PathBuf) -> String { + match path.extension().and_then(|s| s.to_str()) { + Some("webm") => "video/webm".to_string(), + Some("mp4") => "video/mp4".to_string(), + Some("mkv") => "video/x-matroska".to_string(), + Some("avi") => "video/x-msvideo".to_string(), + Some("mov") => "video/quicktime".to_string(), + _ => "application/octet-stream".to_string(), + } +} diff --git a/src/actions/index.js b/src/actions/index.js index ab5f79d..5f671f1 100644 --- a/src/actions/index.js +++ b/src/actions/index.js @@ -629,7 +629,8 @@ export const actions = { metadata.duration, 0, // offset action.videoname, - metadata.duration // sourceDuration + metadata.duration, // sourceDuration + metadata // Pass full metadata for browser playback support ); // If video has audio, create linked AudioTrack diff --git a/src/main.js b/src/main.js index e47be2d..7e9db0a 100644 --- a/src/main.js +++ b/src/main.js @@ -220,7 +220,6 @@ let fileExportPath = undefined; let state = "normal"; -let playing = false; let lastFrameTime; let uiDirty = false; @@ -347,6 +346,65 @@ window.actions = actions; window.addKeyframeAtPlayhead = addKeyframeAtPlayhead; window.updateVideoFrames = null; // Will be set after function is defined +// IPC Benchmark function - run from console: testIPCBenchmark() +window.testIPCBenchmark = async function() { + const { invoke, Channel } = window.__TAURI__.core; + + // Test sizes: 1KB, 10KB, 50KB, 100KB, 500KB, 1MB, 2MB, 5MB + const testSizes = [ + 1024, // 1 KB + 10 * 1024, // 10 KB + 50 * 1024, // 50 KB + 100 * 1024, // 100 KB + 500 * 1024, // 500 KB + 1024 * 1024, // 1 MB + 2 * 1024 * 1024, // 2 MB + 5 * 1024 * 1024 // 5 MB + ]; + + console.log('\n=== IPC Benchmark Starting ===\n'); + console.log('Size (KB)\tJS Total (ms)\tJS IPC (ms)\tJS Recv (ms)\tThroughput (MB/s)'); + console.log('─'.repeat(80)); + + for (const sizeBytes of testSizes) { + const t_start = performance.now(); + + let receivedData = null; + const dataPromise = new Promise((resolve, reject) => { + const channel = new Channel(); + + channel.onmessage = (data) => { + const t_recv_start = performance.now(); + receivedData = data; + const t_recv_end = performance.now(); + resolve(t_recv_end - t_recv_start); + }; + + invoke('video_ipc_benchmark', { + sizeBytes: sizeBytes, + channel: channel + }).catch(reject); + }); + + const recv_time = await dataPromise; + const t_after_ipc = performance.now(); + + const total_time = t_after_ipc - t_start; + const ipc_time = total_time - recv_time; + const size_kb = sizeBytes / 1024; + const size_mb = sizeBytes / (1024 * 1024); + const throughput = size_mb / (total_time / 1000); + + console.log(`${size_kb.toFixed(0)}\t\t${total_time.toFixed(2)}\t\t${ipc_time.toFixed(2)}\t\t${recv_time.toFixed(2)}\t\t${throughput.toFixed(2)}`); + + // Small delay between tests + await new Promise(resolve => setTimeout(resolve, 100)); + } + + console.log('\n=== IPC Benchmark Complete ===\n'); + console.log('Run again with: testIPCBenchmark()'); +}; + function uuidv4() { return "10000000-1000-4000-8000-100000000000".replace(/[018]/g, (c) => ( @@ -907,8 +965,8 @@ window.addEventListener("keydown", (e) => { }); async function playPause() { - playing = !playing; - if (playing) { + context.playing = !context.playing; + if (context.playing) { // Reset to start if we're at the end const duration = context.activeObject.duration; if (duration > 0 && context.activeObject.currentTime >= duration) { @@ -966,8 +1024,8 @@ async function playPause() { // Update play/pause button appearance if it exists if (context.playPauseButton) { - context.playPauseButton.className = playing ? "playback-btn playback-btn-pause" : "playback-btn playback-btn-play"; - context.playPauseButton.title = playing ? "Pause" : "Play"; + context.playPauseButton.className = context.playing ? "playback-btn playback-btn-pause" : "playback-btn playback-btn-play"; + context.playPauseButton.title = context.playing ? "Pause" : "Play"; } } @@ -980,7 +1038,7 @@ function playbackLoop() { context.timelineWidget.requestRedraw(); } - if (playing) { + if (context.playing) { const duration = context.activeObject.duration; // Check if we've reached the end (but allow infinite playback when recording) @@ -989,7 +1047,7 @@ function playbackLoop() { requestAnimationFrame(playbackLoop); } else { // Animation finished - playing = false; + context.playing = false; // Stop DAW backend audio playback invoke('audio_stop').catch(error => { @@ -1136,7 +1194,7 @@ async function handleAudioEvent(event) { switch (event.type) { case 'PlaybackPosition': // Sync frontend time with DAW time - if (playing) { + if (context.playing) { // Quantize time to framerate for animation playback const framerate = context.activeObject.frameRate; const frameDuration = 1 / framerate; @@ -1565,7 +1623,7 @@ async function toggleRecording() { console.log('[FRONTEND] MIDI recording started successfully'); // Start playback so the timeline moves (if not already playing) - if (!playing) { + if (!context.playing) { await playPause(); } } catch (error) { @@ -1585,7 +1643,7 @@ async function toggleRecording() { console.log('[FRONTEND] Audio recording started successfully, waiting for RecordingStarted event'); // Start playback so the timeline moves (if not already playing) - if (!playing) { + if (!context.playing) { await playPause(); } } catch (error) { @@ -4528,8 +4586,8 @@ function timelineV2() { // Play/Pause button const playPauseButton = document.createElement("button"); - playPauseButton.className = playing ? "playback-btn playback-btn-pause" : "playback-btn playback-btn-play"; - playPauseButton.title = playing ? "Pause" : "Play"; + playPauseButton.className = context.playing ? "playback-btn playback-btn-pause" : "playback-btn playback-btn-play"; + playPauseButton.title = context.playing ? "Pause" : "Play"; playPauseButton.addEventListener("click", playPause); // Store reference so playPause() can update it @@ -6709,7 +6767,7 @@ async function renderMenu() { }, { text: "Play", - enabled: !playing, + enabled: !context.playing, action: playPause, accelerator: getShortcut("playAnimation"), }, @@ -10916,7 +10974,7 @@ async function renderAll() { renderInProgress = false; // FPS logging (only when playing) - if (playing) { + if (context.playing) { frameCount++; const now = performance.now(); const renderTime = now - renderStartTime; diff --git a/src/models/layer.js b/src/models/layer.js index 31f06b7..9a3e155 100644 --- a/src/models/layer.js +++ b/src/models/layer.js @@ -1281,7 +1281,8 @@ class VideoLayer extends Widget { this.linkedAudioTrack = null; // Reference to AudioTrack // Performance settings - this.useJpegCompression = true; // Enable JPEG compression for faster transfer (default: true) + this.useJpegCompression = false; // JPEG compression adds more overhead than it saves (default: false) + this.prefetchCount = 3; // Number of frames to prefetch ahead of playhead // Timeline display this.collapsed = false; @@ -1291,12 +1292,12 @@ class VideoLayer extends Widget { pointerList[this.idx] = this; } - async addClip(poolIndex, startTime, duration, offset = 0.0, name = '', sourceDuration = null) { + async addClip(poolIndex, startTime, duration, offset = 0.0, name = '', sourceDuration = null, metadata = null) { const poolInfo = await invoke('video_get_pool_info', { poolIndex }); // poolInfo is [width, height, fps] tuple from Rust const [width, height, fps] = poolInfo; - this.clips.push({ + const clip = { clipId: this.clips.length, poolIndex, name: name || `Video ${this.clips.length + 1}`, @@ -1305,10 +1306,124 @@ class VideoLayer extends Widget { offset, width, height, - sourceDuration: sourceDuration || duration // Store original file duration + sourceDuration: sourceDuration || duration, // Store original file duration + httpUrl: metadata?.http_url || null, + isBrowserCompatible: metadata?.is_browser_compatible || false, + transcoding: metadata?.transcoding || false, + videoElement: null, // Will hold HTML5 video element if using browser playback + useBrowserVideo: false, // Switch to true when video element is ready + isPlaying: false, // Track if video element is actively playing + }; + + this.clips.push(clip); + + console.log(`Video clip added: ${name}, ${width}x${height}, duration: ${duration}s, browser-compatible: ${clip.isBrowserCompatible}, http_url: ${clip.httpUrl}`); + + // If HTTP URL is available, create video element immediately + if (clip.httpUrl) { + await this._createVideoElement(clip); + clip.useBrowserVideo = true; + } + // If transcoding is in progress, start polling + else if (clip.transcoding) { + console.log(`[Video] Starting transcode polling for ${clip.name}`); + this._pollTranscodeStatus(clip); + } + } + + async _createVideoElement(clip) { + // Create hidden video element for hardware-accelerated decoding + const video = document.createElement('video'); + // DEBUG: Make video visible on top of everything + video.style.position = 'fixed'; + video.style.top = '10px'; + video.style.right = '10px'; + video.style.width = '400px'; + video.style.height = '225px'; + video.style.zIndex = '99999'; + video.style.border = '3px solid red'; + video.controls = true; // DEBUG: Add controls + video.preload = 'auto'; + video.muted = true; // Mute video element (audio plays separately) + video.playsInline = true; + video.autoplay = false; + video.crossOrigin = 'anonymous'; // Required for canvas drawing - prevent CORS taint + + // Add event listeners for debugging + video.addEventListener('loadedmetadata', () => { + console.log(`[Video] Loaded metadata for ${clip.name}: ${video.videoWidth}x${video.videoHeight}, duration: ${video.duration}s`); }); - console.log(`Video clip added: ${name}, ${width}x${height}, duration: ${duration}s`); + video.addEventListener('loadeddata', () => { + console.log(`[Video] Loaded data for ${clip.name}, readyState: ${video.readyState}`); + }); + + video.addEventListener('canplay', () => { + console.log(`[Video] Can play ${clip.name}, duration: ${video.duration}s`); + // Mark video as ready for seeking once we can play AND have valid duration + if (video.duration > 0 && !isNaN(video.duration) && video.duration !== Infinity) { + clip.videoReady = true; + console.log(`[Video] Video is ready for seeking`); + } + }); + + // When seek completes, trigger UI redraw to show the new frame + video.addEventListener('seeked', () => { + if (updateUI) { + updateUI(); + } + }); + + video.addEventListener('error', (e) => { + const error = video.error; + const errorMessages = { + 1: 'MEDIA_ERR_ABORTED - Fetching aborted', + 2: 'MEDIA_ERR_NETWORK - Network error', + 3: 'MEDIA_ERR_DECODE - Decoding error', + 4: 'MEDIA_ERR_SRC_NOT_SUPPORTED - Format not supported or file not accessible' + }; + const errorMsg = errorMessages[error?.code] || 'Unknown error'; + console.error(`[Video] Error loading ${clip.name}: ${errorMsg}`, error?.message); + }); + + // Use HTTP URL from local server (supports range requests for seeking) + video.src = clip.httpUrl; + + // Try to load the video + video.load(); + + document.body.appendChild(video); + clip.videoElement = video; + + console.log(`[Video] Created video element for clip ${clip.name}: ${clip.httpUrl}`); + } + + async _pollTranscodeStatus(clip) { + // Poll transcode status every 2 seconds + const pollInterval = setInterval(async () => { + try { + const status = await invoke('video_get_transcode_status', { poolIndex: clip.poolIndex }); + + if (status && status[2]) { // [path, progress, completed, httpUrl] + // Transcode complete! + clearInterval(pollInterval); + const [outputPath, progress, completed, httpUrl] = status; + + clip.transcodedPath = outputPath; + clip.httpUrl = httpUrl; + clip.transcoding = false; + clip.useBrowserVideo = true; + + console.log(`[Video] Transcode complete for ${clip.name}, switching to browser playback: ${httpUrl}`); + + // Create video element for browser playback + await this._createVideoElement(clip); + } + } catch (error) { + console.error('Failed to poll transcode status:', error); + clearInterval(pollInterval); + } + }, 2000); } // Pre-fetch frames for current time (call before draw) @@ -1325,126 +1440,224 @@ class VideoLayer extends Widget { if (currentTime < clip.startTime || currentTime >= clip.startTime + clip.duration) { clip.currentFrame = null; + + // Pause video element if we left its time range + if (clip.videoElement && clip.isPlaying) { + clip.videoElement.pause(); + clip.isPlaying = false; + } + continue; } - // Calculate video timestamp from clip time - const clipTime = currentTime - clip.startTime; - const videoTimestamp = clip.offset + clipTime; + // If using browser video element + if (clip.useBrowserVideo && clip.videoElement) { + const videoTime = clip.offset + (currentTime - clip.startTime); - // Only fetch if timestamp changed - if (clip.lastFetchedTimestamp === videoTimestamp && clip.currentFrame) { + // Don't do anything until video is fully ready + if (!clip.videoReady) { + if (!clip._notReadyWarned) { + console.warn(`[Video updateFrame] Video not ready yet (duration=${clip.videoElement.duration})`); + clip._notReadyWarned = true; + } + continue; + } + + // During playback: let video play naturally + if (context.playing) { + // Check if we just entered this clip (need to start playing) + if (!clip.isPlaying) { + // Start playing one frame ahead to compensate for canvas drawing lag + const frameDuration = 1 / (clip.fps || 30); // Use clip's actual framerate + const startTime = videoTime + frameDuration; + console.log(`[Video updateFrame] Starting playback at ${startTime.toFixed(3)}s (compensated by ${frameDuration.toFixed(3)}s for ${clip.fps}fps)`); + clip.videoElement.currentTime = startTime; + clip.videoElement.play().catch(e => console.error('Failed to play video:', e)); + clip.isPlaying = true; + } + // Otherwise, let it play naturally - don't seek! + } + // When scrubbing (not playing): seek to exact position and pause + else { + if (clip.isPlaying) { + clip.videoElement.pause(); + clip.isPlaying = false; + } + + // Only seek if the time is actually different + if (!clip.videoElement.seeking) { + const timeDiff = Math.abs(clip.videoElement.currentTime - videoTime); + if (timeDiff > 0.016) { // ~1 frame tolerance at 60fps + clip.videoElement.currentTime = videoTime; + } + } + } + + continue; // Skip frame fetching + } + + // Use frame batching for frame-based playback + + // Initialize frame cache if needed + if (!clip.frameCache) { + clip.frameCache = new Map(); + } + + // Check if current frame is already cached + if (clip.frameCache.has(currentVideoTimestamp)) { + clip.currentFrame = clip.frameCache.get(currentVideoTimestamp); + clip.lastFetchedTimestamp = currentVideoTimestamp; continue; } - // Skip if already fetching this frame - if (clip.fetchInProgress === videoTimestamp) { + // Skip if already fetching + if (clip.fetchInProgress) { continue; } - clip.fetchInProgress = videoTimestamp; - clip.lastFetchedTimestamp = videoTimestamp; + clip.fetchInProgress = true; try { - // Request frame from Rust backend using IPC Channel for efficient binary transfer + // Calculate timestamps to prefetch (current + next N frames) + const frameDuration = 1 / 30; // Assume 30fps for now, could get from clip metadata + const timestamps = []; + for (let i = 0; i < this.prefetchCount; i++) { + const ts = currentVideoTimestamp + (i * frameDuration); + // Don't exceed clip duration + if (ts <= clip.offset + clip.sourceDuration) { + timestamps.push(ts); + } + } + + if (timestamps.length === 0) { + continue; + } + const t_start = performance.now(); - // Create a promise that resolves when channel receives data - const frameDataPromise = new Promise((resolve, reject) => { + // Request batch of frames using IPC Channel + const batchDataPromise = new Promise((resolve, reject) => { const channel = new Channel(); channel.onmessage = (data) => { resolve(data); }; - // Invoke command with channel - invoke('video_get_frame', { + invoke('video_get_frames_batch', { poolIndex: clip.poolIndex, - timestamp: videoTimestamp, + timestamps: timestamps, useJpeg: this.useJpegCompression, channel: channel }).catch(reject); }); - // Wait for the frame data - let frameData = await frameDataPromise; + let batchData = await batchDataPromise; const t_after_ipc = performance.now(); // Ensure data is Uint8Array - if (!(frameData instanceof Uint8Array)) { - frameData = new Uint8Array(frameData); + if (!(batchData instanceof Uint8Array)) { + batchData = new Uint8Array(batchData); + } + + // Unpack the batch format: [frame_count: u32][frame1_size: u32][frame1_data...][frame2_size: u32][frame2_data...]... + const view = new DataView(batchData.buffer, batchData.byteOffset, batchData.byteLength); + let offset = 0; + + // Read frame count + const frameCount = view.getUint32(offset, true); // little-endian + offset += 4; + + if (frameCount !== timestamps.length) { + console.warn(`Expected ${timestamps.length} frames, got ${frameCount}`); } - let imageData; const t_before_conversion = performance.now(); - if (this.useJpegCompression) { - // Decode JPEG data - const blob = new Blob([frameData], { type: 'image/jpeg' }); - const imageUrl = URL.createObjectURL(blob); + // Process each frame + for (let i = 0; i < frameCount; i++) { + // Read frame size + const frameSize = view.getUint32(offset, true); + offset += 4; - // Load and decode JPEG - const img = new Image(); - await new Promise((resolve, reject) => { - img.onload = resolve; - img.onerror = reject; - img.src = imageUrl; - }); + // Extract frame data + const frameData = new Uint8Array(batchData.buffer, batchData.byteOffset + offset, frameSize); + offset += frameSize; - // Create temporary canvas to extract ImageData - const tempCanvas = document.createElement('canvas'); - tempCanvas.width = clip.width; - tempCanvas.height = clip.height; - const tempCtx = tempCanvas.getContext('2d'); - tempCtx.drawImage(img, 0, 0); - imageData = tempCtx.getImageData(0, 0, clip.width, clip.height); + let imageData; - // Cleanup - URL.revokeObjectURL(imageUrl); - } else { - // Raw RGBA data - const expectedSize = clip.width * clip.height * 4; // RGBA = 4 bytes per pixel + if (this.useJpegCompression) { + // Decode JPEG using createImageBitmap + const blob = new Blob([frameData], { type: 'image/jpeg' }); + const imageBitmap = await createImageBitmap(blob); - if (frameData.length !== expectedSize) { - throw new Error(`Invalid frame data size: got ${frameData.length}, expected ${expectedSize}`); + // Create temporary canvas to extract ImageData + const tempCanvas = document.createElement('canvas'); + tempCanvas.width = clip.width; + tempCanvas.height = clip.height; + const tempCtx = tempCanvas.getContext('2d'); + tempCtx.drawImage(imageBitmap, 0, 0); + imageData = tempCtx.getImageData(0, 0, clip.width, clip.height); + + imageBitmap.close(); + } else { + // Raw RGBA data + const expectedSize = clip.width * clip.height * 4; + + if (frameData.length !== expectedSize) { + console.error(`Invalid frame ${i} data size: got ${frameData.length}, expected ${expectedSize}`); + continue; + } + + imageData = new ImageData( + new Uint8ClampedArray(frameData), + clip.width, + clip.height + ); } - imageData = new ImageData( - new Uint8ClampedArray(frameData), - clip.width, - clip.height - ); + // Create canvas for this frame + const frameCanvas = document.createElement('canvas'); + frameCanvas.width = clip.width; + frameCanvas.height = clip.height; + const frameCtx = frameCanvas.getContext('2d'); + frameCtx.putImageData(imageData, 0, 0); + + // Cache the frame + clip.frameCache.set(timestamps[i], frameCanvas); + + // Set as current frame if it's the first one + if (i === 0) { + clip.currentFrame = frameCanvas; + clip.lastFetchedTimestamp = timestamps[i]; + } } const t_after_conversion = performance.now(); - // Create or reuse temp canvas - if (!clip.frameCanvas) { - clip.frameCanvas = document.createElement('canvas'); - clip.frameCanvas.width = clip.width; - clip.frameCanvas.height = clip.height; + // Limit cache size to avoid memory issues + const maxCacheSize = this.prefetchCount * 2; + if (clip.frameCache.size > maxCacheSize) { + // Remove oldest entries (simple LRU by keeping only recent timestamps) + const sortedKeys = Array.from(clip.frameCache.keys()).sort((a, b) => a - b); + const toRemove = sortedKeys.slice(0, sortedKeys.length - maxCacheSize); + for (let key of toRemove) { + clip.frameCache.delete(key); + } } - const tempCtx = clip.frameCanvas.getContext('2d'); - const t_before_putimage = performance.now(); - tempCtx.putImageData(imageData, 0, 0); - const t_after_putimage = performance.now(); - - clip.currentFrame = clip.frameCanvas; - - // Log detailed timing breakdown - const total_time = t_after_putimage - t_start; + // Log timing breakdown + const total_time = t_after_conversion - t_start; const ipc_time = t_after_ipc - t_start; const conversion_time = t_after_conversion - t_before_conversion; - const putimage_time = t_after_putimage - t_before_putimage; const compression_mode = this.useJpegCompression ? 'JPEG' : 'RAW'; + const avg_per_frame = total_time / frameCount; - console.log(`[JS Video Timing ${compression_mode}] ts=${videoTimestamp.toFixed(3)}s | Total: ${total_time.toFixed(1)}ms | IPC: ${ipc_time.toFixed(1)}ms (${(ipc_time/total_time*100).toFixed(0)}%) | Convert: ${conversion_time.toFixed(1)}ms | PutImage: ${putimage_time.toFixed(1)}ms | Size: ${(frameData.length/1024/1024).toFixed(2)}MB`); + console.log(`[JS Video Batch ${compression_mode}] Fetched ${frameCount} frames | Total: ${total_time.toFixed(1)}ms | IPC: ${ipc_time.toFixed(1)}ms (${(ipc_time/total_time*100).toFixed(0)}%) | Convert: ${conversion_time.toFixed(1)}ms | Avg/frame: ${avg_per_frame.toFixed(1)}ms | Size: ${(batchData.length/1024/1024).toFixed(2)}MB`); } catch (error) { - console.error('Failed to get video frame:', error); + console.error('Failed to get video frames batch:', error); clip.currentFrame = null; } finally { - clip.fetchInProgress = null; + clip.fetchInProgress = false; } } } finally { @@ -1472,8 +1685,89 @@ class VideoLayer extends Widget { continue; } - // Draw cached frame if available - if (clip.currentFrame) { + // Debug: log what path we're taking + if (!clip._drawPathLogged) { + console.log(`[Video Draw] useBrowserVideo=${clip.useBrowserVideo}, videoElement=${!!clip.videoElement}, currentFrame=${!!clip.currentFrame}`); + clip._drawPathLogged = true; + } + + // Prefer browser video element if available + if (clip.useBrowserVideo && clip.videoElement) { + // Debug: log readyState issues + if (clip.videoElement.readyState < 2) { + if (!clip._readyStateWarned) { + console.warn(`[Video] Video not ready: readyState=${clip.videoElement.readyState}, src=${clip.videoElement.src}`); + clip._readyStateWarned = true; + } + } + + // Draw if video is ready (shows last frame while seeking, updates when seek completes) + if (clip.videoElement.readyState >= 2) { + try { + // Calculate expected video time + const expectedVideoTime = clip.offset + (currentTime - clip.startTime); + const actualVideoTime = clip.videoElement.currentTime; + const timeDiff = Math.abs(expectedVideoTime - actualVideoTime); + + // Debug: log if time is significantly different + if (timeDiff > 0.1 && (!clip._lastTimeDiffWarning || Date.now() - clip._lastTimeDiffWarning > 1000)) { + console.warn(`[Video Draw] Time mismatch: expected ${expectedVideoTime.toFixed(2)}s, actual ${actualVideoTime.toFixed(2)}s, diff=${timeDiff.toFixed(2)}s`); + clip._lastTimeDiffWarning = Date.now(); + } + + // Debug: log successful draw periodically + if (!clip._lastDrawLog || Date.now() - clip._lastDrawLog > 1000) { + console.log(`[Video Draw] Drawing at currentTime=${actualVideoTime.toFixed(2)}s (expected ${expectedVideoTime.toFixed(2)}s)`); + clip._lastDrawLog = Date.now(); + } + + // Scale to fit canvas while maintaining aspect ratio + const canvasWidth = config.fileWidth; + const canvasHeight = config.fileHeight; + const scale = Math.min( + canvasWidth / clip.videoElement.videoWidth, + canvasHeight / clip.videoElement.videoHeight + ); + const scaledWidth = clip.videoElement.videoWidth * scale; + const scaledHeight = clip.videoElement.videoHeight * scale; + const x = (canvasWidth - scaledWidth) / 2; + const y = (canvasHeight - scaledHeight) / 2; + + // Debug: draw a test rectangle to verify canvas is working + if (!clip._canvasTestDone) { + ctx.save(); + ctx.fillStyle = 'red'; + ctx.fillRect(10, 10, 100, 100); + ctx.restore(); + console.log(`[Video Draw] Drew test rectangle at (10, 10, 100, 100)`); + console.log(`[Video Draw] Canvas dimensions: ${canvasWidth}x${canvasHeight}`); + console.log(`[Video Draw] Scaled video dimensions: ${scaledWidth}x${scaledHeight} at (${x}, ${y})`); + clip._canvasTestDone = true; + } + + // Debug: Check if video element has dimensions + if (!clip._videoDimensionsLogged) { + console.log(`[Video Draw] Video element dimensions: videoWidth=${clip.videoElement.videoWidth}, videoHeight=${clip.videoElement.videoHeight}, naturalWidth=${clip.videoElement.videoWidth}, naturalHeight=${clip.videoElement.videoHeight}`); + console.log(`[Video Draw] Video element state: paused=${clip.videoElement.paused}, ended=${clip.videoElement.ended}, seeking=${clip.videoElement.seeking}, readyState=${clip.videoElement.readyState}`); + clip._videoDimensionsLogged = true; + } + + ctx.drawImage(clip.videoElement, x, y, scaledWidth, scaledHeight); + + // Debug: Sample a pixel to see if video is actually drawing + if (!clip._pixelTestDone) { + const imageData = ctx.getImageData(canvasWidth / 2, canvasHeight / 2, 1, 1); + const pixel = imageData.data; + console.log(`[Video Draw] Center pixel after drawImage: R=${pixel[0]}, G=${pixel[1]}, B=${pixel[2]}, A=${pixel[3]}`); + clip._pixelTestDone = true; + } + } catch (error) { + console.error('Failed to draw video element:', error); + } + } + } + // Fall back to cached frame if available + else if (clip.currentFrame) { try { // Scale to fit canvas while maintaining aspect ratio const canvasWidth = config.fileWidth; @@ -1500,7 +1794,8 @@ class VideoLayer extends Widget { ctx.font = '24px sans-serif'; ctx.textAlign = 'center'; ctx.textBaseline = 'middle'; - ctx.fillText('Loading...', config.fileWidth / 2, config.fileHeight / 2); + const msg = clip.transcoding ? 'Transcoding...' : 'Loading...'; + ctx.fillText(msg, config.fileWidth / 2, config.fileHeight / 2); ctx.restore(); } } diff --git a/src/state.js b/src/state.js index 674f842..a1788bd 100644 --- a/src/state.js +++ b/src/state.js @@ -36,7 +36,8 @@ export let context = { timelineWidget: null, // Reference to TimelineWindowV2 widget for zoom controls config: null, // Reference to config object (set after config is initialized) mode: "select", // Current tool mode - // Recording state + // Playback and recording state + playing: false, isRecording: false, recordingTrackId: null, recordingClipId: null, From 47e1954efe5dcfb524ea3c85cec021413c902863 Mon Sep 17 00:00:00 2001 From: Skyler Lehmkuhl Date: Thu, 6 Nov 2025 22:36:02 -0500 Subject: [PATCH 5/5] try to improve performance --- src/models/layer.js | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/models/layer.js b/src/models/layer.js index 9a3e155..33599d0 100644 --- a/src/models/layer.js +++ b/src/models/layer.js @@ -1334,15 +1334,15 @@ class VideoLayer extends Widget { async _createVideoElement(clip) { // Create hidden video element for hardware-accelerated decoding const video = document.createElement('video'); - // DEBUG: Make video visible on top of everything + // Hide video element using opacity (browsers may skip decoding if off-screen) video.style.position = 'fixed'; - video.style.top = '10px'; - video.style.right = '10px'; - video.style.width = '400px'; - video.style.height = '225px'; - video.style.zIndex = '99999'; - video.style.border = '3px solid red'; - video.controls = true; // DEBUG: Add controls + video.style.bottom = '0'; + video.style.right = '0'; + video.style.width = '1px'; + video.style.height = '1px'; + video.style.opacity = '0.01'; // Nearly invisible but not 0 (some browsers optimize opacity:0) + video.style.pointerEvents = 'none'; + video.style.zIndex = '-1'; video.preload = 'auto'; video.muted = true; // Mute video element (audio plays separately) video.playsInline = true; @@ -1469,7 +1469,8 @@ class VideoLayer extends Widget { if (!clip.isPlaying) { // Start playing one frame ahead to compensate for canvas drawing lag const frameDuration = 1 / (clip.fps || 30); // Use clip's actual framerate - const startTime = videoTime + frameDuration; + const maxVideoTime = clip.sourceDuration - frameDuration; // Don't seek past end + const startTime = Math.min(videoTime + frameDuration, maxVideoTime); console.log(`[Video updateFrame] Starting playback at ${startTime.toFixed(3)}s (compensated by ${frameDuration.toFixed(3)}s for ${clip.fps}fps)`); clip.videoElement.currentTime = startTime; clip.videoElement.play().catch(e => console.error('Failed to play video:', e));