diff --git a/lightningbeam-core/Cargo.lock b/lightningbeam-core/Cargo.lock index c2984ce..8740cb0 100644 --- a/lightningbeam-core/Cargo.lock +++ b/lightningbeam-core/Cargo.lock @@ -45,6 +45,12 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" +[[package]] +name = "atomic_refcell" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41e67cd8309bbd06cd603a9e693a784ac2e5d1e955f11286e355089fcab3047c" + [[package]] name = "autocfg" version = "1.4.0" @@ -211,6 +217,21 @@ dependencies = [ "windows", ] +[[package]] +name = "crossbeam-channel" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06ba6d68e24814cb8de6bb986db8222d3a027d15872cabc0d18817bc3c0e4471" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + [[package]] name = "dasp_sample" version = "0.11.0" @@ -244,24 +265,45 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af9673d8203fcb076b19dfd17e38b3d4ae9f44959416ea532ce72415a6020365" +[[package]] +name = "futures-channel" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" + [[package]] name = "glob" version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" +[[package]] +name = "gloo-timers" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b995a66bb87bebce9a0f4a95aed01daca4872c050bfcb21653361c03bc35e5c" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", +] + [[package]] name = "hashbrown" version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" -[[package]] -name = "hound" -version = "3.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62adaabb884c94955b19907d60019f4e145d091c75345379e70d1ee696f7854f" - [[package]] name = "indexmap" version = "2.7.1" @@ -349,12 +391,14 @@ name = "lightningbeam-core" version = "0.1.0" dependencies = [ "anyhow", + "atomic_refcell", "console_error_panic_hook", "cpal", - "hound", + "crossbeam-channel", + "gloo-timers", "js-sys", "log", - "minimp3", + "parking_lot", "rubato", "serde", "symphonia", @@ -362,6 +406,17 @@ dependencies = [ "wasm-bindgen-futures", "wasm-logger", "web-sys", + "web-time", +] + +[[package]] +name = "lock_api" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" +dependencies = [ + "autocfg", + "scopeguard", ] [[package]] @@ -370,15 +425,6 @@ version = "0.4.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04cbf5b083de1c7e0222a7a51dbfdba1cbe1c6ab0b15e29fff3f6c077fd9cd9f" -[[package]] -name = "mach" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b823e83b2affd8f40a9ee8c29dbc56404c1e34cd2710921f2801e2cf29527afa" -dependencies = [ - "libc", -] - [[package]] name = "mach2" version = "0.4.2" @@ -400,26 +446,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" -[[package]] -name = "minimp3" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "985438f75febf74c392071a975a29641b420dd84431135a6e6db721de4b74372" -dependencies = [ - "minimp3-sys", - "slice-deque", - "thiserror", -] - -[[package]] -name = "minimp3-sys" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e21c73734c69dc95696c9ed8926a2b393171d98b3f5f5935686a26a487ab9b90" -dependencies = [ - "cc", -] - [[package]] name = "ndk" version = "0.8.0" @@ -547,6 +573,29 @@ version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" +[[package]] +name = "parking_lot" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-targets 0.52.6", +] + [[package]] name = "pkg-config" version = "0.3.31" @@ -599,6 +648,15 @@ dependencies = [ "rustfft", ] +[[package]] +name = "redox_syscall" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03a862b389f93e68874fbf580b9de08dd02facb9a788ebadaf4a3fd33cf58834" +dependencies = [ + "bitflags 2.8.0", +] + [[package]] name = "regex" version = "1.11.1" @@ -676,6 +734,12 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + [[package]] name = "serde" version = "1.0.217" @@ -703,15 +767,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" [[package]] -name = "slice-deque" -version = "0.3.0" +name = "smallvec" +version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31ef6ee280cdefba6d2d0b4b78a84a1c1a3f3a4cec98c2d4231c8bc225de0f25" -dependencies = [ - "libc", - "mach", - "winapi", -] +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" [[package]] name = "strength_reduce" @@ -1087,21 +1146,15 @@ dependencies = [ ] [[package]] -name = "winapi" -version = "0.3.9" +name = "web-time" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +checksum = "aa30049b1c872b72c89866d458eae9f20380ab280ffd1b1e18df2d3e2d98cfe0" dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", + "js-sys", + "wasm-bindgen", ] -[[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - [[package]] name = "winapi-util" version = "0.1.9" @@ -1111,12 +1164,6 @@ dependencies = [ "windows-sys 0.59.0", ] -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" - [[package]] name = "windows" version = "0.54.0" diff --git a/lightningbeam-core/Cargo.toml b/lightningbeam-core/Cargo.toml index ca67190..3eb80d5 100644 --- a/lightningbeam-core/Cargo.toml +++ b/lightningbeam-core/Cargo.toml @@ -15,18 +15,19 @@ wasm-logger = "0.2" log = "0.4" rubato = "0.14.0" symphonia = { version = "0.5", features = ["all"] } +crossbeam-channel = "0.5.4" +atomic_refcell = "0.1.13" # WASM-compatible atomic refcell +parking_lot = "0.12" [dependencies.web-sys] version = "0.3.22" -features = ["console", "AudioContext"] - -[target.'cfg(not(target_arch = "wasm32"))'.dependencies] -minimp3 = "0.5.1" # Only include minimp3 for native platforms -hound = "3.5.1" # Only include hound for native platforms +features = ["console", "AudioContext", "Window", "Performance", "PerformanceTiming"] [target.'cfg(target_arch = "wasm32")'.dependencies] wasm-bindgen-futures = "0.4" js-sys = "0.3" +web-time = "0.2" # WASM-compatible timing +gloo-timers = { version = "0.2", features = ["futures"] } # The `console_error_panic_hook` crate provides better debugging of panics by # logging them with `console.error`. This is great for development, but requires diff --git a/lightningbeam-core/src/audio.rs b/lightningbeam-core/src/audio.rs index 3904192..92d157d 100644 --- a/lightningbeam-core/src/audio.rs +++ b/lightningbeam-core/src/audio.rs @@ -2,18 +2,35 @@ use cpal::traits::{DeviceTrait, HostTrait, StreamTrait}; use cpal::{Sample}; use std::sync::{Arc, Mutex}; use crate::{TrackManager, Timestamp, Duration, SampleCount, AudioOutput, PlaybackState}; +#[cfg(target_arch = "wasm32")] +use web_time::{Instant, Duration as StdDuration}; +#[cfg(not(target_arch = "wasm32"))] +use std::time::{Instant, Duration as StdDuration}; +use std::sync::atomic::Ordering; +use std::sync::atomic::AtomicU32; +use std::cell::Cell; +use std::collections::VecDeque; -#[derive(PartialEq)] +#[derive(PartialEq, Clone)] enum AudioState { - Suspended, - Running, + Suspended, + Running, } -// #[cfg(feature = "wasm")] -// use wasm_bindgen::prelude::*; +const DELAY_HISTORY_SIZE: usize = 5; + +#[derive(Default)] +struct StutterDetector { + delay_history: Mutex>, + desired_buffer_size: AtomicU32, + current_buffer_size: AtomicU32, + max_buffer_size: AtomicU32, + stutter_count: AtomicU32, + max_stutter_count: AtomicU32, + last_callback_time: Cell>, + scheduling_threshold: AtomicU32, +} -// #[cfg(feature="wasm")] -// #[wasm_bindgen] pub struct CpalAudioOutput { track_manager: Option>>, _stream: Option, @@ -22,12 +39,32 @@ pub struct CpalAudioOutput { timestamp: Arc>, chunk_size: usize, sample_rate: u32, + stutter_detector: Arc>, + resize_sender: crossbeam_channel::Sender<()>, // Or other channel implementation + resize_receiver: crossbeam_channel::Receiver<()>, +} + +impl StutterDetector { + fn new() -> Self { + Self { + delay_history: Mutex::new(VecDeque::with_capacity(DELAY_HISTORY_SIZE)), + desired_buffer_size: AtomicU32::new(256), + current_buffer_size: AtomicU32::new(256), + max_buffer_size: AtomicU32::new(8192), + stutter_count: AtomicU32::new(0), + max_stutter_count: AtomicU32::new(3), + last_callback_time: Cell::new(None), + scheduling_threshold: AtomicU32::new(1200), // 1.2 stored in fixed point + } + } + fn get_scheduling_threshold(&self) -> f32 { + self.scheduling_threshold.load(Ordering::Relaxed) as f32 / 1000.0 + } } -// #[cfg(feature="wasm")] -// #[wasm_bindgen] impl CpalAudioOutput { pub fn new() -> Self { + let (tx, rx) = crossbeam_channel::bounded(1); Self { track_manager: None, _stream: None, @@ -35,7 +72,10 @@ impl CpalAudioOutput { audio_state: AudioState::Suspended, timestamp: Arc::new(Mutex::new(Timestamp::from_seconds(0.0))), chunk_size: 0, - sample_rate: 44100, // Default sample rate, updated later + sample_rate: 44100, + stutter_detector: Arc::new(Mutex::new(StutterDetector::new())), + resize_sender: tx, + resize_receiver: rx, } } @@ -43,70 +83,213 @@ impl CpalAudioOutput { &mut self, device: &cpal::Device, config: cpal::SupportedStreamConfig, -) -> Result -where - T: Sample + From + cpal::SizedSample, -{ + ) -> Result + where + T: Sample + From + cpal::SizedSample, + { let supported_config = config.config(); self.sample_rate = supported_config.sample_rate.0; - let num_channels = supported_config.channels as usize; // Get channel count - + let num_channels = supported_config.channels as usize; + + let stutter_detector = self.stutter_detector.clone(); + let resize_sender = self.resize_sender.clone(); + let sample_rate = self.sample_rate; + let buffer_size_range = match config.buffer_size() { cpal::SupportedBufferSize::Range { min, max } => (*min, *max), - cpal::SupportedBufferSize::Unknown => { - // Use a reasonable default range if the device doesn't specify - (256, 4096) - } - }; - - // Define the desired buffer size and clamp it to the supported range - let desired_buffer_size = 2048; + cpal::SupportedBufferSize::Unknown => (256, 4096), + }; + + let detector_guard = self.stutter_detector.lock().unwrap(); + let desired_buffer_size = detector_guard.desired_buffer_size.load(Ordering::Relaxed); + drop(detector_guard); + let clamped_buffer_size = desired_buffer_size.clamp(buffer_size_range.0, buffer_size_range.1); - let mut stream_config = supported_config.clone(); stream_config.buffer_size = cpal::BufferSize::Fixed(clamped_buffer_size); - + + log::info!("Starting stream with buffer size {}", clamped_buffer_size); + let track_manager = self.track_manager.clone(); let timestamp = self.timestamp.clone(); - let sample_rate = self.sample_rate; - - let err_fn = |err| eprintln!("Audio stream error: {:?}", err); - + + let err_fn = |err| log::error!("Audio stream error: {:?}", err); + let stream = device.build_output_stream( - &stream_config, - move |data: &mut [T], _: &cpal::OutputCallbackInfo| { - if let Some(track_manager) = &track_manager { - let num_frames = data.len() / num_channels; // Stereo: divide by 2 - let sample_count = SampleCount::new(num_frames); - let chunk_duration = Duration::new(num_frames as f64 / sample_rate as f64); - - let mut track_manager = track_manager.lock().unwrap(); - - let mut timestamp_guard = timestamp.lock().unwrap(); - let timestamp = &mut *timestamp_guard; - - let chunk = track_manager.update_audio( - timestamp.clone(), - sample_count, - sample_rate, - ); - - // Write samples (interleaved stereo) - for (i, frame) in chunk.iter().enumerate() { - let sample = T::from(*frame); - data[i * num_channels] = sample; // Left channel - data[i * num_channels + 1] = sample; // Right channel (or process separately) + &stream_config, + move |data: &mut [T], _: &cpal::OutputCallbackInfo| { + // Timing measurement + let processing_start = if cfg!(target_arch = "wasm32") { + let perf = web_sys::window() + .and_then(|w| w.performance()) + .expect("performance should be available"); + let now_ms = perf.now(); + Instant::now() + StdDuration::from_secs_f64(now_ms / 1000.0) + } else { + Instant::now() + }; + + // Initialize resize flag outside of lock scope + let mut should_resize = false; + let current_size; + let buffer_duration; + let scheduling_threshold; + + { + let detector = stutter_detector.lock().unwrap(); + + // Update detector state + current_size = detector.current_buffer_size.load(Ordering::Relaxed); + buffer_duration = StdDuration::from_secs_f64(current_size as f64 / sample_rate as f64); + scheduling_threshold = detector.get_scheduling_threshold(); + + // Calculate scheduling delay + let last_time = detector.last_callback_time.get(); + // log::info!("Current size: {}", current_size); + + // Audio processing + if let Some(track_manager) = &track_manager { + let num_frames = data.len() / num_channels; // Stereo: divide by 2 + let sample_count = SampleCount::new(num_frames); + let chunk_duration = Duration::new(num_frames as f64 / sample_rate as f64); + + let mut track_manager = track_manager.lock().unwrap(); + + let mut timestamp_guard = timestamp.lock().unwrap(); + let timestamp = &mut *timestamp_guard; + + let chunk = track_manager.update_audio( + timestamp.clone(), + sample_count, + sample_rate, + ); + + // Write samples (interleaved stereo) + for (i, frame) in chunk.iter().enumerate() { + let sample = T::from(*frame); + for channel in 0..num_channels { + let index = i * num_channels + channel; + if index < data.len() { + data[index] = sample; } - - *timestamp_guard += chunk_duration; + } } - }, - err_fn, - None, + + *timestamp_guard += chunk_duration; + + // Stutter detection logic + let processing_time = processing_start.elapsed(); + let processing_overrun = processing_time > buffer_duration; + + // Update delay history + if let Some(last) = last_time { + let interval = processing_start.duration_since(last); + let mut history = detector.delay_history.lock().unwrap(); + if history.len() >= 5 { + history.pop_front(); + } + history.push_back(interval); + // log::info!("Interval: {:?}", interval); + } + + // Calculate average delay + let avg_delay = { + let history = detector.delay_history.lock().unwrap(); + if history.is_empty() { + StdDuration::ZERO + } else { + history.iter().sum::() / history.len() as u32 + } + }; + + // log::info!("Average delay: {:?}", avg_delay); + + // Determine stutter + let stutter_detected = avg_delay > buffer_duration.mul_f32(scheduling_threshold) + || processing_overrun; + + // Update stutter count with hysteresis + let current_count = detector.stutter_count.load(Ordering::Relaxed); + if stutter_detected { + detector.stutter_count.store( + (current_count + 1).min(detector.max_stutter_count.load(Ordering::Relaxed)), + Ordering::Relaxed + ); + } else { + detector.stutter_count.store( + current_count.saturating_sub(1), + Ordering::Relaxed + ); + } + + // Check for resize + if detector.stutter_count.load(Ordering::Relaxed) >= detector.max_stutter_count.load(Ordering::Relaxed) { + let desired_size = detector.desired_buffer_size.load(Ordering::Relaxed); + let new_size = (desired_size * 2).min(detector.max_buffer_size.load(Ordering::Relaxed)); + + if new_size != desired_size { + detector.desired_buffer_size.store(new_size, Ordering::Relaxed); + detector.stutter_count.store(0, Ordering::Relaxed); + should_resize = true; + } + } + } + + detector.last_callback_time.set(Some(processing_start)); + } + + // Send resize request outside of lock + if should_resize { + let _ = resize_sender.try_send(()); + } + }, + err_fn, + None, )?; + // Update current buffer size after stream creation + let detector = self.stutter_detector.lock().unwrap(); + detector.current_buffer_size.store(clamped_buffer_size, Ordering::Relaxed); + Ok(stream) } + + fn recreate_stream(&mut self) -> Result<(), Box> { + // Stop and destroy old stream first + if let Some(old_stream) = self._stream.take() { + old_stream.pause()?; + // Explicitly drop the stream + drop(old_stream); + } + + // Add a small delay to ensure resources are freed (especially important in WASM) + #[cfg(not(target_arch = "wasm32"))] + std::thread::sleep(std::time::Duration::from_millis(50)); + + #[cfg(target_arch = "wasm32")] + { + use wasm_bindgen_futures::spawn_local; + use gloo_timers::future::sleep; + spawn_local(async { + sleep(std::time::Duration::from_millis(50)).await; + }); + } + + // Recreate stream with current configuration + let host = cpal::default_host(); + let device = host.default_output_device() + .ok_or_else(|| "No output device available")?; + let supported_config = device.default_output_config()?; + + self._stream = Some(self.build_stream::(&device, supported_config)?); + + // Restart playback if needed + if self.audio_state == AudioState::Running { + self._stream.as_ref().unwrap().play()?; + } + + Ok(()) +} } impl AudioOutput for CpalAudioOutput { @@ -128,7 +311,7 @@ impl AudioOutput for CpalAudioOutput { fn stop(&mut self) { self.playback_state = PlaybackState::Stopped; } - + fn resume(&mut self) -> Result<(), anyhow::Error> { if self.audio_state == AudioState::Suspended { if let Some(stream) = &self._stream { @@ -150,4 +333,37 @@ impl AudioOutput for CpalAudioOutput { fn set_chunk_size(&mut self, chunk_size: usize) { self.chunk_size = chunk_size } -} + fn check_resize(&mut self) -> Result<(), Box> { + // Process resize requests with timeout + let timeout = StdDuration::from_millis(10); + while let Ok(()) = self.resize_receiver.try_recv() { + let start = Instant::now(); + + // Try to lock, non-blocking + { + let detector = match self.stutter_detector.try_lock() { + Ok(d) => d, + Err(_) => { + // Couldn't acquire lock immediately, skip this iteration + return Ok(()); + } + }; + + // Quick check before heavy operation + if detector.desired_buffer_size.load(Ordering::Relaxed) == detector.current_buffer_size.load(Ordering::Relaxed) { + continue; + } + detector.current_buffer_size.store(detector.desired_buffer_size.load(Ordering::Relaxed), Ordering::Relaxed); + } + + // Actual stream recreation + log::info!("Restarting stream"); + let _ = self.recreate_stream()?; + + if Instant::now().duration_since(start) > timeout { + break; + } + } + Ok(()) + } +} \ No newline at end of file diff --git a/lightningbeam-core/src/lib.rs b/lightningbeam-core/src/lib.rs index 227d597..de745fd 100644 --- a/lightningbeam-core/src/lib.rs +++ b/lightningbeam-core/src/lib.rs @@ -124,6 +124,7 @@ pub trait AudioOutput { fn register_track_manager(&mut self, track_manager: Arc>); fn get_timestamp(&mut self) -> Timestamp; fn set_chunk_size(&mut self, chunk_size: usize); + fn check_resize(&mut self) -> Result<(), Box>; } pub trait FrameTarget { @@ -372,64 +373,64 @@ fn decode_audio( ) -> Result<(), Box> { // Create a media source from the byte slice let mss = MediaSourceStream::new( - Box::new(Cursor::new(audio_data.to_vec())), - Default::default(), + Box::new(Cursor::new(audio_data.to_vec())), + Default::default(), ); - + // Use a fresh hint (no extension specified) for format detection let hint = Hint::new(); - + // Probe the media source for a supported format let probed = symphonia::default::get_probe() - .format(&hint, mss, &FormatOptions::default(), &MetadataOptions::default())?; - + .format(&hint, mss, &FormatOptions::default(), &MetadataOptions::default())?; + // Get the format reader let mut format = probed.format; - + // Find the first supported audio track let default_track = format - .tracks() - .iter() - .find(|t| t.codec_params.codec != symphonia::core::codecs::CODEC_TYPE_NULL) - .ok_or("No supported audio track found")?; - + .tracks() + .iter() + .find(|t| t.codec_params.codec != symphonia::core::codecs::CODEC_TYPE_NULL) + .ok_or("No supported audio track found")?; + // Create a decoder for the track let mut decoder = symphonia::default::get_codecs() - .make(&default_track.codec_params, &DecoderOptions::default())?; - + .make(&default_track.codec_params, &DecoderOptions::default())?; + // Get the sample rate from the track let sample_rate = default_track.codec_params.sample_rate.ok_or("Unknown sample rate")?; let mut decoded_samples = Vec::new(); - + // Decode loop loop { - let packet = match format.next_packet() { - Ok(packet) => packet, - Err(_) => break, // End of stream - }; - - match decoder.decode(&packet)? { - AudioBufferRef::F32(buf) => { - for i in 0..buf.frames() { - for c in 0..buf.spec().channels.count() { - decoded_samples.push(buf.chan(c)[i]); - } - } + let packet = match format.next_packet() { + Ok(packet) => packet, + Err(_) => break, // End of stream + }; + + match decoder.decode(&packet)? { + AudioBufferRef::F32(buf) => { + for i in 0..buf.frames() { + for c in 0..buf.spec().channels.count() { + decoded_samples.push(buf.chan(c)[i]); } - AudioBufferRef::S16(buf) => { - for i in 0..buf.frames() { - for c in 0..buf.spec().channels.count() { - decoded_samples.push(buf.chan(c)[i] as f32 / 32768.0); - } - } - } - _ => return Err("Unsupported audio format".into()), + } } + AudioBufferRef::S16(buf) => { + for i in 0..buf.frames() { + for c in 0..buf.spec().channels.count() { + decoded_samples.push(buf.chan(c)[i] as f32 / 32768.0); + } + } + } + _ => return Err("Unsupported audio format".into()), + } } - + // Add the decoded audio to the track track.add_buffer(start_time, sample_rate, decoded_samples); - + Ok(()) } @@ -469,7 +470,11 @@ pub struct CoreInterface { #[wasm_bindgen(skip)] track_manager: Arc>, #[wasm_bindgen(skip)] - cpal_audio_output: Box, + cpal_audio_output: Arc>, + #[wasm_bindgen(skip)] + resize_interval_id: Option, + #[wasm_bindgen(skip)] + resize_closure: Option>, } #[cfg(feature="wasm")] @@ -479,14 +484,21 @@ impl CoreInterface { pub fn new() -> Self { Self { track_manager: Arc::new(Mutex::new(TrackManager::new())), - cpal_audio_output: Box::new(CpalAudioOutput::new()) + cpal_audio_output: Arc::new(Mutex::new(CpalAudioOutput::new())), + resize_interval_id: None, + resize_closure: None, } } pub fn init(&mut self) { println!("Init CoreInterface"); - let track_manager_clone = self.track_manager.clone(); - self.cpal_audio_output.register_track_manager(track_manager_clone); - let _ = self.cpal_audio_output.start(); + { + let track_manager_clone = self.track_manager.clone(); + let mut cpal_audio_output = self.cpal_audio_output.lock().unwrap(); + cpal_audio_output.register_track_manager(track_manager_clone); + let _ = cpal_audio_output.start(); + } + + self.start_resize_polling(); } pub fn play(&mut self, timestamp: f64) { // Lock the Mutex to get access to TrackManager @@ -500,9 +512,52 @@ impl CoreInterface { } pub fn resume_audio(&mut self) -> Result<(), JsValue> { // Call this on user gestures if audio gets suspended - self.cpal_audio_output.resume() + self.cpal_audio_output.lock().unwrap().resume() .map_err(|e| JsValue::from_str(&format!("Failed to resume audio: {}", e))) } + // In CoreInterface + fn start_resize_polling(&mut self) { + #[cfg(target_arch = "wasm32")] + { + use wasm_bindgen::{prelude::*, JsCast}; + use js_sys::Array; + + let window = web_sys::window().unwrap(); + let audio_output = Arc::clone(&self.cpal_audio_output); + + // Use weak reference to break cycle + let weak_audio = Arc::downgrade(&audio_output); + let closure = Closure::::new(move || { + if let Some(audio) = weak_audio.upgrade() { + // NON-BLOCKING lock attempt + if let Ok(mut audio) = audio.try_lock() { + let _ = audio.check_resize(); + } + } + }); + + let args = Array::new(); + let interval_id = window + .set_interval_with_callback_and_timeout_and_arguments( + closure.as_ref().unchecked_ref(), + 50, + &args + ) + .unwrap(); + + self.resize_interval_id = Some(interval_id); + self.resize_closure = Some(closure); + } + + #[cfg(not(target_arch = "wasm32"))] + { + let mut audio = self.cpal_audio_output.clone(); + std::thread::spawn(move || loop { + let _ = audio.check_resize(); + std::thread::sleep(std::time::Duration::from_millis(50)); + }); + } + } pub fn add_sine_track(&mut self, frequency: f32) -> Result<(), String> { if frequency.is_nan() || frequency.is_infinite() || frequency <= 0.0 { return Err(format!("Invalid frequency: {}", frequency)); @@ -516,7 +571,7 @@ impl CoreInterface { } pub fn get_timestamp(&mut self) -> f64 { - self.cpal_audio_output.get_timestamp().as_seconds() + self.cpal_audio_output.lock().unwrap().get_timestamp().as_seconds() } pub fn get_tracks(&mut self) -> Vec { let track_manager = self.track_manager.lock().unwrap(); @@ -528,8 +583,20 @@ impl CoreInterface { }) .collect() } + } +// Cleanup implementation +#[cfg(feature = "wasm")] +impl Drop for CoreInterface { + fn drop(&mut self) { + if let Some(interval_id) = self.resize_interval_id { + web_sys::window() + .unwrap() + .clear_interval_with_handle(interval_id); + } + } +} struct PlainTextLogger; diff --git a/src/pkg/lightningbeam_core.d.ts b/src/pkg/lightningbeam_core.d.ts index 3b8c0e8..91bd9d3 100644 --- a/src/pkg/lightningbeam_core.d.ts +++ b/src/pkg/lightningbeam_core.d.ts @@ -42,7 +42,9 @@ export interface InitOutput { readonly __wbindgen_realloc: (a: number, b: number, c: number, d: number) => number; readonly __wbindgen_add_to_stack_pointer: (a: number) => number; readonly __wbindgen_free: (a: number, b: number, c: number) => void; - readonly _dyn_core__ops__function__FnMut_____Output___R_as_wasm_bindgen__closure__WasmClosure___describe__invoke__hfe92e343adbc229b: (a: number, b: number) => void; + readonly _dyn_core__ops__function__FnMut_____Output___R_as_wasm_bindgen__closure__WasmClosure___describe__invoke__h078d17eff0b70a99: (a: number, b: number) => void; + readonly _dyn_core__ops__function__FnMut_____Output___R_as_wasm_bindgen__closure__WasmClosure___describe__invoke__h26b4819ece79f796: (a: number, b: number) => void; + readonly _dyn_core__ops__function__FnMut__A____Output___R_as_wasm_bindgen__closure__WasmClosure___describe__invoke__hb1f2a4138d993283: (a: number, b: number, c: number) => void; readonly __wbindgen_start: () => void; } diff --git a/src/pkg/lightningbeam_core.js b/src/pkg/lightningbeam_core.js index 1df0b89..42bafa0 100644 --- a/src/pkg/lightningbeam_core.js +++ b/src/pkg/lightningbeam_core.js @@ -26,6 +26,18 @@ function handleError(f, args) { } } +function dropObject(idx) { + if (idx < 132) return; + heap[idx] = heap_next; + heap_next = idx; +} + +function takeObject(idx) { + const ret = getObject(idx); + dropObject(idx); + return ret; +} + let cachedFloat32ArrayMemory0 = null; function getFloat32ArrayMemory0() { @@ -62,18 +74,6 @@ function isLikeNone(x) { return x === undefined || x === null; } -function dropObject(idx) { - if (idx < 132) return; - heap[idx] = heap_next; - heap_next = idx; -} - -function takeObject(idx) { - const ret = getObject(idx); - dropObject(idx); - return ret; -} - const CLOSURE_DTORS = (typeof FinalizationRegistry === 'undefined') ? { register: () => {}, unregister: () => {} } : new FinalizationRegistry(state => { @@ -249,8 +249,16 @@ export function main_js() { wasm.main_js(); } -function __wbg_adapter_18(arg0, arg1) { - wasm._dyn_core__ops__function__FnMut_____Output___R_as_wasm_bindgen__closure__WasmClosure___describe__invoke__hfe92e343adbc229b(arg0, arg1); +function __wbg_adapter_20(arg0, arg1) { + wasm._dyn_core__ops__function__FnMut_____Output___R_as_wasm_bindgen__closure__WasmClosure___describe__invoke__h078d17eff0b70a99(arg0, arg1); +} + +function __wbg_adapter_23(arg0, arg1) { + wasm._dyn_core__ops__function__FnMut_____Output___R_as_wasm_bindgen__closure__WasmClosure___describe__invoke__h26b4819ece79f796(arg0, arg1); +} + +function __wbg_adapter_26(arg0, arg1, arg2) { + wasm._dyn_core__ops__function__FnMut__A____Output___R_as_wasm_bindgen__closure__WasmClosure___describe__invoke__hb1f2a4138d993283(arg0, arg1, addHeapObject(arg2)); } const CoreInterfaceFinalization = (typeof FinalizationRegistry === 'undefined') @@ -445,6 +453,13 @@ function __wbg_get_imports() { const ret = getObject(arg0).call(getObject(arg1)); return addHeapObject(ret); }, arguments) }; + imports.wbg.__wbg_clearInterval_ad2594253cc39c4b = function(arg0, arg1) { + getObject(arg0).clearInterval(arg1); + }; + imports.wbg.__wbg_clearTimeout_96804de0ab838f26 = function(arg0) { + const ret = clearTimeout(takeObject(arg0)); + return addHeapObject(ret); + }; imports.wbg.__wbg_close_5a97ef05b337f8ce = function() { return handleError(function (arg0) { const ret = getObject(arg0).close(); return addHeapObject(ret); @@ -501,6 +516,10 @@ function __wbg_get_imports() { const ret = new Object(); return addHeapObject(ret); }; + imports.wbg.__wbg_new_78feb108b6472713 = function() { + const ret = new Array(); + return addHeapObject(ret); + }; imports.wbg.__wbg_newnoargs_105ed471475aaf50 = function(arg0, arg1) { const ret = new Function(getStringFromWasm0(arg0, arg1)); return addHeapObject(ret); @@ -509,10 +528,45 @@ function __wbg_get_imports() { const ret = new lAudioContext(getObject(arg0)); return addHeapObject(ret); }, arguments) }; + imports.wbg.__wbg_now_6f91d421b96ea22a = function(arg0) { + const ret = getObject(arg0).now(); + return ret; + }; + imports.wbg.__wbg_now_d18023d54d4e5500 = function(arg0) { + const ret = getObject(arg0).now(); + return ret; + }; + imports.wbg.__wbg_performance_c185c0cdc2766575 = function(arg0) { + const ret = getObject(arg0).performance; + return isLikeNone(ret) ? 0 : addHeapObject(ret); + }; + imports.wbg.__wbg_performance_f71bd4abe0370171 = function(arg0) { + const ret = getObject(arg0).performance; + return addHeapObject(ret); + }; + imports.wbg.__wbg_queueMicrotask_97d92b4fcc8a61c5 = function(arg0) { + queueMicrotask(getObject(arg0)); + }; + imports.wbg.__wbg_queueMicrotask_d3219def82552485 = function(arg0) { + const ret = getObject(arg0).queueMicrotask; + return addHeapObject(ret); + }; + imports.wbg.__wbg_resolve_4851785c9c5f573d = function(arg0) { + const ret = Promise.resolve(getObject(arg0)); + return addHeapObject(ret); + }; imports.wbg.__wbg_resume_35efdc4ffe13bf18 = function() { return handleError(function (arg0) { const ret = getObject(arg0).resume(); return addHeapObject(ret); }, arguments) }; + imports.wbg.__wbg_setInterval_83d54331ceeda644 = function() { return handleError(function (arg0, arg1, arg2, arg3) { + const ret = getObject(arg0).setInterval(getObject(arg1), arg2, ...getObject(arg3)); + return ret; + }, arguments) }; + imports.wbg.__wbg_setTimeout_eefe7f4c234b0c6b = function() { return handleError(function (arg0, arg1) { + const ret = setTimeout(getObject(arg0), arg1); + return addHeapObject(ret); + }, arguments) }; imports.wbg.__wbg_setTimeout_f2fe5af8e3debeb3 = function() { return handleError(function (arg0, arg1, arg2) { const ret = getObject(arg0).setTimeout(getObject(arg1), arg2); return ret; @@ -548,6 +602,14 @@ function __wbg_get_imports() { const ret = typeof window === 'undefined' ? null : window; return isLikeNone(ret) ? 0 : addHeapObject(ret); }; + imports.wbg.__wbg_suspend_6011a41599f07de4 = function() { return handleError(function (arg0) { + const ret = getObject(arg0).suspend(); + return addHeapObject(ret); + }, arguments) }; + imports.wbg.__wbg_then_44b73946d2fb3e7d = function(arg0, arg1) { + const ret = getObject(arg0).then(getObject(arg1)); + return addHeapObject(ret); + }; imports.wbg.__wbindgen_boolean_get = function(arg0) { const v = getObject(arg0); const ret = typeof(v) === 'boolean' ? (v ? 1 : 0) : 2; @@ -562,8 +624,16 @@ function __wbg_get_imports() { const ret = false; return ret; }; - imports.wbg.__wbindgen_closure_wrapper151 = function(arg0, arg1, arg2) { - const ret = makeMutClosure(arg0, arg1, 73, __wbg_adapter_18); + imports.wbg.__wbindgen_closure_wrapper207 = function(arg0, arg1, arg2) { + const ret = makeMutClosure(arg0, arg1, 82, __wbg_adapter_20); + return addHeapObject(ret); + }; + imports.wbg.__wbindgen_closure_wrapper265 = function(arg0, arg1, arg2) { + const ret = makeMutClosure(arg0, arg1, 109, __wbg_adapter_23); + return addHeapObject(ret); + }; + imports.wbg.__wbindgen_closure_wrapper283 = function(arg0, arg1, arg2) { + const ret = makeMutClosure(arg0, arg1, 116, __wbg_adapter_26); return addHeapObject(ret); }; imports.wbg.__wbindgen_debug_string = function(arg0, arg1) { @@ -573,6 +643,10 @@ function __wbg_get_imports() { getDataViewMemory0().setInt32(arg0 + 4 * 1, len1, true); getDataViewMemory0().setInt32(arg0 + 4 * 0, ptr1, true); }; + imports.wbg.__wbindgen_is_function = function(arg0) { + const ret = typeof(getObject(arg0)) === 'function'; + return ret; + }; imports.wbg.__wbindgen_is_undefined = function(arg0) { const ret = getObject(arg0) === undefined; return ret; diff --git a/src/pkg/lightningbeam_core_bg.wasm b/src/pkg/lightningbeam_core_bg.wasm index 2d2ab1d..e80499d 100644 Binary files a/src/pkg/lightningbeam_core_bg.wasm and b/src/pkg/lightningbeam_core_bg.wasm differ diff --git a/src/pkg/lightningbeam_core_bg.wasm.d.ts b/src/pkg/lightningbeam_core_bg.wasm.d.ts index 7a3debd..882e2b7 100644 --- a/src/pkg/lightningbeam_core_bg.wasm.d.ts +++ b/src/pkg/lightningbeam_core_bg.wasm.d.ts @@ -20,5 +20,7 @@ export const __wbindgen_malloc: (a: number, b: number) => number; export const __wbindgen_realloc: (a: number, b: number, c: number, d: number) => number; export const __wbindgen_add_to_stack_pointer: (a: number) => number; export const __wbindgen_free: (a: number, b: number, c: number) => void; -export const _dyn_core__ops__function__FnMut_____Output___R_as_wasm_bindgen__closure__WasmClosure___describe__invoke__hfe92e343adbc229b: (a: number, b: number) => void; +export const _dyn_core__ops__function__FnMut_____Output___R_as_wasm_bindgen__closure__WasmClosure___describe__invoke__h078d17eff0b70a99: (a: number, b: number) => void; +export const _dyn_core__ops__function__FnMut_____Output___R_as_wasm_bindgen__closure__WasmClosure___describe__invoke__h26b4819ece79f796: (a: number, b: number) => void; +export const _dyn_core__ops__function__FnMut__A____Output___R_as_wasm_bindgen__closure__WasmClosure___describe__invoke__hb1f2a4138d993283: (a: number, b: number, c: number) => void; export const __wbindgen_start: () => void;