From 37613a65c447cbc28de794b983a2072f0c5228bd Mon Sep 17 00:00:00 2001 From: restitux Date: Fri, 5 Dec 2025 01:54:01 -0700 Subject: [PATCH] web: fix audio on firefox --- gui/Cargo.toml | 12 +- ...t_mic_worklet.js => rust_audio_worklet.js} | 46 +++++++- gui/src/imp/desktop.rs | 2 +- gui/src/imp/web.rs | 111 +++++++++--------- gui/src/lib.rs | 2 +- 5 files changed, 109 insertions(+), 64 deletions(-) rename gui/assets/{rust_mic_worklet.js => rust_audio_worklet.js} (59%) diff --git a/gui/Cargo.toml b/gui/Cargo.toml index 71e99ac..1c92c50 100644 --- a/gui/Cargo.toml +++ b/gui/Cargo.toml @@ -30,8 +30,6 @@ web-sys = { version = "^0.3.72", features = [ "EncodedAudioChunkInit", "EncodedAudioChunkType", "CodecState", - "MediaStreamTrackGenerator", - "MediaStreamTrackGeneratorInit", "AudioContext", "AudioContextOptions", "MediaStream", @@ -42,6 +40,7 @@ web-sys = { version = "^0.3.72", features = [ "AudioWorkletNode", "AudioWorklet", "AudioWorkletProcessor", + "MessagePort", "MediaStreamConstraints", "WorkletOptions", "AudioEncoder", @@ -88,7 +87,7 @@ tracing = "^0.1.40" color-eyre = "^0.6.3" crossbeam-queue = "^0.3.11" lol_html = "^2.2.0" -rfd = { git = "https://github.com/samsartor/rfd.git", version = "^0.16.0", default-features = false } +rfd = { git = "https://github.com/samsartor/rfd.git", version = "^0.16.0", default-features = false } base64 = "^0.22" mime_guess = "^2.0.5" async_cell = "^0.2.3" @@ -97,7 +96,9 @@ dioxus-asset-resolver = "0.7.1" # Denoising # ========= -deep_filter = { git = "https://github.com/Rikorose/DeepFilterNet.git", rev = "d375b2d8309e0935d165700c91da9de862a99c31", features = ["tract"] } +deep_filter = { git = "https://github.com/Rikorose/DeepFilterNet.git", rev = "d375b2d8309e0935d165700c91da9de862a99c31", features = [ + "tract", +] } crossbeam = "0.8.4" [patch.crates-io] @@ -130,4 +131,5 @@ desktop = [ "dasp_ring_buffer", "rfd/xdg-portal", "rfd/tokio", -] \ No newline at end of file +] + diff --git a/gui/assets/rust_mic_worklet.js b/gui/assets/rust_audio_worklet.js similarity index 59% rename from gui/assets/rust_mic_worklet.js rename to gui/assets/rust_audio_worklet.js index 2b77709..ab82f08 100644 --- a/gui/assets/rust_mic_worklet.js +++ b/gui/assets/rust_audio_worklet.js @@ -1,7 +1,7 @@ const SAMPLE_RATE = 48000; const PACKET_SAMPLES = 960; -class RustWorklet extends AudioWorkletProcessor { +class RustMicWorklet extends AudioWorkletProcessor { constructor(options) { super(); this.module = options.processorOptions; @@ -31,7 +31,7 @@ class RustWorklet extends AudioWorkletProcessor { } this.buffer_offset -= PACKET_SAMPLES; this.timestamp = null; - } + } process(inputs) { //console.log(inputs); @@ -60,4 +60,44 @@ class RustWorklet extends AudioWorkletProcessor { } }; -registerProcessor("rust_mic_worklet", RustWorklet); + +class RustSpeakerWorklet extends AudioWorkletProcessor { + constructor() { + super(); + this.queue = []; + this.readIndex = 0; + + this.port.onmessage = (event) => { + this.queue.push(event.data) + }; + } + + process(inputs, outputs) { + if (this.queue.length) { + console.log(this.queue[0].samples.length, outputs[0][0].length); + } + + const output = outputs[0]; + + for (let i = 0; i < output[0].length; i++) { + if (!this.queue.length) { + return true; + } + const current = this.queue[0]; + for (let ch = 0; ch < output.length; ch++) { + output[ch][i] = current.samples[this.readIndex]; + } + this.readIndex++; + if (this.readIndex >= current.samples.length) { + this.queue.shift(); + this.readIndex = 0; + } + } + return true; + } +}; + + + +registerProcessor("rust_mic_worklet", RustMicWorklet); +registerProcessor("rust_speaker_worklet", RustSpeakerWorklet); diff --git a/gui/src/imp/desktop.rs b/gui/src/imp/desktop.rs index 8337718..2e9faf2 100644 --- a/gui/src/imp/desktop.rs +++ b/gui/src/imp/desktop.rs @@ -43,7 +43,7 @@ const PACKET_SAMPLES: u32 = 960; type Buffer = Arc>>>; impl AudioSystem { - pub fn new() -> Result { + pub async fn new() -> Result { // TODO let host = cpal::default_host(); let name = host.id(); diff --git a/gui/src/imp/web.rs b/gui/src/imp/web.rs index 297c323..74f2321 100644 --- a/gui/src/imp/web.rs +++ b/gui/src/imp/web.rs @@ -15,7 +15,6 @@ use tracing::{debug, error, info, instrument}; use wasm_bindgen::prelude::*; use wasm_bindgen_futures::JsFuture; use web_sys::js_sys::{Promise, Reflect, Uint8Array}; -use web_sys::AudioContext; use web_sys::AudioContextOptions; use web_sys::AudioData; use web_sys::AudioDecoder; @@ -30,14 +29,13 @@ use web_sys::EncodedAudioChunkInit; use web_sys::EncodedAudioChunkType; use web_sys::MediaStream; use web_sys::MediaStreamConstraints; -use web_sys::MediaStreamTrackGenerator; -use web_sys::MediaStreamTrackGeneratorInit; use web_sys::MessageEvent; use web_sys::WebTransport; use web_sys::WebTransportBidirectionalStream; use web_sys::WebTransportOptions; use web_sys::WorkletOptions; use web_sys::{console, window}; +use web_sys::{AudioContext, AudioDataCopyToOptions}; pub use wasm_bindgen_futures::spawn_local as spawn; @@ -78,12 +76,41 @@ pub struct AudioSystem { processors: AudioProcessorSender, } +async fn attach_worklet(audio_context: &AudioContext) -> Result<(), Error> { + // Create worklets to process mic and speaker audio + // Speaker audio processing worklet only required on + // browsers that don't support MediaStreamTrackGenerator + + let options = WorkletOptions::new(); + Reflect::set( + &options, + &"processorOptions".into(), + &wasm_bindgen::module(), + ) + .ey()?; + + let module = asset!("assets/rust_audio_worklet.js").to_string(); + info!("loading mic worklet from {module:?}"); + audio_context + .audio_worklet() + .ey()? + .add_module_with_options(&module, &options) + .ey()? + .into_future() + .await + .ey()?; + Ok(()) +} + impl AudioSystem { - pub fn new() -> Result { + pub async fn new() -> Result { // Create MediaStreams to playback decoded audio // The audio context is used to reproduce audio. let webctx = configure_audio_context(); + attach_worklet(&webctx).await?; + let processors = AudioProcessorSender::default(); + Ok(AudioSystem { webctx, processors }) } @@ -104,18 +131,10 @@ impl AudioSystem { } pub fn create_player(&mut self) -> Result { - let audio_stream_generator = - MediaStreamTrackGenerator::new(&MediaStreamTrackGeneratorInit::new("audio")).ey()?; + let sink_node = AudioWorkletNode::new(&self.webctx, "rust_speaker_worklet").ey()?; - // Create MediaStream from MediaStreamTrackGenerator - let js_tracks = web_sys::js_sys::Array::new(); - js_tracks.push(&audio_stream_generator); - let media_stream = MediaStream::new_with_tracks(&js_tracks).ey()?; - - // Create MediaStreamAudioSourceNode - let audio_source = self.webctx.create_media_stream_source(&media_stream).ey()?; - // Connect output of audio_source to audio_context (browser audio) - audio_source + // Connect worklet to destination + sink_node .connect_with_audio_node(&self.webctx.destination()) .ey()?; @@ -124,28 +143,31 @@ impl AudioSystem { error!("error decoding audio {:?}", e); }) as Box); - // This knows what MediaStreamTrackGenerator to use as it closes around it + let sink_port = sink_node.port().ey()?; + let output = Closure::wrap(Box::new(move |audio_data: AudioData| { - let writable = audio_stream_generator.writable(); - if writable.locked() { - return; - } - if let Err(e) = writable.get_writer().map(|writer| { - spawn(async move { - if let Err(e) = JsFuture::from(writer.ready()).await.ey() { - error!("write chunk ready error {:?}", e); - } - if let Err(e) = JsFuture::from(writer.write_with_chunk(&audio_data)) - .await - .ey() - { - error!("write chunk error {:?}", e); - }; - writer.release_lock(); - }); - }) { - error!("error writing audio data {:?}", e); + // Extract planar PCM from AudioData into an ArrayBuffer or Float32Array + // Here we assume f32 samples, 1 channel for brevity. + let number_of_frames = audio_data.number_of_frames(); + + let js_buffer = Float32Array::new_with_length(number_of_frames); + + let audio_data_copy_to_options = &AudioDataCopyToOptions::new(0); + audio_data_copy_to_options.set_format(web_sys::AudioSampleFormat::F32); + + if let Err(e) = audio_data + .copy_to_with_buffer_source(&js_buffer.buffer(), &audio_data_copy_to_options) + { + error!("could not copy audio data to array {:?}", e); } + + // Post to the worklet; include sampleRate and channel count if needed. + let msg = js_sys::Object::new(); + js_sys::Reflect::set(&msg, &"samples".into(), &js_buffer).unwrap(); + + sink_port.post_message(&msg).unwrap(); + + audio_data.close(); }) as Box); let audio_decoder = AudioDecoder::new(&AudioDecoderInit::new( @@ -234,25 +256,6 @@ async fn run_encoder_worklet( .map_err(|e| JsError::new(&format!("not a stream: {e:?}"))) .ey()?; - let options = WorkletOptions::new(); - Reflect::set( - &options, - &"processorOptions".into(), - &wasm_bindgen::module(), - ) - .ey()?; - - let module = asset!("assets/rust_mic_worklet.js").to_string(); - info!("loading mic worklet from {module:?}"); - audio_context - .audio_worklet() - .ey()? - .add_module_with_options(&module, &options) - .ey()? - .into_future() - .await - .ey()?; - let source = audio_context.create_media_stream_source(&stream).ey()?; let worklet_node = AudioWorkletNode::new(audio_context, "rust_mic_worklet").ey()?; diff --git a/gui/src/lib.rs b/gui/src/lib.rs index 5b40f62..a8a163d 100644 --- a/gui/src/lib.rs +++ b/gui/src/lib.rs @@ -113,7 +113,7 @@ pub async fn network_loop( }); } - let mut audio = imp::AudioSystem::new()?; + let mut audio = imp::AudioSystem::new().await?; { let send_chan = send_chan.clone(); let mut sequence_num = 0;