web: fix audio on firefox

gui/Cargo.toml

@@ -30,8 +30,6 @@ web-sys = { version = "^0.3.72", features = [
   "EncodedAudioChunkInit",
   "EncodedAudioChunkType",
   "CodecState",
-  "MediaStreamTrackGenerator",
-  "MediaStreamTrackGeneratorInit",
   "AudioContext",
   "AudioContextOptions",
   "MediaStream",
@@ -42,6 +40,7 @@ web-sys = { version = "^0.3.72", features = [
   "AudioWorkletNode",
   "AudioWorklet",
   "AudioWorkletProcessor",
+  "MessagePort",
   "MediaStreamConstraints",
   "WorkletOptions",
   "AudioEncoder",
@@ -88,7 +87,7 @@ tracing = "^0.1.40"
 color-eyre = "^0.6.3"
 crossbeam-queue = "^0.3.11"
 lol_html = "^2.2.0"
-rfd = { git = "https://github.com/samsartor/rfd.git", version = "^0.16.0", default-features = false } 
+rfd = { git = "https://github.com/samsartor/rfd.git", version = "^0.16.0", default-features = false }
 base64 = "^0.22"
 mime_guess = "^2.0.5"
 async_cell = "^0.2.3"
@@ -97,7 +96,9 @@ dioxus-asset-resolver = "0.7.1"
 
 # Denoising
 # =========
-deep_filter = { git = "https://github.com/Rikorose/DeepFilterNet.git", rev = "d375b2d8309e0935d165700c91da9de862a99c31", features = ["tract"] }
+deep_filter = { git = "https://github.com/Rikorose/DeepFilterNet.git", rev = "d375b2d8309e0935d165700c91da9de862a99c31", features = [
+  "tract",
+] }
 crossbeam = "0.8.4"
 
 [patch.crates-io]
@@ -130,4 +131,5 @@ desktop = [
   "dasp_ring_buffer",
   "rfd/xdg-portal",
   "rfd/tokio",
-]
+]
+

gui/assets/rust_audio_worklet.js

@@ -1,7 +1,7 @@
 const SAMPLE_RATE = 48000;
 const PACKET_SAMPLES = 960;
 
-class RustWorklet extends AudioWorkletProcessor {
+class RustMicWorklet extends AudioWorkletProcessor {
   constructor(options) {
     super();
     this.module = options.processorOptions;
@@ -31,7 +31,7 @@ class RustWorklet extends AudioWorkletProcessor {
     }
     this.buffer_offset -= PACKET_SAMPLES;
     this.timestamp = null;
-   }
+  }
 
   process(inputs) {
     //console.log(inputs);
@@ -60,4 +60,44 @@ class RustWorklet extends AudioWorkletProcessor {
   }
 };
 
-registerProcessor("rust_mic_worklet", RustWorklet);
+
+class RustSpeakerWorklet extends AudioWorkletProcessor {
+  constructor() {
+    super();
+    this.queue = [];
+    this.readIndex = 0;
+
+    this.port.onmessage = (event) => {
+      this.queue.push(event.data)
+    };
+  }
+
+  process(inputs, outputs) {
+    if (this.queue.length) {
+      console.log(this.queue[0].samples.length, outputs[0][0].length);
+    }
+
+    const output = outputs[0];
+
+    for (let i = 0; i < output[0].length; i++) {
+      if (!this.queue.length) {
+        return true;
+      }
+      const current = this.queue[0];
+      for (let ch = 0; ch < output.length; ch++) {
+        output[ch][i] = current.samples[this.readIndex];
+      }
+      this.readIndex++;
+      if (this.readIndex >= current.samples.length) {
+        this.queue.shift();
+        this.readIndex = 0;
+      }
+    }
+    return true;
+  }
+};
+
+
+
+registerProcessor("rust_mic_worklet", RustMicWorklet);
+registerProcessor("rust_speaker_worklet", RustSpeakerWorklet);

gui/src/imp/desktop.rs

@@ -43,7 +43,7 @@ const PACKET_SAMPLES: u32 = 960;
 type Buffer = Arc<Mutex<dasp_ring_buffer::Bounded<Vec<i16>>>>;
 
 impl AudioSystem {
-    pub fn new() -> Result<Self, Error> {
+    pub async fn new() -> Result<Self, Error> {
         // TODO
         let host = cpal::default_host();
         let name = host.id();

gui/src/imp/web.rs

@@ -15,7 +15,6 @@ use tracing::{debug, error, info, instrument};
 use wasm_bindgen::prelude::*;
 use wasm_bindgen_futures::JsFuture;
 use web_sys::js_sys::{Promise, Reflect, Uint8Array};
-use web_sys::AudioContext;
 use web_sys::AudioContextOptions;
 use web_sys::AudioData;
 use web_sys::AudioDecoder;
@@ -30,14 +29,13 @@ use web_sys::EncodedAudioChunkInit;
 use web_sys::EncodedAudioChunkType;
 use web_sys::MediaStream;
 use web_sys::MediaStreamConstraints;
-use web_sys::MediaStreamTrackGenerator;
-use web_sys::MediaStreamTrackGeneratorInit;
 use web_sys::MessageEvent;
 use web_sys::WebTransport;
 use web_sys::WebTransportBidirectionalStream;
 use web_sys::WebTransportOptions;
 use web_sys::WorkletOptions;
 use web_sys::{console, window};
+use web_sys::{AudioContext, AudioDataCopyToOptions};
 
 pub use wasm_bindgen_futures::spawn_local as spawn;
 
@@ -78,12 +76,41 @@ pub struct AudioSystem {
     processors: AudioProcessorSender,
 }
 
+async fn attach_worklet(audio_context: &AudioContext) -> Result<(), Error> {
+    // Create worklets to process mic and speaker audio
+    // Speaker audio processing worklet only required on
+    // browsers that don't support MediaStreamTrackGenerator
+
+    let options = WorkletOptions::new();
+    Reflect::set(
+        &options,
+        &"processorOptions".into(),
+        &wasm_bindgen::module(),
+    )
+    .ey()?;
+
+    let module = asset!("assets/rust_audio_worklet.js").to_string();
+    info!("loading mic worklet from {module:?}");
+    audio_context
+        .audio_worklet()
+        .ey()?
+        .add_module_with_options(&module, &options)
+        .ey()?
+        .into_future()
+        .await
+        .ey()?;
+    Ok(())
+}
+
 impl AudioSystem {
-    pub fn new() -> Result<Self, Error> {
+    pub async fn new() -> Result<Self, Error> {
         // Create MediaStreams to playback decoded audio
         // The audio context is used to reproduce audio.
         let webctx = configure_audio_context();
+        attach_worklet(&webctx).await?;
+
         let processors = AudioProcessorSender::default();
+
         Ok(AudioSystem { webctx, processors })
     }
 
@@ -104,18 +131,10 @@ impl AudioSystem {
     }
 
     pub fn create_player(&mut self) -> Result<AudioPlayer, Error> {
-        let audio_stream_generator =
-            MediaStreamTrackGenerator::new(&MediaStreamTrackGeneratorInit::new("audio")).ey()?;
+        let sink_node = AudioWorkletNode::new(&self.webctx, "rust_speaker_worklet").ey()?;
 
-        // Create MediaStream from MediaStreamTrackGenerator
-        let js_tracks = web_sys::js_sys::Array::new();
-        js_tracks.push(&audio_stream_generator);
-        let media_stream = MediaStream::new_with_tracks(&js_tracks).ey()?;
-
-        // Create MediaStreamAudioSourceNode
-        let audio_source = self.webctx.create_media_stream_source(&media_stream).ey()?;
-        // Connect output of audio_source to audio_context (browser audio)
-        audio_source
+        // Connect worklet to destination
+        sink_node
             .connect_with_audio_node(&self.webctx.destination())
             .ey()?;
 
@@ -124,28 +143,31 @@ impl AudioSystem {
             error!("error decoding audio {:?}", e);
         }) as Box<dyn FnMut(JsValue)>);
 
-        // This knows what MediaStreamTrackGenerator to use as it closes around it
+        let sink_port = sink_node.port().ey()?;
+
         let output = Closure::wrap(Box::new(move |audio_data: AudioData| {
-            let writable = audio_stream_generator.writable();
-            if writable.locked() {
-                return;
-            }
-            if let Err(e) = writable.get_writer().map(|writer| {
-                spawn(async move {
-                    if let Err(e) = JsFuture::from(writer.ready()).await.ey() {
-                        error!("write chunk ready error {:?}", e);
-                    }
-                    if let Err(e) = JsFuture::from(writer.write_with_chunk(&audio_data))
-                        .await
-                        .ey()
-                    {
-                        error!("write chunk error {:?}", e);
-                    };
-                    writer.release_lock();
-                });
-            }) {
-                error!("error writing audio data {:?}", e);
+            // Extract planar PCM from AudioData into an ArrayBuffer or Float32Array
+            // Here we assume f32 samples, 1 channel for brevity.
+            let number_of_frames = audio_data.number_of_frames();
+
+            let js_buffer = Float32Array::new_with_length(number_of_frames);
+
+            let audio_data_copy_to_options = &AudioDataCopyToOptions::new(0);
+            audio_data_copy_to_options.set_format(web_sys::AudioSampleFormat::F32);
+
+            if let Err(e) = audio_data
+                .copy_to_with_buffer_source(&js_buffer.buffer(), &audio_data_copy_to_options)
+            {
+                error!("could not copy audio data to array {:?}", e);
             }
+
+            // Post to the worklet; include sampleRate and channel count if needed.
+            let msg = js_sys::Object::new();
+            js_sys::Reflect::set(&msg, &"samples".into(), &js_buffer).unwrap();
+
+            sink_port.post_message(&msg).unwrap();
+
+            audio_data.close();
         }) as Box<dyn FnMut(AudioData)>);
 
         let audio_decoder = AudioDecoder::new(&AudioDecoderInit::new(
@@ -234,25 +256,6 @@ async fn run_encoder_worklet(
         .map_err(|e| JsError::new(&format!("not a stream: {e:?}")))
         .ey()?;
 
-    let options = WorkletOptions::new();
-    Reflect::set(
-        &options,
-        &"processorOptions".into(),
-        &wasm_bindgen::module(),
-    )
-    .ey()?;
-
-    let module = asset!("assets/rust_mic_worklet.js").to_string();
-    info!("loading mic worklet from {module:?}");
-    audio_context
-        .audio_worklet()
-        .ey()?
-        .add_module_with_options(&module, &options)
-        .ey()?
-        .into_future()
-        .await
-        .ey()?;
-
     let source = audio_context.create_media_stream_source(&stream).ey()?;
     let worklet_node = AudioWorkletNode::new(audio_context, "rust_mic_worklet").ey()?;
 

gui/src/lib.rs

@@ -113,7 +113,7 @@ pub async fn network_loop<R: imp::ImpRead, W: imp::ImpWrite>(
         });
     }
 
-    let mut audio = imp::AudioSystem::new()?;
+    let mut audio = imp::AudioSystem::new().await?;
     {
         let send_chan = send_chan.clone();
         let mut sequence_num = 0;