try to run denoising

Cargo.lock

@@ -671,6 +671,12 @@ dependencies = [
  "libloading 0.8.5",
 ]
 
+[[package]]
+name = "claxon"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4bfbf56724aa9eca8afa4fcfadeb479e722935bb2a0900c2d37e0cc477af0688"
+
 [[package]]
 name = "cmake"
 version = "0.1.51"
@@ -1004,6 +1010,19 @@ dependencies = [
  "cfg-if",
 ]
 
+[[package]]
+name = "crossbeam"
+version = "0.8.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1137cd7e7fc0fb5d3c5a8678be38ec56e819125d8d7907411fe24ccb943faca8"
+dependencies = [
+ "crossbeam-channel",
+ "crossbeam-deque",
+ "crossbeam-epoch",
+ "crossbeam-queue",
+ "crossbeam-utils",
+]
+
 [[package]]
 name = "crossbeam-channel"
 version = "0.5.13"
@@ -1013,6 +1032,25 @@ dependencies = [
  "crossbeam-utils",
 ]
 
+[[package]]
+name = "crossbeam-deque"
+version = "0.8.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51"
+dependencies = [
+ "crossbeam-epoch",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-epoch"
+version = "0.9.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
+dependencies = [
+ "crossbeam-utils",
+]
+
 [[package]]
 name = "crossbeam-queue"
 version = "0.3.11"
@@ -1163,6 +1201,20 @@ version = "2.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2"
 
+[[package]]
+name = "deep_filter"
+version = "0.5.7-pre"
+source = "git+https://github.com/Rikorose/DeepFilterNet.git?rev=d375b2d8309e0935d165700c91da9de862a99c31#d375b2d8309e0935d165700c91da9de862a99c31"
+dependencies = [
+ "claxon",
+ "itertools",
+ "lewton",
+ "num-complex",
+ "ogg 0.8.0",
+ "realfft",
+ "rustfft",
+]
+
 [[package]]
 name = "deranged"
 version = "0.3.11"
@@ -1926,9 +1978,9 @@ checksum = "aa9a19cbb55df58761df49b23516a86d432839add4af60fc256da840f66ed35b"
 
 [[package]]
 name = "form_urlencoded"
-version = "1.2.1"
+version = "1.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456"
+checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf"
 dependencies = [
  "percent-encoding",
 ]
@@ -2872,9 +2924,9 @@ checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
 
 [[package]]
 name = "idna"
-version = "1.0.3"
+version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e"
+checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de"
 dependencies = [
  "idna_adapter",
  "smallvec",
@@ -3102,6 +3154,17 @@ version = "1.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
 
+[[package]]
+name = "lewton"
+version = "0.10.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "777b48df9aaab155475a83a7df3070395ea1ac6902f5cd062b8f2b028075c030"
+dependencies = [
+ "byteorder",
+ "ogg 0.8.0",
+ "tinyvec",
+]
+
 [[package]]
 name = "libappindicator"
 version = "0.9.0"
@@ -3561,8 +3624,10 @@ dependencies = [
  "byteorder",
  "color-eyre",
  "cpal",
+ "crossbeam",
  "crossbeam-queue",
  "dasp_ring_buffer",
+ "deep_filter",
  "dioxus",
  "dioxus-desktop",
  "dioxus-web",
@@ -3577,7 +3642,7 @@ dependencies = [
  "mime_guess",
  "mumble-protocol-2x",
  "mumble-web2-common",
- "ogg",
+ "ogg 0.9.1",
  "once_cell",
  "opus",
  "ordermap",
@@ -3759,6 +3824,16 @@ dependencies = [
  "num-traits",
 ]
 
+[[package]]
+name = "num-complex"
+version = "0.4.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495"
+dependencies = [
+ "num-traits",
+ "serde",
+]
+
 [[package]]
 name = "num-conv"
 version = "0.1.0"
@@ -3997,6 +4072,15 @@ dependencies = [
  "cc",
 ]
 
+[[package]]
+name = "ogg"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6951b4e8bf21c8193da321bcce9c9dd2e13c858fe078bf9054a288b419ae5d6e"
+dependencies = [
+ "byteorder",
+]
+
 [[package]]
 name = "ogg"
 version = "0.9.1"
@@ -4187,9 +4271,9 @@ dependencies = [
 
 [[package]]
 name = "percent-encoding"
-version = "2.3.1"
+version = "2.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e"
+checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220"
 
 [[package]]
 name = "phf"
@@ -4437,6 +4521,15 @@ dependencies = [
  "syn 2.0.87",
 ]
 
+[[package]]
+name = "primal-check"
+version = "0.3.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dc0d895b311e3af9902528fbb8f928688abbd95872819320517cc24ca6b2bd08"
+dependencies = [
+ "num-integer",
+]
+
 [[package]]
 name = "proc-macro-crate"
 version = "1.3.1"
@@ -4750,6 +4843,15 @@ dependencies = [
  "yasna",
 ]
 
+[[package]]
+name = "realfft"
+version = "3.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f821338fddb99d089116342c46e9f1fbf3828dba077674613e734e01d6ea8677"
+dependencies = [
+ "rustfft",
+]
+
 [[package]]
 name = "redox_syscall"
 version = "0.5.7"
@@ -4998,6 +5100,20 @@ dependencies = [
  "semver",
 ]
 
+[[package]]
+name = "rustfft"
+version = "6.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "21db5f9893e91f41798c88680037dba611ca6674703c1a18601b01a72c8adb89"
+dependencies = [
+ "num-complex",
+ "num-integer",
+ "num-traits",
+ "primal-check",
+ "strength_reduce",
+ "transpose",
+]
+
 [[package]]
 name = "rustix"
 version = "0.38.44"
@@ -5802,6 +5918,12 @@ version = "3.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0ceb97b7225c713c2fd4db0153cb6b3cab244eb37900c3f634ed4d43310d8c34"
 
+[[package]]
+name = "strength_reduce"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fe895eb47f22e2ddd4dabc02bce419d2e643c8e3b585c78158b349195bc24d82"
+
 [[package]]
 name = "string_cache"
 version = "0.8.7"
@@ -6378,6 +6500,16 @@ dependencies = [
  "web-sys",
 ]
 
+[[package]]
+name = "transpose"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1ad61aed86bc3faea4300c7aee358b4c6d0c8d6ccc36524c96e4c92ccf26e77e"
+dependencies = [
+ "num-integer",
+ "strength_reduce",
+]
+
 [[package]]
 name = "tray-icon"
 version = "0.19.1"
@@ -6506,9 +6638,9 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
 
 [[package]]
 name = "url"
-version = "2.5.3"
+version = "2.5.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8d157f1b96d14500ffdc1f10ba712e780825526c03d9a49b4d0324b0d9113ada"
+checksum = "08bc136a29a3d1758e07a9cca267be308aeebf5cfd5a10f3f67ab2097683ef5b"
 dependencies = [
  "form_urlencoded",
  "idna",

gui/Cargo.toml

@@ -98,6 +98,11 @@ mime_guess = "2.0.5"
 async_cell = "0.2.3"
 reqwest = { version = "0.12.22", features = ["json"] }
 
+# Denoising
+# =========
+deep_filter = { git = "https://github.com/Rikorose/DeepFilterNet.git", rev = "d375b2d8309e0935d165700c91da9de862a99c31" }
+crossbeam = "0.8.4"
+
 [features]
 web = [
   "dioxus/web",

gui/src/app.rs

@@ -48,6 +48,9 @@ pub enum Command {
         channel: ChannelId,
         user: UserId,
     },
+    UpdateMicEffects {
+        denoise: bool,
+    },
     Disconnect,
 }
 
@@ -633,6 +636,7 @@ pub fn ControlView(config: Resource<ClientConfig>) -> Element {
         },
     };
 
+    let denoise = use_signal(|| false);
     rsx!(
         // Server control
         div {
@@ -672,6 +676,23 @@ pub fn ControlView(config: Resource<ClientConfig>) -> Element {
                 }
             }
             span { class: "{spacer}" }
+            button {
+                class: match denoise() {
+                    true => toggle_button_on,
+                    false => toggle_button,
+                },
+                role: "switch",
+                aria_checked: denoise(),
+                onclick: move |_| {
+                    let new_denoise = !denoise();
+                    *denoise.write_unchecked() = new_denoise;
+                    net.send(UpdateMicEffects { denoise: new_denoise })
+                },
+                match denoise() {
+                    true => rsx!(span { class: "material-symbols-outlined", style: "{button_style}", "cadence"}),
+                    false => rsx!(span { class: "material-symbols-outlined", style: "{button_style}", "graphic_eq"}),
+                }
+            }
             button {
                 class: match mute || self_mute {
                     true => toggle_button_on,

gui/src/effects.rs

@@ -0,0 +1,28 @@
+use crossbeam::atomic::AtomicCell;
+use std::sync::Arc;
+
+#[derive(Default)]
+pub struct AudioProcessor {
+    df: Option<::df::DFState>,
+}
+
+impl AudioProcessor {
+    pub fn new_denoising() -> Self {
+        let df = ::df::DFState::default();
+        AudioProcessor { df: Some(df) }
+    }
+}
+
+impl AudioProcessor {
+    pub fn process(&mut self, audio: &[f32]) -> Box<[f32]> {
+        let mut output: Box<[f32]> = vec![0f32; audio.len()].into();
+        if let Some(df) = &mut self.df {
+            df.process_frame(audio, &mut output);
+        } else {
+            output.copy_from_slice(audio);
+        }
+        output
+    }
+}
+
+pub type AudioProcessorSender = Arc<AtomicCell<Option<AudioProcessor>>>;

gui/src/imp/desktop.rs

@@ -1,4 +1,5 @@
 use crate::app::Command;
+use crate::effects::AudioProcessor;
 use color_eyre::eyre::{eyre, Error};
 use cpal::traits::{DeviceTrait, HostTrait};
 use dioxus::hooks::{UnboundedReceiver, UnboundedSender};
@@ -13,7 +14,7 @@ use tokio::net::TcpStream;
 use tokio_rustls::rustls;
 use tokio_rustls::rustls::client::danger::{HandshakeSignatureValid, ServerCertVerifier};
 use tokio_rustls::rustls::pki_types::{CertificateDer, ServerName, UnixTime};
-use tokio_rustls::rustls::ClientConfig;
+use tokio_rustls::rustls::ClientConfig as RlsClientConfig;
 use tokio_rustls::rustls::DigitallySignedStruct;
 use tokio_rustls::TlsConnector;
 use tokio_util::compat::{TokioAsyncReadCompatExt as _, TokioAsyncWriteCompatExt as _};
@@ -50,6 +51,10 @@ impl AudioSystem {
         })
     }
 
+    pub fn set_processor(&self, processor: AudioProcessor) {
+        // TODO
+    }
+
     pub fn start_recording(&mut self, each: impl FnMut(Vec<u8>) + 'static) -> Result<(), Error> {
         // TODO
         Ok(())
@@ -192,7 +197,7 @@ pub async fn network_connect(
 ) -> Result<(), Error> {
     info!("connecting");
 
-    let config = ClientConfig::builder()
+    let config = RlsClientConfig::builder()
         .dangerous()
         .with_custom_certificate_verifier(Arc::new(NoCertificateVerification))
         .with_no_client_auth();

gui/src/imp/web.rs

@@ -1,12 +1,11 @@
 use crate::app::Command;
+use crate::effects::{AudioProcessor, AudioProcessorSender};
 use color_eyre::eyre::{bail, eyre, Error};
 use dioxus::prelude::*;
 use futures::{AsyncRead, AsyncWrite};
-use futures_channel::mpsc::UnboundedSender;
 use gloo_timers::future::TimeoutFuture;
-use mumble_protocol::control::{ClientControlCodec, ControlPacket};
+use js_sys::Float32Array;
-use mumble_protocol::voice::{VoicePacket, VoicePacketPayload};
+use mumble_protocol::control::ClientControlCodec;
-use mumble_protocol::Serverbound;
 use mumble_web2_common::ClientConfig;
 use reqwest::Url;
 use std::time::Duration;
@@ -72,20 +71,33 @@ impl<T> ResultExt<T> for Result<T, JsError> {
         self.map_err(|e| JsValue::from(e)).ey()
     }
 }
-pub struct AudioSystem(AudioContext);
+
+pub struct AudioSystem {
+    webctx: AudioContext,
+    processors: AudioProcessorSender,
+}
 
 impl AudioSystem {
     pub fn new() -> Result<Self, Error> {
         // Create MediaStreams to playback decoded audio
         // The audio context is used to reproduce audio.
-        let audio_context = configure_audio_context();
+        let webctx = configure_audio_context();
-        Ok(AudioSystem(audio_context))
+        let processor = AudioProcessorSender::default();
+        Ok(AudioSystem {
+            webctx,
+            processors: processor,
+        })
+    }
+
+    pub fn set_processor(&self, processor: AudioProcessor) {
+        self.processors.store(Some(processor))
     }
 
     pub fn start_recording(&mut self, each: impl FnMut(Vec<u8>) + 'static) -> Result<(), Error> {
-        let audio_context_worklet = self.0.clone();
+        let audio_context_worklet = self.webctx.clone();
+        let processors = self.processors.clone();
         spawn(async move {
-            match run_encoder_worklet(&audio_context_worklet, each).await {
+            match run_encoder_worklet(&audio_context_worklet, each, processors).await {
                 Ok(node) => info!("created encoder worklet: {:?}", &node),
                 Err(err) => error!("could not create encoder worklet: {err}"),
             }
@@ -94,8 +106,6 @@ impl AudioSystem {
     }
 
     pub fn create_player(&mut self) -> Result<AudioPlayer, Error> {
-        let audio_context = &self.0;
-
         let audio_stream_generator =
             MediaStreamTrackGenerator::new(&MediaStreamTrackGeneratorInit::new("audio")).ey()?;
 
@@ -105,12 +115,10 @@ impl AudioSystem {
         let media_stream = MediaStream::new_with_tracks(&js_tracks).ey()?;
 
         // Create MediaStreamAudioSourceNode
-        let audio_source = audio_context
+        let audio_source = self.webctx.create_media_stream_source(&media_stream).ey()?;
-            .create_media_stream_source(&media_stream)
-            .ey()?;
         // Connect output of audio_source to audio_context (browser audio)
         audio_source
-            .connect_with_audio_node(&audio_context.destination())
+            .connect_with_audio_node(&self.webctx.destination())
             .ey()?;
 
         // Create callback functions for AudioDecoder
@@ -194,9 +202,22 @@ impl PromiseExt for Promise {
     }
 }
 
+fn process_audio(frame: &JsValue, processor: &mut AudioProcessor) {
+    let Ok(samples) = Reflect::get(&frame, &"data".into()) else {
+        return;
+    };
+    let Ok(samples) = samples.dyn_into::<Float32Array>() else {
+        return;
+    };
+    let input = samples.to_vec();
+    let output = processor.process(&input);
+    samples.copy_from(&output);
+}
+
 async fn run_encoder_worklet(
     audio_context: &AudioContext,
     mut each: impl FnMut(Vec<u8>) + 'static,
+    processors: AudioProcessorSender,
 ) -> Result<AudioWorkletNode, Error> {
     let constraints = MediaStreamConstraints::new();
     constraints.set_audio(&JsValue::TRUE);
@@ -264,23 +285,18 @@ async fn run_encoder_worklet(
     audio_encoder.configure(&encoder_config);
     info!("created audio encoder");
 
-    let download_buffer = std::cell::RefCell::new(Vec::new());
+    let mut current_processor = AudioProcessor::default();
-
     let onmessage: Closure<dyn FnMut(MessageEvent)> = Closure::new(move |event: MessageEvent| {
-        match AudioData::new(event.data().unchecked_ref()) {
+        if let Some(new_processor) = processors.take() {
-            Ok(data) => {
+            current_processor = new_processor;
-                let x = web_sys::AudioDataCopyToOptions::new(0);
-                x.set_format(web_sys::AudioSampleFormat::F32);
-                let mut sub_buffer = vec![0; data.allocation_size(&x).unwrap() as usize];
-                data.copy_to_with_u8_slice(&mut sub_buffer, &x);
-                download_buffer.borrow_mut().append(&mut sub_buffer);
-                if download_buffer.borrow().len() > 48000 * 10 * 4 {
-                    //pub fn download_data(data: Vec<u8>, filename: &str) -> Result<(), JsValue> {
-                    //download_data(download_buffer.borrow().to_vec(), "download_buffer.pcm32");
-                    download_buffer.borrow_mut().clear();
         }
 
-                audio_encoder.encode(&data);
+        let frame = event.data();
+        process_audio(&frame, &mut current_processor);
+
+        match AudioData::new(frame.unchecked_ref()) {
+            Ok(data) => {
+                let _ = audio_encoder.encode(&data);
             }
             Err(err) => {
                 error!(

gui/src/lib.rs

@@ -29,7 +29,11 @@ use tracing::debug;
 use tracing::error;
 use tracing::info;
 
+use crate::effects::AudioProcessor;
+use crate::imp::AudioSystem;
+
 pub mod app;
+mod effects;
 pub mod imp;
 mod msghtml;
 
@@ -161,7 +165,7 @@ pub async fn network_loop<R: imp::ImpRead, W: imp::ImpWrite>(
                 match command {
                     Some(Command::Disconnect) => break,
                     Some(command) => {
-                        let res = accept_command(command, &mut send_chan);
+                        let res = accept_command(command, &mut send_chan, &mut audio);
                         if let Err(err) = res {
                             info!("error accepting command {:?}", err)
                         }
@@ -179,6 +183,7 @@ pub async fn network_loop<R: imp::ImpRead, W: imp::ImpWrite>(
 fn accept_command(
     command: Command,
     send_chan: &mut UnboundedSender<ControlPacket<mumble_protocol::Serverbound>>,
+    audio: &mut AudioSystem,
 ) -> Result<(), Error> {
     use Command::*;
     let Some(session) = STATE.server.read().session else {
@@ -280,6 +285,13 @@ fn accept_command(
             let _ = send_chan.unbounded_send(u.into());
         }
         Connect { .. } | Disconnect => (),
+        UpdateMicEffects { denoise } => {
+            if denoise {
+                audio.set_processor(AudioProcessor::new_denoising());
+            } else {
+                audio.set_processor(AudioProcessor::default());
+            }
+        }
     }
 
     Ok(())