From 260decc9af2eed2746a350b54c34c8fcbd611db7 Mon Sep 17 00:00:00 2001 From: Sam Sartor Date: Sat, 25 Oct 2025 18:15:26 -0600 Subject: [PATCH] try to run denoising --- Cargo.lock | 150 ++++++++++++++++++++++++++++++++++++++--- gui/Cargo.toml | 5 ++ gui/src/app.rs | 21 ++++++ gui/src/effects.rs | 28 ++++++++ gui/src/imp/desktop.rs | 9 ++- gui/src/imp/web.rs | 76 ++++++++++++--------- gui/src/lib.rs | 14 +++- 7 files changed, 261 insertions(+), 42 deletions(-) create mode 100644 gui/src/effects.rs diff --git a/Cargo.lock b/Cargo.lock index f3f18a4..9789115 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -671,6 +671,12 @@ dependencies = [ "libloading 0.8.5", ] +[[package]] +name = "claxon" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bfbf56724aa9eca8afa4fcfadeb479e722935bb2a0900c2d37e0cc477af0688" + [[package]] name = "cmake" version = "0.1.51" @@ -1004,6 +1010,19 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1137cd7e7fc0fb5d3c5a8678be38ec56e819125d8d7907411fe24ccb943faca8" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-epoch", + "crossbeam-queue", + "crossbeam-utils", +] + [[package]] name = "crossbeam-channel" version = "0.5.13" @@ -1013,6 +1032,25 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-queue" version = "0.3.11" @@ -1163,6 +1201,20 @@ version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2" +[[package]] +name = "deep_filter" +version = "0.5.7-pre" +source = "git+https://github.com/Rikorose/DeepFilterNet.git?rev=d375b2d8309e0935d165700c91da9de862a99c31#d375b2d8309e0935d165700c91da9de862a99c31" +dependencies = [ + "claxon", + "itertools", + "lewton", + "num-complex", + "ogg 0.8.0", + "realfft", + "rustfft", +] + [[package]] name = "deranged" version = "0.3.11" @@ -1926,9 +1978,9 @@ checksum = "aa9a19cbb55df58761df49b23516a86d432839add4af60fc256da840f66ed35b" [[package]] name = "form_urlencoded" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" dependencies = [ "percent-encoding", ] @@ -2872,9 +2924,9 @@ checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" [[package]] name = "idna" -version = "1.0.3" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" dependencies = [ "idna_adapter", "smallvec", @@ -3102,6 +3154,17 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" +[[package]] +name = "lewton" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "777b48df9aaab155475a83a7df3070395ea1ac6902f5cd062b8f2b028075c030" +dependencies = [ + "byteorder", + "ogg 0.8.0", + "tinyvec", +] + [[package]] name = "libappindicator" version = "0.9.0" @@ -3561,8 +3624,10 @@ dependencies = [ "byteorder", "color-eyre", "cpal", + "crossbeam", "crossbeam-queue", "dasp_ring_buffer", + "deep_filter", "dioxus", "dioxus-desktop", "dioxus-web", @@ -3577,7 +3642,7 @@ dependencies = [ "mime_guess", "mumble-protocol-2x", "mumble-web2-common", - "ogg", + "ogg 0.9.1", "once_cell", "opus", "ordermap", @@ -3759,6 +3824,16 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", + "serde", +] + [[package]] name = "num-conv" version = "0.1.0" @@ -3997,6 +4072,15 @@ dependencies = [ "cc", ] +[[package]] +name = "ogg" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6951b4e8bf21c8193da321bcce9c9dd2e13c858fe078bf9054a288b419ae5d6e" +dependencies = [ + "byteorder", +] + [[package]] name = "ogg" version = "0.9.1" @@ -4187,9 +4271,9 @@ dependencies = [ [[package]] name = "percent-encoding" -version = "2.3.1" +version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "phf" @@ -4437,6 +4521,15 @@ dependencies = [ "syn 2.0.87", ] +[[package]] +name = "primal-check" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc0d895b311e3af9902528fbb8f928688abbd95872819320517cc24ca6b2bd08" +dependencies = [ + "num-integer", +] + [[package]] name = "proc-macro-crate" version = "1.3.1" @@ -4750,6 +4843,15 @@ dependencies = [ "yasna", ] +[[package]] +name = "realfft" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f821338fddb99d089116342c46e9f1fbf3828dba077674613e734e01d6ea8677" +dependencies = [ + "rustfft", +] + [[package]] name = "redox_syscall" version = "0.5.7" @@ -4998,6 +5100,20 @@ dependencies = [ "semver", ] +[[package]] +name = "rustfft" +version = "6.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21db5f9893e91f41798c88680037dba611ca6674703c1a18601b01a72c8adb89" +dependencies = [ + "num-complex", + "num-integer", + "num-traits", + "primal-check", + "strength_reduce", + "transpose", +] + [[package]] name = "rustix" version = "0.38.44" @@ -5802,6 +5918,12 @@ version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ceb97b7225c713c2fd4db0153cb6b3cab244eb37900c3f634ed4d43310d8c34" +[[package]] +name = "strength_reduce" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe895eb47f22e2ddd4dabc02bce419d2e643c8e3b585c78158b349195bc24d82" + [[package]] name = "string_cache" version = "0.8.7" @@ -6378,6 +6500,16 @@ dependencies = [ "web-sys", ] +[[package]] +name = "transpose" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad61aed86bc3faea4300c7aee358b4c6d0c8d6ccc36524c96e4c92ccf26e77e" +dependencies = [ + "num-integer", + "strength_reduce", +] + [[package]] name = "tray-icon" version = "0.19.1" @@ -6506,9 +6638,9 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "url" -version = "2.5.3" +version = "2.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d157f1b96d14500ffdc1f10ba712e780825526c03d9a49b4d0324b0d9113ada" +checksum = "08bc136a29a3d1758e07a9cca267be308aeebf5cfd5a10f3f67ab2097683ef5b" dependencies = [ "form_urlencoded", "idna", diff --git a/gui/Cargo.toml b/gui/Cargo.toml index 39d0434..e51e62b 100644 --- a/gui/Cargo.toml +++ b/gui/Cargo.toml @@ -98,6 +98,11 @@ mime_guess = "2.0.5" async_cell = "0.2.3" reqwest = { version = "0.12.22", features = ["json"] } +# Denoising +# ========= +deep_filter = { git = "https://github.com/Rikorose/DeepFilterNet.git", rev = "d375b2d8309e0935d165700c91da9de862a99c31" } +crossbeam = "0.8.4" + [features] web = [ "dioxus/web", diff --git a/gui/src/app.rs b/gui/src/app.rs index 2325d1f..5a4fadb 100644 --- a/gui/src/app.rs +++ b/gui/src/app.rs @@ -48,6 +48,9 @@ pub enum Command { channel: ChannelId, user: UserId, }, + UpdateMicEffects { + denoise: bool, + }, Disconnect, } @@ -633,6 +636,7 @@ pub fn ControlView(config: Resource) -> Element { }, }; + let denoise = use_signal(|| false); rsx!( // Server control div { @@ -672,6 +676,23 @@ pub fn ControlView(config: Resource) -> Element { } } span { class: "{spacer}" } + button { + class: match denoise() { + true => toggle_button_on, + false => toggle_button, + }, + role: "switch", + aria_checked: denoise(), + onclick: move |_| { + let new_denoise = !denoise(); + *denoise.write_unchecked() = new_denoise; + net.send(UpdateMicEffects { denoise: new_denoise }) + }, + match denoise() { + true => rsx!(span { class: "material-symbols-outlined", style: "{button_style}", "cadence"}), + false => rsx!(span { class: "material-symbols-outlined", style: "{button_style}", "graphic_eq"}), + } + } button { class: match mute || self_mute { true => toggle_button_on, diff --git a/gui/src/effects.rs b/gui/src/effects.rs new file mode 100644 index 0000000..addd072 --- /dev/null +++ b/gui/src/effects.rs @@ -0,0 +1,28 @@ +use crossbeam::atomic::AtomicCell; +use std::sync::Arc; + +#[derive(Default)] +pub struct AudioProcessor { + df: Option<::df::DFState>, +} + +impl AudioProcessor { + pub fn new_denoising() -> Self { + let df = ::df::DFState::default(); + AudioProcessor { df: Some(df) } + } +} + +impl AudioProcessor { + pub fn process(&mut self, audio: &[f32]) -> Box<[f32]> { + let mut output: Box<[f32]> = vec![0f32; audio.len()].into(); + if let Some(df) = &mut self.df { + df.process_frame(audio, &mut output); + } else { + output.copy_from_slice(audio); + } + output + } +} + +pub type AudioProcessorSender = Arc>>; diff --git a/gui/src/imp/desktop.rs b/gui/src/imp/desktop.rs index 4ae446e..ccb6bda 100644 --- a/gui/src/imp/desktop.rs +++ b/gui/src/imp/desktop.rs @@ -1,4 +1,5 @@ use crate::app::Command; +use crate::effects::AudioProcessor; use color_eyre::eyre::{eyre, Error}; use cpal::traits::{DeviceTrait, HostTrait}; use dioxus::hooks::{UnboundedReceiver, UnboundedSender}; @@ -13,7 +14,7 @@ use tokio::net::TcpStream; use tokio_rustls::rustls; use tokio_rustls::rustls::client::danger::{HandshakeSignatureValid, ServerCertVerifier}; use tokio_rustls::rustls::pki_types::{CertificateDer, ServerName, UnixTime}; -use tokio_rustls::rustls::ClientConfig; +use tokio_rustls::rustls::ClientConfig as RlsClientConfig; use tokio_rustls::rustls::DigitallySignedStruct; use tokio_rustls::TlsConnector; use tokio_util::compat::{TokioAsyncReadCompatExt as _, TokioAsyncWriteCompatExt as _}; @@ -50,6 +51,10 @@ impl AudioSystem { }) } + pub fn set_processor(&self, processor: AudioProcessor) { + // TODO + } + pub fn start_recording(&mut self, each: impl FnMut(Vec) + 'static) -> Result<(), Error> { // TODO Ok(()) @@ -192,7 +197,7 @@ pub async fn network_connect( ) -> Result<(), Error> { info!("connecting"); - let config = ClientConfig::builder() + let config = RlsClientConfig::builder() .dangerous() .with_custom_certificate_verifier(Arc::new(NoCertificateVerification)) .with_no_client_auth(); diff --git a/gui/src/imp/web.rs b/gui/src/imp/web.rs index 00f637b..2b71e2f 100644 --- a/gui/src/imp/web.rs +++ b/gui/src/imp/web.rs @@ -1,12 +1,11 @@ use crate::app::Command; +use crate::effects::{AudioProcessor, AudioProcessorSender}; use color_eyre::eyre::{bail, eyre, Error}; use dioxus::prelude::*; use futures::{AsyncRead, AsyncWrite}; -use futures_channel::mpsc::UnboundedSender; use gloo_timers::future::TimeoutFuture; -use mumble_protocol::control::{ClientControlCodec, ControlPacket}; -use mumble_protocol::voice::{VoicePacket, VoicePacketPayload}; -use mumble_protocol::Serverbound; +use js_sys::Float32Array; +use mumble_protocol::control::ClientControlCodec; use mumble_web2_common::ClientConfig; use reqwest::Url; use std::time::Duration; @@ -72,20 +71,33 @@ impl ResultExt for Result { self.map_err(|e| JsValue::from(e)).ey() } } -pub struct AudioSystem(AudioContext); + +pub struct AudioSystem { + webctx: AudioContext, + processors: AudioProcessorSender, +} impl AudioSystem { pub fn new() -> Result { // Create MediaStreams to playback decoded audio // The audio context is used to reproduce audio. - let audio_context = configure_audio_context(); - Ok(AudioSystem(audio_context)) + let webctx = configure_audio_context(); + let processor = AudioProcessorSender::default(); + Ok(AudioSystem { + webctx, + processors: processor, + }) + } + + pub fn set_processor(&self, processor: AudioProcessor) { + self.processors.store(Some(processor)) } pub fn start_recording(&mut self, each: impl FnMut(Vec) + 'static) -> Result<(), Error> { - let audio_context_worklet = self.0.clone(); + let audio_context_worklet = self.webctx.clone(); + let processors = self.processors.clone(); spawn(async move { - match run_encoder_worklet(&audio_context_worklet, each).await { + match run_encoder_worklet(&audio_context_worklet, each, processors).await { Ok(node) => info!("created encoder worklet: {:?}", &node), Err(err) => error!("could not create encoder worklet: {err}"), } @@ -94,8 +106,6 @@ impl AudioSystem { } pub fn create_player(&mut self) -> Result { - let audio_context = &self.0; - let audio_stream_generator = MediaStreamTrackGenerator::new(&MediaStreamTrackGeneratorInit::new("audio")).ey()?; @@ -105,12 +115,10 @@ impl AudioSystem { let media_stream = MediaStream::new_with_tracks(&js_tracks).ey()?; // Create MediaStreamAudioSourceNode - let audio_source = audio_context - .create_media_stream_source(&media_stream) - .ey()?; + let audio_source = self.webctx.create_media_stream_source(&media_stream).ey()?; // Connect output of audio_source to audio_context (browser audio) audio_source - .connect_with_audio_node(&audio_context.destination()) + .connect_with_audio_node(&self.webctx.destination()) .ey()?; // Create callback functions for AudioDecoder @@ -194,9 +202,22 @@ impl PromiseExt for Promise { } } +fn process_audio(frame: &JsValue, processor: &mut AudioProcessor) { + let Ok(samples) = Reflect::get(&frame, &"data".into()) else { + return; + }; + let Ok(samples) = samples.dyn_into::() else { + return; + }; + let input = samples.to_vec(); + let output = processor.process(&input); + samples.copy_from(&output); +} + async fn run_encoder_worklet( audio_context: &AudioContext, mut each: impl FnMut(Vec) + 'static, + processors: AudioProcessorSender, ) -> Result { let constraints = MediaStreamConstraints::new(); constraints.set_audio(&JsValue::TRUE); @@ -264,23 +285,18 @@ async fn run_encoder_worklet( audio_encoder.configure(&encoder_config); info!("created audio encoder"); - let download_buffer = std::cell::RefCell::new(Vec::new()); - + let mut current_processor = AudioProcessor::default(); let onmessage: Closure = Closure::new(move |event: MessageEvent| { - match AudioData::new(event.data().unchecked_ref()) { - Ok(data) => { - let x = web_sys::AudioDataCopyToOptions::new(0); - x.set_format(web_sys::AudioSampleFormat::F32); - let mut sub_buffer = vec![0; data.allocation_size(&x).unwrap() as usize]; - data.copy_to_with_u8_slice(&mut sub_buffer, &x); - download_buffer.borrow_mut().append(&mut sub_buffer); - if download_buffer.borrow().len() > 48000 * 10 * 4 { - //pub fn download_data(data: Vec, filename: &str) -> Result<(), JsValue> { - //download_data(download_buffer.borrow().to_vec(), "download_buffer.pcm32"); - download_buffer.borrow_mut().clear(); - } + if let Some(new_processor) = processors.take() { + current_processor = new_processor; + } - audio_encoder.encode(&data); + let frame = event.data(); + process_audio(&frame, &mut current_processor); + + match AudioData::new(frame.unchecked_ref()) { + Ok(data) => { + let _ = audio_encoder.encode(&data); } Err(err) => { error!( diff --git a/gui/src/lib.rs b/gui/src/lib.rs index 8da2c49..83de958 100644 --- a/gui/src/lib.rs +++ b/gui/src/lib.rs @@ -29,7 +29,11 @@ use tracing::debug; use tracing::error; use tracing::info; +use crate::effects::AudioProcessor; +use crate::imp::AudioSystem; + pub mod app; +mod effects; pub mod imp; mod msghtml; @@ -161,7 +165,7 @@ pub async fn network_loop( match command { Some(Command::Disconnect) => break, Some(command) => { - let res = accept_command(command, &mut send_chan); + let res = accept_command(command, &mut send_chan, &mut audio); if let Err(err) = res { info!("error accepting command {:?}", err) } @@ -179,6 +183,7 @@ pub async fn network_loop( fn accept_command( command: Command, send_chan: &mut UnboundedSender>, + audio: &mut AudioSystem, ) -> Result<(), Error> { use Command::*; let Some(session) = STATE.server.read().session else { @@ -280,6 +285,13 @@ fn accept_command( let _ = send_chan.unbounded_send(u.into()); } Connect { .. } | Disconnect => (), + UpdateMicEffects { denoise } => { + if denoise { + audio.set_processor(AudioProcessor::new_denoising()); + } else { + audio.set_processor(AudioProcessor::default()); + } + } } Ok(())