diff --git a/Cargo.lock b/Cargo.lock index 53fdd55..fdd6f6e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -79,6 +79,28 @@ dependencies = [ "memchr", ] +[[package]] +name = "alsa" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed7572b7ba83a31e20d1b48970ee402d2e3e0537dcfe0a3ff4d6eb7508617d43" +dependencies = [ + "alsa-sys", + "bitflags 2.6.0", + "cfg-if", + "libc", +] + +[[package]] +name = "alsa-sys" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db8fee663d06c4e303404ef5f40488a53e062f89ba8bfed81f42325aafad1527" +dependencies = [ + "libc", + "pkg-config", +] + [[package]] name = "anyhow" version = "1.0.93" @@ -269,6 +291,17 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" +[[package]] +name = "audiopus_sys" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62314a1546a2064e033665d658e88c620a62904be945f8147e6b16c3db9f8651" +dependencies = [ + "cmake", + "log", + "pkg-config", +] + [[package]] name = "autocfg" version = "1.4.0" @@ -293,7 +326,7 @@ version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df7a4168111d7eb622a31b214057b8509c0a7e1794f44c546d742330dc793972" dependencies = [ - "bindgen", + "bindgen 0.69.5", "cc", "cmake", "dunce", @@ -352,6 +385,24 @@ dependencies = [ "which", ] +[[package]] +name = "bindgen" +version = "0.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f49d8fed880d473ea71efb9bf597651e77201bdd4893efe54c9e5d65ae04ce6f" +dependencies = [ + "bitflags 2.6.0", + "cexpr", + "clang-sys", + "itertools", + "proc-macro2", + "quote", + "regex", + "rustc-hash 1.1.0", + "shlex", + "syn 2.0.87", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -805,6 +856,49 @@ dependencies = [ "libc", ] +[[package]] +name = "coreaudio-rs" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "321077172d79c662f64f5071a03120748d5bb652f5231570141be24cfcd2bace" +dependencies = [ + "bitflags 1.3.2", + "core-foundation-sys", + "coreaudio-sys", +] + +[[package]] +name = "coreaudio-sys" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ce857aa0b77d77287acc1ac3e37a05a8c95a2af3647d23b15f263bdaeb7562b" +dependencies = [ + "bindgen 0.70.1", +] + +[[package]] +name = "cpal" +version = "0.15.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "873dab07c8f743075e57f524c583985fbaf745602acbe916a01539364369a779" +dependencies = [ + "alsa", + "core-foundation-sys", + "coreaudio-rs", + "dasp_sample", + "jni", + "js-sys", + "libc", + "mach2", + "ndk 0.8.0", + "ndk-context", + "oboe", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "windows 0.54.0", +] + [[package]] name = "cpufeatures" version = "0.2.15" @@ -832,6 +926,15 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-queue" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df0346b5d5e76ac2fe4e327c5fd1118d6be7c51dfb18f9b7922923f287471e35" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.20" @@ -938,6 +1041,18 @@ dependencies = [ "parking_lot_core", ] +[[package]] +name = "dasp_ring_buffer" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07d79e19b89618a543c4adec9c5a347fe378a19041699b3278e616e387511ea1" + +[[package]] +name = "dasp_sample" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c87e182de0887fd5361989c677c4e8f5000cd9491d6d563161a8f3a5519fc7f" + [[package]] name = "data-encoding" version = "2.6.0" @@ -2972,6 +3087,15 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" +[[package]] +name = "mach2" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19b955cdeb2a02b9117f121ce63aa52d08ade45de53e48fe6a38b39c10f6f709" +dependencies = [ + "libc", +] + [[package]] name = "malloc_buf" version = "0.0.6" @@ -3245,6 +3369,9 @@ dependencies = [ "asynchronous-codec", "byteorder", "color-eyre", + "cpal", + "crossbeam-queue", + "dasp_ring_buffer", "dioxus", "dioxus-desktop", "dioxus-web", @@ -3259,6 +3386,7 @@ dependencies = [ "mumble-web2-common", "ogg", "once_cell", + "opus", "ordermap", "serde", "serde-wasm-bindgen 0.6.5", @@ -3311,6 +3439,20 @@ dependencies = [ "tempfile", ] +[[package]] +name = "ndk" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2076a31b7010b17a38c01907c45b945e8f11495ee4dd588309718901b1f7a5b7" +dependencies = [ + "bitflags 2.6.0", + "jni-sys", + "log", + "ndk-sys 0.5.0+25.2.9519653", + "num_enum", + "thiserror 1.0.69", +] + [[package]] name = "ndk" version = "0.9.0" @@ -3320,7 +3462,7 @@ dependencies = [ "bitflags 2.6.0", "jni-sys", "log", - "ndk-sys", + "ndk-sys 0.6.0+11769913", "num_enum", "raw-window-handle 0.6.2", "thiserror 1.0.69", @@ -3332,6 +3474,15 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" +[[package]] +name = "ndk-sys" +version = "0.5.0+25.2.9519653" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c196769dd60fd4f363e11d948139556a344e79d451aeb2fa2fd040738ef7691" +dependencies = [ + "jni-sys", +] + [[package]] name = "ndk-sys" version = "0.6.0+11769913" @@ -3413,6 +3564,17 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" +[[package]] +name = "num-derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + [[package]] name = "num-integer" version = "0.1.46" @@ -3610,6 +3772,29 @@ dependencies = [ "memchr", ] +[[package]] +name = "oboe" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8b61bebd49e5d43f5f8cc7ee2891c16e0f41ec7954d36bcb6c14c5e0de867fb" +dependencies = [ + "jni", + "ndk 0.8.0", + "ndk-context", + "num-derive", + "num-traits", + "oboe-sys", +] + +[[package]] +name = "oboe-sys" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c8bb09a4a2b1d668170cfe0a7d5bc103f8999fb316c98099b6a9939c9f2e79d" +dependencies = [ + "cc", +] + [[package]] name = "ogg" version = "0.9.1" @@ -3681,6 +3866,16 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" +[[package]] +name = "opus" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6526409b274a7e98e55ff59d96aafd38e6cd34d46b7dbbc32ce126dffcd75e8e" +dependencies = [ + "audiopus_sys", + "libc", +] + [[package]] name = "ordered-stream" version = "0.2.0" @@ -5362,9 +5557,9 @@ dependencies = [ "lazy_static", "libc", "log", - "ndk", + "ndk 0.9.0", "ndk-context", - "ndk-sys", + "ndk-sys 0.6.0+11769913", "objc", "once_cell", "parking_lot", @@ -5374,8 +5569,8 @@ dependencies = [ "tao-macros", "unicode-segmentation", "url", - "windows", - "windows-core", + "windows 0.58.0", + "windows-core 0.58.0", "windows-version", "x11-dl", ] @@ -6193,8 +6388,8 @@ checksum = "6f61ff3d9d0ee4efcb461b14eb3acfda2702d10dc329f339303fc3e57215ae2c" dependencies = [ "webview2-com-macros", "webview2-com-sys", - "windows", - "windows-core", + "windows 0.58.0", + "windows-core 0.58.0", "windows-implement", "windows-interface", ] @@ -6217,8 +6412,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a3a3e2eeb58f82361c93f9777014668eb3d07e7d174ee4c819575a9208011886" dependencies = [ "thiserror 1.0.69", - "windows", - "windows-core", + "windows 0.58.0", + "windows-core 0.58.0", ] [[package]] @@ -6264,13 +6459,33 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.54.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9252e5725dbed82865af151df558e754e4a3c2c30818359eb17465f1346a1b49" +dependencies = [ + "windows-core 0.54.0", + "windows-targets 0.52.6", +] + [[package]] name = "windows" version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd04d41d93c4992d421894c18c8b43496aa748dd4c081bac0dc93eb0489272b6" dependencies = [ - "windows-core", + "windows-core 0.58.0", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-core" +version = "0.54.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12661b9c89351d684a50a8a643ce5f608e20243b9fb84687800163429f161d65" +dependencies = [ + "windows-result 0.1.2", "windows-targets 0.52.6", ] @@ -6282,7 +6497,7 @@ checksum = "6ba6d44ec8c2591c134257ce647b7ea6b20335bf6379a27dac5f1641fcf59f99" dependencies = [ "windows-implement", "windows-interface", - "windows-result", + "windows-result 0.2.0", "windows-strings", "windows-targets 0.52.6", ] @@ -6315,11 +6530,20 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e400001bb720a623c1c69032f8e3e4cf09984deec740f007dd2b03ec864804b0" dependencies = [ - "windows-result", + "windows-result 0.2.0", "windows-strings", "windows-targets 0.52.6", ] +[[package]] +name = "windows-result" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8" +dependencies = [ + "windows-targets 0.52.6", +] + [[package]] name = "windows-result" version = "0.2.0" @@ -6335,7 +6559,7 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10" dependencies = [ - "windows-result", + "windows-result 0.2.0", "windows-targets 0.52.6", ] @@ -6613,7 +6837,7 @@ dependencies = [ "jni", "kuchikiki", "libc", - "ndk", + "ndk 0.9.0", "objc", "objc_id", "once_cell", @@ -6626,8 +6850,8 @@ dependencies = [ "webkit2gtk", "webkit2gtk-sys", "webview2-com", - "windows", - "windows-core", + "windows 0.58.0", + "windows-core 0.58.0", "windows-version", "x11-dl", ] diff --git a/gui/Cargo.toml b/gui/Cargo.toml index 55f118c..4992bae 100644 --- a/gui/Cargo.toml +++ b/gui/Cargo.toml @@ -62,7 +62,9 @@ tracing-web = { version = "0.1.3", optional = true } dioxus-desktop = { version = "0.6.0-alpha.4", optional = true} tokio = { version = "1.41.1", features = ["net", "rt"], optional = true } tokio-rustls = { version = "0.26.0", optional = true } - +opus = { version = "0.3.0", optional = true } +cpal = { version = "0.15.3", optional = true } +dasp_ring_buffer = { version = "0.11.0", optional = true } # Base Dependencies # ================ @@ -86,6 +88,7 @@ serde = { workspace = true } tracing-subscriber = { version = "0.3.18", features = ["ansi"] } tracing = "0.1.40" color-eyre = "0.6.3" +crossbeam-queue = "0.3.11" [features] web = [ @@ -100,4 +103,4 @@ web = [ "gloo-timers", "tracing-web", ] -desktop = ["dioxus/desktop", "tokio", "tokio-rustls", "tracing-subscriber/env-filter"] +desktop = ["dioxus/desktop", "tokio", "tokio-rustls", "tracing-subscriber/env-filter", "opus", "cpal", "dasp_ring_buffer"] diff --git a/gui/src/imp/desktop.rs b/gui/src/imp/desktop.rs index 8d7a1c2..9e4df94 100644 --- a/gui/src/imp/desktop.rs +++ b/gui/src/imp/desktop.rs @@ -1,11 +1,15 @@ use crate::app::Command; -use color_eyre::eyre::Error; +use color_eyre::eyre::{eyre, Error}; +use cpal::traits::{DeviceTrait, HostTrait}; +use crossbeam_queue::ArrayQueue; use dioxus::hooks::{UnboundedReceiver, UnboundedSender}; use futures::io::{AsyncRead, AsyncWrite}; use mumble_protocol::control::{ClientControlCodec, ControlPacket}; use mumble_protocol::Serverbound; use mumble_web2_common::GuiConfig; +use std::collections::VecDeque; use std::net::ToSocketAddrs; +use std::sync::Mutex; use std::{fmt, io, sync::Arc}; use tokio::net::TcpStream; use tokio_rustls::rustls; @@ -15,6 +19,7 @@ use tokio_rustls::rustls::ClientConfig; use tokio_rustls::rustls::DigitallySignedStruct; use tokio_rustls::TlsConnector; use tokio_util::compat::{TokioAsyncReadCompatExt as _, TokioAsyncWriteCompatExt as _}; +use tracing::{error, warn}; pub use tokio::task::spawn; pub use tokio::time::sleep; @@ -25,25 +30,92 @@ impl ImpRead for T {} pub trait ImpWrite: AsyncWrite + Unpin + Send + 'static {} impl ImpWrite for T {} -pub struct AudioSystem(); +pub struct AudioSystem { + output: cpal::Device, + input: cpal::Device, +} + +const BUF_LEN: usize = 480; // 20 ms impl AudioSystem { - pub fn new(sender: UnboundedSender>) -> Result { + pub fn new() -> Result { // TODO - Ok(AudioSystem()) + let host = cpal::default_host(); + let name = host.id(); + Ok(AudioSystem { + output: host + .default_output_device() + .ok_or(eyre!("no output devices from {name:?}"))?, + input: host + .default_input_device() + .ok_or(eyre!("no input devices from {name:?}"))?, + }) + } + + pub fn start_recording(&mut self, each: impl FnMut(Vec) + 'static) -> Result<(), Error> { + // TODO + Ok(()) } pub fn create_player(&mut self) -> Result { - // TODO - Ok(AudioPlayer()) + let queue = Arc::new(ArrayQueue::<[i16; BUF_LEN]>::new(10)); + let decoder = opus::Decoder::new(48_000, opus::Channels::Mono)?; + let stream = { + let queue = queue.clone(); + self.output.build_output_stream( + &cpal::StreamConfig { + channels: 1, + sample_rate: cpal::SampleRate(48_000), + buffer_size: cpal::BufferSize::Fixed(BUF_LEN as u32), // 20ms playback delay + }, + move |out, info| match queue.pop() { + Some(buf) => out.copy_from_slice(&buf[..]), + None => out.fill(0), + }, + move |err| error!("could not create output stream {err:?}"), + None, + )? + }; + Ok(AudioPlayer { + decoder, + stream, + queue, + tmp: vec![0; 2400], + pos: 0, + }) } } -pub struct AudioPlayer(); +pub struct AudioPlayer { + decoder: opus::Decoder, + stream: cpal::Stream, + queue: Arc>, + tmp: Vec, + pos: usize, +} impl AudioPlayer { pub fn play_opus(&mut self, payload: &[u8]) { - // TODO + match self + .decoder + .decode(payload, &mut self.tmp[self.pos..], false) + { + Ok(l) => { + self.pos += l; + } + Err(e) => { + error!("opus decode error {e:?}"); + } + }; + while self.pos >= BUF_LEN { + let mut chunk = [0; BUF_LEN]; + chunk.copy_from_slice(&self.tmp[..BUF_LEN]); + dbg!(&chunk); + let _ = self.queue.push(chunk); + let i = std::cell::Cell::new(0usize); + self.tmp.retain(|_| i.replace(i.get() + 1) >= BUF_LEN); + self.pos -= BUF_LEN; + } } } diff --git a/gui/src/imp/web.rs b/gui/src/imp/web.rs index 4c7d08f..b70e7e9 100644 --- a/gui/src/imp/web.rs +++ b/gui/src/imp/web.rs @@ -74,20 +74,22 @@ impl ResultExt for Result { pub struct AudioSystem(AudioContext); impl AudioSystem { - pub fn new(sender: UnboundedSender>) -> Result { + pub fn new() -> Result { // Create MediaStreams to playback decoded audio // The audio context is used to reproduce audio. let audio_context = configure_audio_context(); + Ok(AudioSystem(audio_context)) + } - let audio_context_worklet = audio_context.clone(); + pub fn start_recording(&mut self, each: impl FnMut(Vec) + 'static) -> Result<(), Error> { + let audio_context_worklet = self.0.clone(); spawn(async move { - match create_encoder_worklet(&audio_context_worklet, sender).await { + match run_encoder_worklet(&audio_context_worklet, each).await { Ok(node) => info!("created encoder worklet: {:?}", &node), Err(err) => error!("could not create encoder worklet: {err}"), } }); - - Ok(AudioSystem(audio_context)) + Ok(()) } pub fn create_player(&mut self) -> Result { @@ -191,9 +193,9 @@ impl PromiseExt for Promise { } } -async fn create_encoder_worklet( +async fn run_encoder_worklet( audio_context: &AudioContext, - packets: UnboundedSender>, + mut each: impl FnMut(Vec) + 'static, ) -> Result { let stream = window() .unwrap() @@ -234,35 +236,12 @@ async fn create_encoder_worklet( let encoder_error: Closure = Closure::new(|e| error!("error encoding audio {:?}", e)); - let download_buffer = std::cell::RefCell::new(Vec::new()); - // This knows what MediaStreamTrackGenerator to use as it closes around it - let mut sequence_num = 0; let output: Closure = Closure::new(move |audio_data: EncodedAudioChunk| { let mut array = vec![0u8; audio_data.byte_length() as usize]; audio_data.copy_to_with_u8_slice(&mut array); - - download_buffer.borrow_mut().push(array.clone()); - if download_buffer.borrow().len() > 200 { - //download_data(download_buffer.borrow().to_vec(), "download_buffer.opus"); - //download_data( - // ass::encode(download_buffer.borrow().to_vec(), 960, 0), - // "download_buffer.opus", - //); - download_buffer.borrow_mut().clear(); - } - - let _ = - packets.unbounded_send(ControlPacket::UDPTunnel(Box::new(VoicePacket::Audio { - _dst: std::marker::PhantomData, - target: 0, - session_id: (), - seq_num: sequence_num, - payload: VoicePacketPayload::Opus(array.into(), false), - position_info: None, - }))); - sequence_num = sequence_num.wrapping_add(2); + each(array); }); let audio_encoder = AudioEncoder::new(&AudioEncoderInit::new( diff --git a/gui/src/lib.rs b/gui/src/lib.rs index 1798681..89fe3aa 100644 --- a/gui/src/lib.rs +++ b/gui/src/lib.rs @@ -15,6 +15,7 @@ pub use imp::spawn; use mumble_protocol::control::msgs; use mumble_protocol::control::ControlCodec; use mumble_protocol::control::ControlPacket; +use mumble_protocol::voice::VoicePacket; use mumble_protocol::voice::VoicePacketPayload; use mumble_protocol::Clientbound; use mumble_protocol::Serverbound; @@ -103,7 +104,23 @@ pub async fn network_loop( }); } - let mut audio = imp::AudioSystem::new(send_chan.clone())?; + let mut audio = imp::AudioSystem::new()?; + { + let send_chan = send_chan.clone(); + let mut sequence_num = 0; + audio.start_recording(move |opus_frame| { + let _ = + send_chan.unbounded_send(ControlPacket::UDPTunnel(Box::new(VoicePacket::Audio { + _dst: std::marker::PhantomData, + target: 0, + session_id: (), + seq_num: sequence_num, + payload: VoicePacketPayload::Opus(opus_frame.into(), false), + position_info: None, + }))); + sequence_num = sequence_num.wrapping_add(2); + }); + } // Create map of session_id -> AudioDecoder let mut decoder_map = HashMap::new();