mumble/mumble-web2
6
0
Fork 0
Code Issues Pull Requests 5 Actions Packages Projects Releases Wiki Activity

attempt desktop audio playback

Browse Source
This commit is contained in:
Sam Sartor
2024-11-12 20:03:58 -07:00
parent b65ec274d8
commit b2ee911c66
5 changed files with 354 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+241 -17
View File
@@ -79,6 +79,28 @@ dependencies = [
"memchr",
]
[[package]]
name = "alsa"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed7572b7ba83a31e20d1b48970ee402d2e3e0537dcfe0a3ff4d6eb7508617d43"
dependencies = [
"alsa-sys",
"bitflags 2.6.0",
"cfg-if",
"libc",
]
[[package]]
name = "alsa-sys"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "db8fee663d06c4e303404ef5f40488a53e062f89ba8bfed81f42325aafad1527"
dependencies = [
"libc",
"pkg-config",
]
[[package]]
name = "anyhow"
version = "1.0.93"
@@ -269,6 +291,17 @@ version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0"
[[package]]
name = "audiopus_sys"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "62314a1546a2064e033665d658e88c620a62904be945f8147e6b16c3db9f8651"
dependencies = [
"cmake",
"log",
"pkg-config",
]
[[package]]
name = "autocfg"
version = "1.4.0"
@@ -293,7 +326,7 @@ version = "0.22.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df7a4168111d7eb622a31b214057b8509c0a7e1794f44c546d742330dc793972"
dependencies = [
"bindgen",
"bindgen 0.69.5",
"cc",
"cmake",
"dunce",
@@ -352,6 +385,24 @@ dependencies = [
"which",
]
[[package]]
name = "bindgen"
version = "0.70.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f49d8fed880d473ea71efb9bf597651e77201bdd4893efe54c9e5d65ae04ce6f"
dependencies = [
"bitflags 2.6.0",
"cexpr",
"clang-sys",
"itertools",
"proc-macro2",
"quote",
"regex",
"rustc-hash 1.1.0",
"shlex",
"syn 2.0.87",
]
[[package]]
name = "bitflags"
version = "1.3.2"
@@ -805,6 +856,49 @@ dependencies = [
"libc",
]
[[package]]
name = "coreaudio-rs"
version = "0.11.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "321077172d79c662f64f5071a03120748d5bb652f5231570141be24cfcd2bace"
dependencies = [
"bitflags 1.3.2",
"core-foundation-sys",
"coreaudio-sys",
]
[[package]]
name = "coreaudio-sys"
version = "0.2.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2ce857aa0b77d77287acc1ac3e37a05a8c95a2af3647d23b15f263bdaeb7562b"
dependencies = [
"bindgen 0.70.1",
]
[[package]]
name = "cpal"
version = "0.15.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "873dab07c8f743075e57f524c583985fbaf745602acbe916a01539364369a779"
dependencies = [
"alsa",
"core-foundation-sys",
"coreaudio-rs",
"dasp_sample",
"jni",
"js-sys",
"libc",
"mach2",
"ndk 0.8.0",
"ndk-context",
"oboe",
"wasm-bindgen",
"wasm-bindgen-futures",
"web-sys",
"windows 0.54.0",
]
[[package]]
name = "cpufeatures"
version = "0.2.15"
@@ -832,6 +926,15 @@ dependencies = [
"crossbeam-utils",
]
[[package]]
name = "crossbeam-queue"
version = "0.3.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df0346b5d5e76ac2fe4e327c5fd1118d6be7c51dfb18f9b7922923f287471e35"
dependencies = [
"crossbeam-utils",
]
[[package]]
name = "crossbeam-utils"
version = "0.8.20"
@@ -938,6 +1041,18 @@ dependencies = [
"parking_lot_core",
]
[[package]]
name = "dasp_ring_buffer"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "07d79e19b89618a543c4adec9c5a347fe378a19041699b3278e616e387511ea1"
[[package]]
name = "dasp_sample"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c87e182de0887fd5361989c677c4e8f5000cd9491d6d563161a8f3a5519fc7f"
[[package]]
name = "data-encoding"
version = "2.6.0"
@@ -2972,6 +3087,15 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
[[package]]
name = "mach2"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "19b955cdeb2a02b9117f121ce63aa52d08ade45de53e48fe6a38b39c10f6f709"
dependencies = [
"libc",
]
[[package]]
name = "malloc_buf"
version = "0.0.6"
@@ -3245,6 +3369,9 @@ dependencies = [
"asynchronous-codec",
"byteorder",
"color-eyre",
"cpal",
"crossbeam-queue",
"dasp_ring_buffer",
"dioxus",
"dioxus-desktop",
"dioxus-web",
@@ -3259,6 +3386,7 @@ dependencies = [
"mumble-web2-common",
"ogg",
"once_cell",
"opus",
"ordermap",
"serde",
"serde-wasm-bindgen 0.6.5",
@@ -3311,6 +3439,20 @@ dependencies = [
"tempfile",
]
[[package]]
name = "ndk"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2076a31b7010b17a38c01907c45b945e8f11495ee4dd588309718901b1f7a5b7"
dependencies = [
"bitflags 2.6.0",
"jni-sys",
"log",
"ndk-sys 0.5.0+25.2.9519653",
"num_enum",
"thiserror 1.0.69",
]
[[package]]
name = "ndk"
version = "0.9.0"
@@ -3320,7 +3462,7 @@ dependencies = [
"bitflags 2.6.0",
"jni-sys",
"log",
"ndk-sys",
"ndk-sys 0.6.0+11769913",
"num_enum",
"raw-window-handle 0.6.2",
"thiserror 1.0.69",
@@ -3332,6 +3474,15 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b"
[[package]]
name = "ndk-sys"
version = "0.5.0+25.2.9519653"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8c196769dd60fd4f363e11d948139556a344e79d451aeb2fa2fd040738ef7691"
dependencies = [
"jni-sys",
]
[[package]]
name = "ndk-sys"
version = "0.6.0+11769913"
@@ -3413,6 +3564,17 @@ version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9"
[[package]]
name = "num-derive"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.87",
]
[[package]]
name = "num-integer"
version = "0.1.46"
@@ -3610,6 +3772,29 @@ dependencies = [
"memchr",
]
[[package]]
name = "oboe"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e8b61bebd49e5d43f5f8cc7ee2891c16e0f41ec7954d36bcb6c14c5e0de867fb"
dependencies = [
"jni",
"ndk 0.8.0",
"ndk-context",
"num-derive",
"num-traits",
"oboe-sys",
]
[[package]]
name = "oboe-sys"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6c8bb09a4a2b1d668170cfe0a7d5bc103f8999fb316c98099b6a9939c9f2e79d"
dependencies = [
"cc",
]
[[package]]
name = "ogg"
version = "0.9.1"
@@ -3681,6 +3866,16 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d"
[[package]]
name = "opus"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6526409b274a7e98e55ff59d96aafd38e6cd34d46b7dbbc32ce126dffcd75e8e"
dependencies = [
"audiopus_sys",
"libc",
]
[[package]]
name = "ordered-stream"
version = "0.2.0"
@@ -5362,9 +5557,9 @@ dependencies = [
"lazy_static",
"libc",
"log",
"ndk",
"ndk 0.9.0",
"ndk-context",
"ndk-sys",
"ndk-sys 0.6.0+11769913",
"objc",
"once_cell",
"parking_lot",
@@ -5374,8 +5569,8 @@ dependencies = [
"tao-macros",
"unicode-segmentation",
"url",
"windows",
"windows-core",
"windows 0.58.0",
"windows-core 0.58.0",
"windows-version",
"x11-dl",
]
@@ -6193,8 +6388,8 @@ checksum = "6f61ff3d9d0ee4efcb461b14eb3acfda2702d10dc329f339303fc3e57215ae2c"
dependencies = [
"webview2-com-macros",
"webview2-com-sys",
"windows",
"windows-core",
"windows 0.58.0",
"windows-core 0.58.0",
"windows-implement",
"windows-interface",
]
@@ -6217,8 +6412,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a3a3e2eeb58f82361c93f9777014668eb3d07e7d174ee4c819575a9208011886"
dependencies = [
"thiserror 1.0.69",
"windows",
"windows-core",
"windows 0.58.0",
"windows-core 0.58.0",
]
[[package]]
@@ -6264,13 +6459,33 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
name = "windows"
version = "0.54.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9252e5725dbed82865af151df558e754e4a3c2c30818359eb17465f1346a1b49"
dependencies = [
"windows-core 0.54.0",
"windows-targets 0.52.6",
]
[[package]]
name = "windows"
version = "0.58.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dd04d41d93c4992d421894c18c8b43496aa748dd4c081bac0dc93eb0489272b6"
dependencies = [
"windows-core",
"windows-core 0.58.0",
"windows-targets 0.52.6",
]
[[package]]
name = "windows-core"
version = "0.54.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "12661b9c89351d684a50a8a643ce5f608e20243b9fb84687800163429f161d65"
dependencies = [
"windows-result 0.1.2",
"windows-targets 0.52.6",
]
@@ -6282,7 +6497,7 @@ checksum = "6ba6d44ec8c2591c134257ce647b7ea6b20335bf6379a27dac5f1641fcf59f99"
dependencies = [
"windows-implement",
"windows-interface",
"windows-result",
"windows-result 0.2.0",
"windows-strings",
"windows-targets 0.52.6",
]
@@ -6315,11 +6530,20 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e400001bb720a623c1c69032f8e3e4cf09984deec740f007dd2b03ec864804b0"
dependencies = [
"windows-result",
"windows-result 0.2.0",
"windows-strings",
"windows-targets 0.52.6",
]
[[package]]
name = "windows-result"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8"
dependencies = [
"windows-targets 0.52.6",
]
[[package]]
name = "windows-result"
version = "0.2.0"
@@ -6335,7 +6559,7 @@ version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10"
dependencies = [
"windows-result",
"windows-result 0.2.0",
"windows-targets 0.52.6",
]
@@ -6613,7 +6837,7 @@ dependencies = [
"jni",
"kuchikiki",
"libc",
"ndk",
"ndk 0.9.0",
"objc",
"objc_id",
"once_cell",
@@ -6626,8 +6850,8 @@ dependencies = [
"webkit2gtk",
"webkit2gtk-sys",
"webview2-com",
"windows",
"windows-core",
"windows 0.58.0",
"windows-core 0.58.0",
"windows-version",
"x11-dl",
]
gui/Cargo.toml
+5 -2
View File
@@ -62,7 +62,9 @@ tracing-web = { version = "0.1.3", optional = true }
dioxus-desktop = { version = "0.6.0-alpha.4", optional = true}
tokio = { version = "1.41.1", features = ["net", "rt"], optional = true }
tokio-rustls = { version = "0.26.0", optional = true }
opus = { version = "0.3.0", optional = true }
cpal = { version = "0.15.3", optional = true }
dasp_ring_buffer = { version = "0.11.0", optional = true }
# Base Dependencies
# ================
@@ -86,6 +88,7 @@ serde = { workspace = true }
tracing-subscriber = { version = "0.3.18", features = ["ansi"] }
tracing = "0.1.40"
color-eyre = "0.6.3"
crossbeam-queue = "0.3.11"
[features]
web = [
@@ -100,4 +103,4 @@ web = [
"gloo-timers",
"tracing-web",
]
desktop = ["dioxus/desktop", "tokio", "tokio-rustls", "tracing-subscriber/env-filter"]
desktop = ["dioxus/desktop", "tokio", "tokio-rustls", "tracing-subscriber/env-filter", "opus", "cpal", "dasp_ring_buffer"]
gui/src/imp/desktop.rs
+80 -8
View File
@@ -1,11 +1,15 @@
use crate::app::Command;
use color_eyre::eyre::Error;
use color_eyre::eyre::{eyre, Error};
use cpal::traits::{DeviceTrait, HostTrait};
use crossbeam_queue::ArrayQueue;
use dioxus::hooks::{UnboundedReceiver, UnboundedSender};
use futures::io::{AsyncRead, AsyncWrite};
use mumble_protocol::control::{ClientControlCodec, ControlPacket};
use mumble_protocol::Serverbound;
use mumble_web2_common::GuiConfig;
use std::collections::VecDeque;
use std::net::ToSocketAddrs;
use std::sync::Mutex;
use std::{fmt, io, sync::Arc};
use tokio::net::TcpStream;
use tokio_rustls::rustls;
@@ -15,6 +19,7 @@ use tokio_rustls::rustls::ClientConfig;
use tokio_rustls::rustls::DigitallySignedStruct;
use tokio_rustls::TlsConnector;
use tokio_util::compat::{TokioAsyncReadCompatExt as _, TokioAsyncWriteCompatExt as _};
use tracing::{error, warn};
pub use tokio::task::spawn;
pub use tokio::time::sleep;
@@ -25,25 +30,92 @@ impl<T: AsyncRead + Unpin + Send + 'static> ImpRead for T {}
pub trait ImpWrite: AsyncWrite + Unpin + Send + 'static {}
impl<T: AsyncWrite + Unpin + Send + 'static> ImpWrite for T {}
pub struct AudioSystem();
pub struct AudioSystem {
output: cpal::Device,
input: cpal::Device,
}
const BUF_LEN: usize = 480; // 20 ms
impl AudioSystem {
pub fn new(sender: UnboundedSender<ControlPacket<Serverbound>>) -> Result<Self, Error> {
pub fn new() -> Result<Self, Error> {
// TODO
Ok(AudioSystem())
let host = cpal::default_host();
let name = host.id();
Ok(AudioSystem {
output: host
.default_output_device()
.ok_or(eyre!("no output devices from {name:?}"))?,
input: host
.default_input_device()
.ok_or(eyre!("no input devices from {name:?}"))?,
})
}
pub fn start_recording(&mut self, each: impl FnMut(Vec<u8>) + 'static) -> Result<(), Error> {
// TODO
Ok(())
}
pub fn create_player(&mut self) -> Result<AudioPlayer, Error> {
// TODO
Ok(AudioPlayer())
let queue = Arc::new(ArrayQueue::<[i16; BUF_LEN]>::new(10));
let decoder = opus::Decoder::new(48_000, opus::Channels::Mono)?;
let stream = {
let queue = queue.clone();
self.output.build_output_stream(
&cpal::StreamConfig {
channels: 1,
sample_rate: cpal::SampleRate(48_000),
buffer_size: cpal::BufferSize::Fixed(BUF_LEN as u32), // 20ms playback delay
},
move |out, info| match queue.pop() {
Some(buf) => out.copy_from_slice(&buf[..]),
None => out.fill(0),
},
move |err| error!("could not create output stream {err:?}"),
None,
)?
};
Ok(AudioPlayer {
decoder,
stream,
queue,
tmp: vec![0; 2400],
pos: 0,
})
}
}
pub struct AudioPlayer();
pub struct AudioPlayer {
decoder: opus::Decoder,
stream: cpal::Stream,
queue: Arc<ArrayQueue<[i16; BUF_LEN]>>,
tmp: Vec<i16>,
pos: usize,
}
impl AudioPlayer {
pub fn play_opus(&mut self, payload: &[u8]) {
// TODO
match self
.decoder
.decode(payload, &mut self.tmp[self.pos..], false)
{
Ok(l) => {
self.pos += l;
}
Err(e) => {
error!("opus decode error {e:?}");
}
};
while self.pos >= BUF_LEN {
let mut chunk = [0; BUF_LEN];
chunk.copy_from_slice(&self.tmp[..BUF_LEN]);
dbg!(&chunk);
let _ = self.queue.push(chunk);
let i = std::cell::Cell::new(0usize);
self.tmp.retain(|_| i.replace(i.get() + 1) >= BUF_LEN);
self.pos -= BUF_LEN;
}
}
}
gui/src/imp/web.rs
+10 -31
View File
@@ -74,20 +74,22 @@ impl<T> ResultExt<T> for Result<T, JsError> {
pub struct AudioSystem(AudioContext);
impl AudioSystem {
pub fn new(sender: UnboundedSender<ControlPacket<Serverbound>>) -> Result<Self, Error> {
pub fn new() -> Result<Self, Error> {
// Create MediaStreams to playback decoded audio
// The audio context is used to reproduce audio.
let audio_context = configure_audio_context();
Ok(AudioSystem(audio_context))
}
let audio_context_worklet = audio_context.clone();
pub fn start_recording(&mut self, each: impl FnMut(Vec<u8>) + 'static) -> Result<(), Error> {
let audio_context_worklet = self.0.clone();
spawn(async move {
match create_encoder_worklet(&audio_context_worklet, sender).await {
match run_encoder_worklet(&audio_context_worklet, each).await {
Ok(node) => info!("created encoder worklet: {:?}", &node),
Err(err) => error!("could not create encoder worklet: {err}"),
}
});
Ok(AudioSystem(audio_context))
Ok(())
}
pub fn create_player(&mut self) -> Result<AudioPlayer, Error> {
@@ -191,9 +193,9 @@ impl PromiseExt for Promise {
}
}
async fn create_encoder_worklet(
async fn run_encoder_worklet(
audio_context: &AudioContext,
packets: UnboundedSender<ControlPacket<mumble_protocol::Serverbound>>,
mut each: impl FnMut(Vec<u8>) + 'static,
) -> Result<AudioWorkletNode, Error> {
let stream = window()
.unwrap()
@@ -234,35 +236,12 @@ async fn create_encoder_worklet(
let encoder_error: Closure<dyn FnMut(JsValue)> =
Closure::new(|e| error!("error encoding audio {:?}", e));
let download_buffer = std::cell::RefCell::new(Vec::new());
// This knows what MediaStreamTrackGenerator to use as it closes around it
let mut sequence_num = 0;
let output: Closure<dyn FnMut(EncodedAudioChunk)> =
Closure::new(move |audio_data: EncodedAudioChunk| {
let mut array = vec![0u8; audio_data.byte_length() as usize];
audio_data.copy_to_with_u8_slice(&mut array);
download_buffer.borrow_mut().push(array.clone());
if download_buffer.borrow().len() > 200 {
//download_data(download_buffer.borrow().to_vec(), "download_buffer.opus");
//download_data(
// ass::encode(download_buffer.borrow().to_vec(), 960, 0),
// "download_buffer.opus",
//);
download_buffer.borrow_mut().clear();
}
let _ =
packets.unbounded_send(ControlPacket::UDPTunnel(Box::new(VoicePacket::Audio {
_dst: std::marker::PhantomData,
target: 0,
session_id: (),
seq_num: sequence_num,
payload: VoicePacketPayload::Opus(array.into(), false),
position_info: None,
})));
sequence_num = sequence_num.wrapping_add(2);
each(array);
});
let audio_encoder = AudioEncoder::new(&AudioEncoderInit::new(
gui/src/lib.rs
+18 -1
View File
@@ -15,6 +15,7 @@ pub use imp::spawn;
use mumble_protocol::control::msgs;
use mumble_protocol::control::ControlCodec;
use mumble_protocol::control::ControlPacket;
use mumble_protocol::voice::VoicePacket;
use mumble_protocol::voice::VoicePacketPayload;
use mumble_protocol::Clientbound;
use mumble_protocol::Serverbound;
@@ -103,7 +104,23 @@ pub async fn network_loop<R: imp::ImpRead, W: imp::ImpWrite>(
});
}
let mut audio = imp::AudioSystem::new(send_chan.clone())?;
let mut audio = imp::AudioSystem::new()?;
{
let send_chan = send_chan.clone();
let mut sequence_num = 0;
audio.start_recording(move |opus_frame| {
let _ =
send_chan.unbounded_send(ControlPacket::UDPTunnel(Box::new(VoicePacket::Audio {
_dst: std::marker::PhantomData,
target: 0,
session_id: (),
seq_num: sequence_num,
payload: VoicePacketPayload::Opus(opus_frame.into(), false),
position_info: None,
})));
sequence_num = sequence_num.wrapping_add(2);
});
}
// Create map of session_id -> AudioDecoder
let mut decoder_map = HashMap::new();