From cf6458aae9265cee1c87b08093e5d2c750b73db7 Mon Sep 17 00:00:00 2001 From: restitux Date: Sat, 25 May 2024 17:54:31 -0600 Subject: [PATCH] Added audio playback support --- Cargo.toml | 2 +- src/lib.rs | 213 +++++++++++++++++++++++++++++++++++++++++++---------- 2 files changed, 177 insertions(+), 38 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 75e5268..62b6341 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,5 +18,5 @@ tokio-util = { version = "0.7.11", features = ["codec"]} wasm-bindgen = "0.2.92" wasm-bindgen-futures = "0.4.42" wasm-streams = "0.4.0" -web-sys = { version = "0.3.69", features = ["WebTransport", "console", "WebTransportOptions", "WebTransportBidirectionalStream", "WebTransportSendStream", "WebTransportReceiveStream", "Navigator"] } +web-sys = { version = "0.3.69", features = ["WebTransport", "console", "WebTransportOptions", "WebTransportBidirectionalStream", "WebTransportSendStream", "WebTransportReceiveStream", "Navigator", "MediaDevices", "AudioDecoder", "AudioDecoderInit", "AudioData", "AudioEncoderConfig", "AudioDecoderConfig", "EncodedAudioChunk", "EncodedAudioChunkInit", "EncodedAudioChunkType", "CodecState", "MediaStreamTrackGenerator", "MediaStreamTrackGeneratorInit", "AudioContext", "AudioContextOptions", "MediaStream", "GainNode", "MediaStreamAudioSourceNode", "BaseAudioContext", "AudioDestinationNode"] } async-std = "1.12.0" diff --git a/src/lib.rs b/src/lib.rs index c74f5c3..2c6a138 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -20,18 +20,55 @@ use mumble_protocol::control::{msgs, ClientControlCodec}; use mumble_protocol::Clientbound; use mumble_protocol::Serverbound; use wasm_bindgen::prelude::*; +use wasm_bindgen_futures::JsFuture; use web_sys::console; +use web_sys::js_sys::Uint8Array; +use web_sys::AudioData; +use web_sys::AudioContext; +use web_sys::AudioContextOptions; +use web_sys::MediaStream; +use web_sys::AudioDecoder; +use web_sys::AudioDecoderConfig; +use web_sys::AudioDecoderInit; +use web_sys::CodecState; +use web_sys::EncodedAudioChunk; +use web_sys::EncodedAudioChunkInit; +use web_sys::EncodedAudioChunkType; +use web_sys::MediaStreamTrackGenerator; +use web_sys::MediaStreamTrackGeneratorInit; use web_sys::WebTransport; use web_sys::WebTransportOptions; use wasm_bindgen_futures::spawn_local as spawn; -#[wasm_bindgen] -extern "C" { - pub fn alert(s: &str); +//#[wasm_bindgen] +//extern "C" { +//} + +fn configure_audio_context( + audio_stream_generator: &MediaStreamTrackGenerator, +) -> AudioContext { + let js_tracks = web_sys::js_sys::Array::new(); + js_tracks.push(audio_stream_generator); + let media_stream = MediaStream::new_with_tracks(&js_tracks).unwrap(); + let mut audio_context_options = AudioContextOptions::new(); + audio_context_options.sample_rate(48000 as f32); + let audio_context = AudioContext::new_with_context_options(&audio_context_options).unwrap(); + let gain_node = audio_context.create_gain().unwrap(); + gain_node.set_channel_count(1); + let source = audio_context + .create_media_stream_source(&media_stream) + .unwrap(); + let _ = source.connect_with_audio_node(&gain_node).unwrap(); + let _ = gain_node + .connect_with_audio_node(&audio_context.destination()) + .unwrap(); + audio_context } pub async fn network_entrypoint() { + async_std::task::sleep(Duration::from_millis(3000)).await; + console::log_1(&"Rust via WASM!".into()); let server_hash = vec![ @@ -104,8 +141,10 @@ pub async fn network_entrypoint() { let read_codec = ClientControlCodec::new(); let write_codec = ClientControlCodec::new(); - let mut reader = asynchronous_codec::FramedRead::new(wasm_stream_readable.into_async_read(), read_codec); - let mut writer = asynchronous_codec::FramedWrite::new(wasm_stream_writable.into_async_write(), write_codec); + let mut reader = + asynchronous_codec::FramedRead::new(wasm_stream_readable.into_async_read(), read_codec); + let mut writer = + asynchronous_codec::FramedWrite::new(wasm_stream_writable.into_async_write(), write_codec); let (send_chan, writer_recv_chan) = async_std::channel::unbounded(); @@ -134,6 +173,7 @@ pub async fn network_entrypoint() { // Send authenticate packet let mut msg = msgs::Authenticate::new(); msg.set_username("mumbleweb2".to_string()); + msg.set_opus(true); send_chan.send(msg.into()).await.unwrap(); console::log_1(&"Sent authenticate packet".into()); @@ -152,42 +192,141 @@ pub async fn network_entrypoint() { }); } + let error = Closure::wrap(Box::new(move |e: JsValue| { + console::log_1(&e); + }) as Box); + + let audio_stream_generator = + MediaStreamTrackGenerator::new(&MediaStreamTrackGeneratorInit::new("audio")).unwrap(); + // The audio context is used to reproduce audio. + let _audio_context = configure_audio_context(&audio_stream_generator); + + let output = Closure::wrap(Box::new(move |audio_data: AudioData| { + console::log_1(&"Got audio".into()); + console::log_1(&audio_data.clone().into()); + let writable = audio_stream_generator.writable(); + if writable.locked() { + return; + } + if let Err(e) = writable.get_writer().map(|writer| { + wasm_bindgen_futures::spawn_local(async move { + if let Err(e) = JsFuture::from(writer.ready()).await { + console::log_1(&format!("write chunk error {:?}", e).into()); + } + if let Err(e) = JsFuture::from(writer.write_with_chunk(&audio_data)).await { + console::log_1(&format!("write chunk error {:?}", e).into()); + }; + writer.release_lock(); + }); + }) { + console::log_1(&e); + } + }) as Box); + + let audio_decoder = AudioDecoder::new(&AudioDecoderInit::new( + error.as_ref().unchecked_ref(), + output.as_ref().unchecked_ref(), + )).unwrap(); + + console::log_1(&"Created Audio Decoder".into()); + console::log_1(&audio_decoder); + + audio_decoder.configure(&AudioDecoderConfig::new( + "opus", + 1, + 48000, + )); + + loop { - match reader.next().await { - Some(Ok(msg)) => { - console::log_1(&format!("{:#?}", msg).into()); - }, - None => { - break; - }, - Some(Err(e)) => { - console::log_1(&e.to_string().into()); - break; - }, - } + match reader.next().await { + Some(Ok(msg)) => { + match msg { + ControlPacket::UDPTunnel(u) => { + match *u.clone() { + mumble_protocol::voice::VoicePacket::Audio { + _dst, + target, + session_id, + seq_num, + payload, + position_info, + } => { + console::log_1(&"Voice packet found".into()); + if let mumble_protocol::voice::VoicePacketPayload::Opus( + audio_payload, + end_bit, + ) = payload + { + let js_audio_payload = Uint8Array::from(audio_payload.as_ref()); + audio_decoder.decode(&EncodedAudioChunk::new( + &EncodedAudioChunkInit::new( + &js_audio_payload.into(), + 0.0, + EncodedAudioChunkType::Key, + ), + ).unwrap()); + console::log_1(&"Oueued audio chunk for decoding".into()); + + + //let mut encoded_audio_chunk_init = + // web_sys::EncodedAudioChunkInit::new( + // &js_audio_payload.into(), + // 0.0, + // web_sys::EncodedAudioChunkType::Delta, + // ); + ////encoded_audio_chunk_init.duration(1.0); + //let encoded_audio_chunk = + // web_sys::EncodedAudioChunk::new(&encoded_audio_chunk_init) + // .unwrap(); + //console::log_1(&encoded_audio_chunk); + + } + } + _ => { + unreachable!("TCP tunnels UDP packets should not contain pings"); + // I think? + } + } + console::log_1(&"Got UDP tunnel".into()); + console::log_1(&format!("{:#?}", u).into()); + } + _ => { + console::log_1(&format!("{:#?}", msg).into()); + } + } + } + None => { + break; + } + Some(Err(e)) => { + console::log_1(&e.to_string().into()); + break; + } + } } -//async fn handle_send( -// mut writer: Rc>>>, -// send_queue: Queue, -//) { -// loop { -// let msg = send_queue.get(); -// client.send(msg).await.unwrap(); -// } -//} + //async fn handle_send( + // mut writer: Rc>>>, + // send_queue: Queue, + //) { + // loop { + // let msg = send_queue.get(); + // client.send(msg).await.unwrap(); + // } + //} -//async fn handle_ping( -//) { -// //pin!(client); -// loop { -// let ping = msgs::Ping::new(); -// client.borrow_mut().send(ping.into()).await.unwrap(); -// console::log_1(&"Sent ping packet".into()); -// -// async_std::task::sleep(Duration::from_millis(3000)).await; -// } -//} + //async fn handle_ping( + //) { + // //pin!(client); + // loop { + // let ping = msgs::Ping::new(); + // client.borrow_mut().send(ping.into()).await.unwrap(); + // console::log_1(&"Sent ping packet".into()); + // + // async_std::task::sleep(Duration::from_millis(3000)).await; + // } + //} //let queue_write, queue_read = Queue::new();