mumble/mumble-web2
6
0
Fork 0
Code Issues Pull Requests 5 Actions Packages Projects Releases Wiki Activity

Added audio playback support

Browse Source
This commit is contained in:
restitux
2024-05-25 17:54:31 -06:00
parent 410e2d8bcd
commit cf6458aae9
2 changed files with 177 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.toml
+1 -1
View File
@@ -18,5 +18,5 @@ tokio-util = { version = "0.7.11", features = ["codec"]}
wasm-bindgen = "0.2.92"
wasm-bindgen-futures = "0.4.42"
wasm-streams = "0.4.0"
web-sys = { version = "0.3.69", features = ["WebTransport", "console", "WebTransportOptions", "WebTransportBidirectionalStream", "WebTransportSendStream", "WebTransportReceiveStream", "Navigator"] }
web-sys = { version = "0.3.69", features = ["WebTransport", "console", "WebTransportOptions", "WebTransportBidirectionalStream", "WebTransportSendStream", "WebTransportReceiveStream", "Navigator", "MediaDevices", "AudioDecoder", "AudioDecoderInit", "AudioData", "AudioEncoderConfig", "AudioDecoderConfig", "EncodedAudioChunk", "EncodedAudioChunkInit", "EncodedAudioChunkType", "CodecState", "MediaStreamTrackGenerator", "MediaStreamTrackGeneratorInit", "AudioContext", "AudioContextOptions", "MediaStream", "GainNode", "MediaStreamAudioSourceNode", "BaseAudioContext", "AudioDestinationNode"] }
async-std = "1.12.0"
src/lib.rs
+176 -37
View File
@@ -20,18 +20,55 @@ use mumble_protocol::control::{msgs, ClientControlCodec};
use mumble_protocol::Clientbound;
use mumble_protocol::Serverbound;
use wasm_bindgen::prelude::*;
use wasm_bindgen_futures::JsFuture;
use web_sys::console;
use web_sys::js_sys::Uint8Array;
use web_sys::AudioData;
use web_sys::AudioContext;
use web_sys::AudioContextOptions;
use web_sys::MediaStream;
use web_sys::AudioDecoder;
use web_sys::AudioDecoderConfig;
use web_sys::AudioDecoderInit;
use web_sys::CodecState;
use web_sys::EncodedAudioChunk;
use web_sys::EncodedAudioChunkInit;
use web_sys::EncodedAudioChunkType;
use web_sys::MediaStreamTrackGenerator;
use web_sys::MediaStreamTrackGeneratorInit;
use web_sys::WebTransport;
use web_sys::WebTransportOptions;
use wasm_bindgen_futures::spawn_local as spawn;
#[wasm_bindgen]
extern "C" {
pub fn alert(s: &str);
//#[wasm_bindgen]
//extern "C" {
//}
fn configure_audio_context(
audio_stream_generator: &MediaStreamTrackGenerator,
) -> AudioContext {
let js_tracks = web_sys::js_sys::Array::new();
js_tracks.push(audio_stream_generator);
let media_stream = MediaStream::new_with_tracks(&js_tracks).unwrap();
let mut audio_context_options = AudioContextOptions::new();
audio_context_options.sample_rate(48000 as f32);
let audio_context = AudioContext::new_with_context_options(&audio_context_options).unwrap();
let gain_node = audio_context.create_gain().unwrap();
gain_node.set_channel_count(1);
let source = audio_context
.create_media_stream_source(&media_stream)
.unwrap();
let _ = source.connect_with_audio_node(&gain_node).unwrap();
let _ = gain_node
.connect_with_audio_node(&audio_context.destination())
.unwrap();
audio_context
}
pub async fn network_entrypoint() {
async_std::task::sleep(Duration::from_millis(3000)).await;
console::log_1(&"Rust via WASM!".into());
let server_hash = vec![
@@ -104,8 +141,10 @@ pub async fn network_entrypoint() {
let read_codec = ClientControlCodec::new();
let write_codec = ClientControlCodec::new();
let mut reader = asynchronous_codec::FramedRead::new(wasm_stream_readable.into_async_read(), read_codec);
let mut writer = asynchronous_codec::FramedWrite::new(wasm_stream_writable.into_async_write(), write_codec);
let mut reader =
asynchronous_codec::FramedRead::new(wasm_stream_readable.into_async_read(), read_codec);
let mut writer =
asynchronous_codec::FramedWrite::new(wasm_stream_writable.into_async_write(), write_codec);
let (send_chan, writer_recv_chan) = async_std::channel::unbounded();
@@ -134,6 +173,7 @@ pub async fn network_entrypoint() {
// Send authenticate packet
let mut msg = msgs::Authenticate::new();
msg.set_username("mumbleweb2".to_string());
msg.set_opus(true);
send_chan.send(msg.into()).await.unwrap();
console::log_1(&"Sent authenticate packet".into());
@@ -152,42 +192,141 @@ pub async fn network_entrypoint() {
});
}
let error = Closure::wrap(Box::new(move |e: JsValue| {
console::log_1(&e);
}) as Box<dyn FnMut(JsValue)>);
let audio_stream_generator =
MediaStreamTrackGenerator::new(&MediaStreamTrackGeneratorInit::new("audio")).unwrap();
// The audio context is used to reproduce audio.
let _audio_context = configure_audio_context(&audio_stream_generator);
let output = Closure::wrap(Box::new(move |audio_data: AudioData| {
console::log_1(&"Got audio".into());
console::log_1(&audio_data.clone().into());
let writable = audio_stream_generator.writable();
if writable.locked() {
return;
}
if let Err(e) = writable.get_writer().map(|writer| {
wasm_bindgen_futures::spawn_local(async move {
if let Err(e) = JsFuture::from(writer.ready()).await {
console::log_1(&format!("write chunk error {:?}", e).into());
}
if let Err(e) = JsFuture::from(writer.write_with_chunk(&audio_data)).await {
console::log_1(&format!("write chunk error {:?}", e).into());
};
writer.release_lock();
});
}) {
console::log_1(&e);
}
}) as Box<dyn FnMut(AudioData)>);
let audio_decoder = AudioDecoder::new(&AudioDecoderInit::new(
error.as_ref().unchecked_ref(),
output.as_ref().unchecked_ref(),
)).unwrap();
console::log_1(&"Created Audio Decoder".into());
console::log_1(&audio_decoder);
audio_decoder.configure(&AudioDecoderConfig::new(
"opus",
1,
48000,
));
loop {
match reader.next().await {
Some(Ok(msg)) => {
console::log_1(&format!("{:#?}", msg).into());
},
None => {
break;
},
Some(Err(e)) => {
console::log_1(&e.to_string().into());
break;
},
}
match reader.next().await {
Some(Ok(msg)) => {
match msg {
ControlPacket::UDPTunnel(u) => {
match *u.clone() {
mumble_protocol::voice::VoicePacket::Audio {
_dst,
target,
session_id,
seq_num,
payload,
position_info,
} => {
console::log_1(&"Voice packet found".into());
if let mumble_protocol::voice::VoicePacketPayload::Opus(
audio_payload,
end_bit,
) = payload
{
let js_audio_payload = Uint8Array::from(audio_payload.as_ref());
audio_decoder.decode(&EncodedAudioChunk::new(
&EncodedAudioChunkInit::new(
&js_audio_payload.into(),
0.0,
EncodedAudioChunkType::Key,
),
).unwrap());
console::log_1(&"Oueued audio chunk for decoding".into());
//let mut encoded_audio_chunk_init =
// web_sys::EncodedAudioChunkInit::new(
// &js_audio_payload.into(),
// 0.0,
// web_sys::EncodedAudioChunkType::Delta,
// );
////encoded_audio_chunk_init.duration(1.0);
//let encoded_audio_chunk =
// web_sys::EncodedAudioChunk::new(&encoded_audio_chunk_init)
// .unwrap();
//console::log_1(&encoded_audio_chunk);
}
}
_ => {
unreachable!("TCP tunnels UDP packets should not contain pings");
// I think?
}
}
console::log_1(&"Got UDP tunnel".into());
console::log_1(&format!("{:#?}", u).into());
}
_ => {
console::log_1(&format!("{:#?}", msg).into());
}
}
}
None => {
break;
}
Some(Err(e)) => {
console::log_1(&e.to_string().into());
break;
}
}
}
//async fn handle_send(
// mut writer: Rc<RefCell<asynchronous_codec::FramedWrite<impl AsyncRead + AsyncWrite + Unpin, ControlCodec<Serverbound, Clientbound>>>>,
// send_queue: Queue<mumble_protocol::Serverbound>,
//) {
// loop {
// let msg = send_queue.get();
// client.send(msg).await.unwrap();
// }
//}
//async fn handle_send(
// mut writer: Rc<RefCell<asynchronous_codec::FramedWrite<impl AsyncRead + AsyncWrite + Unpin, ControlCodec<Serverbound, Clientbound>>>>,
// send_queue: Queue<mumble_protocol::Serverbound>,
//) {
// loop {
// let msg = send_queue.get();
// client.send(msg).await.unwrap();
// }
//}
//async fn handle_ping(
//) {
// //pin!(client);
// loop {
// let ping = msgs::Ping::new();
// client.borrow_mut().send(ping.into()).await.unwrap();
// console::log_1(&"Sent ping packet".into());
//
// async_std::task::sleep(Duration::from_millis(3000)).await;
// }
//}
//async fn handle_ping(
//) {
// //pin!(client);
// loop {
// let ping = msgs::Ping::new();
// client.borrow_mut().send(ping.into()).await.unwrap();
// console::log_1(&"Sent ping packet".into());
//
// async_std::task::sleep(Duration::from_millis(3000)).await;
// }
//}
//let queue_write, queue_read = Queue::new();