mumble/mumble-web2
6
0
Fork 0
Code Issues Pull Requests 5 Actions Packages Projects Releases Wiki Activity

less broken microphone support

Browse Source
This commit is contained in:
restitux
2024-08-30 22:48:49 -06:00
parent 30245b2fb8
commit b0ae26d79a
4 changed files with 180 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+26 -14
View File
@@ -1531,9 +1531,9 @@ checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b"
[[package]]
name = "js-sys"
version = "0.3.69"
version = "0.3.70"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d"
checksum = "1868808506b929d7b0cfa8f75951347aa71bb21144b7791bae35d9bccfcfe37a"
dependencies = [
"wasm-bindgen",
]
@@ -1737,12 +1737,14 @@ dependencies = [
"anyhow",
"async-std",
"asynchronous-codec",
"byteorder",
"dioxus",
"dioxus-web",
"futures",
"manganis",
"merge-io",
"mumble-protocol-2x",
"ogg",
"once_cell",
"serde-wasm-bindgen 0.6.5",
"serde_json",
@@ -1798,6 +1800,15 @@ dependencies = [
"memchr",
]
[[package]]
name = "ogg"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5477016638150530ba21dec7caac835b29ef69b20865751d2973fce6be386cf1"
dependencies = [
"byteorder",
]
[[package]]
name = "once_cell"
version = "1.19.0"
@@ -2797,19 +2808,20 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
[[package]]
name = "wasm-bindgen"
version = "0.2.92"
version = "0.2.93"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8"
checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5"
dependencies = [
"cfg-if",
"once_cell",
"wasm-bindgen-macro",
]
[[package]]
name = "wasm-bindgen-backend"
version = "0.2.92"
version = "0.2.93"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da"
checksum = "9de396da306523044d3302746f1208fa71d7532227f15e347e2d93e4145dd77b"
dependencies = [
"bumpalo",
"log",
@@ -2834,9 +2846,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro"
version = "0.2.92"
version = "0.2.93"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726"
checksum = "585c4c91a46b072c92e908d99cb1dcdf95c5218eeb6f3bf1efa991ee7a68cccf"
dependencies = [
"quote",
"wasm-bindgen-macro-support",
@@ -2844,9 +2856,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro-support"
version = "0.2.92"
version = "0.2.93"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7"
checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836"
dependencies = [
"proc-macro2",
"quote",
@@ -2857,9 +2869,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-shared"
version = "0.2.92"
version = "0.2.93"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96"
checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484"
[[package]]
name = "wasm-streams"
@@ -2876,9 +2888,9 @@ dependencies = [
[[package]]
name = "web-sys"
version = "0.3.69"
version = "0.3.70"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef"
checksum = "26fdeaafd9bd129f65e7c031593c24d62186301e0c72c8978fa1678be7d532c0"
dependencies = [
"js-sys",
"wasm-bindgen",
Cargo.toml
+3 -1
View File
@@ -18,6 +18,8 @@ tokio-util = { version = "0.7.11", features = ["codec"]}
wasm-bindgen = "0.2.92"
wasm-bindgen-futures = "0.4.42"
wasm-streams = "0.4.0"
web-sys = { version = "0.3.69", features = ["WebTransport", "console", "WebTransportOptions", "WebTransportBidirectionalStream", "WebTransportSendStream", "WebTransportReceiveStream", "Navigator", "MediaDevices", "AudioDecoder", "AudioDecoderInit", "AudioData", "AudioEncoderConfig", "AudioDecoderConfig", "EncodedAudioChunk", "EncodedAudioChunkInit", "EncodedAudioChunkType", "CodecState", "MediaStreamTrackGenerator", "MediaStreamTrackGeneratorInit", "AudioContext", "AudioContextOptions", "MediaStream", "GainNode", "MediaStreamAudioSourceNode", "BaseAudioContext", "AudioDestinationNode", "AudioWorkletNode", "AudioWorklet", "AudioWorkletProcessor", "MediaStreamConstraints", "WorkletOptions", "AudioEncoder", "AudioEncoderInit", "AudioDataInit"] }
web-sys = { version = "0.3.70", features = ["WebTransport", "console", "WebTransportOptions", "WebTransportBidirectionalStream", "WebTransportSendStream", "WebTransportReceiveStream", "Navigator", "MediaDevices", "AudioDecoder", "AudioDecoderInit", "AudioData", "AudioEncoderConfig", "AudioDecoderConfig", "EncodedAudioChunk", "EncodedAudioChunkInit", "EncodedAudioChunkType", "CodecState", "MediaStreamTrackGenerator", "MediaStreamTrackGeneratorInit", "AudioContext", "AudioContextOptions", "MediaStream", "GainNode", "MediaStreamAudioSourceNode", "BaseAudioContext", "AudioDestinationNode", "AudioWorkletNode", "AudioWorklet", "AudioWorkletProcessor", "MediaStreamConstraints", "WorkletOptions", "AudioEncoder", "AudioEncoderInit", "AudioDataInit", "HtmlAnchorElement", "Url", "Blob", "AudioDataCopyToOptions", "AudioSampleFormat"] }
async-std = "1.12.0"
anyhow = "1.0.86"
byteorder = "1.5.0"
ogg = "0.9.1"
public/rust_mic_worklet.js
+9 -1
View File
@@ -1,6 +1,7 @@
const SAMPLE_RATE = 48000;
const PACKET_MS = 20;
const PACKET_MS = 10;
const PACKET_FRAMES = PACKET_MS / 1000 * SAMPLE_RATE;
//const PACKET_FRAMES = 2400;
console.log("Frames per packet:", PACKET_FRAMES);
class RustWorklet extends AudioWorkletProcessor {
@@ -22,9 +23,11 @@ class RustWorklet extends AudioWorkletProcessor {
const data = {
format: 'f32',
sampleRate: SAMPLE_RATE,
//numberOfFrames: this.buffer_offset,
numberOfFrames: this.buffer_offset,
numberOfChannels: 1,
timestamp: this.timestamp,
//timestamp: null,
data: this.buffer.slice(0, this.buffer_offset),
};
this.port.postMessage(data);
@@ -33,8 +36,13 @@ class RustWorklet extends AudioWorkletProcessor {
}
process(inputs) {
//console.log(inputs);
if (inputs.length != 1) {
console.log("We got " + inputs.length + " heads?")
}
const input = inputs[0];
if (input.length == 0) {
console.log("We got no ears?")
return true
}
src/lib.rs
+142 -6
View File
@@ -44,6 +44,116 @@ use web_sys::WebTransport;
use web_sys::WebTransportOptions;
use web_sys::WorkletOptions;
mod ass {
use byteorder::{ByteOrder, LittleEndian};
use ogg::PacketWriter;
const VER: &str = "ballz";
const fn to_samples<const S_PS: u32>(ms: u32) -> usize {
((S_PS * ms) / 1000) as usize
}
const fn calc_sr_u64(val: u64, from: u32, to: u32) -> u64 {
(val * to as u64) / from as u64
}
pub fn encode(
pre_encoded_frames: Vec<Vec<u8>>,
frame_size: usize,
skip: u16,
) -> Vec<u8>{
let mut buffer: Vec<u8> = Vec::new();
let mut packet_writer = PacketWriter::new(&mut buffer);
// Hardcoded serial number
let serial = 12345;
let skip_48 = calc_sr_u64(skip.into(), 48000, 48000);
let opus_head: [u8; 19] = [
b'O', b'p', b'u', b's', b'H', b'e', b'a', b'd', 1, 1, // NUM_CHANNELS = 1
0, 0, 0, 0, 0, 0, 0, 0, 0,
];
let mut head = opus_head;
LittleEndian::write_u16(&mut head[10..12], skip_48 as u16);
LittleEndian::write_u32(&mut head[12..16], 48000);
let mut opus_tags: Vec<u8> = Vec::with_capacity(60);
let vendor_str = format!("ogg-opus {}", VER);
opus_tags.extend(b"OpusTags");
let mut len_bf = [0u8; 4];
LittleEndian::write_u32(&mut len_bf, vendor_str.len() as u32);
opus_tags.extend(&len_bf);
opus_tags.extend(vendor_str.bytes());
opus_tags.extend(&[0u8; 4]);
packet_writer.write_packet(&head, serial, ogg::PacketWriteEndInfo::EndPage, 0);
packet_writer.write_packet(
&opus_tags,
serial,
ogg::PacketWriteEndInfo::EndPage,
0,
);
for (i, frame) in pre_encoded_frames.iter().enumerate() {
let is_last = i == pre_encoded_frames.len() - 1;
//let granule_pos = 0;
let granule_pos = calc_sr_u64(
(skip as usize + (i + 1) * frame_size) as u64,
48000,
48000,
);
packet_writer.write_packet(
frame.clone(),
serial,
if is_last {
ogg::PacketWriteEndInfo::EndStream
} else {
ogg::PacketWriteEndInfo::NormalPacket
},
granule_pos,
);
}
buffer
}
}
// Function to download data as a file
pub fn download_data(data: Vec<u8>, filename: &str) -> Result<(), JsValue> {
use wasm_bindgen::prelude::*;
use web_sys::{Blob, Url, HtmlAnchorElement, window};
// Create a new Blob from the data
let array = web_sys::js_sys::Uint8Array::from(&data[..]);
let blob = Blob::new_with_u8_array_sequence(&vec![array].into())?;
// Create a URL for the Blob
let url = Url::create_object_url_with_blob(&blob)?;
// Create an anchor element and set its href to the Blob URL
let document = window().unwrap().document().unwrap();
let a = document.create_element("a")?.dyn_into::<HtmlAnchorElement>()?;
a.set_href(&url);
a.set_download(filename);
// Append the anchor to the document body, click it, and remove it
document.body().unwrap().append_child(&a)?;
a.click();
document.body().unwrap().remove_child(&a)?;
// Revoke the object URL to free resources
Url::revoke_object_url(&url)?;
Ok(())
}
// Borrowed from
// https://github.com/security-union/videocall-rs/blob/main/videocall-client/src/decode/config.rs#L6
fn configure_audio_context() -> AudioContext {
@@ -97,12 +207,25 @@ async fn create_encoder_worklet(
let error: Closure<dyn FnMut(JsValue)> = Closure::new(|e| console::error_1(&e));
let mut download_buffer = std::cell::RefCell::new(Vec::new());
// This knows what MediaStreamTrackGenerator to use as it closes around it
let mut sequence_num = 0;
let output: Closure<dyn FnMut(EncodedAudioChunk)> =
Closure::new(move |audio_data: EncodedAudioChunk| {
let mut array = vec![0u8; audio_data.byte_length() as usize];
audio_data.copy_to_with_u8_array(&mut array);
download_buffer.borrow_mut().push(array.clone());
if download_buffer.borrow().len() > 200 {
//download_data(download_buffer.borrow().to_vec(), "download_buffer.opus");
download_data(ass::encode(download_buffer.borrow().to_vec(), 960, 0), "download_buffer.opus");
download_buffer.borrow_mut().clear();
}
let _ = packets.try_send(ControlPacket::UDPTunnel(Box::new(VoicePacket::Audio {
_dst: std::marker::PhantomData,
target: 0,
@@ -111,7 +234,7 @@ async fn create_encoder_worklet(
payload: VoicePacketPayload::Opus(array.into(), false),
position_info: None,
})));
sequence_num = sequence_num.wrapping_add(1);
sequence_num = sequence_num.wrapping_add(2);
});
let audio_encoder = AudioEncoder::new(&AudioEncoderInit::new(
@@ -123,17 +246,30 @@ async fn create_encoder_worklet(
// This is required to prevent these from being deallocated
error.forget();
output.forget();
let encoder_config = AudioEncoderConfig::new("opus");
encoder_config.set_number_of_channels(1);
encoder_config.set_sample_rate(48000);
encoder_config.set_bitrate(72_000.0);
audio_encoder.configure(
&AudioEncoderConfig::new("opus")
.number_of_channels(1)
.sample_rate(48000),
);
audio_encoder.configure(&encoder_config);
console::log_1(&"Created Audio Encoder".into());
let mut download_buffer = std::cell::RefCell::new(Vec::new());
let onmessage: Closure<dyn FnMut(MessageEvent)> = Closure::new(move |event: MessageEvent| {
match AudioData::new(event.data().unchecked_ref()) {
Ok(data) => {
let x = web_sys::AudioDataCopyToOptions::new(0);
x.set_format(web_sys::AudioSampleFormat::F32);
let mut sub_buffer = vec![0;data.allocation_size(&x).unwrap() as usize];
data.copy_to_with_u8_array(&mut sub_buffer, &x);
download_buffer.borrow_mut().append(&mut sub_buffer);
if download_buffer.borrow().len() > 48000 * 10 * 4 {
//pub fn download_data(data: Vec<u8>, filename: &str) -> Result<(), JsValue> {
//download_data(download_buffer.borrow().to_vec(), "download_buffer.pcm32");
download_buffer.borrow_mut().clear();
}
audio_encoder.encode(&data);
}
Err(err) => {