diff --git a/Cargo.lock b/Cargo.lock index ba8a67d..9f96b5d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1531,9 +1531,9 @@ checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" [[package]] name = "js-sys" -version = "0.3.69" +version = "0.3.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" +checksum = "1868808506b929d7b0cfa8f75951347aa71bb21144b7791bae35d9bccfcfe37a" dependencies = [ "wasm-bindgen", ] @@ -1737,12 +1737,14 @@ dependencies = [ "anyhow", "async-std", "asynchronous-codec", + "byteorder", "dioxus", "dioxus-web", "futures", "manganis", "merge-io", "mumble-protocol-2x", + "ogg", "once_cell", "serde-wasm-bindgen 0.6.5", "serde_json", @@ -1798,6 +1800,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "ogg" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5477016638150530ba21dec7caac835b29ef69b20865751d2973fce6be386cf1" +dependencies = [ + "byteorder", +] + [[package]] name = "once_cell" version = "1.19.0" @@ -2797,19 +2808,20 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" +checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5" dependencies = [ "cfg-if", + "once_cell", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" +checksum = "9de396da306523044d3302746f1208fa71d7532227f15e347e2d93e4145dd77b" dependencies = [ "bumpalo", "log", @@ -2834,9 +2846,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" +checksum = "585c4c91a46b072c92e908d99cb1dcdf95c5218eeb6f3bf1efa991ee7a68cccf" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2844,9 +2856,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" +checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" dependencies = [ "proc-macro2", "quote", @@ -2857,9 +2869,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" +checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484" [[package]] name = "wasm-streams" @@ -2876,9 +2888,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.69" +version = "0.3.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef" +checksum = "26fdeaafd9bd129f65e7c031593c24d62186301e0c72c8978fa1678be7d532c0" dependencies = [ "js-sys", "wasm-bindgen", diff --git a/Cargo.toml b/Cargo.toml index d814927..4ef6625 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,6 +18,8 @@ tokio-util = { version = "0.7.11", features = ["codec"]} wasm-bindgen = "0.2.92" wasm-bindgen-futures = "0.4.42" wasm-streams = "0.4.0" -web-sys = { version = "0.3.69", features = ["WebTransport", "console", "WebTransportOptions", "WebTransportBidirectionalStream", "WebTransportSendStream", "WebTransportReceiveStream", "Navigator", "MediaDevices", "AudioDecoder", "AudioDecoderInit", "AudioData", "AudioEncoderConfig", "AudioDecoderConfig", "EncodedAudioChunk", "EncodedAudioChunkInit", "EncodedAudioChunkType", "CodecState", "MediaStreamTrackGenerator", "MediaStreamTrackGeneratorInit", "AudioContext", "AudioContextOptions", "MediaStream", "GainNode", "MediaStreamAudioSourceNode", "BaseAudioContext", "AudioDestinationNode", "AudioWorkletNode", "AudioWorklet", "AudioWorkletProcessor", "MediaStreamConstraints", "WorkletOptions", "AudioEncoder", "AudioEncoderInit", "AudioDataInit"] } +web-sys = { version = "0.3.70", features = ["WebTransport", "console", "WebTransportOptions", "WebTransportBidirectionalStream", "WebTransportSendStream", "WebTransportReceiveStream", "Navigator", "MediaDevices", "AudioDecoder", "AudioDecoderInit", "AudioData", "AudioEncoderConfig", "AudioDecoderConfig", "EncodedAudioChunk", "EncodedAudioChunkInit", "EncodedAudioChunkType", "CodecState", "MediaStreamTrackGenerator", "MediaStreamTrackGeneratorInit", "AudioContext", "AudioContextOptions", "MediaStream", "GainNode", "MediaStreamAudioSourceNode", "BaseAudioContext", "AudioDestinationNode", "AudioWorkletNode", "AudioWorklet", "AudioWorkletProcessor", "MediaStreamConstraints", "WorkletOptions", "AudioEncoder", "AudioEncoderInit", "AudioDataInit", "HtmlAnchorElement", "Url", "Blob", "AudioDataCopyToOptions", "AudioSampleFormat"] } async-std = "1.12.0" anyhow = "1.0.86" +byteorder = "1.5.0" +ogg = "0.9.1" diff --git a/public/rust_mic_worklet.js b/public/rust_mic_worklet.js index 3ee9d4c..df12eb4 100644 --- a/public/rust_mic_worklet.js +++ b/public/rust_mic_worklet.js @@ -1,6 +1,7 @@ const SAMPLE_RATE = 48000; -const PACKET_MS = 20; +const PACKET_MS = 10; const PACKET_FRAMES = PACKET_MS / 1000 * SAMPLE_RATE; +//const PACKET_FRAMES = 2400; console.log("Frames per packet:", PACKET_FRAMES); class RustWorklet extends AudioWorkletProcessor { @@ -22,9 +23,11 @@ class RustWorklet extends AudioWorkletProcessor { const data = { format: 'f32', sampleRate: SAMPLE_RATE, + //numberOfFrames: this.buffer_offset, numberOfFrames: this.buffer_offset, numberOfChannels: 1, timestamp: this.timestamp, + //timestamp: null, data: this.buffer.slice(0, this.buffer_offset), }; this.port.postMessage(data); @@ -33,8 +36,13 @@ class RustWorklet extends AudioWorkletProcessor { } process(inputs) { + //console.log(inputs); + if (inputs.length != 1) { + console.log("We got " + inputs.length + " heads?") + } const input = inputs[0]; if (input.length == 0) { + console.log("We got no ears?") return true } diff --git a/src/lib.rs b/src/lib.rs index 5532348..b2b7644 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -44,6 +44,116 @@ use web_sys::WebTransport; use web_sys::WebTransportOptions; use web_sys::WorkletOptions; +mod ass { + + +use byteorder::{ByteOrder, LittleEndian}; +use ogg::PacketWriter; + +const VER: &str = "ballz"; + +const fn to_samples(ms: u32) -> usize { + ((S_PS * ms) / 1000) as usize +} + +const fn calc_sr_u64(val: u64, from: u32, to: u32) -> u64 { + (val * to as u64) / from as u64 +} + +pub fn encode( + pre_encoded_frames: Vec>, + frame_size: usize, + skip: u16, +) -> Vec{ + let mut buffer: Vec = Vec::new(); + let mut packet_writer = PacketWriter::new(&mut buffer); + + // Hardcoded serial number + let serial = 12345; + + let skip_48 = calc_sr_u64(skip.into(), 48000, 48000); + + let opus_head: [u8; 19] = [ + b'O', b'p', b'u', b's', b'H', b'e', b'a', b'd', 1, 1, // NUM_CHANNELS = 1 + 0, 0, 0, 0, 0, 0, 0, 0, 0, + ]; + + let mut head = opus_head; + LittleEndian::write_u16(&mut head[10..12], skip_48 as u16); + LittleEndian::write_u32(&mut head[12..16], 48000); + + let mut opus_tags: Vec = Vec::with_capacity(60); + let vendor_str = format!("ogg-opus {}", VER); + opus_tags.extend(b"OpusTags"); + let mut len_bf = [0u8; 4]; + LittleEndian::write_u32(&mut len_bf, vendor_str.len() as u32); + opus_tags.extend(&len_bf); + opus_tags.extend(vendor_str.bytes()); + opus_tags.extend(&[0u8; 4]); + + packet_writer.write_packet(&head, serial, ogg::PacketWriteEndInfo::EndPage, 0); + packet_writer.write_packet( + &opus_tags, + serial, + ogg::PacketWriteEndInfo::EndPage, + 0, + ); + + for (i, frame) in pre_encoded_frames.iter().enumerate() { + let is_last = i == pre_encoded_frames.len() - 1; + //let granule_pos = 0; + let granule_pos = calc_sr_u64( + (skip as usize + (i + 1) * frame_size) as u64, + 48000, + 48000, + ); + + packet_writer.write_packet( + frame.clone(), + serial, + if is_last { + ogg::PacketWriteEndInfo::EndStream + } else { + ogg::PacketWriteEndInfo::NormalPacket + }, + granule_pos, + ); + } + + buffer +} + +} + +// Function to download data as a file +pub fn download_data(data: Vec, filename: &str) -> Result<(), JsValue> { + use wasm_bindgen::prelude::*; + use web_sys::{Blob, Url, HtmlAnchorElement, window}; + // Create a new Blob from the data + let array = web_sys::js_sys::Uint8Array::from(&data[..]); + let blob = Blob::new_with_u8_array_sequence(&vec![array].into())?; + + // Create a URL for the Blob + let url = Url::create_object_url_with_blob(&blob)?; + + // Create an anchor element and set its href to the Blob URL + let document = window().unwrap().document().unwrap(); + let a = document.create_element("a")?.dyn_into::()?; + a.set_href(&url); + a.set_download(filename); + + // Append the anchor to the document body, click it, and remove it + document.body().unwrap().append_child(&a)?; + a.click(); + document.body().unwrap().remove_child(&a)?; + + // Revoke the object URL to free resources + Url::revoke_object_url(&url)?; + + Ok(()) +} + + // Borrowed from // https://github.com/security-union/videocall-rs/blob/main/videocall-client/src/decode/config.rs#L6 fn configure_audio_context() -> AudioContext { @@ -97,12 +207,25 @@ async fn create_encoder_worklet( let error: Closure = Closure::new(|e| console::error_1(&e)); + + let mut download_buffer = std::cell::RefCell::new(Vec::new()); + + // This knows what MediaStreamTrackGenerator to use as it closes around it let mut sequence_num = 0; let output: Closure = Closure::new(move |audio_data: EncodedAudioChunk| { let mut array = vec![0u8; audio_data.byte_length() as usize]; audio_data.copy_to_with_u8_array(&mut array); + + download_buffer.borrow_mut().push(array.clone()); + if download_buffer.borrow().len() > 200 { + //download_data(download_buffer.borrow().to_vec(), "download_buffer.opus"); + download_data(ass::encode(download_buffer.borrow().to_vec(), 960, 0), "download_buffer.opus"); + download_buffer.borrow_mut().clear(); + } + + let _ = packets.try_send(ControlPacket::UDPTunnel(Box::new(VoicePacket::Audio { _dst: std::marker::PhantomData, target: 0, @@ -111,7 +234,7 @@ async fn create_encoder_worklet( payload: VoicePacketPayload::Opus(array.into(), false), position_info: None, }))); - sequence_num = sequence_num.wrapping_add(1); + sequence_num = sequence_num.wrapping_add(2); }); let audio_encoder = AudioEncoder::new(&AudioEncoderInit::new( @@ -123,17 +246,30 @@ async fn create_encoder_worklet( // This is required to prevent these from being deallocated error.forget(); output.forget(); + let encoder_config = AudioEncoderConfig::new("opus"); + encoder_config.set_number_of_channels(1); + encoder_config.set_sample_rate(48000); + encoder_config.set_bitrate(72_000.0); - audio_encoder.configure( - &AudioEncoderConfig::new("opus") - .number_of_channels(1) - .sample_rate(48000), - ); + audio_encoder.configure(&encoder_config); console::log_1(&"Created Audio Encoder".into()); + let mut download_buffer = std::cell::RefCell::new(Vec::new()); + let onmessage: Closure = Closure::new(move |event: MessageEvent| { match AudioData::new(event.data().unchecked_ref()) { Ok(data) => { + let x = web_sys::AudioDataCopyToOptions::new(0); + x.set_format(web_sys::AudioSampleFormat::F32); + let mut sub_buffer = vec![0;data.allocation_size(&x).unwrap() as usize]; + data.copy_to_with_u8_array(&mut sub_buffer, &x); + download_buffer.borrow_mut().append(&mut sub_buffer); + if download_buffer.borrow().len() > 48000 * 10 * 4 { + //pub fn download_data(data: Vec, filename: &str) -> Result<(), JsValue> { + //download_data(download_buffer.borrow().to_vec(), "download_buffer.pcm32"); + download_buffer.borrow_mut().clear(); + } + audio_encoder.encode(&data); } Err(err) => {