From 30245b2fb867b9da772a50c031ba7b7f8c8e3243 Mon Sep 17 00:00:00 2001 From: Sam Sartor Date: Tue, 2 Jul 2024 19:32:43 -0600 Subject: [PATCH] mic sorta working but aweful --- Cargo.lock | 204 +++++++++++++++++-------------------- Cargo.toml | 3 +- public/rust_mic_worklet.js | 63 ++++++++++++ src/lib.rs | 174 ++++++++++++++++++++++++++----- 4 files changed, 310 insertions(+), 134 deletions(-) create mode 100644 public/rust_mic_worklet.js diff --git a/Cargo.lock b/Cargo.lock index e8049e2..ba8a67d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,9 +4,9 @@ version = 3 [[package]] name = "addr2line" -version = "0.21.0" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" +checksum = "6e4503c46a5c0c7844e948c9a4d6acd9f50cccb4de1c48eb9e291ea17470c678" dependencies = [ "gimli", ] @@ -65,16 +65,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89b47800b0be77592da0afd425cc03468052844aff33b84e33cc696f64e77b6a" dependencies = [ "concurrent-queue", - "event-listener-strategy 0.5.2", + "event-listener-strategy", "futures-core", "pin-project-lite", ] [[package]] name = "async-executor" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b10202063978b3351199d68f8b22c4e47e4b1b822f8d43fd862d5ea8c006b29a" +checksum = "c8828ec6e544c02b0d6691d21ed9f9218d0384a82542855073c2a3f58304aaf0" dependencies = [ "async-task", "concurrent-queue", @@ -91,8 +91,8 @@ checksum = "05b1b633a2115cd122d73b955eadd9916c18c8f510ec9cd1686404c60ad1c29c" dependencies = [ "async-channel 2.3.1", "async-executor", - "async-io 2.3.2", - "async-lock 3.3.0", + "async-io 2.3.3", + "async-lock 3.4.0", "blocking", "futures-lite 2.3.0", "once_cell", @@ -120,17 +120,17 @@ dependencies = [ [[package]] name = "async-io" -version = "2.3.2" +version = "2.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcccb0f599cfa2f8ace422d3555572f47424da5648a4382a9dd0310ff8210884" +checksum = "0d6baa8f0178795da0e71bc42c9e5d13261aac7ee549853162e66a241ba17964" dependencies = [ - "async-lock 3.3.0", + "async-lock 3.4.0", "cfg-if", "concurrent-queue", "futures-io", "futures-lite 2.3.0", "parking", - "polling 3.7.0", + "polling 3.7.2", "rustix 0.38.34", "slab", "tracing", @@ -148,12 +148,12 @@ dependencies = [ [[package]] name = "async-lock" -version = "3.3.0" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d034b430882f8381900d3fe6f0aaa3ad94f2cb4ac519b429692a1bc2dda4ae7b" +checksum = "ff6e472cdea888a4bd64f342f09b3f50e1886d32afe8df3d663c01140b811b18" dependencies = [ - "event-listener 4.0.3", - "event-listener-strategy 0.4.0", + "event-listener 5.3.1", + "event-listener-strategy", "pin-project-lite", ] @@ -227,9 +227,9 @@ checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" [[package]] name = "backtrace" -version = "0.3.71" +version = "0.3.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b05800d2e817c8b3b4b54abd461726265fa9789ae34330622f2db9ee696f9d" +checksum = "5cc23269a4f8976d0a4d2e7109211a419fe30e8d88d677cd60b6bc79c5732e0a" dependencies = [ "addr2line", "cc", @@ -263,21 +263,20 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.5.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" +checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" dependencies = [ "serde", ] [[package]] name = "blocking" -version = "1.6.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "495f7104e962b7356f0aeb34247aca1fe7d2e783b346582db7f2904cb5717e88" +checksum = "703f41c54fc768e63e091340b424302bb1c29ef4aa0c7f10fe849dfb114d29ea" dependencies = [ "async-channel 2.3.1", - "async-lock 3.3.0", "async-task", "futures-io", "futures-lite 2.3.0", @@ -336,9 +335,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.98" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41c270e7540d725e65ac7f1b212ac8ce349719624d7bcff99f8e2e488e8cf03f" +checksum = "74b6a57f98764a267ff415d50a25e6e166f3831a5071af4995296ea97d210490" [[package]] name = "cfb" @@ -881,43 +880,22 @@ checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" [[package]] name = "event-listener" -version = "4.0.3" +version = "5.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67b215c49b2b248c855fb73579eb1f4f26c38ffdc12973e20e07b91d78d5646e" +checksum = "6032be9bd27023a771701cc49f9f053c751055f71efb2e0ae5c15809093675ba" dependencies = [ "concurrent-queue", "parking", "pin-project-lite", ] -[[package]] -name = "event-listener" -version = "5.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d9944b8ca13534cdfb2800775f8dd4902ff3fc75a50101466decadfdf322a24" -dependencies = [ - "concurrent-queue", - "parking", - "pin-project-lite", -] - -[[package]] -name = "event-listener-strategy" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "958e4d70b6d5e81971bebec42271ec641e7ff4e170a6fa605f2b8a8b65cb97d3" -dependencies = [ - "event-listener 4.0.3", - "pin-project-lite", -] - [[package]] name = "event-listener-strategy" version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f214dc438f977e6d4e3500aaa277f5ad94ca83fbbd9b1a15713ce2344ccc5a1" dependencies = [ - "event-listener 5.3.0", + "event-listener 5.3.1", "pin-project-lite", ] @@ -1100,9 +1078,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.28.1" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" +checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" [[package]] name = "gloo" @@ -1351,6 +1329,12 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" +[[package]] +name = "hermit-abi" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbf6a919d6cf397374f7dfeeea91d974c7c0a7221d0d0f4f20d859d329e53fcc" + [[package]] name = "home" version = "0.5.9" @@ -1395,9 +1379,9 @@ dependencies = [ [[package]] name = "httparse" -version = "1.8.0" +version = "1.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" +checksum = "0fcc0b4a115bf80b728eb8ea024ad5bd707b615bfed49e0665b6e0f86fd082d9" [[package]] name = "httpdate" @@ -1407,9 +1391,9 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "hyper" -version = "0.14.28" +version = "0.14.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" +checksum = "f361cde2f109281a220d4307746cdfd5ee3f410da58a70377762396775634b33" dependencies = [ "bytes", "futures-channel", @@ -1528,7 +1512,7 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" dependencies = [ - "hermit-abi", + "hermit-abi 0.3.9", "libc", "windows-sys 0.48.0", ] @@ -1560,7 +1544,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b750dcadc39a09dbadd74e118f6dd6598df77fa01df0cfcdc52c28dece74528a" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "serde", "unicode-segmentation", ] @@ -1589,9 +1573,9 @@ dependencies = [ [[package]] name = "lazy_static" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" @@ -1623,9 +1607,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.21" +version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" dependencies = [ "value-bag", ] @@ -1693,9 +1677,9 @@ checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" [[package]] name = "memchr" -version = "2.7.2" +version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] name = "merge-io" @@ -1714,9 +1698,9 @@ checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "miniz_oxide" -version = "0.7.3" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87dfd01fe195c66b572b37921ad8803d010623c0aca821bea2302239d155cdae" +checksum = "b8a240ddb74feaf34a79a7add65a741f3167852fba007066dcac1ca548d89c08" dependencies = [ "adler", ] @@ -1750,6 +1734,7 @@ dependencies = [ name = "mumble-webtransport" version = "0.1.0" dependencies = [ + "anyhow", "async-std", "asynchronous-codec", "dioxus", @@ -1770,11 +1755,10 @@ dependencies = [ [[package]] name = "native-tls" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" +checksum = "a8614eb2c83d59d1c8cc974dd3f920198647674a0a035e1af1fa58707e317466" dependencies = [ - "lazy_static", "libc", "log", "openssl", @@ -1807,9 +1791,9 @@ dependencies = [ [[package]] name = "object" -version = "0.32.2" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" +checksum = "081b846d1d56ddfc18fdf1a922e4f6e07a11768ea1b92dec44e42b72712ccfce" dependencies = [ "memchr", ] @@ -1826,7 +1810,7 @@ version = "0.10.64" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95a0481286a310808298130d22dd1fef0fa571e05a8f44ec801801e84b216b1f" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "cfg-if", "foreign-types", "libc", @@ -1887,9 +1871,9 @@ checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae" [[package]] name = "parking_lot" -version = "0.12.2" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e4af0ca4f6caed20e900d564c242b8e5d4903fdacf31d3daf527b66fe6f42fb" +checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" dependencies = [ "lock_api", "parking_lot_core", @@ -1958,9 +1942,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "piper" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "464db0c665917b13ebb5d453ccdec4add5658ee1adc7affc7677615356a8afaf" +checksum = "ae1d5c74c9876f070d3e8fd503d748c7d974c3e48da8f41350fa5222ef9b4391" dependencies = [ "atomic-waker", "fastrand 2.1.0", @@ -1991,13 +1975,13 @@ dependencies = [ [[package]] name = "polling" -version = "3.7.0" +version = "3.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "645493cf344456ef24219d02a768cf1fb92ddf8c92161679ae3d91b91a637be3" +checksum = "a3ed00ed3fbf728b5816498ecd316d1716eecaced9c0c8d2c5a6740ca214985b" dependencies = [ "cfg-if", "concurrent-queue", - "hermit-abi", + "hermit-abi 0.4.0", "pin-project-lite", "rustix 0.38.34", "tracing", @@ -2016,9 +2000,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.83" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b33eb56c327dec362a9e55b3ad14f9d2f0904fb5a5b03b513ab5465399e9f43" +checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" dependencies = [ "unicode-ident", ] @@ -2059,11 +2043,11 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "469052894dcb553421e483e4209ee581a45100d31b4018de03e5a7ad86374a7e" +checksum = "c82cf8cff14456045f55ec4241383baeff27af886adb72ffb2162f99911de0fd" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", ] [[package]] @@ -2147,7 +2131,7 @@ version = "0.38.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "errno", "libc", "linux-raw-sys 0.4.14", @@ -2190,7 +2174,7 @@ version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c627723fd09706bacdb5cf41499e95098555af3c3c29d014dc3c458ef6be11c0" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "core-foundation", "core-foundation-sys", "libc", @@ -2227,9 +2211,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.202" +version = "1.0.203" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "226b61a0d411b2ba5ff6d7f73a476ac4f8bb900373459cd00fab8512828ba395" +checksum = "7253ab4de971e72fb7be983802300c30b5a7f0c2e56fab8abfc6a214307c0094" dependencies = [ "serde_derive", ] @@ -2268,9 +2252,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.202" +version = "1.0.203" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6048858004bcff69094cd972ed40a32500f153bd3be9f716b2eed2e8217c4838" +checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba" dependencies = [ "proc-macro2", "quote", @@ -2279,9 +2263,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.117" +version = "1.0.120" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "455182ea6142b14f93f4bc5320a2b31c1f266b66a4a5c858b013302a5d8cbfc3" +checksum = "4e0d21c9a8cae1235ad58a00c11cb40d4b1e5c784f1ef2c537876ed6ffd8b7c5" dependencies = [ "itoa", "ryu", @@ -2333,9 +2317,9 @@ dependencies = [ [[package]] name = "server_fn" -version = "0.6.11" +version = "0.6.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "536a5b959673643ee01e59ae41bf01425482c8070dee95d7061ee2d45296b59c" +checksum = "b06e6e5467a2cd93ce1accfdfd8b859404f0b3b2041131ffd774fabf666b8219" dependencies = [ "bytes", "const_format", @@ -2361,9 +2345,9 @@ dependencies = [ [[package]] name = "server_fn_macro" -version = "0.6.11" +version = "0.6.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "064dd9b256e78bf2886774f265cc34d2aefdd05b430c58c78a69eceef21b5e60" +checksum = "09c216bb1c1ac890151397643c663c875a1836adf0b269be4e389cb1b48c173c" dependencies = [ "const_format", "convert_case", @@ -2375,9 +2359,9 @@ dependencies = [ [[package]] name = "server_fn_macro_default" -version = "0.6.11" +version = "0.6.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4ad11700cbccdbd313703916eb8c97301ee423c4a06e5421b77956fdcb36a9f" +checksum = "00783df297ec85ea605779f2fef9cbec98981dffe2e01e1a9845c102ee1f1ae6" dependencies = [ "server_fn_macro", "syn", @@ -2469,9 +2453,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.65" +version = "2.0.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2863d96a84c6439701d7a38f9de935ec562c8832cc55d1dde0f513b52fad106" +checksum = "901fa70d88b9d6c98022e23b4136f9f3e54e4662c3bc1bd1d84a42a9a0f0c1e9" dependencies = [ "proc-macro2", "quote", @@ -2549,9 +2533,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.6.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +checksum = "c55115c6fbe2d2bef26eb09ad74bde02d8255476fc0c7b515ef09fbb35742d82" dependencies = [ "tinyvec_macros", ] @@ -2570,9 +2554,9 @@ checksum = "c7c4ceeeca15c8384bbc3e011dbd8fccb7f068a440b752b7d9b32ceb0ca0e2e8" [[package]] name = "tokio" -version = "1.37.0" +version = "1.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1adbebffeca75fcfd058afa480fb6c0b81e165a0323f9c9d39c9697e37c46787" +checksum = "ba4f4a02a7a80d6f274636f0aa95c7e383b912d41fe721a31f29e29698585a4a" dependencies = [ "backtrace", "bytes", @@ -2744,9 +2728,9 @@ checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" [[package]] name = "url" -version = "2.5.0" +version = "2.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" +checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c" dependencies = [ "form_urlencoded", "idna", @@ -2762,9 +2746,9 @@ checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" [[package]] name = "uuid" -version = "1.8.0" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a183cf7feeba97b4dd1c0d46788634f6221d87fa961b305bed08c851829efcc0" +checksum = "5de17fd2f7da591098415cff336e12965a28061ddace43b59cb3c430179c9439" [[package]] name = "valuable" @@ -3088,18 +3072,18 @@ checksum = "927da81e25be1e1a2901d59b81b37dd2efd1fc9c9345a55007f09bf5a2d3ee03" [[package]] name = "zerocopy" -version = "0.7.34" +version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae87e3fcd617500e5d106f0380cf7b77f3c6092aae37191433159dda23cfb087" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.7.34" +version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15e934569e47891f7d9411f1a451d947a60e000ab3bd24fbb970f000387d1b3b" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index 62b6341..d814927 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,5 +18,6 @@ tokio-util = { version = "0.7.11", features = ["codec"]} wasm-bindgen = "0.2.92" wasm-bindgen-futures = "0.4.42" wasm-streams = "0.4.0" -web-sys = { version = "0.3.69", features = ["WebTransport", "console", "WebTransportOptions", "WebTransportBidirectionalStream", "WebTransportSendStream", "WebTransportReceiveStream", "Navigator", "MediaDevices", "AudioDecoder", "AudioDecoderInit", "AudioData", "AudioEncoderConfig", "AudioDecoderConfig", "EncodedAudioChunk", "EncodedAudioChunkInit", "EncodedAudioChunkType", "CodecState", "MediaStreamTrackGenerator", "MediaStreamTrackGeneratorInit", "AudioContext", "AudioContextOptions", "MediaStream", "GainNode", "MediaStreamAudioSourceNode", "BaseAudioContext", "AudioDestinationNode"] } +web-sys = { version = "0.3.69", features = ["WebTransport", "console", "WebTransportOptions", "WebTransportBidirectionalStream", "WebTransportSendStream", "WebTransportReceiveStream", "Navigator", "MediaDevices", "AudioDecoder", "AudioDecoderInit", "AudioData", "AudioEncoderConfig", "AudioDecoderConfig", "EncodedAudioChunk", "EncodedAudioChunkInit", "EncodedAudioChunkType", "CodecState", "MediaStreamTrackGenerator", "MediaStreamTrackGeneratorInit", "AudioContext", "AudioContextOptions", "MediaStream", "GainNode", "MediaStreamAudioSourceNode", "BaseAudioContext", "AudioDestinationNode", "AudioWorkletNode", "AudioWorklet", "AudioWorkletProcessor", "MediaStreamConstraints", "WorkletOptions", "AudioEncoder", "AudioEncoderInit", "AudioDataInit"] } async-std = "1.12.0" +anyhow = "1.0.86" diff --git a/public/rust_mic_worklet.js b/public/rust_mic_worklet.js new file mode 100644 index 0000000..3ee9d4c --- /dev/null +++ b/public/rust_mic_worklet.js @@ -0,0 +1,63 @@ +const SAMPLE_RATE = 48000; +const PACKET_MS = 20; +const PACKET_FRAMES = PACKET_MS / 1000 * SAMPLE_RATE; +console.log("Frames per packet:", PACKET_FRAMES); + +class RustWorklet extends AudioWorkletProcessor { + constructor(options) { + super(); + this.module = options.processorOptions; + this.timestamp = null; + this.buffer = new Float32Array(PACKET_FRAMES); + this.buffer_offset = 0; + if (sampleRate != SAMPLE_RATE) { + throw Error(`sample rate ${sampleRate} should be ${SAMPLE_RATE}`); + } + + console.log('RustWorklet:', this); + + } + + do_send() { + const data = { + format: 'f32', + sampleRate: SAMPLE_RATE, + numberOfFrames: this.buffer_offset, + numberOfChannels: 1, + timestamp: this.timestamp, + data: this.buffer.slice(0, this.buffer_offset), + }; + this.port.postMessage(data); + this.buffer_offset = 0; + this.timestamp = null; + } + + process(inputs) { + const input = inputs[0]; + if (input.length == 0) { + return true + } + + if (this.timestamp == null) { + this.timestamp = currentFrame / SAMPLE_RATE * 1e6; + } + + const frames = input[0]; + + if (this.buffer_offset + frames.length > this.buffer.length) { + // too full, send now + this.do_send(); + } + + this.buffer.set(frames, this.buffer_offset); + this.buffer_offset += frames.length; + + if (this.buffer_offset + 128 > this.buffer.length) { + // full enough, send now + this.do_send(); + } + return true; + } +}; + +registerProcessor("rust_mic_worklet", RustWorklet); diff --git a/src/lib.rs b/src/lib.rs index f88c082..5532348 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,35 +1,48 @@ pub mod app; -use std::collections::HashMap; -use std::time::Duration; - +use anyhow::Error; use app::STATE; - +use async_std::channel::Sender; use futures::SinkExt; use futures::StreamExt; +use manganis::{file, mg}; use mumble_protocol::control::ControlPacket; use mumble_protocol::control::{msgs, ClientControlCodec}; +use mumble_protocol::voice::VoicePacket; use mumble_protocol::voice::VoicePacketDst; +use mumble_protocol::voice::VoicePacketPayload; +use std::collections::HashMap; +use std::time::Duration; use wasm_bindgen::prelude::*; -use wasm_bindgen_futures::JsFuture; +use wasm_bindgen_futures::spawn_local as spawn; +use wasm_bindgen_futures::{future_to_promise, JsFuture}; use web_sys::console; +use web_sys::js_sys::Promise; +use web_sys::js_sys::Reflect; use web_sys::js_sys::Uint8Array; +use web_sys::window; use web_sys::AudioContext; use web_sys::AudioContextOptions; use web_sys::AudioData; +use web_sys::AudioDataInit; use web_sys::AudioDecoder; use web_sys::AudioDecoderConfig; use web_sys::AudioDecoderInit; +use web_sys::AudioEncoder; +use web_sys::AudioEncoderConfig; +use web_sys::AudioEncoderInit; +use web_sys::AudioWorkletNode; use web_sys::EncodedAudioChunk; use web_sys::EncodedAudioChunkInit; use web_sys::EncodedAudioChunkType; use web_sys::MediaStream; +use web_sys::MediaStreamConstraints; use web_sys::MediaStreamTrackGenerator; use web_sys::MediaStreamTrackGeneratorInit; +use web_sys::MessageEvent; use web_sys::WebTransport; use web_sys::WebTransportOptions; - -use wasm_bindgen_futures::spawn_local as spawn; +use web_sys::WorkletOptions; // Borrowed from // https://github.com/security-union/videocall-rs/blob/main/videocall-client/src/decode/config.rs#L6 @@ -40,6 +53,108 @@ fn configure_audio_context() -> AudioContext { audio_context } +trait PromiseExt { + fn into_future(self) -> JsFuture; +} + +impl PromiseExt for Promise { + fn into_future(self) -> JsFuture { + self.into() + } +} + +async fn create_encoder_worklet( + audio_context: &AudioContext, + packets: Sender>, +) -> Result { + let stream = window() + .unwrap() + .navigator() + .media_devices()? + .get_user_media_with_constraints(MediaStreamConstraints::new().audio(&JsValue::TRUE))? + .into_future() + .await? + .dyn_into() + .map_err(|e| JsError::new(&format!("not a stream: {e:?}")))?; + + let options = WorkletOptions::new(); + Reflect::set( + &options, + &"processorOptions".into(), + &wasm_bindgen::module(), + )?; + + let module = "rust_mic_worklet.js"; + console::log_1(&format!("Loading mic worklet from {module:?}").into()); + audio_context + .audio_worklet()? + .add_module_with_options(module, &options)? + .into_future() + .await?; + + let source = audio_context.create_media_stream_source(&stream)?; + let worklet_node = AudioWorkletNode::new(audio_context, "rust_mic_worklet")?; + + let error: Closure = Closure::new(|e| console::error_1(&e)); + + // This knows what MediaStreamTrackGenerator to use as it closes around it + let mut sequence_num = 0; + let output: Closure = + Closure::new(move |audio_data: EncodedAudioChunk| { + let mut array = vec![0u8; audio_data.byte_length() as usize]; + audio_data.copy_to_with_u8_array(&mut array); + let _ = packets.try_send(ControlPacket::UDPTunnel(Box::new(VoicePacket::Audio { + _dst: std::marker::PhantomData, + target: 0, + session_id: (), + seq_num: sequence_num, + payload: VoicePacketPayload::Opus(array.into(), false), + position_info: None, + }))); + sequence_num = sequence_num.wrapping_add(1); + }); + + let audio_encoder = AudioEncoder::new(&AudioEncoderInit::new( + error.as_ref().unchecked_ref(), + output.as_ref().unchecked_ref(), + )) + .unwrap(); + + // This is required to prevent these from being deallocated + error.forget(); + output.forget(); + + audio_encoder.configure( + &AudioEncoderConfig::new("opus") + .number_of_channels(1) + .sample_rate(48000), + ); + console::log_1(&"Created Audio Encoder".into()); + + let onmessage: Closure = Closure::new(move |event: MessageEvent| { + match AudioData::new(event.data().unchecked_ref()) { + Ok(data) => { + audio_encoder.encode(&data); + } + Err(err) => { + console::error_1(&err); + console::debug_1(&event); + } + } + }); + Reflect::set( + &Reflect::get(&worklet_node, &"port".into())?, + &"onmessage".into(), + onmessage.as_ref(), + )?; + onmessage.forget(); + + source.connect_with_audio_node(&worklet_node)?; + worklet_node.connect_with_audio_node(&audio_context.destination())?; + + Ok(worklet_node) +} + fn create_decoder(audio_context: &AudioContext) -> AudioDecoder { let audio_stream_generator = MediaStreamTrackGenerator::new(&MediaStreamTrackGeneratorInit::new("audio")).unwrap(); @@ -50,17 +165,19 @@ fn create_decoder(audio_context: &AudioContext) -> AudioDecoder { let media_stream = MediaStream::new_with_tracks(&js_tracks).unwrap(); // Create MediaStreamAudioSourceNode - let audio_source = audio_context.create_media_stream_source(&media_stream).unwrap(); + let audio_source = audio_context + .create_media_stream_source(&media_stream) + .unwrap(); // Connect output of audio_source to audio_context (browser audio) - audio_source.connect_with_audio_node(&audio_context.destination()).unwrap(); - + audio_source + .connect_with_audio_node(&audio_context.destination()) + .unwrap(); // Create callback functions for AudioDecoder let error = Closure::wrap(Box::new(move |e: JsValue| { console::log_1(&e); }) as Box); - // This knows what MediaStreamTrackGenerator to use as it closes around it let output = Closure::wrap(Box::new(move |audio_data: AudioData| { let writable = audio_stream_generator.writable(); @@ -106,15 +223,19 @@ pub async fn network_entrypoint() { console::log_1(&"Rust via WASM!".into()); - let server_hash = vec![ - 14, 162, 111, 176, 34, 113, 218, 69, 177, 18, 13, 180, 232, 204, 49, 65, 161, 195, 36, 238, - 23, 95, 174, 190, 24, 216, 105, 89, 236, 147, 206, 139, - ]; + let Ok(server_hash): Result, _> = env!("WEBTRANSPORT_SERVER_HASH") + .trim_matches(&['[', ']']) + .split(',') + .map(|x| x.trim().parse()) + .collect() + else { + panic!("could not parse server hash") + }; let hash = web_sys::js_sys::Uint8Array::from(server_hash.as_slice()); let object = web_sys::js_sys::Object::new(); - web_sys::js_sys::Reflect::set( + Reflect::set( &object, &JsValue::from_str("algorithm"), &JsValue::from_str("sha-256"), @@ -227,10 +348,19 @@ pub async fn network_entrypoint() { }); } - // Create MediaStreams to playback decoded audio // The audio context is used to reproduce audio. let audio_context = configure_audio_context(); + + let audio_context_worklet = audio_context.clone(); + let packet_sender_worklet = send_chan.clone(); + spawn(async move { + match create_encoder_worklet(&audio_context_worklet, packet_sender_worklet).await { + Ok(node) => console::log_2(&"Created audio worklet:".into(), &node), + Err(err) => console::error_1(&err), + } + }); + // Create map of session_id -> AudioDecoder let mut decoder_map = HashMap::new(); @@ -249,17 +379,15 @@ pub async fn network_entrypoint() { position_info, } => { // Get or create audio decoder for this user - let audio_decoder = decoder_map.entry(session_id).or_insert_with(|| create_decoder(&audio_context)); + let audio_decoder = decoder_map + .entry(session_id) + .or_insert_with(|| create_decoder(&audio_context)); // This will over time (as users join and leave) leak // AudioDecoders, MediaStreamTrackGenerators, MediaStreams, and MediaStreamAudioSourceNodes. // A better way to handle this would be to delete and create all the audio // infra on channel join and update it as new users join the channel, dropping // any audio packets that come in the meantime. - if let mumble_protocol::voice::VoicePacketPayload::Opus( - audio_payload, - end_bit, - ) = payload - { + if let VoicePacketPayload::Opus(audio_payload, end_bit) = payload { let js_audio_payload = Uint8Array::from(audio_payload.as_ref()); audio_decoder.decode( &EncodedAudioChunk::new(&EncodedAudioChunkInit::new(