diff options
| -rw-r--r-- | Cargo.lock | 308 | ||||
| -rw-r--r-- | client/Cargo.toml | 1 | ||||
| -rw-r--r-- | client/src/audio.rs | 76 |
3 files changed, 365 insertions, 20 deletions
@@ -216,6 +216,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" [[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi 0.1.19", + "libc", + "winapi", +] + +[[package]] name = "audiopus" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -506,6 +517,22 @@ dependencies = [ [[package]] name = "clap" +version = "3.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123" +dependencies = [ + "atty", + "bitflags 1.3.2", + "clap_lex 0.2.4", + "indexmap 1.9.3", + "once_cell", + "strsim 0.10.0", + "termcolor", + "textwrap", +] + +[[package]] +name = "clap" version = "4.5.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3135e7ec2ef7b10c6ed8950f0f792ed96ee093fa088608f1c76e569722700c84" @@ -522,8 +549,8 @@ checksum = "30582fc632330df2bd26877bde0c1f4470d57c582bbc070376afcd04d8cb4838" dependencies = [ "anstream", "anstyle", - "clap_lex", - "strsim", + "clap_lex 0.7.4", + "strsim 0.11.1", ] [[package]] @@ -540,6 +567,15 @@ dependencies = [ [[package]] name = "clap_lex" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5" +dependencies = [ + "os_str_bytes", +] + +[[package]] +name = "clap_lex" version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" @@ -721,12 +757,125 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96a6ac251f4a2aca6b3f91340350eab87ae57c3f127ffeb585e92bd336717991" [[package]] +name = "dasp" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7381b67da416b639690ac77c73b86a7b5e64a29e31d1f75fb3b1102301ef355a" +dependencies = [ + "dasp_envelope", + "dasp_frame", + "dasp_interpolate", + "dasp_peak", + "dasp_ring_buffer", + "dasp_rms", + "dasp_sample", + "dasp_signal", + "dasp_slice", + "dasp_window", +] + +[[package]] +name = "dasp_envelope" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ec617ce7016f101a87fe85ed44180839744265fae73bb4aa43e7ece1b7668b6" +dependencies = [ + "dasp_frame", + "dasp_peak", + "dasp_ring_buffer", + "dasp_rms", + "dasp_sample", +] + +[[package]] +name = "dasp_frame" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a3937f5fe2135702897535c8d4a5553f8b116f76c1529088797f2eee7c5cd6" +dependencies = [ + "dasp_sample", +] + +[[package]] +name = "dasp_interpolate" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fc975a6563bb7ca7ec0a6c784ead49983a21c24835b0bc96eea11ee407c7486" +dependencies = [ + "dasp_frame", + "dasp_ring_buffer", + "dasp_sample", +] + +[[package]] +name = "dasp_peak" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cf88559d79c21f3d8523d91250c397f9a15b5fc72fbb3f87fdb0a37b79915bf" +dependencies = [ + "dasp_frame", + "dasp_sample", +] + +[[package]] +name = "dasp_ring_buffer" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07d79e19b89618a543c4adec9c5a347fe378a19041699b3278e616e387511ea1" + +[[package]] +name = "dasp_rms" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6c5dcb30b7e5014486e2822537ea2beae50b19722ffe2ed7549ab03774575aa" +dependencies = [ + "dasp_frame", + "dasp_ring_buffer", + "dasp_sample", +] + +[[package]] name = "dasp_sample" version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c87e182de0887fd5361989c677c4e8f5000cd9491d6d563161a8f3a5519fc7f" [[package]] +name = "dasp_signal" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa1ab7d01689c6ed4eae3d38fe1cea08cba761573fbd2d592528d55b421077e7" +dependencies = [ + "dasp_envelope", + "dasp_frame", + "dasp_interpolate", + "dasp_peak", + "dasp_ring_buffer", + "dasp_rms", + "dasp_sample", + "dasp_window", +] + +[[package]] +name = "dasp_slice" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e1c7335d58e7baedafa516cb361360ff38d6f4d3f9d9d5ee2a2fc8e27178fa1" +dependencies = [ + "dasp_frame", + "dasp_sample", +] + +[[package]] +name = "dasp_window" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99ded7b88821d2ce4e8b842c9f1c86ac911891ab89443cc1de750cae764c5076" +dependencies = [ + "dasp_sample", +] + +[[package]] name = "dispatch" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1119,7 +1268,7 @@ checksum = "dcf29e94d6d243368b7a56caa16bc213e4f9f8ed38c4d9557069527b5d5281ca" dependencies = [ "bitflags 2.6.0", "gpu-descriptor-types", - "hashbrown", + "hashbrown 0.15.2", ] [[package]] @@ -1143,6 +1292,12 @@ dependencies = [ [[package]] name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "hashbrown" version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" @@ -1158,6 +1313,15 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "hermit-abi" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fbf6a919d6cf397374f7dfeeea91d974c7c0a7221d0d0f4f20d859d329e53fcc" @@ -1169,6 +1333,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dfa686283ad6dd069f105e5ab091b04c62850d3e4cf5d67debad1933f55023df" [[package]] +name = "hound" +version = "3.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62adaabb884c94955b19907d60019f4e145d091c75345379e70d1ee696f7854f" + +[[package]] name = "humansize" version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1224,12 +1394,22 @@ checksum = "d0263a3d970d5c054ed9312c0057b4f3bde9c0b33836d3637361d4a9e6e7a408" [[package]] name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", +] + +[[package]] +name = "indexmap" version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "62f822373a4fe84d4bb149bf54e584a7f4abec90e072ed49cda0edea5b95471f" dependencies = [ "equivalent", - "hashbrown", + "hashbrown 0.15.2", ] [[package]] @@ -1521,7 +1701,7 @@ dependencies = [ "cfg_aliases 0.1.1", "codespan-reporting", "hexf-parse", - "indexmap", + "indexmap 2.7.0", "log", "rustc-hash", "spirv", @@ -1590,6 +1770,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" [[package]] +name = "nnnoiseless" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23d377ce2fb579ed5c14cfa0d39e70849030fdf673d6d1a764cadb2dfbb02a50" +dependencies = [ + "anyhow", + "clap 3.2.25", + "dasp", + "dasp_interpolate", + "dasp_ring_buffer", + "hound", + "once_cell", + "rustfft", +] + +[[package]] name = "nohash-hasher" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1622,6 +1818,15 @@ dependencies = [ ] [[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] name = "num-derive" version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1933,6 +2138,12 @@ dependencies = [ ] [[package]] +name = "os_str_bytes" +version = "6.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1" + +[[package]] name = "owned_ttf_parser" version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -2029,7 +2240,7 @@ checksum = "a604568c3202727d1507653cb121dbd627a58684eb09a820fd746bee38b4442f" dependencies = [ "cfg-if", "concurrent-queue", - "hermit-abi", + "hermit-abi 0.4.0", "pin-project-lite", "rustix", "tracing", @@ -2058,6 +2269,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8cf8e6a8aa66ce33f63993ffc4ea4271eb5b0530a9002db8455ea6050c77bfa" [[package]] +name = "primal-check" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc0d895b311e3af9902528fbb8f928688abbd95872819320517cc24ca6b2bd08" +dependencies = [ + "num-integer", +] + +[[package]] name = "proc-macro-crate" version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -2345,6 +2565,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" [[package]] +name = "rustfft" +version = "6.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43806561bc506d0c5d160643ad742e3161049ac01027b5e6d7524091fd401d86" +dependencies = [ + "num-complex", + "num-integer", + "num-traits", + "primal-check", + "strength_reduce", + "transpose", + "version_check", +] + +[[package]] name = "rustix" version = "0.38.42" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -2533,6 +2768,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] +name = "strength_reduce" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe895eb47f22e2ddd4dabc02bce419d2e643c8e3b585c78158b349195bc24d82" + +[[package]] name = "strict-num" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -2540,6 +2781,12 @@ checksum = "6637bab7722d379c8b41ba849228d680cc12d0a45ba1fa2b48f2a30577a06731" [[package]] name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "strsim" version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" @@ -2584,6 +2831,12 @@ dependencies = [ ] [[package]] +name = "textwrap" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23d434d3f8967a09480fb04132ebe0a3e088c173e6d0ee7897abbdf4eab0f8b9" + +[[package]] name = "thiserror" version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -2666,7 +2919,7 @@ version = "0.22.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" dependencies = [ - "indexmap", + "indexmap 2.7.0", "serde", "serde_spanned", "toml_datetime", @@ -2690,6 +2943,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c" [[package]] +name = "transpose" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad61aed86bc3faea4300c7aee358b4c6d0c8d6ccc36524c96e4c92ccf26e77e" +dependencies = [ + "num-integer", + "strength_reduce", +] + +[[package]] name = "ttf-parser" version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -2971,7 +3234,7 @@ dependencies = [ "anyhow", "audiopus", "bytemuck", - "clap", + "clap 4.5.23", "cpal", "egui", "egui-wgpu", @@ -2979,6 +3242,7 @@ dependencies = [ "glam", "image", "log", + "nnnoiseless", "pollster", "rand 0.9.0-beta.1", "weareshared", @@ -2992,7 +3256,7 @@ name = "weareserver" version = "0.1.0" dependencies = [ "anyhow", - "clap", + "clap 4.5.23", "env_logger", "log", "weareshared", @@ -3018,7 +3282,7 @@ name = "weareworld" version = "0.1.0" dependencies = [ "anyhow", - "clap", + "clap 4.5.23", "env_logger", "gltf", "humansize", @@ -3091,7 +3355,7 @@ dependencies = [ "bitflags 2.6.0", "cfg_aliases 0.1.1", "document-features", - "indexmap", + "indexmap 2.7.0", "log", "naga", "once_cell", @@ -3162,6 +3426,22 @@ dependencies = [ ] [[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] name = "winapi-util" version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -3171,6 +3451,12 @@ dependencies = [ ] [[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] name = "windows" version = "0.54.0" source = "registry+https://github.com/rust-lang/crates.io-index" diff --git a/client/Cargo.toml b/client/Cargo.toml index 77149b2..c98374a 100644 --- a/client/Cargo.toml +++ b/client/Cargo.toml @@ -21,3 +21,4 @@ egui-wgpu = "0.30.0" egui = { version = "0.30.0", features = ["bytemuck"] } bytemuck = "1.21.0" xdg = "2.5.2" +nnnoiseless = "0.5.1" diff --git a/client/src/audio.rs b/client/src/audio.rs index 5751c0e..f93466d 100644 --- a/client/src/audio.rs +++ b/client/src/audio.rs @@ -17,6 +17,7 @@ use std::{ collections::{HashMap, VecDeque}, sync::mpsc::{Receiver, SyncSender, sync_channel}, + time::Instant, }; use anyhow::{Result, anyhow}; @@ -29,7 +30,8 @@ use cpal::{ traits::{DeviceTrait, HostTrait}, }; use glam::Vec3; -use log::warn; +use log::{debug, info, warn}; +use nnnoiseless::{DenoiseState, RnnModel}; pub struct Audio { _instream: Stream, @@ -105,38 +107,86 @@ pub struct APlayPacket { data: Vec<u8>, } +const AE_FRAME_SIZE: usize = 480; + pub struct AEncoder { encoder: Encoder, sender: SyncSender<Vec<u8>>, buffer: VecDeque<f32>, + noise_rnn: DenoiseState<'static>, + trigger: VadTrigger, +} + +struct VadTrigger { + last_sig: Instant, + transmitting: bool, } + impl AEncoder { pub fn new() -> Result<(Self, Receiver<Vec<u8>>)> { let (sender, rx) = sync_channel(1024); Ok(( Self { + noise_rnn: *DenoiseState::from_model(RnnModel::default()), encoder: Encoder::new(SampleRate::Hz48000, Channels::Mono, Application::Voip)?, sender, buffer: VecDeque::new(), + trigger: VadTrigger { + last_sig: Instant::now(), + transmitting: false, + }, }, rx, )) } pub fn data(&mut self, samples: &[f32]) -> Result<()> { self.buffer.extend(samples); - while self.buffer.len() >= 120 { - let mut out = [0u8; 120]; - let mut inp = [0f32; 120]; - for i in 0..120 { - inp[i] = self.buffer.pop_front().unwrap(); + while self.buffer.len() >= AE_FRAME_SIZE { + let mut out = [0u8; AE_FRAME_SIZE]; + let mut denoise = [0f32; AE_FRAME_SIZE]; + let mut raw = [0f32; AE_FRAME_SIZE]; + for i in 0..AE_FRAME_SIZE { + raw[i] = self.buffer.pop_front().unwrap() * 32768.0; + } + self.noise_rnn.process_frame(&mut denoise, &raw); + for e in &mut denoise { + *e /= 32768.0; + } + let energy = measure_energy(&denoise); + let (tx, end_tx) = self.trigger.update(energy); + if tx { + let size = self.encoder.encode_float(&denoise, &mut out)?; + let _ = self.sender.try_send(out[..size].to_vec()); + } + if end_tx { + // TODO send end frame } - let size = self.encoder.encode_float(&inp, &mut out)?; - let _ = self.sender.try_send(out[..size].to_vec()); } Ok(()) } } +impl VadTrigger { + pub fn update(&mut self, energy: f32) -> (bool, bool) { + debug!("E={energy:.02}"); + let now = Instant::now(); + if energy > 1. { + self.last_sig = now; + } + let last_sig_elapsed = (now - self.last_sig).as_secs_f32(); + let prev_transmitting = self.transmitting; + self.transmitting = last_sig_elapsed < 0.5; + + match (prev_transmitting, self.transmitting) { + (false, true) => info!("start transmit"), + (true, false) => info!("end transmit"), + _ => (), + } + + (self.transmitting, prev_transmitting && !self.transmitting) + } +} + const BUFFER_SIZE: usize = 48_000; pub struct ADecoder { decoder: Decoder, @@ -162,7 +212,7 @@ impl ADecoder { pub fn data(&mut self, samples: &mut [f32]) -> Result<()> { while self.buffer.len() < samples.len() { if let Ok(p) = self.receiver.try_recv() { - let mut output = [0f32; 120]; + let mut output = [0f32; AE_FRAME_SIZE]; let size = self.decoder.decode_float( Some(p.data.as_slice()), output.as_mut_slice(), @@ -191,3 +241,11 @@ impl ADecoder { Ok(()) } } + +fn measure_energy(samples: &[f32]) -> f32 { + let mut e = 0.; + for s in samples { + e += *s * *s; + } + e +} |