From 44220f3ff1015c056730c8ee4c95cb55a0759abc Mon Sep 17 00:00:00 2001 From: metamuffin Date: Thu, 16 Jan 2025 12:23:05 +0100 Subject: rnnoise and vad --- Cargo.lock | 308 ++++++++++++++++++++++++++++++++++++++++++++++++++-- client/Cargo.toml | 1 + client/src/audio.rs | 76 +++++++++++-- 3 files changed, 365 insertions(+), 20 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6f14876..e73b669 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -215,6 +215,17 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi 0.1.19", + "libc", + "winapi", +] + [[package]] name = "audiopus" version = "0.2.0" @@ -504,6 +515,22 @@ dependencies = [ "libloading", ] +[[package]] +name = "clap" +version = "3.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123" +dependencies = [ + "atty", + "bitflags 1.3.2", + "clap_lex 0.2.4", + "indexmap 1.9.3", + "once_cell", + "strsim 0.10.0", + "termcolor", + "textwrap", +] + [[package]] name = "clap" version = "4.5.23" @@ -522,8 +549,8 @@ checksum = "30582fc632330df2bd26877bde0c1f4470d57c582bbc070376afcd04d8cb4838" dependencies = [ "anstream", "anstyle", - "clap_lex", - "strsim", + "clap_lex 0.7.4", + "strsim 0.11.1", ] [[package]] @@ -538,6 +565,15 @@ dependencies = [ "syn", ] +[[package]] +name = "clap_lex" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5" +dependencies = [ + "os_str_bytes", +] + [[package]] name = "clap_lex" version = "0.7.4" @@ -720,12 +756,125 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96a6ac251f4a2aca6b3f91340350eab87ae57c3f127ffeb585e92bd336717991" +[[package]] +name = "dasp" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7381b67da416b639690ac77c73b86a7b5e64a29e31d1f75fb3b1102301ef355a" +dependencies = [ + "dasp_envelope", + "dasp_frame", + "dasp_interpolate", + "dasp_peak", + "dasp_ring_buffer", + "dasp_rms", + "dasp_sample", + "dasp_signal", + "dasp_slice", + "dasp_window", +] + +[[package]] +name = "dasp_envelope" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ec617ce7016f101a87fe85ed44180839744265fae73bb4aa43e7ece1b7668b6" +dependencies = [ + "dasp_frame", + "dasp_peak", + "dasp_ring_buffer", + "dasp_rms", + "dasp_sample", +] + +[[package]] +name = "dasp_frame" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a3937f5fe2135702897535c8d4a5553f8b116f76c1529088797f2eee7c5cd6" +dependencies = [ + "dasp_sample", +] + +[[package]] +name = "dasp_interpolate" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fc975a6563bb7ca7ec0a6c784ead49983a21c24835b0bc96eea11ee407c7486" +dependencies = [ + "dasp_frame", + "dasp_ring_buffer", + "dasp_sample", +] + +[[package]] +name = "dasp_peak" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cf88559d79c21f3d8523d91250c397f9a15b5fc72fbb3f87fdb0a37b79915bf" +dependencies = [ + "dasp_frame", + "dasp_sample", +] + +[[package]] +name = "dasp_ring_buffer" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07d79e19b89618a543c4adec9c5a347fe378a19041699b3278e616e387511ea1" + +[[package]] +name = "dasp_rms" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6c5dcb30b7e5014486e2822537ea2beae50b19722ffe2ed7549ab03774575aa" +dependencies = [ + "dasp_frame", + "dasp_ring_buffer", + "dasp_sample", +] + [[package]] name = "dasp_sample" version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c87e182de0887fd5361989c677c4e8f5000cd9491d6d563161a8f3a5519fc7f" +[[package]] +name = "dasp_signal" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa1ab7d01689c6ed4eae3d38fe1cea08cba761573fbd2d592528d55b421077e7" +dependencies = [ + "dasp_envelope", + "dasp_frame", + "dasp_interpolate", + "dasp_peak", + "dasp_ring_buffer", + "dasp_rms", + "dasp_sample", + "dasp_window", +] + +[[package]] +name = "dasp_slice" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e1c7335d58e7baedafa516cb361360ff38d6f4d3f9d9d5ee2a2fc8e27178fa1" +dependencies = [ + "dasp_frame", + "dasp_sample", +] + +[[package]] +name = "dasp_window" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99ded7b88821d2ce4e8b842c9f1c86ac911891ab89443cc1de750cae764c5076" +dependencies = [ + "dasp_sample", +] + [[package]] name = "dispatch" version = "0.2.0" @@ -1119,7 +1268,7 @@ checksum = "dcf29e94d6d243368b7a56caa16bc213e4f9f8ed38c4d9557069527b5d5281ca" dependencies = [ "bitflags 2.6.0", "gpu-descriptor-types", - "hashbrown", + "hashbrown 0.15.2", ] [[package]] @@ -1141,6 +1290,12 @@ dependencies = [ "crunchy", ] +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + [[package]] name = "hashbrown" version = "0.15.2" @@ -1156,6 +1311,15 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + [[package]] name = "hermit-abi" version = "0.4.0" @@ -1168,6 +1332,12 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dfa686283ad6dd069f105e5ab091b04c62850d3e4cf5d67debad1933f55023df" +[[package]] +name = "hound" +version = "3.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62adaabb884c94955b19907d60019f4e145d091c75345379e70d1ee696f7854f" + [[package]] name = "humansize" version = "2.1.3" @@ -1222,6 +1392,16 @@ version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0263a3d970d5c054ed9312c0057b4f3bde9c0b33836d3637361d4a9e6e7a408" +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", +] + [[package]] name = "indexmap" version = "2.7.0" @@ -1229,7 +1409,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "62f822373a4fe84d4bb149bf54e584a7f4abec90e072ed49cda0edea5b95471f" dependencies = [ "equivalent", - "hashbrown", + "hashbrown 0.15.2", ] [[package]] @@ -1521,7 +1701,7 @@ dependencies = [ "cfg_aliases 0.1.1", "codespan-reporting", "hexf-parse", - "indexmap", + "indexmap 2.7.0", "log", "rustc-hash", "spirv", @@ -1589,6 +1769,22 @@ version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" +[[package]] +name = "nnnoiseless" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23d377ce2fb579ed5c14cfa0d39e70849030fdf673d6d1a764cadb2dfbb02a50" +dependencies = [ + "anyhow", + "clap 3.2.25", + "dasp", + "dasp_interpolate", + "dasp_ring_buffer", + "hound", + "once_cell", + "rustfft", +] + [[package]] name = "nohash-hasher" version = "0.2.0" @@ -1621,6 +1817,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + [[package]] name = "num-derive" version = "0.4.2" @@ -1932,6 +2137,12 @@ dependencies = [ "libredox", ] +[[package]] +name = "os_str_bytes" +version = "6.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1" + [[package]] name = "owned_ttf_parser" version = "0.25.0" @@ -2029,7 +2240,7 @@ checksum = "a604568c3202727d1507653cb121dbd627a58684eb09a820fd746bee38b4442f" dependencies = [ "cfg-if", "concurrent-queue", - "hermit-abi", + "hermit-abi 0.4.0", "pin-project-lite", "rustix", "tracing", @@ -2057,6 +2268,15 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8cf8e6a8aa66ce33f63993ffc4ea4271eb5b0530a9002db8455ea6050c77bfa" +[[package]] +name = "primal-check" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc0d895b311e3af9902528fbb8f928688abbd95872819320517cc24ca6b2bd08" +dependencies = [ + "num-integer", +] + [[package]] name = "proc-macro-crate" version = "3.2.0" @@ -2344,6 +2564,21 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" +[[package]] +name = "rustfft" +version = "6.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43806561bc506d0c5d160643ad742e3161049ac01027b5e6d7524091fd401d86" +dependencies = [ + "num-complex", + "num-integer", + "num-traits", + "primal-check", + "strength_reduce", + "transpose", + "version_check", +] + [[package]] name = "rustix" version = "0.38.42" @@ -2532,12 +2767,24 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "strength_reduce" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe895eb47f22e2ddd4dabc02bce419d2e643c8e3b585c78158b349195bc24d82" + [[package]] name = "strict-num" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6637bab7722d379c8b41ba849228d680cc12d0a45ba1fa2b48f2a30577a06731" +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + [[package]] name = "strsim" version = "0.11.1" @@ -2583,6 +2830,12 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "textwrap" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23d434d3f8967a09480fb04132ebe0a3e088c173e6d0ee7897abbdf4eab0f8b9" + [[package]] name = "thiserror" version = "1.0.69" @@ -2666,7 +2919,7 @@ version = "0.22.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" dependencies = [ - "indexmap", + "indexmap 2.7.0", "serde", "serde_spanned", "toml_datetime", @@ -2689,6 +2942,16 @@ version = "0.1.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c" +[[package]] +name = "transpose" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad61aed86bc3faea4300c7aee358b4c6d0c8d6ccc36524c96e4c92ccf26e77e" +dependencies = [ + "num-integer", + "strength_reduce", +] + [[package]] name = "ttf-parser" version = "0.25.1" @@ -2971,7 +3234,7 @@ dependencies = [ "anyhow", "audiopus", "bytemuck", - "clap", + "clap 4.5.23", "cpal", "egui", "egui-wgpu", @@ -2979,6 +3242,7 @@ dependencies = [ "glam", "image", "log", + "nnnoiseless", "pollster", "rand 0.9.0-beta.1", "weareshared", @@ -2992,7 +3256,7 @@ name = "weareserver" version = "0.1.0" dependencies = [ "anyhow", - "clap", + "clap 4.5.23", "env_logger", "log", "weareshared", @@ -3018,7 +3282,7 @@ name = "weareworld" version = "0.1.0" dependencies = [ "anyhow", - "clap", + "clap 4.5.23", "env_logger", "gltf", "humansize", @@ -3091,7 +3355,7 @@ dependencies = [ "bitflags 2.6.0", "cfg_aliases 0.1.1", "document-features", - "indexmap", + "indexmap 2.7.0", "log", "naga", "once_cell", @@ -3161,6 +3425,22 @@ dependencies = [ "web-sys", ] +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + [[package]] name = "winapi-util" version = "0.1.9" @@ -3170,6 +3450,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows" version = "0.54.0" diff --git a/client/Cargo.toml b/client/Cargo.toml index 77149b2..c98374a 100644 --- a/client/Cargo.toml +++ b/client/Cargo.toml @@ -21,3 +21,4 @@ egui-wgpu = "0.30.0" egui = { version = "0.30.0", features = ["bytemuck"] } bytemuck = "1.21.0" xdg = "2.5.2" +nnnoiseless = "0.5.1" diff --git a/client/src/audio.rs b/client/src/audio.rs index 5751c0e..f93466d 100644 --- a/client/src/audio.rs +++ b/client/src/audio.rs @@ -17,6 +17,7 @@ use std::{ collections::{HashMap, VecDeque}, sync::mpsc::{Receiver, SyncSender, sync_channel}, + time::Instant, }; use anyhow::{Result, anyhow}; @@ -29,7 +30,8 @@ use cpal::{ traits::{DeviceTrait, HostTrait}, }; use glam::Vec3; -use log::warn; +use log::{debug, info, warn}; +use nnnoiseless::{DenoiseState, RnnModel}; pub struct Audio { _instream: Stream, @@ -105,38 +107,86 @@ pub struct APlayPacket { data: Vec, } +const AE_FRAME_SIZE: usize = 480; + pub struct AEncoder { encoder: Encoder, sender: SyncSender>, buffer: VecDeque, + noise_rnn: DenoiseState<'static>, + trigger: VadTrigger, +} + +struct VadTrigger { + last_sig: Instant, + transmitting: bool, } + impl AEncoder { pub fn new() -> Result<(Self, Receiver>)> { let (sender, rx) = sync_channel(1024); Ok(( Self { + noise_rnn: *DenoiseState::from_model(RnnModel::default()), encoder: Encoder::new(SampleRate::Hz48000, Channels::Mono, Application::Voip)?, sender, buffer: VecDeque::new(), + trigger: VadTrigger { + last_sig: Instant::now(), + transmitting: false, + }, }, rx, )) } pub fn data(&mut self, samples: &[f32]) -> Result<()> { self.buffer.extend(samples); - while self.buffer.len() >= 120 { - let mut out = [0u8; 120]; - let mut inp = [0f32; 120]; - for i in 0..120 { - inp[i] = self.buffer.pop_front().unwrap(); + while self.buffer.len() >= AE_FRAME_SIZE { + let mut out = [0u8; AE_FRAME_SIZE]; + let mut denoise = [0f32; AE_FRAME_SIZE]; + let mut raw = [0f32; AE_FRAME_SIZE]; + for i in 0..AE_FRAME_SIZE { + raw[i] = self.buffer.pop_front().unwrap() * 32768.0; + } + self.noise_rnn.process_frame(&mut denoise, &raw); + for e in &mut denoise { + *e /= 32768.0; + } + let energy = measure_energy(&denoise); + let (tx, end_tx) = self.trigger.update(energy); + if tx { + let size = self.encoder.encode_float(&denoise, &mut out)?; + let _ = self.sender.try_send(out[..size].to_vec()); + } + if end_tx { + // TODO send end frame } - let size = self.encoder.encode_float(&inp, &mut out)?; - let _ = self.sender.try_send(out[..size].to_vec()); } Ok(()) } } +impl VadTrigger { + pub fn update(&mut self, energy: f32) -> (bool, bool) { + debug!("E={energy:.02}"); + let now = Instant::now(); + if energy > 1. { + self.last_sig = now; + } + let last_sig_elapsed = (now - self.last_sig).as_secs_f32(); + let prev_transmitting = self.transmitting; + self.transmitting = last_sig_elapsed < 0.5; + + match (prev_transmitting, self.transmitting) { + (false, true) => info!("start transmit"), + (true, false) => info!("end transmit"), + _ => (), + } + + (self.transmitting, prev_transmitting && !self.transmitting) + } +} + const BUFFER_SIZE: usize = 48_000; pub struct ADecoder { decoder: Decoder, @@ -162,7 +212,7 @@ impl ADecoder { pub fn data(&mut self, samples: &mut [f32]) -> Result<()> { while self.buffer.len() < samples.len() { if let Ok(p) = self.receiver.try_recv() { - let mut output = [0f32; 120]; + let mut output = [0f32; AE_FRAME_SIZE]; let size = self.decoder.decode_float( Some(p.data.as_slice()), output.as_mut_slice(), @@ -191,3 +241,11 @@ impl ADecoder { Ok(()) } } + +fn measure_energy(samples: &[f32]) -> f32 { + let mut e = 0.; + for s in samples { + e += *s * *s; + } + e +} -- cgit v1.2.3-70-g09d2