/* wearechat - generic multiplayer game with voip Copyright (C) 2025 metamuffin This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by the Free Software Foundation, version 3 of the License only. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with this program. If not, see . */ use std::{ collections::{HashMap, VecDeque}, sync::mpsc::{Receiver, SyncSender, sync_channel}, time::Instant, }; use anyhow::{Result, anyhow}; use audiopus::{ Application, Channels, SampleRate, coder::{Decoder, Encoder}, }; use cpal::{ Stream, traits::{DeviceTrait, HostTrait}, }; use glam::Vec3; use log::{debug, info, warn}; use nnnoiseless::{DenoiseState, RnnModel}; pub struct Audio { _instream: Stream, _outstream: Stream, rx: Receiver>, tx: SyncSender, } impl Audio { pub fn new() -> Result { let host = cpal::default_host(); let indev = host .default_input_device() .ok_or(anyhow!("no input device"))?; let outdev = host .default_output_device() .ok_or(anyhow!("no output device"))?; let mut inconf = indev.default_input_config()?.config(); inconf.channels = 1; inconf.sample_rate = cpal::SampleRate(48_000); inconf.buffer_size = cpal::BufferSize::Fixed(480); let mut outconf = outdev.default_input_config()?.config(); outconf.channels = 1; outconf.sample_rate = cpal::SampleRate(48_000); outconf.buffer_size = cpal::BufferSize::Fixed(480); let (mut aenc, rx) = AEncoder::new()?; let (mut adec, tx) = ADecoder::new()?; let instream = indev.build_input_stream( &inconf, move |samples: &[f32], _| { if let Err(e) = aenc.data(samples) { warn!("encoder error: {e}"); } }, |err| warn!("audio input error: {err}"), None, )?; let outstream = outdev.build_output_stream( &outconf, move |samples: &mut [f32], _| { if let Err(e) = adec.data(samples) { warn!("decoder error: {e}"); } }, |err| warn!("audio output error: {err}"), None, )?; Ok(Self { _instream: instream, _outstream: outstream, rx, tx, }) } pub fn pop_output(&mut self) -> Option> { self.rx.try_recv().ok() } pub fn incoming_packet(&mut self, channel: u128, pos: Vec3, data: Vec) { if let Err(e) = self.tx.send(APlayPacket { pos, data, channel }) { warn!("audio output buffer overflow: {e:?}") } } } pub struct APlayPacket { pos: Vec3, channel: u128, data: Vec, } const AE_FRAME_SIZE: usize = 480; pub struct AEncoder { encoder: Encoder, sender: SyncSender>, buffer: VecDeque, noise_rnn: DenoiseState<'static>, trigger: VadTrigger, } struct VadTrigger { last_sig: Instant, transmitting: bool, } impl AEncoder { pub fn new() -> Result<(Self, Receiver>)> { let (sender, rx) = sync_channel(1024); Ok(( Self { noise_rnn: *DenoiseState::from_model(RnnModel::default()), encoder: Encoder::new(SampleRate::Hz48000, Channels::Mono, Application::Voip)?, sender, buffer: VecDeque::new(), trigger: VadTrigger { last_sig: Instant::now(), transmitting: false, }, }, rx, )) } pub fn data(&mut self, samples: &[f32]) -> Result<()> { self.buffer.extend(samples); while self.buffer.len() >= AE_FRAME_SIZE { let mut out = [0u8; AE_FRAME_SIZE]; let mut denoise = [0f32; AE_FRAME_SIZE]; let mut raw = [0f32; AE_FRAME_SIZE]; for i in 0..AE_FRAME_SIZE { raw[i] = self.buffer.pop_front().unwrap() * 32768.0; } self.noise_rnn.process_frame(&mut denoise, &raw); for e in &mut denoise { *e /= 32768.0; } let energy = measure_energy(&denoise); let (tx, end_tx) = self.trigger.update(energy); if tx { let size = self.encoder.encode_float(&denoise, &mut out)?; let _ = self.sender.try_send(out[..size].to_vec()); } if end_tx { // TODO send end frame } } Ok(()) } } impl VadTrigger { pub fn update(&mut self, energy: f32) -> (bool, bool) { debug!("E={energy:.02}"); let now = Instant::now(); if energy > 1. { self.last_sig = now; } let last_sig_elapsed = (now - self.last_sig).as_secs_f32(); let prev_transmitting = self.transmitting; self.transmitting = last_sig_elapsed < 0.5; match (prev_transmitting, self.transmitting) { (false, true) => info!("start transmit"), (true, false) => info!("end transmit"), _ => (), } (self.transmitting, prev_transmitting && !self.transmitting) } } const BUFFER_SIZE: usize = 48_000; pub struct ADecoder { decoder: Decoder, receiver: Receiver, channels: HashMap, playback: usize, buffer: Box<[[f32; 2]; BUFFER_SIZE]>, } impl ADecoder { pub fn new() -> Result<(Self, SyncSender)> { let (tx, receiver) = sync_channel(1024); Ok(( Self { decoder: Decoder::new(SampleRate::Hz48000, Channels::Mono)?, receiver, channels: HashMap::new(), playback: 0, buffer: unsafe { Box::new_zeroed().assume_init() }, }, tx, )) } pub fn data(&mut self, samples: &mut [f32]) -> Result<()> { while self.buffer.len() < samples.len() { if let Ok(p) = self.receiver.try_recv() { let mut output = [0f32; AE_FRAME_SIZE]; let size = self.decoder.decode_float( Some(p.data.as_slice()), output.as_mut_slice(), false, )?; let channel_cursor = self.channels.entry(p.channel).or_insert(self.playback); let free_space = *channel_cursor - self.playback; for i in 0..size.min(free_space) { // TODO positional audio let _ = p.pos; self.buffer[*channel_cursor] = [output[i], output[i]]; *channel_cursor += 1; *channel_cursor %= BUFFER_SIZE } } else { break; } } for x in samples.array_chunks_mut::<2>() { *x = self.buffer[self.playback]; self.buffer[self.playback] = [0.; 2]; self.playback += 1; self.playback %= BUFFER_SIZE; } Ok(()) } } fn measure_energy(samples: &[f32]) -> f32 { let mut e = 0.; for s in samples { e += *s * *s; } e }