audio loopback with opus encoder

3 files changed, 161 insertions, 1 deletions

diff --git a/client/src/audio.rs b/client/src/audio.rs
new file mode 100644
index 0000000..c72c6b3
--- /dev/null
+++ b/client/src/audio.rs
@@ -0,0 +1,153 @@
+use std::{
+    collections::VecDeque,
+    sync::mpsc::{Receiver, SyncSender, sync_channel},
+};
+
+use anyhow::{Result, anyhow};
+use audiopus::{
+    Application, Channels, SampleRate,
+    coder::{Decoder, Encoder},
+};
+use cpal::{
+    Stream,
+    traits::{DeviceTrait, HostTrait},
+};
+use log::warn;
+
+pub struct Audio {
+    _instream: Stream,
+    _outstream: Stream,
+    rx: Receiver<Vec<u8>>,
+    tx: SyncSender<Vec<u8>>,
+}
+impl Audio {
+    pub fn new() -> Result<Self> {
+        let host = cpal::default_host();
+
+        let indev = host
+            .default_input_device()
+            .ok_or(anyhow!("no input device"))?;
+        let outdev = host
+            .default_output_device()
+            .ok_or(anyhow!("no output device"))?;
+
+        let mut inconf = indev.default_input_config()?.config();
+        inconf.channels = 1;
+        inconf.sample_rate = cpal::SampleRate(48_000);
+        inconf.buffer_size = cpal::BufferSize::Fixed(480);
+
+        let mut outconf = outdev.default_input_config()?.config();
+        outconf.channels = 1;
+        outconf.sample_rate = cpal::SampleRate(48_000);
+        outconf.buffer_size = cpal::BufferSize::Fixed(480);
+
+        let (mut aenc, rx) = AEncoder::new()?;
+        let (mut adec, tx) = ADecoder::new()?;
+
+        let instream = indev.build_input_stream(
+            &inconf,
+            move |samples: &[f32], _| {
+                if let Err(e) = aenc.data(samples) {
+                    warn!("encoder error: {e}");
+                }
+            },
+            |err| warn!("audio input error: {err}"),
+            None,
+        )?;
+        let outstream = outdev.build_output_stream(
+            &outconf,
+            move |samples: &mut [f32], _| {
+                if let Err(e) = adec.data(samples) {
+                    warn!("decoder error: {e}");
+                }
+            },
+            |err| warn!("audio output error: {err}"),
+            None,
+        )?;
+
+        Ok(Self {
+            _instream: instream,
+            _outstream: outstream,
+            rx,
+            tx,
+        })
+    }
+    pub fn update(&mut self) {
+        for e in self.rx.try_iter() {
+            let _ = self.tx.try_send(e);
+        }
+    }
+}
+
+pub struct AEncoder {
+    encoder: Encoder,
+    sender: SyncSender<Vec<u8>>,
+    buffer: VecDeque<f32>,
+}
+impl AEncoder {
+    pub fn new() -> Result<(Self, Receiver<Vec<u8>>)> {
+        let (sender, rx) = sync_channel(1024);
+        Ok((
+            Self {
+                encoder: Encoder::new(SampleRate::Hz48000, Channels::Mono, Application::Voip)?,
+                sender,
+                buffer: VecDeque::new(),
+            },
+            rx,
+        ))
+    }
+    pub fn data(&mut self, samples: &[f32]) -> Result<()> {
+        self.buffer.extend(samples);
+        while self.buffer.len() >= 120 {
+            let mut out = [0u8; 120];
+            let mut inp = [0f32; 120];
+            for i in 0..120 {
+                inp[i] = self.buffer.pop_front().unwrap();
+            }
+            let size = self.encoder.encode_float(&inp, &mut out)?;
+            eprintln!("enc {size}");
+            let _ = self.sender.try_send(out[..size].to_vec());
+        }
+        Ok(())
+    }
+}
+
+pub struct ADecoder {
+    decoder: Decoder,
+    receiver: Receiver<Vec<u8>>,
+    buffer: VecDeque<f32>,
+}
+impl ADecoder {
+    pub fn new() -> Result<(Self, SyncSender<Vec<u8>>)> {
+        let (tx, receiver) = sync_channel(1024);
+        Ok((
+            Self {
+                decoder: Decoder::new(SampleRate::Hz48000, Channels::Mono)?,
+                receiver,
+                buffer: VecDeque::new(),
+            },
+            tx,
+        ))
+    }
+    pub fn data(&mut self, samples: &mut [f32]) -> Result<()> {
+        while self.buffer.len() < samples.len() {
+            if let Ok(buf) = self.receiver.try_recv() {
+                let mut output = [0f32; 120];
+                let size = self.decoder.decode_float(
+                    Some(buf.as_slice()),
+                    output.as_mut_slice(),
+                    false,
+                )?;
+                eprintln!("dec {size}");
+                self.buffer.extend(&output[..size]);
+            } else {
+                break;
+            }
+        }
+
+        for x in samples {
+            *x = self.buffer.pop_front().unwrap_or(0.);
+        }
+        Ok(())
+    }
+}
diff --git a/client/src/main.rs b/client/src/main.rs
index 091854b..699e09d 100644
--- a/client/src/main.rs
+++ b/client/src/main.rs
@@ -25,6 +25,7 @@ pub mod scene_render;
 pub mod state;
 pub mod ui;
 pub mod window;
+pub mod audio;
 
 use anyhow::Result;
 use clap::Parser;
diff --git a/client/src/state.rs b/client/src/state.rs
index a658f51..c3d8eb1 100644
--- a/client/src/state.rs
+++ b/client/src/state.rs
@@ -14,7 +14,9 @@
     You should have received a copy of the GNU Affero General Public License
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 */
-use crate::{camera::Camera, download::Downloader, network::Network, renderer::Renderer};
+use crate::{
+    audio::Audio, camera::Camera, download::Downloader, network::Network, renderer::Renderer,
+};
 use anyhow::{Context, Result};
 use glam::{Vec2, Vec3};
 use log::{info, warn};
@@ -34,6 +36,7 @@ pub struct State<'a> {
     pub tree: SceneTree,
     pub camera: Camera,
     pub input_state: InputState,
+    pub audio: Audio,
 
     pub prefab_index: Arc<PrefabIndex>,
     pub prefab_index_res_loaded: Option<Resource<PrefabIndex>>,
@@ -54,6 +57,7 @@ impl<'a> State<'a> {
         info!("new state");
         let downloader = Arc::new(Downloader::new(ResourceStore::new_env()?));
         Ok(Self {
+            audio: Audio::new()?,
             camera: Camera::new(),
             network: Network::new(conn).into(),
             tree: SceneTree::default(),
@@ -133,6 +137,8 @@ impl<'a> State<'a> {
         let dt = (now - self.input_state.time).as_secs_f32();
         self.input_state.time = now;
 
+        self.audio.update();
+
         self.camera
             .update(self.input_state.move_dir, self.input_state.mouse_acc, dt);
         self.input_state.mouse_acc = Vec2::ZERO;