multi-channel audio buffering

3 files changed, 77 insertions, 31 deletions

diff --git a/client/src/audio.rs b/client/src/audio.rs
index c72c6b3..5751c0e 100644
--- a/client/src/audio.rs
+++ b/client/src/audio.rs
@@ -1,5 +1,21 @@
+/*
+    wearechat - generic multiplayer game with voip
+    Copyright (C) 2025 metamuffin
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as published by
+    the Free Software Foundation, version 3 of the License only.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+*/
 use std::{
-    collections::VecDeque,
+    collections::{HashMap, VecDeque},
     sync::mpsc::{Receiver, SyncSender, sync_channel},
 };
 
@@ -12,13 +28,14 @@ use cpal::{
     Stream,
     traits::{DeviceTrait, HostTrait},
 };
+use glam::Vec3;
 use log::warn;
 
 pub struct Audio {
     _instream: Stream,
     _outstream: Stream,
     rx: Receiver<Vec<u8>>,
-    tx: SyncSender<Vec<u8>>,
+    tx: SyncSender<APlayPacket>,
 }
 impl Audio {
     pub fn new() -> Result<Self> {
@@ -72,13 +89,22 @@ impl Audio {
             tx,
         })
     }
-    pub fn update(&mut self) {
-        for e in self.rx.try_iter() {
-            let _ = self.tx.try_send(e);
+    pub fn pop_output(&mut self) -> Option<Vec<u8>> {
+        self.rx.try_recv().ok()
+    }
+    pub fn incoming_packet(&mut self, channel: u128, pos: Vec3, data: Vec<u8>) {
+        if let Err(e) = self.tx.send(APlayPacket { pos, data, channel }) {
+            warn!("audio output buffer overflow: {e:?}")
         }
     }
 }
 
+pub struct APlayPacket {
+    pos: Vec3,
+    channel: u128,
+    data: Vec<u8>,
+}
+
 pub struct AEncoder {
     encoder: Encoder,
     sender: SyncSender<Vec<u8>>,
@@ -105,48 +131,62 @@ impl AEncoder {
                 inp[i] = self.buffer.pop_front().unwrap();
             }
             let size = self.encoder.encode_float(&inp, &mut out)?;
-            eprintln!("enc {size}");
             let _ = self.sender.try_send(out[..size].to_vec());
         }
         Ok(())
     }
 }
 
+const BUFFER_SIZE: usize = 48_000;
 pub struct ADecoder {
     decoder: Decoder,
-    receiver: Receiver<Vec<u8>>,
-    buffer: VecDeque<f32>,
+    receiver: Receiver<APlayPacket>,
+    channels: HashMap<u128, usize>,
+    playback: usize,
+    buffer: Box<[[f32; 2]; BUFFER_SIZE]>,
 }
 impl ADecoder {
-    pub fn new() -> Result<(Self, SyncSender<Vec<u8>>)> {
+    pub fn new() -> Result<(Self, SyncSender<APlayPacket>)> {
         let (tx, receiver) = sync_channel(1024);
         Ok((
             Self {
                 decoder: Decoder::new(SampleRate::Hz48000, Channels::Mono)?,
                 receiver,
-                buffer: VecDeque::new(),
+                channels: HashMap::new(),
+                playback: 0,
+                buffer: unsafe { Box::new_zeroed().assume_init() },
             },
             tx,
         ))
     }
     pub fn data(&mut self, samples: &mut [f32]) -> Result<()> {
         while self.buffer.len() < samples.len() {
-            if let Ok(buf) = self.receiver.try_recv() {
+            if let Ok(p) = self.receiver.try_recv() {
                 let mut output = [0f32; 120];
                 let size = self.decoder.decode_float(
-                    Some(buf.as_slice()),
+                    Some(p.data.as_slice()),
                     output.as_mut_slice(),
                     false,
                 )?;
-                eprintln!("dec {size}");
-                self.buffer.extend(&output[..size]);
+
+                let channel_cursor = self.channels.entry(p.channel).or_insert(self.playback);
+                let free_space = *channel_cursor - self.playback;
+                for i in 0..size.min(free_space) {
+                    // TODO positional audio
+                    let _ = p.pos;
+                    self.buffer[*channel_cursor] = [output[i], output[i]];
+                    *channel_cursor += 1;
+                    *channel_cursor %= BUFFER_SIZE
+                }
             } else {
                 break;
             }
         }
-
-        for x in samples {
-            *x = self.buffer.pop_front().unwrap_or(0.);
+        for x in samples.array_chunks_mut::<2>() {
+            *x = self.buffer[self.playback];
+            self.buffer[self.playback] = [0.; 2];
+            self.playback += 1;
+            self.playback %= BUFFER_SIZE;
         }
         Ok(())
     }
diff --git a/client/src/main.rs b/client/src/main.rs
index 699e09d..bfd7d10 100644
--- a/client/src/main.rs
+++ b/client/src/main.rs
@@ -16,6 +16,8 @@
 */
 #![feature(iter_array_chunks)]
 #![feature(mpmc_channel)]
+#![feature(array_chunks)]
+#![feature(new_zeroed_alloc)]
 pub mod camera;
 pub mod download;
 pub mod network;
diff --git a/client/src/state.rs b/client/src/state.rs
index c3d8eb1..e59691a 100644
--- a/client/src/state.rs
+++ b/client/src/state.rs
@@ -22,7 +22,7 @@ use glam::{Vec2, Vec3};
 use log::{info, warn};
 use std::{net::TcpStream, sync::Arc, time::Instant};
 use weareshared::{
-    packets::{Object, Packet, Resource},
+    packets::{Data, Object, Packet, Resource},
     resources::PrefabIndex,
     store::ResourceStore,
     tree::SceneTree,
@@ -37,7 +37,6 @@ pub struct State<'a> {
     pub camera: Camera,
     pub input_state: InputState,
     pub audio: Audio,
-
     pub prefab_index: Arc<PrefabIndex>,
     pub prefab_index_res_loaded: Option<Resource<PrefabIndex>>,
     pub prefab_index_res: Option<Resource<PrefabIndex>>,
@@ -137,8 +136,6 @@ impl<'a> State<'a> {
         let dt = (now - self.input_state.time).as_secs_f32();
         self.input_state.time = now;
 
-        self.audio.update();
-
         self.camera
             .update(self.input_state.move_dir, self.input_state.mouse_acc, dt);
         self.input_state.mouse_acc = Vec2::ZERO;
@@ -146,8 +143,15 @@ impl<'a> State<'a> {
         for p in self.network.packet_recv.try_iter() {
             self.downloader.packet(&p)?;
             self.tree.packet(&p);
-            if let Packet::PrefabIndex(res) = &p {
-                self.prefab_index_res = Some(res.to_owned());
+            match &p {
+                Packet::PrefabIndex(res) => self.prefab_index_res = Some(res.to_owned()),
+                Packet::Sound(ob, data) => {
+                    if let Some(obdata) = self.tree.objects.get(ob) {
+                        self.audio
+                            .incoming_packet(ob.0, obdata.pos.into(), data.0.clone())
+                    }
+                }
+                _ => (),
             }
         }
         self.downloader
@@ -165,14 +169,14 @@ impl<'a> State<'a> {
         }
 
         if let Some(ob) = self.avatar_ob {
-            self.network
-                .packet_send
-                .send(Packet::Position(
-                    ob,
-                    self.camera.position().into(),
-                    self.camera.rotation().into(),
-                ))
-                .unwrap();
+            self.network.packet_send.send(Packet::Position(
+                ob,
+                self.camera.position().into(),
+                self.camera.rotation().into(),
+            ))?;
+            while let Some(p) = self.audio.pop_output() {
+                self.network.packet_send.send(Packet::Sound(ob, Data(p)))?;
+            }
         } else {
             if let Some(res) = self.prefab_index.0.get("avatar_test") {
                 info!("found avatar resource {res}");