zero copy texture write

7 files changed, 107 insertions, 60 deletions

diff --git a/.gitignore b/.gitignore
index 2a8ca7e..19e8735 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1 +1,2 @@
 /target*
+/perf*
diff --git a/client/src/scene_prepare.rs b/client/src/scene_prepare.rs
index cf94bbf..2b3e2e7 100644
--- a/client/src/scene_prepare.rs
+++ b/client/src/scene_prepare.rs
@@ -79,7 +79,7 @@ pub struct ScenePreparer {
     queue: Arc<Queue>,
     texture_bgl: BindGroupLayout,
 
-    textures: DemandMap<Resource<Image>, (Arc<Texture>, Arc<BindGroup>)>,
+    textures: DemandMap<Resource<Image<'static>>, (Arc<Texture>, Arc<BindGroup>)>,
     placeholder_textures: DemandMap<bool, (Arc<Texture>, Arc<BindGroup>)>,
     index_buffers: DemandMap<Resource<Vec<[u32; 3]>>, (Arc<Buffer>, u32)>,
     vertex_buffers: DemandMap<Resource<Vec<f32>>, (Arc<Buffer>, u32)>,
diff --git a/server/src/network.rs b/server/src/network.rs
index b8be0b1..5c6f173 100644
--- a/server/src/network.rs
+++ b/server/src/network.rs
@@ -161,7 +161,7 @@ impl ServerNetwork {
     }
     pub fn broadcast(&self, packet: Packet, reliable: bool) {
         debug!("* -> {packet:?}");
-        let ser = Arc::new(packet.write_alloc());
+        let ser = Arc::new(packet.write_alloc().into_owned());
         for (cid, (tcp, udp)) in self.conns.lock().unwrap().iter() {
             if !reliable {
                 if let Some(addr) = udp {
@@ -175,7 +175,7 @@ impl ServerNetwork {
         }
     }
     pub fn send(&self, conn: u128, packet: Packet, reliable: bool) {
-        let ser = Arc::new(packet.write_alloc());
+        let ser = Arc::new(packet.write_alloc().into_owned());
         if let Some((tcp, udp)) = self.conns.lock().unwrap().get(&conn) {
             if !reliable {
                 if let Some(addr) = udp {
diff --git a/shared/src/helper.rs b/shared/src/helper.rs
index d267f68..eb1f321 100644
--- a/shared/src/helper.rs
+++ b/shared/src/helper.rs
@@ -18,6 +18,7 @@ use crate::packets::{Data, Object, Resource};
 use anyhow::Result;
 use glam::{Affine3A, Vec2, Vec3A};
 use std::{
+    borrow::Cow,
     io::{Read, Write},
     marker::PhantomData,
 };
@@ -26,10 +27,10 @@ pub trait ReadWrite: Sized {
     fn write(&self, w: &mut dyn Write) -> Result<()>;
     fn read(r: &mut dyn Read) -> Result<Self>;
 
-    fn write_alloc(&self) -> Vec<u8> {
+    fn write_alloc<'a>(&'a self) -> Cow<'a, [u8]> {
         let mut buf = Vec::new();
         self.write(&mut buf).unwrap();
-        buf
+        Cow::Owned(buf)
     }
 }
 
@@ -66,6 +67,23 @@ impl ReadWrite for Vec<u8> {
         Ok(buf)
     }
 }
+impl<'a> ReadWrite for Cow<'a, [u8]> {
+    fn write(&self, w: &mut dyn Write) -> Result<()> {
+        w.write_all(&self)?;
+        Ok(())
+    }
+    fn read(r: &mut dyn Read) -> Result<Self> {
+        let mut buf = Vec::new();
+        r.read_to_end(&mut buf)?;
+        Ok(Cow::Owned(buf))
+    }
+    fn write_alloc<'b>(&'b self) -> Cow<'b, [u8]> {
+        match self {
+            Cow::Borrowed(x) => Cow::Borrowed(x),
+            Cow::Owned(x) => Cow::Borrowed(&x),
+        }
+    }
+}
 impl ReadWrite for Vec<Vec3A> {
     fn write(&self, w: &mut dyn Write) -> Result<()> {
         for e in self {
diff --git a/shared/src/resources.rs b/shared/src/resources.rs
index ba45284..40a9cad 100644
--- a/shared/src/resources.rs
+++ b/shared/src/resources.rs
@@ -19,6 +19,7 @@ use anyhow::Result;
 use glam::{Affine3A, Vec2, Vec3A};
 use log::warn;
 use std::{
+    borrow::Cow,
     collections::BTreeMap,
     io::{Read, Write},
 };
@@ -40,7 +41,7 @@ pub struct LightPart {
 
 #[derive(Debug, Default, Clone)]
 pub struct EnvironmentPart {
-    pub skybox: Option<Resource<Image>>,
+    pub skybox: Option<Resource<Image<'static>>>,
     pub sun: Option<(Vec3A, Vec3A)>,
 }
 
@@ -69,15 +70,15 @@ pub struct MeshPart {
     pub va_transmission: Option<Resource<Vec<f32>>>,
     pub va_alpha: Option<Resource<Vec<f32>>>,
     pub va_emission: Option<Resource<Vec<Vec3A>>>,
-    pub tex_normal: Option<Resource<Image>>,
-    pub tex_roughness: Option<Resource<Image>>,
-    pub tex_metallic: Option<Resource<Image>>,
-    pub tex_albedo: Option<Resource<Image>>,
-    pub tex_transmission: Option<Resource<Image>>,
-    pub tex_alpha: Option<Resource<Image>>,
-    pub tex_emission: Option<Resource<Image>>,
-    pub tex_thickness: Option<Resource<Image>>,
-    pub tex_occlusion: Option<Resource<Image>>,
+    pub tex_normal: Option<Resource<Image<'static>>>,
+    pub tex_roughness: Option<Resource<Image<'static>>>,
+    pub tex_metallic: Option<Resource<Image<'static>>>,
+    pub tex_albedo: Option<Resource<Image<'static>>>,
+    pub tex_transmission: Option<Resource<Image<'static>>>,
+    pub tex_alpha: Option<Resource<Image<'static>>>,
+    pub tex_emission: Option<Resource<Image<'static>>>,
+    pub tex_thickness: Option<Resource<Image<'static>>>,
+    pub tex_occlusion: Option<Resource<Image<'static>>>,
 }
 
 #[derive(Debug, Default, Clone)]
@@ -97,7 +98,7 @@ pub struct CollisionPart {
 pub struct PrefabIndex(pub BTreeMap<String, Resource<Prefab>>);
 
 #[derive(Debug, Clone)]
-pub struct Image(pub Vec<u8>);
+pub struct Image<'a>(pub Cow<'a, [u8]>);
 
 impl ReadWrite for PrefabIndex {
     fn write(&self, w: &mut dyn Write) -> Result<()> {
@@ -331,12 +332,14 @@ fn write_kv(w: &mut dyn Write, key: &[u8], value: &[u8]) -> Result<()> {
     Ok(())
 }
 
-impl ReadWrite for Image {
+impl<'a> ReadWrite for Image<'a> {
     fn write(&self, w: &mut dyn Write) -> Result<()> {
         self.0.write(w)
     }
-
     fn read(r: &mut dyn Read) -> Result<Self> {
-        Ok(Self(<Vec<u8> as ReadWrite>::read(r)?))
+        Ok(Self(<Vec<u8> as ReadWrite>::read(r)?.into()))
+    }
+    fn write_alloc<'b>(&'b self) -> Cow<'b, [u8]> {
+        self.0.write_alloc()
     }
 }
diff --git a/world/Cargo.toml b/world/Cargo.toml
index 3a867bb..8bf5e02 100644
--- a/world/Cargo.toml
+++ b/world/Cargo.toml
@@ -21,3 +21,4 @@ log = "0.4.22"
 weareshared = { path = "../shared" }
 rand = "0.9.0-beta.1"
 image = "0.25.5"
+rayon = "1.10.0"
diff --git a/world/src/main.rs b/world/src/main.rs
index 0decba6..efdd16b 100644
--- a/world/src/main.rs
+++ b/world/src/main.rs
@@ -22,13 +22,16 @@ use anyhow::Result;
 use clap::Parser;
 use gltf::{Gltf, image::Source, import_buffers};
 use image::{ImageReader, codecs::webp::WebPEncoder};
-use log::info;
+use log::{debug, info};
 use mesh::import_mesh;
 use physics::import_physics;
 use rand::random;
+use rayon::iter::{ParallelBridge, ParallelIterator};
 use std::{
+    borrow::Cow,
     fs::File,
     io::{Cursor, Read, Write},
+    marker::PhantomData,
     net::{SocketAddr, TcpStream},
     path::{Path, PathBuf},
     thread::{self, sleep},
@@ -73,23 +76,61 @@ pub struct Args {
     scale: Option<f32>,
     #[arg(short, long)]
     z_up: bool,
+    #[arg(short, long)]
+    dry_run: bool,
 }
 
 fn main() -> Result<()> {
     env_logger::init_from_env("LOG");
     let args = Args::parse();
 
-    let mut sock = TcpStream::connect(args.address)?;
     let store = ResourceStore::new_memory();
 
-    Packet::Connect(random()).write(&mut sock)?;
-
     let path_base = args.scene.parent().unwrap();
     let mut gltf = Gltf::from_reader_without_validation(File::open(&args.scene)?)?;
     let blob = gltf.blob.take();
     let buffers = import_buffers(&gltf, Some(path_base), blob)?;
 
-    let mut prefab = Prefab::default();
+    let mut prefab = gltf
+        .nodes()
+        .par_bridge()
+        .map(|node| {
+            let mut prefab = Prefab::default();
+            if let Some(mesh) = node.mesh() {
+                info!("--- MESH ---");
+                import_mesh(mesh, &buffers, &store, path_base, &node, &mut prefab, &args)?;
+            }
+            let (position, _, _) = node.transform().decomposed();
+            if let Some(light) = node.light() {
+                info!("--- LIGHT ---");
+                let emission = Some(Vec3A::from_array(light.color()) * light.intensity());
+                if let Some(e) = emission {
+                    info!("emission is {e}");
+                }
+                prefab.light.push((
+                    Vec3A::from_array(position),
+                    store.set(&LightPart {
+                        emission,
+                        ..Default::default()
+                    })?,
+                ));
+            }
+            import_physics(&gltf, &node, &mut prefab, &store, &buffers, &args)?;
+            Ok::<_, anyhow::Error>(prefab)
+        })
+        .reduce(
+            || Ok(Prefab::default()),
+            |a, b| {
+                let mut a = a?;
+                let b = b?;
+                a.collision.extend(b.collision);
+                a.mesh.extend(b.mesh);
+                a.light.extend(b.light);
+                a.environment = a.environment.or(b.environment);
+                a.name = a.name.or(b.name);
+                Ok(a)
+            },
+        )?;
 
     prefab.name = args.name.clone().or(gltf
         .default_scene()
@@ -97,40 +138,23 @@ fn main() -> Result<()> {
         .flatten()
         .map(|n| n.to_owned()));
 
-    for node in gltf.nodes() {
-        if let Some(mesh) = node.mesh() {
-            info!("--- MESH ---");
-            import_mesh(mesh, &buffers, &store, path_base, &node, &mut prefab, &args)?;
-        }
-        let (position, _, _) = node.transform().decomposed();
-        if let Some(light) = node.light() {
-            info!("--- LIGHT ---");
-            let emission = Some(Vec3A::from_array(light.color()) * light.intensity());
-            if let Some(e) = emission {
-                info!("emission is {e}");
-            }
-            prefab.light.push((
-                Vec3A::from_array(position),
-                store.set(&LightPart {
-                    emission,
-                    ..Default::default()
-                })?,
-            ));
-        }
-        import_physics(&gltf, &node, &mut prefab, &store, &buffers, &args)?;
-    }
-
     if let Some(skybox) = args.skybox {
         let mut buf = Vec::new();
         File::open(skybox)?.read_to_end(&mut buf)?;
         prefab.environment = Some(store.set(&EnvironmentPart {
-            skybox: Some(store.set(&Image(buf))?),
+            skybox: Some(store.set(&Image(Cow::Owned(buf)))?),
             ..Default::default()
         })?);
     }
 
     let pres = store.set(&prefab)?;
 
+    if args.dry_run {
+        return Ok(());
+    }
+
+    let mut sock = TcpStream::connect(args.address)?;
+    Packet::Connect(random()).write(&mut sock)?;
     Packet::AnnouncePrefab(pres.clone()).write(&mut sock)?;
     sock.flush()?;
 
@@ -199,46 +223,46 @@ fn load_texture(
     buffers: &[gltf::buffer::Data],
     source: &Source,
     webp: bool,
-) -> Result<Resource<Image>> {
+) -> Result<Resource<Image<'static>>> {
     let mut image = match source {
         gltf::image::Source::View { view, mime_type } => {
-            info!("{name} texture is embedded and of type {mime_type:?}");
+            debug!("{name} texture is embedded and of type {mime_type:?}");
             let buf =
                 &buffers[view.buffer().index()].0[view.offset()..view.offset() + view.length()];
-            Image(buf.to_vec())
+            Image(Cow::Borrowed(buf))
         }
         gltf::image::Source::Uri {
             uri,
             mime_type: Some(mime_type),
         } => {
-            info!("{name} texture is {uri:?} and of type {mime_type:?}");
+            debug!("{name} texture is {uri:?} and of type {mime_type:?}");
             let path = path.join(uri);
             let mut buf = Vec::new();
             File::open(path)?.read_to_end(&mut buf)?;
-            Image(buf)
+            Image(buf.into())
         }
         gltf::image::Source::Uri {
             uri,
             mime_type: None,
         } => {
-            info!("{name} texture is {uri:?} and has no type");
+            debug!("{name} texture is {uri:?} and has no type");
             let path = path.join(uri);
             let mut buf = Vec::new();
             File::open(path)?.read_to_end(&mut buf)?;
-            Image(buf)
+            Image(buf.into())
         }
     };
 
     if webp {
-        let mut image_out = Image(Vec::new());
+        let mut image_out = Vec::new();
 
         let len = image.0.len();
         ImageReader::new(Cursor::new(image.0))
             .with_guessed_format()?
             .decode()?
-            .write_with_encoder(WebPEncoder::new_lossless(&mut image_out.0))?;
-        info!("webp encode: {len} -> {}", image_out.0.len());
-        image = image_out;
+            .write_with_encoder(WebPEncoder::new_lossless(&mut image_out))?;
+        debug!("webp encode: {len} -> {}", image_out.len());
+        image = Image(Cow::Owned(image_out));
     }
-    Ok(store.set(&image)?)
+    Ok(Resource(store.set(&image)?.0, PhantomData))
 }