respack

8 files changed, 664 insertions, 504 deletions

diff --git a/doc/respack.md b/doc/respack.md
new file mode 100644
index 0000000..eb316c0
--- /dev/null
+++ b/doc/respack.md
@@ -0,0 +1,22 @@
+# Resource Package Format
+
+Multiple related resources can be packages into a single file for easier
+distribution and storage outside of the game.
+
+A resource pack has a header describing the layout of the contained resources.
+It maps resource hashes to absolute offset-size ranges in file. There is no
+guarantee that the resource data is tightly packed in the file. There may also
+be an entry point resource specified which is of type `RespackEntry`, or not in
+which case the field is zeroed.
+
+```
+MMMM EEEEEEEE CC [RRRRRRRR OO SS]... Data...
+^    ^        ^   ^        ^  ^
+|    |        |   |        |  | Size (u64)
+|    | Entry Point Res     | Offset (u64)
+| Magic bytes |   | Resource hash (256-bit)
+              | Resources count
+```
+
+The magic bytes are `0f0c 5745 4152 4501 5245 5350 4143 4b02`
+(`\x0f\x0cWEARE\x01RESPACK\x02`).
diff --git a/shared/src/lib.rs b/shared/src/lib.rs
index 9c3e863..c3d8d6b 100644
--- a/shared/src/lib.rs
+++ b/shared/src/lib.rs
@@ -27,5 +27,6 @@ pub mod packets;
 pub mod resources;
 pub mod store;
 pub mod tree;
+pub mod respack;
 
 pub use glam::{Affine3A, Mat3A, Vec2, Vec3A, Vec4, vec2, vec3a, vec4};
diff --git a/shared/src/resources.rs b/shared/src/resources.rs
index fbc171b..d5ec76c 100644
--- a/shared/src/resources.rs
+++ b/shared/src/resources.rs
@@ -73,6 +73,10 @@ macro_rules! resource_dicts {
 }
 
 resource_dicts!(
+    pub struct RespackEntry {
+        name: String,
+    }
+
     pub struct Prefab {
         name: String,
         mesh[multi]: (Affine3A, Resource<MeshPart>),
diff --git a/shared/src/respack.rs b/shared/src/respack.rs
new file mode 100644
index 0000000..8c0b9d8
--- /dev/null
+++ b/shared/src/respack.rs
@@ -0,0 +1,107 @@
+/*
+    wearechat - generic multiplayer game with voip
+    Copyright (C) 2025 metamuffin
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as published by
+    the Free Software Foundation, version 3 of the License only.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+*/
+use crate::{packets::Resource, resources::RespackEntry, store::ResourceStore};
+use anyhow::{Result, bail};
+use log::{info, warn};
+use std::{
+    io::{Read, Seek, SeekFrom, Write},
+    marker::PhantomData,
+};
+
+const MAGIC: &[u8; 16] = b"\x0f\x0cWEARE\x01RESPACK\x02";
+
+pub fn save_respack(
+    mut output: impl Write,
+    store: &ResourceStore,
+    resources: &[Resource],
+    entry: Option<Resource<RespackEntry>>,
+) -> Result<()> {
+    output.write_all(MAGIC)?;
+    output.write_all(&entry.map(|e| e.0).unwrap_or([0u8; 32]))?;
+    output.write_all(&u64::to_be_bytes(resources.len() as u64))?;
+    let mut off =
+        (MAGIC.len() + 32 + size_of::<u64>() + (32 + size_of::<u64>() * 2) * resources.len())
+            as u64;
+    for r in resources {
+        let size = store.get_raw_size(*r)?.unwrap() as u64;
+        output.write_all(&r.0)?;
+        output.write_all(&u64::to_be_bytes(off))?;
+        output.write_all(&u64::to_be_bytes(size))?;
+        off += size;
+    }
+    for r in resources {
+        output.write_all(&store.get_raw(*r)?.unwrap())?;
+    }
+    Ok(())
+}
+
+pub fn load_respack(
+    mut input: impl Read + Seek,
+    store: &ResourceStore,
+) -> Result<Option<Resource<RespackEntry>>> {
+    let mut magic = [0u8; MAGIC.len()];
+    input.read_exact(&mut magic)?;
+    if magic != *MAGIC {
+        bail!("wrong magic bytes");
+    }
+    let mut entry = [0u8; 32];
+    input.read_exact(&mut entry)?;
+    let entry = if entry != [0u8; 32] {
+        Some(Resource(entry, PhantomData))
+    } else {
+        None
+    };
+
+    let mut count = [0u8; size_of::<u64>()];
+    input.read_exact(&mut count)?;
+    let count = u64::from_be_bytes(count);
+
+    let mut load_queue = Vec::new();
+    let mut found_entry = false;
+    for _ in 0..count {
+        let mut res = [0u8; 32];
+        let mut off = [0u8; size_of::<u64>()];
+        let mut size = [0u8; size_of::<u64>()];
+        input.read_exact(&mut res)?;
+        input.read_exact(&mut off)?;
+        input.read_exact(&mut size)?;
+
+        found_entry |= Some(Resource(res, PhantomData)) == entry;
+        if store.get_raw_size(Resource(res, PhantomData))?.is_none() {
+            load_queue.push((res, u64::from_be_bytes(off), u64::from_be_bytes(size)))
+        }
+    }
+    if !found_entry && entry.is_some() {
+        warn!("respack does not contain its entry resource")
+    }
+    info!(
+        "loading {} of {count} resources from pack",
+        load_queue.len(),
+    );
+
+    for (res, off, size) in load_queue {
+        input.seek(SeekFrom::Start(off))?;
+        let mut buf = Vec::new();
+        input.by_ref().take(size).read_to_end(&mut buf)?;
+        let key = store.set_raw(&buf)?;
+        if key.0 != res {
+            warn!("respack containes mislabeled resources")
+        }
+    }
+
+    Ok(entry)
+}
diff --git a/shared/src/store.rs b/shared/src/store.rs
index 3db6bac..e3e5949 100644
--- a/shared/src/store.rs
+++ b/shared/src/store.rs
@@ -76,6 +76,16 @@ impl ResourceStore {
     pub fn set<T: ReadWrite>(&self, value: &T) -> Result<Resource<T>> {
         Ok(Resource(self.set_raw(&value.write_alloc())?.0, PhantomData))
     }
+    pub fn get_raw_size(&self, key: Resource) -> Result<Option<usize>> {
+        match self {
+            ResourceStore::Redb(_) => todo!(),
+            ResourceStore::Filesystem(_) => todo!(),
+            ResourceStore::Memory(mutex) => {
+                let g = mutex.lock().unwrap();
+                Ok(g.get(&key).map(|s| s.len()))
+            }
+        }
+    }
     pub fn get_raw(&self, key: Resource) -> Result<Option<Vec<u8>>> {
         match self {
             ResourceStore::Redb(database) => {
diff --git a/world/src/animation.rs b/world/src/animation.rs
index 54ce388..ca049e5 100644
--- a/world/src/animation.rs
+++ b/world/src/animation.rs
@@ -61,13 +61,14 @@ pub fn import_animation(
                     ReadOutputs::Rotations(iter) => iter
                         .into_f32()
                         .map(Quat::from_array)
-                        .map(|q| rot.mul_quat(q))
+                        .map(|q| q * rot)
                         .flat_map(|q| q.to_array())
                         .collect(),
-                    ReadOutputs::Scales(iter) => iter
-                        .flat_map(|[x, y, z]| (t.matrix3 * vec3a(x, y, z)).to_array())
-                        .collect(),
+                    // ReadOutputs::Scales(iter) => iter
+                    //     .flat_map(|[x, y, z]| (t.matrix3 * vec3a(x, y, z)).to_array())
+                    //     .collect(),
                     ReadOutputs::MorphTargetWeights(iter) => iter.into_f32().collect(),
+                    _ => continue,
                 }
             } else {
                 match reader.read_outputs().unwrap() {
diff --git a/world/src/main.rs b/world/src/main.rs
index 1a75a61..554a2bd 100644
--- a/world/src/main.rs
+++ b/world/src/main.rs
@@ -19,27 +19,22 @@
 pub mod animation;
 pub mod mesh;
 pub mod physics;
+pub mod prefab;
 pub mod vrm;
 
-use animation::import_animation;
-use anyhow::{Context, Result, anyhow};
+use anyhow::{Result, bail};
 use clap::Parser;
-use glam::Vec3;
-use gltf::{Gltf, Node, image::Source, import_buffers, scene::Transform};
+use gltf::{image::Source, scene::Transform};
 use humansize::BINARY;
 use image::{ImageReader, codecs::webp::WebPEncoder};
 use log::{debug, info};
-use mesh::import_mesh;
-use physics::import_physics;
+use prefab::import_prefab;
 use rand::random;
-use rayon::iter::{IntoParallelRefIterator, ParallelIterator};
-use serde::Deserialize;
-use serde_json::Value;
 use std::{
     borrow::Cow,
-    collections::{BTreeMap, HashMap},
-    fs::{File, read_to_string},
-    io::{Cursor, Read, Write},
+    collections::HashMap,
+    fs::File,
+    io::{BufWriter, Cursor, Read, Write},
     marker::PhantomData,
     net::{SocketAddr, TcpStream},
     path::{Path, PathBuf},
@@ -47,21 +42,25 @@ use std::{
     thread::{self, sleep},
     time::Duration,
 };
-use vrm::extract_vrm_data;
 use weareshared::{
     Affine3A, Vec3A,
     helper::ReadWrite,
     packets::{Data, Object, Packet, Resource},
-    resources::{
-        ArmaturePart, AvatarInfoPart, EnvironmentPart, Image, LightPart, ParticlesPart, Prefab,
-    },
+    resources::{Image, RespackEntry},
+    respack::save_respack,
     store::ResourceStore,
     vec3a,
 };
 
 #[derive(Parser)]
 pub struct Args {
-    address: SocketAddr,
+    #[arg(short, long)]
+    address: Option<SocketAddr>,
+
+    /// Output converted prefab as resource package
+    #[arg(short = 'o', long)]
+    pack: Option<PathBuf>,
+
     /// Path(s) to a glTF file, binary or json format
     scene: Vec<PathBuf>,
     /// Send all resources to the server then quit
@@ -113,9 +112,6 @@ pub struct Args {
     /// Adds a light
     #[arg(long)]
     debug_light: bool,
-    /// Wiggles joint index 6
-    #[arg(long)]
-    debug_armature: bool,
     #[arg(long)]
     no_particles: bool,
     #[arg(long)]
@@ -124,9 +120,9 @@ pub struct Args {
 
 fn main() -> Result<()> {
     env_logger::init_from_env("LOG");
-    let mut args = Args::parse();
+    let args = Args::parse();
 
-    let store = if args.use_cache {
+    let store = if args.use_cache && !args.pack.is_some() {
         ResourceStore::new_env()?
     } else {
         ResourceStore::new_memory()
@@ -135,408 +131,8 @@ fn main() -> Result<()> {
     let mut prefabs = Vec::new();
     let texture_cache = Arc::new(Mutex::new(HashMap::new()));
 
-    let mut root_affine = Affine3A::IDENTITY;
-    root_affine.matrix3 *= args.scale.unwrap_or(1.);
-    root_affine.translation *= args.scale.unwrap_or(1.);
-
-    let mut debug_bone = (0, Affine3A::IDENTITY);
-
     for scenepath in &args.scene {
-        let path_base = scenepath.parent().unwrap();
-        let mut gltf =
-            Gltf::from_reader_without_validation(File::open(scenepath)?).context("gltf parsing")?;
-        let blob = gltf.blob.take();
-        let buffers = import_buffers(&gltf, Some(path_base), blob).context("importing buffers")?;
-
-        let root = gltf.default_scene().ok_or(anyhow!("no default scene"))?;
-
-        // gltf.as_json().asset.copyright
-        // eprintln!("{:?}", gltf.extensions_used());
-        // eprintln!("{:?}", gltf.extensions());
-        // eprintln!("{:?}", root.extensions());
-        // eprintln!("{:?}", root.extras());
-
-        let mut nodes = Vec::new();
-        fn traverse<'a>(out: &mut Vec<(Affine3A, Node<'a>)>, node: Node<'a>, trans: Affine3A) {
-            let trans = trans * transform_to_affine(node.transform());
-            for c in node.children() {
-                traverse(out, c, trans);
-            }
-            out.push((trans, node));
-        }
-
-        for node in root.nodes() {
-            traverse(&mut nodes, node, root_affine);
-        }
-
-        let vrm = extract_vrm_data(&gltf)?;
-
-        let mut skin_index_to_arm_index = BTreeMap::new();
-        let mut joint_index_to_arm_index = BTreeMap::new();
-        let joint_name_to_arm_index: BTreeMap<String, usize>;
-        let armature = {
-            let mut name = Vec::new();
-            let mut parent_pre_map = Vec::new();
-            let mut transform = Vec::new();
-            let mut inverse_bind_transform = Vec::new();
-
-            for skin in gltf.skins() {
-                let mut inverse_bind_mat = skin
-                    .reader(|buf| Some(&buffers[buf.index()]))
-                    .read_inverse_bind_matrices();
-                for (j_ind, j) in skin.joints().enumerate() {
-                    let ibm = inverse_bind_mat.as_mut().map(|x| x.next().unwrap());
-                    let a_ind = match joint_index_to_arm_index.get(&j.index()) {
-                        Some(i) => *i,
-                        None => {
-                            let a_ind = name.len();
-                            // name.push(j.name().unwrap_or("").to_string());
-                            name.push(format!("bone{}", a_ind));
-                            transform.push(transform_to_affine(j.transform()));
-                            parent_pre_map.push(
-                                gltf.nodes()
-                                    .find(|n| n.children().any(|c| c.index() == j.index()))
-                                    .map(|n| n.index()),
-                            );
-                            inverse_bind_transform.push(
-                                ibm.map(|a| transform_to_affine(Transform::Matrix { matrix: a }))
-                                    .unwrap_or(Affine3A::IDENTITY),
-                            );
-
-                            joint_index_to_arm_index.insert(j.index(), a_ind);
-                            a_ind
-                        }
-                    };
-                    skin_index_to_arm_index.insert((skin.index(), j_ind as u16), a_ind as u32);
-                }
-            }
-
-            let parent = parent_pre_map
-                .into_iter()
-                .enumerate()
-                .map(|(i, p)| {
-                    p.and_then(|i| joint_index_to_arm_index.get(&i).copied())
-                        .unwrap_or(i) as u16
-                })
-                .collect::<Vec<_>>();
-
-            for (node, bname) in &vrm.bone_node_names {
-                let ind = joint_index_to_arm_index[node];
-                name[ind] = bname.to_owned();
-            }
-
-            for (i, (name, tr)) in name.iter().zip(transform.iter()).enumerate() {
-                if name == "head" {
-                    debug_bone = (i, *tr);
-                }
-            }
-
-            joint_name_to_arm_index = name
-                .iter()
-                .cloned()
-                .enumerate()
-                .map(|(a, b)| (b, a))
-                .collect();
-
-            ArmaturePart {
-                name: Some(name),
-                parent: Some(parent),
-                transform: Some(transform),
-                inverse_bind_transform: Some(inverse_bind_transform),
-            }
-        };
-
-        let head_bones = {
-            let pa = armature.parent.clone().unwrap_or_default();
-            let na = armature.name.clone().unwrap_or_default();
-            (0..pa.len())
-                .filter(|&(mut i)| {
-                    let mut f = false;
-                    while pa[i] as usize != i {
-                        f |= na[i] == "head";
-                        i = pa[i] as usize;
-                    }
-                    f
-                })
-                .map(|e| e as u32)
-                .collect::<Vec<_>>()
-        };
-
-        let prefab = Mutex::new(Prefab::default());
-        let node_to_meshes = Mutex::new(BTreeMap::<usize, Vec<usize>>::new());
-
-        nodes
-            .par_iter()
-            .map(|(trans, node)| {
-                // if node.name().unwrap_or_default() == "particles" {
-                //     eprintln!("{:?}", node.transform());
-                //     eprintln!("{:?}", node.extensions());
-                //     eprintln!("{:?}", node.extras());
-                // }
-                let name = node.name().unwrap_or_default();
-                let extras: Value = node
-                    .extras()
-                    .to_owned()
-                    .map(|v| serde_json::from_str(v.get()).unwrap())
-                    .unwrap_or(Value::Object(Default::default()));
-
-                if !name.ends_with("-collider") {
-                    if let Some(mesh) = node.mesh() {
-                        let meshes = import_mesh(
-                            mesh,
-                            *trans,
-                            &buffers,
-                            &store,
-                            path_base,
-                            node,
-                            &args,
-                            &texture_cache,
-                            &skin_index_to_arm_index,
-                            &vrm,
-                            &head_bones,
-                        )?;
-                        for (node, trans, mesh) in meshes {
-                            let mut k = prefab.lock().unwrap();
-                            let i = k.mesh.len();
-                            k.mesh.push((trans, mesh));
-                            node_to_meshes
-                                .lock()
-                                .unwrap()
-                                .entry(node)
-                                .or_default()
-                                .push(i)
-                        }
-                    }
-                }
-                if extras.get("particles") == Some(&Value::Bool(true)) && !args.no_particles {
-                    #[derive(Deserialize)]
-                    struct ParticlesAttr {
-                        density: Option<f32>,
-                        lifetime: Option<f32>,
-                        lifetime_spread: Option<f32>,
-                        velocity: Option<Vec3A>,
-                        velocity_spread: Option<Vec3A>,
-                    }
-                    // let sprite = extras
-                    //     .get("sprite")
-                    //     .ok_or(anyhow!("particle volume is missing sprite"))?;
-
-                    let attr: ParticlesAttr =
-                        serde_json::from_value(extras).context("particles attributes")?;
-
-                    info!("adding particles part");
-
-                    let part = store.set(&ParticlesPart {
-                        sprite: None,
-                        density: attr.density,
-                        lifetime: attr.lifetime,
-                        lifetime_spread: attr.lifetime_spread,
-                        velocity: attr.velocity,
-                        velocity_spread: attr.velocity_spread,
-                    })?;
-
-                    prefab
-                        .lock()
-                        .unwrap()
-                        .particles
-                        .push((transform_to_affine(node.transform()), part));
-                }
-
-                if let Some(light) = node.light() {
-                    let name = node.name().map(|e| e.to_owned());
-                    if let Some(name) = &name {
-                        info!("adding light {name:?}");
-                    } else {
-                        info!("adding light");
-                    }
-                    let emission = Some(Vec3A::from_array(light.color()) * light.intensity());
-                    if let Some(e) = emission {
-                        debug!("emission is {e}");
-                    }
-                    let (position, _, _) = node.transform().decomposed();
-                    let part = store.set(&LightPart {
-                        emission,
-                        name,
-                        radius: None,
-                    })?;
-                    prefab
-                        .lock()
-                        .unwrap()
-                        .light
-                        .push((Vec3A::from_array(position), part));
-                }
-                {
-                    let collider = import_physics(&gltf, *trans, node, &store, &buffers)?;
-                    prefab.lock().unwrap().collision.extend(collider);
-                }
-
-                Ok::<_, anyhow::Error>(())
-            })
-            .reduce(
-                || Ok(()),
-                |a, b| match (a, b) {
-                    (Ok(()), Ok(())) => Ok(()),
-                    (Ok(()), a) => a,
-                    (a, _) => a,
-                },
-            )?;
-
-        let mut prefab = prefab.into_inner().unwrap();
-        let node_to_meshes = node_to_meshes.into_inner().unwrap();
-
-        if let Some(apath) = args.animation.take() {
-            let path_base = apath.parent().unwrap();
-            let mut gltf = Gltf::from_reader_without_validation(File::open(&apath)?)
-                .context("gltf parsing")?;
-            let blob = gltf.blob.take();
-            let buffers =
-                import_buffers(&gltf, Some(path_base), blob).context("importing buffers")?;
-
-            let anim_name_map = if let Some(ref map_path) = args.animation_bone_map {
-                let mut map = BTreeMap::new();
-                for l in read_to_string(map_path)?.lines() {
-                    if !l.trim().is_empty() && !l.starts_with(";") {
-                        let (a, b) = l.split_once("=").unwrap();
-                        map.insert(a.to_string(), b.to_string());
-                    }
-                }
-                Some(map)
-            } else {
-                None
-            };
-
-            let mut anim_joint_index_to_arm_index = BTreeMap::new();
-            let mut joint_index_to_ibm = BTreeMap::new();
-            for n in gltf.nodes() {
-                if let Some(name) = n.name() {
-                    let Some(vrm_name) = (if let Some(map) = &anim_name_map {
-                        map.get(name).cloned()
-                    } else {
-                        Some(name.to_string())
-                    }) else {
-                        continue;
-                    };
-                    let anim_node_index = n.index();
-                    if let Some(scene_arm_index) = joint_name_to_arm_index.get(&vrm_name) {
-                        debug!(
-                            "mapping {name:?} (node={}) -> {vrm_name:?} (bone={})",
-                            anim_node_index, scene_arm_index
-                        );
-                        anim_joint_index_to_arm_index.insert(anim_node_index, *scene_arm_index);
-                    }
-                }
-            }
-            if args.animation_apply_ibm {
-                for s in gltf.skins() {
-                    let reader = s.reader(|buf| Some(&buffers[buf.index()]));
-                    if let Some(ibms) = reader.read_inverse_bind_matrices() {
-                        for (jn, ibm) in s.joints().zip(ibms) {
-                            joint_index_to_ibm.insert(
-                                jn.index(),
-                                transform_to_affine(Transform::Matrix { matrix: ibm }),
-                            );
-                        }
-                    }
-                }
-                debug!("{} joint IBMs found", joint_index_to_ibm.len());
-            }
-            let transform = args
-                .animation_rotation_y
-                .map(Affine3A::from_rotation_y)
-                .unwrap_or_default()
-                * args
-                    .animation_scale
-                    .map(Vec3::splat)
-                    .map(Affine3A::from_scale)
-                    .unwrap_or_default();
-            for a in gltf.animations() {
-                prefab.animation.push(import_animation(
-                    a,
-                    &store,
-                    transform,
-                    &joint_index_to_ibm,
-                    &anim_joint_index_to_arm_index,
-                    &BTreeMap::new(),
-                    &buffers,
-                )?);
-            }
-        }
-
-        for a in gltf.animations() {
-            if !args.no_animations {
-                prefab.animation.push(import_animation(
-                    a,
-                    &store,
-                    Affine3A::IDENTITY,
-                    &BTreeMap::new(),
-                    &joint_index_to_arm_index,
-                    &node_to_meshes,
-                    &buffers,
-                )?);
-            };
-        }
-
-        if vrm.camera_mount.is_some()
-            || vrm.camera_mount_offset.is_some()
-            || !vrm.bone_node_names.is_empty()
-        {
-            info!("avatar info enabled");
-            prefab.avatar_info = Some(
-                store.set(&AvatarInfoPart {
-                    armature: Some(0), // TODO
-                    camera_mount: vrm
-                        .camera_mount
-                        .map(|e| joint_index_to_arm_index[&e] as u32),
-                    camera_mount_offset: vrm.camera_mount_offset,
-                    ..Default::default()
-                })?,
-            );
-        }
-
-        prefab.armature = if armature.parent.as_ref().is_some_and(|a| !a.is_empty()) {
-            vec![store.set(&armature)?]
-        } else {
-            vec![]
-        };
-
-        let skybox = if let Some(skybox) = &args.skybox {
-            let mut buf = Vec::new();
-            File::open(skybox)?.read_to_end(&mut buf)?;
-            Some(store.set(&Image(Cow::Owned(buf)))?)
-        } else {
-            None
-        };
-        let sun = if args.with_default_sun {
-            Some((
-                vec3a(1., -5., 1.).normalize(),
-                vec3a(1., 1., 1.).normalize() * 100_000.,
-            ))
-        } else {
-            None
-        };
-        prefab.environment = if skybox.is_some() || sun.is_some() {
-            Some(store.set(&EnvironmentPart { skybox, sun })?)
-        } else {
-            None
-        };
-
-        prefab.name = args.name.clone().or(gltf
-            .default_scene()
-            .and_then(|n| n.name())
-            .map(|n| n.to_owned()));
-
-        if args.debug_light {
-            prefab.light.push((
-                vec3a(5., 5., 5.),
-                store.set(&LightPart {
-                    name: Some("debug light".to_owned()),
-                    emission: Some(vec3a(10., 5., 15.)),
-                    radius: Some(0.3),
-                })?,
-            ));
-        }
-
-        prefabs.push(store.set(&prefab)?);
+        prefabs.push(import_prefab(&store, &texture_cache, scenepath, &args)?);
     }
 
     let mut size = 0;
@@ -549,94 +145,87 @@ fn main() -> Result<()> {
     if args.dry_run {
         return Ok(());
     }
-
-    let mut sock = TcpStream::connect(args.address)?;
-    Packet::Connect(random()).write(&mut sock)?;
-    for p in &prefabs {
-        Packet::AnnouncePrefab(p.clone()).write(&mut sock)?;
-    }
-    sock.flush()?;
-
-    let mut obs = Vec::new();
-    if args.add {
-        for (i, p) in prefabs.iter().enumerate() {
-            let ob = Object::new();
-            info!("adding object {ob}");
-            Packet::Add(ob, p.clone()).write(&mut sock)?;
-            if args.line_up {
-                Packet::Position(ob, vec3a(i as f32 * 1.2, 0., i as f32 * 0.3), Vec3A::ZERO)
-                    .write(&mut sock)?;
-            }
-            obs.push(ob);
+    if let Some(outpath) = args.pack {
+        let entry = store.set(&RespackEntry { name: None })?;
+        let mut resources = Vec::new();
+        store.iter(|r, _| resources.push(r))?;
+        save_respack(
+            BufWriter::new(File::create(outpath)?),
+            &store,
+            &resources,
+            Some(entry),
+        )?;
+    } else if let Some(address) = args.address {
+        let mut sock = TcpStream::connect(address)?;
+        Packet::Connect(random()).write(&mut sock)?;
+        for p in &prefabs {
+            Packet::AnnouncePrefab(p.clone()).write(&mut sock)?;
         }
         sock.flush()?;
-    }
 
-    if args.debug_spin {
-        let ob = obs[0];
-        let mut sock2 = sock.try_clone().unwrap();
-        thread::spawn(move || {
-            let mut x = 0.;
-            loop {
-                Packet::Position(ob, Vec3A::ZERO, vec3a(0., x * 0.1, 0.))
-                    .write(&mut sock2)
-                    .unwrap();
-                sock2.flush().unwrap();
-                x += 0.1;
-                sleep(Duration::from_millis(50));
-            }
-        });
-    }
-    if args.debug_armature {
-        let ob = obs[0];
-        let mut sock2 = sock.try_clone().unwrap();
-        thread::spawn(move || {
-            let mut x = 0f32;
-            loop {
-                let a = Affine3A::IDENTITY
-                    * Affine3A::from_rotation_x(x.cos() * 0.3)
-                    * Affine3A::from_rotation_y(x.sin() * 0.5)
-                    * debug_bone.1;
-                Packet::Pose(ob, vec![(debug_bone.0 as u16, a)])
-                    .write(&mut sock2)
-                    .unwrap();
-                sock2.flush().unwrap();
-                x += 0.2;
-                sleep(Duration::from_millis(50));
+        let mut obs = Vec::new();
+        if args.add {
+            for (i, p) in prefabs.iter().enumerate() {
+                let ob = Object::new();
+                info!("adding object {ob}");
+                Packet::Add(ob, p.clone()).write(&mut sock)?;
+                if args.line_up {
+                    Packet::Position(ob, vec3a(i as f32 * 1.2, 0., i as f32 * 0.3), Vec3A::ZERO)
+                        .write(&mut sock)?;
+                }
+                obs.push(ob);
             }
-        });
-    }
+            sock.flush()?;
+        }
 
-    if args.push {
-        if args.use_cache {
-            return Ok(());
+        if args.debug_spin {
+            let ob = obs[0];
+            let mut sock2 = sock.try_clone().unwrap();
+            thread::spawn(move || {
+                let mut x = 0.;
+                loop {
+                    Packet::Position(ob, Vec3A::ZERO, vec3a(0., x * 0.1, 0.))
+                        .write(&mut sock2)
+                        .unwrap();
+                    sock2.flush().unwrap();
+                    x += 0.1;
+                    sleep(Duration::from_millis(50));
+                }
+            });
         }
-        store.iter(|k, _| {
-            Packet::RespondResource(k, Data(store.get_raw(k).unwrap().unwrap()))
-                .write(&mut sock)
-                .unwrap();
-        })?;
-        sock.flush()?;
-    } else {
-        loop {
-            let packet = Packet::read(&mut sock)?;
-            match packet {
-                Packet::RequestResource(hash) => {
-                    if let Some(d) = store.get_raw(hash)? {
-                        Packet::RespondResource(hash, Data(d)).write(&mut sock)?;
-                        sock.flush()?;
+        if args.push {
+            if args.use_cache {
+                return Ok(());
+            }
+            store.iter(|k, _| {
+                Packet::RespondResource(k, Data(store.get_raw(k).unwrap().unwrap()))
+                    .write(&mut sock)
+                    .unwrap();
+            })?;
+            sock.flush()?;
+        } else {
+            loop {
+                let packet = Packet::read(&mut sock)?;
+                match packet {
+                    Packet::RequestResource(hash) => {
+                        if let Some(d) = store.get_raw(hash)? {
+                            Packet::RespondResource(hash, Data(d)).write(&mut sock)?;
+                            sock.flush()?;
+                        }
                     }
-                }
-                Packet::Add(ob_a, _) => {
-                    if args.clear && !obs.contains(&ob_a) {
-                        info!("removing object {ob_a}");
-                        Packet::Remove(ob_a).write(&mut sock)?;
-                        sock.flush()?;
+                    Packet::Add(ob_a, _) => {
+                        if args.clear && !obs.contains(&ob_a) {
+                            info!("removing object {ob_a}");
+                            Packet::Remove(ob_a).write(&mut sock)?;
+                            sock.flush()?;
+                        }
                     }
+                    _ => (),
                 }
-                _ => (),
             }
         }
+    } else {
+        bail!("no output option specified. either provide an address to a server or use --pack")
     }
     Ok(())
 }
diff --git a/world/src/prefab.rs b/world/src/prefab.rs
new file mode 100644
index 0000000..bf45f01
--- /dev/null
+++ b/world/src/prefab.rs
@@ -0,0 +1,426 @@
+use crate::{
+    Args, TextureCache, animation::import_animation, mesh::import_mesh, physics::import_physics,
+    transform_to_affine, vrm::extract_vrm_data,
+};
+use anyhow::{Context, Result, anyhow};
+use glam::{Affine3A, Vec3, Vec3A, vec3a};
+use gltf::{Gltf, Node, import_buffers, scene::Transform};
+use log::{debug, info};
+use rayon::iter::{IntoParallelRefIterator, ParallelIterator};
+use serde::Deserialize;
+use serde_json::Value;
+use std::{
+    borrow::Cow,
+    collections::BTreeMap,
+    fs::{File, read_to_string},
+    io::Read,
+    path::Path,
+    sync::Mutex,
+};
+use weareshared::{
+    packets::Resource,
+    resources::{
+        ArmaturePart, AvatarInfoPart, EnvironmentPart, Image, LightPart, ParticlesPart, Prefab,
+    },
+    store::ResourceStore,
+};
+
+pub fn import_prefab(
+    store: &ResourceStore,
+    texture_cache: &TextureCache,
+    scenepath: &Path,
+    args: &Args,
+) -> Result<Resource<Prefab>> {
+    let path_base = scenepath.parent().unwrap();
+    let mut gltf =
+        Gltf::from_reader_without_validation(File::open(scenepath)?).context("gltf parsing")?;
+    let blob = gltf.blob.take();
+    let buffers = import_buffers(&gltf, Some(path_base), blob).context("importing buffers")?;
+
+    let root = gltf.default_scene().ok_or(anyhow!("no default scene"))?;
+
+    // gltf.as_json().asset.copyright
+    // eprintln!("{:?}", gltf.extensions_used());
+    // eprintln!("{:?}", gltf.extensions());
+    // eprintln!("{:?}", root.extensions());
+    // eprintln!("{:?}", root.extras());
+
+    let mut nodes = Vec::new();
+    fn traverse<'a>(out: &mut Vec<(Affine3A, Node<'a>)>, node: Node<'a>, trans: Affine3A) {
+        let trans = trans * transform_to_affine(node.transform());
+        for c in node.children() {
+            traverse(out, c, trans);
+        }
+        out.push((trans, node));
+    }
+
+    let mut root_affine = Affine3A::IDENTITY;
+    root_affine.matrix3 *= args.scale.unwrap_or(1.);
+    root_affine.translation *= args.scale.unwrap_or(1.);
+
+    for node in root.nodes() {
+        traverse(&mut nodes, node, root_affine);
+    }
+
+    let vrm = extract_vrm_data(&gltf)?;
+
+    let mut skin_index_to_arm_index = BTreeMap::new();
+    let mut joint_index_to_arm_index = BTreeMap::new();
+    let joint_name_to_arm_index: BTreeMap<String, usize>;
+    let armature = {
+        let mut name = Vec::new();
+        let mut parent_pre_map = Vec::new();
+        let mut transform = Vec::new();
+        let mut inverse_bind_transform = Vec::new();
+
+        for skin in gltf.skins() {
+            let mut inverse_bind_mat = skin
+                .reader(|buf| Some(&buffers[buf.index()]))
+                .read_inverse_bind_matrices();
+            for (j_ind, j) in skin.joints().enumerate() {
+                let ibm = inverse_bind_mat.as_mut().map(|x| x.next().unwrap());
+                let a_ind = match joint_index_to_arm_index.get(&j.index()) {
+                    Some(i) => *i,
+                    None => {
+                        let a_ind = name.len();
+                        // name.push(j.name().unwrap_or("").to_string());
+                        name.push(format!("bone{}", a_ind));
+                        transform.push(transform_to_affine(j.transform()));
+                        parent_pre_map.push(
+                            gltf.nodes()
+                                .find(|n| n.children().any(|c| c.index() == j.index()))
+                                .map(|n| n.index()),
+                        );
+                        inverse_bind_transform.push(
+                            ibm.map(|a| transform_to_affine(Transform::Matrix { matrix: a }))
+                                .unwrap_or(Affine3A::IDENTITY),
+                        );
+
+                        joint_index_to_arm_index.insert(j.index(), a_ind);
+                        a_ind
+                    }
+                };
+                skin_index_to_arm_index.insert((skin.index(), j_ind as u16), a_ind as u32);
+            }
+        }
+
+        let parent = parent_pre_map
+            .into_iter()
+            .enumerate()
+            .map(|(i, p)| {
+                p.and_then(|i| joint_index_to_arm_index.get(&i).copied())
+                    .unwrap_or(i) as u16
+            })
+            .collect::<Vec<_>>();
+
+        for (node, bname) in &vrm.bone_node_names {
+            let ind = joint_index_to_arm_index[node];
+            name[ind] = bname.to_owned();
+        }
+
+        joint_name_to_arm_index = name
+            .iter()
+            .cloned()
+            .enumerate()
+            .map(|(a, b)| (b, a))
+            .collect();
+
+        ArmaturePart {
+            name: Some(name),
+            parent: Some(parent),
+            transform: Some(transform),
+            inverse_bind_transform: Some(inverse_bind_transform),
+        }
+    };
+
+    let head_bones = {
+        let pa = armature.parent.clone().unwrap_or_default();
+        let na = armature.name.clone().unwrap_or_default();
+        (0..pa.len())
+            .filter(|&(mut i)| {
+                let mut f = false;
+                while pa[i] as usize != i {
+                    f |= na[i] == "head";
+                    i = pa[i] as usize;
+                }
+                f
+            })
+            .map(|e| e as u32)
+            .collect::<Vec<_>>()
+    };
+
+    let prefab = Mutex::new(Prefab::default());
+    let node_to_meshes = Mutex::new(BTreeMap::<usize, Vec<usize>>::new());
+
+    nodes
+        .par_iter()
+        .map(|(trans, node)| {
+            // if node.name().unwrap_or_default() == "particles" {
+            //     eprintln!("{:?}", node.transform());
+            //     eprintln!("{:?}", node.extensions());
+            //     eprintln!("{:?}", node.extras());
+            // }
+            let name = node.name().unwrap_or_default();
+            let extras: Value = node
+                .extras()
+                .to_owned()
+                .map(|v| serde_json::from_str(v.get()).unwrap())
+                .unwrap_or(Value::Object(Default::default()));
+
+            if !name.ends_with("-collider") {
+                if let Some(mesh) = node.mesh() {
+                    let meshes = import_mesh(
+                        mesh,
+                        *trans,
+                        &buffers,
+                        &store,
+                        path_base,
+                        node,
+                        &args,
+                        &texture_cache,
+                        &skin_index_to_arm_index,
+                        &vrm,
+                        &head_bones,
+                    )?;
+                    for (node, trans, mesh) in meshes {
+                        let mut k = prefab.lock().unwrap();
+                        let i = k.mesh.len();
+                        k.mesh.push((trans, mesh));
+                        node_to_meshes
+                            .lock()
+                            .unwrap()
+                            .entry(node)
+                            .or_default()
+                            .push(i)
+                    }
+                }
+            }
+            if extras.get("particles") == Some(&Value::Bool(true)) && !args.no_particles {
+                #[derive(Deserialize)]
+                struct ParticlesAttr {
+                    density: Option<f32>,
+                    lifetime: Option<f32>,
+                    lifetime_spread: Option<f32>,
+                    velocity: Option<Vec3A>,
+                    velocity_spread: Option<Vec3A>,
+                }
+                // let sprite = extras
+                //     .get("sprite")
+                //     .ok_or(anyhow!("particle volume is missing sprite"))?;
+
+                let attr: ParticlesAttr =
+                    serde_json::from_value(extras).context("particles attributes")?;
+
+                info!("adding particles part");
+
+                let part = store.set(&ParticlesPart {
+                    sprite: None,
+                    density: attr.density,
+                    lifetime: attr.lifetime,
+                    lifetime_spread: attr.lifetime_spread,
+                    velocity: attr.velocity,
+                    velocity_spread: attr.velocity_spread,
+                })?;
+
+                prefab
+                    .lock()
+                    .unwrap()
+                    .particles
+                    .push((transform_to_affine(node.transform()), part));
+            }
+
+            if let Some(light) = node.light() {
+                let name = node.name().map(|e| e.to_owned());
+                if let Some(name) = &name {
+                    info!("adding light {name:?}");
+                } else {
+                    info!("adding light");
+                }
+                let emission = Some(Vec3A::from_array(light.color()) * light.intensity());
+                if let Some(e) = emission {
+                    debug!("emission is {e}");
+                }
+                let (position, _, _) = node.transform().decomposed();
+                let part = store.set(&LightPart {
+                    emission,
+                    name,
+                    radius: None,
+                })?;
+                prefab
+                    .lock()
+                    .unwrap()
+                    .light
+                    .push((Vec3A::from_array(position), part));
+            }
+            {
+                let collider = import_physics(&gltf, *trans, node, &store, &buffers)?;
+                prefab.lock().unwrap().collision.extend(collider);
+            }
+
+            Ok::<_, anyhow::Error>(())
+        })
+        .reduce(
+            || Ok(()),
+            |a, b| match (a, b) {
+                (Ok(()), Ok(())) => Ok(()),
+                (Ok(()), a) => a,
+                (a, _) => a,
+            },
+        )?;
+
+    let mut prefab = prefab.into_inner().unwrap();
+    let node_to_meshes = node_to_meshes.into_inner().unwrap();
+
+    if let Some(apath) = args.animation.clone() {
+        let path_base = apath.parent().unwrap();
+        let mut gltf =
+            Gltf::from_reader_without_validation(File::open(&apath)?).context("gltf parsing")?;
+        let blob = gltf.blob.take();
+        let buffers = import_buffers(&gltf, Some(path_base), blob).context("importing buffers")?;
+
+        let anim_name_map = if let Some(ref map_path) = args.animation_bone_map {
+            let mut map = BTreeMap::new();
+            for l in read_to_string(map_path)?.lines() {
+                if !l.trim().is_empty() && !l.starts_with(";") {
+                    let (a, b) = l.split_once("=").unwrap();
+                    map.insert(a.to_string(), b.to_string());
+                }
+            }
+            Some(map)
+        } else {
+            None
+        };
+
+        let mut anim_joint_index_to_arm_index = BTreeMap::new();
+        let mut joint_index_to_ibm = BTreeMap::new();
+        for n in gltf.nodes() {
+            if let Some(name) = n.name() {
+                let Some(vrm_name) = (if let Some(map) = &anim_name_map {
+                    map.get(name).cloned()
+                } else {
+                    Some(name.to_string())
+                }) else {
+                    continue;
+                };
+                let anim_node_index = n.index();
+                if let Some(scene_arm_index) = joint_name_to_arm_index.get(&vrm_name) {
+                    debug!(
+                        "mapping {name:?} (node={}) -> {vrm_name:?} (bone={})",
+                        anim_node_index, scene_arm_index
+                    );
+                    anim_joint_index_to_arm_index.insert(anim_node_index, *scene_arm_index);
+                }
+            }
+        }
+        if args.animation_apply_ibm {
+            for s in gltf.skins() {
+                let reader = s.reader(|buf| Some(&buffers[buf.index()]));
+                if let Some(ibms) = reader.read_inverse_bind_matrices() {
+                    for (jn, ibm) in s.joints().zip(ibms) {
+                        joint_index_to_ibm.insert(
+                            jn.index(),
+                            transform_to_affine(Transform::Matrix { matrix: ibm }),
+                        );
+                    }
+                }
+            }
+            debug!("{} joint IBMs found", joint_index_to_ibm.len());
+        }
+        let transform = args
+            .animation_rotation_y
+            .map(Affine3A::from_rotation_y)
+            .unwrap_or_default()
+            * args
+                .animation_scale
+                .map(Vec3::splat)
+                .map(Affine3A::from_scale)
+                .unwrap_or_default();
+        for a in gltf.animations() {
+            prefab.animation.push(import_animation(
+                a,
+                &store,
+                transform,
+                &joint_index_to_ibm,
+                &anim_joint_index_to_arm_index,
+                &BTreeMap::new(),
+                &buffers,
+            )?);
+        }
+    }
+
+    for a in gltf.animations() {
+        if !args.no_animations {
+            prefab.animation.push(import_animation(
+                a,
+                &store,
+                Affine3A::IDENTITY,
+                &BTreeMap::new(),
+                &joint_index_to_arm_index,
+                &node_to_meshes,
+                &buffers,
+            )?);
+        };
+    }
+
+    if vrm.camera_mount.is_some()
+        || vrm.camera_mount_offset.is_some()
+        || !vrm.bone_node_names.is_empty()
+    {
+        info!("avatar info enabled");
+        prefab.avatar_info = Some(
+            store.set(&AvatarInfoPart {
+                armature: Some(0), // TODO
+                camera_mount: vrm
+                    .camera_mount
+                    .map(|e| joint_index_to_arm_index[&e] as u32),
+                camera_mount_offset: vrm.camera_mount_offset,
+                ..Default::default()
+            })?,
+        );
+    }
+
+    prefab.armature = if armature.parent.as_ref().is_some_and(|a| !a.is_empty()) {
+        vec![store.set(&armature)?]
+    } else {
+        vec![]
+    };
+
+    let skybox = if let Some(skybox) = &args.skybox {
+        let mut buf = Vec::new();
+        File::open(skybox)?.read_to_end(&mut buf)?;
+        Some(store.set(&Image(Cow::Owned(buf)))?)
+    } else {
+        None
+    };
+    let sun = if args.with_default_sun {
+        Some((
+            vec3a(1., -5., 1.).normalize(),
+            vec3a(1., 1., 1.).normalize() * 100_000.,
+        ))
+    } else {
+        None
+    };
+    prefab.environment = if skybox.is_some() || sun.is_some() {
+        Some(store.set(&EnvironmentPart { skybox, sun })?)
+    } else {
+        None
+    };
+
+    prefab.name = args.name.clone().or(gltf
+        .default_scene()
+        .and_then(|n| n.name())
+        .map(|n| n.to_owned()));
+
+    if args.debug_light {
+        prefab.light.push((
+            vec3a(5., 5., 5.),
+            store.set(&LightPart {
+                name: Some("debug light".to_owned()),
+                emission: Some(vec3a(10., 5., 15.)),
+                radius: Some(0.3),
+            })?,
+        ));
+    }
+
+    Ok(store.set(&prefab)?)
+}