fix vrm first person bone

3 files changed, 65 insertions, 25 deletions

diff --git a/world/src/animation.rs b/world/src/animation.rs
index 00c716f..cb10123 100644
--- a/world/src/animation.rs
+++ b/world/src/animation.rs
@@ -23,6 +23,7 @@ use gltf::{
 use log::{debug, info};
 use std::collections::BTreeMap;
 use weareshared::{
+    Affine3A,
     packets::Resource,
     resources::{AnimationChannel, AnimationPart},
     store::ResourceStore,
@@ -31,6 +32,7 @@ use weareshared::{
 pub fn import_animation<'a>(
     a: Animation<'a>,
     store: &ResourceStore,
+    joint_index_to_ibm: &BTreeMap<usize, Affine3A>,
     joint_index_to_arm_index: &BTreeMap<usize, usize>,
     node_to_meshes: &BTreeMap<usize, Vec<usize>>,
     buffers: &[Data],
@@ -41,11 +43,30 @@ pub fn import_animation<'a>(
         let node = c.target().node().index();
         let reader = c.reader(|i| Some(&buffers[i.index()].0));
         let inputs: Vec<f32> = reader.read_inputs().unwrap().collect::<Vec<f32>>();
-        let outputs: Vec<f32> = match reader.read_outputs().unwrap() {
-            ReadOutputs::Translations(iter) => iter.flatten().collect(),
-            ReadOutputs::Rotations(iter) => iter.into_f32().flatten().collect(),
-            ReadOutputs::Scales(iter) => iter.flatten().collect(),
-            ReadOutputs::MorphTargetWeights(iter) => iter.into_f32().collect(),
+        let outputs: Vec<f32> = if let Some(_ibm) = joint_index_to_ibm.get(&node) {
+            debug!("pre-applying inverse bind matricies");
+            // let ibm =
+            // match reader.read_outputs().unwrap() {
+            //     // ReadOutputs::Translations(iter) => iter
+            //     //     .flat_map(|[x, y, z]| (ibm.matrix3 * vec3a(x, y, z)).to_array())
+            //     //     .collect(),
+            //     ReadOutputs::Rotations(iter) => iter.into_f32().map(|[a,b,c,d]| {
+
+            //     }).flatten().collect(),
+            //     // ReadOutputs::Scales(iter) => iter
+            //     //     .flat_map(|[x, y, z]| (ibm.matrix3 * vec3a(x, y, z)).to_array())
+            //     //     .collect(),
+            //     ReadOutputs::MorphTargetWeights(iter) => iter.into_f32().collect(),
+            //     _ => continue,
+            // }
+            todo!()
+        } else {
+            match reader.read_outputs().unwrap() {
+                ReadOutputs::Translations(iter) => iter.flatten().collect(),
+                ReadOutputs::Rotations(iter) => iter.into_f32().flatten().collect(),
+                ReadOutputs::Scales(iter) => iter.flatten().collect(),
+                ReadOutputs::MorphTargetWeights(iter) => iter.into_f32().collect(),
+            }
         };
         for x in &inputs {
             max_time = max_time.max(*x)
diff --git a/world/src/main.rs b/world/src/main.rs
index 9222b3e..be39a4a 100644
--- a/world/src/main.rs
+++ b/world/src/main.rs
@@ -403,6 +403,7 @@ fn main() -> Result<()> {
             };
 
             let mut anim_joint_index_to_arm_index = BTreeMap::new();
+            let mut joint_index_to_ibm = BTreeMap::new();
             for n in gltf.nodes() {
                 if let Some(name) = n.name() {
                     let Some(vrm_name) = (if let Some(map) = &anim_name_map {
@@ -422,10 +423,22 @@ fn main() -> Result<()> {
                     }
                 }
             }
+            for s in gltf.skins() {
+                let reader = s.reader(|buf| Some(&buffers[buf.index()]));
+                if let Some(ibms) = reader.read_inverse_bind_matrices() {
+                    for (jn, ibm) in s.joints().zip(ibms) {
+                        joint_index_to_ibm.insert(
+                            jn.index(),
+                            transform_to_affine(Transform::Matrix { matrix: ibm }),
+                        );
+                    }
+                }
+            }
             for a in gltf.animations() {
                 prefab.animation.push(import_animation(
                     a,
                     &store,
+                    &BTreeMap::new(),
                     &anim_joint_index_to_arm_index,
                     &BTreeMap::new(),
                     &buffers,
@@ -438,6 +451,7 @@ fn main() -> Result<()> {
                 prefab.animation.push(import_animation(
                     a,
                     &store,
+                    &BTreeMap::new(),
                     &joint_index_to_arm_index,
                     &node_to_meshes,
                     &buffers,
@@ -450,12 +464,16 @@ fn main() -> Result<()> {
             || !vrm.bone_node_names.is_empty()
         {
             info!("avatar info enabled");
-            prefab.avatar_info = Some(store.set(&AvatarInfoPart {
-                armature: Some(0), // TODO
-                camera_mount: vrm.camera_mount.map(|e| e as u32),
-                camera_mount_offset: vrm.camera_mount_offset,
-                ..Default::default()
-            })?);
+            prefab.avatar_info = Some(
+                store.set(&AvatarInfoPart {
+                    armature: Some(0), // TODO
+                    camera_mount: vrm
+                        .camera_mount
+                        .map(|e| joint_index_to_arm_index[&e] as u32),
+                    camera_mount_offset: vrm.camera_mount_offset,
+                    ..Default::default()
+                })?,
+            );
         }
 
         prefab.armature = if armature.parent.as_ref().is_some_and(|a| !a.is_empty()) {
diff --git a/world/src/vrm.rs b/world/src/vrm.rs
index a97323f..80865cb 100644
--- a/world/src/vrm.rs
+++ b/world/src/vrm.rs
@@ -20,7 +20,7 @@ use serde::Deserialize;
 use std::collections::{BTreeMap, BTreeSet};
 use weareshared::Vec3A;
 
-#[derive(Default)]
+#[derive(Debug, Default)]
 pub struct VrmInfo {
     pub bone_node_names: Vec<(usize, String)>,
     pub hide_first_person: BTreeSet<usize>,
@@ -30,16 +30,14 @@ pub struct VrmInfo {
 
 pub fn extract_vrm_data(gltf: &Gltf) -> Result<VrmInfo> {
     let mut o = VrmInfo::default();
+    let mut fp = None;
     if let Some(vrm) = gltf.extension_value("VRM") {
         // serde_json::to_writer(std::fs::File::create("/tmp/vrm").unwrap(), vrm).unwrap();
         let vrm: Vrm = serde_json::from_value(vrm.clone())?;
         for bone in vrm.humanoid.human_bones {
             o.bone_node_names.push((bone.node, bone.bone))
         }
-        if let Some(fp) = vrm.first_person {
-            o.camera_mount = fp.first_person_bone;
-            o.camera_mount_offset = fp.first_person_bone_offset.map(convert_vrm_vec);
-        }
+        fp = vrm.first_person;
     }
     if let Some(vrm) = gltf.extension_value("VRMC_vrm") {
         // serde_json::to_writer(std::fs::File::create("/tmp/vrmc").unwrap(), vrm).unwrap();
@@ -47,23 +45,26 @@ pub fn extract_vrm_data(gltf: &Gltf) -> Result<VrmInfo> {
         for (name, bone) in vrm.humanoid.human_bones {
             o.bone_node_names.push((bone.node, name))
         }
-        if let Some(fp) = vrm.first_person {
-            o.camera_mount = fp.first_person_bone;
-            o.camera_mount_offset = fp.first_person_bone_offset.map(convert_vrm_vec);
-            for ann in fp.mesh_annotations {
-                match ann.first_person_flag {
-                    FirstPersonFlag::ThirdPersonOnly => {
-                        o.hide_first_person.insert(ann.node);
-                    }
-                    _ => (),
+        fp = vrm.first_person;
+    }
+    if let Some(fp) = fp {
+        o.camera_mount = fp.first_person_bone;
+        o.camera_mount_offset = fp.first_person_bone_offset.map(convert_vrm_vec);
+        for ann in fp.mesh_annotations {
+            match ann.first_person_flag {
+                FirstPersonFlag::ThirdPersonOnly => {
+                    o.hide_first_person.insert(ann.node);
                 }
+                _ => (),
             }
         }
     }
+    eprintln!("{o:#?}");
     Ok(o)
 }
 
 #[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
 struct Vrm {
     humanoid: VrmHumanoid,
     first_person: Option<VrmcFirstPerson>,