/* wearechat - generic multiplayer game with voip Copyright (C) 2025 metamuffin This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by the Free Software Foundation, version 3 of the License only. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with this program. If not, see . */ #![feature(iter_array_chunks)] #![allow(clippy::too_many_arguments, clippy::type_complexity)] pub mod animation; pub mod mesh; pub mod physics; pub mod vrm; use animation::import_animation; use anyhow::{Context, Result, anyhow}; use clap::Parser; use glam::Vec3; use gltf::{Gltf, Node, image::Source, import_buffers, scene::Transform}; use humansize::BINARY; use image::{ImageReader, codecs::webp::WebPEncoder}; use log::{debug, info}; use mesh::import_mesh; use physics::import_physics; use rand::random; use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; use serde::Deserialize; use serde_json::Value; use std::{ borrow::Cow, collections::{BTreeMap, HashMap}, fs::{File, read_to_string}, io::{Cursor, Read, Write}, marker::PhantomData, net::{SocketAddr, TcpStream}, path::{Path, PathBuf}, sync::{Arc, Mutex}, thread::{self, sleep}, time::Duration, }; use vrm::extract_vrm_data; use weareshared::{ Affine3A, Vec3A, helper::ReadWrite, packets::{Data, Object, Packet, Resource}, resources::{ ArmaturePart, AvatarInfoPart, EnvironmentPart, Image, LightPart, ParticlesPart, Prefab, }, store::ResourceStore, vec3a, }; #[derive(Parser)] pub struct Args { address: SocketAddr, /// Path(s) to a glTF file, binary or json format scene: Vec, /// Send all resources to the server then quit #[arg(short, long)] push: bool, /// Remove all other object from the world #[arg(short, long)] clear: bool, /// Add the object to the world #[arg(short, long)] add: bool, /// Transcode all textures to WebP #[arg(short, long)] webp: bool, /// Add skybox #[arg(long)] skybox: Option, /// Override prefab name #[arg(short, long)] name: Option, #[arg(short, long)] scale: Option, #[arg(short, long)] dry_run: bool, #[arg(short, long)] line_up: bool, #[arg(long)] use_cache: bool, #[arg(short = 'S', long)] with_default_sun: bool, #[arg(long)] animation: Option, #[arg(long)] animation_bone_map: Option, #[arg(long)] animation_rotation_y: Option, #[arg(long)] animation_scale: Option, #[arg(long)] animation_apply_ibm: bool, /// Spins the object #[arg(long)] debug_spin: bool, /// Adds a light #[arg(long)] debug_light: bool, /// Wiggles joint index 6 #[arg(long)] debug_armature: bool, #[arg(long)] no_particles: bool, #[arg(long)] no_animations: bool, } fn main() -> Result<()> { env_logger::init_from_env("LOG"); let mut args = Args::parse(); let store = if args.use_cache { ResourceStore::new_env()? } else { ResourceStore::new_memory() }; let mut prefabs = Vec::new(); let texture_cache = Arc::new(Mutex::new(HashMap::new())); let mut root_affine = Affine3A::IDENTITY; root_affine.matrix3 *= args.scale.unwrap_or(1.); root_affine.translation *= args.scale.unwrap_or(1.); let mut debug_bone = (0, Affine3A::IDENTITY); for scenepath in &args.scene { let path_base = scenepath.parent().unwrap(); let mut gltf = Gltf::from_reader_without_validation(File::open(scenepath)?).context("gltf parsing")?; let blob = gltf.blob.take(); let buffers = import_buffers(&gltf, Some(path_base), blob).context("importing buffers")?; let root = gltf.default_scene().ok_or(anyhow!("no default scene"))?; // gltf.as_json().asset.copyright // eprintln!("{:?}", gltf.extensions_used()); // eprintln!("{:?}", gltf.extensions()); // eprintln!("{:?}", root.extensions()); // eprintln!("{:?}", root.extras()); let mut nodes = Vec::new(); fn traverse<'a>(out: &mut Vec<(Affine3A, Node<'a>)>, node: Node<'a>, trans: Affine3A) { let trans = trans * transform_to_affine(node.transform()); for c in node.children() { traverse(out, c, trans); } out.push((trans, node)); } for node in root.nodes() { traverse(&mut nodes, node, root_affine); } let vrm = extract_vrm_data(&gltf)?; let mut skin_index_to_arm_index = BTreeMap::new(); let mut joint_index_to_arm_index = BTreeMap::new(); let joint_name_to_arm_index: BTreeMap; let armature = { let mut name = Vec::new(); let mut parent_pre_map = Vec::new(); let mut transform = Vec::new(); let mut inverse_bind_transform = Vec::new(); for skin in gltf.skins() { let mut inverse_bind_mat = skin .reader(|buf| Some(&buffers[buf.index()])) .read_inverse_bind_matrices(); for (j_ind, j) in skin.joints().enumerate() { let ibm = inverse_bind_mat.as_mut().map(|x| x.next().unwrap()); let a_ind = match joint_index_to_arm_index.get(&j.index()) { Some(i) => *i, None => { let a_ind = name.len(); // name.push(j.name().unwrap_or("").to_string()); name.push(format!("bone{}", a_ind)); transform.push(transform_to_affine(j.transform())); parent_pre_map.push( gltf.nodes() .find(|n| n.children().any(|c| c.index() == j.index())) .map(|n| n.index()), ); inverse_bind_transform.push( ibm.map(|a| transform_to_affine(Transform::Matrix { matrix: a })) .unwrap_or(Affine3A::IDENTITY), ); joint_index_to_arm_index.insert(j.index(), a_ind); a_ind } }; skin_index_to_arm_index.insert((skin.index(), j_ind as u16), a_ind as u32); } } let parent = parent_pre_map .into_iter() .enumerate() .map(|(i, p)| { p.and_then(|i| joint_index_to_arm_index.get(&i).copied()) .unwrap_or(i) as u16 }) .collect::>(); for (node, bname) in &vrm.bone_node_names { let ind = joint_index_to_arm_index[node]; name[ind] = bname.to_owned(); } for (i, (name, tr)) in name.iter().zip(transform.iter()).enumerate() { if name == "head" { debug_bone = (i, *tr); } } joint_name_to_arm_index = name .iter() .cloned() .enumerate() .map(|(a, b)| (b, a)) .collect(); ArmaturePart { name: Some(name), parent: Some(parent), transform: Some(transform), inverse_bind_transform: Some(inverse_bind_transform), } }; let head_bones = { let pa = armature.parent.clone().unwrap_or_default(); let na = armature.name.clone().unwrap_or_default(); (0..pa.len()) .filter(|&(mut i)| { let mut f = false; while pa[i] as usize != i { f |= na[i] == "head"; i = pa[i] as usize; } f }) .map(|e| e as u32) .collect::>() }; let prefab = Mutex::new(Prefab::default()); let node_to_meshes = Mutex::new(BTreeMap::>::new()); nodes .par_iter() .map(|(trans, node)| { // if node.name().unwrap_or_default() == "particles" { // eprintln!("{:?}", node.transform()); // eprintln!("{:?}", node.extensions()); // eprintln!("{:?}", node.extras()); // } let name = node.name().unwrap_or_default(); let extras: Value = node .extras() .to_owned() .map(|v| serde_json::from_str(v.get()).unwrap()) .unwrap_or(Value::Object(Default::default())); if !name.ends_with("-collider") { if let Some(mesh) = node.mesh() { let meshes = import_mesh( mesh, *trans, &buffers, &store, path_base, node, &args, &texture_cache, &skin_index_to_arm_index, &vrm, &head_bones, )?; for (node, trans, mesh) in meshes { let mut k = prefab.lock().unwrap(); let i = k.mesh.len(); k.mesh.push((trans, mesh)); node_to_meshes .lock() .unwrap() .entry(node) .or_default() .push(i) } } } if extras.get("particles") == Some(&Value::Bool(true)) && !args.no_particles { #[derive(Deserialize)] struct ParticlesAttr { density: Option, lifetime: Option, lifetime_spread: Option, velocity: Option, velocity_spread: Option, } // let sprite = extras // .get("sprite") // .ok_or(anyhow!("particle volume is missing sprite"))?; let attr: ParticlesAttr = serde_json::from_value(extras).context("particles attributes")?; info!("adding particles part"); let part = store.set(&ParticlesPart { sprite: None, density: attr.density, lifetime: attr.lifetime, lifetime_spread: attr.lifetime_spread, velocity: attr.velocity, velocity_spread: attr.velocity_spread, })?; prefab .lock() .unwrap() .particles .push((transform_to_affine(node.transform()), part)); } if let Some(light) = node.light() { let name = node.name().map(|e| e.to_owned()); if let Some(name) = &name { info!("adding light {name:?}"); } else { info!("adding light"); } let emission = Some(Vec3A::from_array(light.color()) * light.intensity()); if let Some(e) = emission { debug!("emission is {e}"); } let (position, _, _) = node.transform().decomposed(); let part = store.set(&LightPart { emission, name, radius: None, })?; prefab .lock() .unwrap() .light .push((Vec3A::from_array(position), part)); } { let collider = import_physics(&gltf, *trans, node, &store, &buffers)?; prefab.lock().unwrap().collision.extend(collider); } Ok::<_, anyhow::Error>(()) }) .reduce( || Ok(()), |a, b| match (a, b) { (Ok(()), Ok(())) => Ok(()), (Ok(()), a) => a, (a, _) => a, }, )?; let mut prefab = prefab.into_inner().unwrap(); let node_to_meshes = node_to_meshes.into_inner().unwrap(); if let Some(apath) = args.animation.take() { let path_base = apath.parent().unwrap(); let mut gltf = Gltf::from_reader_without_validation(File::open(&apath)?) .context("gltf parsing")?; let blob = gltf.blob.take(); let buffers = import_buffers(&gltf, Some(path_base), blob).context("importing buffers")?; let anim_name_map = if let Some(ref map_path) = args.animation_bone_map { let mut map = BTreeMap::new(); for l in read_to_string(map_path)?.lines() { if !l.trim().is_empty() && !l.starts_with(";") { let (a, b) = l.split_once("=").unwrap(); map.insert(a.to_string(), b.to_string()); } } Some(map) } else { None }; let mut anim_joint_index_to_arm_index = BTreeMap::new(); let mut joint_index_to_ibm = BTreeMap::new(); for n in gltf.nodes() { if let Some(name) = n.name() { let Some(vrm_name) = (if let Some(map) = &anim_name_map { map.get(name).cloned() } else { Some(name.to_string()) }) else { continue; }; let anim_node_index = n.index(); if let Some(scene_arm_index) = joint_name_to_arm_index.get(&vrm_name) { debug!( "mapping {name:?} (node={}) -> {vrm_name:?} (bone={})", anim_node_index, scene_arm_index ); anim_joint_index_to_arm_index.insert(anim_node_index, *scene_arm_index); } } } if args.animation_apply_ibm { for s in gltf.skins() { let reader = s.reader(|buf| Some(&buffers[buf.index()])); if let Some(ibms) = reader.read_inverse_bind_matrices() { for (jn, ibm) in s.joints().zip(ibms) { joint_index_to_ibm.insert( jn.index(), transform_to_affine(Transform::Matrix { matrix: ibm }), ); } } } debug!("{} joint IBMs found", joint_index_to_ibm.len()); } let transform = args .animation_rotation_y .map(Affine3A::from_rotation_y) .unwrap_or_default() * args .animation_scale .map(Vec3::splat) .map(Affine3A::from_scale) .unwrap_or_default(); for a in gltf.animations() { prefab.animation.push(import_animation( a, &store, transform, &joint_index_to_ibm, &anim_joint_index_to_arm_index, &BTreeMap::new(), &buffers, )?); } } for a in gltf.animations() { if !args.no_animations { prefab.animation.push(import_animation( a, &store, Affine3A::IDENTITY, &BTreeMap::new(), &joint_index_to_arm_index, &node_to_meshes, &buffers, )?); }; } if vrm.camera_mount.is_some() || vrm.camera_mount_offset.is_some() || !vrm.bone_node_names.is_empty() { info!("avatar info enabled"); prefab.avatar_info = Some( store.set(&AvatarInfoPart { armature: Some(0), // TODO camera_mount: vrm .camera_mount .map(|e| joint_index_to_arm_index[&e] as u32), camera_mount_offset: vrm.camera_mount_offset, ..Default::default() })?, ); } prefab.armature = if armature.parent.as_ref().is_some_and(|a| !a.is_empty()) { vec![store.set(&armature)?] } else { vec![] }; let skybox = if let Some(skybox) = &args.skybox { let mut buf = Vec::new(); File::open(skybox)?.read_to_end(&mut buf)?; Some(store.set(&Image(Cow::Owned(buf)))?) } else { None }; let sun = if args.with_default_sun { Some(( vec3a(1., -5., 1.).normalize(), vec3a(1., 1., 1.).normalize() * 100_000., )) } else { None }; prefab.environment = if skybox.is_some() || sun.is_some() { Some(store.set(&EnvironmentPart { skybox, sun })?) } else { None }; prefab.name = args.name.clone().or(gltf .default_scene() .and_then(|n| n.name()) .map(|n| n.to_owned())); if args.debug_light { prefab.light.push(( vec3a(5., 5., 5.), store.set(&LightPart { name: Some("debug light".to_owned()), emission: Some(vec3a(10., 5., 15.)), radius: Some(0.3), })?, )); } prefabs.push(store.set(&prefab)?); } let mut size = 0; store.iter(|_k, len| size += len).unwrap(); info!( "prefab has network size of {}", humansize::format_size(size, BINARY) ); if args.dry_run { return Ok(()); } let mut sock = TcpStream::connect(args.address)?; Packet::Connect(random()).write(&mut sock)?; for p in &prefabs { Packet::AnnouncePrefab(p.clone()).write(&mut sock)?; } sock.flush()?; let mut obs = Vec::new(); if args.add { for (i, p) in prefabs.iter().enumerate() { let ob = Object::new(); info!("adding object {ob}"); Packet::Add(ob, p.clone()).write(&mut sock)?; if args.line_up { Packet::Position(ob, vec3a(i as f32 * 1.2, 0., i as f32 * 0.3), Vec3A::ZERO) .write(&mut sock)?; } obs.push(ob); } sock.flush()?; } if args.debug_spin { let ob = obs[0]; let mut sock2 = sock.try_clone().unwrap(); thread::spawn(move || { let mut x = 0.; loop { Packet::Position(ob, Vec3A::ZERO, vec3a(0., x * 0.1, 0.)) .write(&mut sock2) .unwrap(); sock2.flush().unwrap(); x += 0.1; sleep(Duration::from_millis(50)); } }); } if args.debug_armature { let ob = obs[0]; let mut sock2 = sock.try_clone().unwrap(); thread::spawn(move || { let mut x = 0f32; loop { let a = Affine3A::IDENTITY * Affine3A::from_rotation_x(x.cos() * 0.3) * Affine3A::from_rotation_y(x.sin() * 0.5) * debug_bone.1; Packet::Pose(ob, vec![(debug_bone.0 as u16, a)]) .write(&mut sock2) .unwrap(); sock2.flush().unwrap(); x += 0.2; sleep(Duration::from_millis(50)); } }); } if args.push { if args.use_cache { return Ok(()); } store.iter(|k, _| { Packet::RespondResource(k, Data(store.get_raw(k).unwrap().unwrap())) .write(&mut sock) .unwrap(); })?; sock.flush()?; } else { loop { let packet = Packet::read(&mut sock)?; match packet { Packet::RequestResource(hash) => { if let Some(d) = store.get_raw(hash)? { Packet::RespondResource(hash, Data(d)).write(&mut sock)?; sock.flush()?; } } Packet::Add(ob_a, _) => { if args.clear && !obs.contains(&ob_a) { info!("removing object {ob_a}"); Packet::Remove(ob_a).write(&mut sock)?; sock.flush()?; } } _ => (), } } } Ok(()) } pub type TextureCache = Arc>>>>; fn load_texture( name: &str, store: &ResourceStore, path: &Path, buffers: &[gltf::buffer::Data], source: &Source, webp: bool, texture_cache: &TextureCache, ) -> Result>> { let (mut image, uri) = match source { gltf::image::Source::View { view, mime_type } => { debug!("{name} texture is embedded and of type {mime_type:?}"); let buf = &buffers[view.buffer().index()].0[view.offset()..view.offset() + view.length()]; (Image(Cow::Borrowed(buf)), None) } gltf::image::Source::Uri { uri, mime_type: Some(mime_type), } => { debug!("{name} texture is {uri:?} and of type {mime_type:?}"); if let Some(res) = texture_cache.lock().unwrap().get(*uri) { return Ok(res.to_owned()); } let path = path.join(uri); let mut buf = Vec::new(); File::open(path)?.read_to_end(&mut buf)?; (Image(buf.into()), Some(uri.to_string())) } gltf::image::Source::Uri { uri, mime_type: None, } => { debug!("{name} texture is {uri:?} and has no type"); if let Some(res) = texture_cache.lock().unwrap().get(*uri) { return Ok(res.to_owned()); } let path = path.join(uri); let mut buf = Vec::new(); File::open(path)?.read_to_end(&mut buf)?; (Image(buf.into()), Some(uri.to_string())) } }; if webp { let mut image_out = Vec::new(); let len = image.0.len(); ImageReader::new(Cursor::new(image.0)) .with_guessed_format()? .decode()? .write_with_encoder(WebPEncoder::new_lossless(&mut image_out))?; debug!("webp encode: {len} -> {}", image_out.len()); image = Image(Cow::Owned(image_out)); } let res = Resource(store.set(&image)?.0, PhantomData); if let Some(uri) = uri { texture_cache.lock().unwrap().insert(uri, res.clone()); } Ok(res) } pub fn transform_to_affine(trans: Transform) -> Affine3A { let mat = trans.matrix(); Affine3A::from_cols_array_2d(&[ [mat[0][0], mat[0][1], mat[0][2]], [mat[1][0], mat[1][1], mat[1][2]], [mat[2][0], mat[2][1], mat[2][2]], [mat[3][0], mat[3][1], mat[3][2]], ]) }