/* wearechat - generic multiplayer game with voip Copyright (C) 2025 metamuffin This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by the Free Software Foundation, version 3 of the License only. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with this program. If not, see . */ #![feature(iter_array_chunks)] #![allow(clippy::too_many_arguments, clippy::type_complexity)] pub mod mesh; pub mod physics; pub mod vrm; use anyhow::{Context, Result, anyhow}; use clap::Parser; use gltf::{ Gltf, Node, animation::{Property, util::ReadOutputs}, image::Source, import_buffers, scene::Transform, }; use humansize::BINARY; use image::{ImageReader, codecs::webp::WebPEncoder}; use log::{debug, info}; use mesh::import_mesh; use physics::import_physics; use rand::random; use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; use serde::Deserialize; use serde_json::Value; use std::{ borrow::Cow, collections::{BTreeMap, HashMap}, fs::File, io::{Cursor, Read, Write}, marker::PhantomData, net::{SocketAddr, TcpStream}, path::{Path, PathBuf}, sync::{Arc, Mutex}, thread::{self, sleep}, time::Duration, }; use vrm::extract_vrm_data; use weareshared::{ Affine3A, Vec3A, helper::ReadWrite, packets::{Data, Object, Packet, Resource}, resources::{ AnimationChannel, AnimationPart, ArmaturePart, EnvironmentPart, Image, LightPart, ParticlesPart, Prefab, }, store::ResourceStore, vec3a, }; #[derive(Parser)] pub struct Args { address: SocketAddr, /// Path(s) to a glTF file, binary or json format scene: Vec, /// Send all resources to the server then quit #[arg(short, long)] push: bool, /// Remove all other object from the world #[arg(short, long)] clear: bool, /// Add the object to the world #[arg(short, long)] add: bool, /// Transcode all textures to WebP #[arg(short, long)] webp: bool, /// Add skybox #[arg(long)] skybox: Option, /// Override prefab name #[arg(short, long)] name: Option, #[arg(short, long)] scale: Option, #[arg(short, long)] dry_run: bool, #[arg(short, long)] line_up: bool, #[arg(long)] use_cache: bool, #[arg(short = 'S', long)] with_default_sun: bool, /// Spins the object #[arg(long)] debug_spin: bool, /// Adds a light #[arg(long)] debug_light: bool, /// Wiggles joint index 6 #[arg(long)] debug_armature: bool, } fn main() -> Result<()> { env_logger::init_from_env("LOG"); let args = Args::parse(); let store = if args.use_cache { ResourceStore::new_env()? } else { ResourceStore::new_memory() }; let mut prefabs = Vec::new(); let texture_cache = Arc::new(Mutex::new(HashMap::new())); let mut root_affine = Affine3A::IDENTITY; root_affine.matrix3 *= args.scale.unwrap_or(1.); root_affine.translation *= args.scale.unwrap_or(1.); let mut debug_bone = (0, Affine3A::IDENTITY); for scenepath in &args.scene { let path_base = scenepath.parent().unwrap(); let mut gltf = Gltf::from_reader_without_validation(File::open(scenepath)?)?; let blob = gltf.blob.take(); let buffers = import_buffers(&gltf, Some(path_base), blob)?; let root = gltf.default_scene().ok_or(anyhow!("no default scene"))?; // gltf.as_json().asset.copyright // eprintln!("{:?}", gltf.extensions_used()); // eprintln!("{:?}", gltf.extensions()); // eprintln!("{:?}", root.extensions()); // eprintln!("{:?}", root.extras()); let mut nodes = Vec::new(); fn traverse<'a>(out: &mut Vec<(Affine3A, Node<'a>)>, node: Node<'a>, trans: Affine3A) { let trans = trans * transform_to_affine(node.transform()); for c in node.children() { traverse(out, c, trans); } out.push((trans, node)); } for node in root.nodes() { traverse(&mut nodes, node, root_affine); } let vrm = extract_vrm_data(&gltf)?; let mut skin_index_to_arm_index = BTreeMap::new(); let armature = { let mut joint_index_to_arm_index = BTreeMap::new(); let mut name = Vec::new(); let mut parent_pre_map = Vec::new(); let mut transform = Vec::new(); for skin in gltf.skins() { for (j_ind, j) in skin.joints().enumerate() { let a_ind = match joint_index_to_arm_index.get(&j.index()) { Some(i) => *i, None => { let a_ind = name.len(); // name.push(j.name().unwrap_or("").to_string()); name.push(format!("bone{}", a_ind)); transform.push(transform_to_affine(j.transform())); parent_pre_map.push( gltf.nodes() .find(|n| n.children().any(|c| c.index() == j.index())) .map(|n| n.index()), ); joint_index_to_arm_index.insert(j.index(), a_ind); a_ind } }; skin_index_to_arm_index.insert((skin.index(), j_ind as u16), a_ind as u32); } } let parent = parent_pre_map .into_iter() .enumerate() .map(|(i, p)| { p.and_then(|i| joint_index_to_arm_index.get(&i).copied()) .unwrap_or_else(|| i) as u16 }) .collect::>(); for (node, bname) in &vrm.bone_node_names { let ind = joint_index_to_arm_index[node]; name[ind] = bname.to_owned(); } for (i, (name, tr)) in name.iter().zip(transform.iter()).enumerate() { if name == "head" { debug_bone = (i, *tr); } } ArmaturePart { name: Some(name), parent: Some(parent), transform: Some(transform), } }; let head_bones = { let pa = armature.parent.clone().unwrap_or_default(); let na = armature.name.clone().unwrap_or_default(); (0..pa.len()) .filter(|&(mut i)| { let mut f = false; while pa[i] as usize != i { f |= na[i] == "head"; i = pa[i] as usize; } f }) .map(|e| e as u32) .collect::>() }; let prefab = Mutex::new(Prefab::default()); let node_to_meshes = Mutex::new(BTreeMap::>::new()); nodes .par_iter() .map(|(trans, node)| { // if node.name().unwrap_or_default() == "particles" { // eprintln!("{:?}", node.transform()); // eprintln!("{:?}", node.extensions()); // eprintln!("{:?}", node.extras()); // } let name = node.name().unwrap_or_default(); let extras: Value = node .extras() .to_owned() .map(|v| serde_json::from_str(v.get()).unwrap()) .unwrap_or(Value::Object(Default::default())); if !name.ends_with("-collider") { if let Some(mesh) = node.mesh() { let meshes = import_mesh( mesh, *trans, &buffers, &store, path_base, node, &args, &texture_cache, &skin_index_to_arm_index, &vrm, &head_bones, )?; for (node, trans, mesh) in meshes { let mut k = prefab.lock().unwrap(); let i = k.mesh.len(); k.mesh.push((trans, mesh)); node_to_meshes .lock() .unwrap() .entry(node) .or_default() .push(i) } } } if extras.get("particles") == Some(&Value::Bool(true)) { #[derive(Deserialize)] struct ParticlesAttr { density: Option, lifetime: Option, lifetime_spread: Option, velocity: Option, velocity_spread: Option, } // let sprite = extras // .get("sprite") // .ok_or(anyhow!("particle volume is missing sprite"))?; let attr: ParticlesAttr = serde_json::from_value(extras).context("particles attributes")?; info!("adding particles part"); let part = store.set(&ParticlesPart { sprite: None, density: attr.density, lifetime: attr.lifetime, lifetime_spread: attr.lifetime_spread, velocity: attr.velocity, velocity_spread: attr.velocity_spread, })?; prefab .lock() .unwrap() .particles .push((transform_to_affine(node.transform()), part)); } if let Some(light) = node.light() { let name = node.name().map(|e| e.to_owned()); if let Some(name) = &name { info!("adding light {name:?}"); } else { info!("adding light"); } let emission = Some(Vec3A::from_array(light.color()) * light.intensity()); if let Some(e) = emission { debug!("emission is {e}"); } let (position, _, _) = node.transform().decomposed(); let part = store.set(&LightPart { emission, name, radius: None, })?; prefab .lock() .unwrap() .light .push((Vec3A::from_array(position), part)); } import_physics(&gltf, *trans, node, &store, &buffers)?; Ok::<_, anyhow::Error>(()) }) .reduce( || Ok(()), |a, b| match (a, b) { (Ok(()), Ok(())) => Ok(()), (Ok(()), a) => a, (a, _) => a, }, )?; let mut prefab = prefab.into_inner().unwrap(); let node_to_meshes = node_to_meshes.into_inner().unwrap(); for a in gltf.animations() { let mut max_time = 0f32; let mut channels = Vec::new(); for c in a.channels() { let node = c.target().node().index(); let reader = c.reader(|i| Some(&buffers[i.index()].0)); let inputs: Vec = reader.read_inputs().unwrap().collect::>(); let outputs: Vec = match reader.read_outputs().unwrap() { ReadOutputs::Translations(iter) => iter.flatten().collect(), ReadOutputs::Rotations(iter) => iter.into_f32().flatten().collect(), ReadOutputs::Scales(iter) => iter.flatten().collect(), ReadOutputs::MorphTargetWeights(iter) => iter.into_f32().collect(), }; for x in &inputs { max_time = max_time.max(*x) } let time = store.set(&inputs)?; let value = store.set(&outputs)?; let meshes = node_to_meshes .get(&node) .ok_or(anyhow!("animation ref to meshless node"))?; for &m in meshes { let mut ch = AnimationChannel::default(); match c.target().property() { Property::Translation => ch.t_mesh_translation = Some(m as u32), Property::Rotation => ch.t_mesh_rotation = Some(m as u32), Property::Scale => ch.t_mesh_scale = Some(m as u32), Property::MorphTargetWeights => todo!(), } ch.time = Some(time.clone()); ch.value = Some(value.clone()); debug!( "animation channel {:?} of mesh {m} with {} times and {} component values", c.target().property(), inputs.len(), outputs.len() ); channels.push(ch); } } info!("adding animation with {} channels", channels.len()); prefab.animations.push(store.set(&AnimationPart { name: a.name().map(|n| n.to_string()), channel: channels, duration: Some(max_time), })?); } prefab.armature = if armature.parent.as_ref().is_some_and(|a| !a.is_empty()) { vec![store.set(&armature)?] } else { vec![] }; let skybox = if let Some(skybox) = &args.skybox { let mut buf = Vec::new(); File::open(skybox)?.read_to_end(&mut buf)?; Some(store.set(&Image(Cow::Owned(buf)))?) } else { None }; let sun = if args.with_default_sun { Some(( vec3a(1., -5., 1.).normalize(), vec3a(1., 1., 1.).normalize() * 100_000., )) } else { None }; prefab.environment = if skybox.is_some() || sun.is_some() { Some(store.set(&EnvironmentPart { skybox, sun })?) } else { None }; prefab.name = args.name.clone().or(gltf .default_scene() .and_then(|n| n.name()) .map(|n| n.to_owned())); if args.debug_light { prefab.light.push(( vec3a(5., 5., 5.), store.set(&LightPart { name: Some("debug light".to_owned()), emission: Some(vec3a(10., 5., 15.)), radius: Some(0.3), })?, )); } prefabs.push(store.set(&prefab)?); } let mut size = 0; store.iter(|_k, len| size += len).unwrap(); info!( "prefab has network size of {}", humansize::format_size(size, BINARY) ); if args.dry_run { return Ok(()); } let mut sock = TcpStream::connect(args.address)?; Packet::Connect(random()).write(&mut sock)?; for p in &prefabs { Packet::AnnouncePrefab(p.clone()).write(&mut sock)?; } sock.flush()?; let mut obs = Vec::new(); if args.add { for (i, p) in prefabs.iter().enumerate() { let ob = Object::new(); info!("adding object {ob}"); Packet::Add(ob, p.clone()).write(&mut sock)?; if args.line_up { Packet::Position(ob, vec3a(i as f32 * 1.2, 0., i as f32 * 0.3), Vec3A::ZERO) .write(&mut sock)?; } obs.push(ob); } sock.flush()?; } if args.debug_spin { let ob = obs[0]; let mut sock2 = sock.try_clone().unwrap(); thread::spawn(move || { let mut x = 0.; loop { Packet::Position(ob, Vec3A::ZERO, vec3a(0., x * 0.1, 0.)) .write(&mut sock2) .unwrap(); sock2.flush().unwrap(); x += 0.1; sleep(Duration::from_millis(50)); } }); } if args.debug_armature { let ob = obs[0]; let mut sock2 = sock.try_clone().unwrap(); thread::spawn(move || { let mut x = 0f32; loop { let a = Affine3A::IDENTITY * Affine3A::from_rotation_x(x.cos() * 0.3) * Affine3A::from_rotation_y(x.sin() * 0.5) * debug_bone.1; Packet::Pose(ob, vec![(debug_bone.0 as u16, a)]) .write(&mut sock2) .unwrap(); sock2.flush().unwrap(); x += 0.2; sleep(Duration::from_millis(50)); } }); } if args.push { if args.use_cache { return Ok(()); } store.iter(|k, _| { Packet::RespondResource(k, Data(store.get_raw(k).unwrap().unwrap())) .write(&mut sock) .unwrap(); })?; sock.flush()?; } else { loop { let packet = Packet::read(&mut sock)?; match packet { Packet::RequestResource(hash) => { if let Some(d) = store.get_raw(hash)? { Packet::RespondResource(hash, Data(d)).write(&mut sock)?; sock.flush()?; } } Packet::Add(ob_a, _) => { if args.clear && !obs.contains(&ob_a) { info!("removing object {ob_a}"); Packet::Remove(ob_a).write(&mut sock)?; sock.flush()?; } } _ => (), } } } Ok(()) } pub type TextureCache = Arc>>>>; fn load_texture( name: &str, store: &ResourceStore, path: &Path, buffers: &[gltf::buffer::Data], source: &Source, webp: bool, texture_cache: &TextureCache, ) -> Result>> { let (mut image, uri) = match source { gltf::image::Source::View { view, mime_type } => { debug!("{name} texture is embedded and of type {mime_type:?}"); let buf = &buffers[view.buffer().index()].0[view.offset()..view.offset() + view.length()]; (Image(Cow::Borrowed(buf)), None) } gltf::image::Source::Uri { uri, mime_type: Some(mime_type), } => { debug!("{name} texture is {uri:?} and of type {mime_type:?}"); if let Some(res) = texture_cache.lock().unwrap().get(*uri) { return Ok(res.to_owned()); } let path = path.join(uri); let mut buf = Vec::new(); File::open(path)?.read_to_end(&mut buf)?; (Image(buf.into()), Some(uri.to_string())) } gltf::image::Source::Uri { uri, mime_type: None, } => { debug!("{name} texture is {uri:?} and has no type"); if let Some(res) = texture_cache.lock().unwrap().get(*uri) { return Ok(res.to_owned()); } let path = path.join(uri); let mut buf = Vec::new(); File::open(path)?.read_to_end(&mut buf)?; (Image(buf.into()), Some(uri.to_string())) } }; if webp { let mut image_out = Vec::new(); let len = image.0.len(); ImageReader::new(Cursor::new(image.0)) .with_guessed_format()? .decode()? .write_with_encoder(WebPEncoder::new_lossless(&mut image_out))?; debug!("webp encode: {len} -> {}", image_out.len()); image = Image(Cow::Owned(image_out)); } let res = Resource(store.set(&image)?.0, PhantomData); if let Some(uri) = uri { texture_cache.lock().unwrap().insert(uri, res.clone()); } Ok(res) } pub fn transform_to_affine(trans: Transform) -> Affine3A { let mat = trans.matrix(); Affine3A::from_cols_array_2d(&[ [mat[0][0], mat[0][1], mat[0][2]], [mat[1][0], mat[1][1], mat[1][2]], [mat[2][0], mat[2][1], mat[2][2]], [mat[3][0], mat[3][1], mat[3][2]], ]) }