From 11eb107fbc0a4d8f667b5fb201569ebd5dd49e77 Mon Sep 17 00:00:00 2001 From: metamuffin Date: Wed, 12 Mar 2025 18:38:38 +0100 Subject: cli and respack works --- Cargo.lock | 5 +- cli/Cargo.toml | 1 + cli/src/main.rs | 91 ++++++++++++++++++++++++++++++++++- doc/resources.md | 6 +++ import/src/main.rs | 11 +++-- shared/src/resources.rs | 2 +- shared/src/respack.rs | 123 ++++++++++++++++++++++++++++++++---------------- shared/src/store.rs | 29 +++++++++++- 8 files changed, 217 insertions(+), 51 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 85123c8..fb4f1e5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1654,9 +1654,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.22" +version = "0.4.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" +checksum = "30bde2b3dc3671ae49d8e2e9f044c7c005836e7a023ee57cffa25ab82764bb9e" [[package]] name = "loop9" @@ -3417,6 +3417,7 @@ dependencies = [ "anyhow", "clap 4.5.23", "env_logger", + "log", "weareshared", ] diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 84587ba..535553b 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -12,3 +12,4 @@ anyhow = "1.0.95" clap = { version = "4.5.23", features = ["derive"] } env_logger = "0.11.6" weareshared = { path = "../shared" } +log = "0.4.26" diff --git a/cli/src/main.rs b/cli/src/main.rs index e7a11a9..9cf5e3a 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -1,3 +1,90 @@ -fn main() { - println!("Hello, world!"); +#![feature(random)] +use std::{ + fs::File, + io::Write, + net::{SocketAddr, TcpStream}, + path::PathBuf, + random::random, +}; + +use anyhow::Result; +use clap::{Parser, Subcommand}; +use log::info; +use weareshared::{ + helper::ReadWrite, + packets::{Data, Object, Packet}, + respack::RespackReader, + store::ResourceStore, +}; + +#[derive(Parser)] +struct Args { + #[clap(subcommand)] + action: Action, +} + +#[derive(Subcommand)] +enum Action { + Add(Add), +} + +#[derive(Parser)] +struct Add { + address: SocketAddr, + pack: PathBuf, + #[arg(short = 'A', long)] + announce: bool, + #[arg(short, long)] + add: bool, + #[arg(short, long)] + disown: bool, +} + +fn main() -> Result<()> { + env_logger::init_from_env("LOG"); + let args = Args::parse(); + + match args.action { + Action::Add(a) => action_add(a), + } +} + +fn action_add(args: Add) -> Result<()> { + info!("loading pack"); + let pack = RespackReader::open(File::open(&args.pack)?)?; + let entry = pack.entry().unwrap(); + let store = ResourceStore::new_respack(pack)?; + let entry = store.get(entry)?.unwrap(); + + info!("connecting"); + let mut sock = TcpStream::connect(args.address)?; + Packet::Connect(random()).write(&mut sock)?; + for p in entry.c_prefab { + info!("adding {p}"); + if args.announce { + Packet::AnnouncePrefab(p.clone()).write(&mut sock)?; + } + if args.add { + let ob = Object::new(); + Packet::Add(ob, p.clone()).write(&mut sock)?; + if args.disown { + // TODO disown packet once it exists + } + } + } + sock.flush()?; + + info!("idling for resource requests"); + loop { + let packet = Packet::read(&mut sock)?; + match packet { + Packet::RequestResource(hash) => { + if let Some(d) = store.get_raw(hash)? { + Packet::RespondResource(hash, Data(d)).write(&mut sock)?; + sock.flush()?; + } + } + _ => (), + } + } } diff --git a/doc/resources.md b/doc/resources.md index b86a2f8..5060571 100644 --- a/doc/resources.md +++ b/doc/resources.md @@ -39,6 +39,12 @@ | `avatar_info` | `Res` | | | `environment` | `Res` | | +## RespackEntry + +| Key | Value Type | | +| ---------- | ------------- | --------- | +| `c_prefab` | `Res` | Multi key | + ## MeshPart Vertex attribute arrays (va_\*) are resources that contain a packed float32 diff --git a/import/src/main.rs b/import/src/main.rs index 554a2bd..0a7278b 100644 --- a/import/src/main.rs +++ b/import/src/main.rs @@ -22,7 +22,7 @@ pub mod physics; pub mod prefab; pub mod vrm; -use anyhow::{Result, bail}; +use anyhow::{Context, Result, anyhow, bail}; use clap::Parser; use gltf::{image::Source, scene::Transform}; use humansize::BINARY; @@ -123,7 +123,7 @@ fn main() -> Result<()> { let args = Args::parse(); let store = if args.use_cache && !args.pack.is_some() { - ResourceStore::new_env()? + ResourceStore::new_env().context("opening resource store")? } else { ResourceStore::new_memory() }; @@ -132,7 +132,10 @@ fn main() -> Result<()> { let texture_cache = Arc::new(Mutex::new(HashMap::new())); for scenepath in &args.scene { - prefabs.push(import_prefab(&store, &texture_cache, scenepath, &args)?); + prefabs.push( + import_prefab(&store, &texture_cache, scenepath, &args) + .context(anyhow!("in {scenepath:?}"))?, + ); } let mut size = 0; @@ -146,7 +149,7 @@ fn main() -> Result<()> { return Ok(()); } if let Some(outpath) = args.pack { - let entry = store.set(&RespackEntry { name: None })?; + let entry = store.set(&RespackEntry { c_prefab: prefabs })?; let mut resources = Vec::new(); store.iter(|r, _| resources.push(r))?; save_respack( diff --git a/shared/src/resources.rs b/shared/src/resources.rs index d5ec76c..5fa810d 100644 --- a/shared/src/resources.rs +++ b/shared/src/resources.rs @@ -74,7 +74,7 @@ macro_rules! resource_dicts { resource_dicts!( pub struct RespackEntry { - name: String, + c_prefab[multi]: Resource, } pub struct Prefab { diff --git a/shared/src/respack.rs b/shared/src/respack.rs index 8c0b9d8..dee0cd0 100644 --- a/shared/src/respack.rs +++ b/shared/src/respack.rs @@ -18,6 +18,7 @@ use crate::{packets::Resource, resources::RespackEntry, store::ResourceStore}; use anyhow::{Result, bail}; use log::{info, warn}; use std::{ + collections::HashMap, io::{Read, Seek, SeekFrom, Write}, marker::PhantomData, }; @@ -49,59 +50,101 @@ pub fn save_respack( Ok(()) } +/// Copies the entire respack to the store. This is usually a dumb idea because the pack supportes on-demand reading. pub fn load_respack( - mut input: impl Read + Seek, + input: impl Read + Seek, store: &ResourceStore, ) -> Result>> { - let mut magic = [0u8; MAGIC.len()]; - input.read_exact(&mut magic)?; - if magic != *MAGIC { - bail!("wrong magic bytes"); - } - let mut entry = [0u8; 32]; - input.read_exact(&mut entry)?; - let entry = if entry != [0u8; 32] { - Some(Resource(entry, PhantomData)) - } else { - None - }; - - let mut count = [0u8; size_of::()]; - input.read_exact(&mut count)?; - let count = u64::from_be_bytes(count); - + let mut pack = RespackReader::open(input)?; let mut load_queue = Vec::new(); - let mut found_entry = false; - for _ in 0..count { - let mut res = [0u8; 32]; - let mut off = [0u8; size_of::()]; - let mut size = [0u8; size_of::()]; - input.read_exact(&mut res)?; - input.read_exact(&mut off)?; - input.read_exact(&mut size)?; - - found_entry |= Some(Resource(res, PhantomData)) == entry; - if store.get_raw_size(Resource(res, PhantomData))?.is_none() { - load_queue.push((res, u64::from_be_bytes(off), u64::from_be_bytes(size))) + for res in pack.index.keys() { + if store.get_raw_size(*res)?.is_none() { + load_queue.push(*res); } } - if !found_entry && entry.is_some() { - warn!("respack does not contain its entry resource") - } info!( - "loading {} of {count} resources from pack", + "loading {} of {} resources from pack", load_queue.len(), + pack.index.len() ); - - for (res, off, size) in load_queue { - input.seek(SeekFrom::Start(off))?; + for res in load_queue { let mut buf = Vec::new(); - input.by_ref().take(size).read_to_end(&mut buf)?; + pack.read(res)?.unwrap().read_to_end(&mut buf)?; let key = store.set_raw(&buf)?; - if key.0 != res { + if key != res { warn!("respack containes mislabeled resources") } } - Ok(entry) + Ok(pack.entry) +} + +pub struct RespackReader { + entry: Option>, + index: HashMap, + inner: T, +} +impl RespackReader { + pub fn open(mut inner: T) -> Result { + let mut magic = [0u8; MAGIC.len()]; + inner.read_exact(&mut magic)?; + if magic != *MAGIC { + bail!("wrong magic bytes"); + } + let mut entry = [0u8; 32]; + inner.read_exact(&mut entry)?; + let entry = if entry != [0u8; 32] { + Some(Resource(entry, PhantomData)) + } else { + None + }; + + let mut count = [0u8; size_of::()]; + inner.read_exact(&mut count)?; + let count = u64::from_be_bytes(count); + + let mut index = HashMap::new(); + let mut found_entry = false; + for _ in 0..count { + let mut res = [0u8; 32]; + let mut off = [0u8; size_of::()]; + let mut size = [0u8; size_of::()]; + inner.read_exact(&mut res)?; + inner.read_exact(&mut off)?; + inner.read_exact(&mut size)?; + + found_entry |= Some(Resource(res, PhantomData)) == entry; + index.insert( + Resource(res, PhantomData), + (u64::from_be_bytes(off), u64::from_be_bytes(size)), + ); + } + if !found_entry && entry.is_some() { + warn!("respack does not contain its entry resource") + } + info!("opened respack with {} resources", index.len()); + Ok(Self { + entry, + index, + inner, + }) + } + pub fn entry(&self) -> Option> { + self.entry.clone() + } + pub fn get_size(&self, key: Resource) -> Option { + self.index.get(&key).map(|(_off, size)| *size as usize) + } + pub fn iter(&self, mut cb: impl FnMut(Resource, usize)) { + for (r, (_, s)) in &self.index { + cb(*r, *s as usize) + } + } + pub fn read(&mut self, key: Resource) -> Result> { + let Some((off, size)) = self.index.get(&key).copied() else { + return Ok(None); + }; + self.inner.seek(SeekFrom::Start(off))?; + Ok(Some(self.inner.by_ref().take(size))) + } } diff --git a/shared/src/store.rs b/shared/src/store.rs index e3e5949..61b5dee 100644 --- a/shared/src/store.rs +++ b/shared/src/store.rs @@ -14,8 +14,8 @@ You should have received a copy of the GNU Affero General Public License along with this program. If not, see . */ -use crate::{helper::ReadWrite, packets::Resource}; -use anyhow::Result; +use crate::{helper::ReadWrite, packets::Resource, respack::RespackReader}; +use anyhow::{Result, bail}; use log::info; use redb::{Database, TableDefinition}; use std::{ @@ -34,6 +34,7 @@ pub enum ResourceStore { Redb(Database), Filesystem(PathBuf), Memory(Mutex>>), + Respack(Mutex>), } impl ResourceStore { pub fn new_env() -> Result { @@ -56,6 +57,16 @@ impl ResourceStore { create_dir_all(path)?; Ok(Self::Filesystem(path.to_owned())) } + pub fn new_respack_file(path: &Path) -> Result { + info!("using static respack cache from {path:?}"); + Ok(Self::Respack( + RespackReader::open(File::open(path)?)?.into(), + )) + } + pub fn new_respack(pack: RespackReader) -> Result { + info!("using static respack as store"); + Ok(Self::Respack(pack.into())) + } pub fn new_redb(path: &Path) -> Result { info!("initializing redb resource cache..."); let db = Database::create(path)?; @@ -84,6 +95,7 @@ impl ResourceStore { let g = mutex.lock().unwrap(); Ok(g.get(&key).map(|s| s.len())) } + ResourceStore::Respack(r) => Ok(r.lock().unwrap().get_size(key)), } } pub fn get_raw(&self, key: Resource) -> Result>> { @@ -107,6 +119,17 @@ impl ResourceStore { Ok(None) } } + ResourceStore::Respack(r) => { + let mut r = r.lock().unwrap(); + match r.read(key)? { + Some(mut reader) => { + let mut buf = Vec::new(); + reader.read_to_end(&mut buf)?; + Ok(Some(buf)) + } + None => Ok(None), + } + } } } pub fn set_raw(&self, value: &[u8]) -> Result { @@ -130,6 +153,7 @@ impl ResourceStore { rename(path_temp, path)?; } } + ResourceStore::Respack(_) => bail!("tried to write to resback backed store"), } Ok(key) } @@ -158,6 +182,7 @@ impl ResourceStore { .for_each(|(k, v)| cb(*k, v.len())); Ok(()) } + ResourceStore::Respack(r) => Ok(r.lock().unwrap().iter(cb)), } } } -- cgit v1.2.3-70-g09d2