From d8d00eb146241978ef21ed4d6c35ac9c68b1a86e Mon Sep 17 00:00:00 2001
From: metamuffin <metamuffin@disroot.org>
Date: Mon, 10 Feb 2025 17:38:55 +0100
Subject: read serialized file tables

---
 src/helper.rs          |  35 ++++++-
 src/lib.rs             |   1 +
 src/main.rs            | 138 +--------------------------
 src/serialized_file.rs | 248 +++++++++++++++++++++++++++++++++++++++++++++++++
 src/unityfs.rs         |  29 ++++--
 5 files changed, 309 insertions(+), 142 deletions(-)
 create mode 100644 src/serialized_file.rs

(limited to 'src')

diff --git a/src/helper.rs b/src/helper.rs
index 0b126ad..cb52b8c 100644
--- a/src/helper.rs
+++ b/src/helper.rs
@@ -1,4 +1,4 @@
-use std::io::{Read, Result};
+use std::io::{Read, Result, Seek};
 
 #[derive(Debug, Clone, Copy, PartialEq)]
 pub enum Endianness {
@@ -23,9 +23,13 @@ pub trait ReadExt {
     fn read_u64(&mut self, e: Endianness) -> Result<u64>;
     fn read_u64_be(&mut self) -> Result<u64>;
     fn read_u64_le(&mut self) -> Result<u64>;
+    fn read_i64(&mut self, e: Endianness) -> Result<i64>;
+    fn read_i64_be(&mut self) -> Result<i64>;
+    fn read_i64_le(&mut self) -> Result<i64>;
     fn read_u128_be(&mut self) -> Result<u128>;
     fn read_cstr(&mut self) -> Result<String>;
 }
+
 impl<T: Read> ReadExt for T {
     fn read_u8(&mut self) -> Result<u8> {
         let mut buf = [0; 1];
@@ -112,6 +116,22 @@ impl<T: Read> ReadExt for T {
         self.read_exact(&mut buf)?;
         Ok(u64::from_le_bytes(buf))
     }
+    fn read_i64(&mut self, e: Endianness) -> Result<i64> {
+        match e {
+            Endianness::Big => self.read_i64_be(),
+            Endianness::Little => self.read_i64_le(),
+        }
+    }
+    fn read_i64_be(&mut self) -> Result<i64> {
+        let mut buf = [0; 8];
+        self.read_exact(&mut buf)?;
+        Ok(i64::from_be_bytes(buf))
+    }
+    fn read_i64_le(&mut self) -> Result<i64> {
+        let mut buf = [0; 8];
+        self.read_exact(&mut buf)?;
+        Ok(i64::from_le_bytes(buf))
+    }
     fn read_u128_be(&mut self) -> Result<u128> {
         let mut buf = [0; 16];
         self.read_exact(&mut buf)?;
@@ -130,3 +150,16 @@ impl<T: Read> ReadExt for T {
         Ok(String::from_utf8_lossy(&s).to_string())
     }
 }
+
+pub trait AlignExt {
+    fn align(&mut self, size: u64) -> Result<()>;
+}
+impl<T: Seek> AlignExt for T {
+    fn align(&mut self, size: u64) -> Result<()> {
+        let off = self.stream_position()? % size;
+        if off != 0 {
+            self.seek_relative((size - off) as i64)?;
+        }
+        Ok(())
+    }
+}
diff --git a/src/lib.rs b/src/lib.rs
index d2370b3..5022d0d 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -1,3 +1,4 @@
 pub mod unityfs;
 pub mod helper;
 pub mod common_strings;
+pub mod serialized_file;
diff --git a/src/main.rs b/src/main.rs
index a0b35c3..4d91578 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -1,14 +1,5 @@
-use log::debug;
-use std::{
-    env::args,
-    fs::File,
-    io::{BufReader, Cursor, Read},
-};
-use unity_tools::{
-    common_strings::COMMON_STRINGS,
-    helper::{Endianness, ReadExt},
-    unityfs::UnityFS,
-};
+use std::{env::args, fs::File, io::BufReader};
+use unity_tools::{serialized_file::read_serialized_file, unityfs::UnityFS};
 
 fn main() -> anyhow::Result<()> {
     env_logger::init_from_env("LOG");
@@ -19,134 +10,15 @@ fn main() -> anyhow::Result<()> {
         if node.name.ends_with(".resource") || node.name.ends_with("resS") {
             continue;
         }
-        let mut cab = fs.read(&node)?;
+        let cab = fs.read(&node)?;
         // let mut writer = File::create(format!("/tmp/{}", node.name))?;
         // std::io::copy(&mut cab, &mut writer)?;
         // continue;
 
-        let mut metadata_size = cab.read_u32_be()?;
-        let mut file_size = cab.read_u32_be()? as u64;
-        let format = cab.read_u32_be()?;
-        let mut data_offset = cab.read_u32_be()? as u64;
+        let file = read_serialized_file(cab)?;
+        eprintln!("{file:#?}");
 
-        debug!("format={format}");
-
-        assert!(format >= 9);
-        let e = match cab.read_u32_be()? {
-            0 => Endianness::Little,
-            _ => Endianness::Big,
-        };
-        debug!("endianess={e:?}");
-
-        if format >= 22 {
-            metadata_size = cab.read_u32(e)?;
-            file_size = cab.read_u64(e)?;
-            data_offset = cab.read_u64(e)?;
-            cab.read_u64(e)?;
-        }
-        debug!("metadata_size={metadata_size}");
-        debug!("file_size={file_size}");
-        debug!("data_offset={data_offset}");
-
-        let generator_version = cab.read_cstr()?;
-        let target_platform = cab.read_u32_le()?;
-        debug!("generator_version={generator_version:?}");
-        debug!("target_platform={target_platform}");
-
-        let has_type_trees = cab.read_u8()? != 0;
-        let num_types = cab.read_u32(e)?;
-        debug!("has_type_trees={has_type_trees:?}");
-        debug!("num_types={num_types}");
-
-        for _ in 0..num_types {
-            let class_id = cab.read_i32(e)?;
-            let stripped_type = cab.read_u8()? != 0;
-            let script_index = cab.read_i16(e)?;
-            let mut script_id = 0;
-            // TODO reftype
-            if class_id == 114 {
-                script_id = cab.read_u128_be()?;
-            }
-            let _old_hash = cab.read_u128_be()?;
-
-            eprintln!("class_id={class_id}");
-            eprintln!("stripped_type={stripped_type}");
-            eprintln!("script_index={script_index}");
-            eprintln!("script_id={script_id}");
-
-            if has_type_trees {
-                let num_nodes = cab.read_u32(e)?;
-                let size = cab.read_u32(e)?;
-                eprintln!("tree:num_nodes={num_nodes}");
-                eprintln!("tree:size={size}");
-
-                let mut node_data = vec![0u8; num_nodes as usize * 32];
-                cab.read_exact(&mut node_data)?;
-                let mut node_data = Cursor::new(node_data);
-                let mut string_data = vec![0u8; size as usize];
-                cab.read_exact(&mut string_data)?;
-
-                let get_string = |off: u32| {
-                    let data = if off & 0x80000000 != 0 {
-                        let off = off & 0x7fffffff;
-                        &COMMON_STRINGS[(off & 0x7fffffff) as usize..]
-                    } else {
-                        &string_data[off as usize..]
-                    };
-                    String::from_utf8(
-                        data.iter()
-                            .copied()
-                            .take_while(|e| *e != 0)
-                            .collect::<Vec<u8>>(),
-                    )
-                };
-
-                let mut nodes = Vec::new();
-                for _ in 0..num_nodes {
-                    nodes.push(TypeTreeNode {
-                        version: node_data.read_u16(e)?,
-                        level: node_data.read_u8()?,
-                        type_flags: node_data.read_u8()?,
-                        type_string: get_string(node_data.read_u32(e)?)?,
-                        name_string: get_string(node_data.read_u32(e)?)?,
-                        byte_size: node_data.read_i32(e)?,
-                        index: node_data.read_i32(e)?,
-                        flags: node_data.read_i32(e)?,
-                        ref_type_hash: node_data.read_u64(e)?,
-                    });
-                }
-                eprintln!("{nodes:#?}");
-
-                if format >= 21 {
-                    let num_deps = cab.read_u32(e)?;
-                    for _ in 0..num_deps {
-                        cab.read_u32(e)?;
-                    }
-                }
-            }
-
-            if format > 21 {
-                cab.read_u32_be()?;
-            }
-        }
-
-        // let num_objects = cab.read_u32_le()?;
-        // debug!("num_objects={num_objects}");
-        // for _ in 0..num_objects {}
     }
 
     Ok(())
 }
-
-#[derive(Debug)]
-struct TypeTreeNode {
-    version: u16,
-    level: u8,
-    type_flags: u8,
-    type_string: String,
-    name_string: String,
-    byte_size: i32,
-    index: i32,
-    flags: i32,
-    ref_type_hash: u64,
-}
diff --git a/src/serialized_file.rs b/src/serialized_file.rs
new file mode 100644
index 0000000..becf1d7
--- /dev/null
+++ b/src/serialized_file.rs
@@ -0,0 +1,248 @@
+use crate::{
+    common_strings::COMMON_STRINGS,
+    helper::{AlignExt, Endianness, ReadExt},
+};
+use anyhow::Result;
+use log::{debug, trace};
+use std::io::{Cursor, Read, Seek};
+
+#[derive(Debug)]
+pub struct TypeTreeNode {
+    pub version: u16,
+    pub level: u8,
+    pub type_flags: u8,
+    pub type_string: String,
+    pub name_string: String,
+    pub byte_size: i32,
+    pub index: i32,
+    pub flags: i32,
+    pub ref_type_hash: u64,
+}
+
+#[derive(Debug)]
+pub struct SeralizedType {
+    pub class_id: i32,
+    pub stripped_type: bool,
+    pub script_type_index: i16,
+    pub script_id: u128,
+
+    pub type_tree: Vec<TypeTreeNode>,
+    pub type_deps: Vec<u32>,
+}
+
+#[derive(Debug)]
+pub struct ObjectInfo {
+    path_id: i64,
+    data_offset: u64,
+    data_size: u32,
+    type_id: i32,
+}
+
+#[derive(Debug)]
+pub struct Script {
+    file_index: u32,
+    identifier: i64,
+}
+
+#[derive(Debug)]
+pub struct External {
+    something: String,
+    guid: u128,
+    r#type: i32,
+    path_name: String,
+}
+
+#[derive(Debug)]
+pub struct SerializedFile {
+    pub types: Vec<SeralizedType>,
+    pub externals: Vec<External>,
+    pub scripts: Vec<Script>,
+    pub objects: Vec<ObjectInfo>,
+    pub user_string: String,
+}
+
+pub fn read_serialized_file(mut file: impl Read + Seek) -> Result<SerializedFile> {
+    let mut metadata_size = file.read_u32_be()?;
+    let mut file_size = file.read_u32_be()? as u64;
+    let format = file.read_u32_be()?;
+    let mut data_offset = file.read_u32_be()? as u64;
+
+    debug!("format={format}");
+
+    assert!(format >= 9);
+    let e = match file.read_u32_be()? {
+        0 => Endianness::Little,
+        _ => Endianness::Big,
+    };
+    debug!("endianess={e:?}");
+
+    if format >= 22 {
+        metadata_size = file.read_u32(e)?;
+        file_size = file.read_u64(e)?;
+        data_offset = file.read_u64(e)?;
+        file.read_u64(e)?;
+    }
+    debug!("metadata_size={metadata_size}");
+    debug!("file_size={file_size}");
+    debug!("data_offset={data_offset}");
+
+    let generator_version = file.read_cstr()?;
+    let target_platform = file.read_u32_le()?;
+    debug!("generator_version={generator_version:?}");
+    debug!("target_platform={target_platform}");
+
+    let has_type_trees = file.read_u8()? != 0;
+    let num_types = file.read_u32(e)?;
+    debug!("has_type_trees={has_type_trees:?}");
+    debug!("num_types={num_types}");
+
+    let mut types = Vec::new();
+
+    for _ in 0..num_types {
+        let class_id = file.read_i32(e)?;
+        let stripped_type = file.read_u8()? != 0;
+        let script_type_index = file.read_i16(e)?;
+        let mut script_id = 0;
+        // TODO reftype
+        if class_id == 114 {
+            script_id = file.read_u128_be()?;
+        }
+        let _old_hash = file.read_u128_be()?;
+
+        trace!("class_id={class_id}");
+        trace!("stripped_type={stripped_type}");
+        trace!("script_type_index={script_type_index}");
+        trace!("script_id={script_id}");
+
+        let mut type_deps = Vec::new();
+        let mut type_tree = Vec::new();
+        if has_type_trees {
+            let num_nodes = file.read_u32(e)?;
+            let size = file.read_u32(e)?;
+            trace!("tree:num_nodes={num_nodes}");
+            trace!("tree:size={size}");
+
+            let mut node_data = vec![0u8; num_nodes as usize * 32];
+            file.read_exact(&mut node_data)?;
+            let mut node_data = Cursor::new(node_data);
+            let mut string_data = vec![0u8; size as usize];
+            file.read_exact(&mut string_data)?;
+
+            let get_string = |off: u32| {
+                let data = if off & 0x80000000 != 0 {
+                    let off = off & 0x7fffffff;
+                    &COMMON_STRINGS[off as usize..]
+                } else {
+                    &string_data[off as usize..]
+                };
+                String::from_utf8(
+                    data.iter()
+                        .copied()
+                        .take_while(|e| *e != 0)
+                        .collect::<Vec<u8>>(),
+                )
+            };
+
+            for _ in 0..num_nodes {
+                type_tree.push(TypeTreeNode {
+                    version: node_data.read_u16(e)?,
+                    level: node_data.read_u8()?,
+                    type_flags: node_data.read_u8()?,
+                    type_string: get_string(node_data.read_u32(e)?)?,
+                    name_string: get_string(node_data.read_u32(e)?)?,
+                    byte_size: node_data.read_i32(e)?,
+                    index: node_data.read_i32(e)?,
+                    flags: node_data.read_i32(e)?,
+                    ref_type_hash: node_data.read_u64(e)?,
+                });
+            }
+
+            if format >= 21 {
+                let num_deps = file.read_u32(e)?;
+                trace!("num_deps={num_deps}");
+                for _ in 0..num_deps {
+                    type_deps.push(file.read_u32(e)?);
+                }
+            }
+        }
+
+        types.push(SeralizedType {
+            class_id,
+            script_id,
+            script_type_index,
+            stripped_type,
+            type_deps,
+            type_tree,
+        })
+    }
+
+    let num_objects = file.read_u32(e)?;
+    debug!("num_objects={num_objects}");
+    let mut objects = Vec::new();
+    for _ in 0..num_objects {
+        file.align(4)?;
+        let path_id = file.read_i64(e)?;
+        let data_offset = if format >= 22 {
+            file.align(4)?;
+            file.read_u64(e)?
+        } else {
+            file.read_u32(e)? as u64
+        } + data_offset;
+        let data_size = file.read_u32(e)?;
+        let type_id = file.read_i32(e)?;
+        objects.push(ObjectInfo {
+            data_offset,
+            data_size,
+            path_id,
+            type_id,
+        })
+    }
+
+    let num_scripts = file.read_u32(e)?;
+    debug!("num_scripts={num_scripts}");
+    let mut scripts = Vec::new();
+    for _ in 0..num_scripts {
+        let file_index = file.read_u32(e)?;
+        file.align(4)?;
+        let identifier = file.read_i64(e)?;
+        scripts.push(Script {
+            file_index,
+            identifier,
+        })
+    }
+
+    let num_externals = file.read_u32(e)?;
+    debug!("num_externals={num_externals}");
+    let mut externals = Vec::new();
+    for _ in 0..num_externals {
+        let something = file.read_cstr()?;
+        let guid = file.read_u128_be()?;
+        let r#type = file.read_i32(e)?;
+        let path_name = file.read_cstr()?;
+        externals.push(External {
+            guid,
+            path_name,
+            something,
+            r#type,
+        })
+    }
+
+    if format >= 20 {
+        let num_ref_types = file.read_i32(e)?;
+        debug!("num_ref_types={num_ref_types}");
+        // let mut ref_types = Vec::new();
+        for _ in 0..num_ref_types {
+            todo!()
+        }
+    }
+
+    let user_string = file.read_cstr()?;
+
+    Ok(SerializedFile {
+        types,
+        externals,
+        objects,
+        scripts,
+        user_string,
+    })
+}
diff --git a/src/unityfs.rs b/src/unityfs.rs
index d106ce6..f9a8d34 100644
--- a/src/unityfs.rs
+++ b/src/unityfs.rs
@@ -1,4 +1,4 @@
-use crate::helper::ReadExt;
+use crate::helper::{AlignExt, ReadExt};
 use anyhow::{Result, anyhow, bail};
 use log::{debug, info, trace};
 use std::io::{Cursor, ErrorKind, Read, Seek, SeekFrom, Take};
@@ -24,6 +24,7 @@ struct BlockInfo {
 
 pub struct NodeReader<'a, T> {
     inner: Take<&'a mut BlocksReader<T>>,
+    size: u64,
 }
 
 impl<T: Read + Seek> UnityFS<T> {
@@ -77,13 +78,8 @@ impl<T: Read + Seek> UnityFS<T> {
                 meta_comp_scheme.decompress(blockindex, blockindex_decomp_size as usize)?;
             Cursor::new(blockindex)
         };
-        {
-            // align stream
-            let off = file.stream_position()? % 16;
-            if off != 0 {
-                file.seek_relative(16 - off as i64)?;
-            }
-        }
+
+        file.align(16)?;
 
         blockindex.read_u128_be()?;
 
@@ -134,6 +130,7 @@ impl<T: Read + Seek> UnityFS<T> {
     pub fn read<'a>(&'a mut self, node: &NodeInfo) -> std::io::Result<NodeReader<'a, T>> {
         self.reader.seek(SeekFrom::Start(node.offset))?;
         Ok(NodeReader {
+            size: node.size,
             inner: (&mut self.reader).take(node.size),
         })
     }
@@ -231,6 +228,22 @@ impl<T: Read> Read for NodeReader<'_, T> {
         self.inner.read(buf)
     }
 }
+impl<T: Seek + Read> Seek for NodeReader<'_, T> {
+    fn seek(&mut self, pos: SeekFrom) -> std::io::Result<u64> {
+        match pos {
+            SeekFrom::Current(n) if n >= 0 => {
+                for _ in 0..n {
+                    self.read_exact(&mut [0u8])?;
+                }
+                Ok(self.stream_position()?)
+            }
+            _ => unimplemented!(),
+        }
+    }
+    fn stream_position(&mut self) -> std::io::Result<u64> {
+        Ok(self.size - self.inner.limit())
+    }
+}
 
 #[derive(Debug, Clone, Copy, PartialEq)]
 enum CompressionScheme {
-- 
cgit v1.2.3-70-g09d2