#![feature(array_chunks)] pub mod cache; pub mod mesh; use anyhow::{Result, bail}; use cache::Cache; use clap::Parser; use futures_util::{StreamExt, stream::FuturesUnordered}; use glam::{DMat4, Vec3}; use log::{debug, error, info}; use mesh::{convert_mesh, decode_normal_table}; use prost::{Message, bytes::Bytes}; use proto::{BulkMetadata, NodeData, NodeMetadata, PlanetoidMetadata}; use reqwest::{ Client, header::{HeaderMap, HeaderName, HeaderValue}, }; use std::{f32::consts::PI, path::PathBuf, pin::Pin, sync::Arc}; use tokio::sync::Semaphore; use weareshared::{ Affine3A, helper::AABB, packets::Resource, resources::{Prefab, RespackEntry, SpatialIndex}, respack::save_full_respack, store::ResourceStore, }; #[derive(Parser)] struct Args { #[arg(short, long, default_value = "16")] par_limit: usize, #[arg(short, long)] db_cache: Option, #[clap(subcommand)] action: Action, } #[derive(Parser)] enum Action { Cache { level: usize }, Export { level: usize }, } #[tokio::main] async fn main() -> Result<()> { env_logger::init_from_env("LOG"); let args = Args::parse(); let cache = if let Some(path) = args.db_cache { Cache::new_db(&path)? } else { Cache::new_directory()? }; let c = GeClient::new(16, cache).await?; match args.action { Action::Cache { level } => { let entry = c.planetoid_metdata().await?; let epoch = entry.root_node_metadata.unwrap().bulk_metadata_epoch(); cache_all( Arc::new(c), Arc::new(Semaphore::new(64)), "".to_string(), epoch, level, ) .await?; } Action::Export { level } => { let entry = c.planetoid_metdata().await?; let store = Arc::new(ResourceStore::new_memory()); let (_, root) = do_node( Arc::new(c), "".to_string(), store.clone(), entry.root_node_metadata.unwrap().bulk_metadata_epoch(), level, ) .await?; let entry = store.set(&RespackEntry { c_spatial_index: vec![root], ..Default::default() })?; let file = std::fs::File::create("/tmp/a.respack")?; save_full_respack(file, &store, Some(entry))?; } } Ok(()) } fn do_node( c: Arc, path: String, store: Arc, epoch: u32, level: usize, ) -> Pin)>>>> { Box::pin(async move { let bulk = c.bulk_metadata(&path, epoch).await?; let mut fu = FuturesUnordered::new(); let mut meshes = Vec::new(); let mut children = Vec::new(); for node_meta in &bulk.node_metadata { let (cpath, flags) = unpack_path_and_id(node_meta.path_and_flags()); // eprintln!("{path}+{cpath} {flags:?}"); let abspath = format!("{path}{cpath}"); if flags.has_node { let node = c.node_data(&abspath, flags, &bulk, node_meta).await?; let transform = DMat4::from_cols_slice(&node.matrix_globe_from_mesh); let for_normals = decode_normal_table(node.for_normals()); for m in node.meshes { let mesh = convert_mesh(m, &store, &for_normals)?; meshes.push(( Affine3A::from_rotation_x(-PI / 2.) * Affine3A::from_mat4((transform / 500_000.).as_mat4()), mesh, )) } } if cpath.len() == 4 && flags.has_metadata && abspath.len() < level { fu.push(do_node(c.clone(), abspath, store.clone(), epoch, level)); } } while let Some(res) = fu.next().await { children.push(res?); } let bounds = AABB { min: Vec3::ZERO, max: Vec3::MAX, }; let prefab = store.set(&Prefab { mesh: meshes, ..Default::default() })?; Ok(( bounds, store.set(&SpatialIndex { prefab: Some(prefab), child: children, ..Default::default() })?, )) }) } fn cache_all( c: Arc, par: Arc, path: String, epoch: u32, level: usize, ) -> Pin>>> { Box::pin(async move { let _permit = par.acquire().await?; let bulk = c.bulk_metadata(&path, epoch).await?; let mut fu = FuturesUnordered::new(); for node_meta in &bulk.node_metadata { let (cpath, flags) = unpack_path_and_id(node_meta.path_and_flags()); let abspath = format!("{path}{cpath}"); if flags.has_node { c.node_data(&abspath, flags, &bulk, node_meta).await?; } if cpath.len() == 4 && flags.has_metadata && abspath.len() < level { fu.push(cache_all(c.clone(), par.clone(), abspath, epoch, level)); } } while let Some(res) = fu.next().await { res?; } drop(_permit); Ok(()) }) } struct GeClient { client: Client, cache: Cache, par_limit: Semaphore, } impl GeClient { pub async fn new(par_limit: usize, cache: Cache) -> Result { Ok(Self { par_limit: Semaphore::new(par_limit),cache, client: Client::builder().default_headers(HeaderMap::from_iter([ (HeaderName::from_static("user-agent"), HeaderValue::from_static("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36")), (HeaderName::from_static("referer"), HeaderValue::from_static("https://earth.google.com/")) ])).build().unwrap() }) } pub async fn download(&self, path: &str) -> Result { let _permit = self.par_limit.acquire().await?; if let Some(d) = self.cache.get(path)? { debug!("cached {path:?}"); Ok(d.into()) } else { info!("download {path:?}"); let res = self .client .get(format!("https://kh.google.com/rt/earth/{path}")) .send() .await?; if !res.status().is_success() { error!("error response: {:?}", res.text().await?); bail!("error response") } let buf = res.bytes().await?; self.cache.insert(path, &buf)?; Ok(buf) } } pub async fn planetoid_metdata(&self) -> Result { let buf = self.download("PlanetoidMetadata").await?; Ok(PlanetoidMetadata::decode(buf)?) } pub async fn bulk_metadata(&self, path: &str, epoch: u32) -> Result { let buf = self .download(&format!("BulkMetadata/pb=!1m2!1s{path}!2u{epoch}")) .await?; Ok(BulkMetadata::decode(buf)?) } pub async fn node_data( &self, abspath: &str, flags: Flags, bulk: &BulkMetadata, node: &NodeMetadata, ) -> Result { let texture_format = bulk.default_available_texture_formats(); let imagery_epoch = node.imagery_epoch.unwrap_or(bulk.default_imagery_epoch()); let node_epoch = node .epoch .unwrap_or(bulk.head_node_key.as_ref().unwrap().epoch.unwrap()); // ? let image_epoch_part = if flags.use_image_epoch { format!("!3u{imagery_epoch}") } else { String::new() }; let url = format!( "NodeData/pb=!1m2!1s{abspath}!2u{node_epoch}!2e{texture_format}{image_epoch_part}!4b0" ); let buf = self.download(&url).await?; Ok(NodeData::decode(buf)?) } } #[derive(Debug, Clone, Copy)] struct Flags { has_node: bool, has_metadata: bool, use_image_epoch: bool, } fn unpack_path_and_id(mut path_id: u32) -> (String, Flags) { let level = 1 + (path_id & 3); path_id >>= 2; let mut path = String::new(); for _ in 0..level { path += &(path_id & 7).to_string(); path_id >>= 3; } let flags = path_id; ( path, Flags { has_node: flags & 8 == 0, has_metadata: flags & 4 == 0, use_image_epoch: flags & 16 != 0, }, ) } pub mod proto { include!(concat!(env!("OUT_DIR"), "/earth.proto.rs")); }