a

2 files changed, 357 insertions, 0 deletions

diff --git a/src/earth.proto b/src/earth.proto
new file mode 100644
index 0000000..0f0b3d0
--- /dev/null
+++ b/src/earth.proto
@@ -0,0 +1,157 @@
+syntax = "proto2";
+
+package earth.proto;
+
+message BulkMetadataRequest {
+    optional NodeKey node_key = 1;
+}
+
+message NodeDataRequest {
+    optional NodeKey node_key = 1;
+    optional Texture.Format texture_format = 2;
+    optional uint32 imagery_epoch = 3;
+}
+
+message NodeKey {
+    optional string path = 1;
+    optional uint32 epoch = 2;
+}
+
+message CopyrightRequest {
+    optional uint32 epoch = 1;
+}
+
+message TextureDataRequest {
+    optional NodeKey node_key = 1;
+    optional Texture.Format texture_format = 2;
+    optional Texture.ViewDirection view_direction = 3;
+}
+
+message BulkMetadata {
+    repeated NodeMetadata node_metadata = 1;
+    optional NodeKey head_node_key = 2;
+    repeated double head_node_center = 3 [packed = true];
+    repeated float meters_per_texel = 4 [packed = true];
+    optional uint32 default_imagery_epoch = 5;
+    optional uint32 default_available_texture_formats = 6;
+    optional uint32 default_available_view_dependent_textures = 7;
+    optional uint32 default_available_view_dependent_texture_formats = 8;
+}
+
+message NodeMetadata {
+    optional uint32 path_and_flags = 1;
+    optional uint32 epoch = 2;
+    optional uint32 bulk_metadata_epoch = 5;
+    optional bytes oriented_bounding_box = 3;
+    optional float meters_per_texel = 4;
+    repeated double processing_oriented_bounding_box = 6 [packed = true];
+    optional uint32 imagery_epoch = 7;
+    optional uint32 available_texture_formats = 8;
+    optional uint32 available_view_dependent_textures = 9;
+    optional uint32 available_view_dependent_texture_formats = 10;
+    
+    enum Flags {
+        RICH3D_LEAF = 1;
+        RICH3D_NODATA = 2;
+        LEAF = 4;
+        NODATA = 8;
+        USE_IMAGERY_EPOCH = 16;
+    }
+}
+
+message NodeData {
+    repeated double matrix_globe_from_mesh = 1 [packed = true];
+    repeated Mesh meshes = 2;
+    repeated uint32 copyright_ids = 3;
+    optional NodeKey node_key = 4;
+    repeated double kml_bounding_box = 5 [packed = true];
+    optional Mesh water_mesh = 6;
+    repeated Mesh overlay_surface_meshes = 7;
+    optional bytes for_normals = 8;
+}
+
+message Mesh {
+    optional bytes vertices = 1;
+    optional bytes vertex_alphas = 9;
+    optional bytes texture_coords = 2;
+    repeated int32 indices = 3 [packed = true];
+    optional bytes octant_ranges = 4;
+    optional bytes layer_counts = 5;
+    repeated Texture texture = 6;
+    optional bytes texture_coordinates = 7;
+    repeated float uv_offset_and_scale = 10 [packed = true];
+    optional bytes layer_and_octant_counts = 8;
+    optional bytes normals = 11;
+    optional bytes normals_dev = 16;
+    optional uint32 mesh_id = 12;
+    optional bytes skirt_flags = 13;
+    
+    enum Layer {
+        OVERGROUND = 0;
+        TERRAIN_BELOW_WATER = 1;
+        TERRAIN_ABOVE_WATER = 2;
+        TERRAIN_HIDDEN = 3;
+        WATER = 4;
+        WATER_SKIRTS = 5;
+        WATER_SKIRTS_INVERTED = 6;
+        OVERLAY_SURFACE = 7;
+        OVERLAY_SURFACE_SKIRTS = 8;
+        NUM_LAYERS = 9;
+    }
+    
+    enum LayerMask {
+        TERRAIN_WITH_OVERGROUND = 7;
+        TERRAIN_WITH_WATER = 28;
+        TERRAIN_WITHOUT_WATER = 14;
+    }
+}
+
+message Texture {
+    repeated bytes data = 1;
+    
+    optional Format format = 2;
+    enum Format {
+        JPG = 1;
+        DXT1 = 2;
+        ETC1 = 3;
+        PVRTC2 = 4;
+        PVRTC4 = 5;
+        CRN_DXT1 = 6;
+    }
+    
+    optional uint32 width = 3 [default = 256];
+    optional uint32 height = 4 [default = 256];
+    
+    optional ViewDirection view_direction = 5;
+    enum ViewDirection {
+        NADIR = 0;
+        NORTH_45 = 1;
+        EAST_45 = 2;
+        SOUTH_45 = 3;
+        WEST_45 = 4;
+    }
+    
+    optional uint32 mesh_id = 6;
+}
+
+message TextureData {
+    optional NodeKey node_key = 1;
+    repeated Texture textures = 2;
+}
+
+message Copyrights {
+    repeated Copyright copyrights = 1;
+}
+
+message Copyright {
+    optional uint32 id = 1;
+    optional string text = 2;
+    optional string text_clean = 3;
+}
+
+message PlanetoidMetadata {
+    optional NodeMetadata root_node_metadata = 1;
+    optional float radius = 2;
+    optional float min_terrain_altitude = 3;
+    optional float max_terrain_altitude = 4;
+}
diff --git a/src/main.rs b/src/main.rs
new file mode 100644
index 0000000..733dbcc
--- /dev/null
+++ b/src/main.rs
@@ -0,0 +1,200 @@
+#![feature(array_chunks)]
+use anyhow::{Result, bail};
+use glam::Vec3;
+use log::{debug, error};
+use prost::{Message, bytes::Bytes};
+use proto::{BulkMetadata, NodeData, NodeMetadata, PlanetoidMetadata};
+use reqwest::{
+    Client,
+    header::{HeaderMap, HeaderName, HeaderValue},
+};
+use std::path::PathBuf;
+use tokio::{
+    fs::{File, create_dir, create_dir_all},
+    io::{AsyncReadExt, AsyncWriteExt},
+};
+use weareshared::{
+    Affine3A, Vec3A,
+    resources::{MeshPart, Prefab, RespackEntry},
+    respack::save_full_respack,
+    store::ResourceStore,
+    vec3a,
+};
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    env_logger::init_from_env("LOG");
+
+    let c = GeClient::new().await?;
+    let entry = c.planetoid_metdata().await?;
+
+    eprintln!("{:#?}", entry);
+
+    let mut bulk = c
+        .bulk_metadata("", entry.root_node_metadata.unwrap().bulk_metadata_epoch())
+        .await?;
+
+    eprintln!("{:#?}", bulk);
+
+    let store = ResourceStore::new_memory();
+
+    let mut meshes = Vec::new();
+
+    for node_meta in &bulk.node_metadata {
+        let Ok(mut node_data) = c.node_data(&bulk, &node_meta).await else {
+            continue;
+        };
+
+        for m in node_data.meshes {
+            let mut index_strip = Vec::new();
+
+            let strip_len = m.indices[0];
+            let mut zeros = 0;
+            for i in 0..strip_len {
+                let val = m.indices[i as usize + 1];
+                index_strip.push((zeros - val) as u32);
+                if val == 0 {
+                    zeros += 1;
+                }
+            }
+            let mut index = Vec::new();
+            for i in 0..index_strip.len() - 2 {
+                if i & 1 == 0 {
+                    index.push([index_strip[i + 0], index_strip[i + 1], index_strip[i + 2]]);
+                } else {
+                    index.push([index_strip[i + 0], index_strip[i + 2], index_strip[i + 1]]);
+                }
+            }
+
+            let mut positions = Vec::new();
+            let vert = m.vertices();
+            let vertex_count = vert.len() / 3;
+            let (mut x, mut y, mut z) = (0u8, 0u8, 0u8);
+            for i in 0..vertex_count {
+                x = x.wrapping_add(vert[vertex_count * 0 + i]);
+                y = y.wrapping_add(vert[vertex_count * 1 + i]);
+                z = z.wrapping_add(vert[vertex_count * 2 + i]);
+                positions.push(vec3a(x as f32, y as f32, z as f32));
+            }
+            // eprintln!("{index:?}");
+            // eprintln!("{positions:?}");
+
+            meshes.push((
+                Affine3A::from_scale(Vec3::splat(0.01)),
+                store.set(&MeshPart {
+                    index: Some(store.set(&index)?),
+                    va_position: Some(store.set(&positions)?),
+                    g_double_sided: Some(()),
+                    ..Default::default()
+                })?,
+            ))
+        }
+    }
+    eprintln!("{}", meshes.len());
+
+    let prefab = store.set(&Prefab {
+        mesh: meshes,
+        ..Default::default()
+    })?;
+
+    let entry = store.set(&RespackEntry {
+        c_prefab: vec![prefab],
+        ..Default::default()
+    })?;
+
+    let file = std::fs::File::create("/tmp/a.respack")?;
+    save_full_respack(file, &store, Some(entry))?;
+
+    Ok(())
+}
+
+struct GeClient {
+    client: Client,
+    cachedir: PathBuf,
+}
+impl GeClient {
+    pub async fn new() -> Result<Self> {
+        let cachedir = xdg::BaseDirectories::with_prefix("weareearth")
+            .unwrap()
+            .create_cache_directory("download")
+            .unwrap();
+        create_dir_all(cachedir.join("BulkMetadata")).await?;
+        create_dir_all(cachedir.join("NodeData")).await?;
+        Ok(Self {
+            cachedir,
+            client: Client::builder().default_headers(HeaderMap::from_iter([
+                (HeaderName::from_static("user-agent"), HeaderValue::from_static("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36")),
+                (HeaderName::from_static("referer"), HeaderValue::from_static("https://earth.google.com/"))
+            ])).build().unwrap()
+        })
+    }
+    pub async fn download(&self, path: &str) -> Result<Bytes> {
+        let cachepath = self.cachedir.join(path);
+        if cachepath.exists() {
+            debug!("cached {path:?}");
+            let mut buf = Vec::new();
+            File::open(cachepath).await?.read_to_end(&mut buf).await?;
+            Ok(buf.into())
+        } else {
+            debug!("download {path:?}");
+            let res = self
+                .client
+                .get(format!("https://kh.google.com/rt/earth/{path}"))
+                .send()
+                .await?;
+            if !res.status().is_success() {
+                error!("error response: {:?}", res.text().await?);
+                bail!("error response")
+            }
+            let buf = res.bytes().await?;
+            File::create(cachepath).await?.write_all(&buf).await?;
+            Ok(buf)
+        }
+    }
+
+    pub async fn planetoid_metdata(&self) -> Result<PlanetoidMetadata> {
+        let buf = self.download("PlanetoidMetadata").await?;
+        Ok(PlanetoidMetadata::decode(buf)?)
+    }
+    pub async fn bulk_metadata(&self, path: &str, epoch: u32) -> Result<BulkMetadata> {
+        let buf = self
+            .download(&format!("BulkMetadata/pb=!1m2!1s{path}!2u{epoch}"))
+            .await?;
+        Ok(BulkMetadata::decode(buf)?)
+    }
+    pub async fn node_data(&self, bulk: &BulkMetadata, node: &NodeMetadata) -> Result<NodeData> {
+        let (path, flags) = unpack_path_and_id(node.path_and_flags());
+
+        let texture_format = bulk.default_available_texture_formats();
+        let imagery_epoch = node.imagery_epoch.unwrap_or(bulk.default_imagery_epoch());
+        let node_epoch = bulk.head_node_key.as_ref().unwrap().epoch.unwrap(); // ?
+
+        let image_epoch_part = if flags & 16 != 0 {
+            format!("!3u{imagery_epoch}")
+        } else {
+            String::new()
+        };
+        let url = format!(
+            "NodeData/pb=!1m2!1s{path}!2u{node_epoch}!2e{texture_format}{image_epoch_part}!4b0"
+        );
+
+        let buf = self.download(&url).await?;
+        Ok(NodeData::decode(buf)?)
+    }
+}
+
+fn unpack_path_and_id(mut path_id: u32) -> (String, u32) {
+    let level = 1 + (path_id & 3);
+    path_id >>= 2;
+    let mut path = String::new();
+    for _ in 0..level {
+        path += &(path_id & 7).to_string();
+        path_id >>= 3;
+    }
+    let flags = path_id;
+    (path, flags)
+}
+
+pub mod proto {
+    include!(concat!(env!("OUT_DIR"), "/earth.proto.rs"));
+}