use crate::{ Flags, cache::Cache, proto::{BulkMetadata, NodeData, NodeMetadata, PlanetoidMetadata}, }; use anyhow::{Result, bail}; use log::{debug, error, info}; use prost::{Message, bytes::Bytes}; use reqwest::{ Client, header::{HeaderMap, HeaderName, HeaderValue}, }; use std::sync::atomic::{AtomicUsize, Ordering}; use tokio::sync::Semaphore; pub struct GeClient { download_counter: AtomicUsize, cache_counter: AtomicUsize, client: Client, cache: Cache, par_limit: Semaphore, } impl GeClient { pub async fn new(par_limit: usize, cache: Cache) -> Result { Ok(Self { download_counter: AtomicUsize::new(0), cache_counter: AtomicUsize::new(0), par_limit: Semaphore::new(par_limit),cache, client: Client::builder().default_headers(HeaderMap::from_iter([ (HeaderName::from_static("user-agent"), HeaderValue::from_static("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36")), (HeaderName::from_static("referer"), HeaderValue::from_static("https://earth.google.com/")) ])).build().unwrap() }) } pub async fn download(&self, path: &str) -> Result { let _permit = self.par_limit.acquire().await?; if let Some(d) = self.cache.get(path).await? { let n = self.cache_counter.fetch_add(1, Ordering::Relaxed); debug!("cached #{n} {path:?}"); Ok(d.into()) } else { let n = self.download_counter.fetch_add(1, Ordering::Relaxed); info!("download #{n} {path:?}"); let res = self .client .get(format!("https://kh.google.com/rt/earth/{path}")) .send() .await?; if !res.status().is_success() { error!("error response: {:?}", res.text().await?); bail!("error response") } let buf = res.bytes().await?; self.cache.insert(path, &buf).await?; Ok(buf) } } pub async fn planetoid_metdata(&self) -> Result { let buf = self.download("PlanetoidMetadata").await?; Ok(PlanetoidMetadata::decode(buf)?) } pub async fn bulk_metadata(&self, path: &str, epoch: u32) -> Result { let buf = self .download(&format!("BulkMetadata/pb=!1m2!1s{path}!2u{epoch}")) .await?; Ok(BulkMetadata::decode(buf)?) } pub async fn node_data( &self, abspath: &str, flags: Flags, bulk: &BulkMetadata, node: &NodeMetadata, ) -> Result { let texture_format = bulk.default_available_texture_formats(); let imagery_epoch = node.imagery_epoch.unwrap_or(bulk.default_imagery_epoch()); let node_epoch = node .epoch .unwrap_or(bulk.head_node_key.as_ref().unwrap().epoch.unwrap()); // ? let image_epoch_part = if flags.use_image_epoch { format!("!3u{imagery_epoch}") } else { String::new() }; let url = format!( "NodeData/pb=!1m2!1s{abspath}!2u{node_epoch}!2e{texture_format}{image_epoch_part}!4b0" ); let buf = self.download(&url).await?; Ok(NodeData::decode(buf)?) } }