diff options
| -rw-r--r-- | Cargo.lock | 198 | ||||
| -rw-r--r-- | Cargo.toml | 1 | ||||
| -rw-r--r-- | src/cache.rs | 32 | ||||
| -rw-r--r-- | src/main.rs | 20 |
4 files changed, 231 insertions, 20 deletions
@@ -179,6 +179,44 @@ dependencies = [ ] [[package]] +name = "bindgen" +version = "0.69.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" +dependencies = [ + "bitflags", + "cexpr", + "clang-sys", + "itertools 0.12.1", + "lazy_static", + "lazycell", + "proc-macro2", + "quote", + "regex", + "rustc-hash 1.1.0", + "shlex", + "syn", +] + +[[package]] +name = "bindgen" +version = "0.71.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f58bf3d7db68cfbac37cfc485a8d711e87e064c3d0fe0435b92f7a407f9d6b3" +dependencies = [ + "bitflags", + "cexpr", + "clang-sys", + "itertools 0.13.0", + "proc-macro2", + "quote", + "regex", + "rustc-hash 2.1.1", + "shlex", + "syn", +] + +[[package]] name = "bitflags" version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -231,6 +269,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" [[package]] +name = "bzip2-sys" +version = "0.1.13+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" +dependencies = [ + "cc", + "pkg-config", +] + +[[package]] name = "cc" version = "1.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -242,6 +290,15 @@ dependencies = [ ] [[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + +[[package]] name = "cfg-if" version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -254,6 +311,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] +name = "clang-sys" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +dependencies = [ + "glob", + "libc", + "libloading", +] + +[[package]] name = "clap" version = "4.5.35" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -502,6 +570,12 @@ dependencies = [ ] [[package]] +name = "glob" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" + +[[package]] name = "h2" version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -793,6 +867,24 @@ checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" [[package]] name = "itertools" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" @@ -851,12 +943,60 @@ dependencies = [ ] [[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + +[[package]] name = "libc" version = "0.2.171" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6" [[package]] +name = "libloading" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" +dependencies = [ + "cfg-if", + "windows-targets 0.52.6", +] + +[[package]] +name = "librocksdb-sys" +version = "0.17.1+9.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b7869a512ae9982f4d46ba482c2a304f1efd80c6412a3d4bf57bb79a619679f" +dependencies = [ + "bindgen 0.69.5", + "bzip2-sys", + "cc", + "libc", + "libz-sys", + "lz4-sys", + "zstd-sys", +] + +[[package]] +name = "libz-sys" +version = "1.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b70e7a7df205e92a1a4cd9aaae7898dac0aa555503cc0a649494d0d60e7651d" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + +[[package]] name = "linux-raw-sys" version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -885,6 +1025,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" [[package]] +name = "lz4-sys" +version = "1.11.1+lz4-1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bd8c0d6c6ed0cd30b3652886bb8711dc4bb01d637a68105a3d5158039b418e6" +dependencies = [ + "cc", + "libc", +] + +[[package]] name = "memchr" version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -897,6 +1047,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] name = "miniz_oxide" version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -923,6 +1079,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "defc4c55412d89136f966bbb339008b474350e5e6e78d2714439c386b3137a03" [[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] name = "object" version = "0.36.7" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1054,7 +1220,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" dependencies = [ "heck", - "itertools", + "itertools 0.14.0", "log", "multimap", "once_cell", @@ -1074,7 +1240,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools", + "itertools 0.14.0", "proc-macro2", "quote", "syn", @@ -1100,7 +1266,7 @@ dependencies = [ "pin-project-lite", "quinn-proto", "quinn-udp", - "rustc-hash", + "rustc-hash 2.1.1", "rustls", "socket2", "thiserror", @@ -1119,7 +1285,7 @@ dependencies = [ "getrandom 0.3.2", "rand", "ring", - "rustc-hash", + "rustc-hash 2.1.1", "rustls", "rustls-pki-types", "slab", @@ -1296,6 +1462,16 @@ dependencies = [ ] [[package]] +name = "rocksdb" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26ec73b20525cb235bad420f911473b69f9fe27cc856c5461bccd7e4af037f43" +dependencies = [ + "libc", + "librocksdb-sys", +] + +[[package]] name = "rustc-demangle" version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1303,6 +1479,12 @@ checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" [[package]] name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + +[[package]] +name = "rustc-hash" version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" @@ -1724,6 +1906,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] name = "virtue" version = "0.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1839,6 +2027,7 @@ dependencies = [ "prost-types", "redb", "reqwest", + "rocksdb", "tokio", "weareshared", "xdg", @@ -2212,6 +2401,7 @@ version = "2.0.15+zstd.1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eb81183ddd97d0c74cedf1d50d85c8d08c1b8b68ee863bdee9e706eedba1a237" dependencies = [ + "bindgen 0.71.1", "cc", "pkg-config", ] @@ -22,6 +22,7 @@ glam = "0.30.1" futures-util = "0.3.31" clap = { version = "4.5.35", features = ["derive"] } redb = "2.4.0" +rocksdb = "0.23.0" weareshared = { path = "../wearechat/shared" } [build-dependencies] diff --git a/src/cache.rs b/src/cache.rs index a4d6dfb..ebf5e82 100644 --- a/src/cache.rs +++ b/src/cache.rs @@ -1,7 +1,6 @@ use anyhow::Result; use futures_util::{StreamExt, stream::FuturesUnordered}; use log::{debug, info}; -use redb::{Database, Durability, TableDefinition}; use std::{ fs::create_dir_all, path::{Path, PathBuf}, @@ -14,9 +13,15 @@ use tokio::{ pub enum Cache { Directory(PathBuf), - Redb { db: Database, counter: AtomicUsize }, + Redb { + db: redb::Database, + counter: AtomicUsize, + }, + Rocksdb { + db: rocksdb::DBWithThreadMode<rocksdb::MultiThreaded>, + }, } -const T_DOWNLOAD: TableDefinition<&str, &[u8]> = TableDefinition::new("dl"); +const T_DOWNLOAD: redb::TableDefinition<&str, &[u8]> = redb::TableDefinition::new("dl"); impl Cache { pub fn new_directory() -> Result<Self> { let cachedir = xdg::BaseDirectories::with_prefix("weareearth") @@ -27,9 +32,9 @@ impl Cache { create_dir_all(cachedir.join("NodeData"))?; Ok(Self::Directory(cachedir)) } - pub fn new_db(path: &Path) -> Result<Self> { - info!("opening db..."); - let db = Database::create(path)?; + pub fn new_redb(path: &Path) -> Result<Self> { + info!("opening redb..."); + let db = redb::Database::create(path)?; info!("done"); { let txn = db.begin_write()?; @@ -41,6 +46,12 @@ impl Cache { counter: AtomicUsize::new(0), }) } + pub fn new_rocksdb(path: &Path) -> Result<Self> { + info!("opening rocksdb..."); + let db = rocksdb::DBWithThreadMode::open_default(path)?; + info!("done"); + Ok(Self::Rocksdb { db }) + } pub async fn transfer_entries(&self, other: &Cache) -> Result<()> { match self { @@ -104,6 +115,7 @@ impl Cache { Ok(None) } } + Cache::Rocksdb { db } => Ok(db.get(path)?), } } pub async fn insert(&self, path: &str, data: &[u8]) -> Result<()> { @@ -118,10 +130,10 @@ impl Cache { let mut txn = db.begin_write()?; if n % 100_000 == 0 { info!("flushing db"); - txn.set_durability(Durability::Immediate); + txn.set_durability(redb::Durability::Immediate); txn.set_quick_repair(true); } else { - txn.set_durability(Durability::None) + txn.set_durability(redb::Durability::None) }; let mut table = txn.open_table(T_DOWNLOAD)?; table.insert(path, data)?; @@ -129,6 +141,10 @@ impl Cache { txn.commit()?; Ok(()) } + Cache::Rocksdb { db } => { + db.put(path, data)?; + Ok(()) + } } } } diff --git a/src/main.rs b/src/main.rs index 7ea1afa..66a011f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -26,7 +26,9 @@ struct Args { #[arg(short, long, default_value = "16")] par_limit: usize, #[arg(short, long)] - db_cache: Option<PathBuf>, + redb_cache: Option<PathBuf>, + #[arg(short, long)] + rocksdb_cache: Option<PathBuf>, #[clap(subcommand)] action: Action, } @@ -35,7 +37,7 @@ struct Args { enum Action { Cache { level: usize }, Export { level: usize }, - CacheFsToDb { db: PathBuf }, + CacheFsToDb, } #[tokio::main] @@ -44,15 +46,17 @@ async fn main() -> Result<()> { let args = Args::parse(); - let cache = if let Some(path) = args.db_cache { - Cache::new_db(&path)? + let cache = if let Some(path) = args.rocksdb_cache { + Cache::new_rocksdb(&path)? + } else if let Some(path) = args.redb_cache { + Cache::new_redb(&path)? } else { Cache::new_directory()? }; - let c = GeClient::new(16, cache).await?; match args.action { Action::Cache { level } => { + let c = GeClient::new(16, cache).await?; let entry = c.planetoid_metdata().await?; let epoch = entry.root_node_metadata.unwrap().bulk_metadata_epoch(); cache_all( @@ -65,6 +69,7 @@ async fn main() -> Result<()> { .await?; } Action::Export { level } => { + let c = GeClient::new(16, cache).await?; let entry = c.planetoid_metdata().await?; let store = Arc::new(ResourceStore::new_memory()); @@ -85,10 +90,9 @@ async fn main() -> Result<()> { let file = std::fs::File::create("/tmp/a.respack")?; save_full_respack(file, &store, Some(entry))?; } - Action::CacheFsToDb { db: dbpath } => { + Action::CacheFsToDb => { let source = Cache::new_directory()?; - let dest = Cache::new_db(&dbpath)?; - source.transfer_entries(&dest).await?; + source.transfer_entries(&cache).await?; } } |