From a93e0bf8db710ef9dcd40e1526ddd4b18a7288e9 Mon Sep 17 00:00:00 2001 From: metamuffin Date: Sat, 7 Mar 2026 02:58:23 +0100 Subject: move cache tools to kv crate --- Cargo.lock | 9 +------ Cargo.toml | 1 - cache/tools/Cargo.toml | 16 ------------- cache/tools/cache_fs_to_rocksdb.rs | 38 ------------------------------ cache/tools/cache_rocksdb_delete_prefix.rs | 25 -------------------- kv/Cargo.toml | 16 +++++++++++++ kv/src/bin/fs_to_rocksdb.rs | 38 ++++++++++++++++++++++++++++++ kv/src/bin/rocksdb_remove_prefix.rs | 25 ++++++++++++++++++++ 8 files changed, 80 insertions(+), 88 deletions(-) delete mode 100644 cache/tools/Cargo.toml delete mode 100644 cache/tools/cache_fs_to_rocksdb.rs delete mode 100644 cache/tools/cache_rocksdb_delete_prefix.rs create mode 100644 kv/src/bin/fs_to_rocksdb.rs create mode 100644 kv/src/bin/rocksdb_remove_prefix.rs diff --git a/Cargo.lock b/Cargo.lock index 8cc80bd..fe68879 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2011,6 +2011,7 @@ name = "jellykv" version = "0.1.0" dependencies = [ "anyhow", + "humansize", "rand 0.10.0", "redb", "rocksdb", @@ -4265,14 +4266,6 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" -[[package]] -name = "tools" -version = "0.1.0" -dependencies = [ - "anyhow", - "rocksdb", -] - [[package]] name = "tower" version = "0.5.2" diff --git a/Cargo.toml b/Cargo.toml index 794722f..ab2d749 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,6 @@ [workspace] members = [ "cache", - "cache/tools", "common", "common/object", "database", diff --git a/cache/tools/Cargo.toml b/cache/tools/Cargo.toml deleted file mode 100644 index bf0427d..0000000 --- a/cache/tools/Cargo.toml +++ /dev/null @@ -1,16 +0,0 @@ -[package] -name = "tools" -version = "0.1.0" -edition = "2024" - -[dependencies] -rocksdb = { version = "0.24.0", features = ["multi-threaded-cf"] } -anyhow = { workspace = true } - -[[bin]] -name = "cache_fs_to_rocksdb" -path = "cache_fs_to_rocksdb.rs" - -[[bin]] -name = "cache_rocksdb_delete_prefix" -path = "cache_rocksdb_delete_prefix.rs" diff --git a/cache/tools/cache_fs_to_rocksdb.rs b/cache/tools/cache_fs_to_rocksdb.rs deleted file mode 100644 index d283dcb..0000000 --- a/cache/tools/cache_fs_to_rocksdb.rs +++ /dev/null @@ -1,38 +0,0 @@ -/* - This file is part of jellything (https://codeberg.org/metamuffin/jellything) - which is licensed under the GNU Affero General Public License (version 3); see /COPYING. - Copyright (C) 2026 metamuffin -*/ - -use anyhow::{Result, bail}; -use rocksdb::DB; -use std::{env::args, fs::File, io::Read, path::Path}; - -fn main() -> Result<()> { - let in_path = args().nth(1).unwrap(); - let out_path = args().nth(2).unwrap(); - let db = DB::open_default(out_path)?; - if !in_path.ends_with("/") { - bail!("path needs to end with /") - } - traverse(&db, &in_path, in_path.as_ref())?; - db.flush()?; - Ok(()) -} - -fn traverse(db: &DB, prefix: &str, path: &Path) -> Result<()> { - if path.is_dir() { - for e in path.read_dir()? { - traverse(db, prefix, &e?.path())?; - } - } - if path.is_file() { - let key = path.to_string_lossy(); - let key = key.strip_prefix(prefix).unwrap(); - let mut value = Vec::new(); - File::open(path)?.read_to_end(&mut value)?; - println!("{key}"); - db.put(key, value)?; - } - Ok(()) -} diff --git a/cache/tools/cache_rocksdb_delete_prefix.rs b/cache/tools/cache_rocksdb_delete_prefix.rs deleted file mode 100644 index e09ce61..0000000 --- a/cache/tools/cache_rocksdb_delete_prefix.rs +++ /dev/null @@ -1,25 +0,0 @@ -/* - This file is part of jellything (https://codeberg.org/metamuffin/jellything) - which is licensed under the GNU Affero General Public License (version 3); see /COPYING. - Copyright (C) 2026 metamuffin -*/ - -use anyhow::Result; -use rocksdb::DB; -use std::env::args; - -fn main() -> Result<()> { - let db = DB::open_default(args().nth(1).unwrap())?; - let prefix = args().nth(2).unwrap(); - for r in db.prefix_iterator(&prefix) { - let key = r?.0; - let key_s = String::from_utf8_lossy(&key); - if !key_s.starts_with(&prefix) { - break; - } - println!("{key_s}"); - db.delete(key)?; - } - db.flush()?; - Ok(()) -} diff --git a/kv/Cargo.toml b/kv/Cargo.toml index eafd496..ce29215 100644 --- a/kv/Cargo.toml +++ b/kv/Cargo.toml @@ -10,6 +10,22 @@ rocksdb = { version = "0.24.0", features = [ "multi-threaded-cf", ], optional = true } redb = { version = "3.1.0", optional = true } +humansize = "2.1.3" + +[[bin]] +name = "jellykv_rocksdb_remove_prefix" +path = "src/bin/rocksdb_remove_prefix.rs" +required-features = ["rocksdb"] + +[[bin]] +name = "jellykv_rocksdb_stats" +path = "src/bin/rocksdb_stats.rs" +required-features = ["rocksdb"] + +[[bin]] +name = "jellykv_fs_to_rocksdb" +path = "src/bin/fs_to_rocksdb.rs" +required-features = ["rocksdb"] [features] # default = ["rocksdb", "redb", "memory", "filesystem"] diff --git a/kv/src/bin/fs_to_rocksdb.rs b/kv/src/bin/fs_to_rocksdb.rs new file mode 100644 index 0000000..d283dcb --- /dev/null +++ b/kv/src/bin/fs_to_rocksdb.rs @@ -0,0 +1,38 @@ +/* + This file is part of jellything (https://codeberg.org/metamuffin/jellything) + which is licensed under the GNU Affero General Public License (version 3); see /COPYING. + Copyright (C) 2026 metamuffin +*/ + +use anyhow::{Result, bail}; +use rocksdb::DB; +use std::{env::args, fs::File, io::Read, path::Path}; + +fn main() -> Result<()> { + let in_path = args().nth(1).unwrap(); + let out_path = args().nth(2).unwrap(); + let db = DB::open_default(out_path)?; + if !in_path.ends_with("/") { + bail!("path needs to end with /") + } + traverse(&db, &in_path, in_path.as_ref())?; + db.flush()?; + Ok(()) +} + +fn traverse(db: &DB, prefix: &str, path: &Path) -> Result<()> { + if path.is_dir() { + for e in path.read_dir()? { + traverse(db, prefix, &e?.path())?; + } + } + if path.is_file() { + let key = path.to_string_lossy(); + let key = key.strip_prefix(prefix).unwrap(); + let mut value = Vec::new(); + File::open(path)?.read_to_end(&mut value)?; + println!("{key}"); + db.put(key, value)?; + } + Ok(()) +} diff --git a/kv/src/bin/rocksdb_remove_prefix.rs b/kv/src/bin/rocksdb_remove_prefix.rs new file mode 100644 index 0000000..e09ce61 --- /dev/null +++ b/kv/src/bin/rocksdb_remove_prefix.rs @@ -0,0 +1,25 @@ +/* + This file is part of jellything (https://codeberg.org/metamuffin/jellything) + which is licensed under the GNU Affero General Public License (version 3); see /COPYING. + Copyright (C) 2026 metamuffin +*/ + +use anyhow::Result; +use rocksdb::DB; +use std::env::args; + +fn main() -> Result<()> { + let db = DB::open_default(args().nth(1).unwrap())?; + let prefix = args().nth(2).unwrap(); + for r in db.prefix_iterator(&prefix) { + let key = r?.0; + let key_s = String::from_utf8_lossy(&key); + if !key_s.starts_with(&prefix) { + break; + } + println!("{key_s}"); + db.delete(key)?; + } + db.flush()?; + Ok(()) +} -- cgit v1.3