From 2fc5931a6ce9bbb75757c4a20022b19778bd91c5 Mon Sep 17 00:00:00 2001 From: metamuffin Date: Tue, 24 Oct 2023 19:22:01 +0200 Subject: move db to jellybase --- base/src/database.rs | 36 ++++++++++++++++++++++++++++++++++++ base/src/lib.rs | 1 + 2 files changed, 37 insertions(+) create mode 100644 base/src/database.rs (limited to 'base/src') diff --git a/base/src/database.rs b/base/src/database.rs new file mode 100644 index 0000000..b8ba28e --- /dev/null +++ b/base/src/database.rs @@ -0,0 +1,36 @@ +/* + This file is part of jellything (https://codeberg.org/metamuffin/jellything) + which is licensed under the GNU Affero General Public License (version 3); see /COPYING. + Copyright (C) 2023 metamuffin +*/ +use anyhow::Context; +use jellycommon::{user::User, Node}; +use log::info; +use std::path::Path; +use typed_sled::Tree; + +pub use sled; + +pub struct Database { + pub db: sled::Db, + + pub user: Tree, + pub invite: Tree, + pub node: Tree, +} + +impl Database { + pub fn open(path: &Path) -> Result { + info!("opening database… (might take up to O(n) time)"); + let db = sled::open(path).context("opening database")?; + info!("creating trees"); + let r = Ok(Self { + user: Tree::open(&db, "user"), + invite: Tree::open(&db, "invite"), + node: Tree::open(&db, "node"), + db, + }); + info!("ready"); + r + } +} diff --git a/base/src/lib.rs b/base/src/lib.rs index 132fd45..5d96b1a 100644 --- a/base/src/lib.rs +++ b/base/src/lib.rs @@ -7,6 +7,7 @@ pub mod cache; pub mod permission; pub mod temp; +pub mod database; use jellycommon::{config::GlobalConfig, AssetLocation}; use std::{fs::File, path::PathBuf, sync::LazyLock}; -- cgit v1.2.3-70-g09d2 From 5ee01d06c0b067f2f07d0288c499897cd0df29f7 Mon Sep 17 00:00:00 2001 From: metamuffin Date: Tue, 24 Oct 2023 20:04:19 +0200 Subject: migration tool --- Cargo.lock | 52 +++++++++++++++++++++++++++++ base/src/database.rs | 1 + base/src/lib.rs | 12 +++---- tool/Cargo.toml | 2 ++ tool/src/import/mod.rs | 5 +++ tool/src/main.rs | 16 ++++++++- tool/src/migrate.rs | 89 ++++++++++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 170 insertions(+), 7 deletions(-) create mode 100644 tool/src/migrate.rs (limited to 'base/src') diff --git a/Cargo.lock b/Cargo.lock index 9de2869..c7ba7d1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -535,6 +535,19 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" +[[package]] +name = "console" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c926e00cc70edefdc64d3a5ff31cc65bb97a3460097762bd23afb4d8145fccf8" +dependencies = [ + "encode_unicode", + "lazy_static", + "libc", + "unicode-width", + "windows-sys 0.45.0", +] + [[package]] name = "cookie" version = "0.17.0" @@ -733,6 +746,12 @@ version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" +[[package]] +name = "encode_unicode" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" + [[package]] name = "encoding_rs" version = "0.8.32" @@ -1304,6 +1323,19 @@ dependencies = [ "hashbrown 0.14.0", ] +[[package]] +name = "indicatif" +version = "0.17.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb28741c9db9a713d93deb3bb9515c20788cef5815265bee4980e87bde7e0f25" +dependencies = [ + "console", + "instant", + "number_prefix", + "portable-atomic", + "unicode-width", +] + [[package]] name = "inlinable_string" version = "0.1.15" @@ -1500,6 +1532,8 @@ dependencies = [ "bincode 2.0.0-rc.3", "clap", "env_logger", + "indicatif", + "jellybase", "jellycommon", "jellymatroska", "jellyremuxer", @@ -1894,6 +1928,12 @@ dependencies = [ "libc", ] +[[package]] +name = "number_prefix" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" + [[package]] name = "object" version = "0.32.1" @@ -2122,6 +2162,12 @@ dependencies = [ "universal-hash", ] +[[package]] +name = "portable-atomic" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b559898e0b4931ed2d3b959ab0c2da4d99cc644c4b0b1a35b4d344027f474023" + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -3180,6 +3226,12 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-width" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" + [[package]] name = "unicode-xid" version = "0.2.4" diff --git a/base/src/database.rs b/base/src/database.rs index b8ba28e..739d292 100644 --- a/base/src/database.rs +++ b/base/src/database.rs @@ -10,6 +10,7 @@ use std::path::Path; use typed_sled::Tree; pub use sled; +pub use typed_sled; pub struct Database { pub db: sled::Db, diff --git a/base/src/lib.rs b/base/src/lib.rs index 5d96b1a..cfc5a11 100644 --- a/base/src/lib.rs +++ b/base/src/lib.rs @@ -5,20 +5,20 @@ */ #![feature(lazy_cell)] pub mod cache; +pub mod database; pub mod permission; pub mod temp; -pub mod database; use jellycommon::{config::GlobalConfig, AssetLocation}; use std::{fs::File, path::PathBuf, sync::LazyLock}; pub static CONF: LazyLock = LazyLock::new(|| { serde_yaml::from_reader( - File::open( - std::env::args() - .nth(1) - .expect("First argument must specify the configuration to use."), - ) + File::open(std::env::var("JELLYTHING_CONFIG").unwrap_or_else(|_| { + std::env::args().nth(1).expect( + "First argument or JELLYTHING_CONFIG must specify the configuration to use.", + ) + })) .unwrap(), ) .unwrap() diff --git a/tool/Cargo.toml b/tool/Cargo.toml index c9d46f1..a53e1bd 100644 --- a/tool/Cargo.toml +++ b/tool/Cargo.toml @@ -5,6 +5,7 @@ edition = "2021" [dependencies] jellycommon = { path = "../common" } +jellybase = { path = "../base" } jellymatroska = { path = "../matroska" } jellyremuxer = { path = "../remuxer" } @@ -13,6 +14,7 @@ env_logger = "0.10.0" anyhow = "1.0.75" clap = { version = "4.4.6", features = ["derive"] } reqwest = { version = "0.11.20", features = ["blocking", "json"] } +indicatif = "0.17.7" serde = { version = "1.0.188", features = ["derive"] } serde_json = "1.0.107" diff --git a/tool/src/import/mod.rs b/tool/src/import/mod.rs index e148c98..17f3137 100644 --- a/tool/src/import/mod.rs +++ b/tool/src/import/mod.rs @@ -1,3 +1,8 @@ +/* + This file is part of jellything (https://codeberg.org/metamuffin/jellything) + which is licensed under the GNU Affero General Public License (version 3); see /COPYING. + Copyright (C) 2023 metamuffin +*/ pub mod infojson; pub mod tmdb; diff --git a/tool/src/main.rs b/tool/src/main.rs index 1e84bda..3a670f5 100644 --- a/tool/src/main.rs +++ b/tool/src/main.rs @@ -6,12 +6,14 @@ #![feature(file_create_new)] pub mod import; +pub mod migrate; use base64::Engine; -use clap::{Parser, Subcommand}; +use clap::{Parser, Subcommand, ValueEnum}; use import::import; use jellycommon::{config::GlobalConfig, Node, NodeKind, NodePrivate, NodePublic}; use log::{info, warn}; +use migrate::migrate; use rand::random; use std::{fmt::Debug, fs::File, io::Write, path::PathBuf}; @@ -61,6 +63,17 @@ enum Action { #[arg(short, long)] series: bool, }, + Migrate { + database: PathBuf, + mode: MigrateMode, + save_location: PathBuf, + }, +} + +#[derive(Debug, Clone, Copy, PartialEq, ValueEnum)] +enum MigrateMode { + Import, + Export, } fn main() -> anyhow::Result<()> { @@ -123,6 +136,7 @@ fn main() -> anyhow::Result<()> { Ok(()) } a @ Action::New { .. } => import(a, args.dry), + a @ Action::Migrate { .. } => migrate(a), } } diff --git a/tool/src/migrate.rs b/tool/src/migrate.rs new file mode 100644 index 0000000..7c24087 --- /dev/null +++ b/tool/src/migrate.rs @@ -0,0 +1,89 @@ +/* + This file is part of jellything (https://codeberg.org/metamuffin/jellything) + which is licensed under the GNU Affero General Public License (version 3); see /COPYING. + Copyright (C) 2023 metamuffin +*/ +use crate::{Action, MigrateMode}; +use anyhow::bail; +use indicatif::ProgressIterator; +use jellybase::database::{typed_sled::Tree, Database}; +use log::info; +use serde::Serialize; +use std::{ + fs::File, + io::{BufRead, BufReader, BufWriter, Write}, + path::Path, +}; + +pub(crate) fn migrate(action: Action) -> anyhow::Result<()> { + match action { + Action::Migrate { + mode, + save_location, + database, + } => { + std::fs::create_dir_all(&save_location)?; + let db = Database::open(&database)?; + + info!("processing 'user'"); + process_tree(mode, &save_location.join("user"), &db.user)?; + info!("processing 'invite'"); + process_tree(mode, &save_location.join("invite"), &db.invite)?; + info!("processing 'node'"); + process_tree(mode, &save_location.join("node"), &db.node)?; + info!("done"); + Ok(()) + } + _ => unreachable!(), + } +} + +fn process_tree< + K: Serialize + for<'de> serde::Deserialize<'de>, + V: Serialize + for<'de> serde::Deserialize<'de>, +>( + mode: MigrateMode, + path: &Path, + tree: &Tree, +) -> anyhow::Result<()> { + match mode { + MigrateMode::Export => export_tree(path, tree), + MigrateMode::Import => import_tree(path, tree), + } +} + +fn export_tree< + K: Serialize + for<'de> serde::Deserialize<'de>, + V: Serialize + for<'de> serde::Deserialize<'de>, +>( + path: &Path, + tree: &Tree, +) -> anyhow::Result<()> { + let mut o = BufWriter::new(File::create(path)?); + let len = tree.len(); + for r in tree.iter().progress_count(len.try_into().unwrap()) { + let (k, v) = r?; + serde_json::to_writer(&mut o, &(k, v))?; + writeln!(&mut o)?; + } + Ok(()) +} + +fn import_tree< + K: Serialize + for<'de> serde::Deserialize<'de>, + V: Serialize + for<'de> serde::Deserialize<'de>, +>( + path: &Path, + tree: &Tree, +) -> anyhow::Result<()> { + if !tree.is_empty() { + bail!("tree not empty, `rm -rf` your db please :)") + } + let i = BufReader::new(File::open(path)?); + for l in i.lines() { + let l = l?; + let (k, v) = serde_json::from_str::<(K, V)>(&l)?; + tree.insert(&k, &v)?; + } + Ok(()) +} -- cgit v1.2.3-70-g09d2