From 42568332b5433c97813e8c291db0fc0d15867b76 Mon Sep 17 00:00:00 2001 From: metamuffin Date: Tue, 24 Oct 2023 19:32:26 +0200 Subject: import -> jellytool --- tool/src/import/infojson.rs | 143 +++++++++++++++++++++++ tool/src/import/mod.rs | 277 ++++++++++++++++++++++++++++++++++++++++++++ tool/src/import/tmdb.rs | 95 +++++++++++++++ tool/src/main.rs | 150 ++++++++++++++++++++++++ 4 files changed, 665 insertions(+) create mode 100644 tool/src/import/infojson.rs create mode 100644 tool/src/import/mod.rs create mode 100644 tool/src/import/tmdb.rs create mode 100644 tool/src/main.rs (limited to 'tool/src') diff --git a/tool/src/import/infojson.rs b/tool/src/import/infojson.rs new file mode 100644 index 0000000..dd2151b --- /dev/null +++ b/tool/src/import/infojson.rs @@ -0,0 +1,143 @@ +/* + This file is part of jellything (https://codeberg.org/metamuffin/jellything) + which is licensed under the GNU Affero General Public License (version 3); see /COPYING. + Copyright (C) 2023 metamuffin +*/ + +use anyhow::Context; +use jellycommon::chrono::{format::Parsed, DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; + +#[derive(Debug, Serialize, Deserialize)] +pub struct YVideo { + pub id: String, + pub title: String, + pub formats: Vec, + pub thumbnails: Vec, + pub thumbnail: String, + pub description: String, + pub channel_id: String, + pub duration: Option, + pub view_count: usize, + pub average_rating: Option, + pub age_limit: usize, + pub webpage_url: String, + pub categories: Vec, + pub tags: Vec, + pub playable_in_embed: bool, + pub automatic_captions: HashMap>, + pub comment_count: Option, + pub chapters: Option>, + pub heatmap: Option>, + pub like_count: Option, + pub channel: Option, + pub channel_follower_count: usize, + pub channel_is_verified: Option, + pub uploader: String, + pub uploader_id: String, + pub uploader_url: String, + pub upload_date: String, + pub availability: String, // "public" | "private" | "unlisted", + pub original_url: Option, + pub webpage_url_basename: String, + pub webpage_url_domain: String, + pub extractor: String, + pub extractor_key: String, + pub playlist_count: usize, + pub playlist: String, + pub playlist_id: String, + pub playlist_title: String, + pub playlist_uploader: String, + pub playlist_uploader_id: String, + pub n_entries: usize, + pub playlist_index: usize, + pub display_id: String, + pub fulltitle: String, + pub duration_string: String, + pub is_live: bool, + pub was_live: bool, + pub epoch: usize, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct YCaption { + pub url: Option, + pub ext: String, //"vtt" | "json3" | "srv1" | "srv2" | "srv3" | "ttml", + pub protocol: Option, + pub name: Option, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct YFormat { + pub format_id: String, + pub format_note: Option, + pub ext: String, + pub protocol: String, + pub acodec: Option, + pub vcodec: Option, + pub url: Option, + pub width: Option, + pub height: Option, + pub fps: Option, + pub columns: Option, + pub fragments: Option>, + pub resolution: String, + pub dynamic_range: Option, + pub aspect_ratio: Option, + pub http_headers: HashMap, + pub audio_ext: String, + pub video_ext: String, + pub vbr: Option, + pub abr: Option, + pub format: String, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct YFragment { + pub url: Option, + pub duration: Option, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct YThumbnail { + pub url: String, + pub preference: i32, + pub id: String, + pub height: Option, + pub width: Option, + pub resolution: Option, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct YChapter { + pub start_time: f64, + pub end_time: f64, + pub title: String, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct YHeatmapSample { + pub start_time: f64, + pub end_time: f64, + pub value: f64, +} + +pub fn parse_upload_date(d: &str) -> anyhow::Result> { + let (year, month, day) = (&d[0..4], &d[4..6], &d[6..8]); + let (year, month, day) = ( + year.parse().context("parsing year")?, + month.parse().context("parsing month")?, + day.parse().context("parsing day")?, + ); + + let mut p = Parsed::new(); + p.year = Some(year); + p.month = Some(month); + p.day = Some(day); + p.hour_div_12 = Some(0); + p.hour_mod_12 = Some(0); + p.minute = Some(0); + p.second = Some(0); + Ok(p.to_datetime_with_timezone(&Utc)?) +} diff --git a/tool/src/import/mod.rs b/tool/src/import/mod.rs new file mode 100644 index 0000000..e148c98 --- /dev/null +++ b/tool/src/import/mod.rs @@ -0,0 +1,277 @@ +pub mod infojson; +pub mod tmdb; + +use crate::{make_ident, ok_or_warn, Action}; +use anyhow::Context; +use infojson::{parse_upload_date, YVideo}; +use jellycommon::{ + AssetLocation, LocalTrack, MediaInfo, MediaSource, Node, NodeKind, NodePrivate, NodePublic, + Rating, +}; +use jellymatroska::read::EbmlReader; +use jellyremuxer::import::import_metadata; +use log::{info, warn}; +use std::{ + collections::BTreeMap, + fs::{remove_file, File}, + io::{stdin, BufReader, Write}, +}; +use tmdb::{tmdb_details, tmdb_image}; + +pub(crate) fn import(action: Action, dry: bool) -> anyhow::Result<()> { + match action { + Action::New { + path, + tmdb_id, + tmdb_search, + input, + series, + ident_prefix, + copy, + video, + r#move, + } => { + if std::env::current_dir().unwrap().file_name().unwrap() != "library" { + warn!("new command can only be used in the library directory; what you are doing right now probably wont work.") + } + + let tmdb_kind = if series { "tv" } else { "movie" }; + let tmdb_id = if let Some(id) = tmdb_id { + Some(id.parse().unwrap()) + } else if let Some(title) = tmdb_search { + let tmdb_key = std::env::var("TMDB_API_KEY").context("tmdb api key required")?; + let results = tmdb::tmdb_search(tmdb_kind, &title, &tmdb_key)?; + info!("results:"); + for (i, r) in results.results.iter().enumerate() { + info!( + "\t[{i}] {}: {} ({})", + r.id, + r.title.as_ref().or(r.name.as_ref()).unwrap(), + r.overview.chars().take(100).collect::() + ); + } + let res_index = if results.results.len() > 1 { + stdin() + .lines() + .next() + .unwrap() + .unwrap() + .parse::() + .unwrap() + } else { + 0 + }; + Some(results.results[res_index].id) + } else { + None + }; + + let tmdb_details = tmdb_id + .map(|id| { + let tmdb_key = + std::env::var("TMDB_API_KEY").context("tmdb api key required")?; + let td = tmdb_details(tmdb_kind, id, &tmdb_key) + .context("fetching details") + .unwrap(); + Ok::<_, anyhow::Error>(td) + }) + .transpose()?; + + let mut kind = NodeKind::Series; + let mut file_meta = None; + let mut infojson = None; + + if let Some(input_path) = &input { + file_meta = Some({ + let input = BufReader::new(File::open(&input_path).unwrap()); + let mut input = EbmlReader::new(input); + import_metadata(&mut input)? + }); + + if let Some(ij) = &file_meta.as_ref().unwrap().infojson { + infojson = + Some(serde_json::from_str::(ij).context("parsing info.json")?); + } + + kind = if video { + NodeKind::Video + } else { + NodeKind::Movie + }; + } + + let title = tmdb_details + .as_ref() + .map(|d| d.title.clone().or(d.name.clone())) + .flatten() + .or(file_meta.as_ref().map(|m| m.title.clone()).flatten()) + .expect("no title detected"); + + let ident = format!( + "{}{}", + ident_prefix.unwrap_or(String::new()), + make_ident( + &infojson + .as_ref() + .map(|i| i.id.clone()) + .unwrap_or(title.clone()) + ), + ); + let path = path.join(&ident); + let source_path = input.as_ref().map(|_| path.join(format!("source.mkv"))); + + let (mut poster, mut backdrop) = (None, None); + if !dry { + std::fs::create_dir_all(&path)?; + + poster = file_meta + .as_ref() + .map(|m| { + m.cover + .as_ref() + .map(|(mime, data)| { + let pu = path.join(format!( + "cover.{}", + match mime.as_str() { + "image/webp" => "webp", + "image/jpeg" => "jpeg", + "image/png" => "png", + _ => { + warn!("unknown mime, just using webp"); + "webp" + } + } + )); + if !pu.exists() { + let mut f = File::create(&pu)?; + f.write_all(&data)?; + } + Ok::<_, anyhow::Error>(pu) + }) + .transpose() + }) + .transpose()? + .flatten() + .or(tmdb_details + .as_ref() + .map(|d| { + d.poster_path + .as_ref() + .map(|p| { + let pu = path.join("poster.jpeg"); + let mut f = File::create(&pu)?; + tmdb_image(&p, &mut f)?; + Ok::<_, anyhow::Error>(pu) + }) + .transpose() + }) + .transpose()? + .flatten()); + + backdrop = tmdb_details + .as_ref() + .map(|d| { + d.backdrop_path + .as_ref() + .map(|p| { + let pu = path.join("backdrop.jpeg"); + let mut f = File::create(&pu)?; + tmdb_image(&p, &mut f)?; + Ok::<_, anyhow::Error>(pu) + }) + .transpose() + }) + .transpose()? + .flatten(); + } + + let mut ratings = BTreeMap::new(); + + ratings.extend( + infojson + .as_ref() + .map(|i| (Rating::YoutubeViews, i.view_count as f64)), + ); + ratings.extend( + infojson + .as_ref() + .map(|i| i.like_count.map(|l| (Rating::YoutubeLikes, l as f64))) + .flatten(), + ); + + let node = Node { + private: NodePrivate { + id: Some(ident.clone()), + import: None, + backdrop: backdrop.clone().map(AssetLocation::Library), + poster: poster.clone().map(AssetLocation::Library), + source: file_meta.as_ref().map(|m| MediaSource::Local { + tracks: m + .track_sources + .clone() + .into_iter() + .map(|t| LocalTrack { + path: source_path.clone().unwrap(), + ..t + }) + .collect(), + }), + }, + public: NodePublic { + federated: None, + ratings, + description: file_meta + .as_ref() + .map(|m| m.description.clone()) + .flatten() + .or(tmdb_details.as_ref().map(|d| d.overview.to_owned())), + tagline: file_meta.as_ref().map(|m| m.tagline.clone()).flatten().or( + tmdb_details + .as_ref() + .map(|d| d.tagline.to_owned()) + .flatten(), + ), + title, + index: None, + kind, + children: Vec::new(), + media: file_meta.as_ref().map(|m| MediaInfo { + duration: m.duration, + tracks: m.tracks.clone(), + }), + release_date: infojson + .as_ref() + .and_then(|j| ok_or_warn(parse_upload_date(&j.upload_date))), + ..Default::default() + }, + }; + + if dry { + println!("{}", serde_json::to_string_pretty(&node)?); + } else { + if let Some(source_path) = source_path { + let input = input.clone().unwrap(); + if r#move { + std::fs::rename(&input, &source_path)?; + } else if copy { + std::fs::copy(&input, &source_path)?; + } else { + if source_path.is_symlink() { + remove_file(&source_path)?; + } + std::os::unix::fs::symlink(&input, &source_path)?; + } + } + let f = File::create(path.join(if series { + "directory.json" + } else { + "item.jelly" + }))?; + serde_json::to_writer_pretty(f, &node)?; + } + + Ok(()) + } + _ => unreachable!(), + } +} diff --git a/tool/src/import/tmdb.rs b/tool/src/import/tmdb.rs new file mode 100644 index 0000000..5f21afd --- /dev/null +++ b/tool/src/import/tmdb.rs @@ -0,0 +1,95 @@ +/* + This file is part of jellything (https://codeberg.org/metamuffin/jellything) + which is licensed under the GNU Affero General Public License (version 3); see /COPYING. + Copyright (C) 2023 metamuffin +*/ +use log::info; +use serde::Deserialize; +use std::io::Write; + +#[derive(Debug, Clone, Deserialize)] +pub struct TmdbQuery { + pub page: usize, + pub results: Vec, + pub total_pages: usize, + pub total_results: usize, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct TmdbQueryResult { + pub adult: bool, + pub backdrop_path: Option, + pub genre_ids: Vec, + pub id: u64, + pub original_language: Option, + pub original_title: Option, + pub overview: String, + pub popularity: f64, + pub poster_path: Option, + pub release_date: Option, + pub title: Option, + pub name: Option, + pub vote_average: f64, + pub vote_count: usize, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct TmdbDetails { + pub adult: bool, + pub backdrop_path: Option, + pub genres: Vec, + pub id: u64, + pub original_language: Option, + pub original_title: Option, + pub overview: String, + pub popularity: f64, + pub poster_path: Option, + pub release_date: Option, + pub title: Option, + pub name: Option, + pub vote_average: f64, + pub vote_count: usize, + pub budget: Option, + pub homepage: Option, + pub imdb_id: Option, + pub production_companies: Vec, + pub revenue: Option, + pub tagline: Option, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct TmdbGenre { + pub id: u64, + pub name: String, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct TmdbProductionCompany { + pub id: u64, + pub name: String, + pub logo_path: Option, +} + +pub fn tmdb_search(kind: &str, query: &str, key: &str) -> anyhow::Result { + info!("searching tmdb: {query:?}"); + Ok(reqwest::blocking::get(&format!( + "https://api.themoviedb.org/3/search/{kind}?query={}&api_key={key}", + query.replace(" ", "+") + ))? + .json::()?) +} + +pub fn tmdb_details(kind: &str, id: u64, key: &str) -> anyhow::Result { + info!("fetching details: {id:?}"); + Ok(reqwest::blocking::get(&format!( + "https://api.themoviedb.org/3/{kind}/{id}?api_key={key}" + ))? + .json()?) +} + +pub fn tmdb_image(path: &str, out: &mut impl Write) -> anyhow::Result<()> { + info!("downloading image {path:?}"); + let mut res = reqwest::blocking::get(&format!("https://image.tmdb.org/t/p/original{path}"))?; + res.copy_to(out)?; + Ok(()) +} diff --git a/tool/src/main.rs b/tool/src/main.rs new file mode 100644 index 0000000..1e84bda --- /dev/null +++ b/tool/src/main.rs @@ -0,0 +1,150 @@ +/* + This file is part of jellything (https://codeberg.org/metamuffin/jellything) + which is licensed under the GNU Affero General Public License (version 3); see /COPYING. + Copyright (C) 2023 metamuffin +*/ +#![feature(file_create_new)] + +pub mod import; + +use base64::Engine; +use clap::{Parser, Subcommand}; +use import::import; +use jellycommon::{config::GlobalConfig, Node, NodeKind, NodePrivate, NodePublic}; +use log::{info, warn}; +use rand::random; +use std::{fmt::Debug, fs::File, io::Write, path::PathBuf}; + +#[derive(Parser)] +struct Args { + #[arg(short = 'N', long)] + dry: bool, + #[clap(subcommand)] + action: Action, +} + +#[derive(Subcommand)] +enum Action { + /// Initialize a new jellything instance + Init { + /// Base path of the instance, must either be absolute or relative to the servers pwd + base_path: PathBuf, + #[arg(short, long)] + brand: String, + }, + /// Imports a movie, video or series given media and metadata sources + New { + /// Relative path to the node's parent(!). + path: PathBuf, + /// Search the node by title on TMDB + #[arg(short = 't', long)] + tmdb_search: Option, + /// Search the node by id on TMDB + #[arg(short = 'T', long)] + tmdb_id: Option, + #[arg(long)] + /// Prefix the inferred id with something to avoid collisions + ident_prefix: Option, + /// Copies media into the library + #[arg(long)] + copy: bool, + /// Moves media into the library (potentially destructive operation) + #[arg(long)] + r#move: bool, + /// Marks node as a video + #[arg(long)] + video: bool, + /// Path to the media of the node, required for non-series + #[arg(short, long)] + input: Option, + /// Marks node as a series + #[arg(short, long)] + series: bool, + }, +} + +fn main() -> anyhow::Result<()> { + env_logger::builder() + .filter_level(log::LevelFilter::Info) + .parse_env("LOG") + .init(); + let args = Args::parse(); + + match args.action { + Action::Init { + base_path: path, + brand, + } => { + info!("creating new instance..."); + std::fs::create_dir_all(path.join("library"))?; + std::fs::create_dir_all(path.join("cache"))?; + std::fs::create_dir_all(path.join("assets"))?; + File::create_new(path.join("assets/front.htm"))? + .write_fmt(format_args!("

My very own jellything instance

"))?; + serde_yaml::to_writer( + File::create_new(path.join("config.yaml"))?, + &GlobalConfig { + brand: brand.clone(), + slogan: "Creative slogan here".to_string(), + asset_path: path.join("assets"), + cache_path: path.join("cache"), + library_path: path.join("library"), + database_path: path.join("database"), + temp_path: "/tmp".into(), + cookie_key: Some( + base64::engine::general_purpose::STANDARD + .encode([(); 32].map(|_| random())), + ), + session_key: Some( + base64::engine::general_purpose::STANDARD + .encode([(); 32].map(|_| random())), + ), + admin_username: "admin".to_string(), + admin_password: "hackme".to_string(), + login_expire: 10, + ..Default::default() + }, + )?; + serde_json::to_writer( + File::create_new(path.join("library/directory.json"))?, + &Node { + public: NodePublic { + kind: NodeKind::Collection, + title: "My Library".to_string(), + ..Default::default() + }, + private: NodePrivate { + ..Default::default() + }, + }, + )?; + info!("{brand:?} is ready!"); + warn!("please change the admin password."); + Ok(()) + } + a @ Action::New { .. } => import(a, args.dry), + } +} + +fn make_ident(s: &str) -> String { + let mut out = String::new(); + for s in s.chars() { + match s { + 'a'..='z' | '0'..='9' => out.push(s), + 'A'..='Z' => out.push(s.to_ascii_lowercase()), + '-' | ' ' | '_' | ':' => out.push('-'), + _ => (), + } + } + out +} + +fn ok_or_warn(r: Result) -> Option { + match r { + Ok(t) => Some(t), + Err(e) => { + warn!("{e:?}"); + None + } + } +} -- cgit v1.2.3-70-g09d2 From 5ee01d06c0b067f2f07d0288c499897cd0df29f7 Mon Sep 17 00:00:00 2001 From: metamuffin Date: Tue, 24 Oct 2023 20:04:19 +0200 Subject: migration tool --- Cargo.lock | 52 +++++++++++++++++++++++++++++ base/src/database.rs | 1 + base/src/lib.rs | 12 +++---- tool/Cargo.toml | 2 ++ tool/src/import/mod.rs | 5 +++ tool/src/main.rs | 16 ++++++++- tool/src/migrate.rs | 89 ++++++++++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 170 insertions(+), 7 deletions(-) create mode 100644 tool/src/migrate.rs (limited to 'tool/src') diff --git a/Cargo.lock b/Cargo.lock index 9de2869..c7ba7d1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -535,6 +535,19 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" +[[package]] +name = "console" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c926e00cc70edefdc64d3a5ff31cc65bb97a3460097762bd23afb4d8145fccf8" +dependencies = [ + "encode_unicode", + "lazy_static", + "libc", + "unicode-width", + "windows-sys 0.45.0", +] + [[package]] name = "cookie" version = "0.17.0" @@ -733,6 +746,12 @@ version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" +[[package]] +name = "encode_unicode" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" + [[package]] name = "encoding_rs" version = "0.8.32" @@ -1304,6 +1323,19 @@ dependencies = [ "hashbrown 0.14.0", ] +[[package]] +name = "indicatif" +version = "0.17.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb28741c9db9a713d93deb3bb9515c20788cef5815265bee4980e87bde7e0f25" +dependencies = [ + "console", + "instant", + "number_prefix", + "portable-atomic", + "unicode-width", +] + [[package]] name = "inlinable_string" version = "0.1.15" @@ -1500,6 +1532,8 @@ dependencies = [ "bincode 2.0.0-rc.3", "clap", "env_logger", + "indicatif", + "jellybase", "jellycommon", "jellymatroska", "jellyremuxer", @@ -1894,6 +1928,12 @@ dependencies = [ "libc", ] +[[package]] +name = "number_prefix" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" + [[package]] name = "object" version = "0.32.1" @@ -2122,6 +2162,12 @@ dependencies = [ "universal-hash", ] +[[package]] +name = "portable-atomic" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b559898e0b4931ed2d3b959ab0c2da4d99cc644c4b0b1a35b4d344027f474023" + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -3180,6 +3226,12 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-width" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" + [[package]] name = "unicode-xid" version = "0.2.4" diff --git a/base/src/database.rs b/base/src/database.rs index b8ba28e..739d292 100644 --- a/base/src/database.rs +++ b/base/src/database.rs @@ -10,6 +10,7 @@ use std::path::Path; use typed_sled::Tree; pub use sled; +pub use typed_sled; pub struct Database { pub db: sled::Db, diff --git a/base/src/lib.rs b/base/src/lib.rs index 5d96b1a..cfc5a11 100644 --- a/base/src/lib.rs +++ b/base/src/lib.rs @@ -5,20 +5,20 @@ */ #![feature(lazy_cell)] pub mod cache; +pub mod database; pub mod permission; pub mod temp; -pub mod database; use jellycommon::{config::GlobalConfig, AssetLocation}; use std::{fs::File, path::PathBuf, sync::LazyLock}; pub static CONF: LazyLock = LazyLock::new(|| { serde_yaml::from_reader( - File::open( - std::env::args() - .nth(1) - .expect("First argument must specify the configuration to use."), - ) + File::open(std::env::var("JELLYTHING_CONFIG").unwrap_or_else(|_| { + std::env::args().nth(1).expect( + "First argument or JELLYTHING_CONFIG must specify the configuration to use.", + ) + })) .unwrap(), ) .unwrap() diff --git a/tool/Cargo.toml b/tool/Cargo.toml index c9d46f1..a53e1bd 100644 --- a/tool/Cargo.toml +++ b/tool/Cargo.toml @@ -5,6 +5,7 @@ edition = "2021" [dependencies] jellycommon = { path = "../common" } +jellybase = { path = "../base" } jellymatroska = { path = "../matroska" } jellyremuxer = { path = "../remuxer" } @@ -13,6 +14,7 @@ env_logger = "0.10.0" anyhow = "1.0.75" clap = { version = "4.4.6", features = ["derive"] } reqwest = { version = "0.11.20", features = ["blocking", "json"] } +indicatif = "0.17.7" serde = { version = "1.0.188", features = ["derive"] } serde_json = "1.0.107" diff --git a/tool/src/import/mod.rs b/tool/src/import/mod.rs index e148c98..17f3137 100644 --- a/tool/src/import/mod.rs +++ b/tool/src/import/mod.rs @@ -1,3 +1,8 @@ +/* + This file is part of jellything (https://codeberg.org/metamuffin/jellything) + which is licensed under the GNU Affero General Public License (version 3); see /COPYING. + Copyright (C) 2023 metamuffin +*/ pub mod infojson; pub mod tmdb; diff --git a/tool/src/main.rs b/tool/src/main.rs index 1e84bda..3a670f5 100644 --- a/tool/src/main.rs +++ b/tool/src/main.rs @@ -6,12 +6,14 @@ #![feature(file_create_new)] pub mod import; +pub mod migrate; use base64::Engine; -use clap::{Parser, Subcommand}; +use clap::{Parser, Subcommand, ValueEnum}; use import::import; use jellycommon::{config::GlobalConfig, Node, NodeKind, NodePrivate, NodePublic}; use log::{info, warn}; +use migrate::migrate; use rand::random; use std::{fmt::Debug, fs::File, io::Write, path::PathBuf}; @@ -61,6 +63,17 @@ enum Action { #[arg(short, long)] series: bool, }, + Migrate { + database: PathBuf, + mode: MigrateMode, + save_location: PathBuf, + }, +} + +#[derive(Debug, Clone, Copy, PartialEq, ValueEnum)] +enum MigrateMode { + Import, + Export, } fn main() -> anyhow::Result<()> { @@ -123,6 +136,7 @@ fn main() -> anyhow::Result<()> { Ok(()) } a @ Action::New { .. } => import(a, args.dry), + a @ Action::Migrate { .. } => migrate(a), } } diff --git a/tool/src/migrate.rs b/tool/src/migrate.rs new file mode 100644 index 0000000..7c24087 --- /dev/null +++ b/tool/src/migrate.rs @@ -0,0 +1,89 @@ +/* + This file is part of jellything (https://codeberg.org/metamuffin/jellything) + which is licensed under the GNU Affero General Public License (version 3); see /COPYING. + Copyright (C) 2023 metamuffin +*/ +use crate::{Action, MigrateMode}; +use anyhow::bail; +use indicatif::ProgressIterator; +use jellybase::database::{typed_sled::Tree, Database}; +use log::info; +use serde::Serialize; +use std::{ + fs::File, + io::{BufRead, BufReader, BufWriter, Write}, + path::Path, +}; + +pub(crate) fn migrate(action: Action) -> anyhow::Result<()> { + match action { + Action::Migrate { + mode, + save_location, + database, + } => { + std::fs::create_dir_all(&save_location)?; + let db = Database::open(&database)?; + + info!("processing 'user'"); + process_tree(mode, &save_location.join("user"), &db.user)?; + info!("processing 'invite'"); + process_tree(mode, &save_location.join("invite"), &db.invite)?; + info!("processing 'node'"); + process_tree(mode, &save_location.join("node"), &db.node)?; + info!("done"); + Ok(()) + } + _ => unreachable!(), + } +} + +fn process_tree< + K: Serialize + for<'de> serde::Deserialize<'de>, + V: Serialize + for<'de> serde::Deserialize<'de>, +>( + mode: MigrateMode, + path: &Path, + tree: &Tree, +) -> anyhow::Result<()> { + match mode { + MigrateMode::Export => export_tree(path, tree), + MigrateMode::Import => import_tree(path, tree), + } +} + +fn export_tree< + K: Serialize + for<'de> serde::Deserialize<'de>, + V: Serialize + for<'de> serde::Deserialize<'de>, +>( + path: &Path, + tree: &Tree, +) -> anyhow::Result<()> { + let mut o = BufWriter::new(File::create(path)?); + let len = tree.len(); + for r in tree.iter().progress_count(len.try_into().unwrap()) { + let (k, v) = r?; + serde_json::to_writer(&mut o, &(k, v))?; + writeln!(&mut o)?; + } + Ok(()) +} + +fn import_tree< + K: Serialize + for<'de> serde::Deserialize<'de>, + V: Serialize + for<'de> serde::Deserialize<'de>, +>( + path: &Path, + tree: &Tree, +) -> anyhow::Result<()> { + if !tree.is_empty() { + bail!("tree not empty, `rm -rf` your db please :)") + } + let i = BufReader::new(File::open(path)?); + for l in i.lines() { + let l = l?; + let (k, v) = serde_json::from_str::<(K, V)>(&l)?; + tree.insert(&k, &v)?; + } + Ok(()) +} -- cgit v1.2.3-70-g09d2