From 42568332b5433c97813e8c291db0fc0d15867b76 Mon Sep 17 00:00:00 2001 From: metamuffin Date: Tue, 24 Oct 2023 19:32:26 +0200 Subject: import -> jellytool --- tool/src/import/infojson.rs | 143 +++++++++++++++++++++++ tool/src/import/mod.rs | 277 ++++++++++++++++++++++++++++++++++++++++++++ tool/src/import/tmdb.rs | 95 +++++++++++++++ tool/src/main.rs | 150 ++++++++++++++++++++++++ 4 files changed, 665 insertions(+) create mode 100644 tool/src/import/infojson.rs create mode 100644 tool/src/import/mod.rs create mode 100644 tool/src/import/tmdb.rs create mode 100644 tool/src/main.rs (limited to 'tool/src') diff --git a/tool/src/import/infojson.rs b/tool/src/import/infojson.rs new file mode 100644 index 0000000..dd2151b --- /dev/null +++ b/tool/src/import/infojson.rs @@ -0,0 +1,143 @@ +/* + This file is part of jellything (https://codeberg.org/metamuffin/jellything) + which is licensed under the GNU Affero General Public License (version 3); see /COPYING. + Copyright (C) 2023 metamuffin +*/ + +use anyhow::Context; +use jellycommon::chrono::{format::Parsed, DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; + +#[derive(Debug, Serialize, Deserialize)] +pub struct YVideo { + pub id: String, + pub title: String, + pub formats: Vec, + pub thumbnails: Vec, + pub thumbnail: String, + pub description: String, + pub channel_id: String, + pub duration: Option, + pub view_count: usize, + pub average_rating: Option, + pub age_limit: usize, + pub webpage_url: String, + pub categories: Vec, + pub tags: Vec, + pub playable_in_embed: bool, + pub automatic_captions: HashMap>, + pub comment_count: Option, + pub chapters: Option>, + pub heatmap: Option>, + pub like_count: Option, + pub channel: Option, + pub channel_follower_count: usize, + pub channel_is_verified: Option, + pub uploader: String, + pub uploader_id: String, + pub uploader_url: String, + pub upload_date: String, + pub availability: String, // "public" | "private" | "unlisted", + pub original_url: Option, + pub webpage_url_basename: String, + pub webpage_url_domain: String, + pub extractor: String, + pub extractor_key: String, + pub playlist_count: usize, + pub playlist: String, + pub playlist_id: String, + pub playlist_title: String, + pub playlist_uploader: String, + pub playlist_uploader_id: String, + pub n_entries: usize, + pub playlist_index: usize, + pub display_id: String, + pub fulltitle: String, + pub duration_string: String, + pub is_live: bool, + pub was_live: bool, + pub epoch: usize, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct YCaption { + pub url: Option, + pub ext: String, //"vtt" | "json3" | "srv1" | "srv2" | "srv3" | "ttml", + pub protocol: Option, + pub name: Option, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct YFormat { + pub format_id: String, + pub format_note: Option, + pub ext: String, + pub protocol: String, + pub acodec: Option, + pub vcodec: Option, + pub url: Option, + pub width: Option, + pub height: Option, + pub fps: Option, + pub columns: Option, + pub fragments: Option>, + pub resolution: String, + pub dynamic_range: Option, + pub aspect_ratio: Option, + pub http_headers: HashMap, + pub audio_ext: String, + pub video_ext: String, + pub vbr: Option, + pub abr: Option, + pub format: String, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct YFragment { + pub url: Option, + pub duration: Option, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct YThumbnail { + pub url: String, + pub preference: i32, + pub id: String, + pub height: Option, + pub width: Option, + pub resolution: Option, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct YChapter { + pub start_time: f64, + pub end_time: f64, + pub title: String, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct YHeatmapSample { + pub start_time: f64, + pub end_time: f64, + pub value: f64, +} + +pub fn parse_upload_date(d: &str) -> anyhow::Result> { + let (year, month, day) = (&d[0..4], &d[4..6], &d[6..8]); + let (year, month, day) = ( + year.parse().context("parsing year")?, + month.parse().context("parsing month")?, + day.parse().context("parsing day")?, + ); + + let mut p = Parsed::new(); + p.year = Some(year); + p.month = Some(month); + p.day = Some(day); + p.hour_div_12 = Some(0); + p.hour_mod_12 = Some(0); + p.minute = Some(0); + p.second = Some(0); + Ok(p.to_datetime_with_timezone(&Utc)?) +} diff --git a/tool/src/import/mod.rs b/tool/src/import/mod.rs new file mode 100644 index 0000000..e148c98 --- /dev/null +++ b/tool/src/import/mod.rs @@ -0,0 +1,277 @@ +pub mod infojson; +pub mod tmdb; + +use crate::{make_ident, ok_or_warn, Action}; +use anyhow::Context; +use infojson::{parse_upload_date, YVideo}; +use jellycommon::{ + AssetLocation, LocalTrack, MediaInfo, MediaSource, Node, NodeKind, NodePrivate, NodePublic, + Rating, +}; +use jellymatroska::read::EbmlReader; +use jellyremuxer::import::import_metadata; +use log::{info, warn}; +use std::{ + collections::BTreeMap, + fs::{remove_file, File}, + io::{stdin, BufReader, Write}, +}; +use tmdb::{tmdb_details, tmdb_image}; + +pub(crate) fn import(action: Action, dry: bool) -> anyhow::Result<()> { + match action { + Action::New { + path, + tmdb_id, + tmdb_search, + input, + series, + ident_prefix, + copy, + video, + r#move, + } => { + if std::env::current_dir().unwrap().file_name().unwrap() != "library" { + warn!("new command can only be used in the library directory; what you are doing right now probably wont work.") + } + + let tmdb_kind = if series { "tv" } else { "movie" }; + let tmdb_id = if let Some(id) = tmdb_id { + Some(id.parse().unwrap()) + } else if let Some(title) = tmdb_search { + let tmdb_key = std::env::var("TMDB_API_KEY").context("tmdb api key required")?; + let results = tmdb::tmdb_search(tmdb_kind, &title, &tmdb_key)?; + info!("results:"); + for (i, r) in results.results.iter().enumerate() { + info!( + "\t[{i}] {}: {} ({})", + r.id, + r.title.as_ref().or(r.name.as_ref()).unwrap(), + r.overview.chars().take(100).collect::() + ); + } + let res_index = if results.results.len() > 1 { + stdin() + .lines() + .next() + .unwrap() + .unwrap() + .parse::() + .unwrap() + } else { + 0 + }; + Some(results.results[res_index].id) + } else { + None + }; + + let tmdb_details = tmdb_id + .map(|id| { + let tmdb_key = + std::env::var("TMDB_API_KEY").context("tmdb api key required")?; + let td = tmdb_details(tmdb_kind, id, &tmdb_key) + .context("fetching details") + .unwrap(); + Ok::<_, anyhow::Error>(td) + }) + .transpose()?; + + let mut kind = NodeKind::Series; + let mut file_meta = None; + let mut infojson = None; + + if let Some(input_path) = &input { + file_meta = Some({ + let input = BufReader::new(File::open(&input_path).unwrap()); + let mut input = EbmlReader::new(input); + import_metadata(&mut input)? + }); + + if let Some(ij) = &file_meta.as_ref().unwrap().infojson { + infojson = + Some(serde_json::from_str::(ij).context("parsing info.json")?); + } + + kind = if video { + NodeKind::Video + } else { + NodeKind::Movie + }; + } + + let title = tmdb_details + .as_ref() + .map(|d| d.title.clone().or(d.name.clone())) + .flatten() + .or(file_meta.as_ref().map(|m| m.title.clone()).flatten()) + .expect("no title detected"); + + let ident = format!( + "{}{}", + ident_prefix.unwrap_or(String::new()), + make_ident( + &infojson + .as_ref() + .map(|i| i.id.clone()) + .unwrap_or(title.clone()) + ), + ); + let path = path.join(&ident); + let source_path = input.as_ref().map(|_| path.join(format!("source.mkv"))); + + let (mut poster, mut backdrop) = (None, None); + if !dry { + std::fs::create_dir_all(&path)?; + + poster = file_meta + .as_ref() + .map(|m| { + m.cover + .as_ref() + .map(|(mime, data)| { + let pu = path.join(format!( + "cover.{}", + match mime.as_str() { + "image/webp" => "webp", + "image/jpeg" => "jpeg", + "image/png" => "png", + _ => { + warn!("unknown mime, just using webp"); + "webp" + } + } + )); + if !pu.exists() { + let mut f = File::create(&pu)?; + f.write_all(&data)?; + } + Ok::<_, anyhow::Error>(pu) + }) + .transpose() + }) + .transpose()? + .flatten() + .or(tmdb_details + .as_ref() + .map(|d| { + d.poster_path + .as_ref() + .map(|p| { + let pu = path.join("poster.jpeg"); + let mut f = File::create(&pu)?; + tmdb_image(&p, &mut f)?; + Ok::<_, anyhow::Error>(pu) + }) + .transpose() + }) + .transpose()? + .flatten()); + + backdrop = tmdb_details + .as_ref() + .map(|d| { + d.backdrop_path + .as_ref() + .map(|p| { + let pu = path.join("backdrop.jpeg"); + let mut f = File::create(&pu)?; + tmdb_image(&p, &mut f)?; + Ok::<_, anyhow::Error>(pu) + }) + .transpose() + }) + .transpose()? + .flatten(); + } + + let mut ratings = BTreeMap::new(); + + ratings.extend( + infojson + .as_ref() + .map(|i| (Rating::YoutubeViews, i.view_count as f64)), + ); + ratings.extend( + infojson + .as_ref() + .map(|i| i.like_count.map(|l| (Rating::YoutubeLikes, l as f64))) + .flatten(), + ); + + let node = Node { + private: NodePrivate { + id: Some(ident.clone()), + import: None, + backdrop: backdrop.clone().map(AssetLocation::Library), + poster: poster.clone().map(AssetLocation::Library), + source: file_meta.as_ref().map(|m| MediaSource::Local { + tracks: m + .track_sources + .clone() + .into_iter() + .map(|t| LocalTrack { + path: source_path.clone().unwrap(), + ..t + }) + .collect(), + }), + }, + public: NodePublic { + federated: None, + ratings, + description: file_meta + .as_ref() + .map(|m| m.description.clone()) + .flatten() + .or(tmdb_details.as_ref().map(|d| d.overview.to_owned())), + tagline: file_meta.as_ref().map(|m| m.tagline.clone()).flatten().or( + tmdb_details + .as_ref() + .map(|d| d.tagline.to_owned()) + .flatten(), + ), + title, + index: None, + kind, + children: Vec::new(), + media: file_meta.as_ref().map(|m| MediaInfo { + duration: m.duration, + tracks: m.tracks.clone(), + }), + release_date: infojson + .as_ref() + .and_then(|j| ok_or_warn(parse_upload_date(&j.upload_date))), + ..Default::default() + }, + }; + + if dry { + println!("{}", serde_json::to_string_pretty(&node)?); + } else { + if let Some(source_path) = source_path { + let input = input.clone().unwrap(); + if r#move { + std::fs::rename(&input, &source_path)?; + } else if copy { + std::fs::copy(&input, &source_path)?; + } else { + if source_path.is_symlink() { + remove_file(&source_path)?; + } + std::os::unix::fs::symlink(&input, &source_path)?; + } + } + let f = File::create(path.join(if series { + "directory.json" + } else { + "item.jelly" + }))?; + serde_json::to_writer_pretty(f, &node)?; + } + + Ok(()) + } + _ => unreachable!(), + } +} diff --git a/tool/src/import/tmdb.rs b/tool/src/import/tmdb.rs new file mode 100644 index 0000000..5f21afd --- /dev/null +++ b/tool/src/import/tmdb.rs @@ -0,0 +1,95 @@ +/* + This file is part of jellything (https://codeberg.org/metamuffin/jellything) + which is licensed under the GNU Affero General Public License (version 3); see /COPYING. + Copyright (C) 2023 metamuffin +*/ +use log::info; +use serde::Deserialize; +use std::io::Write; + +#[derive(Debug, Clone, Deserialize)] +pub struct TmdbQuery { + pub page: usize, + pub results: Vec, + pub total_pages: usize, + pub total_results: usize, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct TmdbQueryResult { + pub adult: bool, + pub backdrop_path: Option, + pub genre_ids: Vec, + pub id: u64, + pub original_language: Option, + pub original_title: Option, + pub overview: String, + pub popularity: f64, + pub poster_path: Option, + pub release_date: Option, + pub title: Option, + pub name: Option, + pub vote_average: f64, + pub vote_count: usize, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct TmdbDetails { + pub adult: bool, + pub backdrop_path: Option, + pub genres: Vec, + pub id: u64, + pub original_language: Option, + pub original_title: Option, + pub overview: String, + pub popularity: f64, + pub poster_path: Option, + pub release_date: Option, + pub title: Option, + pub name: Option, + pub vote_average: f64, + pub vote_count: usize, + pub budget: Option, + pub homepage: Option, + pub imdb_id: Option, + pub production_companies: Vec, + pub revenue: Option, + pub tagline: Option, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct TmdbGenre { + pub id: u64, + pub name: String, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct TmdbProductionCompany { + pub id: u64, + pub name: String, + pub logo_path: Option, +} + +pub fn tmdb_search(kind: &str, query: &str, key: &str) -> anyhow::Result { + info!("searching tmdb: {query:?}"); + Ok(reqwest::blocking::get(&format!( + "https://api.themoviedb.org/3/search/{kind}?query={}&api_key={key}", + query.replace(" ", "+") + ))? + .json::()?) +} + +pub fn tmdb_details(kind: &str, id: u64, key: &str) -> anyhow::Result { + info!("fetching details: {id:?}"); + Ok(reqwest::blocking::get(&format!( + "https://api.themoviedb.org/3/{kind}/{id}?api_key={key}" + ))? + .json()?) +} + +pub fn tmdb_image(path: &str, out: &mut impl Write) -> anyhow::Result<()> { + info!("downloading image {path:?}"); + let mut res = reqwest::blocking::get(&format!("https://image.tmdb.org/t/p/original{path}"))?; + res.copy_to(out)?; + Ok(()) +} diff --git a/tool/src/main.rs b/tool/src/main.rs new file mode 100644 index 0000000..1e84bda --- /dev/null +++ b/tool/src/main.rs @@ -0,0 +1,150 @@ +/* + This file is part of jellything (https://codeberg.org/metamuffin/jellything) + which is licensed under the GNU Affero General Public License (version 3); see /COPYING. + Copyright (C) 2023 metamuffin +*/ +#![feature(file_create_new)] + +pub mod import; + +use base64::Engine; +use clap::{Parser, Subcommand}; +use import::import; +use jellycommon::{config::GlobalConfig, Node, NodeKind, NodePrivate, NodePublic}; +use log::{info, warn}; +use rand::random; +use std::{fmt::Debug, fs::File, io::Write, path::PathBuf}; + +#[derive(Parser)] +struct Args { + #[arg(short = 'N', long)] + dry: bool, + #[clap(subcommand)] + action: Action, +} + +#[derive(Subcommand)] +enum Action { + /// Initialize a new jellything instance + Init { + /// Base path of the instance, must either be absolute or relative to the servers pwd + base_path: PathBuf, + #[arg(short, long)] + brand: String, + }, + /// Imports a movie, video or series given media and metadata sources + New { + /// Relative path to the node's parent(!). + path: PathBuf, + /// Search the node by title on TMDB + #[arg(short = 't', long)] + tmdb_search: Option, + /// Search the node by id on TMDB + #[arg(short = 'T', long)] + tmdb_id: Option, + #[arg(long)] + /// Prefix the inferred id with something to avoid collisions + ident_prefix: Option, + /// Copies media into the library + #[arg(long)] + copy: bool, + /// Moves media into the library (potentially destructive operation) + #[arg(long)] + r#move: bool, + /// Marks node as a video + #[arg(long)] + video: bool, + /// Path to the media of the node, required for non-series + #[arg(short, long)] + input: Option, + /// Marks node as a series + #[arg(short, long)] + series: bool, + }, +} + +fn main() -> anyhow::Result<()> { + env_logger::builder() + .filter_level(log::LevelFilter::Info) + .parse_env("LOG") + .init(); + let args = Args::parse(); + + match args.action { + Action::Init { + base_path: path, + brand, + } => { + info!("creating new instance..."); + std::fs::create_dir_all(path.join("library"))?; + std::fs::create_dir_all(path.join("cache"))?; + std::fs::create_dir_all(path.join("assets"))?; + File::create_new(path.join("assets/front.htm"))? + .write_fmt(format_args!("

My very own jellything instance

"))?; + serde_yaml::to_writer( + File::create_new(path.join("config.yaml"))?, + &GlobalConfig { + brand: brand.clone(), + slogan: "Creative slogan here".to_string(), + asset_path: path.join("assets"), + cache_path: path.join("cache"), + library_path: path.join("library"), + database_path: path.join("database"), + temp_path: "/tmp".into(), + cookie_key: Some( + base64::engine::general_purpose::STANDARD + .encode([(); 32].map(|_| random())), + ), + session_key: Some( + base64::engine::general_purpose::STANDARD + .encode([(); 32].map(|_| random())), + ), + admin_username: "admin".to_string(), + admin_password: "hackme".to_string(), + login_expire: 10, + ..Default::default() + }, + )?; + serde_json::to_writer( + File::create_new(path.join("library/directory.json"))?, + &Node { + public: NodePublic { + kind: NodeKind::Collection, + title: "My Library".to_string(), + ..Default::default() + }, + private: NodePrivate { + ..Default::default() + }, + }, + )?; + info!("{brand:?} is ready!"); + warn!("please change the admin password."); + Ok(()) + } + a @ Action::New { .. } => import(a, args.dry), + } +} + +fn make_ident(s: &str) -> String { + let mut out = String::new(); + for s in s.chars() { + match s { + 'a'..='z' | '0'..='9' => out.push(s), + 'A'..='Z' => out.push(s.to_ascii_lowercase()), + '-' | ' ' | '_' | ':' => out.push('-'), + _ => (), + } + } + out +} + +fn ok_or_warn(r: Result) -> Option { + match r { + Ok(t) => Some(t), + Err(e) => { + warn!("{e:?}"); + None + } + } +} -- cgit v1.2.3-70-g09d2