diff options
author | metamuffin <metamuffin@disroot.org> | 2023-12-21 23:57:42 +0100 |
---|---|---|
committer | metamuffin <metamuffin@disroot.org> | 2023-12-21 23:57:42 +0100 |
commit | 3a29113e965a94bdef06655f1583cc6e86edd606 (patch) | |
tree | a0910fa9687a9935ba1ca85a9cb5def1a0bc9069 /tool/src | |
parent | a8b2480e898e269e7e0d41dbd46d9a18c7d1e4ba (diff) | |
download | jellything-3a29113e965a94bdef06655f1583cc6e86edd606.tar jellything-3a29113e965a94bdef06655f1583cc6e86edd606.tar.bz2 jellything-3a29113e965a94bdef06655f1583cc6e86edd606.tar.zst |
rework import system pt. 1
Diffstat (limited to 'tool/src')
-rw-r--r-- | tool/src/import/infojson.rs | 143 | ||||
-rw-r--r-- | tool/src/import/mod.rs | 320 | ||||
-rw-r--r-- | tool/src/import/tmdb.rs | 116 | ||||
-rw-r--r-- | tool/src/main.rs | 93 |
4 files changed, 47 insertions, 625 deletions
diff --git a/tool/src/import/infojson.rs b/tool/src/import/infojson.rs deleted file mode 100644 index 3f0edc9..0000000 --- a/tool/src/import/infojson.rs +++ /dev/null @@ -1,143 +0,0 @@ -/* - This file is part of jellything (https://codeberg.org/metamuffin/jellything) - which is licensed under the GNU Affero General Public License (version 3); see /COPYING. - Copyright (C) 2023 metamuffin <metamuffin.org> -*/ - -use anyhow::Context; -use jellycommon::chrono::{format::Parsed, DateTime, Utc}; -use serde::{Deserialize, Serialize}; -use std::collections::HashMap; - -#[derive(Debug, Serialize, Deserialize)] -pub struct YVideo { - pub id: String, - pub title: String, - pub formats: Vec<YFormat>, - pub thumbnails: Vec<YThumbnail>, - pub thumbnail: String, - pub description: String, - pub channel_id: String, - pub duration: Option<f64>, - pub view_count: usize, - pub average_rating: Option<String>, - pub age_limit: usize, - pub webpage_url: String, - pub categories: Vec<String>, - pub tags: Vec<String>, - pub playable_in_embed: bool, - pub automatic_captions: HashMap<String, Vec<YCaption>>, - pub comment_count: Option<usize>, - pub chapters: Option<Vec<YChapter>>, - pub heatmap: Option<Vec<YHeatmapSample>>, - pub like_count: Option<usize>, - pub channel: Option<String>, - pub channel_follower_count: usize, - pub channel_is_verified: Option<bool>, - pub uploader: String, - pub uploader_id: String, - pub uploader_url: String, - pub upload_date: String, - pub availability: String, // "public" | "private" | "unlisted", - pub original_url: Option<String>, - pub webpage_url_basename: String, - pub webpage_url_domain: String, - pub extractor: String, - pub extractor_key: String, - pub playlist_count: Option<usize>, - pub playlist: Option<String>, - pub playlist_id: Option<String>, - pub playlist_title: Option<String>, - pub playlist_uploader: Option<String>, - pub playlist_uploader_id: Option<String>, - pub n_entries: Option<usize>, - pub playlist_index: Option<usize>, - pub display_id: String, - pub fulltitle: String, - pub duration_string: String, - pub is_live: bool, - pub was_live: bool, - pub epoch: usize, -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct YCaption { - pub url: Option<String>, - pub ext: String, //"vtt" | "json3" | "srv1" | "srv2" | "srv3" | "ttml", - pub protocol: Option<String>, - pub name: Option<String>, -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct YFormat { - pub format_id: String, - pub format_note: Option<String>, - pub ext: String, - pub protocol: String, - pub acodec: Option<String>, - pub vcodec: Option<String>, - pub url: Option<String>, - pub width: Option<u32>, - pub height: Option<u32>, - pub fps: Option<f64>, - pub columns: Option<u32>, - pub fragments: Option<Vec<YFragment>>, - pub resolution: String, - pub dynamic_range: Option<String>, - pub aspect_ratio: Option<f64>, - pub http_headers: HashMap<String, String>, - pub audio_ext: String, - pub video_ext: String, - pub vbr: Option<f64>, - pub abr: Option<f64>, - pub format: String, -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct YFragment { - pub url: Option<String>, - pub duration: Option<f64>, -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct YThumbnail { - pub url: String, - pub preference: i32, - pub id: String, - pub height: Option<u32>, - pub width: Option<u32>, - pub resolution: Option<String>, -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct YChapter { - pub start_time: f64, - pub end_time: f64, - pub title: String, -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct YHeatmapSample { - pub start_time: f64, - pub end_time: f64, - pub value: f64, -} - -pub fn parse_upload_date(d: &str) -> anyhow::Result<DateTime<Utc>> { - let (year, month, day) = (&d[0..4], &d[4..6], &d[6..8]); - let (year, month, day) = ( - year.parse().context("parsing year")?, - month.parse().context("parsing month")?, - day.parse().context("parsing day")?, - ); - - let mut p = Parsed::new(); - p.year = Some(year); - p.month = Some(month); - p.day = Some(day); - p.hour_div_12 = Some(0); - p.hour_mod_12 = Some(0); - p.minute = Some(0); - p.second = Some(0); - Ok(p.to_datetime_with_timezone(&Utc)?) -} diff --git a/tool/src/import/mod.rs b/tool/src/import/mod.rs deleted file mode 100644 index 973629c..0000000 --- a/tool/src/import/mod.rs +++ /dev/null @@ -1,320 +0,0 @@ -/* - This file is part of jellything (https://codeberg.org/metamuffin/jellything) - which is licensed under the GNU Affero General Public License (version 3); see /COPYING. - Copyright (C) 2023 metamuffin <metamuffin.org> -*/ -pub mod infojson; -pub mod tmdb; - -use crate::{make_ident, ok_or_warn, Action}; -use anyhow::Context; -use infojson::YVideo; -use jellycommon::{ - AssetLocation, LocalTrack, MediaInfo, MediaSource, Node, NodeKind, NodePrivate, NodePublic, - Rating, -}; -use jellymatroska::read::EbmlReader; -use jellyremuxer::import::import_metadata; -use log::{debug, info, warn}; -use std::{ - collections::BTreeMap, - fs::{remove_file, File}, - io::{stdin, BufReader, Write}, -}; -use tmdb::{tmdb_details, tmdb_image}; - -pub(crate) fn import(action: Action, dry: bool) -> anyhow::Result<()> { - match action { - Action::New { - path, - tmdb_id, - tmdb_search, - input, - series, - ident_prefix, - ignore_attachments, - copy, - video, - ignore_metadata, - r#move, - title, - skip_existing, - } => { - if std::env::current_dir().unwrap().file_name().unwrap() != "library" { - warn!("new command can only be used in the library directory; what you are doing right now probably wont work.") - } - - if skip_existing { - if let Some(input) = &input { - let guessed_path = path.join(input.file_stem().unwrap_or(input.as_os_str())); - if guessed_path.exists() { - info!("guessed output ({guessed_path:?}) exists, skipping import"); - return Ok(()); - } else { - debug!("guessed output ({guessed_path:?}) missing"); - } - } - } - - let tmdb_kind = if series { "tv" } else { "movie" }; - let tmdb_id = if let Some(id) = tmdb_id { - Some(id.parse().unwrap()) - } else if let Some(title) = tmdb_search { - let tmdb_key = std::env::var("TMDB_API_KEY").context("tmdb api key required")?; - let results = tmdb::tmdb_search(tmdb_kind, &title, &tmdb_key)?; - info!("results:"); - for (i, r) in results.results.iter().enumerate() { - info!( - "\t[{i}] {}: {} ({})", - r.id, - r.title.as_ref().or(r.name.as_ref()).unwrap(), - r.overview.chars().take(100).collect::<String>() - ); - } - let res_index = if results.results.len() > 1 { - stdin() - .lines() - .next() - .unwrap() - .unwrap() - .parse::<usize>() - .unwrap() - } else { - 0 - }; - Some(results.results[res_index].id) - } else { - None - }; - - let tmdb_details = tmdb_id - .map(|id| { - let tmdb_key = - std::env::var("TMDB_API_KEY").context("tmdb api key required")?; - let td = tmdb_details(tmdb_kind, id, &tmdb_key) - .context("fetching details") - .unwrap(); - Ok::<_, anyhow::Error>(td) - }) - .transpose()?; - - let mut kind = NodeKind::Series; - let mut file_meta = None; - let mut infojson = None; - - if let Some(input_path) = &input { - file_meta = Some({ - let input = BufReader::new(File::open(&input_path).unwrap()); - let mut input = EbmlReader::new(input); - import_metadata(&mut input)? - }); - if ignore_attachments { - let file_meta = file_meta.as_mut().unwrap(); - file_meta.cover = None; - file_meta.infojson = None; - } - if ignore_metadata { - let file_meta = file_meta.as_mut().unwrap(); - file_meta.description = None; - file_meta.tagline = None; - file_meta.title = None; - } - - if let Some(ij) = &file_meta.as_ref().unwrap().infojson { - infojson = - Some(serde_json::from_str::<YVideo>(ij).context("parsing info.json")?); - } - - kind = if video { - NodeKind::Video - } else { - NodeKind::Movie - }; - } - - let title = title - .or(tmdb_details - .as_ref() - .map(|d| d.title.clone().or(d.name.clone())) - .flatten()) - .or(file_meta.as_ref().map(|m| m.title.clone()).flatten()) - .expect("no title detected"); - - let ident = format!( - "{}{}", - ident_prefix.unwrap_or(String::new()), - make_ident( - &infojson - .as_ref() - .map(|i| i.id.clone()) - .unwrap_or(title.clone()) - ), - ); - let path = path.join(&ident); - let source_path = input.as_ref().map(|_| path.join(format!("source.mkv"))); - - let (mut poster, mut backdrop) = (None, None); - if !dry { - std::fs::create_dir_all(&path)?; - - poster = file_meta - .as_ref() - .map(|m| { - m.cover - .as_ref() - .map(|(mime, data)| { - let pu = path.join(format!( - "cover.{}", - match mime.as_str() { - "image/webp" => "webp", - "image/jpeg" => "jpeg", - "image/png" => "png", - _ => { - warn!("unknown mime, just using webp"); - "webp" - } - } - )); - if !pu.exists() { - let mut f = File::create(&pu)?; - f.write_all(&data)?; - } - Ok::<_, anyhow::Error>(pu) - }) - .transpose() - }) - .transpose()? - .flatten() - .or(tmdb_details - .as_ref() - .map(|d| { - d.poster_path - .as_ref() - .map(|p| { - let pu = path.join("poster.jpeg"); - let mut f = File::create(&pu)?; - tmdb_image(&p, &mut f)?; - Ok::<_, anyhow::Error>(pu) - }) - .transpose() - }) - .transpose()? - .flatten()); - - backdrop = tmdb_details - .as_ref() - .map(|d| { - d.backdrop_path - .as_ref() - .map(|p| { - let pu = path.join("backdrop.jpeg"); - let mut f = File::create(&pu)?; - tmdb_image(&p, &mut f)?; - Ok::<_, anyhow::Error>(pu) - }) - .transpose() - }) - .transpose()? - .flatten(); - } - - let mut ratings = BTreeMap::new(); - - ratings.extend( - infojson - .as_ref() - .map(|i| (Rating::YoutubeViews, i.view_count as f64)), - ); - ratings.extend( - infojson - .as_ref() - .map(|i| i.like_count.map(|l| (Rating::YoutubeLikes, l as f64))) - .flatten(), - ); - ratings.extend( - tmdb_details - .as_ref() - .map(|d| (Rating::Tmdb, d.vote_average)), - ); - - let node = Node { - private: NodePrivate { - id: Some(ident.clone()), - import: None, - backdrop: backdrop.clone().map(AssetLocation::Library), - poster: poster.clone().map(AssetLocation::Library), - source: file_meta.as_ref().map(|m| MediaSource::Local { - tracks: m - .track_sources - .clone() - .into_iter() - .map(|t| LocalTrack { - path: source_path.clone().unwrap(), - ..t - }) - .collect(), - }), - }, - public: NodePublic { - federated: None, - ratings, - description: file_meta - .as_ref() - .map(|m| m.description.clone()) - .flatten() - .or(tmdb_details.as_ref().map(|d| d.overview.to_owned())), - tagline: file_meta.as_ref().map(|m| m.tagline.clone()).flatten().or( - tmdb_details - .as_ref() - .map(|d| d.tagline.to_owned()) - .flatten(), - ), - title, - index: None, - kind, - children: Vec::new(), - media: file_meta.as_ref().map(|m| MediaInfo { - chapters: m.chapters.clone(), - duration: m.duration, - tracks: m.tracks.clone(), - }), - release_date: tmdb_details - .as_ref() - .map(|d| tmdb::parse_release_date(&d.release_date.clone()?).ok()) - .flatten() - .or(infojson - .as_ref() - .and_then(|j| ok_or_warn(infojson::parse_upload_date(&j.upload_date)))), - ..Default::default() - }, - }; - - if dry { - println!("{}", serde_json::to_string_pretty(&node)?); - } else { - if let Some(source_path) = source_path { - let input = input.clone().unwrap(); - if r#move { - std::fs::rename(&input, &source_path)?; - } else if copy { - std::fs::copy(&input, &source_path)?; - } else { - if source_path.is_symlink() { - remove_file(&source_path)?; - } - std::os::unix::fs::symlink(&input, &source_path)?; - } - } - let f = File::create(path.join(if series { - "directory.json" - } else { - "item.jelly" - }))?; - serde_json::to_writer_pretty(f, &node)?; - } - - Ok(()) - } - _ => unreachable!(), - } -} diff --git a/tool/src/import/tmdb.rs b/tool/src/import/tmdb.rs deleted file mode 100644 index c38d50e..0000000 --- a/tool/src/import/tmdb.rs +++ /dev/null @@ -1,116 +0,0 @@ -/* - This file is part of jellything (https://codeberg.org/metamuffin/jellything) - which is licensed under the GNU Affero General Public License (version 3); see /COPYING. - Copyright (C) 2023 metamuffin <metamuffin.org> -*/ -use anyhow::Context; -use jellycommon::chrono::{format::Parsed, DateTime, Utc}; -use log::info; -use serde::Deserialize; -use std::io::Write; - -#[derive(Debug, Clone, Deserialize)] -pub struct TmdbQuery { - pub page: usize, - pub results: Vec<TmdbQueryResult>, - pub total_pages: usize, - pub total_results: usize, -} - -#[derive(Debug, Clone, Deserialize)] -pub struct TmdbQueryResult { - pub adult: bool, - pub backdrop_path: Option<String>, - pub genre_ids: Vec<u64>, - pub id: u64, - pub original_language: Option<String>, - pub original_title: Option<String>, - pub overview: String, - pub popularity: f64, - pub poster_path: Option<String>, - pub release_date: Option<String>, - pub title: Option<String>, - pub name: Option<String>, - pub vote_average: f64, - pub vote_count: usize, -} - -#[derive(Debug, Clone, Deserialize)] -pub struct TmdbDetails { - pub adult: bool, - pub backdrop_path: Option<String>, - pub genres: Vec<TmdbGenre>, - pub id: u64, - pub original_language: Option<String>, - pub original_title: Option<String>, - pub overview: String, - pub popularity: f64, - pub poster_path: Option<String>, - pub release_date: Option<String>, - pub title: Option<String>, - pub name: Option<String>, - pub vote_average: f64, - pub vote_count: usize, - pub budget: Option<usize>, - pub homepage: Option<String>, - pub imdb_id: Option<String>, - pub production_companies: Vec<TmdbProductionCompany>, - pub revenue: Option<usize>, - pub tagline: Option<String>, -} - -#[derive(Debug, Clone, Deserialize)] -pub struct TmdbGenre { - pub id: u64, - pub name: String, -} - -#[derive(Debug, Clone, Deserialize)] -pub struct TmdbProductionCompany { - pub id: u64, - pub name: String, - pub logo_path: Option<String>, -} - -pub fn tmdb_search(kind: &str, query: &str, key: &str) -> anyhow::Result<TmdbQuery> { - info!("searching tmdb: {query:?}"); - Ok(reqwest::blocking::get(&format!( - "https://api.themoviedb.org/3/search/{kind}?query={}&api_key={key}", - query.replace(" ", "+") - ))? - .json::<TmdbQuery>()?) -} - -pub fn tmdb_details(kind: &str, id: u64, key: &str) -> anyhow::Result<TmdbDetails> { - info!("fetching details: {id:?}"); - Ok(reqwest::blocking::get(&format!( - "https://api.themoviedb.org/3/{kind}/{id}?api_key={key}" - ))? - .json()?) -} - -pub fn tmdb_image(path: &str, out: &mut impl Write) -> anyhow::Result<()> { - info!("downloading image {path:?}"); - let mut res = reqwest::blocking::get(&format!("https://image.tmdb.org/t/p/original{path}"))?; - res.copy_to(out)?; - Ok(()) -} - -pub fn parse_release_date(d: &str) -> anyhow::Result<DateTime<Utc>> { - let (year, month, day) = (&d[0..4], &d[5..7], &d[8..10]); - let (year, month, day) = ( - year.parse().context("parsing year")?, - month.parse().context("parsing month")?, - day.parse().context("parsing day")?, - ); - - let mut p = Parsed::new(); - p.year = Some(year); - p.month = Some(month); - p.day = Some(day); - p.hour_div_12 = Some(0); - p.hour_mod_12 = Some(0); - p.minute = Some(0); - p.second = Some(0); - Ok(p.to_datetime_with_timezone(&Utc)?) -} diff --git a/tool/src/main.rs b/tool/src/main.rs index 31e63b7..34337ce 100644 --- a/tool/src/main.rs +++ b/tool/src/main.rs @@ -5,12 +5,10 @@ */ #![feature(file_create_new)] -pub mod import; pub mod migrate; use base64::Engine; use clap::{Parser, Subcommand, ValueEnum}; -use import::import; use jellyclient::{Instance, LoginDetails}; use jellycommon::{config::GlobalConfig, Node, NodeKind, NodePrivate, NodePublic}; use log::{info, warn}; @@ -37,47 +35,47 @@ enum Action { #[arg(short, long)] hostname: String, }, - /// Imports a movie, video or series given media and metadata sources - New { - /// Relative path to the node's parent(!). - path: PathBuf, - /// Search the node by title on TMDB - #[arg(short = 't', long)] - tmdb_search: Option<String>, - /// Search the node by id on TMDB - #[arg(short = 'T', long)] - tmdb_id: Option<String>, - #[arg(long)] - /// Prefix the inferred id with something to avoid collisions - ident_prefix: Option<String>, - /// Copies media into the library - #[arg(long)] - copy: bool, - /// Moves media into the library (potentially destructive operation) - #[arg(long)] - r#move: bool, - /// Marks node as a video - #[arg(long)] - video: bool, - /// Marks node as a series - #[arg(short, long)] - series: bool, - /// Path to the media of the node, required for non-series - #[arg(short, long)] - input: Option<PathBuf>, - /// Ignore attachments (dont use them as cover) - #[arg(long)] - ignore_attachments: bool, - /// Ignore metadate (no title, description and tagline from input) - #[arg(long)] - ignore_metadata: bool, - /// Skip any action that appears to be run already. - #[arg(long)] - skip_existing: bool, - /// Sets the title - #[arg(long)] - title: Option<String>, - }, + // /// Imports a movie, video or series given media and metadata sources + // New { + // /// Relative path to the node's parent(!). + // path: PathBuf, + // /// Search the node by title on TMDB + // #[arg(short = 't', long)] + // tmdb_search: Option<String>, + // /// Search the node by id on TMDB + // #[arg(short = 'T', long)] + // tmdb_id: Option<String>, + // #[arg(long)] + // /// Prefix the inferred id with something to avoid collisions + // ident_prefix: Option<String>, + // /// Copies media into the library + // #[arg(long)] + // copy: bool, + // /// Moves media into the library (potentially destructive operation) + // #[arg(long)] + // r#move: bool, + // /// Marks node as a video + // #[arg(long)] + // video: bool, + // /// Marks node as a series + // #[arg(short, long)] + // series: bool, + // /// Path to the media of the node, required for non-series + // #[arg(short, long)] + // input: Option<PathBuf>, + // /// Ignore attachments (dont use them as cover) + // #[arg(long)] + // ignore_attachments: bool, + // /// Ignore metadate (no title, description and tagline from input) + // #[arg(long)] + // ignore_metadata: bool, + // /// Skip any action that appears to be run already. + // #[arg(long)] + // skip_existing: bool, + // /// Sets the title + // #[arg(long)] + // title: Option<String>, + // }, Migrate { database: PathBuf, mode: MigrateMode, @@ -118,8 +116,11 @@ fn main() -> anyhow::Result<()> { std::fs::create_dir_all(path.join("library"))?; std::fs::create_dir_all(path.join("cache"))?; std::fs::create_dir_all(path.join("assets"))?; + std::fs::create_dir_all(path.join("media"))?; File::create_new(path.join("assets/front.htm"))? .write_fmt(format_args!("<h1>My very own jellything instance</h1>"))?; + + // TODO: dont fill that serde_yaml::to_writer( File::create_new(path.join("config.yaml"))?, &GlobalConfig { @@ -149,8 +150,8 @@ fn main() -> anyhow::Result<()> { File::create_new(path.join("library/directory.json"))?, &Node { public: NodePublic { - kind: NodeKind::Collection, - title: "My Library".to_string(), + kind: Some(NodeKind::Collection), + title: Some("My Library".to_string()), ..Default::default() }, private: NodePrivate { @@ -162,7 +163,7 @@ fn main() -> anyhow::Result<()> { warn!("please change the admin password."); Ok(()) } - a @ Action::New { .. } => import(a, args.dry), + // a @ Action::New { .. } => import(a, args.dry), a @ Action::Migrate { .. } => migrate(a), Action::Reimport { config, |