From 3a29113e965a94bdef06655f1583cc6e86edd606 Mon Sep 17 00:00:00 2001 From: metamuffin Date: Thu, 21 Dec 2023 23:57:42 +0100 Subject: rework import system pt. 1 --- Cargo.lock | 21 ++- Cargo.toml | 3 +- base/Cargo.toml | 1 + base/src/database.rs | 3 + base/src/federation.rs | 61 +++++++ base/src/lib.rs | 2 + common/src/config.rs | 4 + common/src/lib.rs | 45 +++-- import/Cargo.toml | 24 +++ import/src/infojson.rs | 143 +++++++++++++++ import/src/lib.rs | 376 ++++++++++++++++++++++++++++++++++++++ import/src/mod.rs | 319 ++++++++++++++++++++++++++++++++ import/src/tmdb.rs | 116 ++++++++++++ remuxer/src/import/mod.rs | 1 + remuxer/src/remux.rs | 2 +- remuxer/src/snippet.rs | 5 +- server/Cargo.toml | 2 +- server/src/federation.rs | 61 ------- server/src/import.rs | 254 ------------------------- server/src/main.rs | 5 +- server/src/routes/mod.rs | 4 +- server/src/routes/stream.rs | 65 +++---- server/src/routes/ui/admin/mod.rs | 5 +- server/src/routes/ui/node.rs | 22 ++- server/src/routes/ui/player.rs | 2 +- server/src/routes/ui/sort.rs | 16 +- stream/src/lib.rs | 5 +- stream/src/webvtt.rs | 8 +- tool/src/import/infojson.rs | 143 --------------- tool/src/import/mod.rs | 320 -------------------------------- tool/src/import/tmdb.rs | 116 ------------ tool/src/main.rs | 93 +++++----- 32 files changed, 1222 insertions(+), 1025 deletions(-) create mode 100644 base/src/federation.rs create mode 100644 import/Cargo.toml create mode 100644 import/src/infojson.rs create mode 100644 import/src/lib.rs create mode 100644 import/src/mod.rs create mode 100644 import/src/tmdb.rs delete mode 100644 server/src/federation.rs delete mode 100644 server/src/import.rs delete mode 100644 tool/src/import/infojson.rs delete mode 100644 tool/src/import/mod.rs delete mode 100644 tool/src/import/tmdb.rs diff --git a/Cargo.lock b/Cargo.lock index d6415b4..9a12441 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1347,6 +1347,7 @@ dependencies = [ "anyhow", "base64", "bincode 2.0.0-rc.3", + "jellyclient", "jellycommon", "log", "serde", @@ -1380,6 +1381,24 @@ dependencies = [ "serde", ] +[[package]] +name = "jellyimport" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-recursion", + "futures", + "jellybase", + "jellyclient", + "jellycommon", + "jellyremuxer", + "log", + "reqwest", + "serde_json", + "serde_yaml", + "tokio", +] + [[package]] name = "jellymatroska" version = "0.1.0" @@ -1434,8 +1453,8 @@ dependencies = [ "futures", "glob", "jellybase", - "jellyclient", "jellycommon", + "jellyimport", "jellystream", "jellytranscoder", "log", diff --git a/Cargo.toml b/Cargo.toml index a9969e7..1b0d230 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,8 @@ members = [ "ebml_derive", "client", "transcoder", - "base", + "base", + "import", ] resolver = "2" diff --git a/base/Cargo.toml b/base/Cargo.toml index ac6989c..8109998 100644 --- a/base/Cargo.toml +++ b/base/Cargo.toml @@ -5,6 +5,7 @@ edition = "2021" [dependencies] jellycommon = { path = "../common" } +jellyclient = { path = "../client" } serde = { version = "1.0.193", features = ["derive"] } serde_yaml = "0.9.27" log = { workspace = true } diff --git a/base/src/database.rs b/base/src/database.rs index f46f0fb..3f81cca 100644 --- a/base/src/database.rs +++ b/base/src/database.rs @@ -22,6 +22,8 @@ pub struct Database { pub user_node: Tree<(String, String), NodeUserData>, pub invite: Tree, pub node: Tree, + + pub node_import: Tree, Node)>>, } impl Database { @@ -34,6 +36,7 @@ impl Database { invite: Tree::open(&db, "invite"), node: Tree::open(&db, "node"), user_node: Tree::open(&db, "user_node"), + node_import: Tree::open(&db, "node_import"), db, }); info!("ready"); diff --git a/base/src/federation.rs b/base/src/federation.rs new file mode 100644 index 0000000..509b87c --- /dev/null +++ b/base/src/federation.rs @@ -0,0 +1,61 @@ +/* + This file is part of jellything (https://codeberg.org/metamuffin/jellything) + which is licensed under the GNU Affero General Public License (version 3); see /COPYING. + Copyright (C) 2023 metamuffin +*/ +use anyhow::anyhow; +use crate::CONF; +use jellyclient::{Instance, LoginDetails, Session}; +use std::{collections::HashMap, sync::Arc}; +use tokio::sync::RwLock; + +pub struct Federation { + instances: HashMap, + sessions: RwLock>>, +} + +impl Federation { + pub fn initialize() -> Self { + let instances = CONF + .remote_credentials + .iter() + .map(|(k, (_, _, tls))| (k.to_owned(), Instance::new(k.to_owned(), *tls))) + .collect::>(); + + Self { + instances, + sessions: Default::default(), + } + } + + pub fn get_instance(&self, host: &String) -> anyhow::Result<&Instance> { + Ok(self + .instances + .get(host) + .ok_or(anyhow!("unknown instance"))?) + } + pub async fn get_session(&self, host: &String) -> anyhow::Result> { + let mut w = self.sessions.write().await; + if let Some(s) = w.get(host) { + Ok(s.to_owned()) + } else { + let (username, password, _) = CONF + .remote_credentials + .get(host) + .ok_or(anyhow!("no credentials of the remote server"))?; + let s = Arc::new( + self.get_instance(host)? + .to_owned() + .login(LoginDetails { + username: username.to_owned(), + password: password.to_owned(), + expire: None, + drop_permissions: None, + }) + .await?, + ); + w.insert(host.to_owned(), s.clone()); + Ok(s) + } + } +} diff --git a/base/src/lib.rs b/base/src/lib.rs index cfc5a11..a7b15c5 100644 --- a/base/src/lib.rs +++ b/base/src/lib.rs @@ -6,6 +6,7 @@ #![feature(lazy_cell)] pub mod cache; pub mod database; +pub mod federation; pub mod permission; pub mod temp; @@ -34,6 +35,7 @@ impl AssetLocationExt for AssetLocation { AssetLocation::Cache(p) => CONF.cache_path.join(p), AssetLocation::Library(p) => CONF.library_path.join(p), AssetLocation::Temp(p) => CONF.temp_path.join(p), + AssetLocation::Media(p) => CONF.media_path.join(p), } } } diff --git a/common/src/config.rs b/common/src/config.rs index 3ccf0e8..200249c 100644 --- a/common/src/config.rs +++ b/common/src/config.rs @@ -19,6 +19,7 @@ pub struct GlobalConfig { #[serde(default = "default::library_path")] pub library_path: PathBuf, #[serde(default = "default::temp_path")] pub temp_path: PathBuf, #[serde(default = "default::cache_path")] pub cache_path: PathBuf, + #[serde(default = "default::media_path")] pub media_path: PathBuf, #[serde(default = "default::admin_username")] pub admin_username: String, #[serde(default = "default::transcoding_profiles")] pub transcoding_profiles: Vec, #[serde(default = "default::max_in_memory_cache_size")] pub max_in_memory_cache_size: usize, @@ -53,6 +54,9 @@ mod default { pub fn cache_path() -> PathBuf { "data/cache".into() } + pub fn media_path() -> PathBuf { + "data/media".into() + } pub fn temp_path() -> PathBuf { "/tmp".into() } diff --git a/common/src/lib.rs b/common/src/lib.rs index aedcb07..e953d85 100644 --- a/common/src/lib.rs +++ b/common/src/lib.rs @@ -19,7 +19,7 @@ use rocket::{FromFormField, UriDisplayQuery}; use serde::{Deserialize, Serialize}; use std::{collections::BTreeMap, path::PathBuf}; -#[derive(Debug, Clone, Deserialize, Serialize)] +#[derive(Debug, Clone, Deserialize, Serialize, Default)] pub struct Node { pub public: NodePublic, pub private: NodePrivate, @@ -29,17 +29,16 @@ pub struct Node { #[derive(Debug, Clone, Deserialize, Serialize, Default)] pub struct NodePrivate { #[serde(default)] pub id: Option, - #[serde(default)] pub import: Option, #[serde(default)] pub poster: Option, #[serde(default)] pub backdrop: Option, - #[serde(default)] pub source: Option, + #[serde(default)] pub source: Option>, } #[rustfmt::skip] #[derive(Debug, Clone, Deserialize, Serialize, Default)] pub struct NodePublic { - pub kind: NodeKind, - pub title: String, + #[serde(default)] pub kind: Option, + #[serde(default)] pub title: Option, #[serde(default)] pub id: Option, #[serde(default)] pub path: Vec, #[serde(default)] pub children: Vec, @@ -52,13 +51,25 @@ pub struct NodePublic { #[serde(default)] pub federated: Option, } -#[rustfmt::skip] -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct RemoteImportOptions { - pub host: String, +#[derive(Debug, Clone, Deserialize, Serialize, Default)] +pub struct ImportOptions { pub id: String, - #[serde(default)] pub flatten: bool, - #[serde(default)] pub prefix: Option, + pub sources: Vec, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ImportSource { + Override(Node), + Tmdb(u64), + AutoChildren, + Media { + location: AssetLocation, + // TODO ignore options + }, + Federated { + host: String, + }, } #[derive(Debug, Clone, Deserialize, Serialize, Hash, PartialEq, Eq)] @@ -68,6 +79,7 @@ pub enum AssetLocation { Library(PathBuf), Assets(PathBuf), Temp(PathBuf), + Media(PathBuf), } #[rustfmt::skip] @@ -87,9 +99,9 @@ pub enum NodeKind { #[derive(Debug, Clone, Deserialize, Serialize)] #[serde(rename_all = "snake_case")] -pub enum MediaSource { - Local { tracks: Vec }, - Remote { host: String, remote_id: String }, +pub enum TrackSource { + Local(LocalTrack), + Remote, } pub enum PublicMediaSource { @@ -128,18 +140,19 @@ pub struct SourceTrack { pub codec: String, pub language: String, pub default_duration: Option, + #[serde(default)] pub federated: Vec, } #[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq, PartialOrd, Ord)] #[serde(rename_all = "snake_case")] pub enum Rating { + Imdb, + Tmdb, RottenTomatoes, Metacritic, - Imdb, YoutubeViews, YoutubeLikes, YoutubeFollowers, - Tmdb, } #[derive(Debug, Clone, Deserialize, Serialize)] diff --git a/import/Cargo.toml b/import/Cargo.toml new file mode 100644 index 0000000..f2ba7af --- /dev/null +++ b/import/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "jellyimport" +version = "0.1.0" +edition = "2021" + +[dependencies] +jellycommon = { path = "../common" } +jellybase = { path = "../base" } +jellyclient = { path = "../client" } +# jellymatroska = { path = "../matroska" } +jellyremuxer = { path = "../remuxer" } + +log = { workspace = true } +anyhow = "1.0.75" +reqwest = { version = "0.11.22", features = ["blocking", "json"] } + +# serde = { version = "1.0.193", features = ["derive"] } +serde_json = "1.0.108" +serde_yaml = "0.9.27" +# bincode = { version = "2.0.0-rc.3", features = ["serde"] } + +async-recursion = "1.0.5" +futures = "0.3.29" +tokio = { workspace = true } diff --git a/import/src/infojson.rs b/import/src/infojson.rs new file mode 100644 index 0000000..3f0edc9 --- /dev/null +++ b/import/src/infojson.rs @@ -0,0 +1,143 @@ +/* + This file is part of jellything (https://codeberg.org/metamuffin/jellything) + which is licensed under the GNU Affero General Public License (version 3); see /COPYING. + Copyright (C) 2023 metamuffin +*/ + +use anyhow::Context; +use jellycommon::chrono::{format::Parsed, DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; + +#[derive(Debug, Serialize, Deserialize)] +pub struct YVideo { + pub id: String, + pub title: String, + pub formats: Vec, + pub thumbnails: Vec, + pub thumbnail: String, + pub description: String, + pub channel_id: String, + pub duration: Option, + pub view_count: usize, + pub average_rating: Option, + pub age_limit: usize, + pub webpage_url: String, + pub categories: Vec, + pub tags: Vec, + pub playable_in_embed: bool, + pub automatic_captions: HashMap>, + pub comment_count: Option, + pub chapters: Option>, + pub heatmap: Option>, + pub like_count: Option, + pub channel: Option, + pub channel_follower_count: usize, + pub channel_is_verified: Option, + pub uploader: String, + pub uploader_id: String, + pub uploader_url: String, + pub upload_date: String, + pub availability: String, // "public" | "private" | "unlisted", + pub original_url: Option, + pub webpage_url_basename: String, + pub webpage_url_domain: String, + pub extractor: String, + pub extractor_key: String, + pub playlist_count: Option, + pub playlist: Option, + pub playlist_id: Option, + pub playlist_title: Option, + pub playlist_uploader: Option, + pub playlist_uploader_id: Option, + pub n_entries: Option, + pub playlist_index: Option, + pub display_id: String, + pub fulltitle: String, + pub duration_string: String, + pub is_live: bool, + pub was_live: bool, + pub epoch: usize, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct YCaption { + pub url: Option, + pub ext: String, //"vtt" | "json3" | "srv1" | "srv2" | "srv3" | "ttml", + pub protocol: Option, + pub name: Option, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct YFormat { + pub format_id: String, + pub format_note: Option, + pub ext: String, + pub protocol: String, + pub acodec: Option, + pub vcodec: Option, + pub url: Option, + pub width: Option, + pub height: Option, + pub fps: Option, + pub columns: Option, + pub fragments: Option>, + pub resolution: String, + pub dynamic_range: Option, + pub aspect_ratio: Option, + pub http_headers: HashMap, + pub audio_ext: String, + pub video_ext: String, + pub vbr: Option, + pub abr: Option, + pub format: String, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct YFragment { + pub url: Option, + pub duration: Option, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct YThumbnail { + pub url: String, + pub preference: i32, + pub id: String, + pub height: Option, + pub width: Option, + pub resolution: Option, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct YChapter { + pub start_time: f64, + pub end_time: f64, + pub title: String, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct YHeatmapSample { + pub start_time: f64, + pub end_time: f64, + pub value: f64, +} + +pub fn parse_upload_date(d: &str) -> anyhow::Result> { + let (year, month, day) = (&d[0..4], &d[4..6], &d[6..8]); + let (year, month, day) = ( + year.parse().context("parsing year")?, + month.parse().context("parsing month")?, + day.parse().context("parsing day")?, + ); + + let mut p = Parsed::new(); + p.year = Some(year); + p.month = Some(month); + p.day = Some(day); + p.hour_div_12 = Some(0); + p.hour_mod_12 = Some(0); + p.minute = Some(0); + p.second = Some(0); + Ok(p.to_datetime_with_timezone(&Utc)?) +} diff --git a/import/src/lib.rs b/import/src/lib.rs new file mode 100644 index 0000000..3698f79 --- /dev/null +++ b/import/src/lib.rs @@ -0,0 +1,376 @@ +/* + This file is part of jellything (https://codeberg.org/metamuffin/jellything) + which is licensed under the GNU Affero General Public License (version 3); see /COPYING. + Copyright (C) 2023 metamuffin +*/ +#![feature(lazy_cell)] +use anyhow::{Context, Ok}; +use async_recursion::async_recursion; +use futures::{stream::FuturesUnordered, StreamExt}; +use jellybase::{ + cache::async_cache_file, database::Database, federation::Federation, AssetLocationExt, CONF, +}; +use jellyclient::Session; +use jellycommon::{ + AssetLocation, AssetRole, ImportOptions, ImportSource, Node, NodePrivate, NodePublic, +}; +use log::{debug, info}; +use std::{ + cmp::Ordering, + ffi::OsStr, + fs::File, + os::unix::prelude::OsStrExt, + path::{Path, PathBuf}, + sync::{Arc, LazyLock}, +}; +use tokio::sync::Semaphore; + +static IMPORT_SEM: LazyLock = LazyLock::new(|| Semaphore::new(1)); + +pub async fn import(db: &Database, fed: &Federation) -> anyhow::Result<()> { + let permit = IMPORT_SEM.try_acquire()?; + info!("loading sources..."); + import_path(CONF.library_path.clone(), vec![], db, fed) + .await + .context("indexing")?; + info!("merging nodes..."); + merge_nodes(db).context("merging nodes")?; + info!("clearing temporary node tree"); + db.node_import.clear()?; + info!("import completed"); + drop(permit); + Ok(()) +} + +pub fn merge_nodes(db: &Database) -> anyhow::Result<()> { + for r in db.node_import.iter() { + let (id, mut nodes) = r?; + + nodes.sort_by(|(x, _), (y, _)| compare_index_path(x, y)); + + let node = nodes + .into_iter() + .map(|(_, x)| x) + .reduce(|x, y| merge_node(x, y)) + .unwrap(); + + db.node.insert(&id, &node)?; + } + Ok(()) +} + +fn compare_index_path(x: &[usize], y: &[usize]) -> Ordering { + if x.is_empty() { + Ordering::Greater + } else if y.is_empty() { + Ordering::Less + } else { + match x[0].cmp(&y[0]) { + o @ (Ordering::Less | Ordering::Greater) => o, + Ordering::Equal => compare_index_path(&x[1..], &y[1..]), + } + } +} + +#[async_recursion] +pub async fn import_path( + path: PathBuf, + index_path: Vec, + db: &Database, + fed: &Federation, +) -> anyhow::Result<()> { + if path.is_dir() { + let mut children_paths = path + .read_dir()? + .map(Result::unwrap) + .filter_map(|e| { + if e.path().extension() == Some(&OsStr::from_bytes(b"yaml")) + || e.metadata().unwrap().is_dir() + { + Some(e.path()) + } else { + None + } + }) + .collect::>(); + + children_paths.sort(); + + let mut children: FuturesUnordered<_> = children_paths + .into_iter() + .enumerate() + .map(|(i, p)| { + import_path( + p.clone(), + { + let mut path = index_path.clone(); + path.push(i); + path + }, + db, + fed, + ) + }) + .collect(); + + while let Some(k) = children.next().await { + k? + } + } else { + let opts: ImportOptions = serde_yaml::from_reader(File::open(&path)?)?; + + for s in opts.sources { + process_source(opts.id.clone(), s, &path, &index_path, db, fed).await?; + } + } + Ok(()) +} + +async fn process_source( + id: String, + s: ImportSource, + path: &Path, + index_path: &[usize], + db: &Database, + fed: &Federation, +) -> anyhow::Result<()> { + let insert_node = move |id: &String, n: Node| -> anyhow::Result<()> { + db.node_import.fetch_and_update(id, |l| { + let mut l = l.unwrap_or_default(); + l.push((index_path.to_vec(), n.clone())); + Some(l) + })?; + Ok(()) + }; + match s { + ImportSource::Override(n) => insert_node(&id, n)?, + ImportSource::Tmdb(_) => todo!(), + ImportSource::Media { location } => { + let path = location.path(); + } + ImportSource::Federated { host } => { + let session = fed.get_session(&host).await.context("creating session")?; + + import_remote(id, &host, db, &session, index_path) + .await + .context("federated import")? + } + ImportSource::AutoChildren => { + // TODO dont forget to update path of children + } + } + Ok(()) +} + +fn merge_node(x: Node, y: Node) -> Node { + Node { + public: NodePublic { + kind: x.public.kind.or(y.public.kind), + title: x.public.title.or(y.public.title), + id: x.public.id.or(y.public.id), + path: vec![], + children: if x.public.children.is_empty() { + x.public.children + } else { + y.public.children + }, + tagline: x.public.tagline.or(y.public.tagline), + description: x.public.description.or(y.public.description), + release_date: x.public.release_date.or(y.public.release_date), + index: x.public.index.or(y.public.index), + media: x.public.media.or(y.public.media), // TODO proper media merging + ratings: x + .public + .ratings + .into_iter() + .chain(y.public.ratings) + .collect(), + federated: x.public.federated.or(y.public.federated), + }, + private: NodePrivate { + id: x.private.id.or(y.private.id), + poster: x.private.poster.or(y.private.poster), + backdrop: x.private.backdrop.or(y.private.backdrop), + source: x.private.source.or(y.private.source), // TODO here too + }, + } +} + +// #[async_recursion] +// pub async fn import_path( +// path: PathBuf, +// db: &Database, +// fed: &Federation, +// mut node_path: Vec, +// ) -> anyhow::Result<(Vec, usize)> { +// if path.is_dir() { +// let mpath = path.join("directory.json"); +// let children_paths = path.read_dir()?.map(Result::unwrap).filter_map(|e| { +// if e.path().extension() == Some(&OsStr::from_bytes(b"jelly")) +// || e.metadata().unwrap().is_dir() +// { +// Some(e.path()) +// } else { +// None +// } +// }); +// let identifier = if mpath.exists() { +// path.file_name().unwrap().to_str().unwrap().to_string() +// } else { +// node_path +// .last() +// .cloned() +// .ok_or(anyhow!("non-root node requires parent"))? +// }; + +// node_path.push(identifier.clone()); +// let mut all: FuturesUnordered<_> = children_paths +// .into_iter() +// .map(|p| import_path(p.clone(), db, fed, node_path.clone()).map_err(|e| (p, e))) +// .collect(); +// node_path.pop(); // we will set the dirs path later and need it to not be included + +// let mut children_ids = Vec::new(); +// let mut errs = 0; +// while let Some(k) = all.next().await { +// match k { +// core::result::Result::Ok((els, errs2)) => { +// errs += errs2; +// children_ids.extend(els) +// } +// Err((p, e)) => { +// errs += 1; +// error!("import of {p:?} failed: {e:?}") +// } +// } +// } +// if mpath.exists() { +// let mut node: Node = +// serde_json::from_reader(File::open(mpath).context("metadata missing")?)?; + +// node.public.children = children_ids; +// node.public.path = node_path; +// node.public.id = Some(identifier.to_owned()); +// info!("adding {identifier}"); +// db.node.insert(&identifier, &node)?; +// Ok((vec![identifier], errs)) +// } else { +// Ok((children_ids, errs)) +// } +// } else if path.is_file() { +// info!("loading {path:?}"); +// let datafile = File::open(path.clone()).context("cant load metadata")?; +// let mut node: Node = serde_json::from_reader(datafile).context("invalid metadata")?; +// let identifier = node.private.id.clone().unwrap_or_else(|| { +// path.file_name() +// .unwrap() +// .to_str() +// .unwrap() +// .strip_suffix(".json") +// .unwrap() +// .to_string() +// }); + +// let idents = if let Some(io) = node.private.import.take() { +// let session = fed +// .get_session(&io.host) +// .await +// .context("creating session")?; + +// import_remote(io, db, &session, identifier.clone(), node_path) +// .await +// .context("federated import")? +// } else { +// debug!("adding {identifier}"); +// node.public.path = node_path; +// node.public.id = Some(identifier.to_owned()); +// let did_insert = db.node.insert(&identifier, &node)?.is_none(); +// if did_insert { +// vec![identifier] +// } else { +// vec![] +// } +// }; +// Ok((idents, 0)) +// } else { +// bail!("did somebody really put a fifo or socket in the library?!") +// } +// } + +static SEM_REMOTE_IMPORT: LazyLock = LazyLock::new(|| Semaphore::new(16)); + +#[async_recursion] +async fn import_remote( + id: String, + host: &str, + db: &Database, + session: &Arc, + index_path: &[usize], +) -> anyhow::Result<()> { + let insert_node = move |id: &String, n: Node| -> anyhow::Result<()> { + db.node_import.fetch_and_update(id, |l| { + let mut l = l.unwrap_or_default(); + l.push((index_path.to_vec(), n.clone())); + Some(l) + })?; + Ok(()) + }; + let _permit = SEM_REMOTE_IMPORT.acquire().await.unwrap(); + info!("loading federated node {id:?}"); + + let node = session.node(&id).await.context("fetching remote node")?; + + if node.federated.as_ref() == Some(&CONF.hostname) { + return Ok(()); + } + + // TODO maybe use lazy download + let poster = cache_federation_asset(session.to_owned(), id.clone(), AssetRole::Poster).await?; + let backdrop = + cache_federation_asset(session.to_owned(), id.clone(), AssetRole::Backdrop).await?; + + drop(_permit); + + let node = Node { + public: node.clone(), + private: NodePrivate { + backdrop: Some(backdrop), + poster: Some(poster), + id: None, + source: None, // TODO + }, + }; + + debug!("adding {id}"); + insert_node(&id, node.clone())?; + + let mut children: FuturesUnordered<_> = node + .public + .children + .iter() + .map(|c| import_remote(c.to_owned(), host, db, session, index_path)) + .collect(); + + while let Some(r) = children.next().await { + r?; + } + + Ok(()) +} + +async fn cache_federation_asset( + session: Arc, + identifier: String, + role: AssetRole, +) -> anyhow::Result { + async_cache_file( + &["fed-asset", role.as_str(), &identifier.clone()], + move |out| async move { + let session = session; + session + .node_asset(identifier.as_str(), role, 1024, out) + .await + }, + ) + .await +} diff --git a/import/src/mod.rs b/import/src/mod.rs new file mode 100644 index 0000000..0c43cde --- /dev/null +++ b/import/src/mod.rs @@ -0,0 +1,319 @@ +/* + This file is part of jellything (https://codeberg.org/metamuffin/jellything) + which is licensed under the GNU Affero General Public License (version 3); see /COPYING. + Copyright (C) 2023 metamuffin +*/ +pub mod infojson; +pub mod tmdb; + +use crate::{make_ident, ok_or_warn, Action}; +use anyhow::Context; +use infojson::YVideo; +use jellycommon::{ + AssetLocation, LocalTrack, MediaInfo, Node, NodeKind, NodePrivate, NodePublic, Rating, + TrackSource, +}; +use jellymatroska::read::EbmlReader; +use jellyremuxer::import::import_metadata; +use log::{debug, info, warn}; +use std::{ + collections::BTreeMap, + fs::{remove_file, File}, + io::{stdin, BufReader, Write}, +}; +use tmdb::{tmdb_details, tmdb_image}; + +pub(crate) fn import(action: Action, dry: bool) -> anyhow::Result<()> { + match action { + Action::New { + path, + tmdb_id, + tmdb_search, + input, + series, + ident_prefix, + ignore_attachments, + copy, + video, + ignore_metadata, + r#move, + title, + skip_existing, + } => { + if std::env::current_dir().unwrap().file_name().unwrap() != "library" { + warn!("new command can only be used in the library directory; what you are doing right now probably wont work.") + } + + if skip_existing { + if let Some(input) = &input { + let guessed_path = path.join(input.file_stem().unwrap_or(input.as_os_str())); + if guessed_path.exists() { + info!("guessed output ({guessed_path:?}) exists, skipping import"); + return Ok(()); + } else { + debug!("guessed output ({guessed_path:?}) missing"); + } + } + } + + let tmdb_kind = if series { "tv" } else { "movie" }; + let tmdb_id = if let Some(id) = tmdb_id { + Some(id.parse().unwrap()) + } else if let Some(title) = tmdb_search { + let tmdb_key = std::env::var("TMDB_API_KEY").context("tmdb api key required")?; + let results = tmdb::tmdb_search(tmdb_kind, &title, &tmdb_key)?; + info!("results:"); + for (i, r) in results.results.iter().enumerate() { + info!( + "\t[{i}] {}: {} ({})", + r.id, + r.title.as_ref().or(r.name.as_ref()).unwrap(), + r.overview.chars().take(100).collect::() + ); + } + let res_index = if results.results.len() > 1 { + stdin() + .lines() + .next() + .unwrap() + .unwrap() + .parse::() + .unwrap() + } else { + 0 + }; + Some(results.results[res_index].id) + } else { + None + }; + + let tmdb_details = tmdb_id + .map(|id| { + let tmdb_key = + std::env::var("TMDB_API_KEY").context("tmdb api key required")?; + let td = tmdb_details(tmdb_kind, id, &tmdb_key) + .context("fetching details") + .unwrap(); + Ok::<_, anyhow::Error>(td) + }) + .transpose()?; + + let mut kind = NodeKind::Series; + let mut file_meta = None; + let mut infojson = None; + + if let Some(input_path) = &input { + file_meta = Some({ + let input = BufReader::new(File::open(&input_path).unwrap()); + let mut input = EbmlReader::new(input); + import_metadata(&mut input)? + }); + if ignore_attachments { + let file_meta = file_meta.as_mut().unwrap(); + file_meta.cover = None; + file_meta.infojson = None; + } + if ignore_metadata { + let file_meta = file_meta.as_mut().unwrap(); + file_meta.description = None; + file_meta.tagline = None; + file_meta.title = None; + } + + if let Some(ij) = &file_meta.as_ref().unwrap().infojson { + infojson = + Some(serde_json::from_str::(ij).context("parsing info.json")?); + } + + kind = if video { + NodeKind::Video + } else { + NodeKind::Movie + }; + } + + let title = title + .or(tmdb_details + .as_ref() + .map(|d| d.title.clone().or(d.name.clone())) + .flatten()) + .or(file_meta.as_ref().map(|m| m.title.clone()).flatten()) + .expect("no title detected"); + + let ident = format!( + "{}{}", + ident_prefix.unwrap_or(String::new()), + make_ident( + &infojson + .as_ref() + .map(|i| i.id.clone()) + .unwrap_or(title.clone()) + ), + ); + let path = path.join(&ident); + let source_path = input.as_ref().map(|_| path.join(format!("source.mkv"))); + + let (mut poster, mut backdrop) = (None, None); + if !dry { + std::fs::create_dir_all(&path)?; + + poster = file_meta + .as_ref() + .map(|m| { + m.cover + .as_ref() + .map(|(mime, data)| { + let pu = path.join(format!( + "cover.{}", + match mime.as_str() { + "image/webp" => "webp", + "image/jpeg" => "jpeg", + "image/png" => "png", + _ => { + warn!("unknown mime, just using webp"); + "webp" + } + } + )); + if !pu.exists() { + let mut f = File::create(&pu)?; + f.write_all(&data)?; + } + Ok::<_, anyhow::Error>(pu) + }) + .transpose() + }) + .transpose()? + .flatten() + .or(tmdb_details + .as_ref() + .map(|d| { + d.poster_path + .as_ref() + .map(|p| { + let pu = path.join("poster.jpeg"); + let mut f = File::create(&pu)?; + tmdb_image(&p, &mut f)?; + Ok::<_, anyhow::Error>(pu) + }) + .transpose() + }) + .transpose()? + .flatten()); + + backdrop = tmdb_details + .as_ref() + .map(|d| { + d.backdrop_path + .as_ref() + .map(|p| { + let pu = path.join("backdrop.jpeg"); + let mut f = File::create(&pu)?; + tmdb_image(&p, &mut f)?; + Ok::<_, anyhow::Error>(pu) + }) + .transpose() + }) + .transpose()? + .flatten(); + } + + let mut ratings = BTreeMap::new(); + + ratings.extend( + infojson + .as_ref() + .map(|i| (Rating::YoutubeViews, i.view_count as f64)), + ); + ratings.extend( + infojson + .as_ref() + .map(|i| i.like_count.map(|l| (Rating::YoutubeLikes, l as f64))) + .flatten(), + ); + ratings.extend( + tmdb_details + .as_ref() + .map(|d| (Rating::Tmdb, d.vote_average)), + ); + + let node = Node { + private: NodePrivate { + id: Some(ident.clone()), + backdrop: backdrop.clone().map(AssetLocation::Library), + poster: poster.clone().map(AssetLocation::Library), + source: file_meta.as_ref().map(|m| MediaSource::Local { + tracks: m + .track_sources + .clone() + .into_iter() + .map(|t| LocalTrack { + path: source_path.clone().unwrap(), + ..t + }) + .collect(), + }), + }, + public: NodePublic { + federated: None, + ratings, + description: file_meta + .as_ref() + .map(|m| m.description.clone()) + .flatten() + .or(tmdb_details.as_ref().map(|d| d.overview.to_owned())), + tagline: file_meta.as_ref().map(|m| m.tagline.clone()).flatten().or( + tmdb_details + .as_ref() + .map(|d| d.tagline.to_owned()) + .flatten(), + ), + title: Some(title), + index: None, + kind: Some(kind), + children: Vec::new(), + media: file_meta.as_ref().map(|m| MediaInfo { + chapters: m.chapters.clone(), + duration: m.duration, + tracks: m.tracks.clone(), + }), + release_date: tmdb_details + .as_ref() + .map(|d| tmdb::parse_release_date(&d.release_date.clone()?).ok()) + .flatten() + .or(infojson + .as_ref() + .and_then(|j| ok_or_warn(infojson::parse_upload_date(&j.upload_date)))), + ..Default::default() + }, + }; + + if dry { + println!("{}", serde_json::to_string_pretty(&node)?); + } else { + if let Some(source_path) = source_path { + let input = input.clone().unwrap(); + if r#move { + std::fs::rename(&input, &source_path)?; + } else if copy { + std::fs::copy(&input, &source_path)?; + } else { + if source_path.is_symlink() { + remove_file(&source_path)?; + } + std::os::unix::fs::symlink(&input, &source_path)?; + } + } + let f = File::create(path.join(if series { + "directory.json" + } else { + "item.jelly" + }))?; + serde_json::to_writer_pretty(f, &node)?; + } + + Ok(()) + } + _ => unreachable!(), + } +} diff --git a/import/src/tmdb.rs b/import/src/tmdb.rs new file mode 100644 index 0000000..c38d50e --- /dev/null +++ b/import/src/tmdb.rs @@ -0,0 +1,116 @@ +/* + This file is part of jellything (https://codeberg.org/metamuffin/jellything) + which is licensed under the GNU Affero General Public License (version 3); see /COPYING. + Copyright (C) 2023 metamuffin +*/ +use anyhow::Context; +use jellycommon::chrono::{format::Parsed, DateTime, Utc}; +use log::info; +use serde::Deserialize; +use std::io::Write; + +#[derive(Debug, Clone, Deserialize)] +pub struct TmdbQuery { + pub page: usize, + pub results: Vec, + pub total_pages: usize, + pub total_results: usize, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct TmdbQueryResult { + pub adult: bool, + pub backdrop_path: Option, + pub genre_ids: Vec, + pub id: u64, + pub original_language: Option, + pub original_title: Option, + pub overview: String, + pub popularity: f64, + pub poster_path: Option, + pub release_date: Option, + pub title: Option, + pub name: Option, + pub vote_average: f64, + pub vote_count: usize, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct TmdbDetails { + pub adult: bool, + pub backdrop_path: Option, + pub genres: Vec, + pub id: u64, + pub original_language: Option, + pub original_title: Option, + pub overview: String, + pub popularity: f64, + pub poster_path: Option, + pub release_date: Option, + pub title: Option, + pub name: Option, + pub vote_average: f64, + pub vote_count: usize, + pub budget: Option, + pub homepage: Option, + pub imdb_id: Option, + pub production_companies: Vec, + pub revenue: Option, + pub tagline: Option, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct TmdbGenre { + pub id: u64, + pub name: String, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct TmdbProductionCompany { + pub id: u64, + pub name: String, + pub logo_path: Option, +} + +pub fn tmdb_search(kind: &str, query: &str, key: &str) -> anyhow::Result { + info!("searching tmdb: {query:?}"); + Ok(reqwest::blocking::get(&format!( + "https://api.themoviedb.org/3/search/{kind}?query={}&api_key={key}", + query.replace(" ", "+") + ))? + .json::()?) +} + +pub fn tmdb_details(kind: &str, id: u64, key: &str) -> anyhow::Result { + info!("fetching details: {id:?}"); + Ok(reqwest::blocking::get(&format!( + "https://api.themoviedb.org/3/{kind}/{id}?api_key={key}" + ))? + .json()?) +} + +pub fn tmdb_image(path: &str, out: &mut impl Write) -> anyhow::Result<()> { + info!("downloading image {path:?}"); + let mut res = reqwest::blocking::get(&format!("https://image.tmdb.org/t/p/original{path}"))?; + res.copy_to(out)?; + Ok(()) +} + +pub fn parse_release_date(d: &str) -> anyhow::Result> { + let (year, month, day) = (&d[0..4], &d[5..7], &d[8..10]); + let (year, month, day) = ( + year.parse().context("parsing year")?, + month.parse().context("parsing month")?, + day.parse().context("parsing day")?, + ); + + let mut p = Parsed::new(); + p.year = Some(year); + p.month = Some(month); + p.day = Some(day); + p.hour_div_12 = Some(0); + p.hour_mod_12 = Some(0); + p.minute = Some(0); + p.second = Some(0); + Ok(p.to_datetime_with_timezone(&Utc)?) +} diff --git a/remuxer/src/import/mod.rs b/remuxer/src/import/mod.rs index cf37b78..92e601e 100644 --- a/remuxer/src/import/mod.rs +++ b/remuxer/src/import/mod.rs @@ -325,6 +325,7 @@ fn import_read_segment(segment: &mut Unflatten) -> Result { _ => bail!("invalid track type"), }; m.tracks.push(SourceTrack { + federated: vec![], default_duration, name: name.unwrap_or_else(|| "unnamed".to_string()), codec: codec.unwrap(), diff --git a/remuxer/src/remux.rs b/remuxer/src/remux.rs index 0e7877d..b705572 100644 --- a/remuxer/src/remux.rs +++ b/remuxer/src/remux.rs @@ -108,7 +108,7 @@ pub fn remux_stream_into( output.write_tag(&MatroskaTag::Info(Master::Collected(vec![ MatroskaTag::TimestampScale(1_000_000), MatroskaTag::Duration(item.media.unwrap().duration * 1000.0), - MatroskaTag::Title(item.title.clone()), + MatroskaTag::Title(item.title.unwrap_or_default().clone()), MatroskaTag::MuxingApp("jellyremux".to_string()), MatroskaTag::WritingApp("jellything".to_string()), ])))?; diff --git a/remuxer/src/snippet.rs b/remuxer/src/snippet.rs index afda1f3..cd45f8b 100644 --- a/remuxer/src/snippet.rs +++ b/remuxer/src/snippet.rs @@ -110,7 +110,10 @@ pub fn write_snippet_into( output.write_tag(&ebml_header(webm))?; output.write_tag(&MatroskaTag::Segment(Master::Start))?; output.write_tag(&ebml_segment_info( - format!("{} (track {track}; snippet {n})", item.title), + format!( + "{} (track {track}; snippet {n})", + item.title.clone().unwrap_or_default() + ), (last_block.pts - start_block.pts) as f64 / 1000., ))?; output.write_tag(&MatroskaTag::Tags(Master::Collected(vec![])))?; diff --git a/server/Cargo.toml b/server/Cargo.toml index da8603d..68ae861 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -7,8 +7,8 @@ edition = "2021" jellycommon = { path = "../common", features = ["rocket"] } jellybase = { path = "../base" } jellystream = { path = "../stream" } -jellyclient = { path = "../client" } jellytranscoder = { path = "../transcoder" } +jellyimport = { path = "../import" } serde = { version = "1.0.193", features = ["derive"] } bincode = { version = "2.0.0-rc.3", features = ["serde", "derive"] } diff --git a/server/src/federation.rs b/server/src/federation.rs deleted file mode 100644 index eb2a1ac..0000000 --- a/server/src/federation.rs +++ /dev/null @@ -1,61 +0,0 @@ -/* - This file is part of jellything (https://codeberg.org/metamuffin/jellything) - which is licensed under the GNU Affero General Public License (version 3); see /COPYING. - Copyright (C) 2023 metamuffin -*/ -use anyhow::anyhow; -use jellybase::CONF; -use jellyclient::{Instance, LoginDetails, Session}; -use std::{collections::HashMap, sync::Arc}; -use tokio::sync::RwLock; - -pub struct Federation { - instances: HashMap, - sessions: RwLock>>, -} - -impl Federation { - pub fn initialize() -> Self { - let instances = CONF - .remote_credentials - .iter() - .map(|(k, (_, _, tls))| (k.to_owned(), Instance::new(k.to_owned(), *tls))) - .collect::>(); - - Self { - instances, - sessions: Default::default(), - } - } - - pub fn get_instance(&self, host: &String) -> anyhow::Result<&Instance> { - Ok(self - .instances - .get(host) - .ok_or(anyhow!("unknown instance"))?) - } - pub async fn get_session(&self, host: &String) -> anyhow::Result> { - let mut w = self.sessions.write().await; - if let Some(s) = w.get(host) { - Ok(s.to_owned()) - } else { - let (username, password, _) = CONF - .remote_credentials - .get(host) - .ok_or(anyhow!("no credentials of the remote server"))?; - let s = Arc::new( - self.get_instance(host)? - .to_owned() - .login(LoginDetails { - username: username.to_owned(), - password: password.to_owned(), - expire: None, - drop_permissions: None, - }) - .await?, - ); - w.insert(host.to_owned(), s.clone()); - Ok(s) - } - } -} diff --git a/server/src/import.rs b/server/src/import.rs deleted file mode 100644 index dc32fbf..0000000 --- a/server/src/import.rs +++ /dev/null @@ -1,254 +0,0 @@ -/* - This file is part of jellything (https://codeberg.org/metamuffin/jellything) - which is licensed under the GNU Affero General Public License (version 3); see /COPYING. - Copyright (C) 2023 metamuffin -*/ -use crate::{database::Database, federation::Federation, CONF}; -use anyhow::{anyhow, bail, Context, Ok}; -use async_recursion::async_recursion; -use futures::{stream::FuturesUnordered, StreamExt, TryFutureExt}; -use jellybase::cache::async_cache_file; -use jellyclient::Session; -use jellycommon::{AssetLocation, AssetRole, MediaSource, Node, NodePrivate, RemoteImportOptions}; -use log::{debug, error, info}; -use std::{ - ffi::OsStr, - fs::File, - os::unix::prelude::OsStrExt, - path::PathBuf, - sync::{Arc, LazyLock}, -}; -use tokio::sync::Semaphore; - -static IMPORT_SEM: LazyLock = LazyLock::new(|| Semaphore::new(1)); - -pub async fn import(db: &Database, fed: &Federation) -> anyhow::Result<()> { - info!("clearing node tree"); - let permit = IMPORT_SEM.try_acquire()?; - db.node.clear()?; - info!("importing..."); - let (_, errs) = import_path(CONF.library_path.clone(), db, fed, vec![]) - .await - .context("indexing")?; - info!("import completed"); - drop(permit); - if errs == 0 { - Ok(()) - } else { - Err(anyhow!( - "partial import, {errs} errors occured; see server log" - )) - } -} - -#[async_recursion] -pub async fn import_path( - path: PathBuf, - db: &Database, - fed: &Federation, - mut node_path: Vec, -) -> anyhow::Result<(Vec, usize)> { - if path.is_dir() { - let mpath = path.join("directory.json"); - let children_paths = path.read_dir()?.map(Result::unwrap).filter_map(|e| { - if e.path().extension() == Some(&OsStr::from_bytes(b"jelly")) - || e.metadata().unwrap().is_dir() - { - Some(e.path()) - } else { - None - } - }); - let identifier = if mpath.exists() { - path.file_name().unwrap().to_str().unwrap().to_string() - } else { - node_path - .last() - .cloned() - .ok_or(anyhow!("non-root node requires parent"))? - }; - - node_path.push(identifier.clone()); - let mut all: FuturesUnordered<_> = children_paths - .into_iter() - .map(|p| import_path(p.clone(), db, fed, node_path.clone()).map_err(|e| (p, e))) - .collect(); - node_path.pop(); // we will set the dirs path later and need it to not be included - - let mut children_ids = Vec::new(); - let mut errs = 0; - while let Some(k) = all.next().await { - match k { - core::result::Result::Ok((els, errs2)) => { - errs += errs2; - children_ids.extend(els) - } - Err((p, e)) => { - errs += 1; - error!("import of {p:?} failed: {e:?}") - } - } - } - if mpath.exists() { - let mut node: Node = - serde_json::from_reader(File::open(mpath).context("metadata missing")?)?; - - node.public.children = children_ids; - node.public.path = node_path; - node.public.id = Some(identifier.to_owned()); - info!("adding {identifier}"); - db.node.insert(&identifier, &node)?; - Ok((vec![identifier], errs)) - } else { - Ok((children_ids, errs)) - } - } else if path.is_file() { - info!("loading {path:?}"); - let datafile = File::open(path.clone()).context("cant load metadata")?; - let mut node: Node = serde_json::from_reader(datafile).context("invalid metadata")?; - let identifier = node.private.id.clone().unwrap_or_else(|| { - path.file_name() - .unwrap() - .to_str() - .unwrap() - .strip_suffix(".jelly") - .unwrap() - .to_string() - }); - - let idents = if let Some(io) = node.private.import.take() { - let session = fed - .get_session(&io.host) - .await - .context("creating session")?; - - import_remote(io, db, &session, identifier.clone(), node_path) - .await - .context("federated import")? - } else { - debug!("adding {identifier}"); - node.public.path = node_path; - node.public.id = Some(identifier.to_owned()); - let did_insert = db.node.insert(&identifier, &node)?.is_none(); - if did_insert { - vec![identifier] - } else { - vec![] - } - }; - Ok((idents, 0)) - } else { - bail!("did somebody really put a fifo or socket in the library?!") - } -} - -static SEM_REMOTE_IMPORT: LazyLock = LazyLock::new(|| Semaphore::new(16)); - -#[async_recursion] -async fn import_remote( - mut opts: RemoteImportOptions, - db: &Database, - session: &Arc, - identifier: String, - mut node_path: Vec, -) -> anyhow::Result> { - let _permit = SEM_REMOTE_IMPORT.acquire().await.unwrap(); - info!("loading federated node {identifier:?}"); - - let flatten = opts.flatten; - opts.flatten = false; - - let node = session - .node(&opts.id) - .await - .context("fetching remote node")?; - - if node.federated.as_ref() == Some(&CONF.hostname) { - return Ok(vec![]); // node is federated from us, lets not import it - } - - let poster = - cache_federation_asset(session.to_owned(), opts.id.clone(), AssetRole::Poster).await?; - let backdrop = - cache_federation_asset(session.to_owned(), opts.id.clone(), AssetRole::Backdrop).await?; - - drop(_permit); - - let mut did_insert = false; - if !flatten { - let mut node = Node { - public: node.clone(), - private: NodePrivate { - backdrop: Some(backdrop), - poster: Some(poster), - import: None, - id: None, - source: Some(MediaSource::Remote { - host: opts.host.clone(), - remote_id: opts.id.clone(), - }), - }, - }; - node.public.path = node_path.clone(); - node.public.federated = Some(opts.host.clone()); - node.public - .children - .iter_mut() - .for_each(|c| *c = format!("{}{c}", opts.prefix.clone().unwrap_or(String::new()))); - - debug!("adding {identifier}"); - node.public.id = Some(identifier.to_owned()); - did_insert = db - .node - .fetch_and_update(&identifier, |pnode| Some(pnode.unwrap_or(node.clone())))? - .is_none(); - node_path.push(opts.id.clone()); - } - - let mut children: FuturesUnordered<_> = node - .children - .iter() - .map(|c| { - let prefixed = format!("{}{c}", opts.prefix.clone().unwrap_or(String::new())); - import_remote( - RemoteImportOptions { - id: c.to_owned(), - ..opts.clone() - }, - db, - session, - prefixed, - node_path.clone(), - ) - }) - .collect(); - - let mut children_idents = Vec::new(); - while let Some(r) = children.next().await { - children_idents.extend(r?); - } - Ok(if flatten { - children_idents - } else if did_insert { - vec![identifier] - } else { - vec![] - }) -} - -async fn cache_federation_asset( - session: Arc, - identifier: String, - role: AssetRole, -) -> anyhow::Result { - async_cache_file( - &["federation-asset", role.as_str(), &identifier.clone()], - move |out| async move { - let session = session; - session - .node_asset(identifier.as_str(), role, 1024, out) - .await - }, - ) - .await -} diff --git a/server/src/main.rs b/server/src/main.rs index 6e732bd..d9ddf8b 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -8,16 +8,13 @@ use crate::routes::ui::{account::hash_password, admin::log::enable_logging}; use database::Database; -use federation::Federation; -use jellybase::CONF; +use jellybase::{CONF, federation::Federation}; use jellycommon::user::{PermissionSet, Theme, User}; use log::{error, warn}; use routes::build_rocket; use tokio::fs::create_dir_all; pub use jellybase::database; -pub mod federation; -pub mod import; pub mod routes; #[rocket::main] diff --git a/server/src/routes/mod.rs b/server/src/routes/mod.rs index e95d714..4c7838d 100644 --- a/server/src/routes/mod.rs +++ b/server/src/routes/mod.rs @@ -3,10 +3,10 @@ which is licensed under the GNU Affero General Public License (version 3); see /COPYING. Copyright (C) 2023 metamuffin */ -use crate::{database::Database, federation::Federation, routes::ui::error::MyResult}; +use crate::{database::Database, routes::ui::error::MyResult}; use api::{r_api_account_login, r_api_node_raw, r_api_root, r_api_version}; use base64::Engine; -use jellybase::CONF; +use jellybase::{CONF, federation::Federation}; use log::warn; use progress::{r_player_progress, r_player_watched}; use rand::random; diff --git a/server/src/routes/stream.rs b/server/src/routes/stream.rs index e7b3d54..14db462 100644 --- a/server/src/routes/stream.rs +++ b/server/src/routes/stream.rs @@ -4,14 +4,14 @@ Copyright (C) 2023 metamuffin */ use super::ui::{account::session::Session, error::MyError}; -use crate::{database::Database, federation::Federation}; +use crate::database::Database; use anyhow::{anyhow, Result}; use jellybase::{ + federation::Federation, permission::{NodePermissionExt, PermissionSetExt}, CONF, }; -use jellyclient::LoginDetails; -use jellycommon::{stream::StreamSpec, user::UserPermission, MediaSource}; +use jellycommon::{stream::StreamSpec, user::UserPermission, TrackSource}; use log::{info, warn}; use rocket::{ get, head, @@ -58,35 +58,36 @@ pub async fn r_stream( .as_ref() .ok_or(anyhow!("item does not contain media"))?; - if let MediaSource::Remote { host, remote_id } = source { - session - .user - .permissions - .assert(&UserPermission::FederatedContent)?; - - let (username, password, _) = CONF - .remote_credentials - .get(host) - .ok_or(anyhow!("no credentials on the server-side"))?; - - info!("creating session on {host}"); - let instance = federation.get_instance(&host)?.to_owned(); - let session = instance - .login(LoginDetails { - username: username.to_owned(), - password: password.to_owned(), - expire: Some(60), - drop_permissions: Some(HashSet::from_iter([ - UserPermission::ManageSelf, - UserPermission::Admin, // in case somebody federated the admin :))) - ])), - }) - .await?; - - let uri = session.stream(&remote_id, &spec); - info!("federation redirect"); - return Ok(Either::Right(RedirectResponse(uri))); - } + // TODO federated streams + // if let MediaSource::Remote { host, remote_id } = source { + // session + // .user + // .permissions + // .assert(&UserPermission::FederatedContent)?; + + // let (username, password, _) = CONF + // .remote_credentials + // .get(host) + // .ok_or(anyhow!("no credentials on the server-side"))?; + + // info!("creating session on {host}"); + // let instance = federation.get_instance(&host)?.to_owned(); + // let session = instance + // .login(LoginDetails { + // username: username.to_owned(), + // password: password.to_owned(), + // expire: Some(60), + // drop_permissions: Some(HashSet::from_iter([ + // UserPermission::ManageSelf, + // UserPermission::Admin, // in case somebody federated the admin :))) + // ])), + // }) + // .await?; + + // let uri = session.stream(&remote_id, &spec); + // info!("federation redirect"); + // return Ok(Either::Right(RedirectResponse(uri))); + // } info!( "stream request (range={})", diff --git a/server/src/routes/ui/admin/mod.rs b/server/src/routes/ui/admin/mod.rs index 0d1ee0a..b976192 100644 --- a/server/src/routes/ui/admin/mod.rs +++ b/server/src/routes/ui/admin/mod.rs @@ -9,8 +9,6 @@ pub mod user; use super::account::session::AdminSession; use crate::{ database::Database, - federation::Federation, - import::import, routes::ui::{ admin::log::rocket_uri_macro_r_admin_log, error::MyResult, @@ -19,7 +17,8 @@ use crate::{ uri, }; use anyhow::anyhow; -use jellybase::CONF; +use jellybase::{federation::Federation, CONF}; +use jellyimport::import; use rand::Rng; use rocket::{form::Form, get, post, FromForm, State}; use std::time::Instant; diff --git a/server/src/routes/ui/node.rs b/server/src/routes/ui/node.rs index 6e2f532..bcb7362 100644 --- a/server/src/routes/ui/node.rs +++ b/server/src/routes/ui/node.rs @@ -72,7 +72,7 @@ pub async fn r_library_node_filter<'a>( filter_and_sort_nodes(&filter, &mut children); Ok(Either::Left(LayoutPage { - title: node.title.to_string(), + title: node.title.clone().unwrap_or_default(), content: markup::new! { @NodePage { node: &node, id: &id, udata: &udata, children: &children, filter: &filter } }, @@ -82,14 +82,14 @@ pub async fn r_library_node_filter<'a>( markup::define! { NodeCard<'a>(id: &'a str, node: &'a NodePublic, udata: &'a NodeUserData) { - @let cls = format!("node card poster {}", match node.kind {NodeKind::Channel => "aspect-square", NodeKind::Video => "aspect-thumb", NodeKind::Collection => "aspect-land", _ => "aspect-port"}); + @let cls = format!("node card poster {}", match node.kind.unwrap_or_default() {NodeKind::Channel => "aspect-square", NodeKind::Video => "aspect-thumb", NodeKind::Collection => "aspect-land", _ => "aspect-port"}); div[class=cls] { .poster { a[href=uri!(r_library_node(id))] { img[src=uri!(r_item_assets(id, AssetRole::Poster, Some(1024)))]; } .cardhover.item { - @if !(matches!(node.kind, NodeKind::Collection | NodeKind::Channel)) { + @if !(matches!(node.kind.unwrap_or_default(), NodeKind::Collection | NodeKind::Channel)) { a.play.icon[href=&uri!(r_player(id, PlayerConfig::default()))] { "play_arrow" } } @Props { node, udata } @@ -103,17 +103,17 @@ markup::define! { } } NodePage<'a>(id: &'a str, node: &'a NodePublic, udata: &'a NodeUserData, children: &'a Vec<(String, NodePublic, NodeUserData)>, filter: &'a NodeFilterSort) { - @if !matches!(node.kind, NodeKind::Collection) { + @if !matches!(node.kind.unwrap_or_default(), NodeKind::Collection) { img.backdrop[src=uri!(r_item_assets(id, AssetRole::Backdrop, Some(2048)))]; } .page.node { - @if !matches!(node.kind, NodeKind::Collection) { + @if !matches!(node.kind.unwrap_or_default(), NodeKind::Collection) { div.bigposter { img[src=uri!(r_item_assets(id, AssetRole::Poster, Some(2048)))]; } } .title { h1 { @node.title } @if node.media.is_some() { a.play[href=&uri!(r_player(id, PlayerConfig::default()))] { "Watch now" }} - @if !matches!(node.kind, NodeKind::Collection | NodeKind::Channel) { + @if !matches!(node.kind.unwrap_or_default(), NodeKind::Collection | NodeKind::Channel) { @match udata.watched { WatchedState::None | WatchedState::Progress(_) => { @@ -152,15 +152,15 @@ markup::define! { } } } - @if matches!(node.kind, NodeKind::Collection | NodeKind::Channel) { - @if matches!(node.kind, NodeKind::Collection) { + @if matches!(node.kind.unwrap_or_default(), NodeKind::Collection | NodeKind::Channel) { + @if matches!(node.kind.unwrap_or_default(), NodeKind::Collection) { @if let Some(parent) = &node.path.last().cloned() { a.dirup[href=uri!(r_library_node(parent))] { "Go up" } } } @NodeFilterSortForm { f: filter } } - @match node.kind { + @match node.kind.unwrap_or_default() { NodeKind::Collection | NodeKind::Channel => { ul.children {@for (id, node, udata) in children.iter() { li { @NodeCard { id, node, udata } } @@ -261,7 +261,9 @@ impl MediaInfoExt for MediaInfo { let mut maxdim = 0; for t in &self.tracks { match &t.kind { - SourceTrackKind::Video { width, height, .. } => maxdim = maxdim.max(*width.max(height)), + SourceTrackKind::Video { width, height, .. } => { + maxdim = maxdim.max(*width.max(height)) + } _ => (), } } diff --git a/server/src/routes/ui/player.rs b/server/src/routes/ui/player.rs index 177a5f6..8b8adf6 100644 --- a/server/src/routes/ui/player.rs +++ b/server/src/routes/ui/player.rs @@ -59,7 +59,7 @@ pub fn r_player<'a>( let conf = player_conf(item.clone(), playing)?; Ok(LayoutPage { - title: item.public.title.to_owned(), + title: item.public.title.to_owned().unwrap_or_default(), class: Some("player"), content: markup::new! { @if playing { diff --git a/server/src/routes/ui/sort.rs b/server/src/routes/ui/sort.rs index c7fbfc2..143a101 100644 --- a/server/src/routes/ui/sort.rs +++ b/server/src/routes/ui/sort.rs @@ -139,14 +139,14 @@ pub fn filter_and_sort_nodes( o &= !match p { FilterProperty::FederationLocal => node.federated.is_none(), FilterProperty::FederationRemote => node.federated.is_some(), - FilterProperty::KindMovie => node.kind == NodeKind::Movie, - FilterProperty::KindVideo => node.kind == NodeKind::Video, - FilterProperty::KindCollection => node.kind == NodeKind::Collection, - FilterProperty::KindChannel => node.kind == NodeKind::Channel, - FilterProperty::KindShow => node.kind == NodeKind::Show, - FilterProperty::KindSeries => node.kind == NodeKind::Series, - FilterProperty::KindSeason => node.kind == NodeKind::Season, - FilterProperty::KindEpisode => node.kind == NodeKind::Episode, + FilterProperty::KindMovie => node.kind == Some(NodeKind::Movie), + FilterProperty::KindVideo => node.kind == Some(NodeKind::Video), + FilterProperty::KindCollection => node.kind == Some(NodeKind::Collection), + FilterProperty::KindChannel => node.kind == Some(NodeKind::Channel), + FilterProperty::KindShow => node.kind == Some(NodeKind::Show), + FilterProperty::KindSeries => node.kind == Some(NodeKind::Series), + FilterProperty::KindSeason => node.kind == Some(NodeKind::Season), + FilterProperty::KindEpisode => node.kind == Some(NodeKind::Episode), FilterProperty::Watched => udata.watched == WatchedState::Watched, FilterProperty::Unwatched => udata.watched == WatchedState::None, FilterProperty::WatchProgress => { diff --git a/stream/src/lib.rs b/stream/src/lib.rs index 604e3eb..1ee0690 100644 --- a/stream/src/lib.rs +++ b/stream/src/lib.rs @@ -15,7 +15,7 @@ use jellybase::{permission::PermissionSetExt, CONF}; use jellycommon::{ stream::{StreamFormat, StreamSpec}, user::{PermissionSet, UserPermission}, - LocalTrack, MediaSource, Node, + LocalTrack, Node, }; use jhls::jhls_stream; use segment::segment_stream; @@ -55,13 +55,14 @@ pub async fn stream( let (a, b) = duplex(4096); + // TODO remux of mixed remote and local tracks?! let track_sources = match node .private .source .as_ref() .ok_or(anyhow!("node has no media"))? { - MediaSource::Local { tracks } => tracks.to_owned(), + // MediaSource::Local { tracks } => tracks.to_owned(), _ => bail!("node tracks are not local"), }; diff --git a/stream/src/webvtt.rs b/stream/src/webvtt.rs index 6ee5212..faf0cd3 100644 --- a/stream/src/webvtt.rs +++ b/stream/src/webvtt.rs @@ -41,8 +41,12 @@ pub async fn webvtt_stream( }) .await??; - let webvtt = webvtt_from_ass_blocks(node.public.title, codec_private, ass_blocks) - .context("transcoding subtitles")?; + let webvtt = webvtt_from_ass_blocks( + node.public.title.clone().unwrap_or_default(), + codec_private, + ass_blocks, + ) + .context("transcoding subtitles")?; tokio::task::spawn(async move { let _ = b.write_all(webvtt.as_bytes()).await; diff --git a/tool/src/import/infojson.rs b/tool/src/import/infojson.rs deleted file mode 100644 index 3f0edc9..0000000 --- a/tool/src/import/infojson.rs +++ /dev/null @@ -1,143 +0,0 @@ -/* - This file is part of jellything (https://codeberg.org/metamuffin/jellything) - which is licensed under the GNU Affero General Public License (version 3); see /COPYING. - Copyright (C) 2023 metamuffin -*/ - -use anyhow::Context; -use jellycommon::chrono::{format::Parsed, DateTime, Utc}; -use serde::{Deserialize, Serialize}; -use std::collections::HashMap; - -#[derive(Debug, Serialize, Deserialize)] -pub struct YVideo { - pub id: String, - pub title: String, - pub formats: Vec, - pub thumbnails: Vec, - pub thumbnail: String, - pub description: String, - pub channel_id: String, - pub duration: Option, - pub view_count: usize, - pub average_rating: Option, - pub age_limit: usize, - pub webpage_url: String, - pub categories: Vec, - pub tags: Vec, - pub playable_in_embed: bool, - pub automatic_captions: HashMap>, - pub comment_count: Option, - pub chapters: Option>, - pub heatmap: Option>, - pub like_count: Option, - pub channel: Option, - pub channel_follower_count: usize, - pub channel_is_verified: Option, - pub uploader: String, - pub uploader_id: String, - pub uploader_url: String, - pub upload_date: String, - pub availability: String, // "public" | "private" | "unlisted", - pub original_url: Option, - pub webpage_url_basename: String, - pub webpage_url_domain: String, - pub extractor: String, - pub extractor_key: String, - pub playlist_count: Option, - pub playlist: Option, - pub playlist_id: Option, - pub playlist_title: Option, - pub playlist_uploader: Option, - pub playlist_uploader_id: Option, - pub n_entries: Option, - pub playlist_index: Option, - pub display_id: String, - pub fulltitle: String, - pub duration_string: String, - pub is_live: bool, - pub was_live: bool, - pub epoch: usize, -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct YCaption { - pub url: Option, - pub ext: String, //"vtt" | "json3" | "srv1" | "srv2" | "srv3" | "ttml", - pub protocol: Option, - pub name: Option, -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct YFormat { - pub format_id: String, - pub format_note: Option, - pub ext: String, - pub protocol: String, - pub acodec: Option, - pub vcodec: Option, - pub url: Option, - pub width: Option, - pub height: Option, - pub fps: Option, - pub columns: Option, - pub fragments: Option>, - pub resolution: String, - pub dynamic_range: Option, - pub aspect_ratio: Option, - pub http_headers: HashMap, - pub audio_ext: String, - pub video_ext: String, - pub vbr: Option, - pub abr: Option, - pub format: String, -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct YFragment { - pub url: Option, - pub duration: Option, -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct YThumbnail { - pub url: String, - pub preference: i32, - pub id: String, - pub height: Option, - pub width: Option, - pub resolution: Option, -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct YChapter { - pub start_time: f64, - pub end_time: f64, - pub title: String, -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct YHeatmapSample { - pub start_time: f64, - pub end_time: f64, - pub value: f64, -} - -pub fn parse_upload_date(d: &str) -> anyhow::Result> { - let (year, month, day) = (&d[0..4], &d[4..6], &d[6..8]); - let (year, month, day) = ( - year.parse().context("parsing year")?, - month.parse().context("parsing month")?, - day.parse().context("parsing day")?, - ); - - let mut p = Parsed::new(); - p.year = Some(year); - p.month = Some(month); - p.day = Some(day); - p.hour_div_12 = Some(0); - p.hour_mod_12 = Some(0); - p.minute = Some(0); - p.second = Some(0); - Ok(p.to_datetime_with_timezone(&Utc)?) -} diff --git a/tool/src/import/mod.rs b/tool/src/import/mod.rs deleted file mode 100644 index 973629c..0000000 --- a/tool/src/import/mod.rs +++ /dev/null @@ -1,320 +0,0 @@ -/* - This file is part of jellything (https://codeberg.org/metamuffin/jellything) - which is licensed under the GNU Affero General Public License (version 3); see /COPYING. - Copyright (C) 2023 metamuffin -*/ -pub mod infojson; -pub mod tmdb; - -use crate::{make_ident, ok_or_warn, Action}; -use anyhow::Context; -use infojson::YVideo; -use jellycommon::{ - AssetLocation, LocalTrack, MediaInfo, MediaSource, Node, NodeKind, NodePrivate, NodePublic, - Rating, -}; -use jellymatroska::read::EbmlReader; -use jellyremuxer::import::import_metadata; -use log::{debug, info, warn}; -use std::{ - collections::BTreeMap, - fs::{remove_file, File}, - io::{stdin, BufReader, Write}, -}; -use tmdb::{tmdb_details, tmdb_image}; - -pub(crate) fn import(action: Action, dry: bool) -> anyhow::Result<()> { - match action { - Action::New { - path, - tmdb_id, - tmdb_search, - input, - series, - ident_prefix, - ignore_attachments, - copy, - video, - ignore_metadata, - r#move, - title, - skip_existing, - } => { - if std::env::current_dir().unwrap().file_name().unwrap() != "library" { - warn!("new command can only be used in the library directory; what you are doing right now probably wont work.") - } - - if skip_existing { - if let Some(input) = &input { - let guessed_path = path.join(input.file_stem().unwrap_or(input.as_os_str())); - if guessed_path.exists() { - info!("guessed output ({guessed_path:?}) exists, skipping import"); - return Ok(()); - } else { - debug!("guessed output ({guessed_path:?}) missing"); - } - } - } - - let tmdb_kind = if series { "tv" } else { "movie" }; - let tmdb_id = if let Some(id) = tmdb_id { - Some(id.parse().unwrap()) - } else if let Some(title) = tmdb_search { - let tmdb_key = std::env::var("TMDB_API_KEY").context("tmdb api key required")?; - let results = tmdb::tmdb_search(tmdb_kind, &title, &tmdb_key)?; - info!("results:"); - for (i, r) in results.results.iter().enumerate() { - info!( - "\t[{i}] {}: {} ({})", - r.id, - r.title.as_ref().or(r.name.as_ref()).unwrap(), - r.overview.chars().take(100).collect::() - ); - } - let res_index = if results.results.len() > 1 { - stdin() - .lines() - .next() - .unwrap() - .unwrap() - .parse::() - .unwrap() - } else { - 0 - }; - Some(results.results[res_index].id) - } else { - None - }; - - let tmdb_details = tmdb_id - .map(|id| { - let tmdb_key = - std::env::var("TMDB_API_KEY").context("tmdb api key required")?; - let td = tmdb_details(tmdb_kind, id, &tmdb_key) - .context("fetching details") - .unwrap(); - Ok::<_, anyhow::Error>(td) - }) - .transpose()?; - - let mut kind = NodeKind::Series; - let mut file_meta = None; - let mut infojson = None; - - if let Some(input_path) = &input { - file_meta = Some({ - let input = BufReader::new(File::open(&input_path).unwrap()); - let mut input = EbmlReader::new(input); - import_metadata(&mut input)? - }); - if ignore_attachments { - let file_meta = file_meta.as_mut().unwrap(); - file_meta.cover = None; - file_meta.infojson = None; - } - if ignore_metadata { - let file_meta = file_meta.as_mut().unwrap(); - file_meta.description = None; - file_meta.tagline = None; - file_meta.title = None; - } - - if let Some(ij) = &file_meta.as_ref().unwrap().infojson { - infojson = - Some(serde_json::from_str::(ij).context("parsing info.json")?); - } - - kind = if video { - NodeKind::Video - } else { - NodeKind::Movie - }; - } - - let title = title - .or(tmdb_details - .as_ref() - .map(|d| d.title.clone().or(d.name.clone())) - .flatten()) - .or(file_meta.as_ref().map(|m| m.title.clone()).flatten()) - .expect("no title detected"); - - let ident = format!( - "{}{}", - ident_prefix.unwrap_or(String::new()), - make_ident( - &infojson - .as_ref() - .map(|i| i.id.clone()) - .unwrap_or(title.clone()) - ), - ); - let path = path.join(&ident); - let source_path = input.as_ref().map(|_| path.join(format!("source.mkv"))); - - let (mut poster, mut backdrop) = (None, None); - if !dry { - std::fs::create_dir_all(&path)?; - - poster = file_meta - .as_ref() - .map(|m| { - m.cover - .as_ref() - .map(|(mime, data)| { - let pu = path.join(format!( - "cover.{}", - match mime.as_str() { - "image/webp" => "webp", - "image/jpeg" => "jpeg", - "image/png" => "png", - _ => { - warn!("unknown mime, just using webp"); - "webp" - } - } - )); - if !pu.exists() { - let mut f = File::create(&pu)?; - f.write_all(&data)?; - } - Ok::<_, anyhow::Error>(pu) - }) - .transpose() - }) - .transpose()? - .flatten() - .or(tmdb_details - .as_ref() - .map(|d| { - d.poster_path - .as_ref() - .map(|p| { - let pu = path.join("poster.jpeg"); - let mut f = File::create(&pu)?; - tmdb_image(&p, &mut f)?; - Ok::<_, anyhow::Error>(pu) - }) - .transpose() - }) - .transpose()? - .flatten()); - - backdrop = tmdb_details - .as_ref() - .map(|d| { - d.backdrop_path - .as_ref() - .map(|p| { - let pu = path.join("backdrop.jpeg"); - let mut f = File::create(&pu)?; - tmdb_image(&p, &mut f)?; - Ok::<_, anyhow::Error>(pu) - }) - .transpose() - }) - .transpose()? - .flatten(); - } - - let mut ratings = BTreeMap::new(); - - ratings.extend( - infojson - .as_ref() - .map(|i| (Rating::YoutubeViews, i.view_count as f64)), - ); - ratings.extend( - infojson - .as_ref() - .map(|i| i.like_count.map(|l| (Rating::YoutubeLikes, l as f64))) - .flatten(), - ); - ratings.extend( - tmdb_details - .as_ref() - .map(|d| (Rating::Tmdb, d.vote_average)), - ); - - let node = Node { - private: NodePrivate { - id: Some(ident.clone()), - import: None, - backdrop: backdrop.clone().map(AssetLocation::Library), - poster: poster.clone().map(AssetLocation::Library), - source: file_meta.as_ref().map(|m| MediaSource::Local { - tracks: m - .track_sources - .clone() - .into_iter() - .map(|t| LocalTrack { - path: source_path.clone().unwrap(), - ..t - }) - .collect(), - }), - }, - public: NodePublic { - federated: None, - ratings, - description: file_meta - .as_ref() - .map(|m| m.description.clone()) - .flatten() - .or(tmdb_details.as_ref().map(|d| d.overview.to_owned())), - tagline: file_meta.as_ref().map(|m| m.tagline.clone()).flatten().or( - tmdb_details - .as_ref() - .map(|d| d.tagline.to_owned()) - .flatten(), - ), - title, - index: None, - kind, - children: Vec::new(), - media: file_meta.as_ref().map(|m| MediaInfo { - chapters: m.chapters.clone(), - duration: m.duration, - tracks: m.tracks.clone(), - }), - release_date: tmdb_details - .as_ref() - .map(|d| tmdb::parse_release_date(&d.release_date.clone()?).ok()) - .flatten() - .or(infojson - .as_ref() - .and_then(|j| ok_or_warn(infojson::parse_upload_date(&j.upload_date)))), - ..Default::default() - }, - }; - - if dry { - println!("{}", serde_json::to_string_pretty(&node)?); - } else { - if let Some(source_path) = source_path { - let input = input.clone().unwrap(); - if r#move { - std::fs::rename(&input, &source_path)?; - } else if copy { - std::fs::copy(&input, &source_path)?; - } else { - if source_path.is_symlink() { - remove_file(&source_path)?; - } - std::os::unix::fs::symlink(&input, &source_path)?; - } - } - let f = File::create(path.join(if series { - "directory.json" - } else { - "item.jelly" - }))?; - serde_json::to_writer_pretty(f, &node)?; - } - - Ok(()) - } - _ => unreachable!(), - } -} diff --git a/tool/src/import/tmdb.rs b/tool/src/import/tmdb.rs deleted file mode 100644 index c38d50e..0000000 --- a/tool/src/import/tmdb.rs +++ /dev/null @@ -1,116 +0,0 @@ -/* - This file is part of jellything (https://codeberg.org/metamuffin/jellything) - which is licensed under the GNU Affero General Public License (version 3); see /COPYING. - Copyright (C) 2023 metamuffin -*/ -use anyhow::Context; -use jellycommon::chrono::{format::Parsed, DateTime, Utc}; -use log::info; -use serde::Deserialize; -use std::io::Write; - -#[derive(Debug, Clone, Deserialize)] -pub struct TmdbQuery { - pub page: usize, - pub results: Vec, - pub total_pages: usize, - pub total_results: usize, -} - -#[derive(Debug, Clone, Deserialize)] -pub struct TmdbQueryResult { - pub adult: bool, - pub backdrop_path: Option, - pub genre_ids: Vec, - pub id: u64, - pub original_language: Option, - pub original_title: Option, - pub overview: String, - pub popularity: f64, - pub poster_path: Option, - pub release_date: Option, - pub title: Option, - pub name: Option, - pub vote_average: f64, - pub vote_count: usize, -} - -#[derive(Debug, Clone, Deserialize)] -pub struct TmdbDetails { - pub adult: bool, - pub backdrop_path: Option, - pub genres: Vec, - pub id: u64, - pub original_language: Option, - pub original_title: Option, - pub overview: String, - pub popularity: f64, - pub poster_path: Option, - pub release_date: Option, - pub title: Option, - pub name: Option, - pub vote_average: f64, - pub vote_count: usize, - pub budget: Option, - pub homepage: Option, - pub imdb_id: Option, - pub production_companies: Vec, - pub revenue: Option, - pub tagline: Option, -} - -#[derive(Debug, Clone, Deserialize)] -pub struct TmdbGenre { - pub id: u64, - pub name: String, -} - -#[derive(Debug, Clone, Deserialize)] -pub struct TmdbProductionCompany { - pub id: u64, - pub name: String, - pub logo_path: Option, -} - -pub fn tmdb_search(kind: &str, query: &str, key: &str) -> anyhow::Result { - info!("searching tmdb: {query:?}"); - Ok(reqwest::blocking::get(&format!( - "https://api.themoviedb.org/3/search/{kind}?query={}&api_key={key}", - query.replace(" ", "+") - ))? - .json::()?) -} - -pub fn tmdb_details(kind: &str, id: u64, key: &str) -> anyhow::Result { - info!("fetching details: {id:?}"); - Ok(reqwest::blocking::get(&format!( - "https://api.themoviedb.org/3/{kind}/{id}?api_key={key}" - ))? - .json()?) -} - -pub fn tmdb_image(path: &str, out: &mut impl Write) -> anyhow::Result<()> { - info!("downloading image {path:?}"); - let mut res = reqwest::blocking::get(&format!("https://image.tmdb.org/t/p/original{path}"))?; - res.copy_to(out)?; - Ok(()) -} - -pub fn parse_release_date(d: &str) -> anyhow::Result> { - let (year, month, day) = (&d[0..4], &d[5..7], &d[8..10]); - let (year, month, day) = ( - year.parse().context("parsing year")?, - month.parse().context("parsing month")?, - day.parse().context("parsing day")?, - ); - - let mut p = Parsed::new(); - p.year = Some(year); - p.month = Some(month); - p.day = Some(day); - p.hour_div_12 = Some(0); - p.hour_mod_12 = Some(0); - p.minute = Some(0); - p.second = Some(0); - Ok(p.to_datetime_with_timezone(&Utc)?) -} diff --git a/tool/src/main.rs b/tool/src/main.rs index 31e63b7..34337ce 100644 --- a/tool/src/main.rs +++ b/tool/src/main.rs @@ -5,12 +5,10 @@ */ #![feature(file_create_new)] -pub mod import; pub mod migrate; use base64::Engine; use clap::{Parser, Subcommand, ValueEnum}; -use import::import; use jellyclient::{Instance, LoginDetails}; use jellycommon::{config::GlobalConfig, Node, NodeKind, NodePrivate, NodePublic}; use log::{info, warn}; @@ -37,47 +35,47 @@ enum Action { #[arg(short, long)] hostname: String, }, - /// Imports a movie, video or series given media and metadata sources - New { - /// Relative path to the node's parent(!). - path: PathBuf, - /// Search the node by title on TMDB - #[arg(short = 't', long)] - tmdb_search: Option, - /// Search the node by id on TMDB - #[arg(short = 'T', long)] - tmdb_id: Option, - #[arg(long)] - /// Prefix the inferred id with something to avoid collisions - ident_prefix: Option, - /// Copies media into the library - #[arg(long)] - copy: bool, - /// Moves media into the library (potentially destructive operation) - #[arg(long)] - r#move: bool, - /// Marks node as a video - #[arg(long)] - video: bool, - /// Marks node as a series - #[arg(short, long)] - series: bool, - /// Path to the media of the node, required for non-series - #[arg(short, long)] - input: Option, - /// Ignore attachments (dont use them as cover) - #[arg(long)] - ignore_attachments: bool, - /// Ignore metadate (no title, description and tagline from input) - #[arg(long)] - ignore_metadata: bool, - /// Skip any action that appears to be run already. - #[arg(long)] - skip_existing: bool, - /// Sets the title - #[arg(long)] - title: Option, - }, + // /// Imports a movie, video or series given media and metadata sources + // New { + // /// Relative path to the node's parent(!). + // path: PathBuf, + // /// Search the node by title on TMDB + // #[arg(short = 't', long)] + // tmdb_search: Option, + // /// Search the node by id on TMDB + // #[arg(short = 'T', long)] + // tmdb_id: Option, + // #[arg(long)] + // /// Prefix the inferred id with something to avoid collisions + // ident_prefix: Option, + // /// Copies media into the library + // #[arg(long)] + // copy: bool, + // /// Moves media into the library (potentially destructive operation) + // #[arg(long)] + // r#move: bool, + // /// Marks node as a video + // #[arg(long)] + // video: bool, + // /// Marks node as a series + // #[arg(short, long)] + // series: bool, + // /// Path to the media of the node, required for non-series + // #[arg(short, long)] + // input: Option, + // /// Ignore attachments (dont use them as cover) + // #[arg(long)] + // ignore_attachments: bool, + // /// Ignore metadate (no title, description and tagline from input) + // #[arg(long)] + // ignore_metadata: bool, + // /// Skip any action that appears to be run already. + // #[arg(long)] + // skip_existing: bool, + // /// Sets the title + // #[arg(long)] + // title: Option, + // }, Migrate { database: PathBuf, mode: MigrateMode, @@ -118,8 +116,11 @@ fn main() -> anyhow::Result<()> { std::fs::create_dir_all(path.join("library"))?; std::fs::create_dir_all(path.join("cache"))?; std::fs::create_dir_all(path.join("assets"))?; + std::fs::create_dir_all(path.join("media"))?; File::create_new(path.join("assets/front.htm"))? .write_fmt(format_args!("

My very own jellything instance

"))?; + + // TODO: dont fill that serde_yaml::to_writer( File::create_new(path.join("config.yaml"))?, &GlobalConfig { @@ -149,8 +150,8 @@ fn main() -> anyhow::Result<()> { File::create_new(path.join("library/directory.json"))?, &Node { public: NodePublic { - kind: NodeKind::Collection, - title: "My Library".to_string(), + kind: Some(NodeKind::Collection), + title: Some("My Library".to_string()), ..Default::default() }, private: NodePrivate { @@ -162,7 +163,7 @@ fn main() -> anyhow::Result<()> { warn!("please change the admin password."); Ok(()) } - a @ Action::New { .. } => import(a, args.dry), + // a @ Action::New { .. } => import(a, args.dry), a @ Action::Migrate { .. } => migrate(a), Action::Reimport { config, -- cgit v1.2.3-70-g09d2 From 826c61c9612e855b19c3adb0e93d80bbfb4dc903 Mon Sep 17 00:00:00 2001 From: metamuffin Date: Fri, 22 Dec 2023 00:29:11 +0100 Subject: rework import system pt. 2 --- Cargo.lock | 2 ++ common/src/lib.rs | 4 +-- import/Cargo.toml | 4 +-- import/src/lib.rs | 65 ++++++++++++++++++++++++++++++++++++++++++-- server/src/routes/ui/home.rs | 2 +- tool/src/main.rs | 13 --------- 6 files changed, 69 insertions(+), 21 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9a12441..bd09a54 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1391,9 +1391,11 @@ dependencies = [ "jellybase", "jellyclient", "jellycommon", + "jellymatroska", "jellyremuxer", "log", "reqwest", + "serde", "serde_json", "serde_yaml", "tokio", diff --git a/common/src/lib.rs b/common/src/lib.rs index e953d85..9a3535a 100644 --- a/common/src/lib.rs +++ b/common/src/lib.rs @@ -21,8 +21,8 @@ use std::{collections::BTreeMap, path::PathBuf}; #[derive(Debug, Clone, Deserialize, Serialize, Default)] pub struct Node { - pub public: NodePublic, - pub private: NodePrivate, + #[serde(default)] pub public: NodePublic, + #[serde(default)] pub private: NodePrivate, } #[rustfmt::skip] diff --git a/import/Cargo.toml b/import/Cargo.toml index f2ba7af..a54967c 100644 --- a/import/Cargo.toml +++ b/import/Cargo.toml @@ -7,14 +7,14 @@ edition = "2021" jellycommon = { path = "../common" } jellybase = { path = "../base" } jellyclient = { path = "../client" } -# jellymatroska = { path = "../matroska" } +jellymatroska = { path = "../matroska" } jellyremuxer = { path = "../remuxer" } log = { workspace = true } anyhow = "1.0.75" reqwest = { version = "0.11.22", features = ["blocking", "json"] } -# serde = { version = "1.0.193", features = ["derive"] } +serde = { version = "1.0.193", features = ["derive"] } serde_json = "1.0.108" serde_yaml = "0.9.27" # bincode = { version = "2.0.0-rc.3", features = ["serde"] } diff --git a/import/src/lib.rs b/import/src/lib.rs index 3698f79..ed0af2d 100644 --- a/import/src/lib.rs +++ b/import/src/lib.rs @@ -4,6 +4,9 @@ Copyright (C) 2023 metamuffin */ #![feature(lazy_cell)] +pub mod infojson; +pub mod tmdb; + use anyhow::{Context, Ok}; use async_recursion::async_recursion; use futures::{stream::FuturesUnordered, StreamExt}; @@ -14,16 +17,19 @@ use jellyclient::Session; use jellycommon::{ AssetLocation, AssetRole, ImportOptions, ImportSource, Node, NodePrivate, NodePublic, }; +use jellymatroska::read::EbmlReader; +use jellyremuxer::import::import_metadata; use log::{debug, info}; use std::{ cmp::Ordering, ffi::OsStr, fs::File, + io::BufReader, os::unix::prelude::OsStrExt, path::{Path, PathBuf}, sync::{Arc, LazyLock}, }; -use tokio::sync::Semaphore; +use tokio::{sync::Semaphore, task::spawn_blocking}; static IMPORT_SEM: LazyLock = LazyLock::new(|| Semaphore::new(1)); @@ -48,12 +54,15 @@ pub fn merge_nodes(db: &Database) -> anyhow::Result<()> { nodes.sort_by(|(x, _), (y, _)| compare_index_path(x, y)); - let node = nodes + let mut node = nodes .into_iter() .map(|(_, x)| x) .reduce(|x, y| merge_node(x, y)) .unwrap(); + node.public.id = Some(id.clone()); + node.public.path = vec!["library".to_string()]; // TODO reconstruct from children + db.node.insert(&id, &node)?; } Ok(()) @@ -146,7 +155,44 @@ async fn process_source( ImportSource::Override(n) => insert_node(&id, n)?, ImportSource::Tmdb(_) => todo!(), ImportSource::Media { location } => { - let path = location.path(); + let media_path = location.path(); + + let metadata = spawn_blocking(move || { + let input = BufReader::new(File::open(&media_path).unwrap()); + let mut input = EbmlReader::new(input); + import_metadata(&mut input) + }) + .await??; + + // if let Some(cover) = metadata.cover { + // let pu = path.join(format!( + // "cover.{}", + // match mime.as_str() { + // "image/webp" => "webp", + // "image/jpeg" => "jpeg", + // "image/png" => "png", + // _ => { + // warn!("unknown mime, just using webp"); + // "webp" + // } + // } + // )); + // if !pu.exists() { + // let mut f = tokio::fs::File::create(&pu).await?; + // f.write_all(&data).await?; + // } + // } + + let node = Node { + public: NodePublic { + title: metadata.title, + description: metadata.description, + tagline: metadata.tagline, + ..Default::default() + }, + private: NodePrivate::default(), + }; + insert_node(&id, node)?; } ImportSource::Federated { host } => { let session = fed.get_session(&host).await.context("creating session")?; @@ -374,3 +420,16 @@ async fn cache_federation_asset( ) .await } + +fn make_ident(s: &str) -> String { + let mut out = String::new(); + for s in s.chars() { + match s { + 'a'..='z' | '0'..='9' => out.push(s), + 'A'..='Z' => out.push(s.to_ascii_lowercase()), + '-' | ' ' | '_' | ':' => out.push('-'), + _ => (), + } + } + out +} diff --git a/server/src/routes/ui/home.rs b/server/src/routes/ui/home.rs index bcbc847..74e9b02 100644 --- a/server/src/routes/ui/home.rs +++ b/server/src/routes/ui/home.rs @@ -77,7 +77,7 @@ pub fn r_home(sess: Session, db: &State) -> MyResult { Ok(LayoutPage { title: "Home".to_string(), content: markup::new! { - p { "Welcome back " @sess.user.display_name } + p { "Welcome back, " @sess.user.display_name } h2 { "Explore " @CONF.brand } .homelist { ul {@for (id, node, udata) in &toplevel { li { @NodeCard { id, node, udata } } diff --git a/tool/src/main.rs b/tool/src/main.rs index 34337ce..6384822 100644 --- a/tool/src/main.rs +++ b/tool/src/main.rs @@ -193,19 +193,6 @@ fn main() -> anyhow::Result<()> { } } -fn make_ident(s: &str) -> String { - let mut out = String::new(); - for s in s.chars() { - match s { - 'a'..='z' | '0'..='9' => out.push(s), - 'A'..='Z' => out.push(s.to_ascii_lowercase()), - '-' | ' ' | '_' | ':' => out.push('-'), - _ => (), - } - } - out -} - fn ok_or_warn(r: Result) -> Option { match r { Ok(t) => Some(t), -- cgit v1.2.3-70-g09d2