From 02bbb2741f2c463aadf9d07493ebaeac1d73c11a Mon Sep 17 00:00:00 2001 From: metamuffin Date: Thu, 30 Jan 2025 14:39:20 +0100 Subject: import channel and children --- import/src/lib.rs | 119 ++++++++++++++---------------------------------------- 1 file changed, 30 insertions(+), 89 deletions(-) (limited to 'import/src/lib.rs') diff --git a/import/src/lib.rs b/import/src/lib.rs index add7e4d..10bd0ec 100644 --- a/import/src/lib.rs +++ b/import/src/lib.rs @@ -4,24 +4,19 @@ Copyright (C) 2025 metamuffin */ use anyhow::{anyhow, Context, Result}; -use ebml_struct::{ - ids::*, - matroska::*, - read::{EbmlReadExt, TagRead}, -}; use infojson::YVideo; -use jellybase::{assetfed::AssetInner, cache::cache_file, database::Database, CONF, SECRETS}; +use jellybase::{assetfed::AssetInner, database::Database, CONF, SECRETS}; use jellycommon::{ Chapter, LocalTrack, MediaInfo, Node, NodeID, NodeKind, Rating, SourceTrack, SourceTrackKind, TrackSource, }; use log::info; +use matroska::matroska_metadata; use rayon::iter::{ParallelDrainRange, ParallelIterator}; -use regex::Regex; use std::{ collections::HashMap, fs::File, - io::{BufReader, ErrorKind, Read, Write}, + io::{BufReader, Read}, mem::swap, path::{Path, PathBuf}, sync::LazyLock, @@ -35,14 +30,15 @@ use tokio::{ use trakt::Trakt; pub mod infojson; +pub mod matroska; pub mod tmdb; pub mod trakt; static IMPORT_SEM: LazyLock = LazyLock::new(|| Semaphore::new(1)); pub static IMPORT_ERRORS: RwLock> = RwLock::const_new(Vec::new()); -static RE_EPISODE_FILENAME: LazyLock = - LazyLock::new(|| Regex::new(r#"([sS](\d+))?([eE](\d+))( (.+))?"#).unwrap()); +// static RE_EPISODE_FILENAME: LazyLock = +// LazyLock::new(|| Regex::new(r#"([sS](\d+))?([eE](\d+))( (.+))?"#).unwrap()); struct Apis { trakt: Option, @@ -57,6 +53,7 @@ pub async fn import_wrap(db: Database, incremental: bool) -> Result<()> { let _sem = IMPORT_SEM.try_acquire()?; let jh = spawn_blocking(move || { + *IMPORT_ERRORS.blocking_write() = Vec::new(); if let Err(e) = import(&db, incremental) { IMPORT_ERRORS.blocking_write().push(format!("{e:#}")); } @@ -121,25 +118,26 @@ fn import_iter_inner(path: &Path, db: &Database, incremental: bool) -> Result Result<()> { - let parent = NodeID::from_slug( - &path - .parent() - .ok_or(anyhow!("no parent"))? - .file_name() - .ok_or(anyhow!("parent no filename"))? - .to_string_lossy(), - ); + let parent_slug = path + .parent() + .ok_or(anyhow!("no parent"))? + .file_name() + .ok_or(anyhow!("parent no filename"))? + .to_string_lossy(); + let parent = NodeID::from_slug(&parent_slug); let filename = path.file_name().unwrap().to_string_lossy(); match filename.as_ref() { "poster.jpeg" | "poster.webp" => { db.update_node_init(parent, |node| { + node.slug = parent_slug.to_string(); node.poster = Some(AssetInner::Media(path.to_owned()).ser()); Ok(()) })?; } "backdrop.jpeg" | "backdrop.webp" => { db.update_node_init(parent, |node| { + node.slug = parent_slug.to_string(); node.backdrop = Some(AssetInner::Media(path.to_owned()).ser()); Ok(()) })?; @@ -147,6 +145,7 @@ fn import_file(db: &Database, path: &Path) -> Result<()> { "info.json" | "info.yaml" => { let data = serde_yaml::from_reader::<_, Node>(BufReader::new(File::open(path)?))?; db.update_node_init(parent, |node| { + node.slug = parent_slug.to_string(); fn merge_option(a: &mut Option, b: Option) { if b.is_some() { *a = b; @@ -161,6 +160,7 @@ fn import_file(db: &Database, path: &Path) -> Result<()> { "channel.info.json" => { let data = serde_json::from_reader::<_, YVideo>(BufReader::new(File::open(path)?))?; db.update_node_init(parent, |node| { + node.slug = parent_slug.to_string(); node.title = Some( data.title .strip_suffix(" - Videos") @@ -189,76 +189,14 @@ fn import_file(db: &Database, path: &Path) -> Result<()> { fn import_media_file(db: &Database, path: &Path, parent: NodeID) -> Result<()> { info!("reading media file {path:?}"); - let mut file = BufReader::new(File::open(path)?); - let mut file = file.by_ref().take(u64::MAX); - - let (x, mut ebml) = file.read_tag()?; - assert_eq!(x, EL_EBML); - let ebml = Ebml::read(&mut ebml).unwrap(); - assert!(ebml.doc_type == "matroska" || ebml.doc_type == "webm"); - let (x, mut segment) = file.read_tag()?; - assert_eq!(x, EL_SEGMENT); - let mut info = None; - let mut infojson = None; - let mut tracks = None; - let mut cover = None; - let mut chapters = None; - let mut tags = None; - loop { - let (x, mut seg) = match segment.read_tag() { - Ok(o) => o, - Err(e) if e.kind() == ErrorKind::UnexpectedEof => break, - Err(e) => return Err(e.into()), - }; - match x { - EL_INFO => info = Some(Info::read(&mut seg).context("info")?), - EL_TRACKS => tracks = Some(Tracks::read(&mut seg).context("tracks")?), - EL_CHAPTERS => chapters = Some(Chapters::read(&mut seg).context("chapters")?), - EL_TAGS => tags = Some(Tags::read(&mut seg).context("tags")?), - EL_ATTACHMENTS => { - let attachments = Attachments::read(&mut seg).context("attachments")?; - for f in attachments.files { - match f.name.as_str() { - "info.json" => { - infojson = Some( - serde_json::from_slice::(&f.data) - .context("infojson")?, - ); - } - "cover.webp" | "cover.png" | "cover.jpg" | "cover.jpeg" | "cover.avif" => { - cover = Some( - AssetInner::Cache(cache_file( - &["att-cover", path.to_string_lossy().as_ref()], - move |mut file| { - file.write_all(&f.data)?; - Ok(()) - }, - )?) - .ser(), - ) - } - a => println!("{a:?}"), - } - } - } - EL_VOID | EL_CRC32 | EL_CUES | EL_SEEKHEAD => { - seg.consume()?; - } - EL_CLUSTER => { - break; - } - id => { - eprintln!("unknown top-level element {id:x}"); - seg.consume()?; - } - } - } + let m = (*matroska_metadata(path)?).to_owned(); - let info = info.ok_or(anyhow!("no info"))?; - let tracks = tracks.ok_or(anyhow!("no tracks"))?; + let info = m.info.ok_or(anyhow!("no info"))?; + let tracks = m.tracks.ok_or(anyhow!("no tracks"))?; - let mut tags = tags + let mut tags = m + .tags .map(|tags| { tags.tags .into_iter() @@ -274,7 +212,8 @@ fn import_media_file(db: &Database, path: &Path, parent: NodeID) -> Result<()> { .to_string_lossy() .to_string(); - let slug = infojson + let slug = m + .infojson .as_ref() .map(|ij| format!("youtube-{}", ij.id)) .unwrap_or(make_kebab(&filepath_stem)); @@ -282,13 +221,13 @@ fn import_media_file(db: &Database, path: &Path, parent: NodeID) -> Result<()> { db.update_node_init(NodeID::from_slug(&slug), |node| { node.slug = slug; node.title = info.title; - node.poster = cover; + node.poster = m.cover.clone(); node.description = tags.remove("DESCRIPTION"); node.tagline = tags.remove("COMMENT"); if !node.parents.contains(&parent) { node.parents.push(parent) } - if let Some(infojson) = infojson { + if let Some(infojson) = m.infojson { node.kind = Some( if infojson.duration.unwrap_or(0.) < 600. && infojson.aspect_ratio.unwrap_or(2.) < 1. @@ -314,7 +253,9 @@ fn import_media_file(db: &Database, path: &Path, parent: NodeID) -> Result<()> { } } node.media = Some(MediaInfo { - chapters: chapters + chapters: m + .chapters + .clone() .map(|c| { let mut chaps = Vec::new(); if let Some(ee) = c.edition_entries.first() { -- cgit v1.2.3-70-g09d2