/* This file is part of jellything (https://codeberg.org/metamuffin/jellything) which is licensed under the GNU Affero General Public License (version 3); see /COPYING. Copyright (C) 2025 metamuffin */ use anyhow::{anyhow, Context, Result}; use infojson::YVideo; use jellybase::{ assetfed::AssetInner, common::{ Chapter, LocalTrack, MediaInfo, Node, NodeID, NodeKind, Rating, SourceTrack, SourceTrackKind, TrackSource, }, database::Database, CONF, SECRETS, }; use jellyclient::{Appearance, PeopleGroup, TmdbKind, TraktKind, Visibility}; use log::{info, warn}; use matroska::matroska_metadata; use rayon::iter::{ParallelBridge, ParallelIterator}; use std::{ collections::{BTreeMap, HashMap}, fs::{read_to_string, File}, io::BufReader, path::Path, sync::LazyLock, time::UNIX_EPOCH, }; use tmdb::Tmdb; use tokio::{ runtime::Handle, sync::{RwLock, Semaphore}, task::spawn_blocking, }; use trakt::Trakt; pub mod infojson; pub mod matroska; pub mod tmdb; pub mod trakt; static IMPORT_SEM: LazyLock = LazyLock::new(|| Semaphore::new(1)); pub static IMPORT_ERRORS: RwLock> = RwLock::const_new(Vec::new()); // static RE_EPISODE_FILENAME: LazyLock = // LazyLock::new(|| Regex::new(r#"([sS](\d+))?([eE](\d+))( (.+))?"#).unwrap()); struct Apis { trakt: Option, tmdb: Option, } pub fn is_importing() -> bool { IMPORT_SEM.available_permits() == 0 } pub async fn import_wrap(db: Database, incremental: bool) -> Result<()> { let _sem = IMPORT_SEM.try_acquire()?; let jh = spawn_blocking(move || { *IMPORT_ERRORS.blocking_write() = Vec::new(); if let Err(e) = import(&db, incremental) { IMPORT_ERRORS.blocking_write().push(format!("{e:#}")); } }); let _ = jh.await; Ok(()) } fn import(db: &Database, incremental: bool) -> Result<()> { let apis = Apis { trakt: SECRETS.api.trakt.as_ref().map(|key| Trakt::new(key)), tmdb: SECRETS.api.tmdb.as_ref().map(|key| Tmdb::new(key)), }; let rthandle = Handle::current(); import_traverse( &CONF.media_path, db, &apis, &rthandle, incremental, NodeID::MIN, "", Visibility::Visible, )?; Ok(()) } fn import_traverse( path: &Path, db: &Database, apis: &Apis, rthandle: &Handle, incremental: bool, parent: NodeID, parent_slug_fragment: &str, mut visibility: Visibility, ) -> Result<()> { if path.is_dir() { let slug_fragment = if path == CONF.media_path { "library".to_string() } else { path.file_name().unwrap().to_string_lossy().to_string() }; let slug = if parent_slug_fragment.is_empty() { slug_fragment.clone() } else { format!("{parent_slug_fragment}-{slug_fragment}") }; let id = NodeID::from_slug(&slug); if let Ok(content) = read_to_string(path.join("flags")) { for flag in content.lines() { match flag.trim() { "hidden" => visibility = visibility.min(Visibility::Hidden), "reduced" => visibility = visibility.min(Visibility::Reduced), _ => warn!("unknown flag {flag:?}"), } } } db.update_node_init(id, |n| { n.parents.insert(parent); n.slug = slug; n.visibility = visibility; Ok(()) })?; path.read_dir()?.par_bridge().try_for_each(|e| { let path = e?.path(); if let Err(e) = import_traverse( &path, db, apis, rthandle, incremental, id, &slug_fragment, visibility, ) { IMPORT_ERRORS.blocking_write().push(format!("{e:#}")); } Ok::<_, anyhow::Error>(()) })?; return Ok(()); } if path.is_file() { let meta = path.metadata()?; let mtime = meta.modified()?.duration_since(UNIX_EPOCH)?.as_secs(); if incremental { if let Some(last_mtime) = db.get_import_file_mtime(&path)? { if last_mtime >= mtime { return Ok(()); } } } import_file(&db, apis, rthandle, &path, parent, visibility).context(anyhow!("{path:?}"))?; db.set_import_file_mtime(&path, mtime)?; } return Ok(()); } fn import_file( db: &Database, apis: &Apis, rthandle: &Handle, path: &Path, parent: NodeID, visibility: Visibility, ) -> Result<()> { let filename = path.file_name().unwrap().to_string_lossy(); match filename.as_ref() { "poster.jpeg" | "poster.webp" | "poster.png" => { info!("import poster at {path:?}"); db.update_node_init(parent, |node| { node.poster = Some(AssetInner::Media(path.to_owned()).ser()); Ok(()) })?; } "backdrop.jpeg" | "backdrop.webp" | "backdrop.png" => { info!("import backdrop at {path:?}"); db.update_node_init(parent, |node| { node.backdrop = Some(AssetInner::Media(path.to_owned()).ser()); Ok(()) })?; } "node.yaml" => { info!("import node info at {path:?}"); let data = serde_yaml::from_str::(&read_to_string(path)?)?; db.update_node_init(parent, |node| { fn merge_option(a: &mut Option, b: Option) { if b.is_some() { *a = b; } } if data.kind != NodeKind::Unknown { node.kind = data.kind; } merge_option(&mut node.title, data.title); merge_option(&mut node.tagline, data.tagline); merge_option(&mut node.description, data.description); merge_option(&mut node.index, data.index); merge_option(&mut node.release_date, data.release_date); node.external_ids.extend(data.external_ids); Ok(()) })?; } "children" => { info!("import children at {path:?}"); for line in read_to_string(path)?.lines() { let line = line.trim(); if line.starts_with("#") || line.is_empty() { continue; } db.update_node_init(NodeID::from_slug(line), |n| { n.slug = line.to_owned(); n.parents.insert(parent); Ok(()) })?; } } "channel.info.json" => { info!("import channel info.json at {path:?}"); let data = serde_json::from_reader::<_, YVideo>(BufReader::new(File::open(path)?))?; db.update_node_init(parent, |node| { node.kind = NodeKind::Channel; let mut title = data.title.as_str(); title = title.strip_suffix(" - Videos").unwrap_or(title); title = title.strip_suffix(" - Topic").unwrap_or(title); title = title.strip_prefix("Uploads from ").unwrap_or(title); node.title = Some(title.to_owned()); if let Some(cid) = data.channel_id { node.external_ids.insert("youtube:channel".to_string(), cid); } if let Some(uid) = data.uploader_id { node.external_ids .insert("youtube:channel-name".to_string(), uid); } if let Some(desc) = data.description { node.description = Some(desc); } if let Some(followers) = data.channel_follower_count { node.ratings .insert(Rating::YoutubeFollowers, followers as f64); } Ok(()) })?; } _ => { import_media_file(db, apis, rthandle, path, parent, visibility).context("media file")? } } Ok(()) } fn import_media_file( db: &Database, apis: &Apis, rthandle: &Handle, path: &Path, parent: NodeID, visibility: Visibility, ) -> Result<()> { info!("media file {path:?}"); let Some(m) = (*matroska_metadata(path)?).to_owned() else { return Ok(()); }; let info = m.info.ok_or(anyhow!("no info"))?; let tracks = m.tracks.ok_or(anyhow!("no tracks"))?; let mut tags = m .tags .map(|tags| { tags.tags .into_iter() .flat_map(|t| t.simple_tags) .map(|st| (st.name, st.string.unwrap_or_default())) .collect::>() }) .unwrap_or_default(); let filename = path .file_name() .ok_or(anyhow!("no file stem"))? .to_string_lossy() .to_string(); let mut backdrop = None; let mut poster = None; let mut trakt_data = None; let mut tmdb_data = None; let mut filename_toks = filename.split("."); let filepath_stem = filename_toks.next().unwrap(); for tok in filename_toks { if let Some(trakt_id) = tok.strip_prefix("trakt-") { let trakt_id: u64 = trakt_id.parse().context("parse trakt id")?; if let (Some(trakt), Some(tmdb)) = (&apis.trakt, &apis.tmdb) { let data = rthandle .block_on(trakt.lookup(TraktKind::Movie, trakt_id, true)) .context("trakt lookup")?; let people = rthandle .block_on(trakt.people(TraktKind::Movie, trakt_id, true)) .context("trakt people lookup")?; let mut people_map = BTreeMap::>::new(); for p in people.cast.iter() { people_map.entry(PeopleGroup::Cast).or_default().push(p.a()) } for (group, people) in people.crew.iter() { for p in people { people_map.entry(group.a()).or_default().push(p.a()) } } if let Some(tmdb_id) = data.ids.tmdb { let data = rthandle .block_on(tmdb.details(TmdbKind::Movie, tmdb_id)) .context("tmdb details")?; tmdb_data = Some(data.clone()); if let Some(path) = &data.backdrop_path { let im = rthandle .block_on(tmdb.image(path)) .context("tmdb backdrop image")?; backdrop = Some(AssetInner::Cache(im).ser()); } if let Some(path) = &data.poster_path { let im = rthandle .block_on(tmdb.image(path)) .context("tmdb poster image")?; poster = Some(AssetInner::Cache(im).ser()); } for p in people_map.values_mut().flatten() { if let Some(id) = p.person.ids.tmdb { let k = rthandle.block_on(tmdb.person_image(id))?; if let Some(prof) = k.profiles.first() { let im = rthandle.block_on(tmdb.image(&prof.file_path))?; p.person.headshot = Some(AssetInner::Cache(im).ser()); } } } } trakt_data = Some((data.clone(), people_map)); } } } let slug = m .infojson .as_ref() .map(|ij| format!("youtube-{}", ij.id)) .unwrap_or(make_kebab(&filepath_stem)); db.update_node_init(NodeID::from_slug(&slug), |node| { node.slug = slug; node.title = info.title; node.visibility = visibility; node.poster = m.cover.clone().or(poster); node.backdrop = backdrop; node.description = tags.remove("DESCRIPTION").or(tags.remove("SYNOPSIS")); node.tagline = tags.remove("COMMENT"); node.parents.insert(parent); if let Some(ct) = tags.get("CONTENT_TYPE") { node.kind = match ct.to_lowercase().trim() { "movie" | "documentary" | "film" => NodeKind::Movie, "music" | "recording" => NodeKind::Music, _ => NodeKind::Unknown, } } if let Some(data) = tmdb_data { node.title = data.title.clone(); node.tagline = data.tagline.clone(); node.description = Some(data.overview.clone()); node.ratings.insert(Rating::Tmdb, data.vote_average); if let Some(date) = data.release_date.clone() { node.release_date = tmdb::parse_release_date(&date)?; } } if let Some((data, people)) = trakt_data { node.title = Some(data.title.clone()); node.kind = NodeKind::Movie; // TODO if let Some(overview) = &data.overview { node.description = Some(overview.clone()) } if let Some(tagline) = &data.tagline { node.tagline = Some(tagline.clone()) } if let Some(rating) = &data.rating { node.ratings.insert(Rating::Trakt, *rating); } node.people.extend(people); } let tracks = tracks .entries .into_iter() .map(|track| SourceTrack { codec: track.codec_id, language: track.language, name: track.name.unwrap_or_default(), default_duration: track.default_duration, federated: Vec::new(), codec_delay: track.codec_delay, seek_pre_roll: track.seek_pre_roll, kind: if let Some(video) = track.video { SourceTrackKind::Video { width: video.pixel_width, height: video.pixel_height, display_width: video.display_width, display_height: video.display_height, display_unit: video.display_unit, fps: video.frame_rate, } } else if let Some(audio) = track.audio { SourceTrackKind::Audio { channels: audio.channels as usize, sample_rate: audio.sampling_frequency, bit_depth: audio.bit_depth.map(|r| r as usize), } } else { SourceTrackKind::Subtitles }, source: TrackSource::Local(LocalTrack { codec_private: track.codec_private, path: path.to_owned(), track: track.track_number as usize, }), }) .collect::>(); if let Some(infojson) = m.infojson { node.kind = if !tracks .iter() .any(|t| matches!(t.kind, SourceTrackKind::Video { .. })) { NodeKind::Music } else if infojson.duration.unwrap_or(0.) < 600. && infojson.aspect_ratio.unwrap_or(2.) < 1. { NodeKind::ShortFormVideo } else { NodeKind::Video }; node.title = Some(infojson.title); if let Some(desc) = infojson.description { node.description = Some(desc) } node.tagline = Some(infojson.webpage_url); if let Some(date) = &infojson.upload_date { node.release_date = Some(infojson::parse_upload_date(date).context("parsing upload date")?); } match infojson.extractor.as_str() { "youtube" => { node.external_ids .insert("youtube:video".to_string(), infojson.id); node.ratings.insert( Rating::YoutubeViews, infojson.view_count.unwrap_or_default() as f64, ); if let Some(lc) = infojson.like_count { node.ratings.insert(Rating::YoutubeLikes, lc as f64); } } "Bandcamp" => drop( node.external_ids .insert("bandcamp".to_string(), infojson.id), ), _ => (), } } node.media = Some(MediaInfo { chapters: m .chapters .clone() .map(|c| { let mut chaps = Vec::new(); if let Some(ee) = c.edition_entries.first() { for ca in &ee.chapter_atoms { let mut labels = Vec::new(); for cd in &ca.displays { for lang in &cd.languages { labels.push((lang.to_owned(), cd.string.clone())) } } chaps.push(Chapter { labels, time_start: Some(ca.time_start as f64 * 1e-9), time_end: ca.time_end.map(|ts| ts as f64 * 1e-9), }) } } chaps }) .unwrap_or_default(), duration: (info.duration.unwrap_or_default() * info.timestamp_scale as f64) * 1e-9, tracks, }); Ok(()) })?; Ok(()) } fn make_kebab(i: &str) -> String { let mut o = String::with_capacity(i.len()); for c in i.chars() { o.extend(match c { 'A'..='Z' | 'a'..='z' | '0'..='9' | '_' | '-' => Some(c), ' ' => Some('-'), _ => None, }); } o }