aboutsummaryrefslogtreecommitdiff
path: root/import/src
diff options
context:
space:
mode:
Diffstat (limited to 'import/src')
-rw-r--r--import/src/lib.rs200
1 files changed, 95 insertions, 105 deletions
diff --git a/import/src/lib.rs b/import/src/lib.rs
index 4be2151..c70d357 100644
--- a/import/src/lib.rs
+++ b/import/src/lib.rs
@@ -14,14 +14,15 @@ use jellybase::{
database::Database,
CONF, SECRETS,
};
+use jellyclient::Visibility;
+use log::warn;
use matroska::matroska_metadata;
-use rayon::iter::{ParallelDrainRange, ParallelIterator};
+use rayon::iter::{ParallelBridge, ParallelIterator};
use std::{
collections::HashMap,
fs::{read_to_string, File},
io::BufReader,
- mem::swap,
- path::{Path, PathBuf},
+ path::Path,
sync::LazyLock,
time::UNIX_EPOCH,
};
@@ -68,55 +69,71 @@ pub async fn import_wrap(db: Database, incremental: bool) -> Result<()> {
}
fn import(db: &Database, incremental: bool) -> Result<()> {
- let mut queue_prev = vec![(CONF.media_path.clone(), vec![])];
- let mut queue_next;
-
let apis = Apis {
trakt: SECRETS.api.trakt.as_ref().map(|key| Trakt::new(key)),
tmdb: SECRETS.api.tmdb.as_ref().map(|key| Tmdb::new(key)),
};
drop((apis.tmdb, apis.trakt));
- while !queue_prev.is_empty() {
- queue_next = queue_prev
- .par_drain(..)
- .flat_map_iter(move |(path, slugs)| {
- match import_iter_inner(&path, db, slugs, incremental) {
- Ok(ch) => ch,
- Err(e) => {
- IMPORT_ERRORS.blocking_write().push(format!("{e:#}"));
- Vec::new()
- }
- }
- })
- .collect::<Vec<_>>();
- swap(&mut queue_next, &mut queue_prev);
- }
+ import_traverse(
+ &CONF.media_path,
+ db,
+ incremental,
+ NodeID::MIN,
+ "",
+ Visibility::Visible,
+ )?;
+
Ok(())
}
-fn import_iter_inner(
+fn import_traverse(
path: &Path,
db: &Database,
- mut slugs: Vec<String>,
incremental: bool,
-) -> Result<Vec<(PathBuf, Vec<String>)>> {
+ parent: NodeID,
+ parent_slug_fragment: &str,
+ mut visibility: Visibility,
+) -> Result<()> {
if path.is_dir() {
- let mut o = Vec::new();
- let child_slug = if path == CONF.media_path {
+ let slug_fragment = if path == CONF.media_path {
"library".to_string()
} else {
- path.file_name()
- .ok_or(anyhow!("parent no filename"))?
- .to_string_lossy()
- .to_string()
+ path.file_name().unwrap().to_string_lossy().to_string()
};
- slugs.push(child_slug);
- for e in path.read_dir()? {
- let path = e?.path();
- o.push((path, slugs.clone()));
+ let slug = if parent_slug_fragment.is_empty() {
+ slug_fragment.clone()
+ } else {
+ format!("{parent_slug_fragment}-{slug_fragment}")
+ };
+ let id = NodeID::from_slug(&slug);
+
+ if let Ok(content) = read_to_string(path.join("flags")) {
+ for flag in content.lines() {
+ match flag.trim() {
+ "hidden" => visibility = visibility.min(Visibility::Hidden),
+ "reduced" => visibility = visibility.min(Visibility::Reduced),
+ _ => warn!("unknown flag {flag:?}"),
+ }
+ }
}
- return Ok(o);
+
+ db.update_node_init(id, |n| {
+ n.parents.insert(parent);
+ n.slug = slug;
+ n.visibility = visibility;
+ Ok(())
+ })?;
+
+ path.read_dir()?.par_bridge().try_for_each(|e| {
+ let path = e?.path();
+ if let Err(e) = import_traverse(&path, db, incremental, id, &slug_fragment, visibility)
+ {
+ IMPORT_ERRORS.blocking_write().push(format!("{e:#}"));
+ }
+ Ok::<_, anyhow::Error>(())
+ })?;
+ return Ok(());
}
if path.is_file() {
let meta = path.metadata()?;
@@ -125,72 +142,43 @@ fn import_iter_inner(
if incremental {
if let Some(last_mtime) = db.get_import_file_mtime(&path)? {
if last_mtime >= mtime {
- return Ok(Vec::new());
+ return Ok(());
}
}
}
- let (slug, parent_slug) = if slugs.len() > 2 {
- (
- format!("{}-{}", slugs[slugs.len() - 2], slugs[slugs.len() - 1]),
- Some(format!(
- "{}-{}",
- slugs[slugs.len() - 3],
- slugs[slugs.len() - 2]
- )),
- )
- } else if slugs.len() > 1 {
- (
- format!("{}-{}", slugs[slugs.len() - 2], slugs[slugs.len() - 1]),
- Some(slugs[slugs.len() - 2].to_string()),
- )
- } else {
- (slugs[0].to_string(), None)
- };
-
- import_file(&db, &path, slug, parent_slug).context(anyhow!("{path:?}"))?;
+ import_file(&db, &path, parent, visibility).context(anyhow!("{path:?}"))?;
db.set_import_file_mtime(&path, mtime)?;
}
- return Ok(Vec::new());
+ return Ok(());
}
-fn import_file(
- db: &Database,
- path: &Path,
- slug: String,
- parent_slug: Option<String>,
-) -> Result<()> {
- let id = NodeID::from_slug(&slug);
- let parent_id = parent_slug.map(|e| NodeID::from_slug(&e));
-
+fn import_file(db: &Database, path: &Path, parent: NodeID, visibility: Visibility) -> Result<()> {
let filename = path.file_name().unwrap().to_string_lossy();
match filename.as_ref() {
"poster.jpeg" | "poster.webp" | "poster.png" => {
- db.update_node_init(id, |node| {
- node.slug = slug.to_string();
+ db.update_node_init(parent, |node| {
node.poster = Some(AssetInner::Media(path.to_owned()).ser());
Ok(())
})?;
}
"backdrop.jpeg" | "backdrop.webp" | "backdrop.png" => {
- db.update_node_init(id, |node| {
- node.slug = slug.to_string();
+ db.update_node_init(parent, |node| {
node.backdrop = Some(AssetInner::Media(path.to_owned()).ser());
Ok(())
})?;
}
"node.yaml" => {
- let raw = format!("slug: {slug}\n{}", read_to_string(path)?);
- let data = serde_yaml::from_str::<Node>(&raw)?;
- db.update_node_init(id, |node| {
- node.parents.extend(parent_id);
- node.slug = slug.to_string();
+ let data = serde_yaml::from_str::<Node>(&read_to_string(path)?)?;
+ db.update_node_init(parent, |node| {
fn merge_option<T>(a: &mut Option<T>, b: Option<T>) {
if b.is_some() {
*a = b;
}
}
- merge_option(&mut node.kind, data.kind);
+ if data.kind != NodeKind::Unknown {
+ node.kind = data.kind;
+ }
merge_option(&mut node.title, data.title);
merge_option(&mut node.tagline, data.tagline);
merge_option(&mut node.description, data.description);
@@ -203,10 +191,8 @@ fn import_file(
}
"channel.info.json" => {
let data = serde_json::from_reader::<_, YVideo>(BufReader::new(File::open(path)?))?;
- db.update_node_init(id, |node| {
- node.parents.extend(parent_id);
- node.kind = Some(NodeKind::Channel);
- node.slug = slug.to_string();
+ db.update_node_init(parent, |node| {
+ node.kind = NodeKind::Channel;
let mut title = data.title.as_str();
title = title.strip_suffix(" - Videos").unwrap_or(title);
title = title.strip_suffix(" - Topic").unwrap_or(title);
@@ -229,13 +215,18 @@ fn import_file(
Ok(())
})?;
}
- _ => import_media_file(db, path, id).context("media file")?,
+ _ => import_media_file(db, path, parent, visibility).context("media file")?,
}
Ok(())
}
-fn import_media_file(db: &Database, path: &Path, parent: NodeID) -> Result<()> {
+fn import_media_file(
+ db: &Database,
+ path: &Path,
+ parent: NodeID,
+ visibility: Visibility,
+) -> Result<()> {
let Some(m) = (*matroska_metadata(path)?).to_owned() else {
return Ok(());
};
@@ -269,6 +260,7 @@ fn import_media_file(db: &Database, path: &Path, parent: NodeID) -> Result<()> {
db.update_node_init(NodeID::from_slug(&slug), |node| {
node.slug = slug;
node.title = info.title;
+ node.visibility = visibility;
node.poster = m.cover.clone();
node.description = tags.remove("DESCRIPTION");
node.tagline = tags.remove("COMMENT");
@@ -312,20 +304,18 @@ fn import_media_file(db: &Database, path: &Path, parent: NodeID) -> Result<()> {
.collect::<Vec<_>>();
if let Some(infojson) = m.infojson {
- node.kind = Some(
- if !tracks
- .iter()
- .any(|t| matches!(t.kind, SourceTrackKind::Video { .. }))
- {
- NodeKind::Music
- } else if infojson.duration.unwrap_or(0.) < 600.
- && infojson.aspect_ratio.unwrap_or(2.) < 1.
- {
- NodeKind::ShortFormVideo
- } else {
- NodeKind::Video
- },
- );
+ node.kind = if !tracks
+ .iter()
+ .any(|t| matches!(t.kind, SourceTrackKind::Video { .. }))
+ {
+ NodeKind::Music
+ } else if infojson.duration.unwrap_or(0.) < 600.
+ && infojson.aspect_ratio.unwrap_or(2.) < 1.
+ {
+ NodeKind::ShortFormVideo
+ } else {
+ NodeKind::Video
+ };
node.title = Some(infojson.title);
if let Some(desc) = infojson.description {
node.description = Some(desc)
@@ -336,23 +326,23 @@ fn import_media_file(db: &Database, path: &Path, parent: NodeID) -> Result<()> {
Some(infojson::parse_upload_date(date).context("parsing upload date")?);
}
match infojson.extractor.as_str() {
- "youtube" => drop(
+ "youtube" => {
node.external_ids
- .insert("youtube:video".to_string(), infojson.id),
- ),
+ .insert("youtube:video".to_string(), infojson.id);
+ node.ratings.insert(
+ Rating::YoutubeViews,
+ infojson.view_count.unwrap_or_default() as f64,
+ );
+ if let Some(lc) = infojson.like_count {
+ node.ratings.insert(Rating::YoutubeLikes, lc as f64);
+ }
+ }
"Bandcamp" => drop(
node.external_ids
.insert("bandcamp".to_string(), infojson.id),
),
_ => (),
}
- node.ratings.insert(
- Rating::YoutubeViews,
- infojson.view_count.unwrap_or_default() as f64,
- );
- if let Some(lc) = infojson.like_count {
- node.ratings.insert(Rating::YoutubeLikes, lc as f64);
- }
}
node.media = Some(MediaInfo {
chapters: m
generated by cgit v1.2.3-70-g09d2 (git 2.51.0) at 2025-10-08 09:31:59 +0000