music and proper parent slug

2 files changed, 136 insertions, 88 deletions

diff --git a/import/src/infojson.rs b/import/src/infojson.rs
index 69c28ca..50ec66d 100644
--- a/import/src/infojson.rs
+++ b/import/src/infojson.rs
@@ -14,17 +14,17 @@ pub struct YVideo {
     pub id: String,
     pub title: String,
     pub formats: Option<Vec<YFormat>>,
-    pub thumbnails: Vec<YThumbnail>,
+    pub thumbnails: Option<Vec<YThumbnail>>,
     pub thumbnail: Option<String>,
-    pub description: String,
-    pub channel_id: String,
+    pub description: Option<String>,
+    pub channel_id: Option<String>,
     pub duration: Option<f64>,
     pub view_count: Option<usize>,
     pub average_rating: Option<String>,
     pub age_limit: Option<usize>,
     pub webpage_url: String,
     pub categories: Option<Vec<String>>,
-    pub tags: Vec<String>,
+    pub tags: Option<Vec<String>>,
     pub playable_in_embed: Option<bool>,
     pub aspect_ratio: Option<f32>,
     pub width: Option<i32>,
diff --git a/import/src/lib.rs b/import/src/lib.rs
index 4203ba1..6eceb31 100644
--- a/import/src/lib.rs
+++ b/import/src/lib.rs
@@ -68,7 +68,7 @@ pub async fn import_wrap(db: Database, incremental: bool) -> Result<()> {
 }
 
 fn import(db: &Database, incremental: bool) -> Result<()> {
-    let mut queue_prev = vec![CONF.media_path.clone()];
+    let mut queue_prev = vec![(CONF.media_path.clone(), vec![])];
     let mut queue_next;
 
     let apis = Apis {
@@ -80,26 +80,41 @@ fn import(db: &Database, incremental: bool) -> Result<()> {
     while !queue_prev.is_empty() {
         queue_next = queue_prev
             .par_drain(..)
-            .flat_map_iter(
-                move |path| match import_iter_inner(&path, db, incremental) {
+            .flat_map_iter(move |(path, slugs)| {
+                match import_iter_inner(&path, db, slugs, incremental) {
                     Ok(ch) => ch,
                     Err(e) => {
                         IMPORT_ERRORS.blocking_write().push(format!("{e:#}"));
                         Vec::new()
                     }
-                },
-            )
+                }
+            })
             .collect::<Vec<_>>();
         swap(&mut queue_next, &mut queue_prev);
     }
     Ok(())
 }
 
-fn import_iter_inner(path: &Path, db: &Database, incremental: bool) -> Result<Vec<PathBuf>> {
+fn import_iter_inner(
+    path: &Path,
+    db: &Database,
+    mut slugs: Vec<String>,
+    incremental: bool,
+) -> Result<Vec<(PathBuf, Vec<String>)>> {
     if path.is_dir() {
         let mut o = Vec::new();
+        let child_slug = if path == CONF.media_path {
+            "library".to_string()
+        } else {
+            path.file_name()
+                .ok_or(anyhow!("parent no filename"))?
+                .to_string_lossy()
+                .to_string()
+        };
+        slugs.push(child_slug);
         for e in path.read_dir()? {
-            o.push(e?.path());
+            let path = e?.path();
+            o.push((path, slugs.clone()));
         }
         return Ok(o);
     }
@@ -115,46 +130,61 @@ fn import_iter_inner(path: &Path, db: &Database, incremental: bool) -> Result<Ve
             }
         }
 
-        import_file(&db, &path).context(anyhow!("{path:?}"))?;
+        let (slug, parent_slug) = if slugs.len() > 2 {
+            (
+                format!("{}-{}", slugs[slugs.len() - 2], slugs[slugs.len() - 1]),
+                Some(format!(
+                    "{}-{}",
+                    slugs[slugs.len() - 3],
+                    slugs[slugs.len() - 2]
+                )),
+            )
+        } else if slugs.len() > 1 {
+            (
+                format!("{}-{}", slugs[slugs.len() - 2], slugs[slugs.len() - 1]),
+                Some(slugs[slugs.len() - 2].to_string()),
+            )
+        } else {
+            (slugs[0].to_string(), None)
+        };
+
+        import_file(&db, &path, slug, parent_slug).context(anyhow!("{path:?}"))?;
         db.set_import_file_mtime(&path, mtime)?;
     }
     return Ok(Vec::new());
 }
 
-fn import_file(db: &Database, path: &Path) -> Result<()> {
-    let parent_slug = if path == CONF.media_path {
-        "library".to_string()
-    } else {
-        path.parent()
-            .ok_or(anyhow!("no parent"))?
-            .file_name()
-            .ok_or(anyhow!("parent no filename"))?
-            .to_string_lossy()
-            .to_string()
-    };
-    let parent = NodeID::from_slug(&parent_slug);
+fn import_file(
+    db: &Database,
+    path: &Path,
+    slug: String,
+    parent_slug: Option<String>,
+) -> Result<()> {
+    let id = NodeID::from_slug(&slug);
+    let parent_id = parent_slug.map(|e| NodeID::from_slug(&e));
 
     let filename = path.file_name().unwrap().to_string_lossy();
     match filename.as_ref() {
         "poster.jpeg" | "poster.webp" | "poster.png" => {
-            db.update_node_init(parent, |node| {
-                node.slug = parent_slug.to_string();
+            db.update_node_init(id, |node| {
+                node.slug = slug.to_string();
                 node.poster = Some(AssetInner::Media(path.to_owned()).ser());
                 Ok(())
             })?;
         }
-        "backdrop.jpeg" | "backdrop.webp" | "backdrop#.png" => {
-            db.update_node_init(parent, |node| {
-                node.slug = parent_slug.to_string();
+        "backdrop.jpeg" | "backdrop.webp" | "backdrop.png" => {
+            db.update_node_init(id, |node| {
+                node.slug = slug.to_string();
                 node.backdrop = Some(AssetInner::Media(path.to_owned()).ser());
                 Ok(())
             })?;
         }
-        "node.json" | "node.yaml" => {
-            let raw = format!("slug: {parent_slug}\n{}", read_to_string(path)?);
+        "node.yaml" => {
+            let raw = format!("slug: {slug}\n{}", read_to_string(path)?);
             let data = serde_yaml::from_str::<Node>(&raw)?;
-            db.update_node_init(parent, |node| {
-                node.slug = parent_slug.to_string();
+            db.update_node_init(id, |node| {
+                node.parents.extend(parent_id);
+                node.slug = slug.to_string();
                 fn merge_option<T>(a: &mut Option<T>, b: Option<T>) {
                     if b.is_some() {
                         *a = b;
@@ -173,22 +203,25 @@ fn import_file(db: &Database, path: &Path) -> Result<()> {
         }
         "channel.info.json" => {
             let data = serde_json::from_reader::<_, YVideo>(BufReader::new(File::open(path)?))?;
-            db.update_node_init(parent, |node| {
+            db.update_node_init(id, |node| {
+                node.parents.extend(parent_id);
                 node.kind = Some(NodeKind::Channel);
-                node.slug = parent_slug.to_string();
-                node.title = Some(
-                    data.title
-                        .strip_suffix(" - Videos")
-                        .unwrap_or(&data.title)
-                        .to_owned(),
-                );
-                node.external_ids
-                    .insert("youtube:channel".to_string(), data.channel_id);
+                node.slug = slug.to_string();
+                let mut title = data.title.as_str();
+                title = title.strip_suffix(" - Videos").unwrap_or(title);
+                title = title.strip_suffix(" - Topic").unwrap_or(title);
+                title = title.strip_prefix("Uploads from ").unwrap_or(title);
+                node.title = Some(title.to_owned());
+                if let Some(cid) = data.channel_id {
+                    node.external_ids.insert("youtube:channel".to_string(), cid);
+                }
                 if let Some(uid) = data.uploader_id {
                     node.external_ids
                         .insert("youtube:channel-name".to_string(), uid);
                 }
-                node.description = Some(data.description);
+                if let Some(desc) = data.description {
+                    node.description = Some(desc);
+                }
                 if let Some(followers) = data.channel_follower_count {
                     node.ratings
                         .insert(Rating::YoutubeFollowers, followers as f64);
@@ -196,11 +229,9 @@ fn import_file(db: &Database, path: &Path) -> Result<()> {
                 Ok(())
             })?;
         }
-        _ => (),
+        _ => import_media_file(db, path, id).context("media file")?,
     }
 
-    import_media_file(db, path, parent).context("media file")?;
-
     Ok(())
 }
 
@@ -241,12 +272,51 @@ fn import_media_file(db: &Database, path: &Path, parent: NodeID) -> Result<()> {
         node.poster = m.cover.clone();
         node.description = tags.remove("DESCRIPTION");
         node.tagline = tags.remove("COMMENT");
-        if !node.parents.contains(&parent) {
-            node.parents.push(parent)
-        }
+        node.parents.insert(parent);
+
+        let tracks = tracks
+            .entries
+            .into_iter()
+            .map(|track| SourceTrack {
+                codec: track.codec_id,
+                language: track.language,
+                name: track.name.unwrap_or_default(),
+                default_duration: track.default_duration,
+                federated: Vec::new(),
+                kind: if let Some(video) = track.video {
+                    SourceTrackKind::Video {
+                        width: video.pixel_width,
+                        height: video.pixel_height,
+                        display_width: video.display_width,
+                        display_height: video.display_height,
+                        display_unit: Some(video.display_unit),
+                        fps: video.frame_rate,
+                    }
+                } else if let Some(audio) = track.audio {
+                    SourceTrackKind::Audio {
+                        channels: audio.channels as usize,
+                        sample_rate: audio.sampling_frequency,
+                        bit_depth: audio.bit_depth.map(|r| r as usize),
+                    }
+                } else {
+                    SourceTrackKind::Subtitles
+                },
+                source: TrackSource::Local(LocalTrack {
+                    codec_private: track.codec_private,
+                    path: path.to_owned(),
+                    track: track.track_number as usize,
+                }),
+            })
+            .collect::<Vec<_>>();
+
         if let Some(infojson) = m.infojson {
             node.kind = Some(
-                if infojson.duration.unwrap_or(0.) < 600.
+                if !tracks
+                    .iter()
+                    .any(|t| matches!(t.kind, SourceTrackKind::Video { .. }))
+                {
+                    NodeKind::Music
+                } else if infojson.duration.unwrap_or(0.) < 600.
                     && infojson.aspect_ratio.unwrap_or(2.) < 1.
                 {
                     NodeKind::ShortFormVideo
@@ -255,14 +325,25 @@ fn import_media_file(db: &Database, path: &Path, parent: NodeID) -> Result<()> {
                 },
             );
             node.title = Some(infojson.title);
-            node.description = Some(infojson.description);
+            if let Some(desc) = infojson.description {
+                node.description = Some(desc)
+            }
             node.tagline = Some(infojson.webpage_url);
             if let Some(date) = &infojson.upload_date {
                 node.release_date =
                     Some(infojson::parse_upload_date(date).context("parsing upload date")?);
             }
-            node.external_ids
-                .insert("youtube:video".to_string(), infojson.id);
+            match infojson.extractor.as_str() {
+                "youtube" => drop(
+                    node.external_ids
+                        .insert("youtube:video".to_string(), infojson.id),
+                ),
+                "Bandcamp" => drop(
+                    node.external_ids
+                        .insert("bandcamp".to_string(), infojson.id),
+                ),
+                _ => (),
+            }
             node.ratings.insert(
                 Rating::YoutubeViews,
                 infojson.view_count.unwrap_or_default() as f64,
@@ -296,40 +377,7 @@ fn import_media_file(db: &Database, path: &Path, parent: NodeID) -> Result<()> {
                 })
                 .unwrap_or_default(),
             duration: (info.duration.unwrap_or_default() * info.timestamp_scale as f64) * 1e-9,
-            tracks: tracks
-                .entries
-                .into_iter()
-                .map(|track| SourceTrack {
-                    codec: track.codec_id,
-                    language: track.language,
-                    name: track.name.unwrap_or_default(),
-                    default_duration: track.default_duration,
-                    federated: Vec::new(),
-                    kind: if let Some(video) = track.video {
-                        SourceTrackKind::Video {
-                            width: video.pixel_width,
-                            height: video.pixel_height,
-                            display_width: video.display_width,
-                            display_height: video.display_height,
-                            display_unit: Some(video.display_unit),
-                            fps: video.frame_rate,
-                        }
-                    } else if let Some(audio) = track.audio {
-                        SourceTrackKind::Audio {
-                            channels: audio.channels as usize,
-                            sample_rate: audio.sampling_frequency,
-                            bit_depth: audio.bit_depth.map(|r| r as usize),
-                        }
-                    } else {
-                        SourceTrackKind::Subtitles
-                    },
-                    source: TrackSource::Local(LocalTrack {
-                        codec_private: track.codec_private,
-                        path: path.to_owned(),
-                        track: track.track_number as usize,
-                    }),
-                })
-                .collect(),
+            tracks,
         });
 
         Ok(())