rework import system pt. 7: cache tmdb, media ignore flags

7 files changed, 118 insertions, 78 deletions

diff --git a/Cargo.lock b/Cargo.lock
index 5e752ca..bde53de 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1387,6 +1387,7 @@ version = "0.1.0"
 dependencies = [
  "anyhow",
  "async-recursion",
+ "bincode 2.0.0-rc.3",
  "futures",
  "jellybase",
  "jellyclient",
diff --git a/common/src/lib.rs b/common/src/lib.rs
index 73eb860..a5b3f21 100644
--- a/common/src/lib.rs
+++ b/common/src/lib.rs
@@ -71,9 +71,12 @@ pub enum ImportSource {
     },
     Media {
         location: AssetLocation,
-        #[serde(default)] ignore_attachments: bool,
-        #[serde(default)] ignore_metadata: bool,
-        // TODO all ignore options
+        #[serde(default)]
+        ignore_metadata: bool,
+        #[serde(default)]
+        ignore_attachments: bool,
+        #[serde(default)]
+        ignore_chapters: bool,
     },
     Federated {
         host: String,
@@ -109,7 +112,7 @@ pub enum NodeKind {
 #[serde(rename_all = "snake_case")]
 pub enum TrackSource {
     Local(LocalTrack),
-    Remote,
+    Remote(usize),
 }
 
 pub enum PublicMediaSource {
diff --git a/import/Cargo.toml b/import/Cargo.toml
index 91a7662..64aecb2 100644
--- a/import/Cargo.toml
+++ b/import/Cargo.toml
@@ -14,6 +14,7 @@ log = { workspace = true }
 anyhow = "1.0.75"
 reqwest = { version = "0.11.22", features = ["json"] }
 
+bincode = { version = "2.0.0-rc.3", features = ["derive"] }
 serde = { version = "1.0.193", features = ["derive"] }
 serde_json = "1.0.108"
 serde_yaml = "0.9.27"
diff --git a/import/src/lib.rs b/import/src/lib.rs
index 3f14960..b37ba69 100644
--- a/import/src/lib.rs
+++ b/import/src/lib.rs
@@ -9,16 +9,16 @@ pub mod tmdb;
 
 use anyhow::{anyhow, Context, Ok};
 use async_recursion::async_recursion;
-use futures::{stream::FuturesUnordered, StreamExt};
+use futures::{executor::block_on, stream::FuturesUnordered, StreamExt};
 use jellybase::{
-    cache::{async_cache_file, cache_file},
+    cache::{async_cache_file, cache_memory},
     database::Database,
     federation::Federation,
     AssetLocationExt, CONF,
 };
 use jellyclient::Session;
 use jellycommon::{
-    AssetLocation, AssetRole, ImportOptions, ImportSource, MediaInfo, Node, NodePrivate,
+    AssetLocation, AssetRole, ImportOptions, ImportSource, MediaInfo, Node, NodeKind, NodePrivate,
     NodePublic, TrackSource,
 };
 use jellymatroska::read::EbmlReader;
@@ -191,11 +191,17 @@ async fn process_source(
                 .tmdb_api_key
                 .as_ref()
                 .ok_or(anyhow!("no tmdb api key"))?;
-            let details = tmdb::tmdb_details(TmdbKind::Movie, tid, key).await?;
+
+            let details = tokio::task::spawn_blocking(move || {
+                cache_memory(&["tmdb-details", &format!("{tid}")], || {
+                    block_on(tmdb::tmdb_details(TmdbKind::Movie, tid, key))
+                })
+            })
+            .await??;
 
             let mut node = Node::default();
 
-            if let Some(poster) = details.poster_path {
+            if let Some(poster) = &details.poster_path {
                 node.private.poster = Some(
                     async_cache_file(
                         &["tmdb-asset", "poster", &format!("{tid}")],
@@ -204,7 +210,7 @@ async fn process_source(
                     .await?,
                 );
             }
-            if let Some(backdrop) = details.backdrop_path {
+            if let Some(backdrop) = &details.backdrop_path {
                 node.private.backdrop = Some(
                     async_cache_file(
                         &["tmdb-asset", "backdrop", &format!("{tid}")],
@@ -214,13 +220,18 @@ async fn process_source(
                 );
             }
 
-            node.public.tagline = details.tagline;
-            node.public.title = details.title;
-            node.public.description = Some(details.overview);
+            node.public.tagline = details.tagline.clone();
+            node.public.title = details.title.clone();
+            node.public.description = Some(details.overview.clone());
 
             insert_node(&id, node)?;
         }
-        ImportSource::Media { location, .. } => {
+        ImportSource::Media {
+            location,
+            ignore_attachments,
+            ignore_chapters,
+            ignore_metadata,
+        } => {
             // TODO use ignore options
             let media_path = location.path();
 
@@ -232,48 +243,59 @@ async fn process_source(
             })
             .await??;
 
-            let poster = if let Some((filename, data)) = metadata.cover {
-                Some(
-                    async_cache_file(
-                        &[media_path.to_str().unwrap(), &filename],
-                        |mut f| async move {
-                            f.write_all(&data).await?;
-                            Ok(())
-                        },
-                    )
-                    .await?,
-                )
-            } else {
-                None
-            };
+            let mut node = Node::default();
 
-            let node = Node {
-                public: NodePublic {
-                    title: metadata.title,
-                    description: metadata.description,
-                    tagline: metadata.tagline,
-                    media: Some(MediaInfo {
-                        chapters: metadata.chapters,
-                        duration: metadata.duration,
-                        tracks: metadata.tracks,
-                    }),
-                    ..Default::default()
-                },
-                private: NodePrivate {
-                    poster,
-                    source: Some(
-                        metadata
-                            .track_sources
-                            .into_iter()
-                            .map(|mut ts| {
-                                ts.path = media_path.to_owned();
-                                TrackSource::Local(ts)
-                            })
-                            .collect(),
-                    ),
-                    ..Default::default()
+            if !ignore_metadata {
+                node.public.title = metadata.title;
+                node.public.description = metadata.description;
+                node.public.tagline = metadata.tagline;
+            }
+            node.public.media = Some(MediaInfo {
+                duration: metadata.duration,
+                tracks: metadata.tracks,
+                chapters: if ignore_chapters {
+                    vec![]
+                } else {
+                    metadata.chapters
                 },
-            };
+            });
+            node.private.source = Some(
+                metadata
+                    .track_sources
+                    .into_iter()
+                    .map(|mut ts| {
+                        ts.path = media_path.to_owned();
+                        TrackSource::Local(ts)
+                    })
+                    .collect(),
+            );
+
+            if !ignore_attachments {
+                if let Some((filename, data)) = metadata.cover {
+                    node.private.poster = Some(
+                        async_cache_file(
+                            &[media_path.to_str().unwrap(), &filename],
+                            |mut f| async move {
+                                f.write_all(&data).await?;
+                                Ok(())
+                            },
+                        )
+                        .await?,
+                    );
+                };
+
+                if let Some(infojson) = metadata.infojson {
+                    let infojson: infojson::YVideo =
+                        serde_json::from_str(&infojson).context("parsing infojson")?;
+
+                    node.public.kind = Some(NodeKind::Video);
+                    node.public.title = Some(infojson.title);
+                    node.public.description = Some(infojson.description);
+                    node.public.tagline = Some(infojson.webpage_url);
+                    // TODO infojson
+                }
+            }
+
             insert_node(&id, node)?;
         }
         ImportSource::Federated { host } => {
@@ -369,11 +391,21 @@ async fn import_remote(
     let _permit = SEM_REMOTE_IMPORT.acquire().await.unwrap();
     info!("loading federated node {id:?}");
 
-    let node = session.node(&id).await.context("fetching remote node")?;
+    let mut node = session.node(&id).await.context("fetching remote node")?;
 
-    if node.federated.as_ref() == Some(&CONF.hostname) {
-        return Ok(());
-    }
+    // if node.federated.as_ref() == Some(&CONF.hostname) {
+    //     return Ok(());
+    // }
+    let track_sources = if let Some(media) = &mut node.media {
+        let mut track_sources = Vec::new();
+        for (i, t) in media.tracks.iter_mut().enumerate() {
+            t.federated.push(host.to_owned());
+            track_sources.push(TrackSource::Remote(i))
+        }
+        Some(track_sources)
+    } else {
+        None
+    };
 
     // TODO maybe use lazy download
     let poster = cache_federation_asset(session.to_owned(), id.clone(), AssetRole::Poster).await?;
@@ -388,7 +420,7 @@ async fn import_remote(
             backdrop: Some(backdrop),
             poster: Some(poster),
             id: None,
-            source: None, // TODO
+            source: track_sources,
         },
     };
 
@@ -426,15 +458,15 @@ async fn cache_federation_asset(
     .await
 }
 
-fn make_ident(s: &str) -> String {
-    let mut out = String::new();
-    for s in s.chars() {
-        match s {
-            'a'..='z' | '0'..='9' => out.push(s),
-            'A'..='Z' => out.push(s.to_ascii_lowercase()),
-            '-' | ' ' | '_' | ':' => out.push('-'),
-            _ => (),
-        }
-    }
-    out
-}
+// fn make_ident(s: &str) -> String {
+//     let mut out = String::new();
+//     for s in s.chars() {
+//         match s {
+//             'a'..='z' | '0'..='9' => out.push(s),
+//             'A'..='Z' => out.push(s.to_ascii_lowercase()),
+//             '-' | ' ' | '_' | ':' => out.push('-'),
+//             _ => (),
+//         }
+//     }
+//     out
+// }
diff --git a/import/src/tmdb.rs b/import/src/tmdb.rs
index 3780524..37447e6 100644
--- a/import/src/tmdb.rs
+++ b/import/src/tmdb.rs
@@ -4,10 +4,10 @@
     Copyright (C) 2023 metamuffin <metamuffin.org>
 */
 use anyhow::Context;
+use bincode::{Decode, Encode};
 use jellycommon::chrono::{format::Parsed, DateTime, Utc};
 use log::info;
 use serde::Deserialize;
-use std::io::Write;
 
 #[derive(Debug, Clone, Deserialize)]
 pub struct TmdbQuery {
@@ -17,7 +17,7 @@ pub struct TmdbQuery {
     pub total_results: usize,
 }
 
-#[derive(Debug, Clone, Deserialize)]
+#[derive(Debug, Clone, Deserialize, Encode, Decode)]
 pub struct TmdbQueryResult {
     pub adult: bool,
     pub backdrop_path: Option<String>,
@@ -35,7 +35,7 @@ pub struct TmdbQueryResult {
     pub vote_count: usize,
 }
 
-#[derive(Debug, Clone, Deserialize)]
+#[derive(Debug, Clone, Deserialize, Encode, Decode)]
 pub struct TmdbDetails {
     pub adult: bool,
     pub backdrop_path: Option<String>,
@@ -59,13 +59,13 @@ pub struct TmdbDetails {
     pub tagline: Option<String>,
 }
 
-#[derive(Debug, Clone, Deserialize)]
+#[derive(Debug, Clone, Deserialize, Encode, Decode)]
 pub struct TmdbGenre {
     pub id: u64,
     pub name: String,
 }
 
-#[derive(Debug, Clone, Deserialize)]
+#[derive(Debug, Clone, Deserialize, Encode, Decode)]
 pub struct TmdbProductionCompany {
     pub id: u64,
     pub name: String,
diff --git a/stream/src/jhls.rs b/stream/src/jhls.rs
index 600d945..e61e918 100644
--- a/stream/src/jhls.rs
+++ b/stream/src/jhls.rs
@@ -41,7 +41,10 @@ pub async fn jhls_stream(
                     &node.public,
                     match &track_sources[i] {
                         TrackSource::Local(x) => x,
-                        TrackSource::Remote => return Some(Err(anyhow!("das geht nicht"))),
+                        // TODO fetch seek index from the remote and create a single session to be sent in jhls
+                        TrackSource::Remote(_) => {
+                            return Some(Err(anyhow!("remote tracks dont work yet")))
+                        }
                     },
                 ) {
                     Ok(segments) => Some(Ok::<_, anyhow::Error>(JhlsTrack {
diff --git a/stream/src/lib.rs b/stream/src/lib.rs
index ee5c78a..e2137d3 100644
--- a/stream/src/lib.rs
+++ b/stream/src/lib.rs
@@ -72,7 +72,7 @@ pub async fn stream(
                     .ok_or(anyhow!("track does not exist"))?
                 {
                     TrackSource::Local(t) => t.to_owned(),
-                    TrackSource::Remote => bail!("track is not local"),
+                    TrackSource::Remote(_) => bail!("track is not local"),
                 },
             )
         })