refactor tmdb api, cast&crew, node ext

3 files changed, 313 insertions, 129 deletions

diff --git a/import/src/lib.rs b/import/src/lib.rs
index d6eb54f..8347833 100644
--- a/import/src/lib.rs
+++ b/import/src/lib.rs
@@ -11,17 +11,17 @@ pub mod trakt;
 use crate::tmdb::TmdbKind;
 use anyhow::{anyhow, bail, Context, Ok};
 use async_recursion::async_recursion;
-use futures::{executor::block_on, stream::FuturesUnordered, StreamExt};
+use futures::{stream::FuturesUnordered, StreamExt};
 use jellybase::{
     cache::{async_cache_file, cache_memory},
-    database::{DataAcid, ReadableTable, Ser, T_NODE, T_NODE_IMPORT},
+    database::{DataAcid, ReadableTable, Ser, T_NODE, T_NODE_EXTENDED, T_NODE_IMPORT},
     federation::Federation,
     AssetLocationExt, CONF, SECRETS,
 };
 use jellyclient::Session;
 use jellycommon::{
-    AssetLocation, AssetRole, ImportOptions, ImportSource, MediaInfo, Node, NodeKind, NodePrivate,
-    NodePublic, Rating, TrackSource,
+    AssetLocation, AssetRole, ExtendedNode, ImportOptions, ImportSource, MediaInfo, Node, NodeKind,
+    NodePrivate, NodePublic, PeopleGroup, Rating, TrackSource,
 };
 use jellymatroska::read::EbmlReader;
 use jellyremuxer::import::import_metadata;
@@ -38,7 +38,7 @@ use std::{
     path::{Path, PathBuf},
     sync::{Arc, LazyLock},
 };
-use tmdb::{parse_release_date, tmdb_image};
+use tmdb::{parse_release_date, Tmdb};
 use tokio::{io::AsyncWriteExt, sync::Semaphore, task::spawn_blocking};
 use trakt::Trakt;
 
@@ -46,6 +46,7 @@ static IMPORT_SEM: LazyLock<Semaphore> = LazyLock::new(|| Semaphore::new(1));
 
 struct Apis {
     trakt: Option<Trakt>,
+    tmdb: Option<Tmdb>,
 }
 
 pub async fn import(db: &DataAcid, fed: &Federation) -> anyhow::Result<()> {
@@ -64,6 +65,7 @@ pub async fn import(db: &DataAcid, fed: &Federation) -> anyhow::Result<()> {
 
     let ap = Apis {
         trakt: SECRETS.api.trakt.as_ref().map(|key| Trakt::new(key)),
+        tmdb: SECRETS.api.tmdb.as_ref().map(|key| Tmdb::new(key)),
     };
 
     info!("loading sources...");
@@ -251,28 +253,72 @@ async fn process_source(
         txn.commit()?;
         Ok(())
     };
+    let insert_node_ext = move |id: &str, n: ExtendedNode| -> anyhow::Result<()> {
+        // TODO merge this
+        let txn = db.inner.begin_write()?;
+        let mut table = txn.open_table(T_NODE_EXTENDED)?;
+        table.insert(id, Ser(n))?;
+        drop(table);
+        txn.commit()?;
+        Ok(())
+    };
     match s {
         ImportSource::Override(n) => insert_node(&id, n)?,
         ImportSource::Trakt { id: tid, kind } => {
-            let trakt_object = ap
+            info!("trakt {id}");
+            let trakt = ap
                 .trakt
                 .as_ref()
-                .ok_or(anyhow!("trakt api key is required"))?
+                .ok_or(anyhow!("trakt api key is required"))?;
+            let trakt_object = trakt
                 .lookup(kind, tid, true)
-                .await?;
+                .await
+                .context("looking up metadata")?;
+            let trakt_people = trakt
+                .people(kind, tid, true)
+                .await
+                .context("looking up people")?;
 
             let mut node = Node::default();
-            node.public.title = Some(trakt_object.title.to_owned());
-            if let Some(overview) = &trakt_object.overview {
-                node.public.description = Some(overview.to_owned())
-            }
-            if let Some(tagline) = &trakt_object.tagline {
-                node.public.tagline = Some(tagline.to_owned())
-            }
-            if let Some(rating) = &trakt_object.rating {
-                node.public.ratings.insert(Rating::Trakt, *rating);
+            let mut node_ext = ExtendedNode::default();
+            {
+                node.public.title = Some(trakt_object.title.to_owned());
+                if let Some(overview) = &trakt_object.overview {
+                    node.public.description = Some(overview.to_owned())
+                }
+                if let Some(tagline) = &trakt_object.tagline {
+                    node.public.tagline = Some(tagline.to_owned())
+                }
+                if let Some(rating) = &trakt_object.rating {
+                    node.public.ratings.insert(Rating::Trakt, *rating);
+                }
+                for p in trakt_people.cast.iter() {
+                    node_ext
+                        .people
+                        .entry(PeopleGroup::Cast)
+                        .or_default()
+                        .push(p.a())
+                }
+                for (group, people) in trakt_people.crew.iter() {
+                    for p in people {
+                        node_ext.people.entry(group.a()).or_default().push(p.a())
+                    }
+                }
+                for (_, ps) in &mut node_ext.people {
+                    for p in ps {
+                        if let Some(id) = p.person.ids.tmdb {
+                            if let Some(tmdb) = &ap.tmdb {
+                                let k = tmdb.person_image(id).await?;
+                                if let Some(prof) = k.profiles.get(0) {
+                                    p.person.asset = Some(tmdb.image(&prof.file_path).await?);
+                                }
+                            }
+                        }
+                    }
+                }
             }
             insert_node(&id, node)?;
+            insert_node_ext(&id, node_ext)?;
 
             if let Some(tid) = trakt_object.ids.tmdb {
                 let mut index_path = index_path.to_vec();
@@ -290,39 +336,21 @@ async fn process_source(
             }
         }
         ImportSource::Tmdb { id: tid } => {
-            info!("tmdb lookup {id}");
-            let key = SECRETS
-                .api
+            info!("tmdb {id}");
+            let tmdb = ap
                 .tmdb
                 .as_ref()
-                .ok_or(anyhow!("no tmdb api key"))?;
+                .ok_or(anyhow!("tmdb api key is required"))?;
 
-            let details = tokio::task::spawn_blocking(move || {
-                cache_memory(&["tmdb-details", &format!("{tid}")], || {
-                    block_on(tmdb::tmdb_details(TmdbKind::Movie, tid, key))
-                })
-            })
-            .await??;
+            let details = tmdb.details(TmdbKind::Movie, tid).await?;
 
             let mut node = Node::default();
 
             if let Some(poster) = &details.poster_path {
-                node.private.poster = Some(
-                    async_cache_file(
-                        &["tmdb-asset", "poster", &format!("{tid}")],
-                        |mut f| async move { Ok(f.write_all(&tmdb_image(&poster).await?).await?) },
-                    )
-                    .await?,
-                );
+                node.private.poster = Some(tmdb.image(&poster).await?);
             }
             if let Some(backdrop) = &details.backdrop_path {
-                node.private.backdrop = Some(
-                    async_cache_file(
-                        &["tmdb-asset", "backdrop", &format!("{tid}")],
-                        |mut f| async move { Ok(f.write_all(&tmdb_image(&backdrop).await?).await?) },
-                    )
-                    .await?,
-                );
+                node.private.backdrop = Some(tmdb.image(&backdrop).await?);
             }
 
             node.public.tagline = details.tagline.clone();
diff --git a/import/src/tmdb.rs b/import/src/tmdb.rs
index 95ebef4..b651223 100644
--- a/import/src/tmdb.rs
+++ b/import/src/tmdb.rs
@@ -1,3 +1,5 @@
+use std::{fmt::Display, sync::Arc};
+
 /*
     This file is part of jellything (https://codeberg.org/metamuffin/jellything)
     which is licensed under the GNU Affero General Public License (version 3); see /COPYING.
@@ -5,11 +7,143 @@
 */
 use anyhow::Context;
 use bincode::{Decode, Encode};
-use jellycommon::chrono::{format::Parsed, Utc};
+use jellybase::cache::{async_cache_file, async_cache_memory};
+use jellycommon::{
+    chrono::{format::Parsed, Utc},
+    AssetLocation,
+};
 use log::info;
+use reqwest::{
+    header::{HeaderMap, HeaderName, HeaderValue},
+    Client, ClientBuilder,
+};
 use serde::Deserialize;
+use tokio::io::AsyncWriteExt;
+
+pub struct Tmdb {
+    client: Client,
+    key: String,
+}
+
+impl Tmdb {
+    pub fn new(api_key: &str) -> Self {
+        let client = ClientBuilder::new()
+            .default_headers(HeaderMap::from_iter([(
+                HeaderName::from_static("accept"),
+                HeaderValue::from_static("application/json"),
+            )]))
+            .build()
+            .unwrap();
+        Self {
+            client,
+            key: api_key.to_owned(),
+        }
+    }
+    pub async fn search(&self, kind: TmdbKind, query: &str) -> anyhow::Result<Arc<TmdbQuery>> {
+        async_cache_memory(
+            &["api-tmdb-search", query, &format!("{kind}")],
+            || async move {
+                info!("searching tmdb: {query:?}");
+                Ok(self
+                    .client
+                    .get(&format!(
+                        "https://api.themoviedb.org/3/search/{kind}?query={}?api_key={}",
+                        query.replace(" ", "+"),
+                        self.key
+                    ))
+                    .send()
+                    .await?
+                    .error_for_status()?
+                    .json::<TmdbQuery>()
+                    .await?)
+            },
+        )
+        .await
+    }
+    pub async fn details(&self, kind: TmdbKind, id: u64) -> anyhow::Result<Arc<TmdbDetails>> {
+        async_cache_memory(
+            &["api-tmdb-details", &format!("{kind} {id}")],
+            || async move {
+                info!("fetching details: {id:?}");
+                Ok(self
+                    .client
+                    .get(&format!(
+                        "https://api.themoviedb.org/3/{kind}/{id}?api_key={}",
+                        self.key,
+                    ))
+                    .send()
+                    .await?
+                    .error_for_status()?
+                    .json()
+                    .await?)
+            },
+        )
+        .await
+    }
+    pub async fn person_image(&self, id: u64) -> anyhow::Result<Arc<TmdbPersonImage>> {
+        async_cache_memory(&["api-tmdb-search", &format!("{id}")], || async move {
+            Ok(self
+                .client
+                .get(&format!(
+                    "https://api.themoviedb.org/3/person/{id}/images?api_key={}",
+                    self.key,
+                ))
+                .send()
+                .await?
+                .error_for_status()?
+                .json()
+                .await?)
+        })
+        .await
+    }
+    pub async fn image(&self, path: &str) -> anyhow::Result<AssetLocation> {
+        async_cache_file(&["api-tmdb-image", path], |mut file| async move {
+            info!("downloading image {path:?}");
+            let mut res = reqwest::get(&format!("https://image.tmdb.org/t/p/original{path}"))
+                .await?
+                .error_for_status()?;
+            while let Some(chunk) = res.chunk().await? {
+                file.write_all(&chunk).await?;
+            }
+            Ok(())
+        })
+        .await
+    }
+}
+
+pub fn parse_release_date(d: &str) -> anyhow::Result<i64> {
+    let (year, month, day) = (&d[0..4], &d[5..7], &d[8..10]);
+    let (year, month, day) = (
+        year.parse().context("parsing year")?,
+        month.parse().context("parsing month")?,
+        day.parse().context("parsing day")?,
+    );
+
+    let mut p = Parsed::new();
+    p.year = Some(year);
+    p.month = Some(month);
+    p.day = Some(day);
+    p.hour_div_12 = Some(0);
+    p.hour_mod_12 = Some(0);
+    p.minute = Some(0);
+    p.second = Some(0);
+    Ok(p.to_datetime_with_timezone(&Utc)?.timestamp_millis())
+}
 
-#[derive(Debug, Clone, Deserialize)]
+#[derive(Debug, Clone, Deserialize, Encode, Decode)]
+pub struct TmdbPersonImage {
+    pub id: u64,
+    pub profiles: Vec<TmdbPersonImageProfile>,
+}
+#[derive(Debug, Clone, Deserialize, Encode, Decode)]
+pub struct TmdbPersonImageProfile {
+    pub aspect_ratio: f64,
+    pub height: u32,
+    pub width: u32,
+    pub file_path: String,
+}
+
+#[derive(Debug, Clone, Deserialize, Encode, Decode)]
 pub struct TmdbQuery {
     pub page: usize,
     pub results: Vec<TmdbQueryResult>,
@@ -77,59 +211,11 @@ pub enum TmdbKind {
     Tv,
     Movie,
 }
-impl TmdbKind {
-    pub fn as_str(&self) -> &'static str {
-        match self {
+impl Display for TmdbKind {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.write_str(match self {
             TmdbKind::Tv => "tv",
             TmdbKind::Movie => "movie",
-        }
+        })
     }
 }
-
-pub async fn tmdb_search(kind: TmdbKind, query: &str, key: &str) -> anyhow::Result<TmdbQuery> {
-    info!("searching tmdb: {query:?}");
-    Ok(reqwest::get(&format!(
-        "https://api.themoviedb.org/3/search/{}?query={}&api_key={key}",
-        kind.as_str(),
-        query.replace(" ", "+")
-    ))
-    .await?
-    .json::<TmdbQuery>()
-    .await?)
-}
-
-pub async fn tmdb_details(kind: TmdbKind, id: u64, key: &str) -> anyhow::Result<TmdbDetails> {
-    info!("fetching details: {id:?}");
-    Ok(reqwest::get(&format!(
-        "https://api.themoviedb.org/3/{}/{id}?api_key={key}",
-        kind.as_str()
-    ))
-    .await?
-    .json()
-    .await?)
-}
-
-pub async fn tmdb_image(path: &str) -> anyhow::Result<Vec<u8>> {
-    info!("downloading image {path:?}");
-    let res = reqwest::get(&format!("https://image.tmdb.org/t/p/original{path}")).await?;
-    Ok(res.bytes().await?.to_vec())
-}
-
-pub fn parse_release_date(d: &str) -> anyhow::Result<i64> {
-    let (year, month, day) = (&d[0..4], &d[5..7], &d[8..10]);
-    let (year, month, day) = (
-        year.parse().context("parsing year")?,
-        month.parse().context("parsing month")?,
-        day.parse().context("parsing day")?,
-    );
-
-    let mut p = Parsed::new();
-    p.year = Some(year);
-    p.month = Some(month);
-    p.day = Some(day);
-    p.hour_div_12 = Some(0);
-    p.hour_mod_12 = Some(0);
-    p.minute = Some(0);
-    p.second = Some(0);
-    Ok(p.to_datetime_with_timezone(&Utc)?.timestamp_millis())
-}
diff --git a/import/src/trakt.rs b/import/src/trakt.rs
index 9674351..0441ad0 100644
--- a/import/src/trakt.rs
+++ b/import/src/trakt.rs
@@ -1,12 +1,12 @@
 use bincode::{Decode, Encode};
 use jellybase::cache::async_cache_memory;
-use jellycommon::TraktKind;
+use jellycommon::{Appearance, ObjectIds, PeopleGroup, Person, TraktKind};
 use reqwest::{
     header::{HeaderMap, HeaderName, HeaderValue},
     Client, ClientBuilder,
 };
 use serde::{Deserialize, Serialize};
-use std::{fmt::Display, sync::Arc};
+use std::{collections::BTreeMap, fmt::Display, sync::Arc};
 
 pub struct Trakt {
     client: Client,
@@ -81,6 +81,49 @@ impl Trakt {
         )
         .await
     }
+
+    pub async fn people(
+        &self,
+        kind: TraktKind,
+        id: u64,
+        extended: bool,
+    ) -> anyhow::Result<Arc<TraktPeople>> {
+        async_cache_memory(
+            &["api-trakt-people", &format!("{id} {extended}")],
+            || async move {
+                let url = format!(
+                    "https://api.trakt.tv/{}/{}/people{}",
+                    kind.plural(),
+                    id,
+                    optext2(extended)
+                );
+                let res = self.client.get(url).send().await?.error_for_status()?;
+                Ok(res.json().await?)
+            },
+        )
+        .await
+    }
+}
+
+#[derive(Debug, Clone, Deserialize, Serialize, Default, Encode, Decode)]
+pub struct TraktPeople {
+    pub cast: Vec<TraktAppearance>,
+    pub crew: BTreeMap<TraktPeopleGroup, Vec<TraktAppearance>>,
+}
+
+#[derive(Debug, Clone, Deserialize, Serialize, Default, Encode, Decode)]
+pub struct TraktAppearance {
+    #[serde(default)]
+    pub jobs: Vec<String>,
+    #[serde(default)]
+    pub characters: Vec<String>,
+    pub person: TraktPerson,
+}
+
+#[derive(Debug, Clone, Deserialize, Serialize, Default, Encode, Decode)]
+pub struct TraktPerson {
+    pub name: String,
+    pub ids: ObjectIds,
 }
 
 fn optext(extended: bool) -> &'static str {
@@ -130,11 +173,69 @@ impl TraktKindObject {
     }
 }
 
+#[derive(
+    Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Encode, Decode, Clone, Copy,
+)]
+pub enum TraktPeopleGroup {
+    #[serde(rename = "production")]
+    Production,
+    #[serde(rename = "art")]
+    Art,
+    #[serde(rename = "crew")]
+    Crew,
+    #[serde(rename = "costume & make-up")]
+    CostumeMakeup,
+    #[serde(rename = "directing")]
+    Directing,
+    #[serde(rename = "writing")]
+    Writing,
+    #[serde(rename = "sound")]
+    Sound,
+    #[serde(rename = "camera")]
+    Camera,
+    #[serde(rename = "visual effects")]
+    VisualEffects,
+    #[serde(rename = "lighting")]
+    Lighting,
+    #[serde(rename = "editing")]
+    Editing,
+}
+impl TraktPeopleGroup {
+    pub fn a(self) -> PeopleGroup {
+        match self {
+            TraktPeopleGroup::Production => PeopleGroup::Production,
+            TraktPeopleGroup::Art => PeopleGroup::Art,
+            TraktPeopleGroup::Crew => PeopleGroup::Crew,
+            TraktPeopleGroup::CostumeMakeup => PeopleGroup::CostumeMakeup,
+            TraktPeopleGroup::Directing => PeopleGroup::Directing,
+            TraktPeopleGroup::Writing => PeopleGroup::Writing,
+            TraktPeopleGroup::Sound => PeopleGroup::Sound,
+            TraktPeopleGroup::Camera => PeopleGroup::Camera,
+            TraktPeopleGroup::VisualEffects => PeopleGroup::Vfx,
+            TraktPeopleGroup::Lighting => PeopleGroup::Lighting,
+            TraktPeopleGroup::Editing => PeopleGroup::Editing,
+        }
+    }
+}
+impl TraktAppearance {
+    pub fn a(&self) -> Appearance {
+        Appearance {
+            jobs: self.jobs.to_owned(),
+            characters: self.characters.to_owned(),
+            person: Person {
+                name: self.person.name.to_owned(),
+                asset: None,
+                ids: self.person.ids.to_owned(),
+            },
+        }
+    }
+}
+
 #[derive(Debug, Serialize, Deserialize, Encode, Decode, Clone)]
 pub struct TraktMediaObject {
     pub title: String,
     pub year: Option<u32>,
-    pub ids: TraktMediaObjectIds,
+    pub ids: ObjectIds,
 
     pub tagline: Option<String>,
     pub overview: Option<String>,
@@ -152,16 +253,6 @@ pub struct TraktMediaObject {
     pub genres: Option<Vec<String>>,
 }
 
-#[derive(Debug, Serialize, Deserialize, Encode, Decode, Clone)]
-pub struct TraktMediaObjectIds {
-    pub trakt: u64,
-    pub slug: Option<String>,
-    pub imdb: Option<String>,
-    pub tmdb: Option<u64>,
-    pub omdb: Option<u64>,
-    pub tvdb: Option<u64>,
-}
-
 impl Display for TraktSearchResult {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         f.write_fmt(format_args!(
@@ -173,24 +264,3 @@ impl Display for TraktSearchResult {
         ))
     }
 }
-impl Display for TraktMediaObjectIds {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.write_str("trakt")?;
-        if self.slug.is_some() {
-            f.write_str(",slug")?;
-        }
-        if self.tmdb.is_some() {
-            f.write_str(",tmdb")?;
-        }
-        if self.imdb.is_some() {
-            f.write_str(",imdb")?;
-        }
-        if self.tvdb.is_some() {
-            f.write_str(",tvdb")?;
-        }
-        if self.omdb.is_some() {
-            f.write_str(",omdb")?;
-        }
-        Ok(())
-    }
-}