aboutsummaryrefslogtreecommitdiff
path: root/import
diff options
context:
space:
mode:
Diffstat (limited to 'import')
-rw-r--r--import/src/lib.rs110
-rw-r--r--import/src/tmdb.rs194
-rw-r--r--import/src/trakt.rs138
3 files changed, 313 insertions, 129 deletions
diff --git a/import/src/lib.rs b/import/src/lib.rs
index d6eb54f..8347833 100644
--- a/import/src/lib.rs
+++ b/import/src/lib.rs
@@ -11,17 +11,17 @@ pub mod trakt;
use crate::tmdb::TmdbKind;
use anyhow::{anyhow, bail, Context, Ok};
use async_recursion::async_recursion;
-use futures::{executor::block_on, stream::FuturesUnordered, StreamExt};
+use futures::{stream::FuturesUnordered, StreamExt};
use jellybase::{
cache::{async_cache_file, cache_memory},
- database::{DataAcid, ReadableTable, Ser, T_NODE, T_NODE_IMPORT},
+ database::{DataAcid, ReadableTable, Ser, T_NODE, T_NODE_EXTENDED, T_NODE_IMPORT},
federation::Federation,
AssetLocationExt, CONF, SECRETS,
};
use jellyclient::Session;
use jellycommon::{
- AssetLocation, AssetRole, ImportOptions, ImportSource, MediaInfo, Node, NodeKind, NodePrivate,
- NodePublic, Rating, TrackSource,
+ AssetLocation, AssetRole, ExtendedNode, ImportOptions, ImportSource, MediaInfo, Node, NodeKind,
+ NodePrivate, NodePublic, PeopleGroup, Rating, TrackSource,
};
use jellymatroska::read::EbmlReader;
use jellyremuxer::import::import_metadata;
@@ -38,7 +38,7 @@ use std::{
path::{Path, PathBuf},
sync::{Arc, LazyLock},
};
-use tmdb::{parse_release_date, tmdb_image};
+use tmdb::{parse_release_date, Tmdb};
use tokio::{io::AsyncWriteExt, sync::Semaphore, task::spawn_blocking};
use trakt::Trakt;
@@ -46,6 +46,7 @@ static IMPORT_SEM: LazyLock<Semaphore> = LazyLock::new(|| Semaphore::new(1));
struct Apis {
trakt: Option<Trakt>,
+ tmdb: Option<Tmdb>,
}
pub async fn import(db: &DataAcid, fed: &Federation) -> anyhow::Result<()> {
@@ -64,6 +65,7 @@ pub async fn import(db: &DataAcid, fed: &Federation) -> anyhow::Result<()> {
let ap = Apis {
trakt: SECRETS.api.trakt.as_ref().map(|key| Trakt::new(key)),
+ tmdb: SECRETS.api.tmdb.as_ref().map(|key| Tmdb::new(key)),
};
info!("loading sources...");
@@ -251,28 +253,72 @@ async fn process_source(
txn.commit()?;
Ok(())
};
+ let insert_node_ext = move |id: &str, n: ExtendedNode| -> anyhow::Result<()> {
+ // TODO merge this
+ let txn = db.inner.begin_write()?;
+ let mut table = txn.open_table(T_NODE_EXTENDED)?;
+ table.insert(id, Ser(n))?;
+ drop(table);
+ txn.commit()?;
+ Ok(())
+ };
match s {
ImportSource::Override(n) => insert_node(&id, n)?,
ImportSource::Trakt { id: tid, kind } => {
- let trakt_object = ap
+ info!("trakt {id}");
+ let trakt = ap
.trakt
.as_ref()
- .ok_or(anyhow!("trakt api key is required"))?
+ .ok_or(anyhow!("trakt api key is required"))?;
+ let trakt_object = trakt
.lookup(kind, tid, true)
- .await?;
+ .await
+ .context("looking up metadata")?;
+ let trakt_people = trakt
+ .people(kind, tid, true)
+ .await
+ .context("looking up people")?;
let mut node = Node::default();
- node.public.title = Some(trakt_object.title.to_owned());
- if let Some(overview) = &trakt_object.overview {
- node.public.description = Some(overview.to_owned())
- }
- if let Some(tagline) = &trakt_object.tagline {
- node.public.tagline = Some(tagline.to_owned())
- }
- if let Some(rating) = &trakt_object.rating {
- node.public.ratings.insert(Rating::Trakt, *rating);
+ let mut node_ext = ExtendedNode::default();
+ {
+ node.public.title = Some(trakt_object.title.to_owned());
+ if let Some(overview) = &trakt_object.overview {
+ node.public.description = Some(overview.to_owned())
+ }
+ if let Some(tagline) = &trakt_object.tagline {
+ node.public.tagline = Some(tagline.to_owned())
+ }
+ if let Some(rating) = &trakt_object.rating {
+ node.public.ratings.insert(Rating::Trakt, *rating);
+ }
+ for p in trakt_people.cast.iter() {
+ node_ext
+ .people
+ .entry(PeopleGroup::Cast)
+ .or_default()
+ .push(p.a())
+ }
+ for (group, people) in trakt_people.crew.iter() {
+ for p in people {
+ node_ext.people.entry(group.a()).or_default().push(p.a())
+ }
+ }
+ for (_, ps) in &mut node_ext.people {
+ for p in ps {
+ if let Some(id) = p.person.ids.tmdb {
+ if let Some(tmdb) = &ap.tmdb {
+ let k = tmdb.person_image(id).await?;
+ if let Some(prof) = k.profiles.get(0) {
+ p.person.asset = Some(tmdb.image(&prof.file_path).await?);
+ }
+ }
+ }
+ }
+ }
}
insert_node(&id, node)?;
+ insert_node_ext(&id, node_ext)?;
if let Some(tid) = trakt_object.ids.tmdb {
let mut index_path = index_path.to_vec();
@@ -290,39 +336,21 @@ async fn process_source(
}
}
ImportSource::Tmdb { id: tid } => {
- info!("tmdb lookup {id}");
- let key = SECRETS
- .api
+ info!("tmdb {id}");
+ let tmdb = ap
.tmdb
.as_ref()
- .ok_or(anyhow!("no tmdb api key"))?;
+ .ok_or(anyhow!("tmdb api key is required"))?;
- let details = tokio::task::spawn_blocking(move || {
- cache_memory(&["tmdb-details", &format!("{tid}")], || {
- block_on(tmdb::tmdb_details(TmdbKind::Movie, tid, key))
- })
- })
- .await??;
+ let details = tmdb.details(TmdbKind::Movie, tid).await?;
let mut node = Node::default();
if let Some(poster) = &details.poster_path {
- node.private.poster = Some(
- async_cache_file(
- &["tmdb-asset", "poster", &format!("{tid}")],
- |mut f| async move { Ok(f.write_all(&tmdb_image(&poster).await?).await?) },
- )
- .await?,
- );
+ node.private.poster = Some(tmdb.image(&poster).await?);
}
if let Some(backdrop) = &details.backdrop_path {
- node.private.backdrop = Some(
- async_cache_file(
- &["tmdb-asset", "backdrop", &format!("{tid}")],
- |mut f| async move { Ok(f.write_all(&tmdb_image(&backdrop).await?).await?) },
- )
- .await?,
- );
+ node.private.backdrop = Some(tmdb.image(&backdrop).await?);
}
node.public.tagline = details.tagline.clone();
diff --git a/import/src/tmdb.rs b/import/src/tmdb.rs
index 95ebef4..b651223 100644
--- a/import/src/tmdb.rs
+++ b/import/src/tmdb.rs
@@ -1,3 +1,5 @@
+use std::{fmt::Display, sync::Arc};
+
/*
This file is part of jellything (https://codeberg.org/metamuffin/jellything)
which is licensed under the GNU Affero General Public License (version 3); see /COPYING.
@@ -5,11 +7,143 @@
*/
use anyhow::Context;
use bincode::{Decode, Encode};
-use jellycommon::chrono::{format::Parsed, Utc};
+use jellybase::cache::{async_cache_file, async_cache_memory};
+use jellycommon::{
+ chrono::{format::Parsed, Utc},
+ AssetLocation,
+};
use log::info;
+use reqwest::{
+ header::{HeaderMap, HeaderName, HeaderValue},
+ Client, ClientBuilder,
+};
use serde::Deserialize;
+use tokio::io::AsyncWriteExt;
+
+pub struct Tmdb {
+ client: Client,
+ key: String,
+}
+
+impl Tmdb {
+ pub fn new(api_key: &str) -> Self {
+ let client = ClientBuilder::new()
+ .default_headers(HeaderMap::from_iter([(
+ HeaderName::from_static("accept"),
+ HeaderValue::from_static("application/json"),
+ )]))
+ .build()
+ .unwrap();
+ Self {
+ client,
+ key: api_key.to_owned(),
+ }
+ }
+ pub async fn search(&self, kind: TmdbKind, query: &str) -> anyhow::Result<Arc<TmdbQuery>> {
+ async_cache_memory(
+ &["api-tmdb-search", query, &format!("{kind}")],
+ || async move {
+ info!("searching tmdb: {query:?}");
+ Ok(self
+ .client
+ .get(&format!(
+ "https://api.themoviedb.org/3/search/{kind}?query={}?api_key={}",
+ query.replace(" ", "+"),
+ self.key
+ ))
+ .send()
+ .await?
+ .error_for_status()?
+ .json::<TmdbQuery>()
+ .await?)
+ },
+ )
+ .await
+ }
+ pub async fn details(&self, kind: TmdbKind, id: u64) -> anyhow::Result<Arc<TmdbDetails>> {
+ async_cache_memory(
+ &["api-tmdb-details", &format!("{kind} {id}")],
+ || async move {
+ info!("fetching details: {id:?}");
+ Ok(self
+ .client
+ .get(&format!(
+ "https://api.themoviedb.org/3/{kind}/{id}?api_key={}",
+ self.key,
+ ))
+ .send()
+ .await?
+ .error_for_status()?
+ .json()
+ .await?)
+ },
+ )
+ .await
+ }
+ pub async fn person_image(&self, id: u64) -> anyhow::Result<Arc<TmdbPersonImage>> {
+ async_cache_memory(&["api-tmdb-search", &format!("{id}")], || async move {
+ Ok(self
+ .client
+ .get(&format!(
+ "https://api.themoviedb.org/3/person/{id}/images?api_key={}",
+ self.key,
+ ))
+ .send()
+ .await?
+ .error_for_status()?
+ .json()
+ .await?)
+ })
+ .await
+ }
+ pub async fn image(&self, path: &str) -> anyhow::Result<AssetLocation> {
+ async_cache_file(&["api-tmdb-image", path], |mut file| async move {
+ info!("downloading image {path:?}");
+ let mut res = reqwest::get(&format!("https://image.tmdb.org/t/p/original{path}"))
+ .await?
+ .error_for_status()?;
+ while let Some(chunk) = res.chunk().await? {
+ file.write_all(&chunk).await?;
+ }
+ Ok(())
+ })
+ .await
+ }
+}
+
+pub fn parse_release_date(d: &str) -> anyhow::Result<i64> {
+ let (year, month, day) = (&d[0..4], &d[5..7], &d[8..10]);
+ let (year, month, day) = (
+ year.parse().context("parsing year")?,
+ month.parse().context("parsing month")?,
+ day.parse().context("parsing day")?,
+ );
+
+ let mut p = Parsed::new();
+ p.year = Some(year);
+ p.month = Some(month);
+ p.day = Some(day);
+ p.hour_div_12 = Some(0);
+ p.hour_mod_12 = Some(0);
+ p.minute = Some(0);
+ p.second = Some(0);
+ Ok(p.to_datetime_with_timezone(&Utc)?.timestamp_millis())
+}
-#[derive(Debug, Clone, Deserialize)]
+#[derive(Debug, Clone, Deserialize, Encode, Decode)]
+pub struct TmdbPersonImage {
+ pub id: u64,
+ pub profiles: Vec<TmdbPersonImageProfile>,
+}
+#[derive(Debug, Clone, Deserialize, Encode, Decode)]
+pub struct TmdbPersonImageProfile {
+ pub aspect_ratio: f64,
+ pub height: u32,
+ pub width: u32,
+ pub file_path: String,
+}
+
+#[derive(Debug, Clone, Deserialize, Encode, Decode)]
pub struct TmdbQuery {
pub page: usize,
pub results: Vec<TmdbQueryResult>,
@@ -77,59 +211,11 @@ pub enum TmdbKind {
Tv,
Movie,
}
-impl TmdbKind {
- pub fn as_str(&self) -> &'static str {
- match self {
+impl Display for TmdbKind {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ f.write_str(match self {
TmdbKind::Tv => "tv",
TmdbKind::Movie => "movie",
- }
+ })
}
}
-
-pub async fn tmdb_search(kind: TmdbKind, query: &str, key: &str) -> anyhow::Result<TmdbQuery> {
- info!("searching tmdb: {query:?}");
- Ok(reqwest::get(&format!(
- "https://api.themoviedb.org/3/search/{}?query={}&api_key={key}",
- kind.as_str(),
- query.replace(" ", "+")
- ))
- .await?
- .json::<TmdbQuery>()
- .await?)
-}
-
-pub async fn tmdb_details(kind: TmdbKind, id: u64, key: &str) -> anyhow::Result<TmdbDetails> {
- info!("fetching details: {id:?}");
- Ok(reqwest::get(&format!(
- "https://api.themoviedb.org/3/{}/{id}?api_key={key}",
- kind.as_str()
- ))
- .await?
- .json()
- .await?)
-}
-
-pub async fn tmdb_image(path: &str) -> anyhow::Result<Vec<u8>> {
- info!("downloading image {path:?}");
- let res = reqwest::get(&format!("https://image.tmdb.org/t/p/original{path}")).await?;
- Ok(res.bytes().await?.to_vec())
-}
-
-pub fn parse_release_date(d: &str) -> anyhow::Result<i64> {
- let (year, month, day) = (&d[0..4], &d[5..7], &d[8..10]);
- let (year, month, day) = (
- year.parse().context("parsing year")?,
- month.parse().context("parsing month")?,
- day.parse().context("parsing day")?,
- );
-
- let mut p = Parsed::new();
- p.year = Some(year);
- p.month = Some(month);
- p.day = Some(day);
- p.hour_div_12 = Some(0);
- p.hour_mod_12 = Some(0);
- p.minute = Some(0);
- p.second = Some(0);
- Ok(p.to_datetime_with_timezone(&Utc)?.timestamp_millis())
-}
diff --git a/import/src/trakt.rs b/import/src/trakt.rs
index 9674351..0441ad0 100644
--- a/import/src/trakt.rs
+++ b/import/src/trakt.rs
@@ -1,12 +1,12 @@
use bincode::{Decode, Encode};
use jellybase::cache::async_cache_memory;
-use jellycommon::TraktKind;
+use jellycommon::{Appearance, ObjectIds, PeopleGroup, Person, TraktKind};
use reqwest::{
header::{HeaderMap, HeaderName, HeaderValue},
Client, ClientBuilder,
};
use serde::{Deserialize, Serialize};
-use std::{fmt::Display, sync::Arc};
+use std::{collections::BTreeMap, fmt::Display, sync::Arc};
pub struct Trakt {
client: Client,
@@ -81,6 +81,49 @@ impl Trakt {
)
.await
}
+
+ pub async fn people(
+ &self,
+ kind: TraktKind,
+ id: u64,
+ extended: bool,
+ ) -> anyhow::Result<Arc<TraktPeople>> {
+ async_cache_memory(
+ &["api-trakt-people", &format!("{id} {extended}")],
+ || async move {
+ let url = format!(
+ "https://api.trakt.tv/{}/{}/people{}",
+ kind.plural(),
+ id,
+ optext2(extended)
+ );
+ let res = self.client.get(url).send().await?.error_for_status()?;
+ Ok(res.json().await?)
+ },
+ )
+ .await
+ }
+}
+
+#[derive(Debug, Clone, Deserialize, Serialize, Default, Encode, Decode)]
+pub struct TraktPeople {
+ pub cast: Vec<TraktAppearance>,
+ pub crew: BTreeMap<TraktPeopleGroup, Vec<TraktAppearance>>,
+}
+
+#[derive(Debug, Clone, Deserialize, Serialize, Default, Encode, Decode)]
+pub struct TraktAppearance {
+ #[serde(default)]
+ pub jobs: Vec<String>,
+ #[serde(default)]
+ pub characters: Vec<String>,
+ pub person: TraktPerson,
+}
+
+#[derive(Debug, Clone, Deserialize, Serialize, Default, Encode, Decode)]
+pub struct TraktPerson {
+ pub name: String,
+ pub ids: ObjectIds,
}
fn optext(extended: bool) -> &'static str {
@@ -130,11 +173,69 @@ impl TraktKindObject {
}
}
+#[derive(
+ Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Encode, Decode, Clone, Copy,
+)]
+pub enum TraktPeopleGroup {
+ #[serde(rename = "production")]
+ Production,
+ #[serde(rename = "art")]
+ Art,
+ #[serde(rename = "crew")]
+ Crew,
+ #[serde(rename = "costume & make-up")]
+ CostumeMakeup,
+ #[serde(rename = "directing")]
+ Directing,
+ #[serde(rename = "writing")]
+ Writing,
+ #[serde(rename = "sound")]
+ Sound,
+ #[serde(rename = "camera")]
+ Camera,
+ #[serde(rename = "visual effects")]
+ VisualEffects,
+ #[serde(rename = "lighting")]
+ Lighting,
+ #[serde(rename = "editing")]
+ Editing,
+}
+impl TraktPeopleGroup {
+ pub fn a(self) -> PeopleGroup {
+ match self {
+ TraktPeopleGroup::Production => PeopleGroup::Production,
+ TraktPeopleGroup::Art => PeopleGroup::Art,
+ TraktPeopleGroup::Crew => PeopleGroup::Crew,
+ TraktPeopleGroup::CostumeMakeup => PeopleGroup::CostumeMakeup,
+ TraktPeopleGroup::Directing => PeopleGroup::Directing,
+ TraktPeopleGroup::Writing => PeopleGroup::Writing,
+ TraktPeopleGroup::Sound => PeopleGroup::Sound,
+ TraktPeopleGroup::Camera => PeopleGroup::Camera,
+ TraktPeopleGroup::VisualEffects => PeopleGroup::Vfx,
+ TraktPeopleGroup::Lighting => PeopleGroup::Lighting,
+ TraktPeopleGroup::Editing => PeopleGroup::Editing,
+ }
+ }
+}
+impl TraktAppearance {
+ pub fn a(&self) -> Appearance {
+ Appearance {
+ jobs: self.jobs.to_owned(),
+ characters: self.characters.to_owned(),
+ person: Person {
+ name: self.person.name.to_owned(),
+ asset: None,
+ ids: self.person.ids.to_owned(),
+ },
+ }
+ }
+}
+
#[derive(Debug, Serialize, Deserialize, Encode, Decode, Clone)]
pub struct TraktMediaObject {
pub title: String,
pub year: Option<u32>,
- pub ids: TraktMediaObjectIds,
+ pub ids: ObjectIds,
pub tagline: Option<String>,
pub overview: Option<String>,
@@ -152,16 +253,6 @@ pub struct TraktMediaObject {
pub genres: Option<Vec<String>>,
}
-#[derive(Debug, Serialize, Deserialize, Encode, Decode, Clone)]
-pub struct TraktMediaObjectIds {
- pub trakt: u64,
- pub slug: Option<String>,
- pub imdb: Option<String>,
- pub tmdb: Option<u64>,
- pub omdb: Option<u64>,
- pub tvdb: Option<u64>,
-}
-
impl Display for TraktSearchResult {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_fmt(format_args!(
@@ -173,24 +264,3 @@ impl Display for TraktSearchResult {
))
}
}
-impl Display for TraktMediaObjectIds {
- fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
- f.write_str("trakt")?;
- if self.slug.is_some() {
- f.write_str(",slug")?;
- }
- if self.tmdb.is_some() {
- f.write_str(",tmdb")?;
- }
- if self.imdb.is_some() {
- f.write_str(",imdb")?;
- }
- if self.tvdb.is_some() {
- f.write_str(",tvdb")?;
- }
- if self.omdb.is_some() {
- f.write_str(",omdb")?;
- }
- Ok(())
- }
-}
generated by cgit v1.2.3-70-g09d2 (git 2.51.0) at 2025-10-08 13:09:40 +0000