aboutsummaryrefslogtreecommitdiff
path: root/import/src
diff options
context:
space:
mode:
Diffstat (limited to 'import/src')
-rw-r--r--import/src/acoustid.rs75
-rw-r--r--import/src/lib.rs138
-rw-r--r--import/src/musicbrainz.rs172
-rw-r--r--import/src/tmdb.rs163
-rw-r--r--import/src/trakt.rs116
-rw-r--r--import/src/vgmdb.rs85
-rw-r--r--import/src/wikidata.rs53
-rw-r--r--import/src/wikimedia_commons.rs42
8 files changed, 446 insertions, 398 deletions
diff --git a/import/src/acoustid.rs b/import/src/acoustid.rs
index c35708d..809d964 100644
--- a/import/src/acoustid.rs
+++ b/import/src/acoustid.rs
@@ -5,17 +5,22 @@
*/
use crate::USER_AGENT;
use anyhow::{Context, Result};
-use jellycache::async_cache_memory;
+use jellycache::{cache_memory, CacheKey};
use log::info;
use reqwest::{
header::{HeaderMap, HeaderName, HeaderValue},
Client, ClientBuilder,
};
use serde::{Deserialize, Serialize};
-use std::{path::Path, process::Stdio, sync::Arc, time::Duration};
+use std::{
+ io::Read,
+ path::Path,
+ process::{Command, Stdio},
+ sync::Arc,
+ time::Duration,
+};
use tokio::{
- io::AsyncReadExt,
- process::Command,
+ runtime::Handle,
sync::Semaphore,
time::{sleep_until, Instant},
};
@@ -79,8 +84,8 @@ impl AcoustID {
}
}
- pub async fn get_atid_mbid(&self, fp: &Fingerprint) -> Result<Option<(String, String)>> {
- let res = self.lookup(fp.to_owned()).await?;
+ pub fn get_atid_mbid(&self, fp: &Fingerprint, rt: &Handle) -> Result<Option<(String, String)>> {
+ let res = self.lookup(fp.to_owned(), rt)?;
for r in &res.results {
if let Some(k) = r.recordings.first() {
return Ok(Some((r.id.clone(), k.id.clone())));
@@ -89,8 +94,8 @@ impl AcoustID {
Ok(None)
}
- pub async fn lookup(&self, fp: Fingerprint) -> Result<Arc<AcoustIDLookupResponse>> {
- async_cache_memory("api-acoustid", fp.clone(), || async move {
+ pub fn lookup(&self, fp: Fingerprint, rt: &Handle) -> Result<Arc<AcoustIDLookupResponse>> {
+ cache_memory(CacheKey::new_json(("acoustid-lookup", &fp)) , move || rt.block_on(async {
let _permit = self.rate_limit.clone().acquire_owned().await?;
let permit_drop_ts = Instant::now() + Duration::SECOND;
info!("acoustid lookup");
@@ -114,36 +119,36 @@ impl AcoustID {
});
Ok(resp)
- })
- .await.context("acoustid lookup")
+ }))
+ .context("acoustid lookup")
}
}
-#[allow(unused)]
-pub(crate) async fn acoustid_fingerprint(path: &Path) -> Result<Arc<Fingerprint>> {
- async_cache_memory("fpcalc", path, || async move {
- let child = Command::new("fpcalc")
- .arg("-json")
- .arg(path)
- .stdout(Stdio::piped())
- .spawn()
- .context("fpcalc")?;
-
- let mut buf = Vec::new();
- child
- .stdout
- .unwrap()
- .read_to_end(&mut buf)
- .await
- .context("read fpcalc output")?;
+pub(crate) fn acoustid_fingerprint(path: &Path) -> Result<Arc<Fingerprint>> {
+ cache_memory(
+ CacheKey::new_json(("acoustid-fingerprint", path)),
+ move || {
+ let child = Command::new("fpcalc")
+ .arg("-json")
+ .arg(path)
+ .stdout(Stdio::piped())
+ .spawn()
+ .context("fpcalc")?;
- let out: FpCalcOutput = serde_json::from_slice(&buf).context("parsing fpcalc output")?;
- let out = Fingerprint {
- duration: out.duration as u32,
- fingerprint: out.fingerprint,
- };
+ let mut buf = Vec::new();
+ child
+ .stdout
+ .unwrap()
+ .read_to_end(&mut buf)
+ .context("read fpcalc output")?;
- Ok(out)
- })
- .await
+ let out: FpCalcOutput =
+ serde_json::from_slice(&buf).context("parsing fpcalc output")?;
+ let out = Fingerprint {
+ duration: out.duration as u32,
+ fingerprint: out.fingerprint,
+ };
+ Ok(out)
+ },
+ )
}
diff --git a/import/src/lib.rs b/import/src/lib.rs
index d12913f..3cbabdc 100644
--- a/import/src/lib.rs
+++ b/import/src/lib.rs
@@ -20,11 +20,10 @@ use crate::{tmdb::TmdbKind, trakt::TraktKind};
use acoustid::{acoustid_fingerprint, AcoustID};
use anyhow::{anyhow, bail, Context, Result};
use infojson::YVideo;
-use jellycache::{cache_file, cache_memory};
+use jellycache::{cache, cache_memory, cache_store, CacheKey};
use jellycommon::{
- Appearance, Asset, Chapter, CreditCategory, IdentifierType, LocalTrack, MediaInfo, Node,
- NodeID, NodeKind, PictureSlot, RatingType, SourceTrack, SourceTrackKind, TrackSource,
- Visibility,
+ Appearance, Chapter, CreditCategory, IdentifierType, MediaInfo, Node, NodeID, NodeKind,
+ Picture, PictureSlot, RatingType, SourceTrack, SourceTrackKind, TrackSource, Visibility,
};
use jellyimport_fallback_generator::generate_fallback;
use jellyremuxer::{
@@ -39,7 +38,7 @@ use serde::{Deserialize, Serialize};
use std::{
collections::{BTreeMap, HashMap},
fs::{read_to_string, File},
- io::{self, BufReader, Write},
+ io::{BufReader, Read},
path::{Path, PathBuf},
sync::{Arc, LazyLock, Mutex},
time::UNIX_EPOCH,
@@ -262,23 +261,27 @@ fn import_file(
match filename.as_ref() {
"poster.jpeg" | "poster.webp" | "poster.png" => {
info!("import poster at {path:?}");
- let path = cache_file("picture-file", path, |mut f| {
- io::copy(&mut File::open(path)?, &mut f)?;
- Ok(())
+ let key = CacheKey::new_image(path);
+ cache(key, || {
+ let mut data = Vec::new();
+ File::open(path)?.read_to_end(&mut data)?;
+ Ok(data)
})?;
db.update_node_init(parent, |node| {
- node.pictures.insert(PictureSlot::Poster, Asset(path.0));
+ node.pictures.insert(PictureSlot::Cover, Picture(key.0));
Ok(())
})?;
}
"backdrop.jpeg" | "backdrop.webp" | "backdrop.png" => {
info!("import backdrop at {path:?}");
- let path = cache_file("picture-file", path, |mut f| {
- io::copy(&mut File::open(path)?, &mut f)?;
- Ok(())
+ let key = CacheKey::new_image(path);
+ cache(key, || {
+ let mut data = Vec::new();
+ File::open(path)?.read_to_end(&mut data)?;
+ Ok(data)
})?;
db.update_node_init(parent, |node| {
- node.pictures.insert(PictureSlot::Backdrop, Asset(path.0));
+ node.pictures.insert(PictureSlot::Backdrop, Picture(key.0));
Ok(())
})?;
}
@@ -353,7 +356,7 @@ fn import_file(
}
pub fn read_media_metadata(path: &Path) -> Result<Arc<matroska::Segment>> {
- cache_memory("mkmeta-v4", path, move || {
+ cache_memory(CacheKey::new_json(path), move || {
let media = File::open(path)?;
let mut media =
create_demuxer_autodetect(Box::new(media))?.ok_or(anyhow!("media format unknown"))?;
@@ -400,9 +403,8 @@ fn import_media_file(
.flat_map(|a| &a.files)
.find(|a| a.name.starts_with("cover") && a.media_type.starts_with("image/"))
.map(|att| {
- cache_file("att-cover-v2", path, move |mut file| {
- file.write_all(&att.data)?;
- Ok(())
+ cache_store(CacheKey::new_image(("cover", path)), || {
+ Ok(att.data.clone())
})
})
.transpose()?;
@@ -488,13 +490,13 @@ fn import_media_file(
}
if iflags.use_acoustid {
- let fp = rthandle.block_on(acoustid_fingerprint(path))?;
- if let Some((atid, mbid)) = rthandle.block_on(
- apis.acoustid
- .as_ref()
- .ok_or(anyhow!("need acoustid"))?
- .get_atid_mbid(&fp),
- )? {
+ let fp = acoustid_fingerprint(path)?;
+ if let Some((atid, mbid)) = apis
+ .acoustid
+ .as_ref()
+ .ok_or(anyhow!("need acoustid"))?
+ .get_atid_mbid(&fp, rthandle)?
+ {
eids.insert(IdentifierType::AcoustIdTrack, atid);
eids.insert(IdentifierType::MusicbrainzRecording, mbid);
};
@@ -517,7 +519,7 @@ fn import_media_file(
node.identifiers.extend(eids);
if let Some(cover) = cover {
- node.pictures.insert(PictureSlot::Cover, Asset(cover.0));
+ node.pictures.insert(PictureSlot::Cover, Picture(cover.0));
}
if let Some(ct) = tags.get("CONTENT_TYPE") {
@@ -664,18 +666,17 @@ fn import_media_file(
}
if let Some(trakt_id) = trakt_id {
let trakt = apis.trakt.as_ref().ok_or(anyhow!("trakt required"))?;
- let seasons = rthandle.block_on(trakt.show_seasons(trakt_id))?;
+ let seasons = trakt.show_seasons(trakt_id, rthandle)?;
if seasons.iter().any(|x| x.number == season) {
- let episodes = rthandle.block_on(trakt.show_season_episodes(trakt_id, season))?;
+ let episodes = trakt.show_season_episodes(trakt_id, season, rthandle)?;
let mut poster = None;
if let Some(tmdb) = &apis.tmdb {
- let trakt_details =
- rthandle.block_on(trakt.lookup(TraktKind::Show, trakt_id))?;
+ let trakt_details = trakt.lookup(TraktKind::Show, trakt_id, rthandle)?;
if let Some(tmdb_id) = trakt_details.ids.tmdb {
let tmdb_details =
- rthandle.block_on(tmdb.episode_details(tmdb_id, season, episode))?;
+ tmdb.episode_details(tmdb_id, season, episode, rthandle)?;
if let Some(still) = &tmdb_details.still_path {
- poster = Some(Asset(rthandle.block_on(tmdb.image(still))?.0))
+ poster = Some(Picture(tmdb.image(still, rthandle)?.0))
}
}
}
@@ -685,7 +686,7 @@ fn import_media_file(
node.index = Some(episode.number);
node.title = Some(episode.title.clone());
if let Some(poster) = poster {
- node.pictures.insert(PictureSlot::Poster, poster);
+ node.pictures.insert(PictureSlot::Cover, poster);
}
node.description = episode.overview.clone().or(node.description.clone());
node.ratings.insert(RatingType::Trakt, episode.rating);
@@ -768,7 +769,7 @@ fn apply_musicbrainz_recording(
node: NodeID,
mbid: String,
) -> Result<()> {
- let rec = rthandle.block_on(apis.musicbrainz.lookup_recording(mbid))?;
+ let rec = apis.musicbrainz.lookup_recording(mbid, rthandle)?;
db.update_node_init(node, |node| {
node.title = Some(rec.title.clone());
@@ -800,8 +801,9 @@ fn apply_musicbrainz_recording(
if let Some((note, group)) = a {
let artist = rel.artist.as_ref().unwrap();
- let artist =
- rthandle.block_on(apis.musicbrainz.lookup_artist(artist.id.clone()))?;
+ let artist = apis
+ .musicbrainz
+ .lookup_artist(artist.id.clone(), rthandle)?;
let mut image_1 = None;
let mut image_2 = None;
@@ -811,13 +813,13 @@ fn apply_musicbrainz_recording(
WIKIDATA => {
let url = rel.url.as_ref().unwrap().resource.clone();
if let Some(id) = url.strip_prefix("https://www.wikidata.org/wiki/") {
- if let Some(filename) = rthandle
- .block_on(apis.wikidata.query_image_path(id.to_owned()))?
+ if let Some(filename) =
+ apis.wikidata.query_image_path(id.to_owned(), rthandle)?
{
- let path = rthandle.block_on(
- apis.wikimedia_commons.image_by_filename(filename),
- )?;
- image_1 = Some(Asset(path.0));
+ let path = apis
+ .wikimedia_commons
+ .image_by_filename(filename, rthandle)?;
+ image_1 = Some(Picture(path.0));
}
}
}
@@ -825,10 +827,8 @@ fn apply_musicbrainz_recording(
let url = rel.url.as_ref().unwrap().resource.clone();
if let Some(id) = url.strip_prefix("https://vgmdb.net/artist/") {
let id = id.parse::<u64>().context("parse vgmdb id")?;
- if let Some(path) =
- rthandle.block_on(apis.vgmdb.get_artist_image(id))?
- {
- image_2 = Some(Asset(path.0));
+ if let Some(path) = apis.vgmdb.get_artist_image(id, rthandle)? {
+ image_2 = Some(Picture(path.0));
}
}
}
@@ -843,10 +843,9 @@ fn apply_musicbrainz_recording(
let headshot = match image_1.or(image_2) {
Some(x) => x,
- None => Asset(
- cache_file("person-headshot-fallback", &artist.sort_name, |mut file| {
- generate_fallback(&artist.sort_name, &mut file)?;
- Ok(())
+ None => Picture(
+ cache_store(CacheKey::new_image(("fallback", &artist.sort_name)), || {
+ generate_fallback(&artist.sort_name)
})?
.0,
),
@@ -879,12 +878,8 @@ fn apply_trakt_tmdb(
) -> Result<()> {
let trakt_id: u64 = trakt_id.parse().context("parse trakt id")?;
if let (Some(trakt), Some(tmdb)) = (&apis.trakt, &apis.tmdb) {
- let data = rthandle
- .block_on(trakt.lookup(trakt_kind, trakt_id))
- .context("trakt lookup")?;
- let people = rthandle
- .block_on(trakt.people(trakt_kind, trakt_id))
- .context("trakt people lookup")?;
+ let data = trakt.lookup(trakt_kind, trakt_id, rthandle)?;
+ let people = trakt.people(trakt_kind, trakt_id, rthandle)?;
let mut people_map = BTreeMap::<CreditCategory, Vec<Appearance>>::new();
for p in people.cast.iter() {
@@ -903,29 +898,24 @@ fn apply_trakt_tmdb(
let mut backdrop = None;
let mut poster = None;
if let Some(tmdb_id) = data.ids.tmdb {
- let data = rthandle
- .block_on(tmdb.details(
- match trakt_kind {
- TraktKind::Movie => TmdbKind::Movie,
- TraktKind::Show => TmdbKind::Tv,
- _ => TmdbKind::Movie,
- },
- tmdb_id,
- ))
- .context("tmdb details")?;
+ let data = tmdb.details(
+ match trakt_kind {
+ TraktKind::Movie => TmdbKind::Movie,
+ TraktKind::Show => TmdbKind::Tv,
+ _ => TmdbKind::Movie,
+ },
+ tmdb_id,
+ rthandle,
+ )?;
tmdb_data = Some(data.clone());
if let Some(path) = &data.backdrop_path {
- let im = rthandle
- .block_on(tmdb.image(path))
- .context("tmdb backdrop image")?;
- backdrop = Some(Asset(im.0));
+ let im = tmdb.image(path, rthandle).context("tmdb backdrop image")?;
+ backdrop = Some(Picture(im.0));
}
if let Some(path) = &data.poster_path {
- let im = rthandle
- .block_on(tmdb.image(path))
- .context("tmdb poster image")?;
- poster = Some(Asset(im.0));
+ let im = tmdb.image(path, rthandle).context("tmdb poster image")?;
+ poster = Some(Picture(im.0));
}
// for p in people_map.values_mut().flatten() {
@@ -960,7 +950,7 @@ fn apply_trakt_tmdb(
node.ratings.insert(RatingType::Trakt, *rating);
}
if let Some(poster) = poster {
- node.pictures.insert(PictureSlot::Poster, poster);
+ node.pictures.insert(PictureSlot::Cover, poster);
}
if let Some(backdrop) = backdrop {
node.pictures.insert(PictureSlot::Backdrop, backdrop);
diff --git a/import/src/musicbrainz.rs b/import/src/musicbrainz.rs
index 934a9e8..92df703 100644
--- a/import/src/musicbrainz.rs
+++ b/import/src/musicbrainz.rs
@@ -6,7 +6,7 @@
use crate::USER_AGENT;
use anyhow::{Context, Result};
-use jellycache::async_cache_memory;
+use jellycache::{cache_memory, CacheContentType, CacheKey};
use log::info;
use reqwest::{
header::{HeaderMap, HeaderName, HeaderValue},
@@ -15,6 +15,7 @@ use reqwest::{
use serde::{Deserialize, Serialize};
use std::{collections::BTreeMap, sync::Arc, time::Duration};
use tokio::{
+ runtime::Handle,
sync::Semaphore,
time::{sleep_until, Instant},
};
@@ -221,95 +222,108 @@ impl MusicBrainz {
}
}
- pub async fn lookup_recording(&self, id: String) -> Result<Arc<MbRecordingRel>> {
- async_cache_memory("api-musicbrainz-recording", id.clone(), || async move {
- let _permit = self.rate_limit.clone().acquire_owned().await?;
- let permit_drop_ts = Instant::now() + Duration::from_secs(Self::MAX_PAR_REQ as u64);
- info!("recording lookup: {id}");
+ pub fn lookup_recording(&self, id: String, rt: &Handle) -> Result<Arc<MbRecordingRel>> {
+ cache_memory(
+ CacheKey::new(
+ CacheContentType::Json,
+ ("musicbrainz-recording-lookup", &id),
+ ),
+ move || {
+ rt.block_on(async {
+ let _permit = self.rate_limit.clone().acquire_owned().await?;
+ let permit_drop_ts =
+ Instant::now() + Duration::from_secs(Self::MAX_PAR_REQ as u64);
+ info!("recording lookup: {id}");
- let inc = [
- "isrcs",
- "artists",
- "area-rels",
- "artist-rels",
- "event-rels",
- "genre-rels",
- "instrument-rels",
- "label-rels",
- "place-rels",
- "recording-rels",
- "release-rels",
- "release-group-rels",
- "series-rels",
- "url-rels",
- "work-rels",
- ]
- .join("+");
+ let inc = [
+ "isrcs",
+ "artists",
+ "area-rels",
+ "artist-rels",
+ "event-rels",
+ "genre-rels",
+ "instrument-rels",
+ "label-rels",
+ "place-rels",
+ "recording-rels",
+ "release-rels",
+ "release-group-rels",
+ "series-rels",
+ "url-rels",
+ "work-rels",
+ ]
+ .join("+");
- let resp = self
- .client
- .get(format!(
- "https://musicbrainz.org/ws/2/recording/{id}?inc={inc}"
- ))
- .send()
- .await?
- .error_for_status()?
- .json::<MbRecordingRel>()
- .await?;
+ let resp = self
+ .client
+ .get(format!(
+ "https://musicbrainz.org/ws/2/recording/{id}?inc={inc}"
+ ))
+ .send()
+ .await?
+ .error_for_status()?
+ .json::<MbRecordingRel>()
+ .await?;
- tokio::task::spawn(async move {
- sleep_until(permit_drop_ts).await;
- drop(_permit);
- });
+ tokio::task::spawn(async move {
+ sleep_until(permit_drop_ts).await;
+ drop(_permit);
+ });
- Ok(resp)
- })
- .await
+ Ok(resp)
+ })
+ },
+ )
.context("musicbrainz recording lookup")
}
- pub async fn lookup_artist(&self, id: String) -> Result<Arc<MbArtistRel>> {
- async_cache_memory("api-musicbrainz-artist", id.clone(), || async move {
- let _permit = self.rate_limit.clone().acquire_owned().await?;
- let permit_drop_ts = Instant::now() + Duration::from_secs(Self::MAX_PAR_REQ as u64);
- info!("artist lookup: {id}");
+ pub fn lookup_artist(&self, id: String, rt: &Handle) -> Result<Arc<MbArtistRel>> {
+ cache_memory(
+ CacheKey::new(CacheContentType::Json, ("musicbrainz-artist-lookup", &id)),
+ move || {
+ rt.block_on(async {
+ let _permit = self.rate_limit.clone().acquire_owned().await?;
+ let permit_drop_ts =
+ Instant::now() + Duration::from_secs(Self::MAX_PAR_REQ as u64);
+ info!("artist lookup: {id}");
- let inc = [
- "area-rels",
- "artist-rels",
- "event-rels",
- "genre-rels",
- "instrument-rels",
- "label-rels",
- "place-rels",
- "recording-rels",
- "release-rels",
- "release-group-rels",
- "series-rels",
- "url-rels",
- "work-rels",
- ]
- .join("+");
+ let inc = [
+ "area-rels",
+ "artist-rels",
+ "event-rels",
+ "genre-rels",
+ "instrument-rels",
+ "label-rels",
+ "place-rels",
+ "recording-rels",
+ "release-rels",
+ "release-group-rels",
+ "series-rels",
+ "url-rels",
+ "work-rels",
+ ]
+ .join("+");
- let resp = self
- .client
- .get(format!(
- "https://musicbrainz.org/ws/2/artist/{id}?inc={inc}"
- ))
- .send()
- .await?
- .error_for_status()?
- .json::<MbArtistRel>()
- .await?;
+ let resp = self
+ .client
+ .get(format!(
+ "https://musicbrainz.org/ws/2/artist/{id}?inc={inc}"
+ ))
+ .send()
+ .await?
+ .error_for_status()?
+ .json::<MbArtistRel>()
+ .await?;
- tokio::task::spawn(async move {
- sleep_until(permit_drop_ts).await;
- drop(_permit);
- });
+ tokio::task::spawn(async move {
+ sleep_until(permit_drop_ts).await;
+ drop(_permit);
+ });
- Ok(resp)
- })
- .await
+ Ok(resp)
+ })
+ },
+ )
.context("musicbrainz artist lookup")
}
}
diff --git a/import/src/tmdb.rs b/import/src/tmdb.rs
index 414058f..d4a0c25 100644
--- a/import/src/tmdb.rs
+++ b/import/src/tmdb.rs
@@ -4,8 +4,8 @@
Copyright (C) 2025 metamuffin <metamuffin.org>
*/
use crate::USER_AGENT;
-use anyhow::{anyhow, bail, Context};
-use jellycache::{async_cache_file, async_cache_memory, CachePath};
+use anyhow::{anyhow, bail, Context, Result};
+use jellycache::{cache_memory, cache_store, CacheKey};
use jellycommon::chrono::{format::Parsed, Utc};
use log::info;
use reqwest::{
@@ -14,7 +14,7 @@ use reqwest::{
};
use serde::{Deserialize, Serialize};
use std::{fmt::Display, sync::Arc};
-use tokio::io::AsyncWriteExt;
+use tokio::runtime::Handle;
pub struct Tmdb {
client: Client,
@@ -44,100 +44,109 @@ impl Tmdb {
key: api_key.to_owned(),
}
}
- pub async fn search(&self, kind: TmdbKind, query: &str) -> anyhow::Result<Arc<TmdbQuery>> {
- async_cache_memory("api-tmdb-search", (kind, query), || async move {
- info!("searching tmdb: {query:?}");
- Ok(self
- .client
- .get(format!(
- "https://api.themoviedb.org/3/search/{kind}?query={}?api_key={}",
- query.replace(" ", "+"),
- self.key
- ))
- .send()
- .await?
- .error_for_status()?
- .json::<TmdbQuery>()
- .await?)
- })
- .await
+ pub fn search(&self, kind: TmdbKind, query: &str, rt: &Handle) -> Result<Arc<TmdbQuery>> {
+ cache_memory(
+ CacheKey::new_json(("api-tmdb-search", kind, query)),
+ move || {
+ rt.block_on(async {
+ info!("searching tmdb: {query:?}");
+ Ok(self
+ .client
+ .get(format!(
+ "https://api.themoviedb.org/3/search/{kind}?query={}?api_key={}",
+ query.replace(" ", "+"),
+ self.key
+ ))
+ .send()
+ .await?
+ .error_for_status()?
+ .json::<TmdbQuery>()
+ .await?)
+ })
+ },
+ )
.context("tmdb search")
}
- pub async fn details(&self, kind: TmdbKind, id: u64) -> anyhow::Result<Arc<TmdbDetails>> {
- async_cache_memory("api-tmdb-details", (kind, id), || async move {
- info!("fetching details: {id:?}");
- Ok(self
- .client
- .get(format!(
- "https://api.themoviedb.org/3/{kind}/{id}?api_key={}",
- self.key,
- ))
- .send()
- .await?
- .error_for_status()?
- .json()
- .await?)
+ pub fn details(&self, kind: TmdbKind, id: u64, rt: &Handle) -> Result<Arc<TmdbDetails>> {
+ cache_memory(CacheKey::new_json(("tmdb-details", kind, id)), move || {
+ rt.block_on(async {
+ info!("fetching details: {id:?}");
+ Ok(self
+ .client
+ .get(format!(
+ "https://api.themoviedb.org/3/{kind}/{id}?api_key={}",
+ self.key,
+ ))
+ .send()
+ .await?
+ .error_for_status()?
+ .json()
+ .await?)
+ })
})
- .await
.context("tmdb details")
}
- pub async fn person_image(&self, id: u64) -> anyhow::Result<Arc<TmdbPersonImage>> {
- async_cache_memory("api-tmdb-search", id, || async move {
- Ok(self
- .client
- .get(format!(
- "https://api.themoviedb.org/3/person/{id}/images?api_key={}",
- self.key,
- ))
- .send()
- .await?
- .error_for_status()?
- .json()
- .await?)
+ pub fn person_image(&self, id: u64, rt: &Handle) -> Result<Arc<TmdbPersonImage>> {
+ cache_memory(CacheKey::new_image(("tmdb-person-image", id)), move || {
+ rt.block_on(async {
+ Ok(self
+ .client
+ .get(format!(
+ "https://api.themoviedb.org/3/person/{id}/images?api_key={}",
+ self.key,
+ ))
+ .send()
+ .await?
+ .error_for_status()?
+ .json()
+ .await?)
+ })
})
- .await
.context("tmdb person images")
}
- pub async fn image(&self, path: &str) -> anyhow::Result<CachePath> {
- async_cache_file("api-tmdb-image", path, |mut file| async move {
- info!("downloading image {path:?}");
- let mut res = self
- .image_client
- .get(format!("https://image.tmdb.org/t/p/original{path}"))
- .send()
- .await?
- .error_for_status()?;
- while let Some(chunk) = res.chunk().await? {
- file.write_all(&chunk).await?;
- }
- Ok(())
+ pub fn image(&self, path: &str, rt: &Handle) -> Result<CacheKey> {
+ cache_store(CacheKey::new_image(("tmdb-image", path)), move || {
+ rt.block_on(async {
+ info!("downloading image {path:?}");
+ Ok(self
+ .image_client
+ .get(format!("https://image.tmdb.org/t/p/original{path}"))
+ .send()
+ .await?
+ .error_for_status()?
+ .bytes()
+ .await?
+ .to_vec())
+ })
})
- .await
.context("tmdb image download")
}
- pub async fn episode_details(
+ pub fn episode_details(
&self,
series_id: u64,
season: usize,
episode: usize,
- ) -> anyhow::Result<Arc<TmdbEpisode>> {
- async_cache_memory("api-tmdb-episode-details", (series_id,season,episode), || async move {
- info!("tmdb episode details {series_id} S={season} E={episode}");
- Ok(self
- .image_client
- .get(format!("https://api.themoviedb.org/3/tv/{series_id}/season/{season}/episode/{episode}?api_key={}", self.key))
- .send()
- .await?
- .error_for_status()?
- .json()
- .await?)
+ rt: &Handle,
+ ) -> Result<Arc<TmdbEpisode>> {
+ cache_memory(CacheKey::new_json(("tmdb-episode-details", series_id, season, episode)), move || {
+ rt.block_on(async {
+ info!("tmdb episode details {series_id} S={season} E={episode}");
+ Ok(self
+ .image_client
+ .get(format!("https://api.themoviedb.org/3/tv/{series_id}/season/{season}/episode/{episode}?api_key={}", self.key))
+ .send()
+ .await?
+ .error_for_status()?
+ .json()
+ .await?)
+ })
})
- .await.context("tmdb episode details")
+ .context("tmdb episode details")
}
}
-pub fn parse_release_date(d: &str) -> anyhow::Result<Option<i64>> {
+pub fn parse_release_date(d: &str) -> Result<Option<i64>> {
if d.is_empty() {
return Ok(None);
} else if d.len() < 10 {
diff --git a/import/src/trakt.rs b/import/src/trakt.rs
index 3f0ad47..1640ca5 100644
--- a/import/src/trakt.rs
+++ b/import/src/trakt.rs
@@ -5,7 +5,7 @@
*/
use crate::USER_AGENT;
use anyhow::Context;
-use jellycache::async_cache_memory;
+use jellycache::{cache_memory, CacheKey};
use jellycommon::{Appearance, CreditCategory, NodeID};
use log::info;
use reqwest::{
@@ -14,6 +14,7 @@ use reqwest::{
};
use serde::{Deserialize, Serialize};
use std::{collections::BTreeMap, fmt::Display, sync::Arc};
+use tokio::runtime::Handle;
pub struct Trakt {
client: Client,
@@ -45,76 +46,97 @@ impl Trakt {
Self { client }
}
- pub async fn search(
+ pub fn search(
&self,
kinds: &[TraktKind],
query: &str,
+ rt: &Handle,
) -> anyhow::Result<Arc<Vec<TraktSearchResult>>> {
- async_cache_memory("api-trakt-lookup", (kinds, query), || async move {
- let url = format!(
- "https://api.trakt.tv/search/{}?query={}&extended=full",
- kinds
- .iter()
- .map(|t| t.singular())
- .collect::<Vec<_>>()
- .join(","),
- urlencoding::encode(query),
- );
- let res = self.client.get(url).send().await?.error_for_status()?;
- Ok(res.json().await?)
+ cache_memory(CacheKey::new_json(("trakt-lookup", query)), move || {
+ rt.block_on(async {
+ let url = format!(
+ "https://api.trakt.tv/search/{}?query={}&extended=full",
+ kinds
+ .iter()
+ .map(|t| t.singular())
+ .collect::<Vec<_>>()
+ .join(","),
+ urlencoding::encode(query),
+ );
+ let res = self.client.get(url).send().await?.error_for_status()?;
+ Ok(res.json().await?)
+ })
})
- .await
.context("trakt search")
}
- pub async fn lookup(&self, kind: TraktKind, id: u64) -> anyhow::Result<Arc<TraktMediaObject>> {
- async_cache_memory("api-trakt-lookup", (kind, id), || async move {
- info!("trakt lookup {kind:?}:{id:?}");
- let url = format!("https://api.trakt.tv/{}/{id}?extended=full", kind.plural());
- let res = self.client.get(url).send().await?.error_for_status()?;
- Ok(res.json().await?)
+ pub fn lookup(
+ &self,
+ kind: TraktKind,
+ id: u64,
+ rt: &Handle,
+ ) -> anyhow::Result<Arc<TraktMediaObject>> {
+ cache_memory(CacheKey::new_json(("trakt-lookup", kind, id)), move || {
+ rt.block_on(async {
+ info!("trakt lookup {kind:?}:{id:?}");
+ let url = format!("https://api.trakt.tv/{}/{id}?extended=full", kind.plural());
+ let res = self.client.get(url).send().await?.error_for_status()?;
+ Ok(res.json().await?)
+ })
})
- .await
.context("trakt lookup")
}
- pub async fn people(&self, kind: TraktKind, id: u64) -> anyhow::Result<Arc<TraktPeople>> {
- async_cache_memory("api-trakt-people", (kind, id), || async move {
- info!("trakt people {kind:?}:{id:?}");
- let url = format!(
- "https://api.trakt.tv/{}/{id}/people?extended=full",
- kind.plural()
- );
- let res = self.client.get(url).send().await?.error_for_status()?;
- Ok(res.json().await?)
+ pub fn people(
+ &self,
+ kind: TraktKind,
+ id: u64,
+ rt: &Handle,
+ ) -> anyhow::Result<Arc<TraktPeople>> {
+ cache_memory(CacheKey::new_json(("trakt-people", kind, id)), move || {
+ rt.block_on(async {
+ info!("trakt people {kind:?}:{id:?}");
+ let url = format!(
+ "https://api.trakt.tv/{}/{id}/people?extended=full",
+ kind.plural()
+ );
+ let res = self.client.get(url).send().await?.error_for_status()?;
+ Ok(res.json().await?)
+ })
})
- .await
.context("trakt people")
}
- pub async fn show_seasons(&self, id: u64) -> anyhow::Result<Arc<Vec<TraktSeason>>> {
- async_cache_memory("api-trakt-seasons", id, || async move {
- info!("trakt seasons {id:?}");
- let url = format!("https://api.trakt.tv/shows/{id}/seasons?extended=full");
- let res = self.client.get(url).send().await?.error_for_status()?;
- Ok(res.json().await?)
+ pub fn show_seasons(&self, id: u64, rt: &Handle) -> anyhow::Result<Arc<Vec<TraktSeason>>> {
+ cache_memory(CacheKey::new_json(("trakt-seasons", id)), move || {
+ rt.block_on(async {
+ info!("trakt seasons {id:?}");
+ let url = format!("https://api.trakt.tv/shows/{id}/seasons?extended=full");
+ let res = self.client.get(url).send().await?.error_for_status()?;
+ Ok(res.json().await?)
+ })
})
- .await
.context("trakt show seasons")
}
- pub async fn show_season_episodes(
+ pub fn show_season_episodes(
&self,
id: u64,
season: usize,
+ rt: &Handle,
) -> anyhow::Result<Arc<Vec<TraktEpisode>>> {
- async_cache_memory("api-trakt-episodes", (id, season), || async move {
- info!("trakt episodes {id:?} season={season}");
- let url = format!("https://api.trakt.tv/shows/{id}/seasons/{season}?extended=full");
- let res = self.client.get(url).send().await?.error_for_status()?;
- Ok(res.json().await?)
- })
- .await
+ cache_memory(
+ CacheKey::new_json(("trakt-episodes", id, season)),
+ move || {
+ rt.block_on(async {
+ info!("trakt episodes {id:?} season={season}");
+ let url =
+ format!("https://api.trakt.tv/shows/{id}/seasons/{season}?extended=full");
+ let res = self.client.get(url).send().await?.error_for_status()?;
+ Ok(res.json().await?)
+ })
+ },
+ )
.context("trakt show season episodes")
}
}
diff --git a/import/src/vgmdb.rs b/import/src/vgmdb.rs
index 20a0038..4e37ba3 100644
--- a/import/src/vgmdb.rs
+++ b/import/src/vgmdb.rs
@@ -6,7 +6,7 @@
use crate::USER_AGENT;
use anyhow::{Context, Result};
-use jellycache::{async_cache_file, async_cache_memory, CachePath};
+use jellycache::{cache_memory, cache_store, CacheContentType, CacheKey};
use log::info;
use regex::Regex;
use reqwest::{
@@ -18,7 +18,7 @@ use std::{
time::Duration,
};
use tokio::{
- io::AsyncWriteExt,
+ runtime::Handle,
sync::Semaphore,
time::{sleep_until, Instant},
};
@@ -59,18 +59,26 @@ impl Vgmdb {
}
}
- pub async fn get_artist_image(&self, id: u64) -> Result<Option<CachePath>> {
- if let Some(url) = self.get_artist_image_url(id).await? {
+ pub fn get_artist_image(&self, id: u64, rt: &Handle) -> Result<Option<CacheKey>> {
+ if let Some(url) = self.get_artist_image_url(id, rt)? {
Ok(Some(
- async_cache_file("api-vgmdb-media", url.clone(), |mut file| async move {
- info!("downloading image {url:?}");
- let mut res = self.client.get(url).send().await?.error_for_status()?;
- while let Some(chunk) = res.chunk().await? {
- file.write_all(&chunk).await?;
- }
- Ok(())
- })
- .await
+ cache_store(
+ CacheKey::new_image(("vgmdb-artist-media", &url)),
+ move || {
+ rt.block_on(async {
+ info!("downloading image {url:?}");
+ Ok(self
+ .client
+ .get(url)
+ .send()
+ .await?
+ .error_for_status()?
+ .bytes()
+ .await?
+ .to_vec())
+ })
+ },
+ )
.context("vgmdb media download")?,
))
} else {
@@ -78,9 +86,9 @@ impl Vgmdb {
}
}
- pub async fn get_artist_image_url(&self, id: u64) -> Result<Option<String>> {
- let html = self.scrape_artist_page(id).await?;
- if let Some(cap) = RE_IMAGE_URL_FROM_HTML.captures(&html) {
+ pub fn get_artist_image_url(&self, id: u64, rt: &Handle) -> Result<Option<String>> {
+ let html = self.scrape_artist_page(id, rt)?;
+ if let Some(cap) = RE_IMAGE_URL_FROM_HTML.captures(&str::from_utf8(&html).unwrap()) {
if let Some(url) = cap.name("url").map(|m| m.as_str()) {
return Ok(Some(url.to_string()));
}
@@ -88,29 +96,34 @@ impl Vgmdb {
Ok(None)
}
- pub async fn scrape_artist_page(&self, id: u64) -> Result<Arc<String>> {
- async_cache_memory("api-vgmdb-artist", id, || async move {
- let _permit = self.rate_limit.clone().acquire_owned().await?;
- let permit_drop_ts = Instant::now() + Duration::from_secs(1);
- info!("scrape artist: {id}");
+ pub fn scrape_artist_page(&self, id: u64, rt: &Handle) -> Result<Arc<Vec<u8>>> {
+ cache_memory(
+ CacheKey::new(CacheContentType::Unknown, ("vgmdb-artist-page", id)),
+ move || {
+ rt.block_on(async {
+ let _permit = self.rate_limit.clone().acquire_owned().await?;
+ let permit_drop_ts = Instant::now() + Duration::from_secs(1);
+ info!("scrape artist: {id}");
- let resp = self
- .client
- .get(format!("https://vgmdb.net/artist/{id}"))
- .send()
- .await?
- .error_for_status()?
- .text()
- .await?;
+ let resp = self
+ .client
+ .get(format!("https://vgmdb.net/artist/{id}"))
+ .send()
+ .await?
+ .error_for_status()?
+ .bytes()
+ .await?
+ .to_vec();
- tokio::task::spawn(async move {
- sleep_until(permit_drop_ts).await;
- drop(_permit);
- });
+ tokio::task::spawn(async move {
+ sleep_until(permit_drop_ts).await;
+ drop(_permit);
+ });
- Ok(resp)
- })
- .await
+ Ok(resp)
+ })
+ },
+ )
.context("vgmdb artist page scrape")
}
}
diff --git a/import/src/wikidata.rs b/import/src/wikidata.rs
index 71eef9a..40077b9 100644
--- a/import/src/wikidata.rs
+++ b/import/src/wikidata.rs
@@ -6,26 +6,27 @@
use crate::USER_AGENT;
use anyhow::{bail, Context, Result};
-use jellycache::async_cache_memory;
+use jellycache::{cache_memory, CacheKey};
use log::info;
use reqwest::{
header::{HeaderMap, HeaderName, HeaderValue},
Client, ClientBuilder,
};
-use serde::Deserialize;
+use serde::{Deserialize, Serialize};
use serde_json::Value;
-use std::collections::BTreeMap;
+use std::{collections::BTreeMap, sync::Arc};
+use tokio::runtime::Handle;
pub struct Wikidata {
client: Client,
}
-#[derive(Debug, Deserialize, Clone)]
+#[derive(Debug, Deserialize, Serialize, Clone)]
pub struct WikidataResponse {
entities: BTreeMap<String, WikidataEntity>,
}
-#[derive(Debug, Deserialize, Clone)]
+#[derive(Debug, Deserialize, Serialize, Clone)]
pub struct WikidataEntity {
pub pageid: u64,
pub ns: u64,
@@ -36,7 +37,7 @@ pub struct WikidataEntity {
pub id: String,
pub claims: BTreeMap<String, Vec<WikidataClaim>>,
}
-#[derive(Debug, Deserialize, Clone)]
+#[derive(Debug, Deserialize, Serialize, Clone)]
pub struct WikidataClaim {
pub r#type: String,
pub id: String,
@@ -44,7 +45,7 @@ pub struct WikidataClaim {
pub mainsnak: WikidataSnak,
}
-#[derive(Debug, Deserialize, Clone)]
+#[derive(Debug, Deserialize, Serialize, Clone)]
pub struct WikidataSnak {
pub snaktype: String,
pub property: String,
@@ -53,7 +54,7 @@ pub struct WikidataSnak {
pub datatype: String,
}
-#[derive(Debug, Deserialize, Clone)]
+#[derive(Debug, Deserialize, Serialize, Clone)]
pub struct WikidataValue {
pub value: Value,
pub r#type: String,
@@ -87,8 +88,8 @@ impl Wikidata {
Self { client }
}
- pub async fn query_image_path(&self, id: String) -> Result<Option<String>> {
- let response = self.query(id.clone()).await?;
+ pub fn query_image_path(&self, id: String, rt: &Handle) -> Result<Option<String>> {
+ let response = self.query(id.clone(), rt)?;
if let Some(entity) = response.entities.get(&id) {
if let Some(images) = entity.claims.get(properties::IMAGE) {
for image in images {
@@ -106,24 +107,20 @@ impl Wikidata {
Ok(None)
}
- pub async fn query(&self, id: String) -> Result<WikidataResponse> {
- let json = async_cache_memory("api-wikidata", id.clone(), || async move {
- info!("entity query: {id}");
-
- let resp = self
- .client
- .get(format!("https://www.wikidata.org/entity/{id}"))
- .send()
- .await?
- .error_for_status()?
- .text()
- .await?;
-
- Ok(resp)
+ pub fn query(&self, id: String, rt: &Handle) -> Result<Arc<WikidataResponse>> {
+ cache_memory(CacheKey::new_json(("wikidata", &id)), move || {
+ rt.block_on(async {
+ info!("entity query: {id}");
+ Ok(self
+ .client
+ .get(format!("https://www.wikidata.org/entity/{id}"))
+ .send()
+ .await?
+ .error_for_status()?
+ .json()
+ .await?)
+ })
})
- .await
- .context("wikidata entity")?;
-
- serde_json::from_str(&json).context("parse wikidata entity")
+ .context("wikidata entity")
}
}
diff --git a/import/src/wikimedia_commons.rs b/import/src/wikimedia_commons.rs
index f8d8f53..0d716f0 100644
--- a/import/src/wikimedia_commons.rs
+++ b/import/src/wikimedia_commons.rs
@@ -6,13 +6,13 @@
use crate::USER_AGENT;
use anyhow::{Context, Result};
-use jellycache::{async_cache_file, CachePath};
+use jellycache::{cache_store, CacheKey};
use reqwest::{
header::{HeaderMap, HeaderName, HeaderValue},
redirect::Policy,
Client, ClientBuilder,
};
-use tokio::io::AsyncWriteExt;
+use tokio::runtime::Handle;
pub struct WikimediaCommons {
client: Client,
@@ -36,28 +36,26 @@ impl WikimediaCommons {
Self { client }
}
- pub async fn image_by_filename(&self, filename: String) -> Result<CachePath> {
- async_cache_file(
- "api-wikimedia-commons-image",
- filename.clone(),
- |mut file| async move {
- let mut res = self
- .client
- .get(format!(
- "https://commons.wikimedia.org/wiki/Special:FilePath/{}",
- filename.replace(" ", "_")
- ))
- .send()
- .await?
- .error_for_status()?;
-
- while let Some(chunk) = res.chunk().await? {
- file.write_all(&chunk).await?;
- }
- Ok(())
+ pub fn image_by_filename(&self, filename: String, rt: &Handle) -> Result<CacheKey> {
+ cache_store(
+ CacheKey::new_image(("wikimedia-commons-image", &filename)),
+ move || {
+ rt.block_on(async {
+ Ok(self
+ .client
+ .get(format!(
+ "https://commons.wikimedia.org/wiki/Special:FilePath/{}",
+ filename.replace(" ", "_")
+ ))
+ .send()
+ .await?
+ .error_for_status()?
+ .bytes()
+ .await?
+ .to_vec())
+ })
},
)
- .await
.context("mediawiki image by filename")
}
}
generated by cgit v1.3 (git 2.53.0) at 2026-03-25 13:31:39 +0000