aboutsummaryrefslogtreecommitdiff
path: root/import/src
diff options
context:
space:
mode:
authormetamuffin <metamuffin@disroot.org>2026-01-24 23:06:33 +0100
committermetamuffin <metamuffin@disroot.org>2026-01-24 23:06:33 +0100
commit2bcccb18a6cb8bf836f57c3d86f759b19699def2 (patch)
treeef55a10c6d9703677a983b8ca900fb4578a08eb3 /import/src
parentb2e88a8beabf04adc28947cf82996e8692a68b71 (diff)
downloadjellything-2bcccb18a6cb8bf836f57c3d86f759b19699def2.tar
jellything-2bcccb18a6cb8bf836f57c3d86f759b19699def2.tar.bz2
jellything-2bcccb18a6cb8bf836f57c3d86f759b19699def2.tar.zst
cache as object
Diffstat (limited to 'import/src')
-rw-r--r--import/src/lib.rs10
-rw-r--r--import/src/plugins/acoustid.rs30
-rw-r--r--import/src/plugins/infojson.rs15
-rw-r--r--import/src/plugins/misc.rs6
-rw-r--r--import/src/plugins/musicbrainz.rs176
-rw-r--r--import/src/plugins/tmdb.rs160
-rw-r--r--import/src/plugins/trakt.rs155
-rw-r--r--import/src/plugins/vgmdb.rs99
-rw-r--r--import/src/plugins/wikidata.rs48
-rw-r--r--import/src/plugins/wikimedia_commons.rs50
10 files changed, 418 insertions, 331 deletions
diff --git a/import/src/lib.rs b/import/src/lib.rs
index 7d48867..bec5786 100644
--- a/import/src/lib.rs
+++ b/import/src/lib.rs
@@ -9,7 +9,7 @@ pub mod plugins;
pub mod reporting;
use crate::{
- plugins::{PluginContext, ImportPlugin, infojson::is_info_json, init_plugins, misc::is_cover},
+ plugins::{ImportPlugin, PluginContext, infojson::is_info_json, init_plugins, misc::is_cover},
reporting::IMPORT_PROGRESS,
};
use anyhow::{Context, Result, anyhow};
@@ -377,9 +377,9 @@ fn import_file(
}
reporting::set_task(format!("demuxer meta: {path:?}"));
- let Some(seg) =
- reporting::catch(read_media_metadata(path).context(anyhow!("media {path:?}")))
- else {
+ let Some(seg) = reporting::catch(
+ read_media_metadata(&ct.dba.cache, path).context(anyhow!("media {path:?}")),
+ ) else {
return;
};
for p in plugins {
@@ -542,7 +542,7 @@ pub fn read_media_metadata(cache: &Cache, path: &Path) -> Result<Arc<matroska::S
if let Some(attachments) = &mut attachments {
for att in &mut attachments.files {
if let Some(fname) = is_useful_attachment(&att) {
- let key = cache.cache_store(
+ let key = cache.store(
format!("media/attachment/{}-{fname}", HashKey(path)),
|| Ok(att.data.clone()),
)?;
diff --git a/import/src/plugins/acoustid.rs b/import/src/plugins/acoustid.rs
index 9891927..0e16670 100644
--- a/import/src/plugins/acoustid.rs
+++ b/import/src/plugins/acoustid.rs
@@ -5,10 +5,10 @@
*/
use crate::{
USER_AGENT,
- plugins::{PluginContext, ImportPlugin, PluginInfo},
+ plugins::{ImportPlugin, PluginContext, PluginInfo},
};
use anyhow::{Context, Result};
-use jellycache::{HashKey, cache_memory};
+use jellycache::{Cache, HashKey};
use jellycommon::{
IDENT_ACOUST_ID_TRACK, IDENT_MUSICBRAINZ_RECORDING, NO_IDENTIFIERS, jellyobject::Object,
};
@@ -92,8 +92,13 @@ impl AcoustID {
}
}
- pub fn get_atid_mbid(&self, fp: &Fingerprint, rt: &Handle) -> Result<Option<(String, String)>> {
- let res = self.lookup(fp.to_owned(), rt)?;
+ pub fn get_atid_mbid(
+ &self,
+ cache: &Cache,
+ fp: &Fingerprint,
+ rt: &Handle,
+ ) -> Result<Option<(String, String)>> {
+ let res = self.lookup(cache, fp.to_owned(), rt)?;
for r in &res.results {
if let Some(k) = r.recordings.first() {
return Ok(Some((r.id.clone(), k.id.clone())));
@@ -102,8 +107,13 @@ impl AcoustID {
Ok(None)
}
- pub fn lookup(&self, fp: Fingerprint, rt: &Handle) -> Result<Arc<AcoustIDLookupResponse>> {
- cache_memory(&format!("ext/acoustid/{}.json", HashKey(&fp)) , move || rt.block_on(async {
+ pub fn lookup(
+ &self,
+ cache: &Cache,
+ fp: Fingerprint,
+ rt: &Handle,
+ ) -> Result<Arc<AcoustIDLookupResponse>> {
+ cache.cache_memory(&format!("ext/acoustid/{}.json", HashKey(&fp)) , move || rt.block_on(async {
let _permit = self.rate_limit.clone().acquire_owned().await?;
let permit_drop_ts = Instant::now() + Duration::SECOND;
info!("acoustid lookup");
@@ -132,8 +142,8 @@ impl AcoustID {
}
}
-pub(crate) fn acoustid_fingerprint(path: &Path) -> Result<Arc<Fingerprint>> {
- cache_memory(
+pub(crate) fn acoustid_fingerprint(cache: &Cache, path: &Path) -> Result<Arc<Fingerprint>> {
+ cache.cache_memory(
&format!("media/chromaprint/{}.json", HashKey(path)),
move || {
let child = Command::new("fpcalc")
@@ -173,9 +183,9 @@ impl ImportPlugin for AcoustID {
if !ct.iflags.use_acoustid {
return Ok(());
}
- let fp = acoustid_fingerprint(path)?;
+ let fp = acoustid_fingerprint(&ct.dba.cache, path)?;
- if let Some((atid, mbid)) = self.get_atid_mbid(&fp, &ct.rt)? {
+ if let Some((atid, mbid)) = self.get_atid_mbid(&ct.dba.cache, &fp, &ct.rt)? {
ct.dba.db.write_transaction(&mut |txn| {
let ob = ct.dba.nodes.get(txn, node)?.unwrap();
let ob = ob.as_object();
diff --git a/import/src/plugins/infojson.rs b/import/src/plugins/infojson.rs
index d62983d..3426679 100644
--- a/import/src/plugins/infojson.rs
+++ b/import/src/plugins/infojson.rs
@@ -3,16 +3,10 @@
which is licensed under the GNU Affero General Public License (version 3); see /COPYING.
Copyright (C) 2026 metamuffin <metamuffin.org>
*/
-use crate::plugins::{PluginContext, ImportPlugin, PluginInfo};
+use crate::plugins::{ImportPlugin, PluginContext, PluginInfo};
use anyhow::{Context, Result, anyhow};
use chrono::{Utc, format::Parsed};
-use jellycache::cache_read;
-use jellycommon::{
- IDENT_BANDCAMP, IDENT_YOUTUBE_CHANNEL, IDENT_YOUTUBE_CHANNEL_HANDLE, IDENT_YOUTUBE_VIDEO,
- KIND_CHANNEL, KIND_MUSIC, KIND_SHORTFORMVIDEO, KIND_VIDEO, NO_DESCRIPTION, NO_IDENTIFIERS,
- NO_KIND, NO_RATINGS, NO_RELEASEDATE, NO_SUBTITLE, NO_TAG, NO_TITLE, RTYP_YOUTUBE_FOLLOWERS,
- RTYP_YOUTUBE_LIKES, RTYP_YOUTUBE_VIEWS,
-};
+use jellycommon::*;
use jellydb::table::RowNum;
use jellyremuxer::matroska::{AttachedFile, Segment};
use log::info;
@@ -215,7 +209,10 @@ impl ImportPlugin for Infojson {
.flat_map(|a| &a.files)
.find(is_info_json)
.map(|att| {
- let data = cache_read(str::from_utf8(&att.data).unwrap())?
+ let data = ct
+ .dba
+ .cache
+ .read(str::from_utf8(&att.data).unwrap())?
.ok_or(anyhow!("info json cache missing"))?;
anyhow::Ok(serde_json::from_slice::<YVideo>(&data)?)
})
diff --git a/import/src/plugins/misc.rs b/import/src/plugins/misc.rs
index babbcec..554f473 100644
--- a/import/src/plugins/misc.rs
+++ b/import/src/plugins/misc.rs
@@ -3,9 +3,9 @@
which is licensed under the GNU Affero General Public License (version 3); see /COPYING.
Copyright (C) 2026 metamuffin <metamuffin.org>
*/
-use crate::plugins::{PluginContext, ImportPlugin, PluginInfo};
+use crate::plugins::{ImportPlugin, PluginContext, PluginInfo};
use anyhow::{Context, Result, bail};
-use jellycache::{HashKey, cache_store};
+use jellycache::HashKey;
use jellycommon::{jellyobject::inspect::Inspector, *};
use jellydb::table::RowNum;
use jellyremuxer::matroska::{AttachedFile, Segment};
@@ -30,7 +30,7 @@ impl ImportPlugin for ImageFiles {
_ => return Ok(()),
};
info!("import {:?} at {path:?}", Inspector(&TAGREG, slot));
- let asset = cache_store(
+ let asset = ct.dba.cache.store(
format!(
"media/literal/{}-{}.image",
HashKey(path),
diff --git a/import/src/plugins/musicbrainz.rs b/import/src/plugins/musicbrainz.rs
index 413e487..27e4d0f 100644
--- a/import/src/plugins/musicbrainz.rs
+++ b/import/src/plugins/musicbrainz.rs
@@ -9,7 +9,7 @@ use crate::{
plugins::{ImportPlugin, PluginInfo},
};
use anyhow::{Context, Result};
-use jellycache::cache_memory;
+use jellycache::Cache;
use log::info;
use reqwest::{
Client, ClientBuilder,
@@ -225,98 +225,112 @@ impl MusicBrainz {
}
}
- pub fn lookup_recording(&self, id: String, rt: &Handle) -> Result<Arc<MbRecordingRel>> {
- cache_memory(&format!("ext/musicbrainz/recording/{id}.json"), move || {
- rt.block_on(async {
- let _permit = self.rate_limit.clone().acquire_owned().await?;
- let permit_drop_ts = Instant::now() + Duration::from_secs(Self::MAX_PAR_REQ as u64);
- info!("recording lookup: {id}");
+ pub fn lookup_recording(
+ &self,
+ cache: &Cache,
+ id: String,
+ rt: &Handle,
+ ) -> Result<Arc<MbRecordingRel>> {
+ cache
+ .cache_memory(&format!("ext/musicbrainz/recording/{id}.json"), move || {
+ rt.block_on(async {
+ let _permit = self.rate_limit.clone().acquire_owned().await?;
+ let permit_drop_ts =
+ Instant::now() + Duration::from_secs(Self::MAX_PAR_REQ as u64);
+ info!("recording lookup: {id}");
- let inc = [
- "isrcs",
- "artists",
- "area-rels",
- "artist-rels",
- "event-rels",
- "genre-rels",
- "instrument-rels",
- "label-rels",
- "place-rels",
- "recording-rels",
- "release-rels",
- "release-group-rels",
- "series-rels",
- "url-rels",
- "work-rels",
- ]
- .join("+");
+ let inc = [
+ "isrcs",
+ "artists",
+ "area-rels",
+ "artist-rels",
+ "event-rels",
+ "genre-rels",
+ "instrument-rels",
+ "label-rels",
+ "place-rels",
+ "recording-rels",
+ "release-rels",
+ "release-group-rels",
+ "series-rels",
+ "url-rels",
+ "work-rels",
+ ]
+ .join("+");
- let resp = self
- .client
- .get(format!(
- "https://musicbrainz.org/ws/2/recording/{id}?inc={inc}"
- ))
- .send()
- .await?
- .error_for_status()?
- .json::<MbRecordingRel>()
- .await?;
+ let resp = self
+ .client
+ .get(format!(
+ "https://musicbrainz.org/ws/2/recording/{id}?inc={inc}"
+ ))
+ .send()
+ .await?
+ .error_for_status()?
+ .json::<MbRecordingRel>()
+ .await?;
- tokio::task::spawn(async move {
- sleep_until(permit_drop_ts).await;
- drop(_permit);
- });
+ tokio::task::spawn(async move {
+ sleep_until(permit_drop_ts).await;
+ drop(_permit);
+ });
- Ok(resp)
+ Ok(resp)
+ })
})
- })
- .context("musicbrainz recording lookup")
+ .context("musicbrainz recording lookup")
}
- pub fn lookup_artist(&self, id: String, rt: &Handle) -> Result<Arc<MbArtistRel>> {
- cache_memory(&format!("ext/musicbrainz/artist/{id}.json"), move || {
- rt.block_on(async {
- let _permit = self.rate_limit.clone().acquire_owned().await?;
- let permit_drop_ts = Instant::now() + Duration::from_secs(Self::MAX_PAR_REQ as u64);
- info!("artist lookup: {id}");
+ pub fn lookup_artist(
+ &self,
+ cache: &Cache,
+ id: String,
+ rt: &Handle,
+ ) -> Result<Arc<MbArtistRel>> {
+ cache
+ .cache_memory(&format!("ext/musicbrainz/artist/{id}.json"), move || {
+ rt.block_on(async {
+ let _permit = self.rate_limit.clone().acquire_owned().await?;
+ let permit_drop_ts =
+ Instant::now() + Duration::from_secs(Self::MAX_PAR_REQ as u64);
+ info!("artist lookup: {id}");
- let inc = [
- "area-rels",
- "artist-rels",
- "event-rels",
- "genre-rels",
- "instrument-rels",
- "label-rels",
- "place-rels",
- "recording-rels",
- "release-rels",
- "release-group-rels",
- "series-rels",
- "url-rels",
- "work-rels",
- ]
- .join("+");
+ let inc = [
+ "area-rels",
+ "artist-rels",
+ "event-rels",
+ "genre-rels",
+ "instrument-rels",
+ "label-rels",
+ "place-rels",
+ "recording-rels",
+ "release-rels",
+ "release-group-rels",
+ "series-rels",
+ "url-rels",
+ "work-rels",
+ ]
+ .join("+");
- let resp = self
- .client
- .get(format!(
- "https://musicbrainz.org/ws/2/artist/{id}?inc={inc}"
- ))
- .send()
- .await?
- .error_for_status()?
- .json::<MbArtistRel>()
- .await?;
+ let resp = self
+ .client
+ .get(format!(
+ "https://musicbrainz.org/ws/2/artist/{id}?inc={inc}"
+ ))
+ .send()
+ .await?
+ .error_for_status()?
+ .json::<MbArtistRel>()
+ .await?;
- tokio::task::spawn(async move {
- sleep_until(permit_drop_ts).await;
- drop(_permit);
- });
+ tokio::task::spawn(async move {
+ sleep_until(permit_drop_ts).await;
+ drop(_permit);
+ });
- Ok(resp)
+ Ok(resp)
+ })
})
- })
- .context("musicbrainz artist lookup")
+ .context("musicbrainz artist lookup")
}
}
diff --git a/import/src/plugins/tmdb.rs b/import/src/plugins/tmdb.rs
index cf50938..ea32c40 100644
--- a/import/src/plugins/tmdb.rs
+++ b/import/src/plugins/tmdb.rs
@@ -5,11 +5,11 @@
*/
use crate::{
USER_AGENT,
- plugins::{PluginContext, ImportPlugin, PluginInfo},
+ plugins::{ImportPlugin, PluginContext, PluginInfo},
};
use anyhow::{Context, Result, anyhow, bail};
use chrono::{Utc, format::Parsed};
-use jellycache::{EscapeKey, HashKey, cache_memory, cache_store};
+use jellycache::{Cache, EscapeKey, HashKey};
use jellycommon::*;
use jellydb::table::RowNum;
use log::info;
@@ -49,95 +49,117 @@ impl Tmdb {
key: api_key.to_owned(),
}
}
- pub fn search(&self, kind: TmdbKind, query: &str, rt: &Handle) -> Result<Arc<TmdbQuery>> {
- cache_memory(
- &format!("ext/tmdb/search/{kind}-{}.json", HashKey(query)),
- move || {
+ pub fn search(
+ &self,
+ cache: &Cache,
+ kind: TmdbKind,
+ query: &str,
+ rt: &Handle,
+ ) -> Result<Arc<TmdbQuery>> {
+ cache
+ .cache_memory(
+ &format!("ext/tmdb/search/{kind}-{}.json", HashKey(query)),
+ move || {
+ rt.block_on(async {
+ info!("searching tmdb: {query:?}");
+ Ok(self
+ .client
+ .get(format!(
+ "https://api.themoviedb.org/3/search/{kind}?query={}?api_key={}",
+ query.replace(" ", "+"),
+ self.key
+ ))
+ .send()
+ .await?
+ .error_for_status()?
+ .json::<TmdbQuery>()
+ .await?)
+ })
+ },
+ )
+ .context("tmdb search")
+ }
+ pub fn details(
+ &self,
+ cache: &Cache,
+ kind: TmdbKind,
+ id: u64,
+ rt: &Handle,
+ ) -> Result<Arc<TmdbDetails>> {
+ cache
+ .cache_memory(&format!("ext/tmdb/details/{kind}-{id}.json"), move || {
rt.block_on(async {
- info!("searching tmdb: {query:?}");
+ info!("fetching details: {id:?}");
Ok(self
.client
.get(format!(
- "https://api.themoviedb.org/3/search/{kind}?query={}?api_key={}",
- query.replace(" ", "+"),
- self.key
+ "https://api.themoviedb.org/3/{kind}/{id}?api_key={}",
+ self.key,
))
.send()
.await?
.error_for_status()?
- .json::<TmdbQuery>()
+ .json()
.await?)
})
- },
- )
- .context("tmdb search")
- }
- pub fn details(&self, kind: TmdbKind, id: u64, rt: &Handle) -> Result<Arc<TmdbDetails>> {
- cache_memory(&format!("ext/tmdb/details/{kind}-{id}.json"), move || {
- rt.block_on(async {
- info!("fetching details: {id:?}");
- Ok(self
- .client
- .get(format!(
- "https://api.themoviedb.org/3/{kind}/{id}?api_key={}",
- self.key,
- ))
- .send()
- .await?
- .error_for_status()?
- .json()
- .await?)
})
- })
- .context("tmdb details")
+ .context("tmdb details")
}
- pub fn person_image(&self, id: u64, rt: &Handle) -> Result<Arc<TmdbPersonImage>> {
- cache_memory(&format!("ext/tmdb/person/images/{id}.json"), move || {
- rt.block_on(async {
- Ok(self
- .client
- .get(format!(
- "https://api.themoviedb.org/3/person/{id}/images?api_key={}",
- self.key,
- ))
- .send()
- .await?
- .error_for_status()?
- .json()
- .await?)
- })
- })
- .context("tmdb person images")
- }
- pub fn image(&self, path: &str, rt: &Handle) -> Result<String> {
- cache_store(
- format!("ext/tmdb/image/{}.image", EscapeKey(path)),
- move || {
+ pub fn person_image(
+ &self,
+ cache: &Cache,
+ id: u64,
+ rt: &Handle,
+ ) -> Result<Arc<TmdbPersonImage>> {
+ cache
+ .cache_memory(&format!("ext/tmdb/person/images/{id}.json"), move || {
rt.block_on(async {
- info!("downloading image {path:?}");
Ok(self
- .image_client
- .get(format!("https://image.tmdb.org/t/p/original{path}"))
+ .client
+ .get(format!(
+ "https://api.themoviedb.org/3/person/{id}/images?api_key={}",
+ self.key,
+ ))
.send()
.await?
.error_for_status()?
- .bytes()
- .await?
- .to_vec())
+ .json()
+ .await?)
})
- },
- )
- .context("tmdb image download")
+ })
+ .context("tmdb person images")
+ }
+ pub fn image(&self, cache: &Cache, path: &str, rt: &Handle) -> Result<String> {
+ cache
+ .store(
+ format!("ext/tmdb/image/{}.image", EscapeKey(path)),
+ move || {
+ rt.block_on(async {
+ info!("downloading image {path:?}");
+ Ok(self
+ .image_client
+ .get(format!("https://image.tmdb.org/t/p/original{path}"))
+ .send()
+ .await?
+ .error_for_status()?
+ .bytes()
+ .await?
+ .to_vec())
+ })
+ },
+ )
+ .context("tmdb image download")
}
pub fn episode_details(
&self,
+ cache: &Cache,
series_id: u64,
season: u64,
episode: u64,
rt: &Handle,
) -> Result<Arc<TmdbEpisode>> {
- cache_memory(&format!("ext/tmdb/episode-details/{series_id}-S{season}-E{episode}.json"), move || {
+ cache.cache_memory(&format!("ext/tmdb/episode-details/{series_id}-S{season}-E{episode}.json"), move || {
rt.block_on(async {
info!("tmdb episode details {series_id} S={season} E={episode}");
Ok(self
@@ -189,17 +211,17 @@ impl Tmdb {
return Ok(());
};
- let details = self.details(tmdb_kind, tmdb_id, ct.rt)?;
+ let details = self.details(&ct.dba.cache, tmdb_kind, tmdb_id, ct.rt)?;
let backdrop = details
.backdrop_path
.as_ref()
- .map(|path| self.image(&path, ct.rt))
+ .map(|path| self.image(&ct.dba.cache, &path, ct.rt))
.transpose()
.context("backdrop image")?;
let poster = details
.poster_path
.as_ref()
- .map(|path| self.image(&path, ct.rt))
+ .map(|path| self.image(&ct.dba.cache, &path, ct.rt))
.transpose()
.context("poster image")?;
@@ -268,11 +290,11 @@ impl Tmdb {
let Some(series_id) = series_id else {
return Ok(());
};
- let details = self.episode_details(series_id, season, episode, ct.rt)?;
+ let details = self.episode_details(&ct.dba.cache, series_id, season, episode, ct.rt)?;
let cover = details
.still_path
.as_ref()
- .map(|path| self.image(&path, ct.rt))
+ .map(|path| self.image(&ct.dba.cache, &path, ct.rt))
.transpose()
.context("still image download")?;
let release_date = parse_release_date(&details.air_date)?;
diff --git a/import/src/plugins/trakt.rs b/import/src/plugins/trakt.rs
index 3569454..2cb063d 100644
--- a/import/src/plugins/trakt.rs
+++ b/import/src/plugins/trakt.rs
@@ -5,10 +5,10 @@
*/
use crate::{
USER_AGENT,
- plugins::{PluginContext, ImportPlugin, PluginInfo},
+ plugins::{ImportPlugin, PluginContext, PluginInfo},
};
use anyhow::{Context, Result, anyhow, bail};
-use jellycache::{HashKey, cache_memory};
+use jellycache::{Cache, HashKey};
use jellycommon::{jellyobject::Tag, *};
use jellydb::table::RowNum;
use log::info;
@@ -52,89 +52,114 @@ impl Trakt {
pub fn search(
&self,
+ cache: &Cache,
kinds: &[TraktKind],
query: &str,
rt: &Handle,
) -> Result<Arc<Vec<TraktSearchResult>>> {
- cache_memory(
- &format!("ext/trakt/search/{}.json", HashKey(query)),
- move || {
+ cache
+ .cache_memory(
+ &format!("ext/trakt/search/{}.json", HashKey(query)),
+ move || {
+ rt.block_on(async {
+ let url = format!(
+ "https://api.trakt.tv/search/{}?query={}&extended=full",
+ kinds
+ .iter()
+ .map(|t| t.singular())
+ .collect::<Vec<_>>()
+ .join(","),
+ urlencoding::encode(query),
+ );
+ let res = self.client.get(url).send().await?.error_for_status()?;
+ Ok(res.json().await?)
+ })
+ },
+ )
+ .context("trakt search")
+ }
+
+ pub fn lookup(
+ &self,
+ cache: &Cache,
+ kind: TraktKind,
+ id: u64,
+ rt: &Handle,
+ ) -> Result<Arc<TraktMediaObject>> {
+ cache
+ .cache_memory(&format!("ext/trakt/lookup/{kind}-{id}.json"), move || {
rt.block_on(async {
- let url = format!(
- "https://api.trakt.tv/search/{}?query={}&extended=full",
- kinds
- .iter()
- .map(|t| t.singular())
- .collect::<Vec<_>>()
- .join(","),
- urlencoding::encode(query),
- );
+ info!("trakt lookup {kind:?}:{id:?}");
+ let url = format!("https://api.trakt.tv/{}/{id}?extended=full", kind.plural());
let res = self.client.get(url).send().await?.error_for_status()?;
Ok(res.json().await?)
})
- },
- )
- .context("trakt search")
- }
-
- pub fn lookup(&self, kind: TraktKind, id: u64, rt: &Handle) -> Result<Arc<TraktMediaObject>> {
- cache_memory(&format!("ext/trakt/lookup/{kind}-{id}.json"), move || {
- rt.block_on(async {
- info!("trakt lookup {kind:?}:{id:?}");
- let url = format!("https://api.trakt.tv/{}/{id}?extended=full", kind.plural());
- let res = self.client.get(url).send().await?.error_for_status()?;
- Ok(res.json().await?)
})
- })
- .context("trakt lookup")
+ .context("trakt lookup")
}
- pub fn people(&self, kind: TraktKind, id: u64, rt: &Handle) -> Result<Arc<TraktPeople>> {
- cache_memory(&format!("ext/trakt/people/{kind}-{id}.json"), move || {
- rt.block_on(async {
- info!("trakt people {kind:?}:{id:?}");
- let url = format!(
- "https://api.trakt.tv/{}/{id}/people?extended=full",
- kind.plural()
- );
- let res = self.client.get(url).send().await?.error_for_status()?;
- Ok(res.json().await?)
+ pub fn people(
+ &self,
+ cache: &Cache,
+ kind: TraktKind,
+ id: u64,
+ rt: &Handle,
+ ) -> Result<Arc<TraktPeople>> {
+ cache
+ .cache_memory(&format!("ext/trakt/people/{kind}-{id}.json"), move || {
+ rt.block_on(async {
+ info!("trakt people {kind:?}:{id:?}");
+ let url = format!(
+ "https://api.trakt.tv/{}/{id}/people?extended=full",
+ kind.plural()
+ );
+ let res = self.client.get(url).send().await?.error_for_status()?;
+ Ok(res.json().await?)
+ })
})
- })
- .context("trakt people")
+ .context("trakt people")
}
- pub fn show_seasons(&self, id: u64, rt: &Handle) -> Result<Arc<Vec<TraktSeason>>> {
- cache_memory(&format!("ext/trakt/seasons/{id}.json"), move || {
- rt.block_on(async {
- info!("trakt seasons {id:?}");
- let url = format!("https://api.trakt.tv/shows/{id}/seasons?extended=full");
- let res = self.client.get(url).send().await?.error_for_status()?;
- Ok(res.json().await?)
+ pub fn show_seasons(
+ &self,
+ cache: &Cache,
+ id: u64,
+ rt: &Handle,
+ ) -> Result<Arc<Vec<TraktSeason>>> {
+ cache
+ .cache_memory(&format!("ext/trakt/seasons/{id}.json"), move || {
+ rt.block_on(async {
+ info!("trakt seasons {id:?}");
+ let url = format!("https://api.trakt.tv/shows/{id}/seasons?extended=full");
+ let res = self.client.get(url).send().await?.error_for_status()?;
+ Ok(res.json().await?)
+ })
})
- })
- .context("trakt show seasons")
+ .context("trakt show seasons")
}
pub fn show_season_episodes(
&self,
+ cache: &Cache,
id: u64,
season: u64,
rt: &Handle,
) -> Result<Arc<Vec<TraktEpisode>>> {
- cache_memory(
- &format!("ext/trakt/episodes/{id}-S{season}.json"),
- move || {
- rt.block_on(async {
- info!("trakt episodes {id:?} season={season}");
- let url =
- format!("https://api.trakt.tv/shows/{id}/seasons/{season}?extended=full");
- let res = self.client.get(url).send().await?.error_for_status()?;
- Ok(res.json().await?)
- })
- },
- )
- .context("trakt show season episodes")
+ cache
+ .cache_memory(
+ &format!("ext/trakt/episodes/{id}-S{season}.json"),
+ move || {
+ rt.block_on(async {
+ info!("trakt episodes {id:?} season={season}");
+ let url = format!(
+ "https://api.trakt.tv/shows/{id}/seasons/{season}?extended=full"
+ );
+ let res = self.client.get(url).send().await?.error_for_status()?;
+ Ok(res.json().await?)
+ })
+ },
+ )
+ .context("trakt show season episodes")
}
}
@@ -425,7 +450,7 @@ impl Trakt {
return Ok(());
};
- let details = self.lookup(trakt_kind, trakt_id, ct.rt)?;
+ let details = self.lookup(&ct.dba.cache, trakt_kind, trakt_id, ct.rt)?;
// let people = self.people(trakt_kind, trakt_id, ct.rt)?;
// let mut people_map = BTreeMap::<CreditCategory, Vec<Appearance>>::new();
@@ -525,9 +550,9 @@ impl Trakt {
return Ok(());
};
- let seasons = self.show_seasons(show_id, ct.rt)?;
+ let seasons = self.show_seasons(&ct.dba.cache, show_id, ct.rt)?;
if seasons.iter().any(|x| x.number == season) {
- let episodes = self.show_season_episodes(show_id, season, ct.rt)?;
+ let episodes = self.show_season_episodes(&ct.dba.cache, show_id, season, ct.rt)?;
if let Some(episode) = episodes.get(episode.saturating_sub(1) as usize) {
ct.dba.update_node(node, |mut node| {
node = node.as_object().insert(NO_KIND, KIND_EPISODE);
diff --git a/import/src/plugins/vgmdb.rs b/import/src/plugins/vgmdb.rs
index c62eb90..534b241 100644
--- a/import/src/plugins/vgmdb.rs
+++ b/import/src/plugins/vgmdb.rs
@@ -9,7 +9,7 @@ use crate::{
plugins::{ImportPlugin, PluginInfo},
};
use anyhow::{Context, Result};
-use jellycache::{HashKey, cache, cache_store};
+use jellycache::{Cache, HashKey};
use log::info;
use regex::Regex;
use reqwest::{
@@ -62,34 +62,40 @@ impl Vgmdb {
}
}
- pub fn get_artist_image(&self, id: u64, rt: &Handle) -> Result<Option<String>> {
- if let Some(url) = self.get_artist_image_url(id, rt)? {
- cache_store(
- format!("ext/vgmdb/artist-image/{}.image", HashKey(&url)),
- move || {
- rt.block_on(async {
- info!("downloading image {url:?}");
- Ok(self
- .client
- .get(url)
- .send()
- .await?
- .error_for_status()?
- .bytes()
- .await?
- .to_vec())
- })
- },
- )
- .context("vgmdb media download")
- .map(Some)
+ pub fn get_artist_image(&self, cache: &Cache, id: u64, rt: &Handle) -> Result<Option<String>> {
+ if let Some(url) = self.get_artist_image_url(cache, id, rt)? {
+ cache
+ .store(
+ format!("ext/vgmdb/artist-image/{}.image", HashKey(&url)),
+ move || {
+ rt.block_on(async {
+ info!("downloading image {url:?}");
+ Ok(self
+ .client
+ .get(url)
+ .send()
+ .await?
+ .error_for_status()?
+ .bytes()
+ .await?
+ .to_vec())
+ })
+ },
+ )
+ .context("vgmdb media download")
+ .map(Some)
} else {
Ok(None)
}
}
- pub fn get_artist_image_url(&self, id: u64, rt: &Handle) -> Result<Option<String>> {
- let html = self.scrape_artist_page(id, rt)?;
+ pub fn get_artist_image_url(
+ &self,
+ cache: &Cache,
+ id: u64,
+ rt: &Handle,
+ ) -> Result<Option<String>> {
+ let html = self.scrape_artist_page(cache, id, rt)?;
if let Some(cap) = RE_IMAGE_URL_FROM_HTML.captures(&str::from_utf8(&html).unwrap()) {
if let Some(url) = cap.name("url").map(|m| m.as_str()) {
return Ok(Some(url.to_string()));
@@ -98,32 +104,33 @@ impl Vgmdb {
Ok(None)
}
- pub fn scrape_artist_page(&self, id: u64, rt: &Handle) -> Result<Vec<u8>> {
- cache(&format!("ext/vgmdb/artist-page/{id}.html"), move || {
- rt.block_on(async {
- let _permit = self.rate_limit.clone().acquire_owned().await?;
- let permit_drop_ts = Instant::now() + Duration::from_secs(1);
- info!("scrape artist: {id}");
+ pub fn scrape_artist_page(&self, cache: &Cache, id: u64, rt: &Handle) -> Result<Vec<u8>> {
+ cache
+ .cache(&format!("ext/vgmdb/artist-page/{id}.html"), move || {
+ rt.block_on(async {
+ let _permit = self.rate_limit.clone().acquire_owned().await?;
+ let permit_drop_ts = Instant::now() + Duration::from_secs(1);
+ info!("scrape artist: {id}");
- let resp = self
- .client
- .get(format!("https://vgmdb.net/artist/{id}"))
- .send()
- .await?
- .error_for_status()?
- .bytes()
- .await?
- .to_vec();
+ let resp = self
+ .client
+ .get(format!("https://vgmdb.net/artist/{id}"))
+ .send()
+ .await?
+ .error_for_status()?
+ .bytes()
+ .await?
+ .to_vec();
- tokio::task::spawn(async move {
- sleep_until(permit_drop_ts).await;
- drop(_permit);
- });
+ tokio::task::spawn(async move {
+ sleep_until(permit_drop_ts).await;
+ drop(_permit);
+ });
- Ok(resp)
+ Ok(resp)
+ })
})
- })
- .context("vgmdb artist page scrape")
+ .context("vgmdb artist page scrape")
}
}
diff --git a/import/src/plugins/wikidata.rs b/import/src/plugins/wikidata.rs
index e3077e0..3afd393 100644
--- a/import/src/plugins/wikidata.rs
+++ b/import/src/plugins/wikidata.rs
@@ -9,7 +9,7 @@ use crate::{
plugins::{ImportPlugin, PluginInfo},
};
use anyhow::{Context, Result, bail};
-use jellycache::{EscapeKey, cache_memory};
+use jellycache::{Cache, EscapeKey};
use log::info;
use reqwest::{
Client, ClientBuilder,
@@ -91,8 +91,13 @@ impl Wikidata {
Self { client }
}
- pub fn query_image_path(&self, id: String, rt: &Handle) -> Result<Option<String>> {
- let response = self.query(id.clone(), rt)?;
+ pub fn query_image_path(
+ &self,
+ cache: &Cache,
+ id: String,
+ rt: &Handle,
+ ) -> Result<Option<String>> {
+ let response = self.query(cache, id.clone(), rt)?;
if let Some(entity) = response.entities.get(&id) {
if let Some(images) = entity.claims.get(properties::IMAGE) {
for image in images {
@@ -110,24 +115,25 @@ impl Wikidata {
Ok(None)
}
- pub fn query(&self, id: String, rt: &Handle) -> Result<Arc<WikidataResponse>> {
- cache_memory(
- &format!("ext/wikidata/{}.json", EscapeKey(&id)),
- move || {
- rt.block_on(async {
- info!("entity query: {id}");
- Ok(self
- .client
- .get(format!("https://www.wikidata.org/entity/{id}"))
- .send()
- .await?
- .error_for_status()?
- .json()
- .await?)
- })
- },
- )
- .context("wikidata entity")
+ pub fn query(&self, cache: &Cache, id: String, rt: &Handle) -> Result<Arc<WikidataResponse>> {
+ cache
+ .cache_memory(
+ &format!("ext/wikidata/{}.json", EscapeKey(&id)),
+ move || {
+ rt.block_on(async {
+ info!("entity query: {id}");
+ Ok(self
+ .client
+ .get(format!("https://www.wikidata.org/entity/{id}"))
+ .send()
+ .await?
+ .error_for_status()?
+ .json()
+ .await?)
+ })
+ },
+ )
+ .context("wikidata entity")
}
}
diff --git a/import/src/plugins/wikimedia_commons.rs b/import/src/plugins/wikimedia_commons.rs
index c849b61..aebf5dd 100644
--- a/import/src/plugins/wikimedia_commons.rs
+++ b/import/src/plugins/wikimedia_commons.rs
@@ -9,7 +9,7 @@ use crate::{
plugins::{ImportPlugin, PluginInfo},
};
use anyhow::{Context, Result};
-use jellycache::{EscapeKey, cache_store};
+use jellycache::{Cache, EscapeKey};
use reqwest::{
Client, ClientBuilder,
header::{HeaderMap, HeaderName, HeaderValue},
@@ -39,27 +39,33 @@ impl WikimediaCommons {
Self { client }
}
- pub fn image_by_filename(&self, filename: String, rt: &Handle) -> Result<String> {
- cache_store(
- format!("ext/wikimedia-commons/image/{}.image", EscapeKey(&filename)),
- move || {
- rt.block_on(async {
- Ok(self
- .client
- .get(format!(
- "https://commons.wikimedia.org/wiki/Special:FilePath/{}",
- filename.replace(" ", "_")
- ))
- .send()
- .await?
- .error_for_status()?
- .bytes()
- .await?
- .to_vec())
- })
- },
- )
- .context("mediawiki image by filename")
+ pub fn image_by_filename(
+ &self,
+ cache: &Cache,
+ filename: String,
+ rt: &Handle,
+ ) -> Result<String> {
+ cache
+ .store(
+ format!("ext/wikimedia-commons/image/{}.image", EscapeKey(&filename)),
+ move || {
+ rt.block_on(async {
+ Ok(self
+ .client
+ .get(format!(
+ "https://commons.wikimedia.org/wiki/Special:FilePath/{}",
+ filename.replace(" ", "_")
+ ))
+ .send()
+ .await?
+ .error_for_status()?
+ .bytes()
+ .await?
+ .to_vec())
+ })
+ },
+ )
+ .context("mediawiki image by filename")
}
}