diff options
| author | metamuffin <metamuffin@disroot.org> | 2026-01-24 23:06:33 +0100 |
|---|---|---|
| committer | metamuffin <metamuffin@disroot.org> | 2026-01-24 23:06:33 +0100 |
| commit | 2bcccb18a6cb8bf836f57c3d86f759b19699def2 (patch) | |
| tree | ef55a10c6d9703677a983b8ca900fb4578a08eb3 /import/src | |
| parent | b2e88a8beabf04adc28947cf82996e8692a68b71 (diff) | |
| download | jellything-2bcccb18a6cb8bf836f57c3d86f759b19699def2.tar jellything-2bcccb18a6cb8bf836f57c3d86f759b19699def2.tar.bz2 jellything-2bcccb18a6cb8bf836f57c3d86f759b19699def2.tar.zst | |
cache as object
Diffstat (limited to 'import/src')
| -rw-r--r-- | import/src/lib.rs | 10 | ||||
| -rw-r--r-- | import/src/plugins/acoustid.rs | 30 | ||||
| -rw-r--r-- | import/src/plugins/infojson.rs | 15 | ||||
| -rw-r--r-- | import/src/plugins/misc.rs | 6 | ||||
| -rw-r--r-- | import/src/plugins/musicbrainz.rs | 176 | ||||
| -rw-r--r-- | import/src/plugins/tmdb.rs | 160 | ||||
| -rw-r--r-- | import/src/plugins/trakt.rs | 155 | ||||
| -rw-r--r-- | import/src/plugins/vgmdb.rs | 99 | ||||
| -rw-r--r-- | import/src/plugins/wikidata.rs | 48 | ||||
| -rw-r--r-- | import/src/plugins/wikimedia_commons.rs | 50 |
10 files changed, 418 insertions, 331 deletions
diff --git a/import/src/lib.rs b/import/src/lib.rs index 7d48867..bec5786 100644 --- a/import/src/lib.rs +++ b/import/src/lib.rs @@ -9,7 +9,7 @@ pub mod plugins; pub mod reporting; use crate::{ - plugins::{PluginContext, ImportPlugin, infojson::is_info_json, init_plugins, misc::is_cover}, + plugins::{ImportPlugin, PluginContext, infojson::is_info_json, init_plugins, misc::is_cover}, reporting::IMPORT_PROGRESS, }; use anyhow::{Context, Result, anyhow}; @@ -377,9 +377,9 @@ fn import_file( } reporting::set_task(format!("demuxer meta: {path:?}")); - let Some(seg) = - reporting::catch(read_media_metadata(path).context(anyhow!("media {path:?}"))) - else { + let Some(seg) = reporting::catch( + read_media_metadata(&ct.dba.cache, path).context(anyhow!("media {path:?}")), + ) else { return; }; for p in plugins { @@ -542,7 +542,7 @@ pub fn read_media_metadata(cache: &Cache, path: &Path) -> Result<Arc<matroska::S if let Some(attachments) = &mut attachments { for att in &mut attachments.files { if let Some(fname) = is_useful_attachment(&att) { - let key = cache.cache_store( + let key = cache.store( format!("media/attachment/{}-{fname}", HashKey(path)), || Ok(att.data.clone()), )?; diff --git a/import/src/plugins/acoustid.rs b/import/src/plugins/acoustid.rs index 9891927..0e16670 100644 --- a/import/src/plugins/acoustid.rs +++ b/import/src/plugins/acoustid.rs @@ -5,10 +5,10 @@ */ use crate::{ USER_AGENT, - plugins::{PluginContext, ImportPlugin, PluginInfo}, + plugins::{ImportPlugin, PluginContext, PluginInfo}, }; use anyhow::{Context, Result}; -use jellycache::{HashKey, cache_memory}; +use jellycache::{Cache, HashKey}; use jellycommon::{ IDENT_ACOUST_ID_TRACK, IDENT_MUSICBRAINZ_RECORDING, NO_IDENTIFIERS, jellyobject::Object, }; @@ -92,8 +92,13 @@ impl AcoustID { } } - pub fn get_atid_mbid(&self, fp: &Fingerprint, rt: &Handle) -> Result<Option<(String, String)>> { - let res = self.lookup(fp.to_owned(), rt)?; + pub fn get_atid_mbid( + &self, + cache: &Cache, + fp: &Fingerprint, + rt: &Handle, + ) -> Result<Option<(String, String)>> { + let res = self.lookup(cache, fp.to_owned(), rt)?; for r in &res.results { if let Some(k) = r.recordings.first() { return Ok(Some((r.id.clone(), k.id.clone()))); @@ -102,8 +107,13 @@ impl AcoustID { Ok(None) } - pub fn lookup(&self, fp: Fingerprint, rt: &Handle) -> Result<Arc<AcoustIDLookupResponse>> { - cache_memory(&format!("ext/acoustid/{}.json", HashKey(&fp)) , move || rt.block_on(async { + pub fn lookup( + &self, + cache: &Cache, + fp: Fingerprint, + rt: &Handle, + ) -> Result<Arc<AcoustIDLookupResponse>> { + cache.cache_memory(&format!("ext/acoustid/{}.json", HashKey(&fp)) , move || rt.block_on(async { let _permit = self.rate_limit.clone().acquire_owned().await?; let permit_drop_ts = Instant::now() + Duration::SECOND; info!("acoustid lookup"); @@ -132,8 +142,8 @@ impl AcoustID { } } -pub(crate) fn acoustid_fingerprint(path: &Path) -> Result<Arc<Fingerprint>> { - cache_memory( +pub(crate) fn acoustid_fingerprint(cache: &Cache, path: &Path) -> Result<Arc<Fingerprint>> { + cache.cache_memory( &format!("media/chromaprint/{}.json", HashKey(path)), move || { let child = Command::new("fpcalc") @@ -173,9 +183,9 @@ impl ImportPlugin for AcoustID { if !ct.iflags.use_acoustid { return Ok(()); } - let fp = acoustid_fingerprint(path)?; + let fp = acoustid_fingerprint(&ct.dba.cache, path)?; - if let Some((atid, mbid)) = self.get_atid_mbid(&fp, &ct.rt)? { + if let Some((atid, mbid)) = self.get_atid_mbid(&ct.dba.cache, &fp, &ct.rt)? { ct.dba.db.write_transaction(&mut |txn| { let ob = ct.dba.nodes.get(txn, node)?.unwrap(); let ob = ob.as_object(); diff --git a/import/src/plugins/infojson.rs b/import/src/plugins/infojson.rs index d62983d..3426679 100644 --- a/import/src/plugins/infojson.rs +++ b/import/src/plugins/infojson.rs @@ -3,16 +3,10 @@ which is licensed under the GNU Affero General Public License (version 3); see /COPYING. Copyright (C) 2026 metamuffin <metamuffin.org> */ -use crate::plugins::{PluginContext, ImportPlugin, PluginInfo}; +use crate::plugins::{ImportPlugin, PluginContext, PluginInfo}; use anyhow::{Context, Result, anyhow}; use chrono::{Utc, format::Parsed}; -use jellycache::cache_read; -use jellycommon::{ - IDENT_BANDCAMP, IDENT_YOUTUBE_CHANNEL, IDENT_YOUTUBE_CHANNEL_HANDLE, IDENT_YOUTUBE_VIDEO, - KIND_CHANNEL, KIND_MUSIC, KIND_SHORTFORMVIDEO, KIND_VIDEO, NO_DESCRIPTION, NO_IDENTIFIERS, - NO_KIND, NO_RATINGS, NO_RELEASEDATE, NO_SUBTITLE, NO_TAG, NO_TITLE, RTYP_YOUTUBE_FOLLOWERS, - RTYP_YOUTUBE_LIKES, RTYP_YOUTUBE_VIEWS, -}; +use jellycommon::*; use jellydb::table::RowNum; use jellyremuxer::matroska::{AttachedFile, Segment}; use log::info; @@ -215,7 +209,10 @@ impl ImportPlugin for Infojson { .flat_map(|a| &a.files) .find(is_info_json) .map(|att| { - let data = cache_read(str::from_utf8(&att.data).unwrap())? + let data = ct + .dba + .cache + .read(str::from_utf8(&att.data).unwrap())? .ok_or(anyhow!("info json cache missing"))?; anyhow::Ok(serde_json::from_slice::<YVideo>(&data)?) }) diff --git a/import/src/plugins/misc.rs b/import/src/plugins/misc.rs index babbcec..554f473 100644 --- a/import/src/plugins/misc.rs +++ b/import/src/plugins/misc.rs @@ -3,9 +3,9 @@ which is licensed under the GNU Affero General Public License (version 3); see /COPYING. Copyright (C) 2026 metamuffin <metamuffin.org> */ -use crate::plugins::{PluginContext, ImportPlugin, PluginInfo}; +use crate::plugins::{ImportPlugin, PluginContext, PluginInfo}; use anyhow::{Context, Result, bail}; -use jellycache::{HashKey, cache_store}; +use jellycache::HashKey; use jellycommon::{jellyobject::inspect::Inspector, *}; use jellydb::table::RowNum; use jellyremuxer::matroska::{AttachedFile, Segment}; @@ -30,7 +30,7 @@ impl ImportPlugin for ImageFiles { _ => return Ok(()), }; info!("import {:?} at {path:?}", Inspector(&TAGREG, slot)); - let asset = cache_store( + let asset = ct.dba.cache.store( format!( "media/literal/{}-{}.image", HashKey(path), diff --git a/import/src/plugins/musicbrainz.rs b/import/src/plugins/musicbrainz.rs index 413e487..27e4d0f 100644 --- a/import/src/plugins/musicbrainz.rs +++ b/import/src/plugins/musicbrainz.rs @@ -9,7 +9,7 @@ use crate::{ plugins::{ImportPlugin, PluginInfo}, }; use anyhow::{Context, Result}; -use jellycache::cache_memory; +use jellycache::Cache; use log::info; use reqwest::{ Client, ClientBuilder, @@ -225,98 +225,112 @@ impl MusicBrainz { } } - pub fn lookup_recording(&self, id: String, rt: &Handle) -> Result<Arc<MbRecordingRel>> { - cache_memory(&format!("ext/musicbrainz/recording/{id}.json"), move || { - rt.block_on(async { - let _permit = self.rate_limit.clone().acquire_owned().await?; - let permit_drop_ts = Instant::now() + Duration::from_secs(Self::MAX_PAR_REQ as u64); - info!("recording lookup: {id}"); + pub fn lookup_recording( + &self, + cache: &Cache, + id: String, + rt: &Handle, + ) -> Result<Arc<MbRecordingRel>> { + cache + .cache_memory(&format!("ext/musicbrainz/recording/{id}.json"), move || { + rt.block_on(async { + let _permit = self.rate_limit.clone().acquire_owned().await?; + let permit_drop_ts = + Instant::now() + Duration::from_secs(Self::MAX_PAR_REQ as u64); + info!("recording lookup: {id}"); - let inc = [ - "isrcs", - "artists", - "area-rels", - "artist-rels", - "event-rels", - "genre-rels", - "instrument-rels", - "label-rels", - "place-rels", - "recording-rels", - "release-rels", - "release-group-rels", - "series-rels", - "url-rels", - "work-rels", - ] - .join("+"); + let inc = [ + "isrcs", + "artists", + "area-rels", + "artist-rels", + "event-rels", + "genre-rels", + "instrument-rels", + "label-rels", + "place-rels", + "recording-rels", + "release-rels", + "release-group-rels", + "series-rels", + "url-rels", + "work-rels", + ] + .join("+"); - let resp = self - .client - .get(format!( - "https://musicbrainz.org/ws/2/recording/{id}?inc={inc}" - )) - .send() - .await? - .error_for_status()? - .json::<MbRecordingRel>() - .await?; + let resp = self + .client + .get(format!( + "https://musicbrainz.org/ws/2/recording/{id}?inc={inc}" + )) + .send() + .await? + .error_for_status()? + .json::<MbRecordingRel>() + .await?; - tokio::task::spawn(async move { - sleep_until(permit_drop_ts).await; - drop(_permit); - }); + tokio::task::spawn(async move { + sleep_until(permit_drop_ts).await; + drop(_permit); + }); - Ok(resp) + Ok(resp) + }) }) - }) - .context("musicbrainz recording lookup") + .context("musicbrainz recording lookup") } - pub fn lookup_artist(&self, id: String, rt: &Handle) -> Result<Arc<MbArtistRel>> { - cache_memory(&format!("ext/musicbrainz/artist/{id}.json"), move || { - rt.block_on(async { - let _permit = self.rate_limit.clone().acquire_owned().await?; - let permit_drop_ts = Instant::now() + Duration::from_secs(Self::MAX_PAR_REQ as u64); - info!("artist lookup: {id}"); + pub fn lookup_artist( + &self, + cache: &Cache, + id: String, + rt: &Handle, + ) -> Result<Arc<MbArtistRel>> { + cache + .cache_memory(&format!("ext/musicbrainz/artist/{id}.json"), move || { + rt.block_on(async { + let _permit = self.rate_limit.clone().acquire_owned().await?; + let permit_drop_ts = + Instant::now() + Duration::from_secs(Self::MAX_PAR_REQ as u64); + info!("artist lookup: {id}"); - let inc = [ - "area-rels", - "artist-rels", - "event-rels", - "genre-rels", - "instrument-rels", - "label-rels", - "place-rels", - "recording-rels", - "release-rels", - "release-group-rels", - "series-rels", - "url-rels", - "work-rels", - ] - .join("+"); + let inc = [ + "area-rels", + "artist-rels", + "event-rels", + "genre-rels", + "instrument-rels", + "label-rels", + "place-rels", + "recording-rels", + "release-rels", + "release-group-rels", + "series-rels", + "url-rels", + "work-rels", + ] + .join("+"); - let resp = self - .client - .get(format!( - "https://musicbrainz.org/ws/2/artist/{id}?inc={inc}" - )) - .send() - .await? - .error_for_status()? - .json::<MbArtistRel>() - .await?; + let resp = self + .client + .get(format!( + "https://musicbrainz.org/ws/2/artist/{id}?inc={inc}" + )) + .send() + .await? + .error_for_status()? + .json::<MbArtistRel>() + .await?; - tokio::task::spawn(async move { - sleep_until(permit_drop_ts).await; - drop(_permit); - }); + tokio::task::spawn(async move { + sleep_until(permit_drop_ts).await; + drop(_permit); + }); - Ok(resp) + Ok(resp) + }) }) - }) - .context("musicbrainz artist lookup") + .context("musicbrainz artist lookup") } } diff --git a/import/src/plugins/tmdb.rs b/import/src/plugins/tmdb.rs index cf50938..ea32c40 100644 --- a/import/src/plugins/tmdb.rs +++ b/import/src/plugins/tmdb.rs @@ -5,11 +5,11 @@ */ use crate::{ USER_AGENT, - plugins::{PluginContext, ImportPlugin, PluginInfo}, + plugins::{ImportPlugin, PluginContext, PluginInfo}, }; use anyhow::{Context, Result, anyhow, bail}; use chrono::{Utc, format::Parsed}; -use jellycache::{EscapeKey, HashKey, cache_memory, cache_store}; +use jellycache::{Cache, EscapeKey, HashKey}; use jellycommon::*; use jellydb::table::RowNum; use log::info; @@ -49,95 +49,117 @@ impl Tmdb { key: api_key.to_owned(), } } - pub fn search(&self, kind: TmdbKind, query: &str, rt: &Handle) -> Result<Arc<TmdbQuery>> { - cache_memory( - &format!("ext/tmdb/search/{kind}-{}.json", HashKey(query)), - move || { + pub fn search( + &self, + cache: &Cache, + kind: TmdbKind, + query: &str, + rt: &Handle, + ) -> Result<Arc<TmdbQuery>> { + cache + .cache_memory( + &format!("ext/tmdb/search/{kind}-{}.json", HashKey(query)), + move || { + rt.block_on(async { + info!("searching tmdb: {query:?}"); + Ok(self + .client + .get(format!( + "https://api.themoviedb.org/3/search/{kind}?query={}?api_key={}", + query.replace(" ", "+"), + self.key + )) + .send() + .await? + .error_for_status()? + .json::<TmdbQuery>() + .await?) + }) + }, + ) + .context("tmdb search") + } + pub fn details( + &self, + cache: &Cache, + kind: TmdbKind, + id: u64, + rt: &Handle, + ) -> Result<Arc<TmdbDetails>> { + cache + .cache_memory(&format!("ext/tmdb/details/{kind}-{id}.json"), move || { rt.block_on(async { - info!("searching tmdb: {query:?}"); + info!("fetching details: {id:?}"); Ok(self .client .get(format!( - "https://api.themoviedb.org/3/search/{kind}?query={}?api_key={}", - query.replace(" ", "+"), - self.key + "https://api.themoviedb.org/3/{kind}/{id}?api_key={}", + self.key, )) .send() .await? .error_for_status()? - .json::<TmdbQuery>() + .json() .await?) }) - }, - ) - .context("tmdb search") - } - pub fn details(&self, kind: TmdbKind, id: u64, rt: &Handle) -> Result<Arc<TmdbDetails>> { - cache_memory(&format!("ext/tmdb/details/{kind}-{id}.json"), move || { - rt.block_on(async { - info!("fetching details: {id:?}"); - Ok(self - .client - .get(format!( - "https://api.themoviedb.org/3/{kind}/{id}?api_key={}", - self.key, - )) - .send() - .await? - .error_for_status()? - .json() - .await?) }) - }) - .context("tmdb details") + .context("tmdb details") } - pub fn person_image(&self, id: u64, rt: &Handle) -> Result<Arc<TmdbPersonImage>> { - cache_memory(&format!("ext/tmdb/person/images/{id}.json"), move || { - rt.block_on(async { - Ok(self - .client - .get(format!( - "https://api.themoviedb.org/3/person/{id}/images?api_key={}", - self.key, - )) - .send() - .await? - .error_for_status()? - .json() - .await?) - }) - }) - .context("tmdb person images") - } - pub fn image(&self, path: &str, rt: &Handle) -> Result<String> { - cache_store( - format!("ext/tmdb/image/{}.image", EscapeKey(path)), - move || { + pub fn person_image( + &self, + cache: &Cache, + id: u64, + rt: &Handle, + ) -> Result<Arc<TmdbPersonImage>> { + cache + .cache_memory(&format!("ext/tmdb/person/images/{id}.json"), move || { rt.block_on(async { - info!("downloading image {path:?}"); Ok(self - .image_client - .get(format!("https://image.tmdb.org/t/p/original{path}")) + .client + .get(format!( + "https://api.themoviedb.org/3/person/{id}/images?api_key={}", + self.key, + )) .send() .await? .error_for_status()? - .bytes() - .await? - .to_vec()) + .json() + .await?) }) - }, - ) - .context("tmdb image download") + }) + .context("tmdb person images") + } + pub fn image(&self, cache: &Cache, path: &str, rt: &Handle) -> Result<String> { + cache + .store( + format!("ext/tmdb/image/{}.image", EscapeKey(path)), + move || { + rt.block_on(async { + info!("downloading image {path:?}"); + Ok(self + .image_client + .get(format!("https://image.tmdb.org/t/p/original{path}")) + .send() + .await? + .error_for_status()? + .bytes() + .await? + .to_vec()) + }) + }, + ) + .context("tmdb image download") } pub fn episode_details( &self, + cache: &Cache, series_id: u64, season: u64, episode: u64, rt: &Handle, ) -> Result<Arc<TmdbEpisode>> { - cache_memory(&format!("ext/tmdb/episode-details/{series_id}-S{season}-E{episode}.json"), move || { + cache.cache_memory(&format!("ext/tmdb/episode-details/{series_id}-S{season}-E{episode}.json"), move || { rt.block_on(async { info!("tmdb episode details {series_id} S={season} E={episode}"); Ok(self @@ -189,17 +211,17 @@ impl Tmdb { return Ok(()); }; - let details = self.details(tmdb_kind, tmdb_id, ct.rt)?; + let details = self.details(&ct.dba.cache, tmdb_kind, tmdb_id, ct.rt)?; let backdrop = details .backdrop_path .as_ref() - .map(|path| self.image(&path, ct.rt)) + .map(|path| self.image(&ct.dba.cache, &path, ct.rt)) .transpose() .context("backdrop image")?; let poster = details .poster_path .as_ref() - .map(|path| self.image(&path, ct.rt)) + .map(|path| self.image(&ct.dba.cache, &path, ct.rt)) .transpose() .context("poster image")?; @@ -268,11 +290,11 @@ impl Tmdb { let Some(series_id) = series_id else { return Ok(()); }; - let details = self.episode_details(series_id, season, episode, ct.rt)?; + let details = self.episode_details(&ct.dba.cache, series_id, season, episode, ct.rt)?; let cover = details .still_path .as_ref() - .map(|path| self.image(&path, ct.rt)) + .map(|path| self.image(&ct.dba.cache, &path, ct.rt)) .transpose() .context("still image download")?; let release_date = parse_release_date(&details.air_date)?; diff --git a/import/src/plugins/trakt.rs b/import/src/plugins/trakt.rs index 3569454..2cb063d 100644 --- a/import/src/plugins/trakt.rs +++ b/import/src/plugins/trakt.rs @@ -5,10 +5,10 @@ */ use crate::{ USER_AGENT, - plugins::{PluginContext, ImportPlugin, PluginInfo}, + plugins::{ImportPlugin, PluginContext, PluginInfo}, }; use anyhow::{Context, Result, anyhow, bail}; -use jellycache::{HashKey, cache_memory}; +use jellycache::{Cache, HashKey}; use jellycommon::{jellyobject::Tag, *}; use jellydb::table::RowNum; use log::info; @@ -52,89 +52,114 @@ impl Trakt { pub fn search( &self, + cache: &Cache, kinds: &[TraktKind], query: &str, rt: &Handle, ) -> Result<Arc<Vec<TraktSearchResult>>> { - cache_memory( - &format!("ext/trakt/search/{}.json", HashKey(query)), - move || { + cache + .cache_memory( + &format!("ext/trakt/search/{}.json", HashKey(query)), + move || { + rt.block_on(async { + let url = format!( + "https://api.trakt.tv/search/{}?query={}&extended=full", + kinds + .iter() + .map(|t| t.singular()) + .collect::<Vec<_>>() + .join(","), + urlencoding::encode(query), + ); + let res = self.client.get(url).send().await?.error_for_status()?; + Ok(res.json().await?) + }) + }, + ) + .context("trakt search") + } + + pub fn lookup( + &self, + cache: &Cache, + kind: TraktKind, + id: u64, + rt: &Handle, + ) -> Result<Arc<TraktMediaObject>> { + cache + .cache_memory(&format!("ext/trakt/lookup/{kind}-{id}.json"), move || { rt.block_on(async { - let url = format!( - "https://api.trakt.tv/search/{}?query={}&extended=full", - kinds - .iter() - .map(|t| t.singular()) - .collect::<Vec<_>>() - .join(","), - urlencoding::encode(query), - ); + info!("trakt lookup {kind:?}:{id:?}"); + let url = format!("https://api.trakt.tv/{}/{id}?extended=full", kind.plural()); let res = self.client.get(url).send().await?.error_for_status()?; Ok(res.json().await?) }) - }, - ) - .context("trakt search") - } - - pub fn lookup(&self, kind: TraktKind, id: u64, rt: &Handle) -> Result<Arc<TraktMediaObject>> { - cache_memory(&format!("ext/trakt/lookup/{kind}-{id}.json"), move || { - rt.block_on(async { - info!("trakt lookup {kind:?}:{id:?}"); - let url = format!("https://api.trakt.tv/{}/{id}?extended=full", kind.plural()); - let res = self.client.get(url).send().await?.error_for_status()?; - Ok(res.json().await?) }) - }) - .context("trakt lookup") + .context("trakt lookup") } - pub fn people(&self, kind: TraktKind, id: u64, rt: &Handle) -> Result<Arc<TraktPeople>> { - cache_memory(&format!("ext/trakt/people/{kind}-{id}.json"), move || { - rt.block_on(async { - info!("trakt people {kind:?}:{id:?}"); - let url = format!( - "https://api.trakt.tv/{}/{id}/people?extended=full", - kind.plural() - ); - let res = self.client.get(url).send().await?.error_for_status()?; - Ok(res.json().await?) + pub fn people( + &self, + cache: &Cache, + kind: TraktKind, + id: u64, + rt: &Handle, + ) -> Result<Arc<TraktPeople>> { + cache + .cache_memory(&format!("ext/trakt/people/{kind}-{id}.json"), move || { + rt.block_on(async { + info!("trakt people {kind:?}:{id:?}"); + let url = format!( + "https://api.trakt.tv/{}/{id}/people?extended=full", + kind.plural() + ); + let res = self.client.get(url).send().await?.error_for_status()?; + Ok(res.json().await?) + }) }) - }) - .context("trakt people") + .context("trakt people") } - pub fn show_seasons(&self, id: u64, rt: &Handle) -> Result<Arc<Vec<TraktSeason>>> { - cache_memory(&format!("ext/trakt/seasons/{id}.json"), move || { - rt.block_on(async { - info!("trakt seasons {id:?}"); - let url = format!("https://api.trakt.tv/shows/{id}/seasons?extended=full"); - let res = self.client.get(url).send().await?.error_for_status()?; - Ok(res.json().await?) + pub fn show_seasons( + &self, + cache: &Cache, + id: u64, + rt: &Handle, + ) -> Result<Arc<Vec<TraktSeason>>> { + cache + .cache_memory(&format!("ext/trakt/seasons/{id}.json"), move || { + rt.block_on(async { + info!("trakt seasons {id:?}"); + let url = format!("https://api.trakt.tv/shows/{id}/seasons?extended=full"); + let res = self.client.get(url).send().await?.error_for_status()?; + Ok(res.json().await?) + }) }) - }) - .context("trakt show seasons") + .context("trakt show seasons") } pub fn show_season_episodes( &self, + cache: &Cache, id: u64, season: u64, rt: &Handle, ) -> Result<Arc<Vec<TraktEpisode>>> { - cache_memory( - &format!("ext/trakt/episodes/{id}-S{season}.json"), - move || { - rt.block_on(async { - info!("trakt episodes {id:?} season={season}"); - let url = - format!("https://api.trakt.tv/shows/{id}/seasons/{season}?extended=full"); - let res = self.client.get(url).send().await?.error_for_status()?; - Ok(res.json().await?) - }) - }, - ) - .context("trakt show season episodes") + cache + .cache_memory( + &format!("ext/trakt/episodes/{id}-S{season}.json"), + move || { + rt.block_on(async { + info!("trakt episodes {id:?} season={season}"); + let url = format!( + "https://api.trakt.tv/shows/{id}/seasons/{season}?extended=full" + ); + let res = self.client.get(url).send().await?.error_for_status()?; + Ok(res.json().await?) + }) + }, + ) + .context("trakt show season episodes") } } @@ -425,7 +450,7 @@ impl Trakt { return Ok(()); }; - let details = self.lookup(trakt_kind, trakt_id, ct.rt)?; + let details = self.lookup(&ct.dba.cache, trakt_kind, trakt_id, ct.rt)?; // let people = self.people(trakt_kind, trakt_id, ct.rt)?; // let mut people_map = BTreeMap::<CreditCategory, Vec<Appearance>>::new(); @@ -525,9 +550,9 @@ impl Trakt { return Ok(()); }; - let seasons = self.show_seasons(show_id, ct.rt)?; + let seasons = self.show_seasons(&ct.dba.cache, show_id, ct.rt)?; if seasons.iter().any(|x| x.number == season) { - let episodes = self.show_season_episodes(show_id, season, ct.rt)?; + let episodes = self.show_season_episodes(&ct.dba.cache, show_id, season, ct.rt)?; if let Some(episode) = episodes.get(episode.saturating_sub(1) as usize) { ct.dba.update_node(node, |mut node| { node = node.as_object().insert(NO_KIND, KIND_EPISODE); diff --git a/import/src/plugins/vgmdb.rs b/import/src/plugins/vgmdb.rs index c62eb90..534b241 100644 --- a/import/src/plugins/vgmdb.rs +++ b/import/src/plugins/vgmdb.rs @@ -9,7 +9,7 @@ use crate::{ plugins::{ImportPlugin, PluginInfo}, }; use anyhow::{Context, Result}; -use jellycache::{HashKey, cache, cache_store}; +use jellycache::{Cache, HashKey}; use log::info; use regex::Regex; use reqwest::{ @@ -62,34 +62,40 @@ impl Vgmdb { } } - pub fn get_artist_image(&self, id: u64, rt: &Handle) -> Result<Option<String>> { - if let Some(url) = self.get_artist_image_url(id, rt)? { - cache_store( - format!("ext/vgmdb/artist-image/{}.image", HashKey(&url)), - move || { - rt.block_on(async { - info!("downloading image {url:?}"); - Ok(self - .client - .get(url) - .send() - .await? - .error_for_status()? - .bytes() - .await? - .to_vec()) - }) - }, - ) - .context("vgmdb media download") - .map(Some) + pub fn get_artist_image(&self, cache: &Cache, id: u64, rt: &Handle) -> Result<Option<String>> { + if let Some(url) = self.get_artist_image_url(cache, id, rt)? { + cache + .store( + format!("ext/vgmdb/artist-image/{}.image", HashKey(&url)), + move || { + rt.block_on(async { + info!("downloading image {url:?}"); + Ok(self + .client + .get(url) + .send() + .await? + .error_for_status()? + .bytes() + .await? + .to_vec()) + }) + }, + ) + .context("vgmdb media download") + .map(Some) } else { Ok(None) } } - pub fn get_artist_image_url(&self, id: u64, rt: &Handle) -> Result<Option<String>> { - let html = self.scrape_artist_page(id, rt)?; + pub fn get_artist_image_url( + &self, + cache: &Cache, + id: u64, + rt: &Handle, + ) -> Result<Option<String>> { + let html = self.scrape_artist_page(cache, id, rt)?; if let Some(cap) = RE_IMAGE_URL_FROM_HTML.captures(&str::from_utf8(&html).unwrap()) { if let Some(url) = cap.name("url").map(|m| m.as_str()) { return Ok(Some(url.to_string())); @@ -98,32 +104,33 @@ impl Vgmdb { Ok(None) } - pub fn scrape_artist_page(&self, id: u64, rt: &Handle) -> Result<Vec<u8>> { - cache(&format!("ext/vgmdb/artist-page/{id}.html"), move || { - rt.block_on(async { - let _permit = self.rate_limit.clone().acquire_owned().await?; - let permit_drop_ts = Instant::now() + Duration::from_secs(1); - info!("scrape artist: {id}"); + pub fn scrape_artist_page(&self, cache: &Cache, id: u64, rt: &Handle) -> Result<Vec<u8>> { + cache + .cache(&format!("ext/vgmdb/artist-page/{id}.html"), move || { + rt.block_on(async { + let _permit = self.rate_limit.clone().acquire_owned().await?; + let permit_drop_ts = Instant::now() + Duration::from_secs(1); + info!("scrape artist: {id}"); - let resp = self - .client - .get(format!("https://vgmdb.net/artist/{id}")) - .send() - .await? - .error_for_status()? - .bytes() - .await? - .to_vec(); + let resp = self + .client + .get(format!("https://vgmdb.net/artist/{id}")) + .send() + .await? + .error_for_status()? + .bytes() + .await? + .to_vec(); - tokio::task::spawn(async move { - sleep_until(permit_drop_ts).await; - drop(_permit); - }); + tokio::task::spawn(async move { + sleep_until(permit_drop_ts).await; + drop(_permit); + }); - Ok(resp) + Ok(resp) + }) }) - }) - .context("vgmdb artist page scrape") + .context("vgmdb artist page scrape") } } diff --git a/import/src/plugins/wikidata.rs b/import/src/plugins/wikidata.rs index e3077e0..3afd393 100644 --- a/import/src/plugins/wikidata.rs +++ b/import/src/plugins/wikidata.rs @@ -9,7 +9,7 @@ use crate::{ plugins::{ImportPlugin, PluginInfo}, }; use anyhow::{Context, Result, bail}; -use jellycache::{EscapeKey, cache_memory}; +use jellycache::{Cache, EscapeKey}; use log::info; use reqwest::{ Client, ClientBuilder, @@ -91,8 +91,13 @@ impl Wikidata { Self { client } } - pub fn query_image_path(&self, id: String, rt: &Handle) -> Result<Option<String>> { - let response = self.query(id.clone(), rt)?; + pub fn query_image_path( + &self, + cache: &Cache, + id: String, + rt: &Handle, + ) -> Result<Option<String>> { + let response = self.query(cache, id.clone(), rt)?; if let Some(entity) = response.entities.get(&id) { if let Some(images) = entity.claims.get(properties::IMAGE) { for image in images { @@ -110,24 +115,25 @@ impl Wikidata { Ok(None) } - pub fn query(&self, id: String, rt: &Handle) -> Result<Arc<WikidataResponse>> { - cache_memory( - &format!("ext/wikidata/{}.json", EscapeKey(&id)), - move || { - rt.block_on(async { - info!("entity query: {id}"); - Ok(self - .client - .get(format!("https://www.wikidata.org/entity/{id}")) - .send() - .await? - .error_for_status()? - .json() - .await?) - }) - }, - ) - .context("wikidata entity") + pub fn query(&self, cache: &Cache, id: String, rt: &Handle) -> Result<Arc<WikidataResponse>> { + cache + .cache_memory( + &format!("ext/wikidata/{}.json", EscapeKey(&id)), + move || { + rt.block_on(async { + info!("entity query: {id}"); + Ok(self + .client + .get(format!("https://www.wikidata.org/entity/{id}")) + .send() + .await? + .error_for_status()? + .json() + .await?) + }) + }, + ) + .context("wikidata entity") } } diff --git a/import/src/plugins/wikimedia_commons.rs b/import/src/plugins/wikimedia_commons.rs index c849b61..aebf5dd 100644 --- a/import/src/plugins/wikimedia_commons.rs +++ b/import/src/plugins/wikimedia_commons.rs @@ -9,7 +9,7 @@ use crate::{ plugins::{ImportPlugin, PluginInfo}, }; use anyhow::{Context, Result}; -use jellycache::{EscapeKey, cache_store}; +use jellycache::{Cache, EscapeKey}; use reqwest::{ Client, ClientBuilder, header::{HeaderMap, HeaderName, HeaderValue}, @@ -39,27 +39,33 @@ impl WikimediaCommons { Self { client } } - pub fn image_by_filename(&self, filename: String, rt: &Handle) -> Result<String> { - cache_store( - format!("ext/wikimedia-commons/image/{}.image", EscapeKey(&filename)), - move || { - rt.block_on(async { - Ok(self - .client - .get(format!( - "https://commons.wikimedia.org/wiki/Special:FilePath/{}", - filename.replace(" ", "_") - )) - .send() - .await? - .error_for_status()? - .bytes() - .await? - .to_vec()) - }) - }, - ) - .context("mediawiki image by filename") + pub fn image_by_filename( + &self, + cache: &Cache, + filename: String, + rt: &Handle, + ) -> Result<String> { + cache + .store( + format!("ext/wikimedia-commons/image/{}.image", EscapeKey(&filename)), + move || { + rt.block_on(async { + Ok(self + .client + .get(format!( + "https://commons.wikimedia.org/wiki/Special:FilePath/{}", + filename.replace(" ", "_") + )) + .send() + .await? + .error_for_status()? + .bytes() + .await? + .to_vec()) + }) + }, + ) + .context("mediawiki image by filename") } } |