diff options
| author | metamuffin <metamuffin@disroot.org> | 2026-01-24 23:06:33 +0100 |
|---|---|---|
| committer | metamuffin <metamuffin@disroot.org> | 2026-01-24 23:06:33 +0100 |
| commit | 2bcccb18a6cb8bf836f57c3d86f759b19699def2 (patch) | |
| tree | ef55a10c6d9703677a983b8ca900fb4578a08eb3 | |
| parent | b2e88a8beabf04adc28947cf82996e8692a68b71 (diff) | |
| download | jellything-2bcccb18a6cb8bf836f57c3d86f759b19699def2.tar jellything-2bcccb18a6cb8bf836f57c3d86f759b19699def2.tar.bz2 jellything-2bcccb18a6cb8bf836f57c3d86f759b19699def2.tar.zst | |
cache as object
30 files changed, 547 insertions, 410 deletions
@@ -3988,6 +3988,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" [[package]] +name = "tools" +version = "0.1.0" +dependencies = [ + "anyhow", + "rocksdb", +] + +[[package]] name = "tower" version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -18,6 +18,7 @@ members = [ "remuxer", "common/object", "kv", + "cache/tools", ] resolver = "3" diff --git a/cache/src/lib.rs b/cache/src/lib.rs index be2b331..4d6bbcf 100644 --- a/cache/src/lib.rs +++ b/cache/src/lib.rs @@ -81,14 +81,10 @@ impl Cache { Ok(out) } - pub fn cache_read(&self, key: &str) -> Result<Option<Vec<u8>>> { + pub fn read(&self, key: &str) -> Result<Option<Vec<u8>>> { self.storage.read(key) } - pub fn cache_store( - &self, - key: String, - generate: impl FnOnce() -> Result<Vec<u8>>, - ) -> Result<String> { + pub fn store(&self, key: String, generate: impl FnOnce() -> Result<Vec<u8>>) -> Result<String> { self.cache(&key, generate)?; Ok(key) } diff --git a/cache/tools/Cargo.toml b/cache/tools/Cargo.toml new file mode 100644 index 0000000..8f249b5 --- /dev/null +++ b/cache/tools/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "tools" +version = "0.1.0" +edition = "2024" + +[dependencies] +rocksdb = { version = "0.24.0", features = ["multi-threaded-cf"] } +anyhow = "1.0.100" + +[[bin]] +name = "cache_fs_to_rocksdb" +path = "cache_fs_to_rocksdb.rs" + +[[bin]] +name = "cache_rocksdb_delete_prefix" +path = "cache_rocksdb_delete_prefix.rs" diff --git a/cache/src/bin/cache_fs_to_rocksdb.rs b/cache/tools/cache_fs_to_rocksdb.rs index d283dcb..d283dcb 100644 --- a/cache/src/bin/cache_fs_to_rocksdb.rs +++ b/cache/tools/cache_fs_to_rocksdb.rs diff --git a/cache/src/bin/cache_rocksdb_delete_prefix.rs b/cache/tools/cache_rocksdb_delete_prefix.rs index e09ce61..e09ce61 100644 --- a/cache/src/bin/cache_rocksdb_delete_prefix.rs +++ b/cache/tools/cache_rocksdb_delete_prefix.rs diff --git a/database/Cargo.toml b/database/Cargo.toml index 078694e..5d629a3 100644 --- a/database/Cargo.toml +++ b/database/Cargo.toml @@ -11,3 +11,6 @@ serde_json = "1.0.145" anyhow = "1.0.100" bytemuck = { version = "1.24.0", features = ["derive"] } jellykv = { path = "../kv" } + +[dev-dependencies] +jellykv = { path = "../kv", features = ["memory"] } diff --git a/database/src/table.rs b/database/src/table.rs index c3b4342..0b6382a 100644 --- a/database/src/table.rs +++ b/database/src/table.rs @@ -112,7 +112,7 @@ mod test { #[test] pub fn insert_get() -> Result<()> { - let db = jellykv_memory::new(); + let db = jellykv::memory::new(); let table = Table::new(5); let mut bob_row = 0; @@ -133,7 +133,7 @@ mod test { #[test] pub fn update() -> Result<()> { - let db = jellykv_memory::new(); + let db = jellykv::memory::new(); let table = Table::new(5); let mut bob_row = 0; diff --git a/import/src/lib.rs b/import/src/lib.rs index 7d48867..bec5786 100644 --- a/import/src/lib.rs +++ b/import/src/lib.rs @@ -9,7 +9,7 @@ pub mod plugins; pub mod reporting; use crate::{ - plugins::{PluginContext, ImportPlugin, infojson::is_info_json, init_plugins, misc::is_cover}, + plugins::{ImportPlugin, PluginContext, infojson::is_info_json, init_plugins, misc::is_cover}, reporting::IMPORT_PROGRESS, }; use anyhow::{Context, Result, anyhow}; @@ -377,9 +377,9 @@ fn import_file( } reporting::set_task(format!("demuxer meta: {path:?}")); - let Some(seg) = - reporting::catch(read_media_metadata(path).context(anyhow!("media {path:?}"))) - else { + let Some(seg) = reporting::catch( + read_media_metadata(&ct.dba.cache, path).context(anyhow!("media {path:?}")), + ) else { return; }; for p in plugins { @@ -542,7 +542,7 @@ pub fn read_media_metadata(cache: &Cache, path: &Path) -> Result<Arc<matroska::S if let Some(attachments) = &mut attachments { for att in &mut attachments.files { if let Some(fname) = is_useful_attachment(&att) { - let key = cache.cache_store( + let key = cache.store( format!("media/attachment/{}-{fname}", HashKey(path)), || Ok(att.data.clone()), )?; diff --git a/import/src/plugins/acoustid.rs b/import/src/plugins/acoustid.rs index 9891927..0e16670 100644 --- a/import/src/plugins/acoustid.rs +++ b/import/src/plugins/acoustid.rs @@ -5,10 +5,10 @@ */ use crate::{ USER_AGENT, - plugins::{PluginContext, ImportPlugin, PluginInfo}, + plugins::{ImportPlugin, PluginContext, PluginInfo}, }; use anyhow::{Context, Result}; -use jellycache::{HashKey, cache_memory}; +use jellycache::{Cache, HashKey}; use jellycommon::{ IDENT_ACOUST_ID_TRACK, IDENT_MUSICBRAINZ_RECORDING, NO_IDENTIFIERS, jellyobject::Object, }; @@ -92,8 +92,13 @@ impl AcoustID { } } - pub fn get_atid_mbid(&self, fp: &Fingerprint, rt: &Handle) -> Result<Option<(String, String)>> { - let res = self.lookup(fp.to_owned(), rt)?; + pub fn get_atid_mbid( + &self, + cache: &Cache, + fp: &Fingerprint, + rt: &Handle, + ) -> Result<Option<(String, String)>> { + let res = self.lookup(cache, fp.to_owned(), rt)?; for r in &res.results { if let Some(k) = r.recordings.first() { return Ok(Some((r.id.clone(), k.id.clone()))); @@ -102,8 +107,13 @@ impl AcoustID { Ok(None) } - pub fn lookup(&self, fp: Fingerprint, rt: &Handle) -> Result<Arc<AcoustIDLookupResponse>> { - cache_memory(&format!("ext/acoustid/{}.json", HashKey(&fp)) , move || rt.block_on(async { + pub fn lookup( + &self, + cache: &Cache, + fp: Fingerprint, + rt: &Handle, + ) -> Result<Arc<AcoustIDLookupResponse>> { + cache.cache_memory(&format!("ext/acoustid/{}.json", HashKey(&fp)) , move || rt.block_on(async { let _permit = self.rate_limit.clone().acquire_owned().await?; let permit_drop_ts = Instant::now() + Duration::SECOND; info!("acoustid lookup"); @@ -132,8 +142,8 @@ impl AcoustID { } } -pub(crate) fn acoustid_fingerprint(path: &Path) -> Result<Arc<Fingerprint>> { - cache_memory( +pub(crate) fn acoustid_fingerprint(cache: &Cache, path: &Path) -> Result<Arc<Fingerprint>> { + cache.cache_memory( &format!("media/chromaprint/{}.json", HashKey(path)), move || { let child = Command::new("fpcalc") @@ -173,9 +183,9 @@ impl ImportPlugin for AcoustID { if !ct.iflags.use_acoustid { return Ok(()); } - let fp = acoustid_fingerprint(path)?; + let fp = acoustid_fingerprint(&ct.dba.cache, path)?; - if let Some((atid, mbid)) = self.get_atid_mbid(&fp, &ct.rt)? { + if let Some((atid, mbid)) = self.get_atid_mbid(&ct.dba.cache, &fp, &ct.rt)? { ct.dba.db.write_transaction(&mut |txn| { let ob = ct.dba.nodes.get(txn, node)?.unwrap(); let ob = ob.as_object(); diff --git a/import/src/plugins/infojson.rs b/import/src/plugins/infojson.rs index d62983d..3426679 100644 --- a/import/src/plugins/infojson.rs +++ b/import/src/plugins/infojson.rs @@ -3,16 +3,10 @@ which is licensed under the GNU Affero General Public License (version 3); see /COPYING. Copyright (C) 2026 metamuffin <metamuffin.org> */ -use crate::plugins::{PluginContext, ImportPlugin, PluginInfo}; +use crate::plugins::{ImportPlugin, PluginContext, PluginInfo}; use anyhow::{Context, Result, anyhow}; use chrono::{Utc, format::Parsed}; -use jellycache::cache_read; -use jellycommon::{ - IDENT_BANDCAMP, IDENT_YOUTUBE_CHANNEL, IDENT_YOUTUBE_CHANNEL_HANDLE, IDENT_YOUTUBE_VIDEO, - KIND_CHANNEL, KIND_MUSIC, KIND_SHORTFORMVIDEO, KIND_VIDEO, NO_DESCRIPTION, NO_IDENTIFIERS, - NO_KIND, NO_RATINGS, NO_RELEASEDATE, NO_SUBTITLE, NO_TAG, NO_TITLE, RTYP_YOUTUBE_FOLLOWERS, - RTYP_YOUTUBE_LIKES, RTYP_YOUTUBE_VIEWS, -}; +use jellycommon::*; use jellydb::table::RowNum; use jellyremuxer::matroska::{AttachedFile, Segment}; use log::info; @@ -215,7 +209,10 @@ impl ImportPlugin for Infojson { .flat_map(|a| &a.files) .find(is_info_json) .map(|att| { - let data = cache_read(str::from_utf8(&att.data).unwrap())? + let data = ct + .dba + .cache + .read(str::from_utf8(&att.data).unwrap())? .ok_or(anyhow!("info json cache missing"))?; anyhow::Ok(serde_json::from_slice::<YVideo>(&data)?) }) diff --git a/import/src/plugins/misc.rs b/import/src/plugins/misc.rs index babbcec..554f473 100644 --- a/import/src/plugins/misc.rs +++ b/import/src/plugins/misc.rs @@ -3,9 +3,9 @@ which is licensed under the GNU Affero General Public License (version 3); see /COPYING. Copyright (C) 2026 metamuffin <metamuffin.org> */ -use crate::plugins::{PluginContext, ImportPlugin, PluginInfo}; +use crate::plugins::{ImportPlugin, PluginContext, PluginInfo}; use anyhow::{Context, Result, bail}; -use jellycache::{HashKey, cache_store}; +use jellycache::HashKey; use jellycommon::{jellyobject::inspect::Inspector, *}; use jellydb::table::RowNum; use jellyremuxer::matroska::{AttachedFile, Segment}; @@ -30,7 +30,7 @@ impl ImportPlugin for ImageFiles { _ => return Ok(()), }; info!("import {:?} at {path:?}", Inspector(&TAGREG, slot)); - let asset = cache_store( + let asset = ct.dba.cache.store( format!( "media/literal/{}-{}.image", HashKey(path), diff --git a/import/src/plugins/musicbrainz.rs b/import/src/plugins/musicbrainz.rs index 413e487..27e4d0f 100644 --- a/import/src/plugins/musicbrainz.rs +++ b/import/src/plugins/musicbrainz.rs @@ -9,7 +9,7 @@ use crate::{ plugins::{ImportPlugin, PluginInfo}, }; use anyhow::{Context, Result}; -use jellycache::cache_memory; +use jellycache::Cache; use log::info; use reqwest::{ Client, ClientBuilder, @@ -225,98 +225,112 @@ impl MusicBrainz { } } - pub fn lookup_recording(&self, id: String, rt: &Handle) -> Result<Arc<MbRecordingRel>> { - cache_memory(&format!("ext/musicbrainz/recording/{id}.json"), move || { - rt.block_on(async { - let _permit = self.rate_limit.clone().acquire_owned().await?; - let permit_drop_ts = Instant::now() + Duration::from_secs(Self::MAX_PAR_REQ as u64); - info!("recording lookup: {id}"); + pub fn lookup_recording( + &self, + cache: &Cache, + id: String, + rt: &Handle, + ) -> Result<Arc<MbRecordingRel>> { + cache + .cache_memory(&format!("ext/musicbrainz/recording/{id}.json"), move || { + rt.block_on(async { + let _permit = self.rate_limit.clone().acquire_owned().await?; + let permit_drop_ts = + Instant::now() + Duration::from_secs(Self::MAX_PAR_REQ as u64); + info!("recording lookup: {id}"); - let inc = [ - "isrcs", - "artists", - "area-rels", - "artist-rels", - "event-rels", - "genre-rels", - "instrument-rels", - "label-rels", - "place-rels", - "recording-rels", - "release-rels", - "release-group-rels", - "series-rels", - "url-rels", - "work-rels", - ] - .join("+"); + let inc = [ + "isrcs", + "artists", + "area-rels", + "artist-rels", + "event-rels", + "genre-rels", + "instrument-rels", + "label-rels", + "place-rels", + "recording-rels", + "release-rels", + "release-group-rels", + "series-rels", + "url-rels", + "work-rels", + ] + .join("+"); - let resp = self - .client - .get(format!( - "https://musicbrainz.org/ws/2/recording/{id}?inc={inc}" - )) - .send() - .await? - .error_for_status()? - .json::<MbRecordingRel>() - .await?; + let resp = self + .client + .get(format!( + "https://musicbrainz.org/ws/2/recording/{id}?inc={inc}" + )) + .send() + .await? + .error_for_status()? + .json::<MbRecordingRel>() + .await?; - tokio::task::spawn(async move { - sleep_until(permit_drop_ts).await; - drop(_permit); - }); + tokio::task::spawn(async move { + sleep_until(permit_drop_ts).await; + drop(_permit); + }); - Ok(resp) + Ok(resp) + }) }) - }) - .context("musicbrainz recording lookup") + .context("musicbrainz recording lookup") } - pub fn lookup_artist(&self, id: String, rt: &Handle) -> Result<Arc<MbArtistRel>> { - cache_memory(&format!("ext/musicbrainz/artist/{id}.json"), move || { - rt.block_on(async { - let _permit = self.rate_limit.clone().acquire_owned().await?; - let permit_drop_ts = Instant::now() + Duration::from_secs(Self::MAX_PAR_REQ as u64); - info!("artist lookup: {id}"); + pub fn lookup_artist( + &self, + cache: &Cache, + id: String, + rt: &Handle, + ) -> Result<Arc<MbArtistRel>> { + cache + .cache_memory(&format!("ext/musicbrainz/artist/{id}.json"), move || { + rt.block_on(async { + let _permit = self.rate_limit.clone().acquire_owned().await?; + let permit_drop_ts = + Instant::now() + Duration::from_secs(Self::MAX_PAR_REQ as u64); + info!("artist lookup: {id}"); - let inc = [ - "area-rels", - "artist-rels", - "event-rels", - "genre-rels", - "instrument-rels", - "label-rels", - "place-rels", - "recording-rels", - "release-rels", - "release-group-rels", - "series-rels", - "url-rels", - "work-rels", - ] - .join("+"); + let inc = [ + "area-rels", + "artist-rels", + "event-rels", + "genre-rels", + "instrument-rels", + "label-rels", + "place-rels", + "recording-rels", + "release-rels", + "release-group-rels", + "series-rels", + "url-rels", + "work-rels", + ] + .join("+"); - let resp = self - .client - .get(format!( - "https://musicbrainz.org/ws/2/artist/{id}?inc={inc}" - )) - .send() - .await? - .error_for_status()? - .json::<MbArtistRel>() - .await?; + let resp = self + .client + .get(format!( + "https://musicbrainz.org/ws/2/artist/{id}?inc={inc}" + )) + .send() + .await? + .error_for_status()? + .json::<MbArtistRel>() + .await?; - tokio::task::spawn(async move { - sleep_until(permit_drop_ts).await; - drop(_permit); - }); + tokio::task::spawn(async move { + sleep_until(permit_drop_ts).await; + drop(_permit); + }); - Ok(resp) + Ok(resp) + }) }) - }) - .context("musicbrainz artist lookup") + .context("musicbrainz artist lookup") } } diff --git a/import/src/plugins/tmdb.rs b/import/src/plugins/tmdb.rs index cf50938..ea32c40 100644 --- a/import/src/plugins/tmdb.rs +++ b/import/src/plugins/tmdb.rs @@ -5,11 +5,11 @@ */ use crate::{ USER_AGENT, - plugins::{PluginContext, ImportPlugin, PluginInfo}, + plugins::{ImportPlugin, PluginContext, PluginInfo}, }; use anyhow::{Context, Result, anyhow, bail}; use chrono::{Utc, format::Parsed}; -use jellycache::{EscapeKey, HashKey, cache_memory, cache_store}; +use jellycache::{Cache, EscapeKey, HashKey}; use jellycommon::*; use jellydb::table::RowNum; use log::info; @@ -49,95 +49,117 @@ impl Tmdb { key: api_key.to_owned(), } } - pub fn search(&self, kind: TmdbKind, query: &str, rt: &Handle) -> Result<Arc<TmdbQuery>> { - cache_memory( - &format!("ext/tmdb/search/{kind}-{}.json", HashKey(query)), - move || { + pub fn search( + &self, + cache: &Cache, + kind: TmdbKind, + query: &str, + rt: &Handle, + ) -> Result<Arc<TmdbQuery>> { + cache + .cache_memory( + &format!("ext/tmdb/search/{kind}-{}.json", HashKey(query)), + move || { + rt.block_on(async { + info!("searching tmdb: {query:?}"); + Ok(self + .client + .get(format!( + "https://api.themoviedb.org/3/search/{kind}?query={}?api_key={}", + query.replace(" ", "+"), + self.key + )) + .send() + .await? + .error_for_status()? + .json::<TmdbQuery>() + .await?) + }) + }, + ) + .context("tmdb search") + } + pub fn details( + &self, + cache: &Cache, + kind: TmdbKind, + id: u64, + rt: &Handle, + ) -> Result<Arc<TmdbDetails>> { + cache + .cache_memory(&format!("ext/tmdb/details/{kind}-{id}.json"), move || { rt.block_on(async { - info!("searching tmdb: {query:?}"); + info!("fetching details: {id:?}"); Ok(self .client .get(format!( - "https://api.themoviedb.org/3/search/{kind}?query={}?api_key={}", - query.replace(" ", "+"), - self.key + "https://api.themoviedb.org/3/{kind}/{id}?api_key={}", + self.key, )) .send() .await? .error_for_status()? - .json::<TmdbQuery>() + .json() .await?) }) - }, - ) - .context("tmdb search") - } - pub fn details(&self, kind: TmdbKind, id: u64, rt: &Handle) -> Result<Arc<TmdbDetails>> { - cache_memory(&format!("ext/tmdb/details/{kind}-{id}.json"), move || { - rt.block_on(async { - info!("fetching details: {id:?}"); - Ok(self - .client - .get(format!( - "https://api.themoviedb.org/3/{kind}/{id}?api_key={}", - self.key, - )) - .send() - .await? - .error_for_status()? - .json() - .await?) }) - }) - .context("tmdb details") + .context("tmdb details") } - pub fn person_image(&self, id: u64, rt: &Handle) -> Result<Arc<TmdbPersonImage>> { - cache_memory(&format!("ext/tmdb/person/images/{id}.json"), move || { - rt.block_on(async { - Ok(self - .client - .get(format!( - "https://api.themoviedb.org/3/person/{id}/images?api_key={}", - self.key, - )) - .send() - .await? - .error_for_status()? - .json() - .await?) - }) - }) - .context("tmdb person images") - } - pub fn image(&self, path: &str, rt: &Handle) -> Result<String> { - cache_store( - format!("ext/tmdb/image/{}.image", EscapeKey(path)), - move || { + pub fn person_image( + &self, + cache: &Cache, + id: u64, + rt: &Handle, + ) -> Result<Arc<TmdbPersonImage>> { + cache + .cache_memory(&format!("ext/tmdb/person/images/{id}.json"), move || { rt.block_on(async { - info!("downloading image {path:?}"); Ok(self - .image_client - .get(format!("https://image.tmdb.org/t/p/original{path}")) + .client + .get(format!( + "https://api.themoviedb.org/3/person/{id}/images?api_key={}", + self.key, + )) .send() .await? .error_for_status()? - .bytes() - .await? - .to_vec()) + .json() + .await?) }) - }, - ) - .context("tmdb image download") + }) + .context("tmdb person images") + } + pub fn image(&self, cache: &Cache, path: &str, rt: &Handle) -> Result<String> { + cache + .store( + format!("ext/tmdb/image/{}.image", EscapeKey(path)), + move || { + rt.block_on(async { + info!("downloading image {path:?}"); + Ok(self + .image_client + .get(format!("https://image.tmdb.org/t/p/original{path}")) + .send() + .await? + .error_for_status()? + .bytes() + .await? + .to_vec()) + }) + }, + ) + .context("tmdb image download") } pub fn episode_details( &self, + cache: &Cache, series_id: u64, season: u64, episode: u64, rt: &Handle, ) -> Result<Arc<TmdbEpisode>> { - cache_memory(&format!("ext/tmdb/episode-details/{series_id}-S{season}-E{episode}.json"), move || { + cache.cache_memory(&format!("ext/tmdb/episode-details/{series_id}-S{season}-E{episode}.json"), move || { rt.block_on(async { info!("tmdb episode details {series_id} S={season} E={episode}"); Ok(self @@ -189,17 +211,17 @@ impl Tmdb { return Ok(()); }; - let details = self.details(tmdb_kind, tmdb_id, ct.rt)?; + let details = self.details(&ct.dba.cache, tmdb_kind, tmdb_id, ct.rt)?; let backdrop = details .backdrop_path .as_ref() - .map(|path| self.image(&path, ct.rt)) + .map(|path| self.image(&ct.dba.cache, &path, ct.rt)) .transpose() .context("backdrop image")?; let poster = details .poster_path .as_ref() - .map(|path| self.image(&path, ct.rt)) + .map(|path| self.image(&ct.dba.cache, &path, ct.rt)) .transpose() .context("poster image")?; @@ -268,11 +290,11 @@ impl Tmdb { let Some(series_id) = series_id else { return Ok(()); }; - let details = self.episode_details(series_id, season, episode, ct.rt)?; + let details = self.episode_details(&ct.dba.cache, series_id, season, episode, ct.rt)?; let cover = details .still_path .as_ref() - .map(|path| self.image(&path, ct.rt)) + .map(|path| self.image(&ct.dba.cache, &path, ct.rt)) .transpose() .context("still image download")?; let release_date = parse_release_date(&details.air_date)?; diff --git a/import/src/plugins/trakt.rs b/import/src/plugins/trakt.rs index 3569454..2cb063d 100644 --- a/import/src/plugins/trakt.rs +++ b/import/src/plugins/trakt.rs @@ -5,10 +5,10 @@ */ use crate::{ USER_AGENT, - plugins::{PluginContext, ImportPlugin, PluginInfo}, + plugins::{ImportPlugin, PluginContext, PluginInfo}, }; use anyhow::{Context, Result, anyhow, bail}; -use jellycache::{HashKey, cache_memory}; +use jellycache::{Cache, HashKey}; use jellycommon::{jellyobject::Tag, *}; use jellydb::table::RowNum; use log::info; @@ -52,89 +52,114 @@ impl Trakt { pub fn search( &self, + cache: &Cache, kinds: &[TraktKind], query: &str, rt: &Handle, ) -> Result<Arc<Vec<TraktSearchResult>>> { - cache_memory( - &format!("ext/trakt/search/{}.json", HashKey(query)), - move || { + cache + .cache_memory( + &format!("ext/trakt/search/{}.json", HashKey(query)), + move || { + rt.block_on(async { + let url = format!( + "https://api.trakt.tv/search/{}?query={}&extended=full", + kinds + .iter() + .map(|t| t.singular()) + .collect::<Vec<_>>() + .join(","), + urlencoding::encode(query), + ); + let res = self.client.get(url).send().await?.error_for_status()?; + Ok(res.json().await?) + }) + }, + ) + .context("trakt search") + } + + pub fn lookup( + &self, + cache: &Cache, + kind: TraktKind, + id: u64, + rt: &Handle, + ) -> Result<Arc<TraktMediaObject>> { + cache + .cache_memory(&format!("ext/trakt/lookup/{kind}-{id}.json"), move || { rt.block_on(async { - let url = format!( - "https://api.trakt.tv/search/{}?query={}&extended=full", - kinds - .iter() - .map(|t| t.singular()) - .collect::<Vec<_>>() - .join(","), - urlencoding::encode(query), - ); + info!("trakt lookup {kind:?}:{id:?}"); + let url = format!("https://api.trakt.tv/{}/{id}?extended=full", kind.plural()); let res = self.client.get(url).send().await?.error_for_status()?; Ok(res.json().await?) }) - }, - ) - .context("trakt search") - } - - pub fn lookup(&self, kind: TraktKind, id: u64, rt: &Handle) -> Result<Arc<TraktMediaObject>> { - cache_memory(&format!("ext/trakt/lookup/{kind}-{id}.json"), move || { - rt.block_on(async { - info!("trakt lookup {kind:?}:{id:?}"); - let url = format!("https://api.trakt.tv/{}/{id}?extended=full", kind.plural()); - let res = self.client.get(url).send().await?.error_for_status()?; - Ok(res.json().await?) }) - }) - .context("trakt lookup") + .context("trakt lookup") } - pub fn people(&self, kind: TraktKind, id: u64, rt: &Handle) -> Result<Arc<TraktPeople>> { - cache_memory(&format!("ext/trakt/people/{kind}-{id}.json"), move || { - rt.block_on(async { - info!("trakt people {kind:?}:{id:?}"); - let url = format!( - "https://api.trakt.tv/{}/{id}/people?extended=full", - kind.plural() - ); - let res = self.client.get(url).send().await?.error_for_status()?; - Ok(res.json().await?) + pub fn people( + &self, + cache: &Cache, + kind: TraktKind, + id: u64, + rt: &Handle, + ) -> Result<Arc<TraktPeople>> { + cache + .cache_memory(&format!("ext/trakt/people/{kind}-{id}.json"), move || { + rt.block_on(async { + info!("trakt people {kind:?}:{id:?}"); + let url = format!( + "https://api.trakt.tv/{}/{id}/people?extended=full", + kind.plural() + ); + let res = self.client.get(url).send().await?.error_for_status()?; + Ok(res.json().await?) + }) }) - }) - .context("trakt people") + .context("trakt people") } - pub fn show_seasons(&self, id: u64, rt: &Handle) -> Result<Arc<Vec<TraktSeason>>> { - cache_memory(&format!("ext/trakt/seasons/{id}.json"), move || { - rt.block_on(async { - info!("trakt seasons {id:?}"); - let url = format!("https://api.trakt.tv/shows/{id}/seasons?extended=full"); - let res = self.client.get(url).send().await?.error_for_status()?; - Ok(res.json().await?) + pub fn show_seasons( + &self, + cache: &Cache, + id: u64, + rt: &Handle, + ) -> Result<Arc<Vec<TraktSeason>>> { + cache + .cache_memory(&format!("ext/trakt/seasons/{id}.json"), move || { + rt.block_on(async { + info!("trakt seasons {id:?}"); + let url = format!("https://api.trakt.tv/shows/{id}/seasons?extended=full"); + let res = self.client.get(url).send().await?.error_for_status()?; + Ok(res.json().await?) + }) }) - }) - .context("trakt show seasons") + .context("trakt show seasons") } pub fn show_season_episodes( &self, + cache: &Cache, id: u64, season: u64, rt: &Handle, ) -> Result<Arc<Vec<TraktEpisode>>> { - cache_memory( - &format!("ext/trakt/episodes/{id}-S{season}.json"), - move || { - rt.block_on(async { - info!("trakt episodes {id:?} season={season}"); - let url = - format!("https://api.trakt.tv/shows/{id}/seasons/{season}?extended=full"); - let res = self.client.get(url).send().await?.error_for_status()?; - Ok(res.json().await?) - }) - }, - ) - .context("trakt show season episodes") + cache + .cache_memory( + &format!("ext/trakt/episodes/{id}-S{season}.json"), + move || { + rt.block_on(async { + info!("trakt episodes {id:?} season={season}"); + let url = format!( + "https://api.trakt.tv/shows/{id}/seasons/{season}?extended=full" + ); + let res = self.client.get(url).send().await?.error_for_status()?; + Ok(res.json().await?) + }) + }, + ) + .context("trakt show season episodes") } } @@ -425,7 +450,7 @@ impl Trakt { return Ok(()); }; - let details = self.lookup(trakt_kind, trakt_id, ct.rt)?; + let details = self.lookup(&ct.dba.cache, trakt_kind, trakt_id, ct.rt)?; // let people = self.people(trakt_kind, trakt_id, ct.rt)?; // let mut people_map = BTreeMap::<CreditCategory, Vec<Appearance>>::new(); @@ -525,9 +550,9 @@ impl Trakt { return Ok(()); }; - let seasons = self.show_seasons(show_id, ct.rt)?; + let seasons = self.show_seasons(&ct.dba.cache, show_id, ct.rt)?; if seasons.iter().any(|x| x.number == season) { - let episodes = self.show_season_episodes(show_id, season, ct.rt)?; + let episodes = self.show_season_episodes(&ct.dba.cache, show_id, season, ct.rt)?; if let Some(episode) = episodes.get(episode.saturating_sub(1) as usize) { ct.dba.update_node(node, |mut node| { node = node.as_object().insert(NO_KIND, KIND_EPISODE); diff --git a/import/src/plugins/vgmdb.rs b/import/src/plugins/vgmdb.rs index c62eb90..534b241 100644 --- a/import/src/plugins/vgmdb.rs +++ b/import/src/plugins/vgmdb.rs @@ -9,7 +9,7 @@ use crate::{ plugins::{ImportPlugin, PluginInfo}, }; use anyhow::{Context, Result}; -use jellycache::{HashKey, cache, cache_store}; +use jellycache::{Cache, HashKey}; use log::info; use regex::Regex; use reqwest::{ @@ -62,34 +62,40 @@ impl Vgmdb { } } - pub fn get_artist_image(&self, id: u64, rt: &Handle) -> Result<Option<String>> { - if let Some(url) = self.get_artist_image_url(id, rt)? { - cache_store( - format!("ext/vgmdb/artist-image/{}.image", HashKey(&url)), - move || { - rt.block_on(async { - info!("downloading image {url:?}"); - Ok(self - .client - .get(url) - .send() - .await? - .error_for_status()? - .bytes() - .await? - .to_vec()) - }) - }, - ) - .context("vgmdb media download") - .map(Some) + pub fn get_artist_image(&self, cache: &Cache, id: u64, rt: &Handle) -> Result<Option<String>> { + if let Some(url) = self.get_artist_image_url(cache, id, rt)? { + cache + .store( + format!("ext/vgmdb/artist-image/{}.image", HashKey(&url)), + move || { + rt.block_on(async { + info!("downloading image {url:?}"); + Ok(self + .client + .get(url) + .send() + .await? + .error_for_status()? + .bytes() + .await? + .to_vec()) + }) + }, + ) + .context("vgmdb media download") + .map(Some) } else { Ok(None) } } - pub fn get_artist_image_url(&self, id: u64, rt: &Handle) -> Result<Option<String>> { - let html = self.scrape_artist_page(id, rt)?; + pub fn get_artist_image_url( + &self, + cache: &Cache, + id: u64, + rt: &Handle, + ) -> Result<Option<String>> { + let html = self.scrape_artist_page(cache, id, rt)?; if let Some(cap) = RE_IMAGE_URL_FROM_HTML.captures(&str::from_utf8(&html).unwrap()) { if let Some(url) = cap.name("url").map(|m| m.as_str()) { return Ok(Some(url.to_string())); @@ -98,32 +104,33 @@ impl Vgmdb { Ok(None) } - pub fn scrape_artist_page(&self, id: u64, rt: &Handle) -> Result<Vec<u8>> { - cache(&format!("ext/vgmdb/artist-page/{id}.html"), move || { - rt.block_on(async { - let _permit = self.rate_limit.clone().acquire_owned().await?; - let permit_drop_ts = Instant::now() + Duration::from_secs(1); - info!("scrape artist: {id}"); + pub fn scrape_artist_page(&self, cache: &Cache, id: u64, rt: &Handle) -> Result<Vec<u8>> { + cache + .cache(&format!("ext/vgmdb/artist-page/{id}.html"), move || { + rt.block_on(async { + let _permit = self.rate_limit.clone().acquire_owned().await?; + let permit_drop_ts = Instant::now() + Duration::from_secs(1); + info!("scrape artist: {id}"); - let resp = self - .client - .get(format!("https://vgmdb.net/artist/{id}")) - .send() - .await? - .error_for_status()? - .bytes() - .await? - .to_vec(); + let resp = self + .client + .get(format!("https://vgmdb.net/artist/{id}")) + .send() + .await? + .error_for_status()? + .bytes() + .await? + .to_vec(); - tokio::task::spawn(async move { - sleep_until(permit_drop_ts).await; - drop(_permit); - }); + tokio::task::spawn(async move { + sleep_until(permit_drop_ts).await; + drop(_permit); + }); - Ok(resp) + Ok(resp) + }) }) - }) - .context("vgmdb artist page scrape") + .context("vgmdb artist page scrape") } } diff --git a/import/src/plugins/wikidata.rs b/import/src/plugins/wikidata.rs index e3077e0..3afd393 100644 --- a/import/src/plugins/wikidata.rs +++ b/import/src/plugins/wikidata.rs @@ -9,7 +9,7 @@ use crate::{ plugins::{ImportPlugin, PluginInfo}, }; use anyhow::{Context, Result, bail}; -use jellycache::{EscapeKey, cache_memory}; +use jellycache::{Cache, EscapeKey}; use log::info; use reqwest::{ Client, ClientBuilder, @@ -91,8 +91,13 @@ impl Wikidata { Self { client } } - pub fn query_image_path(&self, id: String, rt: &Handle) -> Result<Option<String>> { - let response = self.query(id.clone(), rt)?; + pub fn query_image_path( + &self, + cache: &Cache, + id: String, + rt: &Handle, + ) -> Result<Option<String>> { + let response = self.query(cache, id.clone(), rt)?; if let Some(entity) = response.entities.get(&id) { if let Some(images) = entity.claims.get(properties::IMAGE) { for image in images { @@ -110,24 +115,25 @@ impl Wikidata { Ok(None) } - pub fn query(&self, id: String, rt: &Handle) -> Result<Arc<WikidataResponse>> { - cache_memory( - &format!("ext/wikidata/{}.json", EscapeKey(&id)), - move || { - rt.block_on(async { - info!("entity query: {id}"); - Ok(self - .client - .get(format!("https://www.wikidata.org/entity/{id}")) - .send() - .await? - .error_for_status()? - .json() - .await?) - }) - }, - ) - .context("wikidata entity") + pub fn query(&self, cache: &Cache, id: String, rt: &Handle) -> Result<Arc<WikidataResponse>> { + cache + .cache_memory( + &format!("ext/wikidata/{}.json", EscapeKey(&id)), + move || { + rt.block_on(async { + info!("entity query: {id}"); + Ok(self + .client + .get(format!("https://www.wikidata.org/entity/{id}")) + .send() + .await? + .error_for_status()? + .json() + .await?) + }) + }, + ) + .context("wikidata entity") } } diff --git a/import/src/plugins/wikimedia_commons.rs b/import/src/plugins/wikimedia_commons.rs index c849b61..aebf5dd 100644 --- a/import/src/plugins/wikimedia_commons.rs +++ b/import/src/plugins/wikimedia_commons.rs @@ -9,7 +9,7 @@ use crate::{ plugins::{ImportPlugin, PluginInfo}, }; use anyhow::{Context, Result}; -use jellycache::{EscapeKey, cache_store}; +use jellycache::{Cache, EscapeKey}; use reqwest::{ Client, ClientBuilder, header::{HeaderMap, HeaderName, HeaderValue}, @@ -39,27 +39,33 @@ impl WikimediaCommons { Self { client } } - pub fn image_by_filename(&self, filename: String, rt: &Handle) -> Result<String> { - cache_store( - format!("ext/wikimedia-commons/image/{}.image", EscapeKey(&filename)), - move || { - rt.block_on(async { - Ok(self - .client - .get(format!( - "https://commons.wikimedia.org/wiki/Special:FilePath/{}", - filename.replace(" ", "_") - )) - .send() - .await? - .error_for_status()? - .bytes() - .await? - .to_vec()) - }) - }, - ) - .context("mediawiki image by filename") + pub fn image_by_filename( + &self, + cache: &Cache, + filename: String, + rt: &Handle, + ) -> Result<String> { + cache + .store( + format!("ext/wikimedia-commons/image/{}.image", EscapeKey(&filename)), + move || { + rt.block_on(async { + Ok(self + .client + .get(format!( + "https://commons.wikimedia.org/wiki/Special:FilePath/{}", + filename.replace(" ", "_") + )) + .send() + .await? + .error_for_status()? + .bytes() + .await? + .to_vec()) + }) + }, + ) + .context("mediawiki image by filename") } } diff --git a/kv/Cargo.toml b/kv/Cargo.toml index ab24e25..91422c7 100644 --- a/kv/Cargo.toml +++ b/kv/Cargo.toml @@ -5,6 +5,15 @@ edition = "2024" [dependencies] anyhow = "1.0.100" -rand = "0.10.0-rc.7" -rocksdb = { version = "0.24.0", features = ["multi-threaded-cf"] } -redb = "3.1.0" +rand = { version = "0.10.0-rc.7", optional = true } +rocksdb = { version = "0.24.0", features = [ + "multi-threaded-cf", +], optional = true } +redb = { version = "3.1.0", optional = true } + +[features] +# default = ["rocksdb", "redb", "memory", "filesystem"] +rocksdb = ["dep:rocksdb"] +redb = ["dep:redb"] +memory = [] +filesystem = ["dep:rand"] diff --git a/kv/src/lib.rs b/kv/src/lib.rs index 0e9b78e..939391c 100644 --- a/kv/src/lib.rs +++ b/kv/src/lib.rs @@ -5,9 +5,13 @@ */ pub mod dummy; +#[cfg(feature = "filesystem")] pub mod filesystem; +#[cfg(feature = "memory")] pub mod memory; +#[cfg(feature = "redb")] pub mod redb; +#[cfg(feature = "rocksdb")] pub mod rocksdb; pub use anyhow; diff --git a/stream/src/cues.rs b/stream/src/cues.rs index fbcb341..2fd4d70 100644 --- a/stream/src/cues.rs +++ b/stream/src/cues.rs @@ -4,8 +4,8 @@ Copyright (C) 2026 metamuffin <metamuffin.org> */ -use anyhow::{anyhow, Result}; -use jellycache::{cache_memory, HashKey}; +use anyhow::{Result, anyhow}; +use jellycache::{Cache, HashKey}; use jellyremuxer::demuxers::create_demuxer_autodetect; use serde::{Deserialize, Serialize}; use std::{collections::BTreeMap, fs::File, path::Path, sync::Arc}; @@ -28,8 +28,8 @@ pub struct StatsAndCues { pub cues: Vec<GeneratedCue>, } -pub fn generate_cues(path: &Path) -> Result<Arc<StatsAndCues>> { - cache_memory( +pub fn generate_cues(cache: &Cache, path: &Path) -> Result<Arc<StatsAndCues>> { + cache.cache_memory( &format!("media/generated-cues/{}.json", HashKey(path)), move || { let media = File::open(path)?; diff --git a/stream/src/fragment.rs b/stream/src/fragment.rs index 3f6ed98..6902d77 100644 --- a/stream/src/fragment.rs +++ b/stream/src/fragment.rs @@ -4,16 +4,17 @@ Copyright (C) 2026 metamuffin <metamuffin.org> */ use crate::{ - cues::{generate_cues, GeneratedCue}, - stream_info, SMediaInfo, + SMediaInfo, + cues::{GeneratedCue, generate_cues}, + stream_info, }; -use anyhow::{anyhow, Result}; +use anyhow::{Result, anyhow}; use jellycache::HashKey; use jellyremuxer::{ + ContainerFormat, demuxers::create_demuxer_autodetect, matroska::{self, Segment}, muxers::write_fragment, - ContainerFormat, }; use jellystream_types::{FormatNum, IndexNum, StreamContainer, TrackNum}; use jellytranscoder::fragment::transcode; @@ -24,13 +25,13 @@ use std::{ }; pub fn fragment_stream( - info: Arc<SMediaInfo>, + sinfo: Arc<SMediaInfo>, track: TrackNum, index: IndexNum, format_num: FormatNum, container: StreamContainer, ) -> Result<Box<dyn Read + Send + Sync>> { - let (iinfo, info) = stream_info(info)?; + let (iinfo, info) = stream_info(&sinfo)?; let (file_index, track_num) = *iinfo .track_to_file @@ -54,7 +55,7 @@ pub fn fragment_stream( let timestamp_scale = iinfo.metadata[file_index].info.timestamp_scale; let total_duration = iinfo.metadata[file_index].info.duration; - let cue_stat = generate_cues(&media_path)?; + let cue_stat = generate_cues(&sinfo.cache, &media_path)?; let start_cue = cue_stat .cues .get(index) @@ -121,6 +122,7 @@ pub fn fragment_stream( if !format.remux { segment = transcode( + &sinfo.cache, track.kind, &format!("{}-T{track_num}-I{index}", HashKey(media_path)), format, diff --git a/stream/src/fragment_index.rs b/stream/src/fragment_index.rs index 58e5bd7..e2adf41 100644 --- a/stream/src/fragment_index.rs +++ b/stream/src/fragment_index.rs @@ -3,8 +3,8 @@ which is licensed under the GNU Affero General Public License (version 3); see /COPYING. Copyright (C) 2026 metamuffin <metamuffin.org> */ -use crate::{cues::generate_cues, stream_info, SMediaInfo}; -use anyhow::{anyhow, Result}; +use crate::{SMediaInfo, cues::generate_cues, stream_info}; +use anyhow::{Result, anyhow}; use jellystream_types::TrackNum; use std::{ io::{Cursor, Read}, @@ -12,14 +12,14 @@ use std::{ sync::Arc, }; -pub fn fragment_index(info: Arc<SMediaInfo>, track: TrackNum) -> Result<Vec<Range<f64>>> { - let (iinfo, info) = stream_info(info)?; +pub fn fragment_index(sinfo: &SMediaInfo, track: TrackNum) -> Result<Vec<Range<f64>>> { + let (iinfo, info) = stream_info(&sinfo)?; let (file_index, _) = *iinfo .track_to_file .get(track) .ok_or(anyhow!("track not found"))?; - let cue_stat = generate_cues(&iinfo.paths[file_index])?; + let cue_stat = generate_cues(&sinfo.cache, &iinfo.paths[file_index])?; Ok(cue_stat .cues @@ -42,6 +42,6 @@ pub fn fragment_index_stream( track: TrackNum, ) -> Result<Box<dyn Read + Send + Sync>> { Ok(Box::new(Cursor::new(serde_json::to_string( - &fragment_index(info, track)?, + &fragment_index(&info, track)?, )?))) } diff --git a/stream/src/hls.rs b/stream/src/hls.rs index 20f452c..5557e48 100644 --- a/stream/src/hls.rs +++ b/stream/src/hls.rs @@ -4,7 +4,7 @@ Copyright (C) 2026 metamuffin <metamuffin.org> */ -use crate::{fragment_index::fragment_index, stream_info, SMediaInfo}; +use crate::{SMediaInfo, fragment_index::fragment_index, stream_info}; use anyhow::Result; use jellystream_types::{FormatNum, StreamContainer, StreamSpec, TrackKind, TrackNum}; use std::{ @@ -18,7 +18,7 @@ pub fn hls_multivariant_stream( info: Arc<SMediaInfo>, container: StreamContainer, ) -> Result<Box<dyn Read + Send + Sync>> { - let (_iinfo, info) = stream_info(info)?; + let (_iinfo, info) = stream_info(&info)?; let mut out = String::new(); writeln!(out, "#EXTM3U")?; @@ -53,8 +53,8 @@ pub fn hls_variant_stream( format: FormatNum, container: StreamContainer, ) -> Result<Box<dyn Read + Send + Sync>> { - let frags = fragment_index(info.clone(), track)?; - let (_, info) = stream_info(info)?; + let frags = fragment_index(&info, track)?; + let (_, info) = stream_info(&info)?; let mut out = String::new(); writeln!(out, "#EXTM3U")?; diff --git a/stream/src/lib.rs b/stream/src/lib.rs index 717225e..20a5e1c 100644 --- a/stream/src/lib.rs +++ b/stream/src/lib.rs @@ -12,10 +12,11 @@ pub mod metadata; mod stream_info; mod webvtt; -use anyhow::{anyhow, bail, Context, Result}; +use anyhow::{Context, Result, anyhow, bail}; use fragment::fragment_stream; use fragment_index::fragment_index_stream; use hls::{hls_multivariant_stream, hls_variant_stream}; +use jellycache::Cache; use jellystream_types::{StreamContainer, StreamSpec}; use serde::{Deserialize, Serialize}; use std::{ @@ -47,10 +48,10 @@ static CONF: LazyLock<Config> = LazyLock::new(|| { .expect("stream config not preloaded. logic error") }); -#[derive(Debug)] pub struct SMediaInfo { pub title: Option<String>, pub files: BTreeSet<PathBuf>, + pub cache: Arc<Cache>, } pub struct StreamHead { @@ -114,7 +115,7 @@ fn original_stream( track: usize, range: Range<u64>, ) -> Result<Box<dyn Read + Send + Sync>> { - let (iinfo, _info) = stream_info(info)?; + let (iinfo, _info) = stream_info(&info)?; let (file_index, _) = *iinfo .track_to_file .get(track) diff --git a/stream/src/metadata.rs b/stream/src/metadata.rs index 640b851..4dedb76 100644 --- a/stream/src/metadata.rs +++ b/stream/src/metadata.rs @@ -4,13 +4,13 @@ Copyright (C) 2026 metamuffin <metamuffin.org> */ -use anyhow::{anyhow, Result}; -use jellycache::{cache_memory, HashKey}; +use anyhow::{Result, anyhow}; +use jellycache::{Cache, HashKey}; use jellyremuxer::{demuxers::create_demuxer_autodetect, matroska::Segment}; use std::{fs::File, path::Path, sync::Arc}; -pub fn read_metadata(path: &Path) -> Result<Arc<Segment>> { - cache_memory( +pub fn read_metadata(cache: &Cache, path: &Path) -> Result<Arc<Segment>> { + cache.cache_memory( &format!("media/stream-metadata/{}.json", HashKey(path)), move || { let media = File::open(path)?; diff --git a/stream/src/stream_info.rs b/stream/src/stream_info.rs index 216c968..4a2605e 100644 --- a/stream/src/stream_info.rs +++ b/stream/src/stream_info.rs @@ -3,7 +3,7 @@ which is licensed under the GNU Affero General Public License (version 3); see /COPYING. Copyright (C) 2026 metamuffin <metamuffin.org> */ -use crate::{cues::generate_cues, metadata::read_metadata, SMediaInfo, CONF}; +use crate::{CONF, SMediaInfo, cues::generate_cues, metadata::read_metadata}; use anyhow::Result; use jellyremuxer::matroska::{self, Segment, TrackEntry, TrackType}; use jellystream_types::{ @@ -22,14 +22,14 @@ pub(crate) struct InternalStreamInfo { } // TODO cache mem -pub(crate) fn stream_info(info: Arc<SMediaInfo>) -> Result<(InternalStreamInfo, StreamInfo)> { +pub(crate) fn stream_info(info: &SMediaInfo) -> Result<(InternalStreamInfo, StreamInfo)> { let mut tracks = Vec::new(); let mut track_to_file = Vec::new(); let mut metadata_arr = Vec::new(); let mut paths = Vec::new(); for (i, path) in info.files.iter().enumerate() { - let metadata = read_metadata(path)?; - let cue_stat = generate_cues(path)?; + let metadata = read_metadata(&info.cache, path)?; + let cue_stat = generate_cues(&info.cache, path)?; if let Some(t) = &metadata.tracks { let duration = media_duration(&metadata.info); for t in &t.entries { @@ -158,7 +158,7 @@ fn containers_by_codec(codec: &str) -> Vec<StreamContainer> { } pub(crate) fn write_stream_info(info: Arc<SMediaInfo>) -> Result<Box<dyn Read + Send + Sync>> { - let (_, info) = stream_info(info)?; + let (_, info) = stream_info(&info)?; Ok(Box::new(Cursor::new(serde_json::to_vec(&info)?))) } diff --git a/transcoder/src/fragment.rs b/transcoder/src/fragment.rs index 5aca1f7..6603afa 100644 --- a/transcoder/src/fragment.rs +++ b/transcoder/src/fragment.rs @@ -3,10 +3,10 @@ which is licensed under the GNU Affero General Public License (version 3); see /COPYING. Copyright (C) 2026 metamuffin <metamuffin.org> */ -use crate::{Config, CONF, LOCAL_VIDEO_TRANSCODING_TASKS}; +use crate::{CONF, Config, LOCAL_VIDEO_TRANSCODING_TASKS}; use anyhow::Result; -use jellycache::{cache, HashKey}; -use jellyremuxer::{demuxers::create_demuxer, muxers::write_fragment, ContainerFormat}; +use jellycache::{Cache, HashKey}; +use jellyremuxer::{ContainerFormat, demuxers::create_demuxer, muxers::write_fragment}; use jellystream_types::{StreamFormatInfo, TrackKind}; use log::info; use std::{ @@ -15,12 +15,13 @@ use std::{ process::{Command, Stdio}, thread::spawn, }; -use winter_matroska::{block::Block, Cluster, Segment, TrackEntry as MatroskaTrackEntry}; +use winter_matroska::{Cluster, Segment, TrackEntry as MatroskaTrackEntry, block::Block}; // TODO odd video resolutions can cause errors when transcoding to YUV42{0,2} // TODO with an implementation that cant handle it (SVT-AV1 is such an impl). pub fn transcode( + cache: &Cache, kind: TrackKind, input_key: &str, output_format: &StreamFormatInfo, @@ -38,7 +39,7 @@ pub fn transcode( let input_duration = input.info.duration; let had_next_kf = next_kf.is_some(); - let output = cache( + let output = cache.cache( &format!( "transcode/media-fragment/{input_key}-{}.mkv", HashKey(&command) diff --git a/transcoder/src/image.rs b/transcoder/src/image.rs index 14b3e53..eaf8b86 100644 --- a/transcoder/src/image.rs +++ b/transcoder/src/image.rs @@ -3,21 +3,29 @@ which is licensed under the GNU Affero General Public License (version 3); see /COPYING. Copyright (C) 2026 metamuffin <metamuffin.org> */ -use anyhow::{anyhow, Context, Result}; +use anyhow::{Context, Result, anyhow}; use image::imageops::FilterType; -use jellycache::{cache, cache_read, HashKey}; +use jellycache::{Cache, HashKey}; use log::{debug, info}; use rgb::FromSlice; use std::io::Cursor; -pub fn transcode(key: &str, quality: u32, speed: u8, width: usize) -> Result<Vec<u8>> { - cache( +pub fn transcode( + cache: &Cache, + key: &str, + quality: u32, + speed: u8, + width: usize, +) -> Result<Vec<u8>> { + cache.cache( &format!( "transcode/image/{}-W{width}-Q{quality}-S{speed}", HashKey(key) ), move || { - let input = cache_read(key)?.ok_or(anyhow!("transcode cache key missing"))?; + let input = cache + .read(key)? + .ok_or(anyhow!("transcode cache key missing"))?; info!("encoding image (speed={speed}, quality={quality}, width={width})"); // TODO: use better image library that supports AVIF diff --git a/transcoder/src/thumbnail.rs b/transcoder/src/thumbnail.rs index 85cb356..9207c30 100644 --- a/transcoder/src/thumbnail.rs +++ b/transcoder/src/thumbnail.rs @@ -1,5 +1,5 @@ use anyhow::{Context, Result}; -use jellycache::{cache_store, HashKey}; +use jellycache::{Cache, HashKey}; use log::info; use std::{ io::Read, @@ -7,31 +7,32 @@ use std::{ process::{Command, Stdio}, }; -pub fn create_thumbnail(path: &Path, time: f64) -> Result<String> { - cache_store( - format!("media/thumbnail/{}-{}.image", HashKey(path), time as i64), - move || { - info!("creating thumbnail of {path:?} at {time}s",); +pub fn create_thumbnail(cache: &Cache, path: &Path, time: f64) -> Result<String> { + cache + .store( + format!("media/thumbnail/{}-{}.image", HashKey(path), time as i64), + move || { + info!("creating thumbnail of {path:?} at {time}s",); - let mut proc = Command::new("ffmpeg") - .stdout(Stdio::piped()) - .args(["-ss", &format!("{time}")]) - .args(["-f", "matroska", "-i", path.to_str().unwrap()]) - .args(["-frames:v", "1"]) - .args(["-c:v", "qoi"]) - .args(["-f", "image2"]) - .args(["-update", "1"]) - .arg("pipe:1") - .spawn()?; + let mut proc = Command::new("ffmpeg") + .stdout(Stdio::piped()) + .args(["-ss", &format!("{time}")]) + .args(["-f", "matroska", "-i", path.to_str().unwrap()]) + .args(["-frames:v", "1"]) + .args(["-c:v", "qoi"]) + .args(["-f", "image2"]) + .args(["-update", "1"]) + .arg("pipe:1") + .spawn()?; - let mut stdout = proc.stdout.take().unwrap(); - let mut output = Vec::new(); - stdout.read_to_end(&mut output)?; + let mut stdout = proc.stdout.take().unwrap(); + let mut output = Vec::new(); + stdout.read_to_end(&mut output)?; - proc.wait().unwrap().exit_ok()?; - info!("done"); - Ok(output) - }, - ) - .context("creating thumbnail") + proc.wait().unwrap().exit_ok()?; + info!("done"); + Ok(output) + }, + ) + .context("creating thumbnail") } |