diff options
| author | metamuffin <metamuffin@disroot.org> | 2025-12-08 19:53:12 +0100 |
|---|---|---|
| committer | metamuffin <metamuffin@disroot.org> | 2025-12-08 19:53:12 +0100 |
| commit | 6edf0fd93abf7e58b4c0974e3d3e54bcf8517946 (patch) | |
| tree | 32577db9d987897d4037ba9af0084b95b55e145c | |
| parent | e4584a8135584e6591bac7d5397cf227cf3cff92 (diff) | |
| download | jellything-6edf0fd93abf7e58b4c0974e3d3e54bcf8517946.tar jellything-6edf0fd93abf7e58b4c0974e3d3e54bcf8517946.tar.bz2 jellything-6edf0fd93abf7e58b4c0974e3d3e54bcf8517946.tar.zst | |
human-readable cache keys
| -rw-r--r-- | Cargo.lock | 1 | ||||
| -rw-r--r-- | cache/Cargo.toml | 1 | ||||
| -rw-r--r-- | cache/src/backends/filesystem.rs | 20 | ||||
| -rw-r--r-- | cache/src/backends/mod.rs | 5 | ||||
| -rw-r--r-- | cache/src/helper.rs | 46 | ||||
| -rw-r--r-- | cache/src/key.rs | 83 | ||||
| -rw-r--r-- | cache/src/lib.rs | 47 | ||||
| -rw-r--r-- | common/src/helpers.rs | 23 | ||||
| -rw-r--r-- | common/src/lib.rs | 7 | ||||
| -rw-r--r-- | import/src/acoustid.rs | 6 | ||||
| -rw-r--r-- | import/src/lib.rs | 161 | ||||
| -rw-r--r-- | import/src/musicbrainz.rs | 169 | ||||
| -rw-r--r-- | import/src/tmdb.rs | 20 | ||||
| -rw-r--r-- | import/src/trakt.rs | 65 | ||||
| -rw-r--r-- | import/src/vgmdb.rs | 92 | ||||
| -rw-r--r-- | import/src/wikidata.rs | 31 | ||||
| -rw-r--r-- | import/src/wikimedia_commons.rs | 8 | ||||
| -rw-r--r-- | logic/src/assets.rs | 14 | ||||
| -rw-r--r-- | server/src/helper/asset.rs | 37 | ||||
| -rw-r--r-- | server/src/helper/mod.rs | 2 | ||||
| -rw-r--r-- | server/src/helper/picture.rs | 32 | ||||
| -rw-r--r-- | server/src/ui/assets.rs | 17 | ||||
| -rw-r--r-- | stream/src/cues.rs | 4 | ||||
| -rw-r--r-- | stream/src/fragment.rs | 3 | ||||
| -rw-r--r-- | stream/src/metadata.rs | 29 | ||||
| -rw-r--r-- | stream/types/src/lib.rs | 10 | ||||
| -rw-r--r-- | transcoder/src/fragment.rs | 7 | ||||
| -rw-r--r-- | transcoder/src/image.rs | 11 | ||||
| -rw-r--r-- | transcoder/src/thumbnail.rs | 6 |
29 files changed, 478 insertions, 479 deletions
@@ -1869,6 +1869,7 @@ dependencies = [ "base64", "humansize", "log", + "percent-encoding", "rand 0.9.2", "serde", "serde_json", diff --git a/cache/Cargo.toml b/cache/Cargo.toml index 412b1f2..d7fcd81 100644 --- a/cache/Cargo.toml +++ b/cache/Cargo.toml @@ -12,3 +12,4 @@ sha2 = "0.10.9" rand = "0.9.2" serde = "1.0.228" serde_json = "1.0.145" +percent-encoding = "2.3.2" diff --git a/cache/src/backends/filesystem.rs b/cache/src/backends/filesystem.rs index 39fb7a2..9a9db9c 100644 --- a/cache/src/backends/filesystem.rs +++ b/cache/src/backends/filesystem.rs @@ -4,12 +4,11 @@ Copyright (C) 2025 metamuffin <metamuffin.org> */ -use crate::{CacheKey, Config, backends::CacheStorage}; +use crate::{Config, backends::CacheStorage}; use anyhow::Result; -use base64::Engine; use rand::random; use std::{ - fs::{File, rename}, + fs::{File, create_dir_all, rename}, io::{ErrorKind, Read, Write}, path::PathBuf, }; @@ -20,25 +19,22 @@ impl Filesystem { pub fn new(config: &Config) -> Self { Self(config.path.clone()) } - fn path(&self, key: CacheKey) -> PathBuf { - let filename = base64::engine::general_purpose::URL_SAFE.encode(key.0); - let filename = &filename[..30]; // 180 bits - self.0.join(filename) - } fn temp_path(&self) -> PathBuf { self.0.join(format!("temp-{:016x}", random::<u128>())) } } impl CacheStorage for Filesystem { - fn store(&self, key: CacheKey, value: &[u8]) -> Result<()> { + fn store(&self, key: String, value: &[u8]) -> Result<()> { let temp = self.temp_path(); + let out = self.0.join(&key); + create_dir_all(out.parent().unwrap())?; File::create(&temp)?.write_all(value)?; - rename(temp, self.path(key))?; + rename(temp, out)?; Ok(()) } - fn read(&self, key: CacheKey) -> Result<Option<Vec<u8>>> { - match File::open(self.path(key)) { + fn read(&self, key: &str) -> Result<Option<Vec<u8>>> { + match File::open(self.0.join(key)) { Ok(mut f) => { let mut data = Vec::new(); f.read_to_end(&mut data)?; diff --git a/cache/src/backends/mod.rs b/cache/src/backends/mod.rs index 370c5ab..6b7dac3 100644 --- a/cache/src/backends/mod.rs +++ b/cache/src/backends/mod.rs @@ -5,10 +5,9 @@ */ pub mod filesystem; -use crate::CacheKey; use anyhow::Result; pub(crate) trait CacheStorage: Send + Sync + 'static { - fn store(&self, key: CacheKey, value: &[u8]) -> Result<()>; - fn read(&self, key: CacheKey) -> Result<Option<Vec<u8>>>; + fn store(&self, key: String, value: &[u8]) -> Result<()>; + fn read(&self, key: &str) -> Result<Option<Vec<u8>>>; } diff --git a/cache/src/helper.rs b/cache/src/helper.rs new file mode 100644 index 0000000..8f73e1e --- /dev/null +++ b/cache/src/helper.rs @@ -0,0 +1,46 @@ +/* + This file is part of jellything (https://codeberg.org/metamuffin/jellything) + which is licensed under the GNU Affero General Public License (version 3); see /COPYING. + Copyright (C) 2025 metamuffin <metamuffin.org> +*/ + +use base64::{Engine, prelude::BASE64_URL_SAFE}; +use sha2::Sha256; +use std::{ + fmt::Display, + hash::{Hash, Hasher}, +}; + +pub struct HashKey<T>(pub T); +impl<T: Hash> Display for HashKey<T> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + use sha2::Digest; + struct ShaHasher(Sha256); + impl Hasher for ShaHasher { + fn finish(&self) -> u64 { + unreachable!() + } + fn write(&mut self, bytes: &[u8]) { + self.0.update(bytes); + } + } + let mut d = ShaHasher(sha2::Sha256::new()); + self.0.hash(&mut d); + let d = d.0.finalize(); + f.write_str(&BASE64_URL_SAFE.encode(d)) + } +} + +pub struct EscapeKey<T>(pub T); +impl<T: Display> Display for EscapeKey<T> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{}", + percent_encoding::utf8_percent_encode( + &self.0.to_string(), + percent_encoding::NON_ALPHANUMERIC, + ) + ) + } +} diff --git a/cache/src/key.rs b/cache/src/key.rs deleted file mode 100644 index d8ca510..0000000 --- a/cache/src/key.rs +++ /dev/null @@ -1,83 +0,0 @@ -/* - This file is part of jellything (https://codeberg.org/metamuffin/jellything) - which is licensed under the GNU Affero General Public License (version 3); see /COPYING. - Copyright (C) 2025 metamuffin <metamuffin.org> -*/ -use crate::{CACHE_GENERATION_BUCKET_COUNT, CONF}; -use anyhow::bail; -use base64::{Engine, prelude::BASE64_URL_SAFE}; -use sha2::Sha256; -use std::{ - fmt::Display, - hash::{Hash, Hasher}, - str::FromStr, -}; - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub struct CacheKey(pub [u8; 32]); - -impl CacheKey { - pub fn new(ty: CacheContentType, seed: impl Hash) -> Self { - use sha2::Digest; - struct ShaHasher(Sha256); - impl Hasher for ShaHasher { - fn finish(&self) -> u64 { - unreachable!() - } - fn write(&mut self, bytes: &[u8]) { - self.0.update(bytes); - } - } - let mut d = ShaHasher(sha2::Sha256::new()); - d.0.update(CONF.secret.as_bytes()); - seed.hash(&mut d); - let d = d.0.finalize(); - let mut key: [u8; 32] = d.as_slice().try_into().unwrap(); - key[0] = ty as u8; - Self(key) - } - pub fn new_json(seed: impl Hash) -> Self { - Self::new(CacheContentType::Json, seed) - } - pub fn new_image(seed: impl Hash) -> Self { - Self::new(CacheContentType::Image, seed) - } - pub fn content_type(&self) -> CacheContentType { - match self.0[0] { - 1 => CacheContentType::Image, - 2 => CacheContentType::Json, - _ => CacheContentType::Unknown, - } - } - pub(super) fn bucket(&self) -> usize { - (self.0[1] as usize - | ((self.0[2] as usize) << 8) - | ((self.0[3] as usize) << 16) - | ((self.0[4] as usize) << 24)) - % CACHE_GENERATION_BUCKET_COUNT - } -} - -impl Display for CacheKey { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str(&BASE64_URL_SAFE.encode(self.0)) - } -} -impl FromStr for CacheKey { - type Err = anyhow::Error; - fn from_str(s: &str) -> Result<Self, Self::Err> { - let mut out = [0; 32]; - let size = BASE64_URL_SAFE.decode_slice(s, &mut out)?; - if size != out.len() { - bail!("cache key parse invalid size") - } - Ok(Self(out)) - } -} - -#[repr(u8)] -pub enum CacheContentType { - Unknown = 0, - Image = 1, - Json = 2, -} diff --git a/cache/src/lib.rs b/cache/src/lib.rs index fbda2cf..20e1424 100644 --- a/cache/src/lib.rs +++ b/cache/src/lib.rs @@ -3,17 +3,17 @@ which is licensed under the GNU Affero General Public License (version 3); see /COPYING. Copyright (C) 2025 metamuffin <metamuffin.org> */ -pub mod backends; -pub mod key; +mod backends; +mod helper; use crate::backends::{CacheStorage, filesystem::Filesystem}; use anyhow::{Context, Result, anyhow}; -pub use key::*; -use log::info; +use log::{info, warn}; use serde::{Deserialize, Serialize}; use std::{ any::Any, collections::{BTreeMap, HashMap}, + hash::{DefaultHasher, Hash, Hasher}, path::PathBuf, sync::{ Arc, LazyLock, Mutex, OnceLock, RwLock, @@ -22,11 +22,12 @@ use std::{ time::Instant, }; +pub use helper::{EscapeKey, HashKey}; + #[derive(Debug, Deserialize)] pub struct Config { path: PathBuf, max_in_memory_cache_size: usize, - secret: String, } const CACHE_GENERATION_BUCKET_COUNT: usize = 1024; @@ -54,23 +55,29 @@ pub fn init_cache() -> Result<()> { Ok(()) } -pub fn cache(key: CacheKey, generate: impl FnOnce() -> Result<Vec<u8>>) -> Result<Vec<u8>> { +fn bucket(key: &str) -> usize { + let mut h = DefaultHasher::new(); + key.hash(&mut h); + h.finish() as usize % CACHE_GENERATION_BUCKET_COUNT +} + +pub fn cache(key: &str, generate: impl FnOnce() -> Result<Vec<u8>>) -> Result<Vec<u8>> { // we need a lock even if it exists since somebody might be still in the process of writing. let already_within = WITHIN_CACHE_FILE.with(|a| a.swap(true, Ordering::Relaxed)); let _guard = if already_within { // TODO stupid hack to avoid deadlock for nested calls; not locking is fine but might cause double-generating - CACHE_GENERATION_LOCKS[key.bucket()].try_lock().ok() + CACHE_GENERATION_LOCKS[bucket(key)].try_lock().ok() } else { - CACHE_GENERATION_LOCKS[key.bucket()].lock().ok() + CACHE_GENERATION_LOCKS[bucket(key)].lock().ok() }; let store = CACHE_STORE.get().unwrap(); - let out = match store.read(key)? { + let out = match store.read(&key)? { Some(x) => x, None => { let value = generate()?; - store.store(key, &value)?; + store.store(key.to_owned(), &value)?; value } }; @@ -82,11 +89,11 @@ pub fn cache(key: CacheKey, generate: impl FnOnce() -> Result<Vec<u8>>) -> Resul Ok(out) } -pub fn cache_read(key: CacheKey) -> Result<Option<Vec<u8>>> { +pub fn cache_read(key: &str) -> Result<Option<Vec<u8>>> { CACHE_STORE.get().unwrap().read(key) } -pub fn cache_store(key: CacheKey, generate: impl FnOnce() -> Result<Vec<u8>>) -> Result<CacheKey> { - cache(key, generate)?; +pub fn cache_store(key: String, generate: impl FnOnce() -> Result<Vec<u8>>) -> Result<String> { + cache(&key, generate)?; Ok(key) } @@ -95,18 +102,22 @@ pub struct InMemoryCacheEntry { last_access: Instant, object: Arc<dyn Any + Send + Sync + 'static>, } -pub static CACHE_IN_MEMORY_OBJECTS: LazyLock<RwLock<HashMap<CacheKey, InMemoryCacheEntry>>> = +pub static CACHE_IN_MEMORY_OBJECTS: LazyLock<RwLock<HashMap<String, InMemoryCacheEntry>>> = LazyLock::new(|| RwLock::new(HashMap::new())); pub static CACHE_IN_MEMORY_SIZE: AtomicUsize = AtomicUsize::new(0); -pub fn cache_memory<Fun, T>(key: CacheKey, mut generate: Fun) -> Result<Arc<T>, anyhow::Error> +pub fn cache_memory<Fun, T>(key: &str, mut generate: Fun) -> Result<Arc<T>, anyhow::Error> where Fun: FnMut() -> Result<T, anyhow::Error>, T: Serialize + for<'de> Deserialize<'de> + Send + Sync + 'static, { + if !key.ends_with(".json") { + warn!("cache_memory key not ending in .json: {key:?}") + } + { let mut g = CACHE_IN_MEMORY_OBJECTS.write().unwrap(); - if let Some(entry) = g.get_mut(&key) { + if let Some(entry) = g.get_mut(key) { entry.last_access = Instant::now(); let object = entry .object @@ -117,7 +128,7 @@ where } } - let data = cache(key, move || { + let data = cache(&key, move || { let object = generate()?; Ok(serde_json::to_vec(&object)?) })?; @@ -128,7 +139,7 @@ where { let mut g = CACHE_IN_MEMORY_OBJECTS.write().unwrap(); g.insert( - key, + key.to_owned(), InMemoryCacheEntry { size, last_access: Instant::now(), diff --git a/common/src/helpers.rs b/common/src/helpers.rs index 431bf8c..d643aa6 100644 --- a/common/src/helpers.rs +++ b/common/src/helpers.rs @@ -4,9 +4,7 @@ Copyright (C) 2025 metamuffin <metamuffin.org> */ -use base64::{Engine, prelude::BASE64_URL_SAFE}; - -use crate::{CreditCategory, IdentifierType, Picture, PictureSlot}; +use crate::{CreditCategory, IdentifierType, PictureSlot}; use std::{fmt::Display, ops::Deref, str::FromStr}; #[derive(PartialEq)] @@ -149,22 +147,3 @@ impl FromStr for PictureSlot { }) } } - -impl Display for Picture { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str(&BASE64_URL_SAFE.encode(self.0)) - } -} -impl FromStr for Picture { - type Err = &'static str; - fn from_str(s: &str) -> Result<Self, Self::Err> { - let mut out = [0; 32]; - let size = BASE64_URL_SAFE - .decode_slice(s, &mut out) - .map_err(|_| "invalid base64 picture key")?; - if size != out.len() { - return Err("picture key parse invalid size"); - } - Ok(Self(out)) - } -} diff --git a/common/src/lib.rs b/common/src/lib.rs index cf79cc5..670d9af 100644 --- a/common/src/lib.rs +++ b/common/src/lib.rs @@ -68,7 +68,7 @@ pub struct Node { pub federated: Option<String>, pub tags: BTreeSet<String>, pub ratings: BTreeMap<RatingType, f64>, - pub pictures: BTreeMap<PictureSlot, Picture>, + pub pictures: BTreeMap<PictureSlot, Asset>, pub credits: BTreeMap<CreditCategory, Vec<Appearance>>, pub identifiers: BTreeMap<IdentifierType, String>, pub visibility: Visibility, @@ -104,9 +104,8 @@ pub enum IdentifierType { Omdb, } -// TODO custom b64 ser -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone, Copy)] -pub struct Picture(pub [u8; 32]); +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone, Hash)] +pub struct Asset(pub String); #[derive(Debug, Clone, Deserialize, Serialize)] pub struct Appearance { diff --git a/import/src/acoustid.rs b/import/src/acoustid.rs index 809d964..01adb57 100644 --- a/import/src/acoustid.rs +++ b/import/src/acoustid.rs @@ -5,7 +5,7 @@ */ use crate::USER_AGENT; use anyhow::{Context, Result}; -use jellycache::{cache_memory, CacheKey}; +use jellycache::{cache_memory, HashKey}; use log::info; use reqwest::{ header::{HeaderMap, HeaderName, HeaderValue}, @@ -95,7 +95,7 @@ impl AcoustID { } pub fn lookup(&self, fp: Fingerprint, rt: &Handle) -> Result<Arc<AcoustIDLookupResponse>> { - cache_memory(CacheKey::new_json(("acoustid-lookup", &fp)) , move || rt.block_on(async { + cache_memory(&format!("ext/acoustid/{}.json", HashKey(&fp)) , move || rt.block_on(async { let _permit = self.rate_limit.clone().acquire_owned().await?; let permit_drop_ts = Instant::now() + Duration::SECOND; info!("acoustid lookup"); @@ -126,7 +126,7 @@ impl AcoustID { pub(crate) fn acoustid_fingerprint(path: &Path) -> Result<Arc<Fingerprint>> { cache_memory( - CacheKey::new_json(("acoustid-fingerprint", path)), + &format!("media/chromaprint/{}.json", HashKey(path)), move || { let child = Command::new("fpcalc") .arg("-json") diff --git a/import/src/lib.rs b/import/src/lib.rs index af13316..e31127e 100644 --- a/import/src/lib.rs +++ b/import/src/lib.rs @@ -20,10 +20,10 @@ use crate::{tmdb::TmdbKind, trakt::TraktKind}; use acoustid::{acoustid_fingerprint, AcoustID}; use anyhow::{anyhow, bail, Context, Result}; use infojson::YVideo; -use jellycache::{cache, cache_memory, cache_store, CacheKey}; +use jellycache::{cache_memory, cache_read, cache_store, HashKey}; use jellycommon::{ - Appearance, Chapter, CreditCategory, IdentifierType, MediaInfo, Node, NodeID, NodeKind, - Picture, PictureSlot, RatingType, SourceTrack, SourceTrackKind, TrackSource, Visibility, + Appearance, Asset, Chapter, CreditCategory, IdentifierType, MediaInfo, Node, NodeID, NodeKind, + PictureSlot, RatingType, SourceTrack, SourceTrackKind, TrackSource, Visibility, }; use jellyimport_fallback_generator::generate_fallback; use jellyremuxer::{ @@ -261,27 +261,31 @@ fn import_file( match filename.as_ref() { "poster.jpeg" | "poster.webp" | "poster.png" => { info!("import poster at {path:?}"); - let key = CacheKey::new_image(path); - cache(key, || { - let mut data = Vec::new(); - File::open(path)?.read_to_end(&mut data)?; - Ok(data) - })?; + let asset = Asset(cache_store( + format!("media/literal/{}-poster.image", HashKey(path)), + || { + let mut data = Vec::new(); + File::open(path)?.read_to_end(&mut data)?; + Ok(data) + }, + )?); db.update_node_init(parent, |node| { - node.pictures.insert(PictureSlot::Cover, Picture(key.0)); + node.pictures.insert(PictureSlot::Cover, asset); Ok(()) })?; } "backdrop.jpeg" | "backdrop.webp" | "backdrop.png" => { info!("import backdrop at {path:?}"); - let key = CacheKey::new_image(path); - cache(key, || { - let mut data = Vec::new(); - File::open(path)?.read_to_end(&mut data)?; - Ok(data) - })?; + let asset = Asset(cache_store( + format!("media/literal/{}-poster.image", HashKey(path)), + || { + let mut data = Vec::new(); + File::open(path)?.read_to_end(&mut data)?; + Ok(data) + }, + )?); db.update_node_init(parent, |node| { - node.pictures.insert(PictureSlot::Backdrop, Picture(key.0)); + node.pictures.insert(PictureSlot::Backdrop, asset); Ok(()) })?; } @@ -356,25 +360,63 @@ fn import_file( } pub fn read_media_metadata(path: &Path) -> Result<Arc<matroska::Segment>> { - cache_memory(CacheKey::new_json(path), move || { - let media = File::open(path)?; - let mut media = - create_demuxer_autodetect(Box::new(media))?.ok_or(anyhow!("media format unknown"))?; + cache_memory( + &format!("media/metadata/{}.json", HashKey(path)), + move || { + let media = File::open(path)?; + let mut media = create_demuxer_autodetect(Box::new(media))? + .ok_or(anyhow!("media format unknown"))?; - let info = media.info()?; - let tracks = media.tracks()?; - let tags = media.tags()?; - let attachments = media.attachments()?; - let chapters = media.chapters()?; - Ok(Segment { - info, - tracks, - tags: tags.into_iter().collect(), - attachments, - chapters, - ..Default::default() - }) - }) + let info = media.info()?; + let tracks = media.tracks()?; + let tags = media.tags()?; + let mut attachments = media.attachments()?; + let chapters = media.chapters()?; + + // Replace data of useful attachments with cache key; delete data of all others + if let Some(attachments) = &mut attachments { + for att in &mut attachments.files { + if let Some(fname) = attachment_types::is_useful_attachment(&att) { + let key = cache_store( + format!("media/attachment/{}-{fname}", HashKey(path)), + || Ok(att.data.clone()), + )?; + att.data = key.as_bytes().to_vec(); + } else { + att.data.clear(); + } + } + } + + Ok(Segment { + info, + tracks, + tags: tags.into_iter().collect(), + attachments, + chapters, + ..Default::default() + }) + }, + ) +} + +mod attachment_types { + use jellyremuxer::matroska::AttachedFile; + + pub fn is_useful_attachment(a: &AttachedFile) -> Option<&'static str> { + match a { + _ if is_info_json(&a) => Some("info.json"), + _ if is_cover(&a) => Some("cover.image"), + _ => None, + } + } + + pub fn is_info_json(a: &&AttachedFile) -> bool { + a.name == "info.json" && a.media_type == "application/json" + } + pub fn is_cover(a: &&AttachedFile) -> bool { + a.name.starts_with("cover") && a.media_type.starts_with("image/") + } } fn import_media_file( @@ -392,8 +434,12 @@ fn import_media_file( .attachments .iter() .flat_map(|a| &a.files) - .find(|a| a.name == "info.json" && a.media_type == "application/json") - .map(|d| serde_json::from_slice::<infojson::YVideo>(&d.data)) + .find(attachment_types::is_info_json) + .map(|att| { + let data = cache_read(str::from_utf8(&att.data).unwrap())? + .ok_or(anyhow!("info json cache missing"))?; + anyhow::Ok(serde_json::from_slice::<infojson::YVideo>(&data)?) + }) .transpose() .context("infojson parsing")?; @@ -401,13 +447,8 @@ fn import_media_file( .attachments .iter() .flat_map(|a| &a.files) - .find(|a| a.name.starts_with("cover") && a.media_type.starts_with("image/")) - .map(|att| { - cache_store(CacheKey::new_image(("cover", path)), || { - Ok(att.data.clone()) - }) - }) - .transpose()?; + .find(attachment_types::is_cover) + .map(|att| Asset(att.data.clone().try_into().unwrap())); let mut tags = m .tags @@ -519,7 +560,7 @@ fn import_media_file( node.identifiers.extend(eids); if let Some(cover) = cover { - node.pictures.insert(PictureSlot::Cover, Picture(cover.0)); + node.pictures.insert(PictureSlot::Cover, cover); } if let Some(ct) = tags.get("CONTENT_TYPE") { @@ -676,7 +717,7 @@ fn import_media_file( let tmdb_details = tmdb.episode_details(tmdb_id, season, episode, rthandle)?; if let Some(still) = &tmdb_details.still_path { - poster = Some(Picture(tmdb.image(still, rthandle)?.0)) + poster = Some(tmdb.image(still, rthandle)?) } } } @@ -816,10 +857,10 @@ fn apply_musicbrainz_recording( if let Some(filename) = apis.wikidata.query_image_path(id.to_owned(), rthandle)? { - let path = apis - .wikimedia_commons - .image_by_filename(filename, rthandle)?; - image_1 = Some(Picture(path.0)); + image_1 = Some( + apis.wikimedia_commons + .image_by_filename(filename, rthandle)?, + ); } } } @@ -828,7 +869,7 @@ fn apply_musicbrainz_recording( if let Some(id) = url.strip_prefix("https://vgmdb.net/artist/") { let id = id.parse::<u64>().context("parse vgmdb id")?; if let Some(path) = apis.vgmdb.get_artist_image(id, rthandle)? { - image_2 = Some(Picture(path.0)); + image_2 = Some(path); } } } @@ -841,14 +882,12 @@ fn apply_musicbrainz_recording( } jobs.extend(rel.attributes.clone()); - let headshot = match image_1.or(image_2) { + let _headshot = match image_1.or(image_2) { Some(x) => x, - None => Picture( - cache_store(CacheKey::new_image(("fallback", &artist.sort_name)), || { - generate_fallback(&artist.sort_name) - })? - .0, - ), + None => Asset(cache_store( + format!("fallback/{}.image", HashKey(&artist.sort_name)), + || generate_fallback(&artist.sort_name), + )?), }; node.credits.entry(group).or_default().push(Appearance { @@ -913,12 +952,10 @@ fn apply_trakt_tmdb( tmdb_data = Some(data.clone()); if let Some(path) = &data.backdrop_path { - let im = tmdb.image(path, rthandle).context("tmdb backdrop image")?; - backdrop = Some(Picture(im.0)); + backdrop = Some(tmdb.image(path, rthandle).context("tmdb backdrop image")?); } if let Some(path) = &data.poster_path { - let im = tmdb.image(path, rthandle).context("tmdb poster image")?; - poster = Some(Picture(im.0)); + poster = Some(tmdb.image(path, rthandle).context("tmdb poster image")?); } // for p in people_map.values_mut().flatten() { diff --git a/import/src/musicbrainz.rs b/import/src/musicbrainz.rs index 92df703..fe86175 100644 --- a/import/src/musicbrainz.rs +++ b/import/src/musicbrainz.rs @@ -6,7 +6,7 @@ use crate::USER_AGENT; use anyhow::{Context, Result}; -use jellycache::{cache_memory, CacheContentType, CacheKey}; +use jellycache::cache_memory; use log::info; use reqwest::{ header::{HeaderMap, HeaderName, HeaderValue}, @@ -223,107 +223,96 @@ impl MusicBrainz { } pub fn lookup_recording(&self, id: String, rt: &Handle) -> Result<Arc<MbRecordingRel>> { - cache_memory( - CacheKey::new( - CacheContentType::Json, - ("musicbrainz-recording-lookup", &id), - ), - move || { - rt.block_on(async { - let _permit = self.rate_limit.clone().acquire_owned().await?; - let permit_drop_ts = - Instant::now() + Duration::from_secs(Self::MAX_PAR_REQ as u64); - info!("recording lookup: {id}"); + cache_memory(&format!("ext/musicbrainz/recording/{id}.json"), move || { + rt.block_on(async { + let _permit = self.rate_limit.clone().acquire_owned().await?; + let permit_drop_ts = Instant::now() + Duration::from_secs(Self::MAX_PAR_REQ as u64); + info!("recording lookup: {id}"); - let inc = [ - "isrcs", - "artists", - "area-rels", - "artist-rels", - "event-rels", - "genre-rels", - "instrument-rels", - "label-rels", - "place-rels", - "recording-rels", - "release-rels", - "release-group-rels", - "series-rels", - "url-rels", - "work-rels", - ] - .join("+"); + let inc = [ + "isrcs", + "artists", + "area-rels", + "artist-rels", + "event-rels", + "genre-rels", + "instrument-rels", + "label-rels", + "place-rels", + "recording-rels", + "release-rels", + "release-group-rels", + "series-rels", + "url-rels", + "work-rels", + ] + .join("+"); - let resp = self - .client - .get(format!( - "https://musicbrainz.org/ws/2/recording/{id}?inc={inc}" - )) - .send() - .await? - .error_for_status()? - .json::<MbRecordingRel>() - .await?; + let resp = self + .client + .get(format!( + "https://musicbrainz.org/ws/2/recording/{id}?inc={inc}" + )) + .send() + .await? + .error_for_status()? + .json::<MbRecordingRel>() + .await?; - tokio::task::spawn(async move { - sleep_until(permit_drop_ts).await; - drop(_permit); - }); + tokio::task::spawn(async move { + sleep_until(permit_drop_ts).await; + drop(_permit); + }); - Ok(resp) - }) - }, - ) + Ok(resp) + }) + }) .context("musicbrainz recording lookup") } pub fn lookup_artist(&self, id: String, rt: &Handle) -> Result<Arc<MbArtistRel>> { - cache_memory( - CacheKey::new(CacheContentType::Json, ("musicbrainz-artist-lookup", &id)), - move || { - rt.block_on(async { - let _permit = self.rate_limit.clone().acquire_owned().await?; - let permit_drop_ts = - Instant::now() + Duration::from_secs(Self::MAX_PAR_REQ as u64); - info!("artist lookup: {id}"); + cache_memory(&format!("ext/musicbrainz/artist/{id}.json"), move || { + rt.block_on(async { + let _permit = self.rate_limit.clone().acquire_owned().await?; + let permit_drop_ts = Instant::now() + Duration::from_secs(Self::MAX_PAR_REQ as u64); + info!("artist lookup: {id}"); - let inc = [ - "area-rels", - "artist-rels", - "event-rels", - "genre-rels", - "instrument-rels", - "label-rels", - "place-rels", - "recording-rels", - "release-rels", - "release-group-rels", - "series-rels", - "url-rels", - "work-rels", - ] - .join("+"); + let inc = [ + "area-rels", + "artist-rels", + "event-rels", + "genre-rels", + "instrument-rels", + "label-rels", + "place-rels", + "recording-rels", + "release-rels", + "release-group-rels", + "series-rels", + "url-rels", + "work-rels", + ] + .join("+"); - let resp = self - .client - .get(format!( - "https://musicbrainz.org/ws/2/artist/{id}?inc={inc}" - )) - .send() - .await? - .error_for_status()? - .json::<MbArtistRel>() - .await?; + let resp = self + .client + .get(format!( + "https://musicbrainz.org/ws/2/artist/{id}?inc={inc}" + )) + .send() + .await? + .error_for_status()? + .json::<MbArtistRel>() + .await?; - tokio::task::spawn(async move { - sleep_until(permit_drop_ts).await; - drop(_permit); - }); + tokio::task::spawn(async move { + sleep_until(permit_drop_ts).await; + drop(_permit); + }); - Ok(resp) - }) - }, - ) + Ok(resp) + }) + }) .context("musicbrainz artist lookup") } } diff --git a/import/src/tmdb.rs b/import/src/tmdb.rs index d4a0c25..219c1f6 100644 --- a/import/src/tmdb.rs +++ b/import/src/tmdb.rs @@ -5,8 +5,11 @@ */ use crate::USER_AGENT; use anyhow::{anyhow, bail, Context, Result}; -use jellycache::{cache_memory, cache_store, CacheKey}; -use jellycommon::chrono::{format::Parsed, Utc}; +use jellycache::{cache_memory, cache_store, HashKey}; +use jellycommon::{ + chrono::{format::Parsed, Utc}, + Asset, +}; use log::info; use reqwest::{ header::{HeaderMap, HeaderName, HeaderValue}, @@ -46,7 +49,7 @@ impl Tmdb { } pub fn search(&self, kind: TmdbKind, query: &str, rt: &Handle) -> Result<Arc<TmdbQuery>> { cache_memory( - CacheKey::new_json(("api-tmdb-search", kind, query)), + &format!("ext/tmdb/search/{kind}-{}.json", HashKey(query)), move || { rt.block_on(async { info!("searching tmdb: {query:?}"); @@ -68,7 +71,7 @@ impl Tmdb { .context("tmdb search") } pub fn details(&self, kind: TmdbKind, id: u64, rt: &Handle) -> Result<Arc<TmdbDetails>> { - cache_memory(CacheKey::new_json(("tmdb-details", kind, id)), move || { + cache_memory(&format!("ext/tmdb/details/{kind}-{id}.json"), move || { rt.block_on(async { info!("fetching details: {id:?}"); Ok(self @@ -87,7 +90,7 @@ impl Tmdb { .context("tmdb details") } pub fn person_image(&self, id: u64, rt: &Handle) -> Result<Arc<TmdbPersonImage>> { - cache_memory(CacheKey::new_image(("tmdb-person-image", id)), move || { + cache_memory(&format!("ext/tmdb/person/images/{id}.json"), move || { rt.block_on(async { Ok(self .client @@ -104,8 +107,8 @@ impl Tmdb { }) .context("tmdb person images") } - pub fn image(&self, path: &str, rt: &Handle) -> Result<CacheKey> { - cache_store(CacheKey::new_image(("tmdb-image", path)), move || { + pub fn image(&self, path: &str, rt: &Handle) -> Result<Asset> { + cache_store(format!("ext/tmdb/image/{path}.json"), move || { rt.block_on(async { info!("downloading image {path:?}"); Ok(self @@ -120,6 +123,7 @@ impl Tmdb { }) }) .context("tmdb image download") + .map(Asset) } pub fn episode_details( @@ -129,7 +133,7 @@ impl Tmdb { episode: usize, rt: &Handle, ) -> Result<Arc<TmdbEpisode>> { - cache_memory(CacheKey::new_json(("tmdb-episode-details", series_id, season, episode)), move || { + cache_memory(&format!("ext/tmdb/episode-details/{series_id}-S{season}-E{episode}.json"), move || { rt.block_on(async { info!("tmdb episode details {series_id} S={season} E={episode}"); Ok(self diff --git a/import/src/trakt.rs b/import/src/trakt.rs index 4a4beea..a241725 100644 --- a/import/src/trakt.rs +++ b/import/src/trakt.rs @@ -4,8 +4,8 @@ Copyright (C) 2025 metamuffin <metamuffin.org> */ use crate::USER_AGENT; -use anyhow::Context; -use jellycache::{cache_memory, CacheKey}; +use anyhow::{Context, Result}; +use jellycache::{cache_memory, HashKey}; use jellycommon::{Appearance, CreditCategory, NodeID, NodeKind}; use log::info; use reqwest::{ @@ -51,32 +51,30 @@ impl Trakt { kinds: &[TraktKind], query: &str, rt: &Handle, - ) -> anyhow::Result<Arc<Vec<TraktSearchResult>>> { - cache_memory(CacheKey::new_json(("trakt-lookup", query)), move || { - rt.block_on(async { - let url = format!( - "https://api.trakt.tv/search/{}?query={}&extended=full", - kinds - .iter() - .map(|t| t.singular()) - .collect::<Vec<_>>() - .join(","), - urlencoding::encode(query), - ); - let res = self.client.get(url).send().await?.error_for_status()?; - Ok(res.json().await?) - }) - }) + ) -> Result<Arc<Vec<TraktSearchResult>>> { + cache_memory( + &format!("ext/trakt/search/{}.json", HashKey(query)), + move || { + rt.block_on(async { + let url = format!( + "https://api.trakt.tv/search/{}?query={}&extended=full", + kinds + .iter() + .map(|t| t.singular()) + .collect::<Vec<_>>() + .join(","), + urlencoding::encode(query), + ); + let res = self.client.get(url).send().await?.error_for_status()?; + Ok(res.json().await?) + }) + }, + ) .context("trakt search") } - pub fn lookup( - &self, - kind: TraktKind, - id: u64, - rt: &Handle, - ) -> anyhow::Result<Arc<TraktMediaObject>> { - cache_memory(CacheKey::new_json(("trakt-lookup", kind, id)), move || { + pub fn lookup(&self, kind: TraktKind, id: u64, rt: &Handle) -> Result<Arc<TraktMediaObject>> { + cache_memory(&format!("ext/trakt/lookup/{kind}-{id}.json"), move || { rt.block_on(async { info!("trakt lookup {kind:?}:{id:?}"); let url = format!("https://api.trakt.tv/{}/{id}?extended=full", kind.plural()); @@ -87,13 +85,8 @@ impl Trakt { .context("trakt lookup") } - pub fn people( - &self, - kind: TraktKind, - id: u64, - rt: &Handle, - ) -> anyhow::Result<Arc<TraktPeople>> { - cache_memory(CacheKey::new_json(("trakt-people", kind, id)), move || { + pub fn people(&self, kind: TraktKind, id: u64, rt: &Handle) -> Result<Arc<TraktPeople>> { + cache_memory(&format!("ext/trakt/people/{kind}-{id}.json"), move || { rt.block_on(async { info!("trakt people {kind:?}:{id:?}"); let url = format!( @@ -107,8 +100,8 @@ impl Trakt { .context("trakt people") } - pub fn show_seasons(&self, id: u64, rt: &Handle) -> anyhow::Result<Arc<Vec<TraktSeason>>> { - cache_memory(CacheKey::new_json(("trakt-seasons", id)), move || { + pub fn show_seasons(&self, id: u64, rt: &Handle) -> Result<Arc<Vec<TraktSeason>>> { + cache_memory(&format!("ext/trakt/seasons/{id}"), move || { rt.block_on(async { info!("trakt seasons {id:?}"); let url = format!("https://api.trakt.tv/shows/{id}/seasons?extended=full"); @@ -124,9 +117,9 @@ impl Trakt { id: u64, season: usize, rt: &Handle, - ) -> anyhow::Result<Arc<Vec<TraktEpisode>>> { + ) -> Result<Arc<Vec<TraktEpisode>>> { cache_memory( - CacheKey::new_json(("trakt-episodes", id, season)), + &format!("ext/trakt/episodes/{id}-S{season}.json"), move || { rt.block_on(async { info!("trakt episodes {id:?} season={season}"); diff --git a/import/src/vgmdb.rs b/import/src/vgmdb.rs index 4e37ba3..402fd90 100644 --- a/import/src/vgmdb.rs +++ b/import/src/vgmdb.rs @@ -6,7 +6,8 @@ use crate::USER_AGENT; use anyhow::{Context, Result}; -use jellycache::{cache_memory, cache_store, CacheContentType, CacheKey}; +use jellycache::{cache, cache_store, HashKey}; +use jellycommon::Asset; use log::info; use regex::Regex; use reqwest::{ @@ -59,28 +60,28 @@ impl Vgmdb { } } - pub fn get_artist_image(&self, id: u64, rt: &Handle) -> Result<Option<CacheKey>> { + pub fn get_artist_image(&self, id: u64, rt: &Handle) -> Result<Option<Asset>> { if let Some(url) = self.get_artist_image_url(id, rt)? { - Ok(Some( - cache_store( - CacheKey::new_image(("vgmdb-artist-media", &url)), - move || { - rt.block_on(async { - info!("downloading image {url:?}"); - Ok(self - .client - .get(url) - .send() - .await? - .error_for_status()? - .bytes() - .await? - .to_vec()) - }) - }, - ) - .context("vgmdb media download")?, - )) + cache_store( + format!("ext/vgmdb/artist-image/{}.image", HashKey(&url)), + move || { + rt.block_on(async { + info!("downloading image {url:?}"); + Ok(self + .client + .get(url) + .send() + .await? + .error_for_status()? + .bytes() + .await? + .to_vec()) + }) + }, + ) + .context("vgmdb media download") + .map(Asset) + .map(Some) } else { Ok(None) } @@ -96,34 +97,31 @@ impl Vgmdb { Ok(None) } - pub fn scrape_artist_page(&self, id: u64, rt: &Handle) -> Result<Arc<Vec<u8>>> { - cache_memory( - CacheKey::new(CacheContentType::Unknown, ("vgmdb-artist-page", id)), - move || { - rt.block_on(async { - let _permit = self.rate_limit.clone().acquire_owned().await?; - let permit_drop_ts = Instant::now() + Duration::from_secs(1); - info!("scrape artist: {id}"); + pub fn scrape_artist_page(&self, id: u64, rt: &Handle) -> Result<Vec<u8>> { + cache(&format!("ext/vgmdb/artist-page/{id}.html"), move || { + rt.block_on(async { + let _permit = self.rate_limit.clone().acquire_owned().await?; + let permit_drop_ts = Instant::now() + Duration::from_secs(1); + info!("scrape artist: {id}"); - let resp = self - .client - .get(format!("https://vgmdb.net/artist/{id}")) - .send() - .await? - .error_for_status()? - .bytes() - .await? - .to_vec(); + let resp = self + .client + .get(format!("https://vgmdb.net/artist/{id}")) + .send() + .await? + .error_for_status()? + .bytes() + .await? + .to_vec(); - tokio::task::spawn(async move { - sleep_until(permit_drop_ts).await; - drop(_permit); - }); + tokio::task::spawn(async move { + sleep_until(permit_drop_ts).await; + drop(_permit); + }); - Ok(resp) - }) - }, - ) + Ok(resp) + }) + }) .context("vgmdb artist page scrape") } } diff --git a/import/src/wikidata.rs b/import/src/wikidata.rs index 40077b9..3a107fe 100644 --- a/import/src/wikidata.rs +++ b/import/src/wikidata.rs @@ -6,7 +6,7 @@ use crate::USER_AGENT; use anyhow::{bail, Context, Result}; -use jellycache::{cache_memory, CacheKey}; +use jellycache::{cache_memory, EscapeKey}; use log::info; use reqwest::{ header::{HeaderMap, HeaderName, HeaderValue}, @@ -108,19 +108,22 @@ impl Wikidata { } pub fn query(&self, id: String, rt: &Handle) -> Result<Arc<WikidataResponse>> { - cache_memory(CacheKey::new_json(("wikidata", &id)), move || { - rt.block_on(async { - info!("entity query: {id}"); - Ok(self - .client - .get(format!("https://www.wikidata.org/entity/{id}")) - .send() - .await? - .error_for_status()? - .json() - .await?) - }) - }) + cache_memory( + &format!("ext/wikidata/{}.json", EscapeKey(&id)), + move || { + rt.block_on(async { + info!("entity query: {id}"); + Ok(self + .client + .get(format!("https://www.wikidata.org/entity/{id}")) + .send() + .await? + .error_for_status()? + .json() + .await?) + }) + }, + ) .context("wikidata entity") } } diff --git a/import/src/wikimedia_commons.rs b/import/src/wikimedia_commons.rs index 0d716f0..86d934c 100644 --- a/import/src/wikimedia_commons.rs +++ b/import/src/wikimedia_commons.rs @@ -6,7 +6,8 @@ use crate::USER_AGENT; use anyhow::{Context, Result}; -use jellycache::{cache_store, CacheKey}; +use jellycache::{cache_store, EscapeKey}; +use jellycommon::Asset; use reqwest::{ header::{HeaderMap, HeaderName, HeaderValue}, redirect::Policy, @@ -36,9 +37,9 @@ impl WikimediaCommons { Self { client } } - pub fn image_by_filename(&self, filename: String, rt: &Handle) -> Result<CacheKey> { + pub fn image_by_filename(&self, filename: String, rt: &Handle) -> Result<Asset> { cache_store( - CacheKey::new_image(("wikimedia-commons-image", &filename)), + format!("ext/wikimedia-commons/image/{}.image", EscapeKey(&filename)), move || { rt.block_on(async { Ok(self @@ -57,5 +58,6 @@ impl WikimediaCommons { }, ) .context("mediawiki image by filename") + .map(Asset) } } diff --git a/logic/src/assets.rs b/logic/src/assets.rs index e5cfada..1862b46 100644 --- a/logic/src/assets.rs +++ b/logic/src/assets.rs @@ -6,9 +6,9 @@ use crate::{DATABASE, session::Session}; use anyhow::{Result, anyhow}; -use jellycommon::{NodeID, Picture, SourceTrackKind, TrackSource}; +use jellycommon::{Asset, NodeID, SourceTrackKind, TrackSource}; -pub async fn get_node_thumbnail(_session: &Session, id: NodeID, t: f64) -> Result<Picture> { +pub async fn get_node_thumbnail(_session: &Session, id: NodeID, t: f64) -> Result<Asset> { let node = DATABASE .get_node(id)? .ok_or(anyhow!("node does not exist"))?; @@ -33,8 +33,10 @@ pub async fn get_node_thumbnail(_session: &Session, id: NodeID, t: f64) -> Resul let step = 8.; let t = (t / step).floor() * step; - let asset = match thumb_track_source { - TrackSource::Local(path, _) => jellytranscoder::thumbnail::create_thumbnail(&path, t)?, + Ok(match thumb_track_source { + TrackSource::Local(path, _) => { + Asset(jellytranscoder::thumbnail::create_thumbnail(&path, t)?) + } TrackSource::Remote(_) => { // // TODO in the new system this is preferrably a property of node ext for regular fed // let session = fed @@ -52,7 +54,5 @@ pub async fn get_node_thumbnail(_session: &Session, id: NodeID, t: f64) -> Resul // .await? todo!() } - }; - - Ok(Picture(asset.0)) + }) } diff --git a/server/src/helper/asset.rs b/server/src/helper/asset.rs new file mode 100644 index 0000000..244aa70 --- /dev/null +++ b/server/src/helper/asset.rs @@ -0,0 +1,37 @@ +/* + This file is part of jellything (https://codeberg.org/metamuffin/jellything) + which is licensed under the GNU Affero General Public License (version 3); see /COPYING. + Copyright (C) 2025 metamuffin <metamuffin.org> +*/ + +use crate::helper::A; +use jellycommon::Asset; +use rocket::{ + http::uri::fmt::{FromUriParam, Path, UriDisplay}, + request::{FromParam, FromSegments}, +}; +use std::fmt::Write; + +impl<'a> FromParam<'a> for A<Asset> { + type Error = (); + fn from_param(param: &'a str) -> Result<Self, Self::Error> { + Ok(A(Asset(param.to_owned()))) + } +} +impl UriDisplay<Path> for A<Asset> { + fn fmt(&self, f: &mut rocket::http::uri::fmt::Formatter<'_, Path>) -> std::fmt::Result { + write!(f, "{}", self.0 .0) + } +} +impl FromUriParam<Path, Asset> for A<Asset> { + type Target = A<Asset>; + fn from_uri_param(param: Asset) -> Self::Target { + A(param) + } +} +impl<'r> FromSegments<'r> for A<Asset> { + type Error = (); + fn from_segments(segments: rocket::http::uri::Segments<'r, Path>) -> Result<Self, Self::Error> { + Ok(A(Asset(segments.collect::<Vec<_>>().join("/")))) + } +} diff --git a/server/src/helper/mod.rs b/server/src/helper/mod.rs index 6d1c834..cf592d7 100644 --- a/server/src/helper/mod.rs +++ b/server/src/helper/mod.rs @@ -10,7 +10,7 @@ pub mod filter_sort; pub mod language; pub mod node_id; pub mod session; -pub mod picture; +pub mod asset; use crate::ui::error::{MyError, MyResult}; use accept::Accept; diff --git a/server/src/helper/picture.rs b/server/src/helper/picture.rs deleted file mode 100644 index d5887e3..0000000 --- a/server/src/helper/picture.rs +++ /dev/null @@ -1,32 +0,0 @@ -/* - This file is part of jellything (https://codeberg.org/metamuffin/jellything) - which is licensed under the GNU Affero General Public License (version 3); see /COPYING. - Copyright (C) 2025 metamuffin <metamuffin.org> -*/ - -use crate::helper::A; -use jellycommon::Picture; -use rocket::{ - http::uri::fmt::{FromUriParam, Path, UriDisplay}, - request::FromParam, -}; -use std::fmt::Write; -use std::str::FromStr; - -impl<'a> FromParam<'a> for A<Picture> { - type Error = (); - fn from_param(param: &'a str) -> Result<Self, Self::Error> { - Picture::from_str(param).map_err(|_| ()).map(A) - } -} -impl UriDisplay<Path> for A<Picture> { - fn fmt(&self, f: &mut rocket::http::uri::fmt::Formatter<'_, Path>) -> std::fmt::Result { - write!(f, "{}", self.0) - } -} -impl FromUriParam<Path, Picture> for A<Picture> { - type Target = A<Picture>; - fn from_uri_param(param: Picture) -> Self::Target { - A(param) - } -} diff --git a/server/src/ui/assets.rs b/server/src/ui/assets.rs index d7663c3..6bc92ef 100644 --- a/server/src/ui/assets.rs +++ b/server/src/ui/assets.rs @@ -6,31 +6,29 @@ use super::error::MyResult; use crate::helper::{cache::CacheControlImage, A}; use anyhow::{anyhow, Context}; -use jellycache::{CacheContentType, CacheKey}; -use jellycommon::{api::NodeFilterSort, NodeID, Picture, PictureSlot}; +use jellycommon::{api::NodeFilterSort, Asset, NodeID, PictureSlot}; use jellylogic::{assets::get_node_thumbnail, node::get_node, session::Session}; use rocket::{get, http::ContentType, response::Redirect}; use std::str::FromStr; -pub const AVIF_QUALITY: f32 = 50.; +pub const AVIF_QUALITY: u32 = 50; pub const AVIF_SPEED: u8 = 5; -#[get("/image/<key>?<size>")] +#[get("/image/<key..>?<size>")] pub async fn r_image( _session: A<Session>, - key: A<Picture>, + key: A<Asset>, size: Option<usize>, ) -> MyResult<(ContentType, CacheControlImage)> { let size = size.unwrap_or(2048); - let key = CacheKey(key.0 .0); - if !matches!(key.content_type(), CacheContentType::Image) { + if !key.0 .0.ends_with(".image") { Err(anyhow!("request to non-image"))? } // fit the resolution into a finite set so the maximum cache is finite too. let width = 2usize.pow(size.clamp(128, 2048).ilog2()); - let encoded = jellytranscoder::image::transcode(key, AVIF_QUALITY, AVIF_SPEED, width) + let encoded = jellytranscoder::image::transcode(&key.0 .0, AVIF_QUALITY, AVIF_SPEED, width) .context("transcoding asset")?; Ok((ContentType::AVIF, CacheControlImage(encoded))) @@ -49,8 +47,9 @@ pub async fn r_item_poster( .node .pictures .get(&slot) + .cloned() .ok_or(anyhow!("no pic todo"))?; - Ok(Redirect::permanent(rocket::uri!(r_image(*picture, size)))) + Ok(Redirect::permanent(rocket::uri!(r_image(picture, size)))) } #[get("/n/<id>/thumbnail?<t>&<size>")] diff --git a/stream/src/cues.rs b/stream/src/cues.rs index 9291e18..8aac590 100644 --- a/stream/src/cues.rs +++ b/stream/src/cues.rs @@ -5,7 +5,7 @@ */ use anyhow::{anyhow, Result}; -use jellycache::{cache_memory, CacheKey}; +use jellycache::{cache_memory, HashKey}; use jellyremuxer::demuxers::create_demuxer_autodetect; use serde::{Deserialize, Serialize}; use std::{collections::BTreeMap, fs::File, path::Path, sync::Arc}; @@ -30,7 +30,7 @@ pub struct StatsAndCues { pub fn generate_cues(path: &Path) -> Result<Arc<StatsAndCues>> { cache_memory( - CacheKey::new_json(("media-generated-cues", path)), + &format!("media/generated-cues/{}.json", HashKey(path)), move || { let media = File::open(path)?; let mut media = create_demuxer_autodetect(Box::new(media))? diff --git a/stream/src/fragment.rs b/stream/src/fragment.rs index 35a5cd2..ef4f074 100644 --- a/stream/src/fragment.rs +++ b/stream/src/fragment.rs @@ -8,6 +8,7 @@ use crate::{ stream_info, SMediaInfo, }; use anyhow::{anyhow, Result}; +use jellycache::HashKey; use jellyremuxer::{ demuxers::create_demuxer_autodetect, matroska::{self, Segment}, @@ -121,7 +122,7 @@ pub fn fragment_stream( if !format.remux { segment = transcode( track.kind, - &format!("{media_path:?} {track_num} {index}"), + &format!("{}-T{track_num}-I{index}", HashKey(media_path)), format, segment, next_kf, diff --git a/stream/src/metadata.rs b/stream/src/metadata.rs index fe553e2..30d30e5 100644 --- a/stream/src/metadata.rs +++ b/stream/src/metadata.rs @@ -5,22 +5,25 @@ */ use anyhow::{anyhow, Result}; -use jellycache::{cache_memory, CacheKey}; +use jellycache::{cache_memory, HashKey}; use jellyremuxer::{demuxers::create_demuxer_autodetect, matroska::Segment}; use std::{fs::File, path::Path, sync::Arc}; pub fn read_metadata(path: &Path) -> Result<Arc<Segment>> { - cache_memory(CacheKey::new_json(("media-metadata", path)), move || { - let media = File::open(path)?; - let mut media = - create_demuxer_autodetect(Box::new(media))?.ok_or(anyhow!("media format unknown"))?; + cache_memory( + &format!("media/stream-metadata/{}.json", HashKey(path)), + move || { + let media = File::open(path)?; + let mut media = create_demuxer_autodetect(Box::new(media))? + .ok_or(anyhow!("media format unknown"))?; - let info = media.info()?; - let tracks = media.tracks()?; - Ok(Segment { - info, - tracks, - ..Default::default() - }) - }) + let info = media.info()?; + let tracks = media.tracks()?; + Ok(Segment { + info, + tracks, + ..Default::default() + }) + }, + ) } diff --git a/stream/types/src/lib.rs b/stream/types/src/lib.rs index a031f3a..dbbe7b2 100644 --- a/stream/types/src/lib.rs +++ b/stream/types/src/lib.rs @@ -98,6 +98,16 @@ pub enum StreamContainer { JVTT, } +impl Display for TrackKind { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + TrackKind::Video => "video", + TrackKind::Audio => "audio", + TrackKind::Subtitle => "subtitle", + }) + } +} + impl StreamSpec { pub fn to_query(&self) -> String { match self { diff --git a/transcoder/src/fragment.rs b/transcoder/src/fragment.rs index 564c94d..4380df6 100644 --- a/transcoder/src/fragment.rs +++ b/transcoder/src/fragment.rs @@ -5,7 +5,7 @@ */ use crate::{Config, CONF, LOCAL_VIDEO_TRANSCODING_TASKS}; use anyhow::Result; -use jellycache::{cache, CacheContentType, CacheKey}; +use jellycache::{cache, HashKey}; use jellyremuxer::{demuxers::create_demuxer, muxers::write_fragment, ContainerFormat}; use jellystream_types::{StreamFormatInfo, TrackKind}; use log::info; @@ -39,7 +39,10 @@ pub fn transcode( let had_next_kf = next_kf.is_some(); let output = cache( - CacheKey::new(CacheContentType::Unknown, ("frag-tc", input_key, &command)), + &format!( + "transcode/media-fragment/{input_key}-{}.mkv", + HashKey(&command) + ), || { let _permit = LOCAL_VIDEO_TRANSCODING_TASKS.lock().unwrap(); info!("encoding with {command:?}"); diff --git a/transcoder/src/image.rs b/transcoder/src/image.rs index 6943b2b..49f99bd 100644 --- a/transcoder/src/image.rs +++ b/transcoder/src/image.rs @@ -5,14 +5,17 @@ */ use anyhow::{anyhow, Context, Result}; use image::imageops::FilterType; -use jellycache::{cache, cache_read, CacheKey}; +use jellycache::{cache, cache_read, HashKey}; use log::{debug, info}; use rgb::FromSlice; use std::io::Cursor; -pub fn transcode(key: CacheKey, quality: f32, speed: u8, width: usize) -> Result<Vec<u8>> { +pub fn transcode(key: &str, quality: u32, speed: u8, width: usize) -> Result<Vec<u8>> { cache( - CacheKey::new_image(("image-tc", key, width, quality as i32, speed)), + &format!( + "transcode/image/{}-W{width}-Q{quality}-S{speed}", + HashKey(key) + ), move || { let input = cache_read(key)?.ok_or(anyhow!("transcode cache key missing"))?; info!("encoding image (speed={speed}, quality={quality}, width={width})"); @@ -53,7 +56,7 @@ pub fn transcode(key: CacheKey, quality: f32, speed: u8, width: usize) -> Result let pixels = image.to_vec(); let encoded = ravif::Encoder::new() .with_speed(speed.clamp(1, 10)) - .with_quality(quality.clamp(1., 100.)) + .with_quality(quality.clamp(1, 100) as f32) .encode_rgba(imgref::Img::new( pixels.as_rgba(), image.width() as usize, diff --git a/transcoder/src/thumbnail.rs b/transcoder/src/thumbnail.rs index eda9e04..85cb356 100644 --- a/transcoder/src/thumbnail.rs +++ b/transcoder/src/thumbnail.rs @@ -1,5 +1,5 @@ use anyhow::{Context, Result}; -use jellycache::{cache_store, CacheKey}; +use jellycache::{cache_store, HashKey}; use log::info; use std::{ io::Read, @@ -7,9 +7,9 @@ use std::{ process::{Command, Stdio}, }; -pub fn create_thumbnail(path: &Path, time: f64) -> Result<CacheKey> { +pub fn create_thumbnail(path: &Path, time: f64) -> Result<String> { cache_store( - CacheKey::new_image(("thumbnail", path, time as i64)), + format!("media/thumbnail/{}-{}.image", HashKey(path), time as i64), move || { info!("creating thumbnail of {path:?} at {time}s",); |