aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authormetamuffin <metamuffin@disroot.org>2025-12-08 19:53:12 +0100
committermetamuffin <metamuffin@disroot.org>2025-12-08 19:53:12 +0100
commit6edf0fd93abf7e58b4c0974e3d3e54bcf8517946 (patch)
tree32577db9d987897d4037ba9af0084b95b55e145c
parente4584a8135584e6591bac7d5397cf227cf3cff92 (diff)
downloadjellything-6edf0fd93abf7e58b4c0974e3d3e54bcf8517946.tar
jellything-6edf0fd93abf7e58b4c0974e3d3e54bcf8517946.tar.bz2
jellything-6edf0fd93abf7e58b4c0974e3d3e54bcf8517946.tar.zst
human-readable cache keys
-rw-r--r--Cargo.lock1
-rw-r--r--cache/Cargo.toml1
-rw-r--r--cache/src/backends/filesystem.rs20
-rw-r--r--cache/src/backends/mod.rs5
-rw-r--r--cache/src/helper.rs46
-rw-r--r--cache/src/key.rs83
-rw-r--r--cache/src/lib.rs47
-rw-r--r--common/src/helpers.rs23
-rw-r--r--common/src/lib.rs7
-rw-r--r--import/src/acoustid.rs6
-rw-r--r--import/src/lib.rs161
-rw-r--r--import/src/musicbrainz.rs169
-rw-r--r--import/src/tmdb.rs20
-rw-r--r--import/src/trakt.rs65
-rw-r--r--import/src/vgmdb.rs92
-rw-r--r--import/src/wikidata.rs31
-rw-r--r--import/src/wikimedia_commons.rs8
-rw-r--r--logic/src/assets.rs14
-rw-r--r--server/src/helper/asset.rs37
-rw-r--r--server/src/helper/mod.rs2
-rw-r--r--server/src/helper/picture.rs32
-rw-r--r--server/src/ui/assets.rs17
-rw-r--r--stream/src/cues.rs4
-rw-r--r--stream/src/fragment.rs3
-rw-r--r--stream/src/metadata.rs29
-rw-r--r--stream/types/src/lib.rs10
-rw-r--r--transcoder/src/fragment.rs7
-rw-r--r--transcoder/src/image.rs11
-rw-r--r--transcoder/src/thumbnail.rs6
29 files changed, 478 insertions, 479 deletions
diff --git a/Cargo.lock b/Cargo.lock
index 8806cae..49de608 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1869,6 +1869,7 @@ dependencies = [
"base64",
"humansize",
"log",
+ "percent-encoding",
"rand 0.9.2",
"serde",
"serde_json",
diff --git a/cache/Cargo.toml b/cache/Cargo.toml
index 412b1f2..d7fcd81 100644
--- a/cache/Cargo.toml
+++ b/cache/Cargo.toml
@@ -12,3 +12,4 @@ sha2 = "0.10.9"
rand = "0.9.2"
serde = "1.0.228"
serde_json = "1.0.145"
+percent-encoding = "2.3.2"
diff --git a/cache/src/backends/filesystem.rs b/cache/src/backends/filesystem.rs
index 39fb7a2..9a9db9c 100644
--- a/cache/src/backends/filesystem.rs
+++ b/cache/src/backends/filesystem.rs
@@ -4,12 +4,11 @@
Copyright (C) 2025 metamuffin <metamuffin.org>
*/
-use crate::{CacheKey, Config, backends::CacheStorage};
+use crate::{Config, backends::CacheStorage};
use anyhow::Result;
-use base64::Engine;
use rand::random;
use std::{
- fs::{File, rename},
+ fs::{File, create_dir_all, rename},
io::{ErrorKind, Read, Write},
path::PathBuf,
};
@@ -20,25 +19,22 @@ impl Filesystem {
pub fn new(config: &Config) -> Self {
Self(config.path.clone())
}
- fn path(&self, key: CacheKey) -> PathBuf {
- let filename = base64::engine::general_purpose::URL_SAFE.encode(key.0);
- let filename = &filename[..30]; // 180 bits
- self.0.join(filename)
- }
fn temp_path(&self) -> PathBuf {
self.0.join(format!("temp-{:016x}", random::<u128>()))
}
}
impl CacheStorage for Filesystem {
- fn store(&self, key: CacheKey, value: &[u8]) -> Result<()> {
+ fn store(&self, key: String, value: &[u8]) -> Result<()> {
let temp = self.temp_path();
+ let out = self.0.join(&key);
+ create_dir_all(out.parent().unwrap())?;
File::create(&temp)?.write_all(value)?;
- rename(temp, self.path(key))?;
+ rename(temp, out)?;
Ok(())
}
- fn read(&self, key: CacheKey) -> Result<Option<Vec<u8>>> {
- match File::open(self.path(key)) {
+ fn read(&self, key: &str) -> Result<Option<Vec<u8>>> {
+ match File::open(self.0.join(key)) {
Ok(mut f) => {
let mut data = Vec::new();
f.read_to_end(&mut data)?;
diff --git a/cache/src/backends/mod.rs b/cache/src/backends/mod.rs
index 370c5ab..6b7dac3 100644
--- a/cache/src/backends/mod.rs
+++ b/cache/src/backends/mod.rs
@@ -5,10 +5,9 @@
*/
pub mod filesystem;
-use crate::CacheKey;
use anyhow::Result;
pub(crate) trait CacheStorage: Send + Sync + 'static {
- fn store(&self, key: CacheKey, value: &[u8]) -> Result<()>;
- fn read(&self, key: CacheKey) -> Result<Option<Vec<u8>>>;
+ fn store(&self, key: String, value: &[u8]) -> Result<()>;
+ fn read(&self, key: &str) -> Result<Option<Vec<u8>>>;
}
diff --git a/cache/src/helper.rs b/cache/src/helper.rs
new file mode 100644
index 0000000..8f73e1e
--- /dev/null
+++ b/cache/src/helper.rs
@@ -0,0 +1,46 @@
+/*
+ This file is part of jellything (https://codeberg.org/metamuffin/jellything)
+ which is licensed under the GNU Affero General Public License (version 3); see /COPYING.
+ Copyright (C) 2025 metamuffin <metamuffin.org>
+*/
+
+use base64::{Engine, prelude::BASE64_URL_SAFE};
+use sha2::Sha256;
+use std::{
+ fmt::Display,
+ hash::{Hash, Hasher},
+};
+
+pub struct HashKey<T>(pub T);
+impl<T: Hash> Display for HashKey<T> {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ use sha2::Digest;
+ struct ShaHasher(Sha256);
+ impl Hasher for ShaHasher {
+ fn finish(&self) -> u64 {
+ unreachable!()
+ }
+ fn write(&mut self, bytes: &[u8]) {
+ self.0.update(bytes);
+ }
+ }
+ let mut d = ShaHasher(sha2::Sha256::new());
+ self.0.hash(&mut d);
+ let d = d.0.finalize();
+ f.write_str(&BASE64_URL_SAFE.encode(d))
+ }
+}
+
+pub struct EscapeKey<T>(pub T);
+impl<T: Display> Display for EscapeKey<T> {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ write!(
+ f,
+ "{}",
+ percent_encoding::utf8_percent_encode(
+ &self.0.to_string(),
+ percent_encoding::NON_ALPHANUMERIC,
+ )
+ )
+ }
+}
diff --git a/cache/src/key.rs b/cache/src/key.rs
deleted file mode 100644
index d8ca510..0000000
--- a/cache/src/key.rs
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- This file is part of jellything (https://codeberg.org/metamuffin/jellything)
- which is licensed under the GNU Affero General Public License (version 3); see /COPYING.
- Copyright (C) 2025 metamuffin <metamuffin.org>
-*/
-use crate::{CACHE_GENERATION_BUCKET_COUNT, CONF};
-use anyhow::bail;
-use base64::{Engine, prelude::BASE64_URL_SAFE};
-use sha2::Sha256;
-use std::{
- fmt::Display,
- hash::{Hash, Hasher},
- str::FromStr,
-};
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
-pub struct CacheKey(pub [u8; 32]);
-
-impl CacheKey {
- pub fn new(ty: CacheContentType, seed: impl Hash) -> Self {
- use sha2::Digest;
- struct ShaHasher(Sha256);
- impl Hasher for ShaHasher {
- fn finish(&self) -> u64 {
- unreachable!()
- }
- fn write(&mut self, bytes: &[u8]) {
- self.0.update(bytes);
- }
- }
- let mut d = ShaHasher(sha2::Sha256::new());
- d.0.update(CONF.secret.as_bytes());
- seed.hash(&mut d);
- let d = d.0.finalize();
- let mut key: [u8; 32] = d.as_slice().try_into().unwrap();
- key[0] = ty as u8;
- Self(key)
- }
- pub fn new_json(seed: impl Hash) -> Self {
- Self::new(CacheContentType::Json, seed)
- }
- pub fn new_image(seed: impl Hash) -> Self {
- Self::new(CacheContentType::Image, seed)
- }
- pub fn content_type(&self) -> CacheContentType {
- match self.0[0] {
- 1 => CacheContentType::Image,
- 2 => CacheContentType::Json,
- _ => CacheContentType::Unknown,
- }
- }
- pub(super) fn bucket(&self) -> usize {
- (self.0[1] as usize
- | ((self.0[2] as usize) << 8)
- | ((self.0[3] as usize) << 16)
- | ((self.0[4] as usize) << 24))
- % CACHE_GENERATION_BUCKET_COUNT
- }
-}
-
-impl Display for CacheKey {
- fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
- f.write_str(&BASE64_URL_SAFE.encode(self.0))
- }
-}
-impl FromStr for CacheKey {
- type Err = anyhow::Error;
- fn from_str(s: &str) -> Result<Self, Self::Err> {
- let mut out = [0; 32];
- let size = BASE64_URL_SAFE.decode_slice(s, &mut out)?;
- if size != out.len() {
- bail!("cache key parse invalid size")
- }
- Ok(Self(out))
- }
-}
-
-#[repr(u8)]
-pub enum CacheContentType {
- Unknown = 0,
- Image = 1,
- Json = 2,
-}
diff --git a/cache/src/lib.rs b/cache/src/lib.rs
index fbda2cf..20e1424 100644
--- a/cache/src/lib.rs
+++ b/cache/src/lib.rs
@@ -3,17 +3,17 @@
which is licensed under the GNU Affero General Public License (version 3); see /COPYING.
Copyright (C) 2025 metamuffin <metamuffin.org>
*/
-pub mod backends;
-pub mod key;
+mod backends;
+mod helper;
use crate::backends::{CacheStorage, filesystem::Filesystem};
use anyhow::{Context, Result, anyhow};
-pub use key::*;
-use log::info;
+use log::{info, warn};
use serde::{Deserialize, Serialize};
use std::{
any::Any,
collections::{BTreeMap, HashMap},
+ hash::{DefaultHasher, Hash, Hasher},
path::PathBuf,
sync::{
Arc, LazyLock, Mutex, OnceLock, RwLock,
@@ -22,11 +22,12 @@ use std::{
time::Instant,
};
+pub use helper::{EscapeKey, HashKey};
+
#[derive(Debug, Deserialize)]
pub struct Config {
path: PathBuf,
max_in_memory_cache_size: usize,
- secret: String,
}
const CACHE_GENERATION_BUCKET_COUNT: usize = 1024;
@@ -54,23 +55,29 @@ pub fn init_cache() -> Result<()> {
Ok(())
}
-pub fn cache(key: CacheKey, generate: impl FnOnce() -> Result<Vec<u8>>) -> Result<Vec<u8>> {
+fn bucket(key: &str) -> usize {
+ let mut h = DefaultHasher::new();
+ key.hash(&mut h);
+ h.finish() as usize % CACHE_GENERATION_BUCKET_COUNT
+}
+
+pub fn cache(key: &str, generate: impl FnOnce() -> Result<Vec<u8>>) -> Result<Vec<u8>> {
// we need a lock even if it exists since somebody might be still in the process of writing.
let already_within = WITHIN_CACHE_FILE.with(|a| a.swap(true, Ordering::Relaxed));
let _guard = if already_within {
// TODO stupid hack to avoid deadlock for nested calls; not locking is fine but might cause double-generating
- CACHE_GENERATION_LOCKS[key.bucket()].try_lock().ok()
+ CACHE_GENERATION_LOCKS[bucket(key)].try_lock().ok()
} else {
- CACHE_GENERATION_LOCKS[key.bucket()].lock().ok()
+ CACHE_GENERATION_LOCKS[bucket(key)].lock().ok()
};
let store = CACHE_STORE.get().unwrap();
- let out = match store.read(key)? {
+ let out = match store.read(&key)? {
Some(x) => x,
None => {
let value = generate()?;
- store.store(key, &value)?;
+ store.store(key.to_owned(), &value)?;
value
}
};
@@ -82,11 +89,11 @@ pub fn cache(key: CacheKey, generate: impl FnOnce() -> Result<Vec<u8>>) -> Resul
Ok(out)
}
-pub fn cache_read(key: CacheKey) -> Result<Option<Vec<u8>>> {
+pub fn cache_read(key: &str) -> Result<Option<Vec<u8>>> {
CACHE_STORE.get().unwrap().read(key)
}
-pub fn cache_store(key: CacheKey, generate: impl FnOnce() -> Result<Vec<u8>>) -> Result<CacheKey> {
- cache(key, generate)?;
+pub fn cache_store(key: String, generate: impl FnOnce() -> Result<Vec<u8>>) -> Result<String> {
+ cache(&key, generate)?;
Ok(key)
}
@@ -95,18 +102,22 @@ pub struct InMemoryCacheEntry {
last_access: Instant,
object: Arc<dyn Any + Send + Sync + 'static>,
}
-pub static CACHE_IN_MEMORY_OBJECTS: LazyLock<RwLock<HashMap<CacheKey, InMemoryCacheEntry>>> =
+pub static CACHE_IN_MEMORY_OBJECTS: LazyLock<RwLock<HashMap<String, InMemoryCacheEntry>>> =
LazyLock::new(|| RwLock::new(HashMap::new()));
pub static CACHE_IN_MEMORY_SIZE: AtomicUsize = AtomicUsize::new(0);
-pub fn cache_memory<Fun, T>(key: CacheKey, mut generate: Fun) -> Result<Arc<T>, anyhow::Error>
+pub fn cache_memory<Fun, T>(key: &str, mut generate: Fun) -> Result<Arc<T>, anyhow::Error>
where
Fun: FnMut() -> Result<T, anyhow::Error>,
T: Serialize + for<'de> Deserialize<'de> + Send + Sync + 'static,
{
+ if !key.ends_with(".json") {
+ warn!("cache_memory key not ending in .json: {key:?}")
+ }
+
{
let mut g = CACHE_IN_MEMORY_OBJECTS.write().unwrap();
- if let Some(entry) = g.get_mut(&key) {
+ if let Some(entry) = g.get_mut(key) {
entry.last_access = Instant::now();
let object = entry
.object
@@ -117,7 +128,7 @@ where
}
}
- let data = cache(key, move || {
+ let data = cache(&key, move || {
let object = generate()?;
Ok(serde_json::to_vec(&object)?)
})?;
@@ -128,7 +139,7 @@ where
{
let mut g = CACHE_IN_MEMORY_OBJECTS.write().unwrap();
g.insert(
- key,
+ key.to_owned(),
InMemoryCacheEntry {
size,
last_access: Instant::now(),
diff --git a/common/src/helpers.rs b/common/src/helpers.rs
index 431bf8c..d643aa6 100644
--- a/common/src/helpers.rs
+++ b/common/src/helpers.rs
@@ -4,9 +4,7 @@
Copyright (C) 2025 metamuffin <metamuffin.org>
*/
-use base64::{Engine, prelude::BASE64_URL_SAFE};
-
-use crate::{CreditCategory, IdentifierType, Picture, PictureSlot};
+use crate::{CreditCategory, IdentifierType, PictureSlot};
use std::{fmt::Display, ops::Deref, str::FromStr};
#[derive(PartialEq)]
@@ -149,22 +147,3 @@ impl FromStr for PictureSlot {
})
}
}
-
-impl Display for Picture {
- fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
- f.write_str(&BASE64_URL_SAFE.encode(self.0))
- }
-}
-impl FromStr for Picture {
- type Err = &'static str;
- fn from_str(s: &str) -> Result<Self, Self::Err> {
- let mut out = [0; 32];
- let size = BASE64_URL_SAFE
- .decode_slice(s, &mut out)
- .map_err(|_| "invalid base64 picture key")?;
- if size != out.len() {
- return Err("picture key parse invalid size");
- }
- Ok(Self(out))
- }
-}
diff --git a/common/src/lib.rs b/common/src/lib.rs
index cf79cc5..670d9af 100644
--- a/common/src/lib.rs
+++ b/common/src/lib.rs
@@ -68,7 +68,7 @@ pub struct Node {
pub federated: Option<String>,
pub tags: BTreeSet<String>,
pub ratings: BTreeMap<RatingType, f64>,
- pub pictures: BTreeMap<PictureSlot, Picture>,
+ pub pictures: BTreeMap<PictureSlot, Asset>,
pub credits: BTreeMap<CreditCategory, Vec<Appearance>>,
pub identifiers: BTreeMap<IdentifierType, String>,
pub visibility: Visibility,
@@ -104,9 +104,8 @@ pub enum IdentifierType {
Omdb,
}
-// TODO custom b64 ser
-#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone, Copy)]
-pub struct Picture(pub [u8; 32]);
+#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone, Hash)]
+pub struct Asset(pub String);
#[derive(Debug, Clone, Deserialize, Serialize)]
pub struct Appearance {
diff --git a/import/src/acoustid.rs b/import/src/acoustid.rs
index 809d964..01adb57 100644
--- a/import/src/acoustid.rs
+++ b/import/src/acoustid.rs
@@ -5,7 +5,7 @@
*/
use crate::USER_AGENT;
use anyhow::{Context, Result};
-use jellycache::{cache_memory, CacheKey};
+use jellycache::{cache_memory, HashKey};
use log::info;
use reqwest::{
header::{HeaderMap, HeaderName, HeaderValue},
@@ -95,7 +95,7 @@ impl AcoustID {
}
pub fn lookup(&self, fp: Fingerprint, rt: &Handle) -> Result<Arc<AcoustIDLookupResponse>> {
- cache_memory(CacheKey::new_json(("acoustid-lookup", &fp)) , move || rt.block_on(async {
+ cache_memory(&format!("ext/acoustid/{}.json", HashKey(&fp)) , move || rt.block_on(async {
let _permit = self.rate_limit.clone().acquire_owned().await?;
let permit_drop_ts = Instant::now() + Duration::SECOND;
info!("acoustid lookup");
@@ -126,7 +126,7 @@ impl AcoustID {
pub(crate) fn acoustid_fingerprint(path: &Path) -> Result<Arc<Fingerprint>> {
cache_memory(
- CacheKey::new_json(("acoustid-fingerprint", path)),
+ &format!("media/chromaprint/{}.json", HashKey(path)),
move || {
let child = Command::new("fpcalc")
.arg("-json")
diff --git a/import/src/lib.rs b/import/src/lib.rs
index af13316..e31127e 100644
--- a/import/src/lib.rs
+++ b/import/src/lib.rs
@@ -20,10 +20,10 @@ use crate::{tmdb::TmdbKind, trakt::TraktKind};
use acoustid::{acoustid_fingerprint, AcoustID};
use anyhow::{anyhow, bail, Context, Result};
use infojson::YVideo;
-use jellycache::{cache, cache_memory, cache_store, CacheKey};
+use jellycache::{cache_memory, cache_read, cache_store, HashKey};
use jellycommon::{
- Appearance, Chapter, CreditCategory, IdentifierType, MediaInfo, Node, NodeID, NodeKind,
- Picture, PictureSlot, RatingType, SourceTrack, SourceTrackKind, TrackSource, Visibility,
+ Appearance, Asset, Chapter, CreditCategory, IdentifierType, MediaInfo, Node, NodeID, NodeKind,
+ PictureSlot, RatingType, SourceTrack, SourceTrackKind, TrackSource, Visibility,
};
use jellyimport_fallback_generator::generate_fallback;
use jellyremuxer::{
@@ -261,27 +261,31 @@ fn import_file(
match filename.as_ref() {
"poster.jpeg" | "poster.webp" | "poster.png" => {
info!("import poster at {path:?}");
- let key = CacheKey::new_image(path);
- cache(key, || {
- let mut data = Vec::new();
- File::open(path)?.read_to_end(&mut data)?;
- Ok(data)
- })?;
+ let asset = Asset(cache_store(
+ format!("media/literal/{}-poster.image", HashKey(path)),
+ || {
+ let mut data = Vec::new();
+ File::open(path)?.read_to_end(&mut data)?;
+ Ok(data)
+ },
+ )?);
db.update_node_init(parent, |node| {
- node.pictures.insert(PictureSlot::Cover, Picture(key.0));
+ node.pictures.insert(PictureSlot::Cover, asset);
Ok(())
})?;
}
"backdrop.jpeg" | "backdrop.webp" | "backdrop.png" => {
info!("import backdrop at {path:?}");
- let key = CacheKey::new_image(path);
- cache(key, || {
- let mut data = Vec::new();
- File::open(path)?.read_to_end(&mut data)?;
- Ok(data)
- })?;
+ let asset = Asset(cache_store(
+ format!("media/literal/{}-poster.image", HashKey(path)),
+ || {
+ let mut data = Vec::new();
+ File::open(path)?.read_to_end(&mut data)?;
+ Ok(data)
+ },
+ )?);
db.update_node_init(parent, |node| {
- node.pictures.insert(PictureSlot::Backdrop, Picture(key.0));
+ node.pictures.insert(PictureSlot::Backdrop, asset);
Ok(())
})?;
}
@@ -356,25 +360,63 @@ fn import_file(
}
pub fn read_media_metadata(path: &Path) -> Result<Arc<matroska::Segment>> {
- cache_memory(CacheKey::new_json(path), move || {
- let media = File::open(path)?;
- let mut media =
- create_demuxer_autodetect(Box::new(media))?.ok_or(anyhow!("media format unknown"))?;
+ cache_memory(
+ &format!("media/metadata/{}.json", HashKey(path)),
+ move || {
+ let media = File::open(path)?;
+ let mut media = create_demuxer_autodetect(Box::new(media))?
+ .ok_or(anyhow!("media format unknown"))?;
- let info = media.info()?;
- let tracks = media.tracks()?;
- let tags = media.tags()?;
- let attachments = media.attachments()?;
- let chapters = media.chapters()?;
- Ok(Segment {
- info,
- tracks,
- tags: tags.into_iter().collect(),
- attachments,
- chapters,
- ..Default::default()
- })
- })
+ let info = media.info()?;
+ let tracks = media.tracks()?;
+ let tags = media.tags()?;
+ let mut attachments = media.attachments()?;
+ let chapters = media.chapters()?;
+
+ // Replace data of useful attachments with cache key; delete data of all others
+ if let Some(attachments) = &mut attachments {
+ for att in &mut attachments.files {
+ if let Some(fname) = attachment_types::is_useful_attachment(&att) {
+ let key = cache_store(
+ format!("media/attachment/{}-{fname}", HashKey(path)),
+ || Ok(att.data.clone()),
+ )?;
+ att.data = key.as_bytes().to_vec();
+ } else {
+ att.data.clear();
+ }
+ }
+ }
+
+ Ok(Segment {
+ info,
+ tracks,
+ tags: tags.into_iter().collect(),
+ attachments,
+ chapters,
+ ..Default::default()
+ })
+ },
+ )
+}
+
+mod attachment_types {
+ use jellyremuxer::matroska::AttachedFile;
+
+ pub fn is_useful_attachment(a: &AttachedFile) -> Option<&'static str> {
+ match a {
+ _ if is_info_json(&a) => Some("info.json"),
+ _ if is_cover(&a) => Some("cover.image"),
+ _ => None,
+ }
+ }
+
+ pub fn is_info_json(a: &&AttachedFile) -> bool {
+ a.name == "info.json" && a.media_type == "application/json"
+ }
+ pub fn is_cover(a: &&AttachedFile) -> bool {
+ a.name.starts_with("cover") && a.media_type.starts_with("image/")
+ }
}
fn import_media_file(
@@ -392,8 +434,12 @@ fn import_media_file(
.attachments
.iter()
.flat_map(|a| &a.files)
- .find(|a| a.name == "info.json" && a.media_type == "application/json")
- .map(|d| serde_json::from_slice::<infojson::YVideo>(&d.data))
+ .find(attachment_types::is_info_json)
+ .map(|att| {
+ let data = cache_read(str::from_utf8(&att.data).unwrap())?
+ .ok_or(anyhow!("info json cache missing"))?;
+ anyhow::Ok(serde_json::from_slice::<infojson::YVideo>(&data)?)
+ })
.transpose()
.context("infojson parsing")?;
@@ -401,13 +447,8 @@ fn import_media_file(
.attachments
.iter()
.flat_map(|a| &a.files)
- .find(|a| a.name.starts_with("cover") && a.media_type.starts_with("image/"))
- .map(|att| {
- cache_store(CacheKey::new_image(("cover", path)), || {
- Ok(att.data.clone())
- })
- })
- .transpose()?;
+ .find(attachment_types::is_cover)
+ .map(|att| Asset(att.data.clone().try_into().unwrap()));
let mut tags = m
.tags
@@ -519,7 +560,7 @@ fn import_media_file(
node.identifiers.extend(eids);
if let Some(cover) = cover {
- node.pictures.insert(PictureSlot::Cover, Picture(cover.0));
+ node.pictures.insert(PictureSlot::Cover, cover);
}
if let Some(ct) = tags.get("CONTENT_TYPE") {
@@ -676,7 +717,7 @@ fn import_media_file(
let tmdb_details =
tmdb.episode_details(tmdb_id, season, episode, rthandle)?;
if let Some(still) = &tmdb_details.still_path {
- poster = Some(Picture(tmdb.image(still, rthandle)?.0))
+ poster = Some(tmdb.image(still, rthandle)?)
}
}
}
@@ -816,10 +857,10 @@ fn apply_musicbrainz_recording(
if let Some(filename) =
apis.wikidata.query_image_path(id.to_owned(), rthandle)?
{
- let path = apis
- .wikimedia_commons
- .image_by_filename(filename, rthandle)?;
- image_1 = Some(Picture(path.0));
+ image_1 = Some(
+ apis.wikimedia_commons
+ .image_by_filename(filename, rthandle)?,
+ );
}
}
}
@@ -828,7 +869,7 @@ fn apply_musicbrainz_recording(
if let Some(id) = url.strip_prefix("https://vgmdb.net/artist/") {
let id = id.parse::<u64>().context("parse vgmdb id")?;
if let Some(path) = apis.vgmdb.get_artist_image(id, rthandle)? {
- image_2 = Some(Picture(path.0));
+ image_2 = Some(path);
}
}
}
@@ -841,14 +882,12 @@ fn apply_musicbrainz_recording(
}
jobs.extend(rel.attributes.clone());
- let headshot = match image_1.or(image_2) {
+ let _headshot = match image_1.or(image_2) {
Some(x) => x,
- None => Picture(
- cache_store(CacheKey::new_image(("fallback", &artist.sort_name)), || {
- generate_fallback(&artist.sort_name)
- })?
- .0,
- ),
+ None => Asset(cache_store(
+ format!("fallback/{}.image", HashKey(&artist.sort_name)),
+ || generate_fallback(&artist.sort_name),
+ )?),
};
node.credits.entry(group).or_default().push(Appearance {
@@ -913,12 +952,10 @@ fn apply_trakt_tmdb(
tmdb_data = Some(data.clone());
if let Some(path) = &data.backdrop_path {
- let im = tmdb.image(path, rthandle).context("tmdb backdrop image")?;
- backdrop = Some(Picture(im.0));
+ backdrop = Some(tmdb.image(path, rthandle).context("tmdb backdrop image")?);
}
if let Some(path) = &data.poster_path {
- let im = tmdb.image(path, rthandle).context("tmdb poster image")?;
- poster = Some(Picture(im.0));
+ poster = Some(tmdb.image(path, rthandle).context("tmdb poster image")?);
}
// for p in people_map.values_mut().flatten() {
diff --git a/import/src/musicbrainz.rs b/import/src/musicbrainz.rs
index 92df703..fe86175 100644
--- a/import/src/musicbrainz.rs
+++ b/import/src/musicbrainz.rs
@@ -6,7 +6,7 @@
use crate::USER_AGENT;
use anyhow::{Context, Result};
-use jellycache::{cache_memory, CacheContentType, CacheKey};
+use jellycache::cache_memory;
use log::info;
use reqwest::{
header::{HeaderMap, HeaderName, HeaderValue},
@@ -223,107 +223,96 @@ impl MusicBrainz {
}
pub fn lookup_recording(&self, id: String, rt: &Handle) -> Result<Arc<MbRecordingRel>> {
- cache_memory(
- CacheKey::new(
- CacheContentType::Json,
- ("musicbrainz-recording-lookup", &id),
- ),
- move || {
- rt.block_on(async {
- let _permit = self.rate_limit.clone().acquire_owned().await?;
- let permit_drop_ts =
- Instant::now() + Duration::from_secs(Self::MAX_PAR_REQ as u64);
- info!("recording lookup: {id}");
+ cache_memory(&format!("ext/musicbrainz/recording/{id}.json"), move || {
+ rt.block_on(async {
+ let _permit = self.rate_limit.clone().acquire_owned().await?;
+ let permit_drop_ts = Instant::now() + Duration::from_secs(Self::MAX_PAR_REQ as u64);
+ info!("recording lookup: {id}");
- let inc = [
- "isrcs",
- "artists",
- "area-rels",
- "artist-rels",
- "event-rels",
- "genre-rels",
- "instrument-rels",
- "label-rels",
- "place-rels",
- "recording-rels",
- "release-rels",
- "release-group-rels",
- "series-rels",
- "url-rels",
- "work-rels",
- ]
- .join("+");
+ let inc = [
+ "isrcs",
+ "artists",
+ "area-rels",
+ "artist-rels",
+ "event-rels",
+ "genre-rels",
+ "instrument-rels",
+ "label-rels",
+ "place-rels",
+ "recording-rels",
+ "release-rels",
+ "release-group-rels",
+ "series-rels",
+ "url-rels",
+ "work-rels",
+ ]
+ .join("+");
- let resp = self
- .client
- .get(format!(
- "https://musicbrainz.org/ws/2/recording/{id}?inc={inc}"
- ))
- .send()
- .await?
- .error_for_status()?
- .json::<MbRecordingRel>()
- .await?;
+ let resp = self
+ .client
+ .get(format!(
+ "https://musicbrainz.org/ws/2/recording/{id}?inc={inc}"
+ ))
+ .send()
+ .await?
+ .error_for_status()?
+ .json::<MbRecordingRel>()
+ .await?;
- tokio::task::spawn(async move {
- sleep_until(permit_drop_ts).await;
- drop(_permit);
- });
+ tokio::task::spawn(async move {
+ sleep_until(permit_drop_ts).await;
+ drop(_permit);
+ });
- Ok(resp)
- })
- },
- )
+ Ok(resp)
+ })
+ })
.context("musicbrainz recording lookup")
}
pub fn lookup_artist(&self, id: String, rt: &Handle) -> Result<Arc<MbArtistRel>> {
- cache_memory(
- CacheKey::new(CacheContentType::Json, ("musicbrainz-artist-lookup", &id)),
- move || {
- rt.block_on(async {
- let _permit = self.rate_limit.clone().acquire_owned().await?;
- let permit_drop_ts =
- Instant::now() + Duration::from_secs(Self::MAX_PAR_REQ as u64);
- info!("artist lookup: {id}");
+ cache_memory(&format!("ext/musicbrainz/artist/{id}.json"), move || {
+ rt.block_on(async {
+ let _permit = self.rate_limit.clone().acquire_owned().await?;
+ let permit_drop_ts = Instant::now() + Duration::from_secs(Self::MAX_PAR_REQ as u64);
+ info!("artist lookup: {id}");
- let inc = [
- "area-rels",
- "artist-rels",
- "event-rels",
- "genre-rels",
- "instrument-rels",
- "label-rels",
- "place-rels",
- "recording-rels",
- "release-rels",
- "release-group-rels",
- "series-rels",
- "url-rels",
- "work-rels",
- ]
- .join("+");
+ let inc = [
+ "area-rels",
+ "artist-rels",
+ "event-rels",
+ "genre-rels",
+ "instrument-rels",
+ "label-rels",
+ "place-rels",
+ "recording-rels",
+ "release-rels",
+ "release-group-rels",
+ "series-rels",
+ "url-rels",
+ "work-rels",
+ ]
+ .join("+");
- let resp = self
- .client
- .get(format!(
- "https://musicbrainz.org/ws/2/artist/{id}?inc={inc}"
- ))
- .send()
- .await?
- .error_for_status()?
- .json::<MbArtistRel>()
- .await?;
+ let resp = self
+ .client
+ .get(format!(
+ "https://musicbrainz.org/ws/2/artist/{id}?inc={inc}"
+ ))
+ .send()
+ .await?
+ .error_for_status()?
+ .json::<MbArtistRel>()
+ .await?;
- tokio::task::spawn(async move {
- sleep_until(permit_drop_ts).await;
- drop(_permit);
- });
+ tokio::task::spawn(async move {
+ sleep_until(permit_drop_ts).await;
+ drop(_permit);
+ });
- Ok(resp)
- })
- },
- )
+ Ok(resp)
+ })
+ })
.context("musicbrainz artist lookup")
}
}
diff --git a/import/src/tmdb.rs b/import/src/tmdb.rs
index d4a0c25..219c1f6 100644
--- a/import/src/tmdb.rs
+++ b/import/src/tmdb.rs
@@ -5,8 +5,11 @@
*/
use crate::USER_AGENT;
use anyhow::{anyhow, bail, Context, Result};
-use jellycache::{cache_memory, cache_store, CacheKey};
-use jellycommon::chrono::{format::Parsed, Utc};
+use jellycache::{cache_memory, cache_store, HashKey};
+use jellycommon::{
+ chrono::{format::Parsed, Utc},
+ Asset,
+};
use log::info;
use reqwest::{
header::{HeaderMap, HeaderName, HeaderValue},
@@ -46,7 +49,7 @@ impl Tmdb {
}
pub fn search(&self, kind: TmdbKind, query: &str, rt: &Handle) -> Result<Arc<TmdbQuery>> {
cache_memory(
- CacheKey::new_json(("api-tmdb-search", kind, query)),
+ &format!("ext/tmdb/search/{kind}-{}.json", HashKey(query)),
move || {
rt.block_on(async {
info!("searching tmdb: {query:?}");
@@ -68,7 +71,7 @@ impl Tmdb {
.context("tmdb search")
}
pub fn details(&self, kind: TmdbKind, id: u64, rt: &Handle) -> Result<Arc<TmdbDetails>> {
- cache_memory(CacheKey::new_json(("tmdb-details", kind, id)), move || {
+ cache_memory(&format!("ext/tmdb/details/{kind}-{id}.json"), move || {
rt.block_on(async {
info!("fetching details: {id:?}");
Ok(self
@@ -87,7 +90,7 @@ impl Tmdb {
.context("tmdb details")
}
pub fn person_image(&self, id: u64, rt: &Handle) -> Result<Arc<TmdbPersonImage>> {
- cache_memory(CacheKey::new_image(("tmdb-person-image", id)), move || {
+ cache_memory(&format!("ext/tmdb/person/images/{id}.json"), move || {
rt.block_on(async {
Ok(self
.client
@@ -104,8 +107,8 @@ impl Tmdb {
})
.context("tmdb person images")
}
- pub fn image(&self, path: &str, rt: &Handle) -> Result<CacheKey> {
- cache_store(CacheKey::new_image(("tmdb-image", path)), move || {
+ pub fn image(&self, path: &str, rt: &Handle) -> Result<Asset> {
+ cache_store(format!("ext/tmdb/image/{path}.json"), move || {
rt.block_on(async {
info!("downloading image {path:?}");
Ok(self
@@ -120,6 +123,7 @@ impl Tmdb {
})
})
.context("tmdb image download")
+ .map(Asset)
}
pub fn episode_details(
@@ -129,7 +133,7 @@ impl Tmdb {
episode: usize,
rt: &Handle,
) -> Result<Arc<TmdbEpisode>> {
- cache_memory(CacheKey::new_json(("tmdb-episode-details", series_id, season, episode)), move || {
+ cache_memory(&format!("ext/tmdb/episode-details/{series_id}-S{season}-E{episode}.json"), move || {
rt.block_on(async {
info!("tmdb episode details {series_id} S={season} E={episode}");
Ok(self
diff --git a/import/src/trakt.rs b/import/src/trakt.rs
index 4a4beea..a241725 100644
--- a/import/src/trakt.rs
+++ b/import/src/trakt.rs
@@ -4,8 +4,8 @@
Copyright (C) 2025 metamuffin <metamuffin.org>
*/
use crate::USER_AGENT;
-use anyhow::Context;
-use jellycache::{cache_memory, CacheKey};
+use anyhow::{Context, Result};
+use jellycache::{cache_memory, HashKey};
use jellycommon::{Appearance, CreditCategory, NodeID, NodeKind};
use log::info;
use reqwest::{
@@ -51,32 +51,30 @@ impl Trakt {
kinds: &[TraktKind],
query: &str,
rt: &Handle,
- ) -> anyhow::Result<Arc<Vec<TraktSearchResult>>> {
- cache_memory(CacheKey::new_json(("trakt-lookup", query)), move || {
- rt.block_on(async {
- let url = format!(
- "https://api.trakt.tv/search/{}?query={}&extended=full",
- kinds
- .iter()
- .map(|t| t.singular())
- .collect::<Vec<_>>()
- .join(","),
- urlencoding::encode(query),
- );
- let res = self.client.get(url).send().await?.error_for_status()?;
- Ok(res.json().await?)
- })
- })
+ ) -> Result<Arc<Vec<TraktSearchResult>>> {
+ cache_memory(
+ &format!("ext/trakt/search/{}.json", HashKey(query)),
+ move || {
+ rt.block_on(async {
+ let url = format!(
+ "https://api.trakt.tv/search/{}?query={}&extended=full",
+ kinds
+ .iter()
+ .map(|t| t.singular())
+ .collect::<Vec<_>>()
+ .join(","),
+ urlencoding::encode(query),
+ );
+ let res = self.client.get(url).send().await?.error_for_status()?;
+ Ok(res.json().await?)
+ })
+ },
+ )
.context("trakt search")
}
- pub fn lookup(
- &self,
- kind: TraktKind,
- id: u64,
- rt: &Handle,
- ) -> anyhow::Result<Arc<TraktMediaObject>> {
- cache_memory(CacheKey::new_json(("trakt-lookup", kind, id)), move || {
+ pub fn lookup(&self, kind: TraktKind, id: u64, rt: &Handle) -> Result<Arc<TraktMediaObject>> {
+ cache_memory(&format!("ext/trakt/lookup/{kind}-{id}.json"), move || {
rt.block_on(async {
info!("trakt lookup {kind:?}:{id:?}");
let url = format!("https://api.trakt.tv/{}/{id}?extended=full", kind.plural());
@@ -87,13 +85,8 @@ impl Trakt {
.context("trakt lookup")
}
- pub fn people(
- &self,
- kind: TraktKind,
- id: u64,
- rt: &Handle,
- ) -> anyhow::Result<Arc<TraktPeople>> {
- cache_memory(CacheKey::new_json(("trakt-people", kind, id)), move || {
+ pub fn people(&self, kind: TraktKind, id: u64, rt: &Handle) -> Result<Arc<TraktPeople>> {
+ cache_memory(&format!("ext/trakt/people/{kind}-{id}.json"), move || {
rt.block_on(async {
info!("trakt people {kind:?}:{id:?}");
let url = format!(
@@ -107,8 +100,8 @@ impl Trakt {
.context("trakt people")
}
- pub fn show_seasons(&self, id: u64, rt: &Handle) -> anyhow::Result<Arc<Vec<TraktSeason>>> {
- cache_memory(CacheKey::new_json(("trakt-seasons", id)), move || {
+ pub fn show_seasons(&self, id: u64, rt: &Handle) -> Result<Arc<Vec<TraktSeason>>> {
+ cache_memory(&format!("ext/trakt/seasons/{id}"), move || {
rt.block_on(async {
info!("trakt seasons {id:?}");
let url = format!("https://api.trakt.tv/shows/{id}/seasons?extended=full");
@@ -124,9 +117,9 @@ impl Trakt {
id: u64,
season: usize,
rt: &Handle,
- ) -> anyhow::Result<Arc<Vec<TraktEpisode>>> {
+ ) -> Result<Arc<Vec<TraktEpisode>>> {
cache_memory(
- CacheKey::new_json(("trakt-episodes", id, season)),
+ &format!("ext/trakt/episodes/{id}-S{season}.json"),
move || {
rt.block_on(async {
info!("trakt episodes {id:?} season={season}");
diff --git a/import/src/vgmdb.rs b/import/src/vgmdb.rs
index 4e37ba3..402fd90 100644
--- a/import/src/vgmdb.rs
+++ b/import/src/vgmdb.rs
@@ -6,7 +6,8 @@
use crate::USER_AGENT;
use anyhow::{Context, Result};
-use jellycache::{cache_memory, cache_store, CacheContentType, CacheKey};
+use jellycache::{cache, cache_store, HashKey};
+use jellycommon::Asset;
use log::info;
use regex::Regex;
use reqwest::{
@@ -59,28 +60,28 @@ impl Vgmdb {
}
}
- pub fn get_artist_image(&self, id: u64, rt: &Handle) -> Result<Option<CacheKey>> {
+ pub fn get_artist_image(&self, id: u64, rt: &Handle) -> Result<Option<Asset>> {
if let Some(url) = self.get_artist_image_url(id, rt)? {
- Ok(Some(
- cache_store(
- CacheKey::new_image(("vgmdb-artist-media", &url)),
- move || {
- rt.block_on(async {
- info!("downloading image {url:?}");
- Ok(self
- .client
- .get(url)
- .send()
- .await?
- .error_for_status()?
- .bytes()
- .await?
- .to_vec())
- })
- },
- )
- .context("vgmdb media download")?,
- ))
+ cache_store(
+ format!("ext/vgmdb/artist-image/{}.image", HashKey(&url)),
+ move || {
+ rt.block_on(async {
+ info!("downloading image {url:?}");
+ Ok(self
+ .client
+ .get(url)
+ .send()
+ .await?
+ .error_for_status()?
+ .bytes()
+ .await?
+ .to_vec())
+ })
+ },
+ )
+ .context("vgmdb media download")
+ .map(Asset)
+ .map(Some)
} else {
Ok(None)
}
@@ -96,34 +97,31 @@ impl Vgmdb {
Ok(None)
}
- pub fn scrape_artist_page(&self, id: u64, rt: &Handle) -> Result<Arc<Vec<u8>>> {
- cache_memory(
- CacheKey::new(CacheContentType::Unknown, ("vgmdb-artist-page", id)),
- move || {
- rt.block_on(async {
- let _permit = self.rate_limit.clone().acquire_owned().await?;
- let permit_drop_ts = Instant::now() + Duration::from_secs(1);
- info!("scrape artist: {id}");
+ pub fn scrape_artist_page(&self, id: u64, rt: &Handle) -> Result<Vec<u8>> {
+ cache(&format!("ext/vgmdb/artist-page/{id}.html"), move || {
+ rt.block_on(async {
+ let _permit = self.rate_limit.clone().acquire_owned().await?;
+ let permit_drop_ts = Instant::now() + Duration::from_secs(1);
+ info!("scrape artist: {id}");
- let resp = self
- .client
- .get(format!("https://vgmdb.net/artist/{id}"))
- .send()
- .await?
- .error_for_status()?
- .bytes()
- .await?
- .to_vec();
+ let resp = self
+ .client
+ .get(format!("https://vgmdb.net/artist/{id}"))
+ .send()
+ .await?
+ .error_for_status()?
+ .bytes()
+ .await?
+ .to_vec();
- tokio::task::spawn(async move {
- sleep_until(permit_drop_ts).await;
- drop(_permit);
- });
+ tokio::task::spawn(async move {
+ sleep_until(permit_drop_ts).await;
+ drop(_permit);
+ });
- Ok(resp)
- })
- },
- )
+ Ok(resp)
+ })
+ })
.context("vgmdb artist page scrape")
}
}
diff --git a/import/src/wikidata.rs b/import/src/wikidata.rs
index 40077b9..3a107fe 100644
--- a/import/src/wikidata.rs
+++ b/import/src/wikidata.rs
@@ -6,7 +6,7 @@
use crate::USER_AGENT;
use anyhow::{bail, Context, Result};
-use jellycache::{cache_memory, CacheKey};
+use jellycache::{cache_memory, EscapeKey};
use log::info;
use reqwest::{
header::{HeaderMap, HeaderName, HeaderValue},
@@ -108,19 +108,22 @@ impl Wikidata {
}
pub fn query(&self, id: String, rt: &Handle) -> Result<Arc<WikidataResponse>> {
- cache_memory(CacheKey::new_json(("wikidata", &id)), move || {
- rt.block_on(async {
- info!("entity query: {id}");
- Ok(self
- .client
- .get(format!("https://www.wikidata.org/entity/{id}"))
- .send()
- .await?
- .error_for_status()?
- .json()
- .await?)
- })
- })
+ cache_memory(
+ &format!("ext/wikidata/{}.json", EscapeKey(&id)),
+ move || {
+ rt.block_on(async {
+ info!("entity query: {id}");
+ Ok(self
+ .client
+ .get(format!("https://www.wikidata.org/entity/{id}"))
+ .send()
+ .await?
+ .error_for_status()?
+ .json()
+ .await?)
+ })
+ },
+ )
.context("wikidata entity")
}
}
diff --git a/import/src/wikimedia_commons.rs b/import/src/wikimedia_commons.rs
index 0d716f0..86d934c 100644
--- a/import/src/wikimedia_commons.rs
+++ b/import/src/wikimedia_commons.rs
@@ -6,7 +6,8 @@
use crate::USER_AGENT;
use anyhow::{Context, Result};
-use jellycache::{cache_store, CacheKey};
+use jellycache::{cache_store, EscapeKey};
+use jellycommon::Asset;
use reqwest::{
header::{HeaderMap, HeaderName, HeaderValue},
redirect::Policy,
@@ -36,9 +37,9 @@ impl WikimediaCommons {
Self { client }
}
- pub fn image_by_filename(&self, filename: String, rt: &Handle) -> Result<CacheKey> {
+ pub fn image_by_filename(&self, filename: String, rt: &Handle) -> Result<Asset> {
cache_store(
- CacheKey::new_image(("wikimedia-commons-image", &filename)),
+ format!("ext/wikimedia-commons/image/{}.image", EscapeKey(&filename)),
move || {
rt.block_on(async {
Ok(self
@@ -57,5 +58,6 @@ impl WikimediaCommons {
},
)
.context("mediawiki image by filename")
+ .map(Asset)
}
}
diff --git a/logic/src/assets.rs b/logic/src/assets.rs
index e5cfada..1862b46 100644
--- a/logic/src/assets.rs
+++ b/logic/src/assets.rs
@@ -6,9 +6,9 @@
use crate::{DATABASE, session::Session};
use anyhow::{Result, anyhow};
-use jellycommon::{NodeID, Picture, SourceTrackKind, TrackSource};
+use jellycommon::{Asset, NodeID, SourceTrackKind, TrackSource};
-pub async fn get_node_thumbnail(_session: &Session, id: NodeID, t: f64) -> Result<Picture> {
+pub async fn get_node_thumbnail(_session: &Session, id: NodeID, t: f64) -> Result<Asset> {
let node = DATABASE
.get_node(id)?
.ok_or(anyhow!("node does not exist"))?;
@@ -33,8 +33,10 @@ pub async fn get_node_thumbnail(_session: &Session, id: NodeID, t: f64) -> Resul
let step = 8.;
let t = (t / step).floor() * step;
- let asset = match thumb_track_source {
- TrackSource::Local(path, _) => jellytranscoder::thumbnail::create_thumbnail(&path, t)?,
+ Ok(match thumb_track_source {
+ TrackSource::Local(path, _) => {
+ Asset(jellytranscoder::thumbnail::create_thumbnail(&path, t)?)
+ }
TrackSource::Remote(_) => {
// // TODO in the new system this is preferrably a property of node ext for regular fed
// let session = fed
@@ -52,7 +54,5 @@ pub async fn get_node_thumbnail(_session: &Session, id: NodeID, t: f64) -> Resul
// .await?
todo!()
}
- };
-
- Ok(Picture(asset.0))
+ })
}
diff --git a/server/src/helper/asset.rs b/server/src/helper/asset.rs
new file mode 100644
index 0000000..244aa70
--- /dev/null
+++ b/server/src/helper/asset.rs
@@ -0,0 +1,37 @@
+/*
+ This file is part of jellything (https://codeberg.org/metamuffin/jellything)
+ which is licensed under the GNU Affero General Public License (version 3); see /COPYING.
+ Copyright (C) 2025 metamuffin <metamuffin.org>
+*/
+
+use crate::helper::A;
+use jellycommon::Asset;
+use rocket::{
+ http::uri::fmt::{FromUriParam, Path, UriDisplay},
+ request::{FromParam, FromSegments},
+};
+use std::fmt::Write;
+
+impl<'a> FromParam<'a> for A<Asset> {
+ type Error = ();
+ fn from_param(param: &'a str) -> Result<Self, Self::Error> {
+ Ok(A(Asset(param.to_owned())))
+ }
+}
+impl UriDisplay<Path> for A<Asset> {
+ fn fmt(&self, f: &mut rocket::http::uri::fmt::Formatter<'_, Path>) -> std::fmt::Result {
+ write!(f, "{}", self.0 .0)
+ }
+}
+impl FromUriParam<Path, Asset> for A<Asset> {
+ type Target = A<Asset>;
+ fn from_uri_param(param: Asset) -> Self::Target {
+ A(param)
+ }
+}
+impl<'r> FromSegments<'r> for A<Asset> {
+ type Error = ();
+ fn from_segments(segments: rocket::http::uri::Segments<'r, Path>) -> Result<Self, Self::Error> {
+ Ok(A(Asset(segments.collect::<Vec<_>>().join("/"))))
+ }
+}
diff --git a/server/src/helper/mod.rs b/server/src/helper/mod.rs
index 6d1c834..cf592d7 100644
--- a/server/src/helper/mod.rs
+++ b/server/src/helper/mod.rs
@@ -10,7 +10,7 @@ pub mod filter_sort;
pub mod language;
pub mod node_id;
pub mod session;
-pub mod picture;
+pub mod asset;
use crate::ui::error::{MyError, MyResult};
use accept::Accept;
diff --git a/server/src/helper/picture.rs b/server/src/helper/picture.rs
deleted file mode 100644
index d5887e3..0000000
--- a/server/src/helper/picture.rs
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- This file is part of jellything (https://codeberg.org/metamuffin/jellything)
- which is licensed under the GNU Affero General Public License (version 3); see /COPYING.
- Copyright (C) 2025 metamuffin <metamuffin.org>
-*/
-
-use crate::helper::A;
-use jellycommon::Picture;
-use rocket::{
- http::uri::fmt::{FromUriParam, Path, UriDisplay},
- request::FromParam,
-};
-use std::fmt::Write;
-use std::str::FromStr;
-
-impl<'a> FromParam<'a> for A<Picture> {
- type Error = ();
- fn from_param(param: &'a str) -> Result<Self, Self::Error> {
- Picture::from_str(param).map_err(|_| ()).map(A)
- }
-}
-impl UriDisplay<Path> for A<Picture> {
- fn fmt(&self, f: &mut rocket::http::uri::fmt::Formatter<'_, Path>) -> std::fmt::Result {
- write!(f, "{}", self.0)
- }
-}
-impl FromUriParam<Path, Picture> for A<Picture> {
- type Target = A<Picture>;
- fn from_uri_param(param: Picture) -> Self::Target {
- A(param)
- }
-}
diff --git a/server/src/ui/assets.rs b/server/src/ui/assets.rs
index d7663c3..6bc92ef 100644
--- a/server/src/ui/assets.rs
+++ b/server/src/ui/assets.rs
@@ -6,31 +6,29 @@
use super::error::MyResult;
use crate::helper::{cache::CacheControlImage, A};
use anyhow::{anyhow, Context};
-use jellycache::{CacheContentType, CacheKey};
-use jellycommon::{api::NodeFilterSort, NodeID, Picture, PictureSlot};
+use jellycommon::{api::NodeFilterSort, Asset, NodeID, PictureSlot};
use jellylogic::{assets::get_node_thumbnail, node::get_node, session::Session};
use rocket::{get, http::ContentType, response::Redirect};
use std::str::FromStr;
-pub const AVIF_QUALITY: f32 = 50.;
+pub const AVIF_QUALITY: u32 = 50;
pub const AVIF_SPEED: u8 = 5;
-#[get("/image/<key>?<size>")]
+#[get("/image/<key..>?<size>")]
pub async fn r_image(
_session: A<Session>,
- key: A<Picture>,
+ key: A<Asset>,
size: Option<usize>,
) -> MyResult<(ContentType, CacheControlImage)> {
let size = size.unwrap_or(2048);
- let key = CacheKey(key.0 .0);
- if !matches!(key.content_type(), CacheContentType::Image) {
+ if !key.0 .0.ends_with(".image") {
Err(anyhow!("request to non-image"))?
}
// fit the resolution into a finite set so the maximum cache is finite too.
let width = 2usize.pow(size.clamp(128, 2048).ilog2());
- let encoded = jellytranscoder::image::transcode(key, AVIF_QUALITY, AVIF_SPEED, width)
+ let encoded = jellytranscoder::image::transcode(&key.0 .0, AVIF_QUALITY, AVIF_SPEED, width)
.context("transcoding asset")?;
Ok((ContentType::AVIF, CacheControlImage(encoded)))
@@ -49,8 +47,9 @@ pub async fn r_item_poster(
.node
.pictures
.get(&slot)
+ .cloned()
.ok_or(anyhow!("no pic todo"))?;
- Ok(Redirect::permanent(rocket::uri!(r_image(*picture, size))))
+ Ok(Redirect::permanent(rocket::uri!(r_image(picture, size))))
}
#[get("/n/<id>/thumbnail?<t>&<size>")]
diff --git a/stream/src/cues.rs b/stream/src/cues.rs
index 9291e18..8aac590 100644
--- a/stream/src/cues.rs
+++ b/stream/src/cues.rs
@@ -5,7 +5,7 @@
*/
use anyhow::{anyhow, Result};
-use jellycache::{cache_memory, CacheKey};
+use jellycache::{cache_memory, HashKey};
use jellyremuxer::demuxers::create_demuxer_autodetect;
use serde::{Deserialize, Serialize};
use std::{collections::BTreeMap, fs::File, path::Path, sync::Arc};
@@ -30,7 +30,7 @@ pub struct StatsAndCues {
pub fn generate_cues(path: &Path) -> Result<Arc<StatsAndCues>> {
cache_memory(
- CacheKey::new_json(("media-generated-cues", path)),
+ &format!("media/generated-cues/{}.json", HashKey(path)),
move || {
let media = File::open(path)?;
let mut media = create_demuxer_autodetect(Box::new(media))?
diff --git a/stream/src/fragment.rs b/stream/src/fragment.rs
index 35a5cd2..ef4f074 100644
--- a/stream/src/fragment.rs
+++ b/stream/src/fragment.rs
@@ -8,6 +8,7 @@ use crate::{
stream_info, SMediaInfo,
};
use anyhow::{anyhow, Result};
+use jellycache::HashKey;
use jellyremuxer::{
demuxers::create_demuxer_autodetect,
matroska::{self, Segment},
@@ -121,7 +122,7 @@ pub fn fragment_stream(
if !format.remux {
segment = transcode(
track.kind,
- &format!("{media_path:?} {track_num} {index}"),
+ &format!("{}-T{track_num}-I{index}", HashKey(media_path)),
format,
segment,
next_kf,
diff --git a/stream/src/metadata.rs b/stream/src/metadata.rs
index fe553e2..30d30e5 100644
--- a/stream/src/metadata.rs
+++ b/stream/src/metadata.rs
@@ -5,22 +5,25 @@
*/
use anyhow::{anyhow, Result};
-use jellycache::{cache_memory, CacheKey};
+use jellycache::{cache_memory, HashKey};
use jellyremuxer::{demuxers::create_demuxer_autodetect, matroska::Segment};
use std::{fs::File, path::Path, sync::Arc};
pub fn read_metadata(path: &Path) -> Result<Arc<Segment>> {
- cache_memory(CacheKey::new_json(("media-metadata", path)), move || {
- let media = File::open(path)?;
- let mut media =
- create_demuxer_autodetect(Box::new(media))?.ok_or(anyhow!("media format unknown"))?;
+ cache_memory(
+ &format!("media/stream-metadata/{}.json", HashKey(path)),
+ move || {
+ let media = File::open(path)?;
+ let mut media = create_demuxer_autodetect(Box::new(media))?
+ .ok_or(anyhow!("media format unknown"))?;
- let info = media.info()?;
- let tracks = media.tracks()?;
- Ok(Segment {
- info,
- tracks,
- ..Default::default()
- })
- })
+ let info = media.info()?;
+ let tracks = media.tracks()?;
+ Ok(Segment {
+ info,
+ tracks,
+ ..Default::default()
+ })
+ },
+ )
}
diff --git a/stream/types/src/lib.rs b/stream/types/src/lib.rs
index a031f3a..dbbe7b2 100644
--- a/stream/types/src/lib.rs
+++ b/stream/types/src/lib.rs
@@ -98,6 +98,16 @@ pub enum StreamContainer {
JVTT,
}
+impl Display for TrackKind {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ f.write_str(match self {
+ TrackKind::Video => "video",
+ TrackKind::Audio => "audio",
+ TrackKind::Subtitle => "subtitle",
+ })
+ }
+}
+
impl StreamSpec {
pub fn to_query(&self) -> String {
match self {
diff --git a/transcoder/src/fragment.rs b/transcoder/src/fragment.rs
index 564c94d..4380df6 100644
--- a/transcoder/src/fragment.rs
+++ b/transcoder/src/fragment.rs
@@ -5,7 +5,7 @@
*/
use crate::{Config, CONF, LOCAL_VIDEO_TRANSCODING_TASKS};
use anyhow::Result;
-use jellycache::{cache, CacheContentType, CacheKey};
+use jellycache::{cache, HashKey};
use jellyremuxer::{demuxers::create_demuxer, muxers::write_fragment, ContainerFormat};
use jellystream_types::{StreamFormatInfo, TrackKind};
use log::info;
@@ -39,7 +39,10 @@ pub fn transcode(
let had_next_kf = next_kf.is_some();
let output = cache(
- CacheKey::new(CacheContentType::Unknown, ("frag-tc", input_key, &command)),
+ &format!(
+ "transcode/media-fragment/{input_key}-{}.mkv",
+ HashKey(&command)
+ ),
|| {
let _permit = LOCAL_VIDEO_TRANSCODING_TASKS.lock().unwrap();
info!("encoding with {command:?}");
diff --git a/transcoder/src/image.rs b/transcoder/src/image.rs
index 6943b2b..49f99bd 100644
--- a/transcoder/src/image.rs
+++ b/transcoder/src/image.rs
@@ -5,14 +5,17 @@
*/
use anyhow::{anyhow, Context, Result};
use image::imageops::FilterType;
-use jellycache::{cache, cache_read, CacheKey};
+use jellycache::{cache, cache_read, HashKey};
use log::{debug, info};
use rgb::FromSlice;
use std::io::Cursor;
-pub fn transcode(key: CacheKey, quality: f32, speed: u8, width: usize) -> Result<Vec<u8>> {
+pub fn transcode(key: &str, quality: u32, speed: u8, width: usize) -> Result<Vec<u8>> {
cache(
- CacheKey::new_image(("image-tc", key, width, quality as i32, speed)),
+ &format!(
+ "transcode/image/{}-W{width}-Q{quality}-S{speed}",
+ HashKey(key)
+ ),
move || {
let input = cache_read(key)?.ok_or(anyhow!("transcode cache key missing"))?;
info!("encoding image (speed={speed}, quality={quality}, width={width})");
@@ -53,7 +56,7 @@ pub fn transcode(key: CacheKey, quality: f32, speed: u8, width: usize) -> Result
let pixels = image.to_vec();
let encoded = ravif::Encoder::new()
.with_speed(speed.clamp(1, 10))
- .with_quality(quality.clamp(1., 100.))
+ .with_quality(quality.clamp(1, 100) as f32)
.encode_rgba(imgref::Img::new(
pixels.as_rgba(),
image.width() as usize,
diff --git a/transcoder/src/thumbnail.rs b/transcoder/src/thumbnail.rs
index eda9e04..85cb356 100644
--- a/transcoder/src/thumbnail.rs
+++ b/transcoder/src/thumbnail.rs
@@ -1,5 +1,5 @@
use anyhow::{Context, Result};
-use jellycache::{cache_store, CacheKey};
+use jellycache::{cache_store, HashKey};
use log::info;
use std::{
io::Read,
@@ -7,9 +7,9 @@ use std::{
process::{Command, Stdio},
};
-pub fn create_thumbnail(path: &Path, time: f64) -> Result<CacheKey> {
+pub fn create_thumbnail(path: &Path, time: f64) -> Result<String> {
cache_store(
- CacheKey::new_image(("thumbnail", path, time as i64)),
+ format!("media/thumbnail/{}-{}.image", HashKey(path), time as i64),
move || {
info!("creating thumbnail of {path:?} at {time}s",);