diff options
| author | metamuffin <metamuffin@disroot.org> | 2025-11-30 12:32:44 +0100 |
|---|---|---|
| committer | metamuffin <metamuffin@disroot.org> | 2025-11-30 12:32:44 +0100 |
| commit | 8174d129fbabd2d39323678d11d868893ddb429a (patch) | |
| tree | 7979a528114cd5fb827f748f678a916e8e8eeddc | |
| parent | 5db15c323d76dca9ae71b0204d63dcb09fbbcbc5 (diff) | |
| download | jellything-8174d129fbabd2d39323678d11d868893ddb429a.tar jellything-8174d129fbabd2d39323678d11d868893ddb429a.tar.bz2 jellything-8174d129fbabd2d39323678d11d868893ddb429a.tar.zst | |
new sync cache
44 files changed, 1044 insertions, 1055 deletions
@@ -1873,13 +1873,13 @@ dependencies = [ "serde", "serde_json", "sha2", - "tokio", ] [[package]] name = "jellycommon" version = "0.1.0" dependencies = [ + "base64", "blake3", "chrono", "hex", diff --git a/cache/Cargo.toml b/cache/Cargo.toml index c171147..412b1f2 100644 --- a/cache/Cargo.toml +++ b/cache/Cargo.toml @@ -8,7 +8,6 @@ base64 = "0.22.1" humansize = "2.1.3" anyhow = "1.0.100" log = { workspace = true } -tokio = { workspace = true } sha2 = "0.10.9" rand = "0.9.2" serde = "1.0.228" diff --git a/cache/src/backends/filesystem.rs b/cache/src/backends/filesystem.rs new file mode 100644 index 0000000..39fb7a2 --- /dev/null +++ b/cache/src/backends/filesystem.rs @@ -0,0 +1,51 @@ +/* + This file is part of jellything (https://codeberg.org/metamuffin/jellything) + which is licensed under the GNU Affero General Public License (version 3); see /COPYING. + Copyright (C) 2025 metamuffin <metamuffin.org> +*/ + +use crate::{CacheKey, Config, backends::CacheStorage}; +use anyhow::Result; +use base64::Engine; +use rand::random; +use std::{ + fs::{File, rename}, + io::{ErrorKind, Read, Write}, + path::PathBuf, +}; + +pub struct Filesystem(PathBuf); + +impl Filesystem { + pub fn new(config: &Config) -> Self { + Self(config.path.clone()) + } + fn path(&self, key: CacheKey) -> PathBuf { + let filename = base64::engine::general_purpose::URL_SAFE.encode(key.0); + let filename = &filename[..30]; // 180 bits + self.0.join(filename) + } + fn temp_path(&self) -> PathBuf { + self.0.join(format!("temp-{:016x}", random::<u128>())) + } +} + +impl CacheStorage for Filesystem { + fn store(&self, key: CacheKey, value: &[u8]) -> Result<()> { + let temp = self.temp_path(); + File::create(&temp)?.write_all(value)?; + rename(temp, self.path(key))?; + Ok(()) + } + fn read(&self, key: CacheKey) -> Result<Option<Vec<u8>>> { + match File::open(self.path(key)) { + Ok(mut f) => { + let mut data = Vec::new(); + f.read_to_end(&mut data)?; + Ok(Some(data)) + } + Err(e) if e.kind() == ErrorKind::NotFound => Ok(None), + Err(e) => Err(e.into()), + } + } +} diff --git a/cache/src/backends/mod.rs b/cache/src/backends/mod.rs new file mode 100644 index 0000000..370c5ab --- /dev/null +++ b/cache/src/backends/mod.rs @@ -0,0 +1,14 @@ +/* + This file is part of jellything (https://codeberg.org/metamuffin/jellything) + which is licensed under the GNU Affero General Public License (version 3); see /COPYING. + Copyright (C) 2025 metamuffin <metamuffin.org> +*/ +pub mod filesystem; + +use crate::CacheKey; +use anyhow::Result; + +pub(crate) trait CacheStorage: Send + Sync + 'static { + fn store(&self, key: CacheKey, value: &[u8]) -> Result<()>; + fn read(&self, key: CacheKey) -> Result<Option<Vec<u8>>>; +} diff --git a/cache/src/key.rs b/cache/src/key.rs new file mode 100644 index 0000000..d8ca510 --- /dev/null +++ b/cache/src/key.rs @@ -0,0 +1,83 @@ +/* + This file is part of jellything (https://codeberg.org/metamuffin/jellything) + which is licensed under the GNU Affero General Public License (version 3); see /COPYING. + Copyright (C) 2025 metamuffin <metamuffin.org> +*/ +use crate::{CACHE_GENERATION_BUCKET_COUNT, CONF}; +use anyhow::bail; +use base64::{Engine, prelude::BASE64_URL_SAFE}; +use sha2::Sha256; +use std::{ + fmt::Display, + hash::{Hash, Hasher}, + str::FromStr, +}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct CacheKey(pub [u8; 32]); + +impl CacheKey { + pub fn new(ty: CacheContentType, seed: impl Hash) -> Self { + use sha2::Digest; + struct ShaHasher(Sha256); + impl Hasher for ShaHasher { + fn finish(&self) -> u64 { + unreachable!() + } + fn write(&mut self, bytes: &[u8]) { + self.0.update(bytes); + } + } + let mut d = ShaHasher(sha2::Sha256::new()); + d.0.update(CONF.secret.as_bytes()); + seed.hash(&mut d); + let d = d.0.finalize(); + let mut key: [u8; 32] = d.as_slice().try_into().unwrap(); + key[0] = ty as u8; + Self(key) + } + pub fn new_json(seed: impl Hash) -> Self { + Self::new(CacheContentType::Json, seed) + } + pub fn new_image(seed: impl Hash) -> Self { + Self::new(CacheContentType::Image, seed) + } + pub fn content_type(&self) -> CacheContentType { + match self.0[0] { + 1 => CacheContentType::Image, + 2 => CacheContentType::Json, + _ => CacheContentType::Unknown, + } + } + pub(super) fn bucket(&self) -> usize { + (self.0[1] as usize + | ((self.0[2] as usize) << 8) + | ((self.0[3] as usize) << 16) + | ((self.0[4] as usize) << 24)) + % CACHE_GENERATION_BUCKET_COUNT + } +} + +impl Display for CacheKey { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&BASE64_URL_SAFE.encode(self.0)) + } +} +impl FromStr for CacheKey { + type Err = anyhow::Error; + fn from_str(s: &str) -> Result<Self, Self::Err> { + let mut out = [0; 32]; + let size = BASE64_URL_SAFE.decode_slice(s, &mut out)?; + if size != out.len() { + bail!("cache key parse invalid size") + } + Ok(Self(out)) + } +} + +#[repr(u8)] +pub enum CacheContentType { + Unknown = 0, + Image = 1, + Json = 2, +} diff --git a/cache/src/lib.rs b/cache/src/lib.rs index 115741c..fbda2cf 100644 --- a/cache/src/lib.rs +++ b/cache/src/lib.rs @@ -3,37 +3,38 @@ which is licensed under the GNU Affero General Public License (version 3); see /COPYING. Copyright (C) 2025 metamuffin <metamuffin.org> */ -use anyhow::{Context, anyhow}; -use base64::Engine; -use log::{info, warn}; -use rand::random; +pub mod backends; +pub mod key; + +use crate::backends::{CacheStorage, filesystem::Filesystem}; +use anyhow::{Context, Result, anyhow}; +pub use key::*; +use log::info; use serde::{Deserialize, Serialize}; -use sha2::Sha512; use std::{ any::Any, collections::{BTreeMap, HashMap}, - fs::rename, - future::Future, - hash::{Hash, Hasher}, - io::{Seek, Write}, path::PathBuf, sync::{ - Arc, LazyLock, RwLock, + Arc, LazyLock, Mutex, OnceLock, RwLock, atomic::{AtomicBool, AtomicUsize, Ordering}, }, time::Instant, }; -use tokio::{ - io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt}, - sync::Mutex, -}; #[derive(Debug, Deserialize)] pub struct Config { path: PathBuf, max_in_memory_cache_size: usize, + secret: String, } +const CACHE_GENERATION_BUCKET_COUNT: usize = 1024; +pub static CACHE_GENERATION_LOCKS: LazyLock<[Mutex<()>; CACHE_GENERATION_BUCKET_COUNT]> = + LazyLock::new(|| [(); CACHE_GENERATION_BUCKET_COUNT].map(|_| Mutex::new(()))); + +thread_local! { pub static WITHIN_CACHE_FILE: AtomicBool = const { AtomicBool::new(false) }; } + pub static CONF_PRELOAD: std::sync::Mutex<Option<Config>> = std::sync::Mutex::new(None); static CONF: LazyLock<Config> = LazyLock::new(|| { CONF_PRELOAD @@ -43,126 +44,50 @@ static CONF: LazyLock<Config> = LazyLock::new(|| { .expect("cache config not preloaded. logic error") }); -#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] -pub struct CachePath(pub PathBuf); -impl CachePath { - pub fn abs(&self) -> PathBuf { - CONF.path.join(&self.0) - } -} - -pub fn cache_location(kind: &str, key: impl Hash) -> (usize, CachePath) { - use sha2::Digest; - struct ShaHasher(Sha512); - impl Hasher for ShaHasher { - fn finish(&self) -> u64 { - unreachable!() - } - fn write(&mut self, bytes: &[u8]) { - self.0.update(bytes); - } - } - let mut d = ShaHasher(sha2::Sha512::new()); - d.0.update(kind); - d.0.update(b"\0"); - key.hash(&mut d); - - let d = d.0.finalize(); - let n = - d[0] as usize | ((d[1] as usize) << 8) | ((d[2] as usize) << 16) | ((d[3] as usize) << 24); - let fname = base64::engine::general_purpose::URL_SAFE.encode(d); - let fname = &fname[..30]; // 180 bits - let fname = format!("{}/{}", kind, fname); - (n, CachePath(fname.into())) -} - -const CACHE_GENERATION_BUCKET_COUNT: usize = 1024; -pub static CACHE_GENERATION_LOCKS: LazyLock<[Mutex<()>; CACHE_GENERATION_BUCKET_COUNT]> = - LazyLock::new(|| [(); CACHE_GENERATION_BUCKET_COUNT].map(|_| Mutex::new(()))); +static CACHE_STORE: OnceLock<Box<dyn CacheStorage>> = OnceLock::new(); -pub async fn async_cache_file<Fun, Fut>( - kind: &str, - key: impl Hash, - generate: Fun, -) -> Result<CachePath, anyhow::Error> -where - Fun: FnOnce(tokio::fs::File) -> Fut, - Fut: Future<Output = Result<(), anyhow::Error>>, -{ - let (bucket, location) = cache_location(kind, key); - let loc_abs = location.abs(); - // we need a lock even if it exists since somebody might be still in the process of writing. - let _guard = CACHE_GENERATION_LOCKS[bucket % CACHE_GENERATION_BUCKET_COUNT] - .lock() - .await; - let exists = tokio::fs::try_exists(&loc_abs) - .await - .context("unable to test for cache file existance")?; - if !exists { - let temp_path = CONF.path.join(format!("temp-{:x}", random::<u128>())); - let f = tokio::fs::File::create(&temp_path) - .await - .context("creating new cache file")?; - match generate(f).await { - Ok(()) => (), - Err(e) => { - warn!("cache generation failed, unlinking entry"); - tokio::fs::remove_file(temp_path).await?; - return Err(e); - } - } - tokio::fs::create_dir_all(loc_abs.parent().unwrap()) - .await - .context("create kind dir")?; - tokio::fs::rename(temp_path, &loc_abs) - .await - .context("rename cache")?; - } - drop(_guard); - Ok(location) +pub fn init_cache() -> Result<()> { + CACHE_STORE + .set(Box::new(Filesystem::new(&CONF))) + .map_err(|_| ()) + .unwrap(); + Ok(()) } -thread_local! { pub static WITHIN_CACHE_FILE: AtomicBool = const { AtomicBool::new(false) }; } - -pub fn cache_file<Fun>( - kind: &str, - key: impl Hash, - generate: Fun, -) -> Result<CachePath, anyhow::Error> -where - Fun: FnOnce(std::fs::File) -> Result<(), anyhow::Error>, -{ - let (bucket, location) = cache_location(kind, key); - let loc_abs = location.abs(); +pub fn cache(key: CacheKey, generate: impl FnOnce() -> Result<Vec<u8>>) -> Result<Vec<u8>> { // we need a lock even if it exists since somebody might be still in the process of writing. let already_within = WITHIN_CACHE_FILE.with(|a| a.swap(true, Ordering::Relaxed)); let _guard = if already_within { - // TODO stupid hack to avoid deadlock for nested cache_file. proper solution needed - CACHE_GENERATION_LOCKS[bucket % CACHE_GENERATION_BUCKET_COUNT] - .try_lock() - .ok() + // TODO stupid hack to avoid deadlock for nested calls; not locking is fine but might cause double-generating + CACHE_GENERATION_LOCKS[key.bucket()].try_lock().ok() } else { - Some(CACHE_GENERATION_LOCKS[bucket % CACHE_GENERATION_BUCKET_COUNT].blocking_lock()) + CACHE_GENERATION_LOCKS[key.bucket()].lock().ok() }; - if !loc_abs.exists() { - let temp_path = CONF.path.join(format!("temp-{:x}", random::<u128>())); - let f = std::fs::File::create(&temp_path).context("creating new cache file")?; - match generate(f) { - Ok(()) => (), - Err(e) => { - warn!("cache generation failed, unlinking entry"); - std::fs::remove_file(temp_path)?; - return Err(e); - } + + let store = CACHE_STORE.get().unwrap(); + + let out = match store.read(key)? { + Some(x) => x, + None => { + let value = generate()?; + store.store(key, &value)?; + value } - std::fs::create_dir_all(loc_abs.parent().unwrap()).context("create kind dir")?; - rename(temp_path, loc_abs).context("rename cache")?; - } + }; + if !already_within { WITHIN_CACHE_FILE.with(|a| a.swap(false, Ordering::Relaxed)); } drop(_guard); - Ok(location) + Ok(out) +} + +pub fn cache_read(key: CacheKey) -> Result<Option<Vec<u8>>> { + CACHE_STORE.get().unwrap().read(key) +} +pub fn cache_store(key: CacheKey, generate: impl FnOnce() -> Result<Vec<u8>>) -> Result<CacheKey> { + cache(key, generate)?; + Ok(key) } pub struct InMemoryCacheEntry { @@ -170,23 +95,18 @@ pub struct InMemoryCacheEntry { last_access: Instant, object: Arc<dyn Any + Send + Sync + 'static>, } -pub static CACHE_IN_MEMORY_OBJECTS: LazyLock<RwLock<HashMap<PathBuf, InMemoryCacheEntry>>> = +pub static CACHE_IN_MEMORY_OBJECTS: LazyLock<RwLock<HashMap<CacheKey, InMemoryCacheEntry>>> = LazyLock::new(|| RwLock::new(HashMap::new())); pub static CACHE_IN_MEMORY_SIZE: AtomicUsize = AtomicUsize::new(0); -pub fn cache_memory<Fun, T>( - kind: &str, - key: impl Hash, - mut generate: Fun, -) -> Result<Arc<T>, anyhow::Error> +pub fn cache_memory<Fun, T>(key: CacheKey, mut generate: Fun) -> Result<Arc<T>, anyhow::Error> where Fun: FnMut() -> Result<T, anyhow::Error>, T: Serialize + for<'de> Deserialize<'de> + Send + Sync + 'static, { - let (_, location) = cache_location(kind, &key); { let mut g = CACHE_IN_MEMORY_OBJECTS.write().unwrap(); - if let Some(entry) = g.get_mut(&location.abs()) { + if let Some(entry) = g.get_mut(&key) { entry.last_access = Instant::now(); let object = entry .object @@ -197,82 +117,18 @@ where } } - let location = cache_file(kind, &key, move |file| { + let data = cache(key, move || { let object = generate()?; - let mut file = std::io::BufWriter::new(file); - serde_json::to_writer(&mut file, &object).context("encoding cache object")?; - file.flush()?; - Ok(()) + Ok(serde_json::to_vec(&object)?) })?; - let mut file = std::io::BufReader::new(std::fs::File::open(location.abs())?); - let object = serde_json::from_reader::<_, T>(&mut file).context("decoding cache object")?; - let object = Arc::new(object); - let size = file.stream_position()? as usize; // this is an approximation mainly since varint is used in bincode - - { - let mut g = CACHE_IN_MEMORY_OBJECTS.write().unwrap(); - g.insert( - location.abs(), - InMemoryCacheEntry { - size, - last_access: Instant::now(), - object: object.clone(), - }, - ); - CACHE_IN_MEMORY_SIZE.fetch_add(size, Ordering::Relaxed); - } - - cleanup_cache(); - - Ok(object) -} - -pub async fn async_cache_memory<Fun, Fut, T>( - kind: &str, - key: impl Hash, - generate: Fun, -) -> Result<Arc<T>, anyhow::Error> -where - Fun: FnOnce() -> Fut, - Fut: Future<Output = Result<T, anyhow::Error>>, - T: Serialize + for<'de> Deserialize<'de> + Send + Sync + 'static, -{ - let (_, location) = cache_location(kind, &key); - { - let mut g = CACHE_IN_MEMORY_OBJECTS.write().unwrap(); - if let Some(entry) = g.get_mut(&location.abs()) { - entry.last_access = Instant::now(); - let object = entry - .object - .clone() - .downcast::<T>() - .map_err(|_| anyhow!("inconsistent types for in-memory cache"))?; - return Ok(object); - } - } - - let location = async_cache_file(kind, &key, move |mut file| async move { - let object = generate().await?; - let data = serde_json::to_vec(&object).context("encoding cache object")?; - - file.write_all(&data).await?; - - Ok(()) - }) - .await?; - let mut file = tokio::fs::File::open(&location.abs()).await?; - let mut data = Vec::new(); - file.read_to_end(&mut data) - .await - .context("reading cache object")?; + let size = data.len(); let object = serde_json::from_slice::<T>(&data).context("decoding cache object")?; let object = Arc::new(object); - let size = file.stream_position().await? as usize; // this is an approximation mainly since varint is used in bincode { let mut g = CACHE_IN_MEMORY_OBJECTS.write().unwrap(); g.insert( - location.abs(), + key, InMemoryCacheEntry { size, last_access: Instant::now(), diff --git a/common/Cargo.toml b/common/Cargo.toml index 37e9c6c..7e25933 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -9,3 +9,4 @@ chrono = { version = "0.4.42", features = ["serde"] } blake3 = "1.8.2" hex = "0.4.3" jellystream-types = { path = "../stream/types" } +base64 = "0.22.1" diff --git a/common/src/helpers.rs b/common/src/helpers.rs index db75ba9..431bf8c 100644 --- a/common/src/helpers.rs +++ b/common/src/helpers.rs @@ -4,7 +4,9 @@ Copyright (C) 2025 metamuffin <metamuffin.org> */ -use crate::{CreditCategory, IdentifierType}; +use base64::{Engine, prelude::BASE64_URL_SAFE}; + +use crate::{CreditCategory, IdentifierType, Picture, PictureSlot}; use std::{fmt::Display, ops::Deref, str::FromStr}; #[derive(PartialEq)] @@ -129,3 +131,40 @@ impl FromStr for IdentifierType { }) } } +impl Display for PictureSlot { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + PictureSlot::Backdrop => "backdrop", + PictureSlot::Cover => "cover", + }) + } +} +impl FromStr for PictureSlot { + type Err = (); + fn from_str(s: &str) -> Result<Self, Self::Err> { + Ok(match s { + "backdrop" => PictureSlot::Backdrop, + "cover" => PictureSlot::Cover, + _ => return Err(()), + }) + } +} + +impl Display for Picture { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&BASE64_URL_SAFE.encode(self.0)) + } +} +impl FromStr for Picture { + type Err = &'static str; + fn from_str(s: &str) -> Result<Self, Self::Err> { + let mut out = [0; 32]; + let size = BASE64_URL_SAFE + .decode_slice(s, &mut out) + .map_err(|_| "invalid base64 picture key")?; + if size != out.len() { + return Err("picture key parse invalid size"); + } + Ok(Self(out)) + } +} diff --git a/common/src/lib.rs b/common/src/lib.rs index f3d8416..3f535fd 100644 --- a/common/src/lib.rs +++ b/common/src/lib.rs @@ -68,7 +68,7 @@ pub struct Node { pub federated: Option<String>, pub tags: BTreeSet<String>, pub ratings: BTreeMap<RatingType, f64>, - pub pictures: BTreeMap<PictureSlot, Asset>, + pub pictures: BTreeMap<PictureSlot, Picture>, pub credits: BTreeMap<CreditCategory, Vec<Appearance>>, pub identifiers: BTreeMap<IdentifierType, String>, pub visibility: Visibility, @@ -78,10 +78,8 @@ pub struct Node { #[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq, PartialOrd, Ord)] #[serde(rename_all = "snake_case")] pub enum PictureSlot { - Poster, Cover, Backdrop, - Headshot, } #[derive(Debug, Clone, Copy, Deserialize, Serialize, PartialEq, Eq, PartialOrd, Ord)] @@ -105,8 +103,9 @@ pub enum IdentifierType { Omdb, } -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone)] -pub struct Asset(pub PathBuf); +// TODO custom b64 ser +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone, Copy)] +pub struct Picture(pub [u8; 32]); #[derive(Debug, Clone, Deserialize, Serialize)] pub struct Appearance { diff --git a/common/src/routes.rs b/common/src/routes.rs index f78c7f0..19b0206 100644 --- a/common/src/routes.rs +++ b/common/src/routes.rs @@ -3,7 +3,7 @@ which is licensed under the GNU Affero General Public License (version 3); see /COPYING. Copyright (C) 2025 metamuffin <metamuffin.org> */ -use crate::{api::NodeFilterSort, user::ApiWatchedState, NodeID, CreditCategory}; +use crate::{CreditCategory, NodeID, PictureSlot, api::NodeFilterSort, user::ApiWatchedState}; pub fn u_home() -> String { "/home".to_owned() @@ -20,11 +20,8 @@ pub fn u_node_slug_player(node: &str) -> String { pub fn u_node_slug_player_time(node: &str, time: f64) -> String { format!("/n/{node}/player?t={time}") } -pub fn u_node_slug_poster(node: &str, width: usize) -> String { - format!("/n/{node}/poster?width={width}") -} -pub fn u_node_slug_backdrop(node: &str, width: usize) -> String { - format!("/n/{node}/backdrop?width={width}") +pub fn u_node_image(node: &str, slot: PictureSlot, width: usize) -> String { + format!("/n/{node}/image/{slot}?width={width}") } pub fn u_node_slug_watched(node: &str, state: ApiWatchedState) -> String { format!("/n/{node}/watched?state={state}") diff --git a/import/fallback_generator/src/lib.rs b/import/fallback_generator/src/lib.rs index 0fcfaa1..60b8cc5 100644 --- a/import/fallback_generator/src/lib.rs +++ b/import/fallback_generator/src/lib.rs @@ -3,12 +3,9 @@ use ab_glyph::{FontRef, PxScale}; use anyhow::Result; use image::{DynamicImage, ImageBuffer, ImageEncoder, Rgba, codecs::qoi::QoiEncoder}; use imageproc::drawing::{draw_text_mut, text_size}; -use std::{ - hash::{Hash, Hasher}, - io::Write, -}; +use std::hash::{Hash, Hasher}; -pub fn generate_fallback(name: &str, output: &mut dyn Write) -> Result<()> { +pub fn generate_fallback(name: &str) -> Result<Vec<u8>> { let width = 1024; let height = (width * 1000) / 707; @@ -65,13 +62,14 @@ pub fn generate_fallback(name: &str, output: &mut dyn Write) -> Result<()> { let image = DynamicImage::from(image).to_rgb8(); - QoiEncoder::new(output).write_image( + let mut output = Vec::new(); + QoiEncoder::new(&mut output).write_image( image.as_raw(), image.width(), image.height(), image::ExtendedColorType::Rgb8, )?; - Ok(()) + Ok(output) } struct XorshiftHasher(u64); @@ -104,5 +102,5 @@ fn random_accent(text: &str, y: f32) -> Rgba<f32> { #[test] fn generate_fallback_test() { - generate_fallback("Hello world!", &mut Vec::new()).unwrap(); + generate_fallback("Hello world!").unwrap(); } diff --git a/import/src/acoustid.rs b/import/src/acoustid.rs index c35708d..809d964 100644 --- a/import/src/acoustid.rs +++ b/import/src/acoustid.rs @@ -5,17 +5,22 @@ */ use crate::USER_AGENT; use anyhow::{Context, Result}; -use jellycache::async_cache_memory; +use jellycache::{cache_memory, CacheKey}; use log::info; use reqwest::{ header::{HeaderMap, HeaderName, HeaderValue}, Client, ClientBuilder, }; use serde::{Deserialize, Serialize}; -use std::{path::Path, process::Stdio, sync::Arc, time::Duration}; +use std::{ + io::Read, + path::Path, + process::{Command, Stdio}, + sync::Arc, + time::Duration, +}; use tokio::{ - io::AsyncReadExt, - process::Command, + runtime::Handle, sync::Semaphore, time::{sleep_until, Instant}, }; @@ -79,8 +84,8 @@ impl AcoustID { } } - pub async fn get_atid_mbid(&self, fp: &Fingerprint) -> Result<Option<(String, String)>> { - let res = self.lookup(fp.to_owned()).await?; + pub fn get_atid_mbid(&self, fp: &Fingerprint, rt: &Handle) -> Result<Option<(String, String)>> { + let res = self.lookup(fp.to_owned(), rt)?; for r in &res.results { if let Some(k) = r.recordings.first() { return Ok(Some((r.id.clone(), k.id.clone()))); @@ -89,8 +94,8 @@ impl AcoustID { Ok(None) } - pub async fn lookup(&self, fp: Fingerprint) -> Result<Arc<AcoustIDLookupResponse>> { - async_cache_memory("api-acoustid", fp.clone(), || async move { + pub fn lookup(&self, fp: Fingerprint, rt: &Handle) -> Result<Arc<AcoustIDLookupResponse>> { + cache_memory(CacheKey::new_json(("acoustid-lookup", &fp)) , move || rt.block_on(async { let _permit = self.rate_limit.clone().acquire_owned().await?; let permit_drop_ts = Instant::now() + Duration::SECOND; info!("acoustid lookup"); @@ -114,36 +119,36 @@ impl AcoustID { }); Ok(resp) - }) - .await.context("acoustid lookup") + })) + .context("acoustid lookup") } } -#[allow(unused)] -pub(crate) async fn acoustid_fingerprint(path: &Path) -> Result<Arc<Fingerprint>> { - async_cache_memory("fpcalc", path, || async move { - let child = Command::new("fpcalc") - .arg("-json") - .arg(path) - .stdout(Stdio::piped()) - .spawn() - .context("fpcalc")?; - - let mut buf = Vec::new(); - child - .stdout - .unwrap() - .read_to_end(&mut buf) - .await - .context("read fpcalc output")?; +pub(crate) fn acoustid_fingerprint(path: &Path) -> Result<Arc<Fingerprint>> { + cache_memory( + CacheKey::new_json(("acoustid-fingerprint", path)), + move || { + let child = Command::new("fpcalc") + .arg("-json") + .arg(path) + .stdout(Stdio::piped()) + .spawn() + .context("fpcalc")?; - let out: FpCalcOutput = serde_json::from_slice(&buf).context("parsing fpcalc output")?; - let out = Fingerprint { - duration: out.duration as u32, - fingerprint: out.fingerprint, - }; + let mut buf = Vec::new(); + child + .stdout + .unwrap() + .read_to_end(&mut buf) + .context("read fpcalc output")?; - Ok(out) - }) - .await + let out: FpCalcOutput = + serde_json::from_slice(&buf).context("parsing fpcalc output")?; + let out = Fingerprint { + duration: out.duration as u32, + fingerprint: out.fingerprint, + }; + Ok(out) + }, + ) } diff --git a/import/src/lib.rs b/import/src/lib.rs index d12913f..3cbabdc 100644 --- a/import/src/lib.rs +++ b/import/src/lib.rs @@ -20,11 +20,10 @@ use crate::{tmdb::TmdbKind, trakt::TraktKind}; use acoustid::{acoustid_fingerprint, AcoustID}; use anyhow::{anyhow, bail, Context, Result}; use infojson::YVideo; -use jellycache::{cache_file, cache_memory}; +use jellycache::{cache, cache_memory, cache_store, CacheKey}; use jellycommon::{ - Appearance, Asset, Chapter, CreditCategory, IdentifierType, LocalTrack, MediaInfo, Node, - NodeID, NodeKind, PictureSlot, RatingType, SourceTrack, SourceTrackKind, TrackSource, - Visibility, + Appearance, Chapter, CreditCategory, IdentifierType, MediaInfo, Node, NodeID, NodeKind, + Picture, PictureSlot, RatingType, SourceTrack, SourceTrackKind, TrackSource, Visibility, }; use jellyimport_fallback_generator::generate_fallback; use jellyremuxer::{ @@ -39,7 +38,7 @@ use serde::{Deserialize, Serialize}; use std::{ collections::{BTreeMap, HashMap}, fs::{read_to_string, File}, - io::{self, BufReader, Write}, + io::{BufReader, Read}, path::{Path, PathBuf}, sync::{Arc, LazyLock, Mutex}, time::UNIX_EPOCH, @@ -262,23 +261,27 @@ fn import_file( match filename.as_ref() { "poster.jpeg" | "poster.webp" | "poster.png" => { info!("import poster at {path:?}"); - let path = cache_file("picture-file", path, |mut f| { - io::copy(&mut File::open(path)?, &mut f)?; - Ok(()) + let key = CacheKey::new_image(path); + cache(key, || { + let mut data = Vec::new(); + File::open(path)?.read_to_end(&mut data)?; + Ok(data) })?; db.update_node_init(parent, |node| { - node.pictures.insert(PictureSlot::Poster, Asset(path.0)); + node.pictures.insert(PictureSlot::Cover, Picture(key.0)); Ok(()) })?; } "backdrop.jpeg" | "backdrop.webp" | "backdrop.png" => { info!("import backdrop at {path:?}"); - let path = cache_file("picture-file", path, |mut f| { - io::copy(&mut File::open(path)?, &mut f)?; - Ok(()) + let key = CacheKey::new_image(path); + cache(key, || { + let mut data = Vec::new(); + File::open(path)?.read_to_end(&mut data)?; + Ok(data) })?; db.update_node_init(parent, |node| { - node.pictures.insert(PictureSlot::Backdrop, Asset(path.0)); + node.pictures.insert(PictureSlot::Backdrop, Picture(key.0)); Ok(()) })?; } @@ -353,7 +356,7 @@ fn import_file( } pub fn read_media_metadata(path: &Path) -> Result<Arc<matroska::Segment>> { - cache_memory("mkmeta-v4", path, move || { + cache_memory(CacheKey::new_json(path), move || { let media = File::open(path)?; let mut media = create_demuxer_autodetect(Box::new(media))?.ok_or(anyhow!("media format unknown"))?; @@ -400,9 +403,8 @@ fn import_media_file( .flat_map(|a| &a.files) .find(|a| a.name.starts_with("cover") && a.media_type.starts_with("image/")) .map(|att| { - cache_file("att-cover-v2", path, move |mut file| { - file.write_all(&att.data)?; - Ok(()) + cache_store(CacheKey::new_image(("cover", path)), || { + Ok(att.data.clone()) }) }) .transpose()?; @@ -488,13 +490,13 @@ fn import_media_file( } if iflags.use_acoustid { - let fp = rthandle.block_on(acoustid_fingerprint(path))?; - if let Some((atid, mbid)) = rthandle.block_on( - apis.acoustid - .as_ref() - .ok_or(anyhow!("need acoustid"))? - .get_atid_mbid(&fp), - )? { + let fp = acoustid_fingerprint(path)?; + if let Some((atid, mbid)) = apis + .acoustid + .as_ref() + .ok_or(anyhow!("need acoustid"))? + .get_atid_mbid(&fp, rthandle)? + { eids.insert(IdentifierType::AcoustIdTrack, atid); eids.insert(IdentifierType::MusicbrainzRecording, mbid); }; @@ -517,7 +519,7 @@ fn import_media_file( node.identifiers.extend(eids); if let Some(cover) = cover { - node.pictures.insert(PictureSlot::Cover, Asset(cover.0)); + node.pictures.insert(PictureSlot::Cover, Picture(cover.0)); } if let Some(ct) = tags.get("CONTENT_TYPE") { @@ -664,18 +666,17 @@ fn import_media_file( } if let Some(trakt_id) = trakt_id { let trakt = apis.trakt.as_ref().ok_or(anyhow!("trakt required"))?; - let seasons = rthandle.block_on(trakt.show_seasons(trakt_id))?; + let seasons = trakt.show_seasons(trakt_id, rthandle)?; if seasons.iter().any(|x| x.number == season) { - let episodes = rthandle.block_on(trakt.show_season_episodes(trakt_id, season))?; + let episodes = trakt.show_season_episodes(trakt_id, season, rthandle)?; let mut poster = None; if let Some(tmdb) = &apis.tmdb { - let trakt_details = - rthandle.block_on(trakt.lookup(TraktKind::Show, trakt_id))?; + let trakt_details = trakt.lookup(TraktKind::Show, trakt_id, rthandle)?; if let Some(tmdb_id) = trakt_details.ids.tmdb { let tmdb_details = - rthandle.block_on(tmdb.episode_details(tmdb_id, season, episode))?; + tmdb.episode_details(tmdb_id, season, episode, rthandle)?; if let Some(still) = &tmdb_details.still_path { - poster = Some(Asset(rthandle.block_on(tmdb.image(still))?.0)) + poster = Some(Picture(tmdb.image(still, rthandle)?.0)) } } } @@ -685,7 +686,7 @@ fn import_media_file( node.index = Some(episode.number); node.title = Some(episode.title.clone()); if let Some(poster) = poster { - node.pictures.insert(PictureSlot::Poster, poster); + node.pictures.insert(PictureSlot::Cover, poster); } node.description = episode.overview.clone().or(node.description.clone()); node.ratings.insert(RatingType::Trakt, episode.rating); @@ -768,7 +769,7 @@ fn apply_musicbrainz_recording( node: NodeID, mbid: String, ) -> Result<()> { - let rec = rthandle.block_on(apis.musicbrainz.lookup_recording(mbid))?; + let rec = apis.musicbrainz.lookup_recording(mbid, rthandle)?; db.update_node_init(node, |node| { node.title = Some(rec.title.clone()); @@ -800,8 +801,9 @@ fn apply_musicbrainz_recording( if let Some((note, group)) = a { let artist = rel.artist.as_ref().unwrap(); - let artist = - rthandle.block_on(apis.musicbrainz.lookup_artist(artist.id.clone()))?; + let artist = apis + .musicbrainz + .lookup_artist(artist.id.clone(), rthandle)?; let mut image_1 = None; let mut image_2 = None; @@ -811,13 +813,13 @@ fn apply_musicbrainz_recording( WIKIDATA => { let url = rel.url.as_ref().unwrap().resource.clone(); if let Some(id) = url.strip_prefix("https://www.wikidata.org/wiki/") { - if let Some(filename) = rthandle - .block_on(apis.wikidata.query_image_path(id.to_owned()))? + if let Some(filename) = + apis.wikidata.query_image_path(id.to_owned(), rthandle)? { - let path = rthandle.block_on( - apis.wikimedia_commons.image_by_filename(filename), - )?; - image_1 = Some(Asset(path.0)); + let path = apis + .wikimedia_commons + .image_by_filename(filename, rthandle)?; + image_1 = Some(Picture(path.0)); } } } @@ -825,10 +827,8 @@ fn apply_musicbrainz_recording( let url = rel.url.as_ref().unwrap().resource.clone(); if let Some(id) = url.strip_prefix("https://vgmdb.net/artist/") { let id = id.parse::<u64>().context("parse vgmdb id")?; - if let Some(path) = - rthandle.block_on(apis.vgmdb.get_artist_image(id))? - { - image_2 = Some(Asset(path.0)); + if let Some(path) = apis.vgmdb.get_artist_image(id, rthandle)? { + image_2 = Some(Picture(path.0)); } } } @@ -843,10 +843,9 @@ fn apply_musicbrainz_recording( let headshot = match image_1.or(image_2) { Some(x) => x, - None => Asset( - cache_file("person-headshot-fallback", &artist.sort_name, |mut file| { - generate_fallback(&artist.sort_name, &mut file)?; - Ok(()) + None => Picture( + cache_store(CacheKey::new_image(("fallback", &artist.sort_name)), || { + generate_fallback(&artist.sort_name) })? .0, ), @@ -879,12 +878,8 @@ fn apply_trakt_tmdb( ) -> Result<()> { let trakt_id: u64 = trakt_id.parse().context("parse trakt id")?; if let (Some(trakt), Some(tmdb)) = (&apis.trakt, &apis.tmdb) { - let data = rthandle - .block_on(trakt.lookup(trakt_kind, trakt_id)) - .context("trakt lookup")?; - let people = rthandle - .block_on(trakt.people(trakt_kind, trakt_id)) - .context("trakt people lookup")?; + let data = trakt.lookup(trakt_kind, trakt_id, rthandle)?; + let people = trakt.people(trakt_kind, trakt_id, rthandle)?; let mut people_map = BTreeMap::<CreditCategory, Vec<Appearance>>::new(); for p in people.cast.iter() { @@ -903,29 +898,24 @@ fn apply_trakt_tmdb( let mut backdrop = None; let mut poster = None; if let Some(tmdb_id) = data.ids.tmdb { - let data = rthandle - .block_on(tmdb.details( - match trakt_kind { - TraktKind::Movie => TmdbKind::Movie, - TraktKind::Show => TmdbKind::Tv, - _ => TmdbKind::Movie, - }, - tmdb_id, - )) - .context("tmdb details")?; + let data = tmdb.details( + match trakt_kind { + TraktKind::Movie => TmdbKind::Movie, + TraktKind::Show => TmdbKind::Tv, + _ => TmdbKind::Movie, + }, + tmdb_id, + rthandle, + )?; tmdb_data = Some(data.clone()); if let Some(path) = &data.backdrop_path { - let im = rthandle - .block_on(tmdb.image(path)) - .context("tmdb backdrop image")?; - backdrop = Some(Asset(im.0)); + let im = tmdb.image(path, rthandle).context("tmdb backdrop image")?; + backdrop = Some(Picture(im.0)); } if let Some(path) = &data.poster_path { - let im = rthandle - .block_on(tmdb.image(path)) - .context("tmdb poster image")?; - poster = Some(Asset(im.0)); + let im = tmdb.image(path, rthandle).context("tmdb poster image")?; + poster = Some(Picture(im.0)); } // for p in people_map.values_mut().flatten() { @@ -960,7 +950,7 @@ fn apply_trakt_tmdb( node.ratings.insert(RatingType::Trakt, *rating); } if let Some(poster) = poster { - node.pictures.insert(PictureSlot::Poster, poster); + node.pictures.insert(PictureSlot::Cover, poster); } if let Some(backdrop) = backdrop { node.pictures.insert(PictureSlot::Backdrop, backdrop); diff --git a/import/src/musicbrainz.rs b/import/src/musicbrainz.rs index 934a9e8..92df703 100644 --- a/import/src/musicbrainz.rs +++ b/import/src/musicbrainz.rs @@ -6,7 +6,7 @@ use crate::USER_AGENT; use anyhow::{Context, Result}; -use jellycache::async_cache_memory; +use jellycache::{cache_memory, CacheContentType, CacheKey}; use log::info; use reqwest::{ header::{HeaderMap, HeaderName, HeaderValue}, @@ -15,6 +15,7 @@ use reqwest::{ use serde::{Deserialize, Serialize}; use std::{collections::BTreeMap, sync::Arc, time::Duration}; use tokio::{ + runtime::Handle, sync::Semaphore, time::{sleep_until, Instant}, }; @@ -221,95 +222,108 @@ impl MusicBrainz { } } - pub async fn lookup_recording(&self, id: String) -> Result<Arc<MbRecordingRel>> { - async_cache_memory("api-musicbrainz-recording", id.clone(), || async move { - let _permit = self.rate_limit.clone().acquire_owned().await?; - let permit_drop_ts = Instant::now() + Duration::from_secs(Self::MAX_PAR_REQ as u64); - info!("recording lookup: {id}"); + pub fn lookup_recording(&self, id: String, rt: &Handle) -> Result<Arc<MbRecordingRel>> { + cache_memory( + CacheKey::new( + CacheContentType::Json, + ("musicbrainz-recording-lookup", &id), + ), + move || { + rt.block_on(async { + let _permit = self.rate_limit.clone().acquire_owned().await?; + let permit_drop_ts = + Instant::now() + Duration::from_secs(Self::MAX_PAR_REQ as u64); + info!("recording lookup: {id}"); - let inc = [ - "isrcs", - "artists", - "area-rels", - "artist-rels", - "event-rels", - "genre-rels", - "instrument-rels", - "label-rels", - "place-rels", - "recording-rels", - "release-rels", - "release-group-rels", - "series-rels", - "url-rels", - "work-rels", - ] - .join("+"); + let inc = [ + "isrcs", + "artists", + "area-rels", + "artist-rels", + "event-rels", + "genre-rels", + "instrument-rels", + "label-rels", + "place-rels", + "recording-rels", + "release-rels", + "release-group-rels", + "series-rels", + "url-rels", + "work-rels", + ] + .join("+"); - let resp = self - .client - .get(format!( - "https://musicbrainz.org/ws/2/recording/{id}?inc={inc}" - )) - .send() - .await? - .error_for_status()? - .json::<MbRecordingRel>() - .await?; + let resp = self + .client + .get(format!( + "https://musicbrainz.org/ws/2/recording/{id}?inc={inc}" + )) + .send() + .await? + .error_for_status()? + .json::<MbRecordingRel>() + .await?; - tokio::task::spawn(async move { - sleep_until(permit_drop_ts).await; - drop(_permit); - }); + tokio::task::spawn(async move { + sleep_until(permit_drop_ts).await; + drop(_permit); + }); - Ok(resp) - }) - .await + Ok(resp) + }) + }, + ) .context("musicbrainz recording lookup") } - pub async fn lookup_artist(&self, id: String) -> Result<Arc<MbArtistRel>> { - async_cache_memory("api-musicbrainz-artist", id.clone(), || async move { - let _permit = self.rate_limit.clone().acquire_owned().await?; - let permit_drop_ts = Instant::now() + Duration::from_secs(Self::MAX_PAR_REQ as u64); - info!("artist lookup: {id}"); + pub fn lookup_artist(&self, id: String, rt: &Handle) -> Result<Arc<MbArtistRel>> { + cache_memory( + CacheKey::new(CacheContentType::Json, ("musicbrainz-artist-lookup", &id)), + move || { + rt.block_on(async { + let _permit = self.rate_limit.clone().acquire_owned().await?; + let permit_drop_ts = + Instant::now() + Duration::from_secs(Self::MAX_PAR_REQ as u64); + info!("artist lookup: {id}"); - let inc = [ - "area-rels", - "artist-rels", - "event-rels", - "genre-rels", - "instrument-rels", - "label-rels", - "place-rels", - "recording-rels", - "release-rels", - "release-group-rels", - "series-rels", - "url-rels", - "work-rels", - ] - .join("+"); + let inc = [ + "area-rels", + "artist-rels", + "event-rels", + "genre-rels", + "instrument-rels", + "label-rels", + "place-rels", + "recording-rels", + "release-rels", + "release-group-rels", + "series-rels", + "url-rels", + "work-rels", + ] + .join("+"); - let resp = self - .client - .get(format!( - "https://musicbrainz.org/ws/2/artist/{id}?inc={inc}" - )) - .send() - .await? - .error_for_status()? - .json::<MbArtistRel>() - .await?; + let resp = self + .client + .get(format!( + "https://musicbrainz.org/ws/2/artist/{id}?inc={inc}" + )) + .send() + .await? + .error_for_status()? + .json::<MbArtistRel>() + .await?; - tokio::task::spawn(async move { - sleep_until(permit_drop_ts).await; - drop(_permit); - }); + tokio::task::spawn(async move { + sleep_until(permit_drop_ts).await; + drop(_permit); + }); - Ok(resp) - }) - .await + Ok(resp) + }) + }, + ) .context("musicbrainz artist lookup") } } diff --git a/import/src/tmdb.rs b/import/src/tmdb.rs index 414058f..d4a0c25 100644 --- a/import/src/tmdb.rs +++ b/import/src/tmdb.rs @@ -4,8 +4,8 @@ Copyright (C) 2025 metamuffin <metamuffin.org> */ use crate::USER_AGENT; -use anyhow::{anyhow, bail, Context}; -use jellycache::{async_cache_file, async_cache_memory, CachePath}; +use anyhow::{anyhow, bail, Context, Result}; +use jellycache::{cache_memory, cache_store, CacheKey}; use jellycommon::chrono::{format::Parsed, Utc}; use log::info; use reqwest::{ @@ -14,7 +14,7 @@ use reqwest::{ }; use serde::{Deserialize, Serialize}; use std::{fmt::Display, sync::Arc}; -use tokio::io::AsyncWriteExt; +use tokio::runtime::Handle; pub struct Tmdb { client: Client, @@ -44,100 +44,109 @@ impl Tmdb { key: api_key.to_owned(), } } - pub async fn search(&self, kind: TmdbKind, query: &str) -> anyhow::Result<Arc<TmdbQuery>> { - async_cache_memory("api-tmdb-search", (kind, query), || async move { - info!("searching tmdb: {query:?}"); - Ok(self - .client - .get(format!( - "https://api.themoviedb.org/3/search/{kind}?query={}?api_key={}", - query.replace(" ", "+"), - self.key - )) - .send() - .await? - .error_for_status()? - .json::<TmdbQuery>() - .await?) - }) - .await + pub fn search(&self, kind: TmdbKind, query: &str, rt: &Handle) -> Result<Arc<TmdbQuery>> { + cache_memory( + CacheKey::new_json(("api-tmdb-search", kind, query)), + move || { + rt.block_on(async { + info!("searching tmdb: {query:?}"); + Ok(self + .client + .get(format!( + "https://api.themoviedb.org/3/search/{kind}?query={}?api_key={}", + query.replace(" ", "+"), + self.key + )) + .send() + .await? + .error_for_status()? + .json::<TmdbQuery>() + .await?) + }) + }, + ) .context("tmdb search") } - pub async fn details(&self, kind: TmdbKind, id: u64) -> anyhow::Result<Arc<TmdbDetails>> { - async_cache_memory("api-tmdb-details", (kind, id), || async move { - info!("fetching details: {id:?}"); - Ok(self - .client - .get(format!( - "https://api.themoviedb.org/3/{kind}/{id}?api_key={}", - self.key, - )) - .send() - .await? - .error_for_status()? - .json() - .await?) + pub fn details(&self, kind: TmdbKind, id: u64, rt: &Handle) -> Result<Arc<TmdbDetails>> { + cache_memory(CacheKey::new_json(("tmdb-details", kind, id)), move || { + rt.block_on(async { + info!("fetching details: {id:?}"); + Ok(self + .client + .get(format!( + "https://api.themoviedb.org/3/{kind}/{id}?api_key={}", + self.key, + )) + .send() + .await? + .error_for_status()? + .json() + .await?) + }) }) - .await .context("tmdb details") } - pub async fn person_image(&self, id: u64) -> anyhow::Result<Arc<TmdbPersonImage>> { - async_cache_memory("api-tmdb-search", id, || async move { - Ok(self - .client - .get(format!( - "https://api.themoviedb.org/3/person/{id}/images?api_key={}", - self.key, - )) - .send() - .await? - .error_for_status()? - .json() - .await?) + pub fn person_image(&self, id: u64, rt: &Handle) -> Result<Arc<TmdbPersonImage>> { + cache_memory(CacheKey::new_image(("tmdb-person-image", id)), move || { + rt.block_on(async { + Ok(self + .client + .get(format!( + "https://api.themoviedb.org/3/person/{id}/images?api_key={}", + self.key, + )) + .send() + .await? + .error_for_status()? + .json() + .await?) + }) }) - .await .context("tmdb person images") } - pub async fn image(&self, path: &str) -> anyhow::Result<CachePath> { - async_cache_file("api-tmdb-image", path, |mut file| async move { - info!("downloading image {path:?}"); - let mut res = self - .image_client - .get(format!("https://image.tmdb.org/t/p/original{path}")) - .send() - .await? - .error_for_status()?; - while let Some(chunk) = res.chunk().await? { - file.write_all(&chunk).await?; - } - Ok(()) + pub fn image(&self, path: &str, rt: &Handle) -> Result<CacheKey> { + cache_store(CacheKey::new_image(("tmdb-image", path)), move || { + rt.block_on(async { + info!("downloading image {path:?}"); + Ok(self + .image_client + .get(format!("https://image.tmdb.org/t/p/original{path}")) + .send() + .await? + .error_for_status()? + .bytes() + .await? + .to_vec()) + }) }) - .await .context("tmdb image download") } - pub async fn episode_details( + pub fn episode_details( &self, series_id: u64, season: usize, episode: usize, - ) -> anyhow::Result<Arc<TmdbEpisode>> { - async_cache_memory("api-tmdb-episode-details", (series_id,season,episode), || async move { - info!("tmdb episode details {series_id} S={season} E={episode}"); - Ok(self - .image_client - .get(format!("https://api.themoviedb.org/3/tv/{series_id}/season/{season}/episode/{episode}?api_key={}", self.key)) - .send() - .await? - .error_for_status()? - .json() - .await?) + rt: &Handle, + ) -> Result<Arc<TmdbEpisode>> { + cache_memory(CacheKey::new_json(("tmdb-episode-details", series_id, season, episode)), move || { + rt.block_on(async { + info!("tmdb episode details {series_id} S={season} E={episode}"); + Ok(self + .image_client + .get(format!("https://api.themoviedb.org/3/tv/{series_id}/season/{season}/episode/{episode}?api_key={}", self.key)) + .send() + .await? + .error_for_status()? + .json() + .await?) + }) }) - .await.context("tmdb episode details") + .context("tmdb episode details") } } -pub fn parse_release_date(d: &str) -> anyhow::Result<Option<i64>> { +pub fn parse_release_date(d: &str) -> Result<Option<i64>> { if d.is_empty() { return Ok(None); } else if d.len() < 10 { diff --git a/import/src/trakt.rs b/import/src/trakt.rs index 3f0ad47..1640ca5 100644 --- a/import/src/trakt.rs +++ b/import/src/trakt.rs @@ -5,7 +5,7 @@ */ use crate::USER_AGENT; use anyhow::Context; -use jellycache::async_cache_memory; +use jellycache::{cache_memory, CacheKey}; use jellycommon::{Appearance, CreditCategory, NodeID}; use log::info; use reqwest::{ @@ -14,6 +14,7 @@ use reqwest::{ }; use serde::{Deserialize, Serialize}; use std::{collections::BTreeMap, fmt::Display, sync::Arc}; +use tokio::runtime::Handle; pub struct Trakt { client: Client, @@ -45,76 +46,97 @@ impl Trakt { Self { client } } - pub async fn search( + pub fn search( &self, kinds: &[TraktKind], query: &str, + rt: &Handle, ) -> anyhow::Result<Arc<Vec<TraktSearchResult>>> { - async_cache_memory("api-trakt-lookup", (kinds, query), || async move { - let url = format!( - "https://api.trakt.tv/search/{}?query={}&extended=full", - kinds - .iter() - .map(|t| t.singular()) - .collect::<Vec<_>>() - .join(","), - urlencoding::encode(query), - ); - let res = self.client.get(url).send().await?.error_for_status()?; - Ok(res.json().await?) + cache_memory(CacheKey::new_json(("trakt-lookup", query)), move || { + rt.block_on(async { + let url = format!( + "https://api.trakt.tv/search/{}?query={}&extended=full", + kinds + .iter() + .map(|t| t.singular()) + .collect::<Vec<_>>() + .join(","), + urlencoding::encode(query), + ); + let res = self.client.get(url).send().await?.error_for_status()?; + Ok(res.json().await?) + }) }) - .await .context("trakt search") } - pub async fn lookup(&self, kind: TraktKind, id: u64) -> anyhow::Result<Arc<TraktMediaObject>> { - async_cache_memory("api-trakt-lookup", (kind, id), || async move { - info!("trakt lookup {kind:?}:{id:?}"); - let url = format!("https://api.trakt.tv/{}/{id}?extended=full", kind.plural()); - let res = self.client.get(url).send().await?.error_for_status()?; - Ok(res.json().await?) + pub fn lookup( + &self, + kind: TraktKind, + id: u64, + rt: &Handle, + ) -> anyhow::Result<Arc<TraktMediaObject>> { + cache_memory(CacheKey::new_json(("trakt-lookup", kind, id)), move || { + rt.block_on(async { + info!("trakt lookup {kind:?}:{id:?}"); + let url = format!("https://api.trakt.tv/{}/{id}?extended=full", kind.plural()); + let res = self.client.get(url).send().await?.error_for_status()?; + Ok(res.json().await?) + }) }) - .await .context("trakt lookup") } - pub async fn people(&self, kind: TraktKind, id: u64) -> anyhow::Result<Arc<TraktPeople>> { - async_cache_memory("api-trakt-people", (kind, id), || async move { - info!("trakt people {kind:?}:{id:?}"); - let url = format!( - "https://api.trakt.tv/{}/{id}/people?extended=full", - kind.plural() - ); - let res = self.client.get(url).send().await?.error_for_status()?; - Ok(res.json().await?) + pub fn people( + &self, + kind: TraktKind, + id: u64, + rt: &Handle, + ) -> anyhow::Result<Arc<TraktPeople>> { + cache_memory(CacheKey::new_json(("trakt-people", kind, id)), move || { + rt.block_on(async { + info!("trakt people {kind:?}:{id:?}"); + let url = format!( + "https://api.trakt.tv/{}/{id}/people?extended=full", + kind.plural() + ); + let res = self.client.get(url).send().await?.error_for_status()?; + Ok(res.json().await?) + }) }) - .await .context("trakt people") } - pub async fn show_seasons(&self, id: u64) -> anyhow::Result<Arc<Vec<TraktSeason>>> { - async_cache_memory("api-trakt-seasons", id, || async move { - info!("trakt seasons {id:?}"); - let url = format!("https://api.trakt.tv/shows/{id}/seasons?extended=full"); - let res = self.client.get(url).send().await?.error_for_status()?; - Ok(res.json().await?) + pub fn show_seasons(&self, id: u64, rt: &Handle) -> anyhow::Result<Arc<Vec<TraktSeason>>> { + cache_memory(CacheKey::new_json(("trakt-seasons", id)), move || { + rt.block_on(async { + info!("trakt seasons {id:?}"); + let url = format!("https://api.trakt.tv/shows/{id}/seasons?extended=full"); + let res = self.client.get(url).send().await?.error_for_status()?; + Ok(res.json().await?) + }) }) - .await .context("trakt show seasons") } - pub async fn show_season_episodes( + pub fn show_season_episodes( &self, id: u64, season: usize, + rt: &Handle, ) -> anyhow::Result<Arc<Vec<TraktEpisode>>> { - async_cache_memory("api-trakt-episodes", (id, season), || async move { - info!("trakt episodes {id:?} season={season}"); - let url = format!("https://api.trakt.tv/shows/{id}/seasons/{season}?extended=full"); - let res = self.client.get(url).send().await?.error_for_status()?; - Ok(res.json().await?) - }) - .await + cache_memory( + CacheKey::new_json(("trakt-episodes", id, season)), + move || { + rt.block_on(async { + info!("trakt episodes {id:?} season={season}"); + let url = + format!("https://api.trakt.tv/shows/{id}/seasons/{season}?extended=full"); + let res = self.client.get(url).send().await?.error_for_status()?; + Ok(res.json().await?) + }) + }, + ) .context("trakt show season episodes") } } diff --git a/import/src/vgmdb.rs b/import/src/vgmdb.rs index 20a0038..4e37ba3 100644 --- a/import/src/vgmdb.rs +++ b/import/src/vgmdb.rs @@ -6,7 +6,7 @@ use crate::USER_AGENT; use anyhow::{Context, Result}; -use jellycache::{async_cache_file, async_cache_memory, CachePath}; +use jellycache::{cache_memory, cache_store, CacheContentType, CacheKey}; use log::info; use regex::Regex; use reqwest::{ @@ -18,7 +18,7 @@ use std::{ time::Duration, }; use tokio::{ - io::AsyncWriteExt, + runtime::Handle, sync::Semaphore, time::{sleep_until, Instant}, }; @@ -59,18 +59,26 @@ impl Vgmdb { } } - pub async fn get_artist_image(&self, id: u64) -> Result<Option<CachePath>> { - if let Some(url) = self.get_artist_image_url(id).await? { + pub fn get_artist_image(&self, id: u64, rt: &Handle) -> Result<Option<CacheKey>> { + if let Some(url) = self.get_artist_image_url(id, rt)? { Ok(Some( - async_cache_file("api-vgmdb-media", url.clone(), |mut file| async move { - info!("downloading image {url:?}"); - let mut res = self.client.get(url).send().await?.error_for_status()?; - while let Some(chunk) = res.chunk().await? { - file.write_all(&chunk).await?; - } - Ok(()) - }) - .await + cache_store( + CacheKey::new_image(("vgmdb-artist-media", &url)), + move || { + rt.block_on(async { + info!("downloading image {url:?}"); + Ok(self + .client + .get(url) + .send() + .await? + .error_for_status()? + .bytes() + .await? + .to_vec()) + }) + }, + ) .context("vgmdb media download")?, )) } else { @@ -78,9 +86,9 @@ impl Vgmdb { } } - pub async fn get_artist_image_url(&self, id: u64) -> Result<Option<String>> { - let html = self.scrape_artist_page(id).await?; - if let Some(cap) = RE_IMAGE_URL_FROM_HTML.captures(&html) { + pub fn get_artist_image_url(&self, id: u64, rt: &Handle) -> Result<Option<String>> { + let html = self.scrape_artist_page(id, rt)?; + if let Some(cap) = RE_IMAGE_URL_FROM_HTML.captures(&str::from_utf8(&html).unwrap()) { if let Some(url) = cap.name("url").map(|m| m.as_str()) { return Ok(Some(url.to_string())); } @@ -88,29 +96,34 @@ impl Vgmdb { Ok(None) } - pub async fn scrape_artist_page(&self, id: u64) -> Result<Arc<String>> { - async_cache_memory("api-vgmdb-artist", id, || async move { - let _permit = self.rate_limit.clone().acquire_owned().await?; - let permit_drop_ts = Instant::now() + Duration::from_secs(1); - info!("scrape artist: {id}"); + pub fn scrape_artist_page(&self, id: u64, rt: &Handle) -> Result<Arc<Vec<u8>>> { + cache_memory( + CacheKey::new(CacheContentType::Unknown, ("vgmdb-artist-page", id)), + move || { + rt.block_on(async { + let _permit = self.rate_limit.clone().acquire_owned().await?; + let permit_drop_ts = Instant::now() + Duration::from_secs(1); + info!("scrape artist: {id}"); - let resp = self - .client - .get(format!("https://vgmdb.net/artist/{id}")) - .send() - .await? - .error_for_status()? - .text() - .await?; + let resp = self + .client + .get(format!("https://vgmdb.net/artist/{id}")) + .send() + .await? + .error_for_status()? + .bytes() + .await? + .to_vec(); - tokio::task::spawn(async move { - sleep_until(permit_drop_ts).await; - drop(_permit); - }); + tokio::task::spawn(async move { + sleep_until(permit_drop_ts).await; + drop(_permit); + }); - Ok(resp) - }) - .await + Ok(resp) + }) + }, + ) .context("vgmdb artist page scrape") } } diff --git a/import/src/wikidata.rs b/import/src/wikidata.rs index 71eef9a..40077b9 100644 --- a/import/src/wikidata.rs +++ b/import/src/wikidata.rs @@ -6,26 +6,27 @@ use crate::USER_AGENT; use anyhow::{bail, Context, Result}; -use jellycache::async_cache_memory; +use jellycache::{cache_memory, CacheKey}; use log::info; use reqwest::{ header::{HeaderMap, HeaderName, HeaderValue}, Client, ClientBuilder, }; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use serde_json::Value; -use std::collections::BTreeMap; +use std::{collections::BTreeMap, sync::Arc}; +use tokio::runtime::Handle; pub struct Wikidata { client: Client, } -#[derive(Debug, Deserialize, Clone)] +#[derive(Debug, Deserialize, Serialize, Clone)] pub struct WikidataResponse { entities: BTreeMap<String, WikidataEntity>, } -#[derive(Debug, Deserialize, Clone)] +#[derive(Debug, Deserialize, Serialize, Clone)] pub struct WikidataEntity { pub pageid: u64, pub ns: u64, @@ -36,7 +37,7 @@ pub struct WikidataEntity { pub id: String, pub claims: BTreeMap<String, Vec<WikidataClaim>>, } -#[derive(Debug, Deserialize, Clone)] +#[derive(Debug, Deserialize, Serialize, Clone)] pub struct WikidataClaim { pub r#type: String, pub id: String, @@ -44,7 +45,7 @@ pub struct WikidataClaim { pub mainsnak: WikidataSnak, } -#[derive(Debug, Deserialize, Clone)] +#[derive(Debug, Deserialize, Serialize, Clone)] pub struct WikidataSnak { pub snaktype: String, pub property: String, @@ -53,7 +54,7 @@ pub struct WikidataSnak { pub datatype: String, } -#[derive(Debug, Deserialize, Clone)] +#[derive(Debug, Deserialize, Serialize, Clone)] pub struct WikidataValue { pub value: Value, pub r#type: String, @@ -87,8 +88,8 @@ impl Wikidata { Self { client } } - pub async fn query_image_path(&self, id: String) -> Result<Option<String>> { - let response = self.query(id.clone()).await?; + pub fn query_image_path(&self, id: String, rt: &Handle) -> Result<Option<String>> { + let response = self.query(id.clone(), rt)?; if let Some(entity) = response.entities.get(&id) { if let Some(images) = entity.claims.get(properties::IMAGE) { for image in images { @@ -106,24 +107,20 @@ impl Wikidata { Ok(None) } - pub async fn query(&self, id: String) -> Result<WikidataResponse> { - let json = async_cache_memory("api-wikidata", id.clone(), || async move { - info!("entity query: {id}"); - - let resp = self - .client - .get(format!("https://www.wikidata.org/entity/{id}")) - .send() - .await? - .error_for_status()? - .text() - .await?; - - Ok(resp) + pub fn query(&self, id: String, rt: &Handle) -> Result<Arc<WikidataResponse>> { + cache_memory(CacheKey::new_json(("wikidata", &id)), move || { + rt.block_on(async { + info!("entity query: {id}"); + Ok(self + .client + .get(format!("https://www.wikidata.org/entity/{id}")) + .send() + .await? + .error_for_status()? + .json() + .await?) + }) }) - .await - .context("wikidata entity")?; - - serde_json::from_str(&json).context("parse wikidata entity") + .context("wikidata entity") } } diff --git a/import/src/wikimedia_commons.rs b/import/src/wikimedia_commons.rs index f8d8f53..0d716f0 100644 --- a/import/src/wikimedia_commons.rs +++ b/import/src/wikimedia_commons.rs @@ -6,13 +6,13 @@ use crate::USER_AGENT; use anyhow::{Context, Result}; -use jellycache::{async_cache_file, CachePath}; +use jellycache::{cache_store, CacheKey}; use reqwest::{ header::{HeaderMap, HeaderName, HeaderValue}, redirect::Policy, Client, ClientBuilder, }; -use tokio::io::AsyncWriteExt; +use tokio::runtime::Handle; pub struct WikimediaCommons { client: Client, @@ -36,28 +36,26 @@ impl WikimediaCommons { Self { client } } - pub async fn image_by_filename(&self, filename: String) -> Result<CachePath> { - async_cache_file( - "api-wikimedia-commons-image", - filename.clone(), - |mut file| async move { - let mut res = self - .client - .get(format!( - "https://commons.wikimedia.org/wiki/Special:FilePath/{}", - filename.replace(" ", "_") - )) - .send() - .await? - .error_for_status()?; - - while let Some(chunk) = res.chunk().await? { - file.write_all(&chunk).await?; - } - Ok(()) + pub fn image_by_filename(&self, filename: String, rt: &Handle) -> Result<CacheKey> { + cache_store( + CacheKey::new_image(("wikimedia-commons-image", &filename)), + move || { + rt.block_on(async { + Ok(self + .client + .get(format!( + "https://commons.wikimedia.org/wiki/Special:FilePath/{}", + filename.replace(" ", "_") + )) + .send() + .await? + .error_for_status()? + .bytes() + .await? + .to_vec()) + }) }, ) - .await .context("mediawiki image by filename") } } diff --git a/logic/src/assets.rs b/logic/src/assets.rs index 068576b..e5cfada 100644 --- a/logic/src/assets.rs +++ b/logic/src/assets.rs @@ -6,125 +6,53 @@ use crate::{DATABASE, session::Session}; use anyhow::{Result, anyhow}; -use jellycommon::{Asset, LocalTrack, NodeID, SourceTrackKind, TrackSource}; +use jellycommon::{NodeID, Picture, SourceTrackKind, TrackSource}; -// pub fn get_node_backdrop(_session: &Session, id: NodeID) -> Result<Asset> { -// // TODO perm -// let node = DATABASE -// .get_node(id)? -// .ok_or(anyhow!("node does not exist"))?; +pub async fn get_node_thumbnail(_session: &Session, id: NodeID, t: f64) -> Result<Picture> { + let node = DATABASE + .get_node(id)? + .ok_or(anyhow!("node does not exist"))?; -// let mut asset = node.backdrop.clone(); -// if asset.is_none() -// && let Some(parent) = node.parents.last().copied() -// { -// let parent = DATABASE -// .get_node(parent)? -// .ok_or(anyhow!("node does not exist"))?; -// asset = parent.backdrop.clone(); -// }; -// Ok(asset.unwrap_or_else(|| { -// AssetInner::Assets(format!("fallback-{:?}.avif", node.kind).into()).ser() -// })) -// } -// pub fn get_node_poster(_session: &Session, id: NodeID) -> Result<Asset> { -// // TODO perm -// let node = DATABASE -// .get_node(id)? -// .ok_or(anyhow!("node does not exist"))?; + let media = node.media.as_ref().ok_or(anyhow!("no media"))?; + let (thumb_track_index, _thumb_track) = media + .tracks + .iter() + .enumerate() + .find(|(_i, t)| matches!(t.kind, SourceTrackKind::Video { .. })) + .ok_or(anyhow!("no video track to create a thumbnail of"))?; + let source = media + .tracks + .get(thumb_track_index) + .ok_or(anyhow!("no source"))?; + let thumb_track_source = source.source.clone(); -// let mut asset = node.poster.clone(); -// if asset.is_none() -// && let Some(parent) = node.parents.last().copied() -// { -// let parent = DATABASE -// .get_node(parent)? -// .ok_or(anyhow!("node does not exist"))?; -// asset = parent.poster.clone(); -// }; -// Ok(asset.unwrap_or_else(|| { -// AssetInner::Assets(format!("fallback-{:?}.avif", node.kind).into()).ser() -// })) -// } + if t < 0. || t > media.duration { + Err(anyhow!("thumbnail instant not within media duration"))? + } -// pub fn get_node_person_asset( -// _session: &Session, -// id: NodeID, -// group: PeopleGroup, -// index: usize, -// ) -> Result<Asset> { -// // TODO perm + let step = 8.; + let t = (t / step).floor() * step; -// let node = DATABASE -// .get_node(id)? -// .ok_or(anyhow!("node does not exist"))?; -// let app = node -// .credits -// .get(&group) -// .ok_or(anyhow!("group has no members"))? -// .get(index) -// .ok_or(anyhow!("person does not exist"))?; + let asset = match thumb_track_source { + TrackSource::Local(path, _) => jellytranscoder::thumbnail::create_thumbnail(&path, t)?, + TrackSource::Remote(_) => { + // // TODO in the new system this is preferrably a property of node ext for regular fed + // let session = fed + // .get_session( + // thumb_track + // .federated + // .last() + // .ok_or(anyhow!("federation broken"))?, + // ) + // .await?; -// let asset = app -// .person -// .headshot -// .to_owned() -// .unwrap_or(AssetInner::Assets("fallback-Person.avif".into()).ser()); + // async_cache_file("fed-thumb", (id.0, t as i64), |out| { + // session.node_thumbnail(out, id.0.into(), 2048, t) + // }) + // .await? + todo!() + } + }; -// Ok(asset) -// } - -// pub async fn get_node_thumbnail(_session: &Session, id: NodeID, t: f64) -> Result<Asset> { -// let node = DATABASE -// .get_node(id)? -// .ok_or(anyhow!("node does not exist"))?; - -// let media = node.media.as_ref().ok_or(anyhow!("no media"))?; -// let (thumb_track_index, _thumb_track) = media -// .tracks -// .iter() -// .enumerate() -// .find(|(_i, t)| matches!(t.kind, SourceTrackKind::Video { .. })) -// .ok_or(anyhow!("no video track to create a thumbnail of"))?; -// let source = media -// .tracks -// .get(thumb_track_index) -// .ok_or(anyhow!("no source"))?; -// let thumb_track_source = source.source.clone(); - -// if t < 0. || t > media.duration { -// Err(anyhow!("thumbnail instant not within media duration"))? -// } - -// let step = 8.; -// let t = (t / step).floor() * step; - -// let asset = match thumb_track_source { -// TrackSource::Local(a) => { -// let AssetInner::LocalTrack(LocalTrack { path, .. }) = AssetInner::deser(&a.0)? else { -// return Err(anyhow!("track set to wrong asset type")); -// }; -// // the track selected might be different from thumb_track -// jellytranscoder::thumbnail::create_thumbnail(&path, t).await? -// } -// TrackSource::Remote(_) => { -// // // TODO in the new system this is preferrably a property of node ext for regular fed -// // let session = fed -// // .get_session( -// // thumb_track -// // .federated -// // .last() -// // .ok_or(anyhow!("federation broken"))?, -// // ) -// // .await?; - -// // async_cache_file("fed-thumb", (id.0, t as i64), |out| { -// // session.node_thumbnail(out, id.0.into(), 2048, t) -// // }) -// // .await? -// todo!() -// } -// }; - -// Ok(AssetInner::Cache(asset).ser()) -// } + Ok(Picture(asset.0)) +} diff --git a/logic/src/filter_sort.rs b/logic/src/filter_sort.rs index d3244af..af88333 100644 --- a/logic/src/filter_sort.rs +++ b/logic/src/filter_sort.rs @@ -70,10 +70,12 @@ pub fn filter_and_sort_nodes( SortProperty::RatingMetacritic => nodes.sort_by_cached_key(|(n, _)| { SortAnyway(*n.ratings.get(&RatingType::Metacritic).unwrap_or(&0.)) }), - SortProperty::RatingImdb => nodes - .sort_by_cached_key(|(n, _)| SortAnyway(*n.ratings.get(&RatingType::Imdb).unwrap_or(&0.))), - SortProperty::RatingTmdb => nodes - .sort_by_cached_key(|(n, _)| SortAnyway(*n.ratings.get(&RatingType::Tmdb).unwrap_or(&0.))), + SortProperty::RatingImdb => nodes.sort_by_cached_key(|(n, _)| { + SortAnyway(*n.ratings.get(&RatingType::Imdb).unwrap_or(&0.)) + }), + SortProperty::RatingTmdb => nodes.sort_by_cached_key(|(n, _)| { + SortAnyway(*n.ratings.get(&RatingType::Tmdb).unwrap_or(&0.)) + }), SortProperty::RatingYoutubeViews => nodes.sort_by_cached_key(|(n, _)| { SortAnyway(*n.ratings.get(&RatingType::YoutubeViews).unwrap_or(&0.)) }), diff --git a/remuxer/src/demuxers/flac.rs b/remuxer/src/demuxers/flac.rs index c309589..18ed7f8 100644 --- a/remuxer/src/demuxers/flac.rs +++ b/remuxer/src/demuxers/flac.rs @@ -252,8 +252,6 @@ impl Demuxer for FlacDemuxer { let mut crc_buf = [0u8; 1]; self.reader.read_exact(&mut crc_buf)?; - - Ok(None) } } diff --git a/remuxer/src/muxers/matroska.rs b/remuxer/src/muxers/matroska.rs index e1216df..fd13bcc 100644 --- a/remuxer/src/muxers/matroska.rs +++ b/remuxer/src/muxers/matroska.rs @@ -13,18 +13,17 @@ use winter_matroska::{MatroskaFile, Segment}; fn write_fragment_shared(out: &mut dyn Write, mut segment: Segment, webm: bool) -> Result<()> { segment.info.muxing_app = concat!(env!("CARGO_PKG_NAME"), "-", env!("CARGO_PKG_VERSION")).to_string(); - if webm - && let Some(tracks) = &mut segment.tracks { - for track in &mut tracks.entries { - if let Some(video) = &mut track.video { - video.colour = None; - video.projection = None; - video.display_unit = 0; // pixels - video.display_width = Some(video.pixel_width); - video.display_height = Some(video.pixel_height); - } + if webm && let Some(tracks) = &mut segment.tracks { + for track in &mut tracks.entries { + if let Some(video) = &mut track.video { + video.colour = None; + video.projection = None; + video.display_unit = 0; // pixels + video.display_width = Some(video.pixel_width); + video.display_height = Some(video.pixel_height); } } + } let file = MatroskaFile { ebml_header: EbmlHeader { diff --git a/server/src/api.rs b/server/src/api.rs index 217cd9f..2cfdbbd 100644 --- a/server/src/api.rs +++ b/server/src/api.rs @@ -54,7 +54,6 @@ pub fn r_api_account_login(data: Json<CreateSessionParams>) -> MyResult<Value> { Ok(json!(token)) } - #[get("/nodes_modified?<since>")] pub fn r_nodes_modified_since(session: A<Session>, since: u64) -> MyResult<Json<Vec<NodeID>>> { let nodes = get_nodes_modified_since(&session.0, since)?; diff --git a/server/src/compat/jellyfin/mod.rs b/server/src/compat/jellyfin/mod.rs index 27df1aa..b18d304 100644 --- a/server/src/compat/jellyfin/mod.rs +++ b/server/src/compat/jellyfin/mod.rs @@ -9,10 +9,10 @@ use crate::{helper::A, ui::error::MyResult}; use anyhow::anyhow; use jellycommon::{ api::{NodeFilterSort, SortOrder, SortProperty}, - routes::{u_asset, u_node_slug_backdrop, u_node_slug_poster}, + routes::{u_asset, u_node_image}, stream::{StreamContainer, StreamSpec}, user::{NodeUserData, WatchedState}, - MediaInfo, Node, NodeID, NodeKind, SourceTrack, SourceTrackKind, Visibility, + MediaInfo, Node, NodeID, NodeKind, PictureSlot, SourceTrack, SourceTrackKind, Visibility, }; use jellylogic::{ login::login_logic, @@ -169,7 +169,11 @@ pub fn r_jellyfin_items_image_primary( tag: String, ) -> Redirect { if tag == "poster" { - Redirect::permanent(u_node_slug_poster(id, fillWidth.unwrap_or(1024))) + Redirect::permanent(u_node_image( + id, + PictureSlot::Cover, + fillWidth.unwrap_or(1024), + )) } else { Redirect::permanent(u_asset(&tag, fillWidth.unwrap_or(1024))) } @@ -182,7 +186,11 @@ pub fn r_jellyfin_items_images_backdrop( id: &str, maxWidth: Option<usize>, ) -> Redirect { - Redirect::permanent(u_node_slug_backdrop(id, maxWidth.unwrap_or(1024))) + Redirect::permanent(u_node_image( + id, + PictureSlot::Backdrop, + maxWidth.unwrap_or(1024), + )) } #[get("/Items/<id>")] diff --git a/server/src/config.rs b/server/src/config.rs index b663c78..bcb3de6 100644 --- a/server/src/config.rs +++ b/server/src/config.rs @@ -5,6 +5,7 @@ */ use anyhow::{anyhow, Context, Result}; +use jellycache::init_cache; use jellylogic::init_database; use serde::Deserialize; use std::env::{args, var}; @@ -40,6 +41,7 @@ pub async fn load_config() -> Result<()> { *crate::CONF_PRELOAD.lock().unwrap() = Some(config.server); *jellyui::CONF_PRELOAD.lock().unwrap() = Some(config.ui); + init_cache()?; init_database()?; Ok(()) diff --git a/server/src/helper/cache.rs b/server/src/helper/cache.rs index d4c0595..93743e7 100644 --- a/server/src/helper/cache.rs +++ b/server/src/helper/cache.rs @@ -12,6 +12,7 @@ use rocket::{ }; use std::{ hash::{DefaultHasher, Hash, Hasher}, + io::Cursor, os::unix::fs::MetadataExt, path::Path, }; @@ -54,3 +55,17 @@ impl<'r> Responder<'r, 'static> for CacheControlFile { } } } + +pub struct CacheControlImage(pub Vec<u8>); +impl<'r> Responder<'r, 'static> for CacheControlImage { + fn respond_to(self, _req: &'r Request<'_>) -> response::Result<'static> { + Response::build() + .status(Status::Ok) + .header(Header::new( + "cache-control", + "private, immutable, maxage=86400", + )) + .sized_body(self.0.len(), Cursor::new(self.0)) + .ok() + } +} diff --git a/server/src/helper/mod.rs b/server/src/helper/mod.rs index a4e0e1f..6d1c834 100644 --- a/server/src/helper/mod.rs +++ b/server/src/helper/mod.rs @@ -10,6 +10,7 @@ pub mod filter_sort; pub mod language; pub mod node_id; pub mod session; +pub mod picture; use crate::ui::error::{MyError, MyResult}; use accept::Accept; diff --git a/server/src/helper/picture.rs b/server/src/helper/picture.rs new file mode 100644 index 0000000..d5887e3 --- /dev/null +++ b/server/src/helper/picture.rs @@ -0,0 +1,32 @@ +/* + This file is part of jellything (https://codeberg.org/metamuffin/jellything) + which is licensed under the GNU Affero General Public License (version 3); see /COPYING. + Copyright (C) 2025 metamuffin <metamuffin.org> +*/ + +use crate::helper::A; +use jellycommon::Picture; +use rocket::{ + http::uri::fmt::{FromUriParam, Path, UriDisplay}, + request::FromParam, +}; +use std::fmt::Write; +use std::str::FromStr; + +impl<'a> FromParam<'a> for A<Picture> { + type Error = (); + fn from_param(param: &'a str) -> Result<Self, Self::Error> { + Picture::from_str(param).map_err(|_| ()).map(A) + } +} +impl UriDisplay<Path> for A<Picture> { + fn fmt(&self, f: &mut rocket::http::uri::fmt::Formatter<'_, Path>) -> std::fmt::Result { + write!(f, "{}", self.0) + } +} +impl FromUriParam<Path, Picture> for A<Picture> { + type Target = A<Picture>; + fn from_uri_param(param: Picture) -> Self::Target { + A(param) + } +} diff --git a/server/src/routes.rs b/server/src/routes.rs index 9a35105..b777788 100644 --- a/server/src/routes.rs +++ b/server/src/routes.rs @@ -16,7 +16,7 @@ use crate::ui::{ r_admin_update_search, user::{r_admin_remove_user, r_admin_user, r_admin_user_permission, r_admin_users}, }, - assets::{r_asset, r_item_backdrop, r_item_poster, r_node_thumbnail, r_person_asset}, + assets::{r_image, r_item_poster, r_node_thumbnail}, error::{r_api_catch, r_catch}, home::r_home, items::r_items, @@ -29,10 +29,7 @@ use crate::ui::{ }; use crate::CONF; use crate::{ - api::{ - r_api_account_login, r_api_root, r_nodes_modified_since, - r_translations, r_version, - }, + api::{r_api_account_login, r_api_root, r_nodes_modified_since, r_translations, r_version}, compat::{ jellyfin::{ r_jellyfin_artists, r_jellyfin_branding_configuration, r_jellyfin_branding_css, @@ -136,7 +133,7 @@ pub fn build_rocket() -> Rocket<Build> { r_admin_user, r_admin_users, r_items, - r_asset, + r_image, r_assets_font, r_assets_js_map, r_assets_js, @@ -144,7 +141,6 @@ pub fn build_rocket() -> Rocket<Build> { r_favicon, r_home, r_index, - r_item_backdrop, r_item_poster, r_node, r_node_thumbnail, @@ -152,7 +148,6 @@ pub fn build_rocket() -> Rocket<Build> { r_node_userdata_rating, r_node_userdata_watched, r_node_userdata, - r_person_asset, r_player, r_playersync, r_search, diff --git a/server/src/ui/admin/user.rs b/server/src/ui/admin/user.rs index dd68383..4df5e80 100644 --- a/server/src/ui/admin/user.rs +++ b/server/src/ui/admin/user.rs @@ -87,10 +87,7 @@ pub fn r_admin_user_permission( Ok(Flash::success( Redirect::to(u_admin_user(name)), - tr( - ri.lang, - "admin.users.permission_update_success", - ), + tr(ri.lang, "admin.users.permission_update_success"), )) } diff --git a/server/src/ui/assets.rs b/server/src/ui/assets.rs index 969f3ed..d7663c3 100644 --- a/server/src/ui/assets.rs +++ b/server/src/ui/assets.rs @@ -4,108 +4,62 @@ Copyright (C) 2025 metamuffin <metamuffin.org> */ use super::error::MyResult; -use crate::{ - helper::{cache::CacheControlFile, A}, - CONF, -}; -use anyhow::{anyhow, bail, Context}; -use jellycommon::NodeID; -use jellylogic::session::Session; -use log::info; +use crate::helper::{cache::CacheControlImage, A}; +use anyhow::{anyhow, Context}; +use jellycache::{CacheContentType, CacheKey}; +use jellycommon::{api::NodeFilterSort, NodeID, Picture, PictureSlot}; +use jellylogic::{assets::get_node_thumbnail, node::get_node, session::Session}; use rocket::{get, http::ContentType, response::Redirect}; -use std::path::PathBuf; +use std::str::FromStr; pub const AVIF_QUALITY: f32 = 50.; pub const AVIF_SPEED: u8 = 5; -#[get("/asset/<token>?<width>")] -pub async fn r_asset( +#[get("/image/<key>?<size>")] +pub async fn r_image( _session: A<Session>, - token: &str, - width: Option<usize>, -) -> MyResult<(ContentType, CacheControlFile)> { - // let width = width.unwrap_or(2048); - // let asset = AssetInner::deser(token)?; + key: A<Picture>, + size: Option<usize>, +) -> MyResult<(ContentType, CacheControlImage)> { + let size = size.unwrap_or(2048); - // // if let AssetInner::Federated { host, asset } = asset { - // // let session = fed.get_session(&host).await?; + let key = CacheKey(key.0 .0); + if !matches!(key.content_type(), CacheContentType::Image) { + Err(anyhow!("request to non-image"))? + } - // // let asset = base64::engine::general_purpose::URL_SAFE.encode(asset); - // // async_cache_file("fed-asset", &asset, |out| async { - // // session.asset(out, &asset, width).await - // // }) - // // .await? - // // } else - // let path = { - // let source = resolve_asset(asset).await.context("resolving asset")?; + // fit the resolution into a finite set so the maximum cache is finite too. + let width = 2usize.pow(size.clamp(128, 2048).ilog2()); + let encoded = jellytranscoder::image::transcode(key, AVIF_QUALITY, AVIF_SPEED, width) + .context("transcoding asset")?; - // // fit the resolution into a finite set so the maximum cache is finite too. - // let width = 2usize.pow(width.clamp(128, 2048).ilog2()); - // jellytranscoder::image::transcode(&source, AVIF_QUALITY, AVIF_SPEED, width) - // .await - // .context("transcoding asset")? - // }; - // info!("loading asset from {path:?}"); - // Ok(( - // ContentType::AVIF, - // CacheControlFile::new_cachekey(&path.abs()).await?, - // )) - todo!() + Ok((ContentType::AVIF, CacheControlImage(encoded))) } -// pub async fn resolve_asset(asset: AssetInner) -> anyhow::Result<PathBuf> { -// match asset { -// AssetInner::Cache(c) => Ok(c.abs()), -// AssetInner::Assets(c) => Ok(CONF.asset_path.join(c)), -// AssetInner::Media(c) => Ok(c), -// _ => bail!("wrong asset type"), -// } -// } - -#[get("/n/<id>/poster?<width>")] +#[get("/n/<id>/image/<slot>?<size>")] pub async fn r_item_poster( session: A<Session>, id: A<NodeID>, - width: Option<usize>, -) -> MyResult<Redirect> { - // let asset = get_node_poster(&session.0, id.0)?; - // Ok(Redirect::permanent(rocket::uri!(r_asset(asset.0, width)))) - Err(anyhow!("a").into()) -} - -#[get("/n/<id>/backdrop?<width>")] -pub async fn r_item_backdrop( - session: A<Session>, - id: A<NodeID>, - width: Option<usize>, -) -> MyResult<Redirect> { - // let asset = get_node_backdrop(&session.0, id.0)?; - // Ok(Redirect::permanent(rocket::uri!(r_asset(asset.0, width)))) - Err(anyhow!("a").into()) -} - -#[get("/n/<id>/person/<index>/asset?<group>&<width>")] -pub async fn r_person_asset( - session: A<Session>, - id: A<NodeID>, - index: usize, - group: String, - width: Option<usize>, + slot: &str, + size: Option<usize>, ) -> MyResult<Redirect> { - // let group = PeopleGroup::from_str_opt(&group).ok_or(anyhow!("unknown people group"))?; - // let asset = get_node_person_asset(&session.0, id.0, group, index)?; - // Ok(Redirect::permanent(rocket::uri!(r_asset(asset.0, width)))) - Err(anyhow!("a").into()) + let slot = PictureSlot::from_str(slot).map_err(|_| anyhow!("slot invalid"))?; + let node = get_node(&session.0, id.0, false, false, NodeFilterSort::default())?; + let picture = node + .node + .pictures + .get(&slot) + .ok_or(anyhow!("no pic todo"))?; + Ok(Redirect::permanent(rocket::uri!(r_image(*picture, size)))) } -#[get("/n/<id>/thumbnail?<t>&<width>")] +#[get("/n/<id>/thumbnail?<t>&<size>")] pub async fn r_node_thumbnail( session: A<Session>, id: A<NodeID>, t: f64, - width: Option<usize>, + size: Option<usize>, ) -> MyResult<Redirect> { - // let asset = get_node_thumbnail(&session.0, id.0, t).await?; - // Ok(Redirect::temporary(rocket::uri!(r_asset(asset.0, width)))) - Err(anyhow!("a").into()) + let picture = get_node_thumbnail(&session.0, id.0, t).await?; + Ok(Redirect::permanent(rocket::uri!(r_image(picture, size)))) } diff --git a/stream/src/cues.rs b/stream/src/cues.rs index 646db5b..9291e18 100644 --- a/stream/src/cues.rs +++ b/stream/src/cues.rs @@ -5,7 +5,7 @@ */ use anyhow::{anyhow, Result}; -use jellycache::cache_memory; +use jellycache::{cache_memory, CacheKey}; use jellyremuxer::demuxers::create_demuxer_autodetect; use serde::{Deserialize, Serialize}; use std::{collections::BTreeMap, fs::File, path::Path, sync::Arc}; @@ -29,29 +29,32 @@ pub struct StatsAndCues { } pub fn generate_cues(path: &Path) -> Result<Arc<StatsAndCues>> { - cache_memory("generated-cues", path, move || { - let media = File::open(path)?; - let mut media = - create_demuxer_autodetect(Box::new(media))?.ok_or(anyhow!("media format unknown"))?; + cache_memory( + CacheKey::new_json(("media-generated-cues", path)), + move || { + let media = File::open(path)?; + let mut media = create_demuxer_autodetect(Box::new(media))? + .ok_or(anyhow!("media format unknown"))?; - let info = media.info()?; - media.seek_cluster(None)?; + let info = media.info()?; + media.seek_cluster(None)?; - let mut stats = BTreeMap::<u64, TrackStat>::new(); - let mut cues = Vec::new(); + let mut stats = BTreeMap::<u64, TrackStat>::new(); + let mut cues = Vec::new(); - while let Some((position, cluster)) = media.read_cluster()? { - cues.push(GeneratedCue { - position, - time: cluster.timestamp * info.timestamp_scale, - }); - for block in cluster.simple_blocks { - let e = stats.entry(block.track).or_default(); - e.num_blocks += 1; - e.total_size += block.data.len() as u64; + while let Some((position, cluster)) = media.read_cluster()? { + cues.push(GeneratedCue { + position, + time: cluster.timestamp * info.timestamp_scale, + }); + for block in cluster.simple_blocks { + let e = stats.entry(block.track).or_default(); + e.num_blocks += 1; + e.total_size += block.data.len() as u64; + } } - } - Ok(StatsAndCues { stats, cues }) - }) + Ok(StatsAndCues { stats, cues }) + }, + ) } diff --git a/stream/src/fragment.rs b/stream/src/fragment.rs index 97ca2db..35a5cd2 100644 --- a/stream/src/fragment.rs +++ b/stream/src/fragment.rs @@ -94,13 +94,12 @@ pub fn fragment_stream( cluster.simple_blocks.retain(|b| b.track == track_num); cluster.block_groups.retain(|b| b.block.track == track_num); - let next_kf = next_cluster - .and_then(|x| { - x.simple_blocks - .iter() - .find(|b| b.track == track_num) - .cloned() - }); + let next_kf = next_cluster.and_then(|x| { + x.simple_blocks + .iter() + .find(|b| b.track == track_num) + .cloned() + }); let jr_container = match container { StreamContainer::WebM => ContainerFormat::Webm, diff --git a/stream/src/lib.rs b/stream/src/lib.rs index 724a593..6c6db8f 100644 --- a/stream/src/lib.rs +++ b/stream/src/lib.rs @@ -113,7 +113,7 @@ fn original_stream( info: Arc<SMediaInfo>, track: usize, range: Range<u64>, -) -> Result<Box<dyn Read+ Send + Sync>> { +) -> Result<Box<dyn Read + Send + Sync>> { let (iinfo, _info) = stream_info(info)?; let (file_index, _) = *iinfo .track_to_file diff --git a/stream/src/metadata.rs b/stream/src/metadata.rs index 9bfa3aa..fe553e2 100644 --- a/stream/src/metadata.rs +++ b/stream/src/metadata.rs @@ -5,12 +5,12 @@ */ use anyhow::{anyhow, Result}; -use jellycache::cache_memory; +use jellycache::{cache_memory, CacheKey}; use jellyremuxer::{demuxers::create_demuxer_autodetect, matroska::Segment}; use std::{fs::File, path::Path, sync::Arc}; pub fn read_metadata(path: &Path) -> Result<Arc<Segment>> { - cache_memory("mkmeta-v4", path, move || { + cache_memory(CacheKey::new_json(("media-metadata", path)), move || { let media = File::open(path)?; let mut media = create_demuxer_autodetect(Box::new(media))?.ok_or(anyhow!("media format unknown"))?; diff --git a/tool/src/add.rs b/tool/src/add.rs index fc43b76..40cc579 100644 --- a/tool/src/add.rs +++ b/tool/src/add.rs @@ -9,14 +9,13 @@ use jellyimport::{get_trakt, trakt::TraktKind}; use log::warn; use std::{ fmt::Display, - path::{Path, PathBuf}, -}; -use tokio::{ fs::{rename, OpenOptions}, - io::AsyncWriteExt, + io::Write, + path::{Path, PathBuf}, }; +use tokio::runtime::Handle; -pub async fn add(action: Action) -> anyhow::Result<()> { +pub fn add(action: Action, rt: &Handle) -> anyhow::Result<()> { match action { Action::Add { media } => { let theme = ColorfulTheme::default(); @@ -36,7 +35,7 @@ pub async fn add(action: Action) -> anyhow::Result<()> { let trakt = get_trakt()?; - let results = trakt.search(search_kinds, &name).await?; + let results = trakt.search(search_kinds, &name, rt)?; if results.is_empty() { warn!("no search results"); @@ -79,10 +78,8 @@ pub async fn add(action: Action) -> anyhow::Result<()> { .append(true) .write(true) .create(true) - .open(flagspath) - .await? - .write_all(flag.as_bytes()) - .await?; + .open(flagspath)? + .write_all(flag.as_bytes())?; } } else { let ext = media @@ -111,7 +108,7 @@ pub async fn add(action: Action) -> anyhow::Result<()> { .interact() .unwrap() { - rename(media, newpath).await?; + rename(media, newpath)?; } } diff --git a/tool/src/main.rs b/tool/src/main.rs index adf0a35..6aeb617 100644 --- a/tool/src/main.rs +++ b/tool/src/main.rs @@ -19,11 +19,13 @@ fn main() -> anyhow::Result<()> { let args = Args::parse(); match args { - a @ Action::Add { .. } => tokio::runtime::Builder::new_multi_thread() - .enable_all() - .build() - .unwrap() - .block_on(add(a)), + a @ Action::Add { .. } => { + let rt = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .unwrap(); + add(a, rt.handle()) + } a @ Action::Migrate { .. } => migrate(a), _ => Ok(()), // Action::Reimport { diff --git a/transcoder/src/fragment.rs b/transcoder/src/fragment.rs index 152e163..564c94d 100644 --- a/transcoder/src/fragment.rs +++ b/transcoder/src/fragment.rs @@ -5,17 +5,17 @@ */ use crate::{Config, CONF, LOCAL_VIDEO_TRANSCODING_TASKS}; use anyhow::Result; -use jellycache::cache_file; +use jellycache::{cache, CacheContentType, CacheKey}; use jellyremuxer::{demuxers::create_demuxer, muxers::write_fragment, ContainerFormat}; use jellystream_types::{StreamFormatInfo, TrackKind}; use log::info; -use std::fmt::Write; -use std::fs::File; -use std::io::{copy, Write as W2}; -use std::process::{Command, Stdio}; -use std::thread::spawn; -use winter_matroska::block::Block; -use winter_matroska::{Cluster, Segment, TrackEntry as MatroskaTrackEntry}; +use std::{ + fmt::Write, + io::{Cursor, Read, Write as W2}, + process::{Command, Stdio}, + thread::spawn, +}; +use winter_matroska::{block::Block, Cluster, Segment, TrackEntry as MatroskaTrackEntry}; // TODO odd video resolutions can cause errors when transcoding to YUV42{0,2} // TODO with an implementation that cant handle it (SVT-AV1 is such an impl). @@ -38,41 +38,40 @@ pub fn transcode( let input_duration = input.info.duration; let had_next_kf = next_kf.is_some(); - let output = cache_file("frag-tc", (input_key, &command), |mut output| { - let _permit = LOCAL_VIDEO_TRANSCODING_TASKS.lock().unwrap(); - info!("encoding with {command:?}"); - let mut args = command.split(" "); - let mut proc = Command::new(args.next().unwrap()) - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .args(args) - .spawn()?; + let output = cache( + CacheKey::new(CacheContentType::Unknown, ("frag-tc", input_key, &command)), + || { + let _permit = LOCAL_VIDEO_TRANSCODING_TASKS.lock().unwrap(); + info!("encoding with {command:?}"); + let mut args = command.split(" "); + let mut proc = Command::new(args.next().unwrap()) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .args(args) + .spawn()?; - let mut stdin = proc.stdin.take().unwrap(); - let mut stdout = proc.stdout.take().unwrap(); + let mut stdin = proc.stdin.take().unwrap(); + let mut stdout = proc.stdout.take().unwrap(); - spawn(move || { - copy(&mut stdout, &mut output).unwrap(); - }); + spawn(move || { + input.clusters.extend(next_kf.map(|kf| Cluster { + simple_blocks: vec![kf], + ..Default::default() + })); + write_fragment(ContainerFormat::Matroska, &mut stdin, input).unwrap(); // TODO + stdin.flush().unwrap(); + drop(stdin); + }); - input.clusters.extend(next_kf.map(|kf| Cluster { - simple_blocks: vec![kf], - ..Default::default() - })); + let mut output = Vec::new(); + stdout.read_to_end(&mut output)?; + proc.wait().unwrap().exit_ok()?; + info!("done"); + Ok(output) + }, + )?; - write_fragment(ContainerFormat::Matroska, &mut stdin, input)?; - stdin.flush()?; - drop(stdin); - - proc.wait().unwrap().exit_ok()?; - info!("done"); - Ok(()) - })?; - - let mut demuxer = create_demuxer( - ContainerFormat::Matroska, - Box::new(File::open(output.abs())?), - ); + let mut demuxer = create_demuxer(ContainerFormat::Matroska, Box::new(Cursor::new(output))); let mut info = demuxer.info()?; info.duration = input_duration; diff --git a/transcoder/src/image.rs b/transcoder/src/image.rs index 6a7f693..8366ece 100644 --- a/transcoder/src/image.rs +++ b/transcoder/src/image.rs @@ -3,96 +3,66 @@ which is licensed under the GNU Affero General Public License (version 3); see /COPYING. Copyright (C) 2025 metamuffin <metamuffin.org> */ -use crate::LOCAL_IMAGE_TRANSCODING_TASKS; -use anyhow::Context; +use anyhow::{anyhow, Context, Result}; use image::imageops::FilterType; -use jellycache::{async_cache_file, CachePath}; +use jellycache::{cache, cache_read, CacheKey}; use log::{debug, info}; use rgb::FromSlice; -use std::{ - fs::File, - io::{BufReader, Read, Seek, SeekFrom}, - path::Path, -}; -use tokio::io::AsyncWriteExt; +use std::io::Cursor; -pub async fn transcode( - path: &Path, - quality: f32, - speed: u8, - width: usize, -) -> anyhow::Result<CachePath> { - async_cache_file( - "image-tc", - (path, width, quality as i32, speed), - |mut output| async move { - let _permit = LOCAL_IMAGE_TRANSCODING_TASKS.acquire().await?; - info!("encoding {path:?} (speed={speed}, quality={quality}, width={width})"); - let path = path.to_owned(); - let encoded = tokio::task::spawn_blocking(move || { - let mut file = BufReader::new(File::open(&path).context("opening source")?); +pub fn transcode(key: CacheKey, quality: f32, speed: u8, width: usize) -> Result<Vec<u8>> { + cache( + CacheKey::new_image(("image-tc", key, width, quality as i32, speed)), + move || { + let input = cache_read(key)?.ok_or(anyhow!("transcode cache key missing"))?; + info!("encoding image (speed={speed}, quality={quality}, width={width})"); - // TODO: use better image library that supports AVIF - let is_avif = { - let mut magic = [0u8; 12]; - let _ = file.read_exact(&mut magic); - file.seek(SeekFrom::Start(0)) - .context("seeking back to start")?; - // TODO: magic experimentally found and probably not working in all cases but fine as long as our avif enc uses that - matches!( - magic, - [ - 0x00, - 0x00, - 0x00, - _, - b'f', - b't', - b'y', - b'p', - b'a', - b'v', - b'i', - b'f', - ] - ) - }; - let original = if is_avif { - let mut buf = Vec::new(); - file.read_to_end(&mut buf).context("reading image")?; - libavif_image::read(&buf).unwrap().to_rgba8() - } else { - let reader = image::ImageReader::new(file); - let reader = reader.with_guessed_format().context("guessing format")?; - debug!("guessed format (or fallback): {:?}", reader.format()); - reader.decode().context("decoding image")?.to_rgba8() - }; - let image = image::imageops::resize( - &original, - width as u32, - width as u32 * original.height() / original.width(), - FilterType::Lanczos3, - ); - let pixels = image.to_vec(); - let encoded = ravif::Encoder::new() - .with_speed(speed.clamp(1, 10)) - .with_quality(quality.clamp(1., 100.)) - .encode_rgba(imgref::Img::new( - pixels.as_rgba(), - image.width() as usize, - image.height() as usize, - )) - .context("avif encoding")?; - info!("transcode finished"); - Ok::<_, anyhow::Error>(encoded) - }) - .await??; - output - .write_all(&encoded.avif_file) - .await - .context("writing encoded image")?; - Ok(()) + // TODO: use better image library that supports AVIF + // TODO: magic experimentally found and probably not working in all cases but fine as long as our avif enc uses that + let is_avif = matches!( + input[0..input.len().min(12)], + [ + 0x00, + 0x00, + 0x00, + _, + b'f', + b't', + b'y', + b'p', + b'a', + b'v', + b'i', + b'f', + ] + ); + let original = if is_avif { + libavif_image::read(&input).unwrap().to_rgba8() + } else { + let reader = image::ImageReader::new(Cursor::new(input)); + let reader = reader.with_guessed_format().context("guessing format")?; + debug!("guessed format (or fallback): {:?}", reader.format()); + reader.decode().context("decoding image")?.to_rgba8() + }; + let image = image::imageops::resize( + &original, + width as u32, + width as u32 * original.height() / original.width(), + FilterType::Lanczos3, + ); + let pixels = image.to_vec(); + let encoded = ravif::Encoder::new() + .with_speed(speed.clamp(1, 10)) + .with_quality(quality.clamp(1., 100.)) + .encode_rgba(imgref::Img::new( + pixels.as_rgba(), + image.width() as usize, + image.height() as usize, + )) + .context("avif encoding")?; + + info!("transcode finished"); + Ok(encoded.avif_file) }, ) - .await } diff --git a/transcoder/src/lib.rs b/transcoder/src/lib.rs index 1eac15b..cd8edbf 100644 --- a/transcoder/src/lib.rs +++ b/transcoder/src/lib.rs @@ -7,7 +7,6 @@ use serde::{Deserialize, Serialize}; use std::sync::{LazyLock, Mutex}; -use tokio::sync::Semaphore; pub mod fragment; pub mod image; @@ -36,5 +35,4 @@ static CONF: LazyLock<Config> = LazyLock::new(|| { .expect("transcoder config not preloaded. logic error") }); -static LOCAL_IMAGE_TRANSCODING_TASKS: Semaphore = Semaphore::const_new(8); static LOCAL_VIDEO_TRANSCODING_TASKS: Mutex<()> = Mutex::new(()); diff --git a/transcoder/src/thumbnail.rs b/transcoder/src/thumbnail.rs index 8cefac3..eda9e04 100644 --- a/transcoder/src/thumbnail.rs +++ b/transcoder/src/thumbnail.rs @@ -1,31 +1,37 @@ -use crate::LOCAL_IMAGE_TRANSCODING_TASKS; -use jellycache::{async_cache_file, CachePath}; +use anyhow::{Context, Result}; +use jellycache::{cache_store, CacheKey}; use log::info; -use std::{path::Path, process::Stdio}; -use tokio::{io::copy, process::Command}; +use std::{ + io::Read, + path::Path, + process::{Command, Stdio}, +}; -pub async fn create_thumbnail(path: &Path, time: f64) -> anyhow::Result<CachePath> { - async_cache_file("thumb", (path, time as i64), move |mut output| async move { - let _permit = LOCAL_IMAGE_TRANSCODING_TASKS.acquire().await?; - info!("creating thumbnail of {path:?} at {time}s",); +pub fn create_thumbnail(path: &Path, time: f64) -> Result<CacheKey> { + cache_store( + CacheKey::new_image(("thumbnail", path, time as i64)), + move || { + info!("creating thumbnail of {path:?} at {time}s",); - let mut proc = Command::new("ffmpeg") - .stdout(Stdio::piped()) - .args(["-ss", &format!("{time}")]) - .args(["-f", "matroska", "-i", path.to_str().unwrap()]) - .args(["-frames:v", "1"]) - .args(["-c:v", "qoi"]) - .args(["-f", "image2"]) - .args(["-update", "1"]) - .arg("pipe:1") - .spawn()?; + let mut proc = Command::new("ffmpeg") + .stdout(Stdio::piped()) + .args(["-ss", &format!("{time}")]) + .args(["-f", "matroska", "-i", path.to_str().unwrap()]) + .args(["-frames:v", "1"]) + .args(["-c:v", "qoi"]) + .args(["-f", "image2"]) + .args(["-update", "1"]) + .arg("pipe:1") + .spawn()?; - let mut stdout = proc.stdout.take().unwrap(); - copy(&mut stdout, &mut output).await?; + let mut stdout = proc.stdout.take().unwrap(); + let mut output = Vec::new(); + stdout.read_to_end(&mut output)?; - proc.wait().await.unwrap().exit_ok()?; - info!("done"); - Ok(()) - }) - .await + proc.wait().unwrap().exit_ok()?; + info!("done"); + Ok(output) + }, + ) + .context("creating thumbnail") } diff --git a/ui/src/node_card.rs b/ui/src/node_card.rs index b4481b7..dfcd37c 100644 --- a/ui/src/node_card.rs +++ b/ui/src/node_card.rs @@ -6,8 +6,8 @@ use crate::{locale::Language, node_page::aspect_class, props::Props}; use jellycommon::{ - Node, - routes::{u_node_slug, u_node_slug_player, u_node_slug_poster}, + Node, PictureSlot, + routes::{u_node_image, u_node_slug, u_node_slug_player}, user::NodeUserData, }; @@ -17,7 +17,7 @@ markup::define! { div[class=cls] { .poster { a[href=u_node_slug(&node.slug)] { - img[src=u_node_slug_poster(&node.slug, 1024), loading="lazy"]; + img[src=u_node_image(&node.slug, PictureSlot::Cover, 1024), loading="lazy"]; } .cardhover.item { @if node.media.is_some() { @@ -42,7 +42,7 @@ markup::define! { div[class="node card widecard poster"] { div[class=&format!("poster {}", aspect_class(node.kind))] { a[href=u_node_slug(&node.slug)] { - img[src=u_node_slug_poster(&node.slug, 1024), loading="lazy"]; + img[src=u_node_image(&node.slug, PictureSlot::Cover, 1024), loading="lazy"]; } .cardhover.item { @if node.media.is_some() { diff --git a/ui/src/node_page.rs b/ui/src/node_page.rs index e7e7b1d..a72eb9a 100644 --- a/ui/src/node_page.rs +++ b/ui/src/node_page.rs @@ -13,12 +13,12 @@ use crate::{ props::Props, }; use jellycommon::{ - Chapter, CreditCategory, IdentifierType, Node, NodeKind, + Chapter, CreditCategory, IdentifierType, Node, NodeKind, PictureSlot, api::NodeFilterSort, routes::{ - u_node_slug, u_node_slug_backdrop, u_node_slug_person_asset, u_node_slug_player, - u_node_slug_player_time, u_node_slug_poster, u_node_slug_thumbnail, - u_node_slug_update_rating, u_node_slug_watched, + u_node_image, u_node_slug, u_node_slug_person_asset, u_node_slug_player, + u_node_slug_player_time, u_node_slug_thumbnail, u_node_slug_update_rating, + u_node_slug_watched, }, user::{ApiWatchedState, NodeUserData, WatchedState}, }; @@ -52,12 +52,12 @@ markup::define! { player: bool, ) { @if !matches!(node.kind, NodeKind::Collection) && !player { - img.backdrop[src=u_node_slug_backdrop(&node.slug, 2048), loading="lazy"]; + img.backdrop[src=u_node_image(&node.slug, PictureSlot::Backdrop, 2048), loading="lazy"]; } .page.node { @if !matches!(node.kind, NodeKind::Collection) && !player { @let cls = format!("bigposter {}", aspect_class(node.kind)); - div[class=cls] { img[src=u_node_slug_poster(&node.slug, 2048), loading="lazy"]; } + div[class=cls] { img[src=u_node_image(&node.slug, PictureSlot::Cover, 2048), loading="lazy"]; } } .title { h1 { @node.title } |