aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--Cargo.lock1
-rw-r--r--base/Cargo.toml1
-rw-r--r--base/src/assetfed.rs71
-rw-r--r--base/src/cache.rs51
-rw-r--r--base/src/lib.rs24
-rw-r--r--base/src/temp.rs30
-rw-r--r--client/src/lib.rs9
-rw-r--r--common/src/lib.rs24
-rw-r--r--import/src/lib.rs113
-rw-r--r--import/src/tmdb.rs14
-rw-r--r--import/src/trakt.rs7
-rw-r--r--server/src/routes/mod.rs3
-rw-r--r--server/src/routes/ui/assets.rs115
-rw-r--r--server/src/routes/ui/error.rs7
-rw-r--r--stream/src/hls.rs2
-rw-r--r--stream/src/jhls.rs2
-rw-r--r--stream/src/lib.rs4
-rw-r--r--stream/src/segment.rs8
-rw-r--r--tool/src/add.rs4
-rw-r--r--transcoder/src/image.rs18
-rw-r--r--transcoder/src/snippet.rs6
-rw-r--r--transcoder/src/thumbnail.rs5
22 files changed, 279 insertions, 240 deletions
diff --git a/Cargo.lock b/Cargo.lock
index 34f5e40..dc46449 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1351,6 +1351,7 @@ checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c"
name = "jellybase"
version = "0.1.0"
dependencies = [
+ "aes-gcm-siv",
"anyhow",
"base64",
"bincode",
diff --git a/base/Cargo.toml b/base/Cargo.toml
index 36b93bc..bd1a748 100644
--- a/base/Cargo.toml
+++ b/base/Cargo.toml
@@ -17,6 +17,7 @@ bincode = "2.0.0-rc.3"
rand = "0.8.5"
redb = "1.5.0"
serde_json = "1.0.111"
+aes-gcm-siv = "0.11.1"
[features]
db_json = []
diff --git a/base/src/assetfed.rs b/base/src/assetfed.rs
new file mode 100644
index 0000000..800e458
--- /dev/null
+++ b/base/src/assetfed.rs
@@ -0,0 +1,71 @@
+use aes_gcm_siv::{
+ aead::{generic_array::GenericArray, Aead},
+ Aes256GcmSiv, KeyInit,
+};
+use anyhow::{anyhow, bail, Context};
+use base64::Engine;
+use bincode::{Decode, Encode};
+use jellycommon::Asset;
+use log::warn;
+use std::{path::PathBuf, sync::LazyLock};
+
+use crate::{cache::CachePath, SECRETS};
+
+const VERSION: u32 = 2;
+
+static ASSET_KEY: LazyLock<Aes256GcmSiv> = LazyLock::new(|| {
+ if let Some(sk) = &SECRETS.session_key {
+ let r = base64::engine::general_purpose::STANDARD
+ .decode(sk)
+ .expect("key invalid; should be valid base64");
+ aes_gcm_siv::Aes256GcmSiv::new_from_slice(&r)
+ .expect("key has the wrong length; should be 32 bytes")
+ } else {
+ warn!("session_key not configured; generating a random one.");
+ aes_gcm_siv::Aes256GcmSiv::new_from_slice(&[(); 32].map(|_| rand::random())).unwrap()
+ }
+});
+
+#[derive(Debug, Encode, Decode)]
+pub enum AssetInner {
+ Federated { host: String, asset: Vec<u8> },
+ Cache(CachePath),
+ Assets(PathBuf),
+}
+
+impl AssetInner {
+ pub fn ser(&self) -> Asset {
+ let mut plaintext = Vec::new();
+ plaintext.extend(u32::to_le_bytes(VERSION));
+ plaintext.extend(bincode::encode_to_vec(&self, bincode::config::standard()).unwrap());
+
+ while plaintext.len() % 16 == 0 {
+ plaintext.push(0);
+ }
+
+ let nonce = [(); 12].map(|_| rand::random());
+ let mut ciphertext = ASSET_KEY
+ .encrypt(&GenericArray::from(nonce), plaintext.as_slice())
+ .unwrap();
+ ciphertext.extend(nonce);
+
+ Asset(base64::engine::general_purpose::URL_SAFE.encode(&ciphertext))
+ }
+ pub fn deser(s: &str) -> anyhow::Result<Self> {
+ let ciphertext = base64::engine::general_purpose::URL_SAFE.decode(&s)?;
+ let (ciphertext, nonce) = ciphertext.split_at(ciphertext.len() - 12);
+ let plaintext = ASSET_KEY
+ .decrypt(nonce.into(), ciphertext)
+ .map_err(|_| anyhow!("asset token decrypt failed"))?;
+
+ let version = u32::from_le_bytes(plaintext[0..4].try_into().unwrap());
+ if version != VERSION {
+ bail!("asset token version mismatch");
+ }
+
+ let (data, _): (AssetInner, _) =
+ bincode::decode_from_slice(&plaintext[4..], bincode::config::standard())
+ .context("asset token has invalid format")?;
+ Ok(data)
+ }
+}
diff --git a/base/src/cache.rs b/base/src/cache.rs
index d1c3e4d..3c4ae81 100644
--- a/base/src/cache.rs
+++ b/base/src/cache.rs
@@ -3,11 +3,10 @@
which is licensed under the GNU Affero General Public License (version 3); see /COPYING.
Copyright (C) 2023 metamuffin <metamuffin.org>
*/
-use crate::{AssetLocationExt, CONF};
+use crate::CONF;
use anyhow::{anyhow, Context};
use base64::Engine;
use bincode::{Decode, Encode};
-use jellycommon::AssetLocation;
use log::{info, warn};
use rand::random;
use std::{
@@ -17,7 +16,6 @@ use std::{
future::Future,
io::Seek,
path::PathBuf,
- str::FromStr,
sync::{
atomic::{AtomicUsize, Ordering},
Arc, LazyLock, RwLock,
@@ -29,7 +27,15 @@ use tokio::{
sync::Mutex,
};
-pub fn cache_location(seed: &[&str]) -> (usize, AssetLocation) {
+#[derive(Debug, Encode, Decode)]
+pub struct CachePath(pub PathBuf);
+impl CachePath {
+ pub fn abs(&self) -> PathBuf {
+ CONF.cache_path.join(&self.0)
+ }
+}
+
+pub fn cache_location(seed: &[&str]) -> (usize, CachePath) {
use sha2::Digest;
let mut d = sha2::Sha512::new();
for s in seed {
@@ -41,7 +47,7 @@ pub fn cache_location(seed: &[&str]) -> (usize, AssetLocation) {
let fname = base64::engine::general_purpose::URL_SAFE.encode(d);
let fname = &fname[..22];
let fname = format!("{}-{}", seed[0], fname); // about 128 bits
- (n, AssetLocation::Cache(fname.into()))
+ (n, CachePath(fname.into()))
}
const CACHE_GENERATION_BUCKET_COUNT: usize = 1024;
@@ -51,7 +57,7 @@ pub static CACHE_GENERATION_LOCKS: LazyLock<[Mutex<()>; CACHE_GENERATION_BUCKET_
pub async fn async_cache_file<Fun, Fut>(
seed: &[&str],
generate: Fun,
-) -> Result<AssetLocation, anyhow::Error>
+) -> Result<CachePath, anyhow::Error>
where
Fun: FnOnce(tokio::fs::File) -> Fut,
Fut: Future<Output = Result<(), anyhow::Error>>,
@@ -61,13 +67,11 @@ where
let _guard = CACHE_GENERATION_LOCKS[bucket % CACHE_GENERATION_BUCKET_COUNT]
.lock()
.await;
- let exists = tokio::fs::try_exists(location.path())
+ let exists = tokio::fs::try_exists(&location.abs())
.await
.context("unable to test for cache file existance")?;
if !exists {
- let temp_path =
- AssetLocation::Cache(PathBuf::from_str(&format!("temp-{:x}", random::<u128>()))?)
- .path();
+ let temp_path = CONF.cache_path.join(format!("temp-{:x}", random::<u128>()));
let f = tokio::fs::File::create(&temp_path)
.await
.context("creating new cache file")?;
@@ -79,7 +83,7 @@ where
return Err(e);
}
}
- tokio::fs::rename(temp_path, location.path())
+ tokio::fs::rename(temp_path, &location.abs())
.await
.context("rename cache")?;
}
@@ -87,18 +91,15 @@ where
Ok(location)
}
-pub fn cache_file<Fun>(seed: &[&str], mut generate: Fun) -> Result<AssetLocation, anyhow::Error>
+pub fn cache_file<Fun>(seed: &[&str], mut generate: Fun) -> Result<CachePath, anyhow::Error>
where
Fun: FnMut(std::fs::File) -> Result<(), anyhow::Error>,
{
let (bucket, location) = cache_location(seed);
// we need a lock even if it exists since somebody might be still in the process of writing.
let _guard = CACHE_GENERATION_LOCKS[bucket % CACHE_GENERATION_BUCKET_COUNT].blocking_lock();
- let exists = location.path().exists();
- if !exists {
- let temp_path =
- AssetLocation::Cache(PathBuf::from_str(&format!("temp-{:x}", random::<u128>()))?)
- .path();
+ if !location.abs().exists() {
+ let temp_path = CONF.cache_path.join(format!("temp-{:x}", random::<u128>()));
let f = std::fs::File::create(&temp_path).context("creating new cache file")?;
match generate(f) {
Ok(()) => (),
@@ -108,7 +109,7 @@ where
return Err(e);
}
}
- rename(temp_path, location.path()).context("rename cache")?;
+ rename(temp_path, &location.abs()).context("rename cache")?;
}
drop(_guard);
Ok(location)
@@ -119,7 +120,7 @@ pub struct InMemoryCacheEntry {
last_access: Instant,
object: Arc<dyn Any + Send + Sync + 'static>,
}
-pub static CACHE_IN_MEMORY_OBJECTS: LazyLock<RwLock<HashMap<AssetLocation, InMemoryCacheEntry>>> =
+pub static CACHE_IN_MEMORY_OBJECTS: LazyLock<RwLock<HashMap<PathBuf, InMemoryCacheEntry>>> =
LazyLock::new(|| RwLock::new(HashMap::new()));
pub static CACHE_IN_MEMORY_SIZE: AtomicUsize = AtomicUsize::new(0);
@@ -131,7 +132,7 @@ where
let (_, location) = cache_location(seed);
{
let mut g = CACHE_IN_MEMORY_OBJECTS.write().unwrap();
- if let Some(entry) = g.get_mut(&location) {
+ if let Some(entry) = g.get_mut(&location.abs()) {
entry.last_access = Instant::now();
let object = entry
.object
@@ -148,7 +149,7 @@ where
.context("encoding cache object")?;
Ok(())
})?;
- let mut file = std::fs::File::open(location.path())?;
+ let mut file = std::fs::File::open(&location.abs())?;
let object = bincode::decode_from_std_read::<T, _, _>(&mut file, bincode::config::standard())
.context("decoding cache object")?;
let object = Arc::new(object);
@@ -157,7 +158,7 @@ where
{
let mut g = CACHE_IN_MEMORY_OBJECTS.write().unwrap();
g.insert(
- location,
+ location.abs(),
InMemoryCacheEntry {
size,
last_access: Instant::now(),
@@ -184,7 +185,7 @@ where
let (_, location) = cache_location(seed);
{
let mut g = CACHE_IN_MEMORY_OBJECTS.write().unwrap();
- if let Some(entry) = g.get_mut(&location) {
+ if let Some(entry) = g.get_mut(&location.abs()) {
entry.last_access = Instant::now();
let object = entry
.object
@@ -205,7 +206,7 @@ where
Ok(())
})
.await?;
- let mut file = tokio::fs::File::open(location.path()).await?;
+ let mut file = tokio::fs::File::open(&location.abs()).await?;
let mut data = Vec::new();
file.read_to_end(&mut data)
.await
@@ -218,7 +219,7 @@ where
{
let mut g = CACHE_IN_MEMORY_OBJECTS.write().unwrap();
g.insert(
- location,
+ location.abs(),
InMemoryCacheEntry {
size,
last_access: Instant::now(),
diff --git a/base/src/lib.rs b/base/src/lib.rs
index 015b62b..90ac27b 100644
--- a/base/src/lib.rs
+++ b/base/src/lib.rs
@@ -8,13 +8,10 @@ pub mod cache;
pub mod database;
pub mod federation;
pub mod permission;
-pub mod temp;
+pub mod assetfed;
-use jellycommon::{
- config::{GlobalConfig, SecretsConfig},
- AssetLocation,
-};
-use std::{fs::File, path::PathBuf, sync::LazyLock};
+use jellycommon::config::{GlobalConfig, SecretsConfig};
+use std::{fs::File, sync::LazyLock};
pub static CONF: LazyLock<GlobalConfig> = LazyLock::new(|| {
serde_yaml::from_reader(
@@ -39,18 +36,3 @@ pub static SECRETS: LazyLock<SecretsConfig> = LazyLock::new(|| {
serde_yaml::from_reader(File::open(&CONF.secrets_path).expect("secrets file missing"))
.expect("secrets config invalid")
});
-
-pub trait AssetLocationExt {
- fn path(&self) -> PathBuf;
-}
-impl AssetLocationExt for AssetLocation {
- fn path(&self) -> PathBuf {
- match self {
- AssetLocation::Assets(p) => CONF.asset_path.join(p),
- AssetLocation::Cache(p) => CONF.cache_path.join(p),
- AssetLocation::Library(p) => CONF.library_path.join(p),
- AssetLocation::Temp(p) => CONF.temp_path.join(p),
- AssetLocation::Media(p) => CONF.media_path.join(p),
- }
- }
-}
diff --git a/base/src/temp.rs b/base/src/temp.rs
deleted file mode 100644
index ee44004..0000000
--- a/base/src/temp.rs
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- This file is part of jellything (https://codeberg.org/metamuffin/jellything)
- which is licensed under the GNU Affero General Public License (version 3); see /COPYING.
- Copyright (C) 2023 metamuffin <metamuffin.org>
-*/
-use crate::AssetLocationExt;
-use anyhow::Context;
-use jellycommon::AssetLocation;
-use std::{fs::File, sync::atomic::AtomicUsize};
-
-static TEMP_COUNTER: AtomicUsize = AtomicUsize::new(0);
-
-pub struct TempFile(pub AssetLocation);
-
-impl TempFile {
- pub fn new(generate: impl FnOnce(File) -> anyhow::Result<()>) -> anyhow::Result<Self> {
- let i = TEMP_COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
- let loc = AssetLocation::Temp(format!("jellything-temp-{i}").into());
-
- let file = File::create(loc.path()).context("creating temp file")?;
- generate(file).context("tempfile generation")?;
-
- Ok(Self(loc))
- }
-}
-impl Drop for TempFile {
- fn drop(&mut self) {
- std::fs::remove_file(self.0.path()).expect("cant unlink tempfile")
- }
-}
diff --git a/client/src/lib.rs b/client/src/lib.rs
index 2af230f..c770a83 100644
--- a/client/src/lib.rs
+++ b/client/src/lib.rs
@@ -131,6 +131,15 @@ impl Session {
.await
}
+ pub async fn asset(&self, writer: impl UnpinWrite, token: &str, width: usize) -> Result<()> {
+ debug!("downloading asset {token:?} (w={width})");
+ self.download_url(
+ writer,
+ format!("{}/asset/{token}?width={width}", self.instance.base(),),
+ )
+ .await
+ }
+
pub async fn stream(
&self,
writer: impl UnpinWrite,
diff --git a/common/src/lib.rs b/common/src/lib.rs
index c836b95..7e5b68c 100644
--- a/common/src/lib.rs
+++ b/common/src/lib.rs
@@ -14,7 +14,6 @@ pub mod user;
pub use chrono;
use bincode::{Decode, Encode};
-
#[cfg(feature = "rocket")]
use rocket::{FromFormField, UriDisplayQuery};
use serde::{Deserialize, Serialize};
@@ -32,8 +31,6 @@ pub struct Node {
#[derive(Debug, Clone, Deserialize, Serialize, Default, Encode,Decode)]
pub struct NodePrivate {
#[serde(default)] pub id: Option<String>,
- #[serde(default)] pub poster: Option<AssetLocation>,
- #[serde(default)] pub backdrop: Option<AssetLocation>,
#[serde(default)] pub source: Option<Vec<TrackSource>>,
}
@@ -41,6 +38,10 @@ pub struct NodePrivate {
#[derive(Debug, Clone, Deserialize, Serialize, Default, Encode, Decode)]
pub struct NodePublic {
#[serde(default)] pub kind: Option<NodeKind>,
+
+ #[serde(default)] pub poster: Option<Asset>,
+ #[serde(default)] pub backdrop: Option<Asset>,
+
#[serde(default)] pub title: Option<String>,
#[serde(default)] pub id: Option<String>,
#[serde(default)] pub path: Vec<String>,
@@ -54,6 +55,9 @@ pub struct NodePublic {
#[serde(default)] pub federated: Option<String>,
}
+#[derive(Debug, Serialize, Deserialize, Encode, Decode, PartialEq, Eq, Clone)]
+pub struct Asset(pub String);
+
#[derive(Debug, Clone, Deserialize, Serialize, Default, Encode, Decode)]
pub struct ExtendedNode {
pub ids: ObjectIds,
@@ -72,7 +76,7 @@ pub struct Appearance {
#[derive(Debug, Clone, Deserialize, Serialize, Default, Encode, Decode)]
pub struct Person {
pub name: String,
- pub asset: Option<AssetLocation>,
+ pub headshot: Option<Asset>,
pub ids: ObjectIds,
}
@@ -126,7 +130,7 @@ pub enum ImportSource {
path: Option<PathBuf>,
},
Media {
- location: AssetLocation,
+ path: PathBuf,
#[serde(default)]
ignore_metadata: bool,
#[serde(default)]
@@ -139,16 +143,6 @@ pub enum ImportSource {
},
}
-#[derive(Debug, Clone, Deserialize, Serialize, Hash, PartialEq, Eq, Encode, Decode)]
-#[serde(rename_all = "snake_case")]
-pub enum AssetLocation {
- Cache(PathBuf),
- Library(PathBuf),
- Assets(PathBuf),
- Temp(PathBuf),
- Media(PathBuf),
-}
-
#[rustfmt::skip]
#[derive(Debug, Clone, Copy, Deserialize, Serialize, PartialEq, Eq, Default, Encode,Decode)]
#[cfg_attr(feature = "rocket", derive(FromFormField, UriDisplayQuery))]
diff --git a/import/src/lib.rs b/import/src/lib.rs
index d1605fe..c30fe37 100644
--- a/import/src/lib.rs
+++ b/import/src/lib.rs
@@ -13,15 +13,16 @@ use anyhow::{anyhow, bail, Context, Ok};
use async_recursion::async_recursion;
use futures::{stream::FuturesUnordered, StreamExt};
use jellybase::{
+ assetfed::AssetInner,
cache::{async_cache_file, cache_memory},
database::{DataAcid, ReadableTable, Ser, T_NODE, T_NODE_EXTENDED, T_NODE_IMPORT},
federation::Federation,
- AssetLocationExt, CONF, SECRETS,
+ CONF, SECRETS,
};
use jellyclient::Session;
use jellycommon::{
- AssetLocation, AssetRole, ExtendedNode, ImportOptions, ImportSource, MediaInfo, Node, NodeKind,
- NodePrivate, NodePublic, PeopleGroup, Rating, SourceTrack, TrackSource,
+ ExtendedNode, ImportOptions, ImportSource, MediaInfo, Node, NodeKind, NodePrivate, NodePublic,
+ PeopleGroup, Rating, SourceTrack, TrackSource,
};
use jellymatroska::read::EbmlReader;
use jellyremuxer::import::import_metadata;
@@ -325,13 +326,16 @@ async fn process_source(
node_ext.people.entry(group.a()).or_default().push(p.a())
}
}
+ // TODO lazy assets
for (_, ps) in &mut node_ext.people {
for p in ps {
if let Some(id) = p.person.ids.tmdb {
if let Some(tmdb) = &ap.tmdb {
let k = tmdb.person_image(id).await?;
if let Some(prof) = k.profiles.get(0) {
- p.person.asset = Some(tmdb.image(&prof.file_path).await?);
+ p.person.headshot = Some(
+ AssetInner::Cache(tmdb.image(&prof.file_path).await?).ser(),
+ );
}
}
}
@@ -367,11 +371,12 @@ async fn process_source(
let mut node = Node::default();
+ // TODO lazy assets
if let Some(poster) = &details.poster_path {
- node.private.poster = Some(tmdb.image(&poster).await?);
+ node.public.poster = Some(AssetInner::Cache(tmdb.image(&poster).await?).ser());
}
if let Some(backdrop) = &details.backdrop_path {
- node.private.backdrop = Some(tmdb.image(&backdrop).await?);
+ node.public.backdrop = Some(AssetInner::Cache(tmdb.image(&backdrop).await?).ser());
}
node.public.tagline = details.tagline.clone();
@@ -388,16 +393,19 @@ async fn process_source(
insert_node(&id, node)?;
}
ImportSource::Media {
- location,
+ path: mpath,
ignore_attachments,
ignore_chapters,
ignore_metadata,
} => {
- info!("media import {location:?}");
- let media_path = location.path();
- if media_path.is_dir() {
+ info!("media import {mpath:?}");
+ let abspath = CONF.media_path.join(&mpath);
+ if !abspath.exists() {
+ bail!("media missing at {abspath:?}");
+ }
+ if abspath.is_dir() {
let mut node = Node::default();
- for f in media_path.read_dir()? {
+ for f in abspath.read_dir()? {
let f = f?;
let child_path = f.path();
if child_path.is_dir()
@@ -411,12 +419,7 @@ async fn process_source(
process_source(
inf_id.clone(),
ImportSource::Media {
- location: match &location {
- AssetLocation::Media(p) => {
- AssetLocation::Media(p.join(f.file_name()))
- }
- _ => bail!("non media path media"),
- },
+ path: mpath.join(f.file_name()),
ignore_attachments,
ignore_chapters,
ignore_metadata,
@@ -433,25 +436,21 @@ async fn process_source(
}
}
insert_node(&id, node)?;
- } else if media_path.is_file() {
+ } else if abspath.is_file() {
let _permit = SEM_IMPORT.acquire().await.unwrap();
- let location_path = location.path();
let metadata = {
+ let abspath = abspath.clone();
spawn_blocking(move || {
- cache_memory(
- &["mkv-probe", location.path().to_str().unwrap()],
- move || {
- let input = BufReader::new(
- File::open(&location.path()).context("opening media file")?,
- );
- let mut input = EbmlReader::new(input);
- import_metadata(&mut input)
- },
- )
+ cache_memory(&["mkv-probe", abspath.to_str().unwrap()], || {
+ let input =
+ BufReader::new(File::open(&abspath).context("opening media file")?);
+ let mut input = EbmlReader::new(input);
+ import_metadata(&mut input)
+ })
})
}
.await?
- .context(anyhow!("probing {location_path:?}"))?
+ .context(anyhow!("probing {abspath:?}"))?
.deref()
.to_owned();
@@ -476,7 +475,7 @@ async fn process_source(
.track_sources
.into_iter()
.map(|mut ts| {
- ts.path = media_path.to_owned();
+ ts.path = mpath.to_owned();
TrackSource::Local(ts)
})
.collect(),
@@ -484,16 +483,19 @@ async fn process_source(
if !ignore_attachments {
if let Some((filename, data)) = metadata.cover {
- node.private.poster = Some(
- async_cache_file(
- &["att-cover", media_path.to_str().unwrap(), &filename],
- |mut f| async move {
- f.write_all(&data).await?;
- Ok(())
- },
+ node.public.poster = Some(
+ AssetInner::Cache(
+ async_cache_file(
+ &["att-cover", mpath.to_str().unwrap(), &filename],
+ |mut f| async move {
+ f.write_all(&data).await?;
+ Ok(())
+ },
+ )
+ .await?,
)
- .await?,
- );
+ .ser(),
+ )
};
if let Some(infojson) = metadata.infojson {
@@ -521,7 +523,7 @@ async fn process_source(
drop(_permit);
insert_node(&id, node)?;
} else {
- warn!("non file/dir import ignored: {media_path:?}")
+ warn!("non file/dir import ignored: {abspath:?}")
}
}
ImportSource::Federated { host } => {
@@ -618,6 +620,7 @@ fn merge_node(x: Node, y: Node) -> anyhow::Result<Node> {
title: x.public.title.or(y.public.title),
id: x.public.id.or(y.public.id),
path: vec![],
+
children: merge_children(x.public.children, y.public.children),
tagline: x.public.tagline.or(y.public.tagline),
description: x.public.description.or(y.public.description),
@@ -631,11 +634,11 @@ fn merge_node(x: Node, y: Node) -> anyhow::Result<Node> {
.chain(y.public.ratings)
.collect(),
federated: x.public.federated.or(y.public.federated),
+ poster: x.public.poster.or(y.public.poster),
+ backdrop: x.public.backdrop.or(y.public.backdrop),
},
private: NodePrivate {
id: x.private.id.or(y.private.id),
- poster: x.private.poster.or(y.private.poster),
- backdrop: x.private.backdrop.or(y.private.backdrop),
source,
},
})
@@ -754,18 +757,11 @@ async fn import_remote(
None
};
- // TODO maybe use lazy download
- let poster = cache_federation_asset(session.to_owned(), id.clone(), AssetRole::Poster).await?;
- let backdrop =
- cache_federation_asset(session.to_owned(), id.clone(), AssetRole::Backdrop).await?;
-
drop(_permit);
let node = Node {
public: node.clone(),
private: NodePrivate {
- backdrop: Some(backdrop),
- poster: Some(poster),
id: None,
source: track_sources,
},
@@ -788,20 +784,3 @@ async fn import_remote(
Ok(())
}
-
-async fn cache_federation_asset(
- session: Arc<Session>,
- identifier: String,
- role: AssetRole,
-) -> anyhow::Result<AssetLocation> {
- async_cache_file(
- &["fed-asset", role.as_str(), &identifier.clone()],
- move |out| async move {
- let session = session;
- session
- .node_asset(out, identifier.as_str(), role, 1024)
- .await
- },
- )
- .await
-}
diff --git a/import/src/tmdb.rs b/import/src/tmdb.rs
index 5dbc335..3470b90 100644
--- a/import/src/tmdb.rs
+++ b/import/src/tmdb.rs
@@ -1,23 +1,19 @@
-use std::{fmt::Display, sync::Arc};
-
/*
This file is part of jellything (https://codeberg.org/metamuffin/jellything)
which is licensed under the GNU Affero General Public License (version 3); see /COPYING.
- Copyright (C) 2023 metamuffin <metamuffin.org>
+ Copyright (C) 2024 metamuffin <metamuffin.org>
*/
use anyhow::{anyhow, bail, Context};
use bincode::{Decode, Encode};
-use jellybase::cache::{async_cache_file, async_cache_memory};
-use jellycommon::{
- chrono::{format::Parsed, Utc},
- AssetLocation,
-};
+use jellybase::cache::{async_cache_file, async_cache_memory, CachePath};
+use jellycommon::chrono::{format::Parsed, Utc};
use log::info;
use reqwest::{
header::{HeaderMap, HeaderName, HeaderValue},
Client, ClientBuilder,
};
use serde::Deserialize;
+use std::{fmt::Display, sync::Arc};
use tokio::io::AsyncWriteExt;
pub struct Tmdb {
@@ -99,7 +95,7 @@ impl Tmdb {
})
.await
}
- pub async fn image(&self, path: &str) -> anyhow::Result<AssetLocation> {
+ pub async fn image(&self, path: &str) -> anyhow::Result<CachePath> {
async_cache_file(&["api-tmdb-image", path], |mut file| async move {
info!("downloading image {path:?}");
let mut res = self
diff --git a/import/src/trakt.rs b/import/src/trakt.rs
index 08962c6..2bec992 100644
--- a/import/src/trakt.rs
+++ b/import/src/trakt.rs
@@ -1,3 +1,8 @@
+/*
+ This file is part of jellything (https://codeberg.org/metamuffin/jellything)
+ which is licensed under the GNU Affero General Public License (version 3); see /COPYING.
+ Copyright (C) 2024 metamuffin <metamuffin.org>
+*/
use bincode::{Decode, Encode};
use jellybase::cache::async_cache_memory;
use jellycommon::{Appearance, ObjectIds, PeopleGroup, Person, TraktKind};
@@ -226,7 +231,7 @@ impl TraktAppearance {
characters: self.characters.to_owned(),
person: Person {
name: self.person.name.to_owned(),
- asset: None,
+ headshot: None,
ids: self.person.ids.to_owned(),
},
}
diff --git a/server/src/routes/mod.rs b/server/src/routes/mod.rs
index ca8c009..dec0858 100644
--- a/server/src/routes/mod.rs
+++ b/server/src/routes/mod.rs
@@ -33,7 +33,7 @@ use ui::{
r_admin_remove_invite,
user::{r_admin_remove_user, r_admin_user, r_admin_user_permission, r_admin_users},
},
- assets::{r_item_assets, r_node_thumbnail, r_person_asset},
+ assets::{r_asset, r_item_assets, r_node_thumbnail, r_person_asset},
browser::r_all_items_filter,
error::{r_api_catch, r_catch},
home::{r_home, r_home_unpriv},
@@ -98,6 +98,7 @@ pub fn build_rocket(database: DataAcid, federation: Federation) -> Rocket<Build>
r_home_unpriv,
r_streamsync,
r_favicon,
+ r_asset,
r_item_assets,
r_person_asset,
r_search,
diff --git a/server/src/routes/ui/assets.rs b/server/src/routes/ui/assets.rs
index 7973b0c..97396f3 100644
--- a/server/src/routes/ui/assets.rs
+++ b/server/src/routes/ui/assets.rs
@@ -3,22 +3,62 @@
which is licensed under the GNU Affero General Public License (version 3); see /COPYING.
Copyright (C) 2023 metamuffin <metamuffin.org>
*/
-use crate::{
- database::DataAcid,
- routes::ui::{account::session::Session, error::MyResult, CacheControlFile},
-};
+use crate::routes::ui::{account::session::Session, error::MyResult, CacheControlFile};
use anyhow::{anyhow, Context};
+use base64::Engine;
use jellybase::{
+ assetfed::AssetInner,
cache::async_cache_file,
- database::{TableExt, T_NODE, T_NODE_EXTENDED},
+ database::{DataAcid, TableExt, T_NODE, T_NODE_EXTENDED},
federation::Federation,
permission::NodePermissionExt,
- AssetLocationExt,
+ CONF,
};
pub use jellycommon::AssetRole;
-use jellycommon::{AssetLocation, LocalTrack, PeopleGroup, SourceTrackKind, TrackSource};
+use jellycommon::{LocalTrack, PeopleGroup, SourceTrackKind, TrackSource};
use log::info;
-use rocket::{get, http::ContentType, State};
+use rocket::{get, http::ContentType, response::Redirect, State};
+use std::path::PathBuf;
+
+#[get("/asset/<token>?<width>")]
+pub async fn r_asset(
+ _session: Session,
+ fed: &State<Federation>,
+ token: &str,
+ width: Option<usize>,
+) -> MyResult<(ContentType, CacheControlFile)> {
+ let asset = AssetInner::deser(token)?;
+
+ let source = resolve_asset(asset, fed).await.context("resolving asset")?;
+
+ // fit the resolution into a finite set so the maximum cache is finite too.
+ let width = 2usize.pow(width.unwrap_or(2048).clamp(128, 2048).ilog2());
+ let path = jellytranscoder::image::transcode(source, 50., 5, width)
+ .await
+ .context("transcoding asset")?;
+ info!("loading asset from {path:?}");
+ Ok((
+ ContentType::AVIF,
+ CacheControlFile::new_cachekey(&path.abs()).await?,
+ ))
+}
+
+pub async fn resolve_asset(asset: AssetInner, fed: &State<Federation>) -> anyhow::Result<PathBuf> {
+ match asset {
+ AssetInner::Federated { host, asset } => {
+ let session = fed.get_session(&host).await?;
+
+ let asset = base64::engine::general_purpose::URL_SAFE.encode(asset);
+ Ok(async_cache_file(&["fed-asset", &asset], |out| async {
+ session.asset(out, &asset, 2048).await
+ })
+ .await?
+ .abs())
+ }
+ AssetInner::Cache(c) => Ok(c.abs()),
+ AssetInner::Assets(c) => Ok(CONF.asset_path.join(c)),
+ }
+}
#[get("/n/<id>/asset?<role>&<width>")]
pub async fn r_item_assets(
@@ -27,15 +67,15 @@ pub async fn r_item_assets(
id: &str,
role: AssetRole,
width: Option<usize>,
-) -> MyResult<(ContentType, CacheControlFile)> {
+) -> MyResult<Redirect> {
let node = T_NODE
.get(&db, id)?
.only_if_permitted(&session.user.permissions)
.ok_or(anyhow!("node does not exist"))?;
let mut asset = match role {
- AssetRole::Backdrop => node.private.backdrop,
- AssetRole::Poster => node.private.poster,
+ AssetRole::Backdrop => node.public.backdrop,
+ AssetRole::Poster => node.public.poster,
};
if let None = asset {
if let Some(parent) = &node.public.path.last() {
@@ -43,15 +83,18 @@ pub async fn r_item_assets(
.get(&db, parent.as_str())?
.ok_or(anyhow!("node does not exist"))?;
asset = match role {
- AssetRole::Backdrop => parent.private.backdrop,
- AssetRole::Poster => parent.private.poster,
+ AssetRole::Backdrop => parent.public.backdrop,
+ AssetRole::Poster => parent.public.poster,
};
}
};
- let asset = asset.unwrap_or(AssetLocation::Assets(
- format!("fallback-{:?}.avif", node.public.kind.unwrap_or_default()).into(),
- ));
- Ok(asset_with_res(asset, width).await?)
+ let asset = asset.unwrap_or_else(|| {
+ AssetInner::Assets(
+ format!("fallback-{:?}.avif", node.public.kind.unwrap_or_default()).into(),
+ )
+ .ser()
+ });
+ Ok(Redirect::temporary(rocket::uri!(r_asset(asset.0, width))))
}
#[get("/n/<id>/person/<index>/asset?<group>&<width>")]
@@ -62,7 +105,7 @@ pub async fn r_person_asset(
index: usize,
group: PeopleGroup,
width: Option<usize>,
-) -> MyResult<(ContentType, CacheControlFile)> {
+) -> MyResult<Redirect> {
T_NODE
.get(&db, id)?
.only_if_permitted(&session.user.permissions)
@@ -76,13 +119,17 @@ pub async fn r_person_asset(
.get(index)
.ok_or(anyhow!("person does not exist"))?;
- let asset = app.person.asset.to_owned().unwrap_or(AssetLocation::Assets(
- format!("fallback-Person.avif").into(),
- ));
- Ok(asset_with_res(asset, width).await?)
+ let asset = app
+ .person
+ .headshot
+ .to_owned()
+ .unwrap_or(AssetInner::Assets(format!("fallback-Person.avif").into()).ser());
+ Ok(Redirect::temporary(rocket::uri!(r_asset(asset.0, width))))
}
// TODO this can create "federation recursion" because track selection cannot be relied on.
+//? TODO is this still relevant?
+
#[get("/n/<id>/thumbnail?<t>&<width>")]
pub async fn r_node_thumbnail(
session: Session,
@@ -91,7 +138,7 @@ pub async fn r_node_thumbnail(
id: &str,
t: f64,
width: Option<usize>,
-) -> MyResult<(ContentType, CacheControlFile)> {
+) -> MyResult<Redirect> {
let node = T_NODE
.get(&db, id)?
.only_if_permitted(&session.user.permissions)
@@ -120,6 +167,7 @@ pub async fn r_node_thumbnail(
jellytranscoder::thumbnail::create_thumbnail(path, t).await?
}
TrackSource::Remote(_) => {
+ // TODO in the new system this is preferrably a property of node ext for regular fed
let session = fed
.get_session(
thumb_track
@@ -136,21 +184,8 @@ pub async fn r_node_thumbnail(
}
};
- Ok(asset_with_res(asset, width).await?)
-}
-
-async fn asset_with_res(
- asset: AssetLocation,
- width: Option<usize>,
-) -> MyResult<(ContentType, CacheControlFile)> {
- // fit the resolution into a finite set so the maximum cache is finite too.
- let width = 2usize.pow(width.unwrap_or(2048).clamp(128, 2048).ilog2());
- let path = jellytranscoder::image::transcode(asset, 50., 5, width)
- .await
- .context("transcoding asset")?;
- info!("loading asset from {path:?}");
- Ok((
- ContentType::AVIF,
- CacheControlFile::new_cachekey(&path.path()).await?,
- ))
+ Ok(Redirect::temporary(rocket::uri!(r_asset(
+ AssetInner::Cache(asset).ser().0,
+ width
+ ))))
}
diff --git a/server/src/routes/ui/error.rs b/server/src/routes/ui/error.rs
index d111041..8154209 100644
--- a/server/src/routes/ui/error.rs
+++ b/server/src/routes/ui/error.rs
@@ -5,8 +5,7 @@
*/
use super::layout::{DynLayoutPage, LayoutPage};
use crate::{routes::ui::account::rocket_uri_macro_r_account_login, uri};
-use jellybase::AssetLocationExt;
-use jellycommon::AssetLocation;
+use jellybase::CONF;
use log::info;
use rocket::{
catch,
@@ -15,11 +14,11 @@ use rocket::{
Request,
};
use serde_json::{json, Value};
-use std::{fmt::Display, fs::File, io::Read, path::PathBuf, str::FromStr, sync::LazyLock};
+use std::{fmt::Display, fs::File, io::Read, sync::LazyLock};
static ERROR_IMAGE: LazyLock<Vec<u8>> = LazyLock::new(|| {
info!("loading error image");
- let mut f = File::open(AssetLocation::Assets(PathBuf::from_str("error.avif").unwrap()).path())
+ let mut f = File::open(CONF.asset_path.join("error.avif"))
.expect("please create error.avif in the asset dir");
let mut o = Vec::new();
f.read_to_end(&mut o).unwrap();
diff --git a/stream/src/hls.rs b/stream/src/hls.rs
index fb7276d..61d031c 100644
--- a/stream/src/hls.rs
+++ b/stream/src/hls.rs
@@ -51,7 +51,7 @@ pub async fn hls_variant_stream(
) -> Result<()> {
let snips = spawn_blocking(move || {
jellyremuxer::snippet::snippet_index(
- &CONF.library_path,
+ &CONF.media_path,
&node.public,
local_tracks.get(0).ok_or(anyhow!("no track"))?,
)
diff --git a/stream/src/jhls.rs b/stream/src/jhls.rs
index e58aafe..7a7b3af 100644
--- a/stream/src/jhls.rs
+++ b/stream/src/jhls.rs
@@ -26,7 +26,7 @@ pub async fn jhls_index(
.to_owned();
let segments = tokio::task::spawn_blocking(move || {
- jellyremuxer::snippet::snippet_index(&CONF.library_path, &node.public, &local_track)
+ jellyremuxer::snippet::snippet_index(&CONF.media_path, &node.public, &local_track)
})
.await??;
diff --git a/stream/src/lib.rs b/stream/src/lib.rs
index fcd3f9d..3d12f74 100644
--- a/stream/src/lib.rs
+++ b/stream/src/lib.rs
@@ -106,7 +106,7 @@ async fn remux_stream(
jellyremuxer::remux_stream_into(
b,
range,
- CONF.library_path.to_owned(),
+ CONF.media_path.to_owned(),
node.public,
local_tracks,
spec.tracks,
@@ -128,7 +128,7 @@ async fn original_stream(
}
let source = local_tracks[spec.tracks[0]].clone();
- let mut file = File::open(CONF.library_path.join(source.path))
+ let mut file = File::open(CONF.media_path.join(source.path))
.await
.context("opening source")?;
file.seek(SeekFrom::Start(range.start as u64))
diff --git a/stream/src/segment.rs b/stream/src/segment.rs
index a2553bc..fc4aaf4 100644
--- a/stream/src/segment.rs
+++ b/stream/src/segment.rs
@@ -4,7 +4,7 @@
Copyright (C) 2023 metamuffin <metamuffin.org>
*/
use anyhow::{anyhow, bail, Result};
-use jellybase::{permission::PermissionSetExt, AssetLocationExt, CONF};
+use jellybase::{permission::PermissionSetExt, CONF};
use jellycommon::{
stream::StreamSpec,
user::{PermissionSet, UserPermission},
@@ -44,7 +44,7 @@ pub async fn segment_stream(
tokio::task::spawn_blocking(move || {
if let Err(err) = jellyremuxer::write_snippet_into(
SyncIoBridge::new(b),
- &CONF.library_path,
+ &CONF.media_path,
&node.public,
&local_track,
track,
@@ -57,7 +57,7 @@ pub async fn segment_stream(
},
)
.await?;
- let mut output = File::open(location.path()).await?;
+ let mut output = File::open(location.abs()).await?;
tokio::task::spawn(async move {
if let Err(err) = tokio::io::copy(&mut output, &mut b).await {
warn!("cannot write stream: {err}")
@@ -68,7 +68,7 @@ pub async fn segment_stream(
tokio::task::spawn_blocking(move || {
if let Err(err) = jellyremuxer::write_snippet_into(
b,
- &CONF.library_path,
+ &CONF.media_path,
&node.public,
&local_track,
track,
diff --git a/tool/src/add.rs b/tool/src/add.rs
index 3cf13fd..23924cb 100644
--- a/tool/src/add.rs
+++ b/tool/src/add.rs
@@ -7,7 +7,7 @@ use crate::Action;
use anyhow::{anyhow, bail, Context};
use dialoguer::{theme::ColorfulTheme, Confirm, FuzzySelect, Input, MultiSelect};
use jellybase::{CONF, SECRETS};
-use jellycommon::{AssetLocation, ImportOptions, ImportSource, TraktKind};
+use jellycommon::{ImportOptions, ImportSource, TraktKind};
use jellyimport::trakt::Trakt;
use log::warn;
use tokio::{fs::File, io::AsyncWriteExt};
@@ -122,7 +122,7 @@ pub(crate) async fn add(action: Action) -> anyhow::Result<()> {
});
if let Some(media) = media {
sources.push(ImportSource::Media {
- location: AssetLocation::Media(media),
+ path: media,
ignore_metadata: true,
ignore_attachments: false,
ignore_chapters: false,
diff --git a/transcoder/src/image.rs b/transcoder/src/image.rs
index 5df21a9..20f7e0c 100644
--- a/transcoder/src/image.rs
+++ b/transcoder/src/image.rs
@@ -6,37 +6,33 @@
use crate::LOCAL_IMAGE_TRANSCODING_TASKS;
use anyhow::Context;
use image::imageops::FilterType;
-use jellybase::{cache::async_cache_file, AssetLocationExt};
-use jellycommon::AssetLocation;
+use jellybase::cache::{async_cache_file, CachePath};
use log::{debug, info};
use rgb::FromSlice;
use std::{
fs::File,
io::{BufReader, Read, Seek, SeekFrom},
+ path::PathBuf,
};
use tokio::io::AsyncWriteExt;
pub async fn transcode(
- asset: AssetLocation,
+ path: PathBuf,
quality: f32,
speed: u8,
width: usize,
-) -> anyhow::Result<AssetLocation> {
- let original_path = asset.path();
- let asset = asset.clone();
+) -> anyhow::Result<CachePath> {
Ok(async_cache_file(
&[
"image-tc",
- original_path.as_os_str().to_str().unwrap(),
+ path.clone().as_os_str().to_str().unwrap(),
&format!("{width} {quality} {speed}"),
],
move |mut output| async move {
let _permit = LOCAL_IMAGE_TRANSCODING_TASKS.acquire().await?;
- info!("encoding {asset:?} (speed={speed}, quality={quality}, width={width})");
+ info!("encoding {path:?} (speed={speed}, quality={quality}, width={width})");
let encoded = tokio::task::spawn_blocking(move || {
- let original_path = asset.path();
- let mut file =
- BufReader::new(File::open(&original_path).context("opening source")?);
+ let mut file = BufReader::new(File::open(&path).context("opening source")?);
// TODO: use better image library that supports AVIF
let is_avif = {
diff --git a/transcoder/src/snippet.rs b/transcoder/src/snippet.rs
index 5da1ae7..3632919 100644
--- a/transcoder/src/snippet.rs
+++ b/transcoder/src/snippet.rs
@@ -5,8 +5,8 @@
*/
use crate::LOCAL_VIDEO_TRANSCODING_TASKS;
-use jellybase::cache::async_cache_file;
-use jellycommon::{jhls::EncodingProfile, AssetLocation};
+use jellybase::cache::{async_cache_file, CachePath};
+use jellycommon::jhls::EncodingProfile;
use log::{debug, info};
use std::process::Stdio;
use tokio::{
@@ -21,7 +21,7 @@ pub async fn transcode(
key: &str,
enc: &EncodingProfile,
input: impl FnOnce(ChildStdin),
-) -> anyhow::Result<AssetLocation> {
+) -> anyhow::Result<CachePath> {
Ok(async_cache_file(
&["snip-tc", key, &format!("{enc:?}")],
move |mut output| async move {
diff --git a/transcoder/src/thumbnail.rs b/transcoder/src/thumbnail.rs
index 5baf888..9661fd0 100644
--- a/transcoder/src/thumbnail.rs
+++ b/transcoder/src/thumbnail.rs
@@ -1,11 +1,10 @@
use crate::LOCAL_IMAGE_TRANSCODING_TASKS;
-use jellybase::cache::async_cache_file;
-use jellycommon::AssetLocation;
+use jellybase::cache::{async_cache_file, CachePath};
use log::info;
use std::{path::Path, process::Stdio};
use tokio::{io::copy, process::Command};
-pub async fn create_thumbnail(path: &Path, time: f64) -> anyhow::Result<AssetLocation> {
+pub async fn create_thumbnail(path: &Path, time: f64) -> anyhow::Result<CachePath> {
Ok(async_cache_file(
&["thumb", path.to_str().unwrap(), &format!("{time}")],
move |mut output| async move {
generated by cgit v1.2.3-70-g09d2 (git 2.51.0) at 2025-10-08 17:12:22 +0000