aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authormetamuffin <metamuffin@disroot.org>2025-01-30 14:39:20 +0100
committermetamuffin <metamuffin@disroot.org>2025-01-30 14:39:20 +0100
commit02bbb2741f2c463aadf9d07493ebaeac1d73c11a (patch)
tree07cfa4b5ba03bb992b745ff9339c69dc03fca9e9
parent570f24c99af8c9cd1b9050564c32adb85e2c9c0f (diff)
downloadjellything-02bbb2741f2c463aadf9d07493ebaeac1d73c11a.tar
jellything-02bbb2741f2c463aadf9d07493ebaeac1d73c11a.tar.bz2
jellything-02bbb2741f2c463aadf9d07493ebaeac1d73c11a.tar.zst
import channel and children
-rw-r--r--Cargo.lock140
-rw-r--r--base/src/database.rs31
-rw-r--r--common/src/impl.rs4
-rw-r--r--import/Cargo.toml4
-rw-r--r--import/src/infojson.rs16
-rw-r--r--import/src/lib.rs119
-rw-r--r--import/src/matroska.rs112
-rw-r--r--server/src/routes/ui/admin/mod.rs42
-rw-r--r--server/src/routes/ui/node.rs54
-rw-r--r--server/src/routes/ui/sort.rs3
10 files changed, 296 insertions, 229 deletions
diff --git a/Cargo.lock b/Cargo.lock
index 4ab33fa..55c12c8 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -241,9 +241,9 @@ dependencies = [
[[package]]
name = "async-trait"
-version = "0.1.83"
+version = "0.1.85"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd"
+checksum = "3f934833b4b7233644e5848f235df3f57ed8c80f1528a26c3dfa13d2147fa056"
dependencies = [
"proc-macro2",
"quote",
@@ -418,9 +418,9 @@ checksum = "c360505aed52b7ec96a3636c3f039d99103c37d1d9b4f7a8c743d3ea9ffcd03b"
[[package]]
name = "bumpalo"
-version = "3.16.0"
+version = "3.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c"
+checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf"
[[package]]
name = "bytemuck"
@@ -544,9 +544,9 @@ dependencies = [
[[package]]
name = "clap_complete"
-version = "4.5.43"
+version = "4.5.44"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0952013545c9c6dca60f491602655b795c6c062ab180c9cb0bccb83135461861"
+checksum = "375f9d8255adeeedd51053574fd8d4ba875ea5fa558e86617b07f09f1680c8b6"
dependencies = [
"clap",
]
@@ -571,9 +571,9 @@ checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6"
[[package]]
name = "cmake"
-version = "0.1.51"
+version = "0.1.53"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fb1e43aa7fd152b1f968787f7dbcdeb306d1867ff373c69955211876c053f91a"
+checksum = "e24a03c8b52922d68a1589ad61032f2c1aa5a8158d2aa0d93c6e9534944bbad6"
dependencies = [
"cc",
]
@@ -634,9 +634,9 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
[[package]]
name = "cpufeatures"
-version = "0.2.14"
+version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "608697df725056feaccfa42cffdaeeec3fccc4ffc38358ecd19b243e716a78e0"
+checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280"
dependencies = [
"libc",
]
@@ -712,9 +712,9 @@ dependencies = [
[[package]]
name = "data-encoding"
-version = "2.6.0"
+version = "2.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2"
+checksum = "0e60eed09d8c01d3cee5b7d30acb059b76614c918fa0f992e0dd6eeb10daad6f"
[[package]]
name = "deranged"
@@ -804,7 +804,10 @@ checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2"
[[package]]
name = "ebml-struct"
version = "0.1.0"
-source = "git+https://codeberg.org/metamuffin/ebml-struct#5574bebaa4e2104a5fd8ce7eba410e8453b99cf7"
+source = "git+https://codeberg.org/metamuffin/ebml-struct#baa1f77aea4accf7a6046bf6b60275e5d942d816"
+dependencies = [
+ "bincode",
+]
[[package]]
name = "ebml_derive"
@@ -866,12 +869,12 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
[[package]]
name = "errno"
-version = "0.3.9"
+version = "0.3.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba"
+checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d"
dependencies = [
"libc",
- "windows-sys 0.52.0",
+ "windows-sys 0.59.0",
]
[[package]]
@@ -897,9 +900,9 @@ checksum = "9afc2bd4d5a73106dd53d10d73d3401c2f32730ba2c0b93ddb888a8983680471"
[[package]]
name = "fastrand"
-version = "2.1.1"
+version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6"
+checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
[[package]]
name = "fdeflate"
@@ -1289,9 +1292,9 @@ dependencies = [
[[package]]
name = "httparse"
-version = "1.9.5"
+version = "1.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7d71d3574edd2771538b901e6549113b4006ece66150fb69c0fb6d9a2adae946"
+checksum = "f2d708df4e7140240a16cd6ab0ab65c972d7433ab77819ea693fde9c43811e2a"
[[package]]
name = "httpdate"
@@ -1316,9 +1319,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
[[package]]
name = "hyper"
-version = "0.14.31"
+version = "0.14.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8c08302e8fa335b151b788c775ff56e7a03ae64ff85c548ee820fecb70356e85"
+checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7"
dependencies = [
"bytes",
"futures-channel",
@@ -1340,9 +1343,9 @@ dependencies = [
[[package]]
name = "hyper"
-version = "1.5.2"
+version = "1.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "256fb8d4bd6413123cc9d91832d78325c48ff41677595be797d90f42969beae0"
+checksum = "cc2b571658e38e0c01b1fdca3bbbe93c00d3d71693ff2770043f8c29bc7d6f80"
dependencies = [
"bytes",
"futures-channel",
@@ -1365,7 +1368,7 @@ checksum = "2d191583f3da1305256f22463b9bb0471acad48a4e534a5218b9963e9c1f59b2"
dependencies = [
"futures-util",
"http 1.2.0",
- "hyper 1.5.2",
+ "hyper 1.6.0",
"hyper-util",
"rustls",
"rustls-pki-types",
@@ -1386,7 +1389,7 @@ dependencies = [
"futures-util",
"http 1.2.0",
"http-body 1.0.1",
- "hyper 1.5.2",
+ "hyper 1.6.0",
"pin-project-lite",
"socket2",
"tokio",
@@ -1659,19 +1662,19 @@ dependencies = [
[[package]]
name = "ipnet"
-version = "2.10.1"
+version = "2.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ddc24109865250148c2e0f3d25d4f0f479571723792d3802153c60922a4fb708"
+checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130"
[[package]]
name = "is-terminal"
-version = "0.4.13"
+version = "0.4.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "261f68e344040fbd0edea105bef17c66edf46f984ddb1115b775ce31be948f4b"
+checksum = "e19b23d53f35ce9f56aebc7d1bb4e6ac1e9c0db7ac85c8d1760c04379edced37"
dependencies = [
"hermit-abi 0.4.0",
"libc",
- "windows-sys 0.52.0",
+ "windows-sys 0.59.0",
]
[[package]]
@@ -1971,9 +1974,9 @@ dependencies = [
[[package]]
name = "libfuzzer-sys"
-version = "0.4.8"
+version = "0.4.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9b9569d2f74e257076d8c6bfa73fb505b46b851e51ddaecc825944aa3bed17fa"
+checksum = "cf78f52d400cf2d84a3a973a78a592b4adc535739e0a5597a0da6f0c357adc75"
dependencies = [
"arbitrary",
"cc",
@@ -1987,9 +1990,9 @@ checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa"
[[package]]
name = "linux-raw-sys"
-version = "0.4.14"
+version = "0.4.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89"
+checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab"
[[package]]
name = "litemap"
@@ -2893,7 +2896,7 @@ dependencies = [
"http 1.2.0",
"http-body 1.0.1",
"http-body-util",
- "hyper 1.5.2",
+ "hyper 1.6.0",
"hyper-rustls",
"hyper-util",
"ipnet",
@@ -3012,7 +3015,7 @@ dependencies = [
"either",
"futures",
"http 0.2.12",
- "hyper 0.14.31",
+ "hyper 0.14.32",
"indexmap",
"log",
"memchr",
@@ -3069,22 +3072,22 @@ checksum = "c7fb8039b3032c191086b10f11f319a6e99e1e82889c5cc6046f515c9db1d497"
[[package]]
name = "rustix"
-version = "0.38.38"
+version = "0.38.44"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aa260229e6538e52293eeb577aabd09945a09d6d9cc0fc550ed7529056c2e32a"
+checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154"
dependencies = [
"bitflags 2.8.0",
"errno",
"libc",
"linux-raw-sys",
- "windows-sys 0.52.0",
+ "windows-sys 0.59.0",
]
[[package]]
name = "rustls"
-version = "0.23.20"
+version = "0.23.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5065c3f250cbd332cd894be57c40fa52387247659b14a2d6041d121547903b1b"
+checksum = "8f287924602bf649d949c63dc8ac8b235fa5387d394020705b80c4eb597ce5b8"
dependencies = [
"once_cell",
"ring",
@@ -3105,9 +3108,9 @@ dependencies = [
[[package]]
name = "rustls-pki-types"
-version = "1.10.1"
+version = "1.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d2bf47e6ff922db3825eb750c4e2ff784c6ff8fb9e13046ef6a1d1c5401b0b37"
+checksum = "917ce264624a4b4db1c364dcc35bfca9ded014d0a958cd47ad3e960e988ea51c"
dependencies = [
"web-time",
]
@@ -3125,15 +3128,15 @@ dependencies = [
[[package]]
name = "rustversion"
-version = "1.0.18"
+version = "1.0.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0e819f2bc632f285be6d7cd36e25940d45b2391dd6d9b939e79de557f7014248"
+checksum = "f7c45b9784283f1b2e7fb61b42047c2fd678ef0960d4f6f1eba131594cc369d4"
[[package]]
name = "ryu"
-version = "1.0.18"
+version = "1.0.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f"
+checksum = "6ea1a2d0a644769cc99faa24c3ad26b379b786fe7c36fd3c546254801650e6dd"
[[package]]
name = "scoped-tls"
@@ -3558,12 +3561,13 @@ checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1"
[[package]]
name = "tempfile"
-version = "3.13.0"
+version = "3.16.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f0f2c9fc62d0beef6951ccffd757e241266a2c833136efbe35af6cd2567dca5b"
+checksum = "38c246215d7d24f48ae091a2902398798e05d978b24315d6efbc00ede9a8bb91"
dependencies = [
"cfg-if",
"fastrand",
+ "getrandom 0.3.1",
"once_cell",
"rustix",
"windows-sys 0.59.0",
@@ -3632,9 +3636,9 @@ dependencies = [
[[package]]
name = "time"
-version = "0.3.36"
+version = "0.3.37"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885"
+checksum = "35e7868883861bd0e56d9ac6efcaaca0d6d5d82a2a7ec8209ff492c07cf37b21"
dependencies = [
"deranged",
"itoa",
@@ -3653,9 +3657,9 @@ checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3"
[[package]]
name = "time-macros"
-version = "0.2.18"
+version = "0.2.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf"
+checksum = "2834e6017e3e5e4b9834939793b282bc03b37a3336245fa820e35e233e2a85de"
dependencies = [
"num-conv",
"time-core",
@@ -3727,9 +3731,9 @@ dependencies = [
[[package]]
name = "tokio-stream"
-version = "0.1.16"
+version = "0.1.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4f4e6ce100d0eb49a2734f8c0812bcd324cf357d21810932c5df6b96ef2b86f1"
+checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047"
dependencies = [
"futures-core",
"pin-project-lite",
@@ -3867,9 +3871,9 @@ dependencies = [
[[package]]
name = "tracing-subscriber"
-version = "0.3.18"
+version = "0.3.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b"
+checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008"
dependencies = [
"matchers",
"nu-ansi-term",
@@ -3935,9 +3939,9 @@ dependencies = [
[[package]]
name = "unicode-ident"
-version = "1.0.15"
+version = "1.0.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "11cd88e12b17c6494200a9c1b683a04fcac9573ed74cd1b62aeb2727c5592243"
+checksum = "a210d160f08b701c8721ba1c726c11662f877ea6b7094007e1ca9a1041945034"
[[package]]
name = "unicode-width"
@@ -4022,9 +4026,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
[[package]]
name = "uuid"
-version = "1.11.0"
+version = "1.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f8c5f0a0af699448548ad1a2fbf920fb4bee257eae39953ba95cb84891a0446a"
+checksum = "b3758f5e68192bb96cc8f9b7e2c2cfdabb435499a28499a42f8f984092adad4b"
dependencies = [
"getrandom 0.2.15",
"serde",
@@ -4043,9 +4047,9 @@ dependencies = [
[[package]]
name = "valuable"
-version = "0.1.0"
+version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d"
+checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65"
[[package]]
name = "version-compare"
@@ -4192,9 +4196,9 @@ dependencies = [
[[package]]
name = "webpki-roots"
-version = "0.26.6"
+version = "0.26.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "841c67bff177718f1d4dfefde8d8f0e78f9b6589319ba88312f567fc5841a958"
+checksum = "5d642ff16b7e79272ae451b7322067cdc17cadf68c23264be9d94a32319efe7e"
dependencies = [
"rustls-pki-types",
]
@@ -4416,9 +4420,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
[[package]]
name = "winnow"
-version = "0.6.24"
+version = "0.6.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c8d71a593cc5c42ad7876e2c1fda56f314f3754c084128833e64f1345ff8a03a"
+checksum = "ad699df48212c6cc6eb4435f35500ac6fd3b9913324f938aea302022ce19d310"
dependencies = [
"memchr",
]
diff --git a/base/src/database.rs b/base/src/database.rs
index e9fe156..bec630a 100644
--- a/base/src/database.rs
+++ b/base/src/database.rs
@@ -10,7 +10,7 @@ use jellycommon::{
Node, NodeID,
};
use log::info;
-use redb::{ReadableTable, TableDefinition};
+use redb::{ReadableTable, StorageError, TableDefinition};
use std::{
fs::create_dir_all,
path::Path,
@@ -29,6 +29,8 @@ const T_USER_NODE: TableDefinition<(&str, [u8; 32]), Ser<NodeUserData>> =
TableDefinition::new("user_node");
const T_INVITE: TableDefinition<&str, ()> = TableDefinition::new("invite");
const T_NODE: TableDefinition<[u8; 32], Ser<Node>> = TableDefinition::new("node");
+const T_NODE_CHILDREN: TableDefinition<([u8; 32], [u8; 32]), ()> =
+ TableDefinition::new("node_children");
const T_IMPORT_FILE_MTIME: TableDefinition<&[u8], u64> = TableDefinition::new("import_file_mtime");
#[derive(Clone)]
@@ -75,11 +77,21 @@ impl Database {
Ok(None)
}
}
+ pub fn get_node_children(&self, id: NodeID) -> Result<Vec<NodeID>> {
+ let txn = self.inner.begin_read()?;
+ let t_node_children = txn.open_table(T_NODE_CHILDREN)?;
+ Ok(t_node_children
+ .range((id.0, NodeID::MIN.0)..(id.0, NodeID::MAX.0))?
+ .map(|r| r.map(|r| NodeID(r.0.value().1)))
+ .collect::<Result<Vec<_>, StorageError>>()?)
+ }
pub fn clear_nodes(&self) -> Result<()> {
let txn = self.inner.begin_write()?;
- let mut table = txn.open_table(T_NODE)?;
- table.retain(|_, _| false)?;
- drop(table);
+ let mut t_node = txn.open_table(T_NODE)?;
+ let mut t_node_children = txn.open_table(T_NODE_CHILDREN)?;
+ t_node.retain(|_, _| false)?;
+ t_node_children.retain(|_, _| false)?;
+ drop((t_node, t_node_children));
txn.commit()?;
Ok(())
}
@@ -209,7 +221,7 @@ impl Database {
txn.commit()?;
Ok(())
}
- pub fn list_nodes_with_udata(&self, username: &str) -> Result<Vec<(Node, NodeUserData)>> {
+ pub fn list_nodes_with_udata(&self, username: &str) -> Result<Vec<(Arc<Node>, NodeUserData)>> {
let txn = self.inner.begin_read()?;
let nodes = txn.open_table(T_NODE)?;
let node_users = txn.open_table(T_USER_NODE)?;
@@ -217,7 +229,7 @@ impl Database {
.iter()?
.map(|a| {
let (x, y) = a.unwrap();
- let (x, y) = (x.value().to_owned(), y.value().0);
+ let (x, y) = (x.value().to_owned(), Arc::new(y.value().0));
let z = node_users
.get(&(username, x))
.unwrap()
@@ -285,12 +297,15 @@ impl Database {
) -> Result<()> {
let txn = self.inner.begin_write()?;
let mut t_nodes = txn.open_table(T_NODE)?;
+ let mut t_node_children = txn.open_table(T_NODE_CHILDREN)?;
let mut node = t_nodes.get(id.0)?.map(|v| v.value().0).unwrap_or_default();
update(&mut node)?;
+ for parent in &node.parents {
+ t_node_children.insert((parent.0, id.0), ())?;
+ }
t_nodes.insert(&id.0, Ser(node))?;
- drop(t_nodes);
+ drop((t_nodes, t_node_children));
txn.commit()?;
-
Ok(())
}
pub fn get_import_file_mtime(&self, path: &Path) -> Result<Option<u64>> {
diff --git a/common/src/impl.rs b/common/src/impl.rs
index a98015a..3814b1d 100644
--- a/common/src/impl.rs
+++ b/common/src/impl.rs
@@ -166,3 +166,7 @@ impl Node {
NodeID::from_node(self)
}
}
+impl NodeID {
+ pub const MIN: NodeID = NodeID([0; 32]);
+ pub const MAX: NodeID = NodeID([255; 32]);
+}
diff --git a/import/Cargo.toml b/import/Cargo.toml
index d0342df..988e626 100644
--- a/import/Cargo.toml
+++ b/import/Cargo.toml
@@ -8,7 +8,9 @@ jellycommon = { path = "../common" }
jellybase = { path = "../base" }
jellyclient = { path = "../client" }
-ebml-struct = { git = "https://codeberg.org/metamuffin/ebml-struct" }
+ebml-struct = { git = "https://codeberg.org/metamuffin/ebml-struct", features = [
+ "bincode",
+] }
rayon = "1.10.0"
crossbeam-channel = "0.5.14"
diff --git a/import/src/infojson.rs b/import/src/infojson.rs
index c2ae305..3e4667e 100644
--- a/import/src/infojson.rs
+++ b/import/src/infojson.rs
@@ -3,13 +3,13 @@
which is licensed under the GNU Affero General Public License (version 3); see /COPYING.
Copyright (C) 2025 metamuffin <metamuffin.org>
*/
-
use anyhow::Context;
+use bincode::{Decode, Encode};
use jellycommon::chrono::{format::Parsed, Utc};
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize, Encode, Decode)]
pub struct YVideo {
pub id: String,
pub title: String,
@@ -63,7 +63,7 @@ pub struct YVideo {
pub epoch: usize,
}
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize, Encode, Decode)]
pub struct YCaption {
pub url: Option<String>,
pub ext: String, //"vtt" | "json3" | "srv1" | "srv2" | "srv3" | "ttml",
@@ -71,7 +71,7 @@ pub struct YCaption {
pub name: Option<String>,
}
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize, Encode, Decode)]
pub struct YFormat {
pub format_id: String,
pub format_note: Option<String>,
@@ -96,13 +96,13 @@ pub struct YFormat {
pub format: String,
}
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize, Encode, Decode)]
pub struct YFragment {
pub url: Option<String>,
pub duration: Option<f64>,
}
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize, Encode, Decode)]
pub struct YThumbnail {
pub url: String,
pub preference: Option<i32>,
@@ -112,14 +112,14 @@ pub struct YThumbnail {
pub resolution: Option<String>,
}
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize, Encode, Decode)]
pub struct YChapter {
pub start_time: f64,
pub end_time: f64,
pub title: String,
}
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize, Encode, Decode)]
pub struct YHeatmapSample {
pub start_time: f64,
pub end_time: f64,
diff --git a/import/src/lib.rs b/import/src/lib.rs
index add7e4d..10bd0ec 100644
--- a/import/src/lib.rs
+++ b/import/src/lib.rs
@@ -4,24 +4,19 @@
Copyright (C) 2025 metamuffin <metamuffin.org>
*/
use anyhow::{anyhow, Context, Result};
-use ebml_struct::{
- ids::*,
- matroska::*,
- read::{EbmlReadExt, TagRead},
-};
use infojson::YVideo;
-use jellybase::{assetfed::AssetInner, cache::cache_file, database::Database, CONF, SECRETS};
+use jellybase::{assetfed::AssetInner, database::Database, CONF, SECRETS};
use jellycommon::{
Chapter, LocalTrack, MediaInfo, Node, NodeID, NodeKind, Rating, SourceTrack, SourceTrackKind,
TrackSource,
};
use log::info;
+use matroska::matroska_metadata;
use rayon::iter::{ParallelDrainRange, ParallelIterator};
-use regex::Regex;
use std::{
collections::HashMap,
fs::File,
- io::{BufReader, ErrorKind, Read, Write},
+ io::{BufReader, Read},
mem::swap,
path::{Path, PathBuf},
sync::LazyLock,
@@ -35,14 +30,15 @@ use tokio::{
use trakt::Trakt;
pub mod infojson;
+pub mod matroska;
pub mod tmdb;
pub mod trakt;
static IMPORT_SEM: LazyLock<Semaphore> = LazyLock::new(|| Semaphore::new(1));
pub static IMPORT_ERRORS: RwLock<Vec<String>> = RwLock::const_new(Vec::new());
-static RE_EPISODE_FILENAME: LazyLock<Regex> =
- LazyLock::new(|| Regex::new(r#"([sS](\d+))?([eE](\d+))( (.+))?"#).unwrap());
+// static RE_EPISODE_FILENAME: LazyLock<Regex> =
+// LazyLock::new(|| Regex::new(r#"([sS](\d+))?([eE](\d+))( (.+))?"#).unwrap());
struct Apis {
trakt: Option<Trakt>,
@@ -57,6 +53,7 @@ pub async fn import_wrap(db: Database, incremental: bool) -> Result<()> {
let _sem = IMPORT_SEM.try_acquire()?;
let jh = spawn_blocking(move || {
+ *IMPORT_ERRORS.blocking_write() = Vec::new();
if let Err(e) = import(&db, incremental) {
IMPORT_ERRORS.blocking_write().push(format!("{e:#}"));
}
@@ -121,25 +118,26 @@ fn import_iter_inner(path: &Path, db: &Database, incremental: bool) -> Result<Ve
}
fn import_file(db: &Database, path: &Path) -> Result<()> {
- let parent = NodeID::from_slug(
- &path
- .parent()
- .ok_or(anyhow!("no parent"))?
- .file_name()
- .ok_or(anyhow!("parent no filename"))?
- .to_string_lossy(),
- );
+ let parent_slug = path
+ .parent()
+ .ok_or(anyhow!("no parent"))?
+ .file_name()
+ .ok_or(anyhow!("parent no filename"))?
+ .to_string_lossy();
+ let parent = NodeID::from_slug(&parent_slug);
let filename = path.file_name().unwrap().to_string_lossy();
match filename.as_ref() {
"poster.jpeg" | "poster.webp" => {
db.update_node_init(parent, |node| {
+ node.slug = parent_slug.to_string();
node.poster = Some(AssetInner::Media(path.to_owned()).ser());
Ok(())
})?;
}
"backdrop.jpeg" | "backdrop.webp" => {
db.update_node_init(parent, |node| {
+ node.slug = parent_slug.to_string();
node.backdrop = Some(AssetInner::Media(path.to_owned()).ser());
Ok(())
})?;
@@ -147,6 +145,7 @@ fn import_file(db: &Database, path: &Path) -> Result<()> {
"info.json" | "info.yaml" => {
let data = serde_yaml::from_reader::<_, Node>(BufReader::new(File::open(path)?))?;
db.update_node_init(parent, |node| {
+ node.slug = parent_slug.to_string();
fn merge_option<T>(a: &mut Option<T>, b: Option<T>) {
if b.is_some() {
*a = b;
@@ -161,6 +160,7 @@ fn import_file(db: &Database, path: &Path) -> Result<()> {
"channel.info.json" => {
let data = serde_json::from_reader::<_, YVideo>(BufReader::new(File::open(path)?))?;
db.update_node_init(parent, |node| {
+ node.slug = parent_slug.to_string();
node.title = Some(
data.title
.strip_suffix(" - Videos")
@@ -189,76 +189,14 @@ fn import_file(db: &Database, path: &Path) -> Result<()> {
fn import_media_file(db: &Database, path: &Path, parent: NodeID) -> Result<()> {
info!("reading media file {path:?}");
- let mut file = BufReader::new(File::open(path)?);
- let mut file = file.by_ref().take(u64::MAX);
-
- let (x, mut ebml) = file.read_tag()?;
- assert_eq!(x, EL_EBML);
- let ebml = Ebml::read(&mut ebml).unwrap();
- assert!(ebml.doc_type == "matroska" || ebml.doc_type == "webm");
- let (x, mut segment) = file.read_tag()?;
- assert_eq!(x, EL_SEGMENT);
- let mut info = None;
- let mut infojson = None;
- let mut tracks = None;
- let mut cover = None;
- let mut chapters = None;
- let mut tags = None;
- loop {
- let (x, mut seg) = match segment.read_tag() {
- Ok(o) => o,
- Err(e) if e.kind() == ErrorKind::UnexpectedEof => break,
- Err(e) => return Err(e.into()),
- };
- match x {
- EL_INFO => info = Some(Info::read(&mut seg).context("info")?),
- EL_TRACKS => tracks = Some(Tracks::read(&mut seg).context("tracks")?),
- EL_CHAPTERS => chapters = Some(Chapters::read(&mut seg).context("chapters")?),
- EL_TAGS => tags = Some(Tags::read(&mut seg).context("tags")?),
- EL_ATTACHMENTS => {
- let attachments = Attachments::read(&mut seg).context("attachments")?;
- for f in attachments.files {
- match f.name.as_str() {
- "info.json" => {
- infojson = Some(
- serde_json::from_slice::<infojson::YVideo>(&f.data)
- .context("infojson")?,
- );
- }
- "cover.webp" | "cover.png" | "cover.jpg" | "cover.jpeg" | "cover.avif" => {
- cover = Some(
- AssetInner::Cache(cache_file(
- &["att-cover", path.to_string_lossy().as_ref()],
- move |mut file| {
- file.write_all(&f.data)?;
- Ok(())
- },
- )?)
- .ser(),
- )
- }
- a => println!("{a:?}"),
- }
- }
- }
- EL_VOID | EL_CRC32 | EL_CUES | EL_SEEKHEAD => {
- seg.consume()?;
- }
- EL_CLUSTER => {
- break;
- }
- id => {
- eprintln!("unknown top-level element {id:x}");
- seg.consume()?;
- }
- }
- }
+ let m = (*matroska_metadata(path)?).to_owned();
- let info = info.ok_or(anyhow!("no info"))?;
- let tracks = tracks.ok_or(anyhow!("no tracks"))?;
+ let info = m.info.ok_or(anyhow!("no info"))?;
+ let tracks = m.tracks.ok_or(anyhow!("no tracks"))?;
- let mut tags = tags
+ let mut tags = m
+ .tags
.map(|tags| {
tags.tags
.into_iter()
@@ -274,7 +212,8 @@ fn import_media_file(db: &Database, path: &Path, parent: NodeID) -> Result<()> {
.to_string_lossy()
.to_string();
- let slug = infojson
+ let slug = m
+ .infojson
.as_ref()
.map(|ij| format!("youtube-{}", ij.id))
.unwrap_or(make_kebab(&filepath_stem));
@@ -282,13 +221,13 @@ fn import_media_file(db: &Database, path: &Path, parent: NodeID) -> Result<()> {
db.update_node_init(NodeID::from_slug(&slug), |node| {
node.slug = slug;
node.title = info.title;
- node.poster = cover;
+ node.poster = m.cover.clone();
node.description = tags.remove("DESCRIPTION");
node.tagline = tags.remove("COMMENT");
if !node.parents.contains(&parent) {
node.parents.push(parent)
}
- if let Some(infojson) = infojson {
+ if let Some(infojson) = m.infojson {
node.kind = Some(
if infojson.duration.unwrap_or(0.) < 600.
&& infojson.aspect_ratio.unwrap_or(2.) < 1.
@@ -314,7 +253,9 @@ fn import_media_file(db: &Database, path: &Path, parent: NodeID) -> Result<()> {
}
}
node.media = Some(MediaInfo {
- chapters: chapters
+ chapters: m
+ .chapters
+ .clone()
.map(|c| {
let mut chaps = Vec::new();
if let Some(ee) = c.edition_entries.first() {
diff --git a/import/src/matroska.rs b/import/src/matroska.rs
new file mode 100644
index 0000000..bb8d927
--- /dev/null
+++ b/import/src/matroska.rs
@@ -0,0 +1,112 @@
+/*
+ This file is part of jellything (https://codeberg.org/metamuffin/jellything)
+ which is licensed under the GNU Affero General Public License (version 3); see /COPYING.
+ Copyright (C) 2025 metamuffin <metamuffin.org>
+*/
+use crate::infojson::{self, YVideo};
+use anyhow::{Context, Result};
+use bincode::{Decode, Encode};
+use ebml_struct::{
+ ids::*,
+ matroska::*,
+ read::{EbmlReadExt, TagRead},
+};
+use jellybase::{
+ assetfed::AssetInner,
+ cache::{cache_file, cache_memory},
+};
+use jellycommon::Asset;
+use std::{
+ fs::File,
+ io::{BufReader, ErrorKind, Read, Write},
+ path::Path,
+ sync::Arc,
+};
+
+#[derive(Encode, Decode, Clone)]
+pub(crate) struct MatroskaMetadata {
+ pub info: Option<Info>,
+ pub tracks: Option<Tracks>,
+ pub cover: Option<Asset>,
+ pub chapters: Option<Chapters>,
+ pub tags: Option<Tags>,
+ pub infojson: Option<YVideo>,
+}
+pub(crate) fn matroska_metadata(path: &Path) -> Result<Arc<MatroskaMetadata>> {
+ cache_memory(&["mkmeta-v1", path.to_string_lossy().as_ref()], || {
+ let mut file = BufReader::new(File::open(path)?);
+ let mut file = file.by_ref().take(u64::MAX);
+
+ let (x, mut ebml) = file.read_tag()?;
+ assert_eq!(x, EL_EBML);
+ let ebml = Ebml::read(&mut ebml).unwrap();
+ assert!(ebml.doc_type == "matroska" || ebml.doc_type == "webm");
+ let (x, mut segment) = file.read_tag()?;
+ assert_eq!(x, EL_SEGMENT);
+
+ let mut info = None;
+ let mut infojson = None;
+ let mut tracks = None;
+ let mut cover = None;
+ let mut chapters = None;
+ let mut tags = None;
+ loop {
+ let (x, mut seg) = match segment.read_tag() {
+ Ok(o) => o,
+ Err(e) if e.kind() == ErrorKind::UnexpectedEof => break,
+ Err(e) => return Err(e.into()),
+ };
+ match x {
+ EL_INFO => info = Some(Info::read(&mut seg).context("info")?),
+ EL_TRACKS => tracks = Some(Tracks::read(&mut seg).context("tracks")?),
+ EL_CHAPTERS => chapters = Some(Chapters::read(&mut seg).context("chapters")?),
+ EL_TAGS => tags = Some(Tags::read(&mut seg).context("tags")?),
+ EL_ATTACHMENTS => {
+ let attachments = Attachments::read(&mut seg).context("attachments")?;
+ for f in attachments.files {
+ match f.name.as_str() {
+ "info.json" => {
+ infojson = Some(
+ serde_json::from_slice::<infojson::YVideo>(&f.data)
+ .context("infojson")?,
+ );
+ }
+ "cover.webp" | "cover.png" | "cover.jpg" | "cover.jpeg"
+ | "cover.avif" => {
+ cover = Some(
+ AssetInner::Cache(cache_file(
+ &["att-cover", path.to_string_lossy().as_ref()],
+ move |mut file| {
+ file.write_all(&f.data)?;
+ Ok(())
+ },
+ )?)
+ .ser(),
+ )
+ }
+ a => println!("{a:?}"),
+ }
+ }
+ }
+ EL_VOID | EL_CRC32 | EL_CUES | EL_SEEKHEAD => {
+ seg.consume()?;
+ }
+ EL_CLUSTER => {
+ break;
+ }
+ id => {
+ eprintln!("unknown top-level element {id:x}");
+ seg.consume()?;
+ }
+ }
+ }
+ Ok(MatroskaMetadata {
+ chapters,
+ cover,
+ info,
+ infojson,
+ tags,
+ tracks,
+ })
+ })
+}
diff --git a/server/src/routes/ui/admin/mod.rs b/server/src/routes/ui/admin/mod.rs
index 5c2c48f..50faa2e 100644
--- a/server/src/routes/ui/admin/mod.rs
+++ b/server/src/routes/ui/admin/mod.rs
@@ -6,7 +6,10 @@
pub mod log;
pub mod user;
-use super::account::session::AdminSession;
+use super::{
+ account::session::AdminSession,
+ assets::{resolve_asset, AVIF_QUALITY, AVIF_SPEED},
+};
use crate::{
database::Database,
routes::ui::{
@@ -17,7 +20,7 @@ use crate::{
uri,
};
use anyhow::{anyhow, Context};
-use jellybase::{federation::Federation, CONF};
+use jellybase::{assetfed::AssetInner, federation::Federation, CONF};
use jellyimport::{import_wrap, is_importing, IMPORT_ERRORS};
use markup::DynRender;
use rand::Rng;
@@ -196,25 +199,22 @@ pub async fn r_admin_transcode_posters(
let t = Instant::now();
- // TODO
- // {
- // let txn = database.begin_read()?;
- // let nodes = txn.open_table(T_NODE)?;
- // for node in nodes.iter()? {
- // let (_, node) = node?;
- // if let Some(poster) = &node.value().0.poster {
- // let asset = AssetInner::deser(&poster.0)?;
- // if asset.is_federated() {
- // continue;
- // }
- // let source = resolve_asset(asset).await.context("resolving asset")?;
- // jellytranscoder::image::transcode(source, AVIF_QUALITY, AVIF_SPEED, 1024)
- // .await
- // .context("transcoding asset")?;
- // }
- // }
- // }
- // drop(_permit);
+ {
+ let nodes = database.list_nodes_with_udata("")?;
+ for (node, _) in nodes {
+ if let Some(poster) = &node.poster {
+ let asset = AssetInner::deser(&poster.0)?;
+ if asset.is_federated() {
+ continue;
+ }
+ let source = resolve_asset(asset).await.context("resolving asset")?;
+ jellytranscoder::image::transcode(source, AVIF_QUALITY, AVIF_SPEED, 1024)
+ .await
+ .context("transcoding asset")?;
+ }
+ }
+ }
+ drop(_permit);
admin_dashboard(
database,
diff --git a/server/src/routes/ui/node.rs b/server/src/routes/ui/node.rs
index 3332483..121896e 100644
--- a/server/src/routes/ui/node.rs
+++ b/server/src/routes/ui/node.rs
@@ -1,5 +1,3 @@
-use std::sync::Arc;
-
/*
This file is part of jellything (https://codeberg.org/metamuffin/jellything)
which is licensed under the GNU Affero General Public License (version 3); see /COPYING.
@@ -37,6 +35,7 @@ use jellycommon::{
Chapter, MediaInfo, Node, NodeID, NodeKind, PeopleGroup, Rating, SourceTrackKind,
};
use rocket::{get, serde::json::Json, Either, State};
+use std::sync::Arc;
/// This function is a stub and only useful for use in the uri! macro.
#[get("/n/<id>")]
@@ -44,43 +43,32 @@ pub fn r_library_node(id: String) {
drop(id)
}
-#[get("/n/<id>?<filter..>")]
+#[get("/n/<slug>?<filter..>")]
pub async fn r_library_node_filter<'a>(
session: Session,
- id: &'a str,
+ slug: &'a str,
db: &'a State<Database>,
aj: AcceptJson,
filter: NodeFilterSort,
) -> MyResult<Either<DynLayoutPage<'a>, Json<Node>>> {
- let (node, udata) = db.get_node_with_userdata(NodeID::from_slug(id), &session)?;
+ let id = NodeID::from_slug(slug);
+ let (node, udata) = db.get_node_with_userdata(id, &session)?;
if *aj {
return Ok(Either::Right(Json((*node).clone())));
}
- // let mut children = node
- // .children
- // .iter()
- // .map(|c| db.get_node_with_userdata(c, &session))
- // .collect::<anyhow::Result<Vec<_>>>()?
- // .into_iter()
- // .collect();
+ let mut children = db
+ .get_node_children(id)?
+ .into_iter()
+ .map(|c| db.get_node_with_userdata(c, &session))
+ .collect::<anyhow::Result<Vec<_>>>()?;
- // let path = node
- // .path
- // .iter()
- // .map(|c| {
- // Ok((
- // c.to_owned(),
- // T_NODE
- // .get(db, c.as_str())?
- // .ok_or(anyhow!("parent node missing"))?
- // .public,
- // ))
- // })
- // .collect::<anyhow::Result<Vec<_>>>()?
- // .into_iter()
- // .collect::<Vec<_>>();
+ let parents = node
+ .parents
+ .iter()
+ .flat_map(|pid| db.get_node(*pid).transpose())
+ .collect::<Result<Vec<_>, _>>()?;
filter_and_sort_nodes(
&filter,
@@ -89,13 +77,13 @@ pub async fn r_library_node_filter<'a>(
_ => (SortProperty::Title, SortOrder::Ascending),
},
// TODO
- &mut Vec::new(),
+ &mut children,
);
Ok(Either::Left(LayoutPage {
title: node.title.clone().unwrap_or_default(),
content: markup::new! {
- @NodePage { node: &node, id, udata: &udata, children: &[], path: &[], filter: &filter }
+ @NodePage { node: &node, id: slug, udata: &udata, children: &children, parents: &parents, filter: &filter }
},
..Default::default()
}))
@@ -128,7 +116,7 @@ markup::define! {
}
}
}
- NodePage<'a>(id: &'a str, node: &'a Node, udata: &'a NodeUserData, children: &'a [(Arc<Node>, NodeUserData)], path: &'a [(String, Node)], filter: &'a NodeFilterSort) {
+ NodePage<'a>(id: &'a str, node: &'a Node, udata: &'a NodeUserData, children: &'a [(Arc<Node>, NodeUserData)], parents: &'a [Arc<Node>], filter: &'a NodeFilterSort) {
@if !matches!(node.kind.unwrap_or_default(), NodeKind::Collection) {
img.backdrop[src=uri!(r_item_backdrop(id, Some(2048))), loading="lazy"];
}
@@ -139,9 +127,9 @@ markup::define! {
}
.title {
h1 { @node.title }
- span.path { @for (cid, cnode) in *path {
- " / " a.component[href=uri!(r_library_node(cid))] { @cnode.title }
- }}
+ ul.parents { @for node in *parents { li {
+ a.component[href=uri!(r_library_node(&node.slug))] { @node.title }
+ }}}
@if node.media.is_some() { a.play[href=&uri!(r_player(id, PlayerConfig::default()))] { "Watch now" }}
@if !matches!(node.kind.unwrap_or_default(), NodeKind::Collection | NodeKind::Channel) {
@if matches!(udata.watched, WatchedState::None | WatchedState::Pending | WatchedState::Progress(_)) {
diff --git a/server/src/routes/ui/sort.rs b/server/src/routes/ui/sort.rs
index 705b616..68bd588 100644
--- a/server/src/routes/ui/sort.rs
+++ b/server/src/routes/ui/sort.rs
@@ -8,6 +8,7 @@ use rocket::{
http::uri::fmt::{Query, UriDisplay},
FromForm, FromFormField, UriDisplayQuery,
};
+use std::sync::Arc;
#[derive(FromForm, UriDisplayQuery, Default, Clone)]
pub struct NodeFilterSort {
@@ -134,7 +135,7 @@ pub enum SortOrder {
pub fn filter_and_sort_nodes(
f: &NodeFilterSort,
default_sort: (SortProperty, SortOrder),
- nodes: &mut Vec<(Node, NodeUserData)>,
+ nodes: &mut Vec<(Arc<Node>, NodeUserData)>,
) {
let sort_prop = f.sort_by.unwrap_or(default_sort.0);
nodes.retain(|(node, udata)| {