diff options
author | metamuffin <metamuffin@disroot.org> | 2025-01-30 14:39:20 +0100 |
---|---|---|
committer | metamuffin <metamuffin@disroot.org> | 2025-01-30 14:39:20 +0100 |
commit | 02bbb2741f2c463aadf9d07493ebaeac1d73c11a (patch) | |
tree | 07cfa4b5ba03bb992b745ff9339c69dc03fca9e9 | |
parent | 570f24c99af8c9cd1b9050564c32adb85e2c9c0f (diff) | |
download | jellything-02bbb2741f2c463aadf9d07493ebaeac1d73c11a.tar jellything-02bbb2741f2c463aadf9d07493ebaeac1d73c11a.tar.bz2 jellything-02bbb2741f2c463aadf9d07493ebaeac1d73c11a.tar.zst |
import channel and children
-rw-r--r-- | Cargo.lock | 140 | ||||
-rw-r--r-- | base/src/database.rs | 31 | ||||
-rw-r--r-- | common/src/impl.rs | 4 | ||||
-rw-r--r-- | import/Cargo.toml | 4 | ||||
-rw-r--r-- | import/src/infojson.rs | 16 | ||||
-rw-r--r-- | import/src/lib.rs | 119 | ||||
-rw-r--r-- | import/src/matroska.rs | 112 | ||||
-rw-r--r-- | server/src/routes/ui/admin/mod.rs | 42 | ||||
-rw-r--r-- | server/src/routes/ui/node.rs | 54 | ||||
-rw-r--r-- | server/src/routes/ui/sort.rs | 3 |
10 files changed, 296 insertions, 229 deletions
@@ -241,9 +241,9 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.83" +version = "0.1.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" +checksum = "3f934833b4b7233644e5848f235df3f57ed8c80f1528a26c3dfa13d2147fa056" dependencies = [ "proc-macro2", "quote", @@ -418,9 +418,9 @@ checksum = "c360505aed52b7ec96a3636c3f039d99103c37d1d9b4f7a8c743d3ea9ffcd03b" [[package]] name = "bumpalo" -version = "3.16.0" +version = "3.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" +checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" [[package]] name = "bytemuck" @@ -544,9 +544,9 @@ dependencies = [ [[package]] name = "clap_complete" -version = "4.5.43" +version = "4.5.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0952013545c9c6dca60f491602655b795c6c062ab180c9cb0bccb83135461861" +checksum = "375f9d8255adeeedd51053574fd8d4ba875ea5fa558e86617b07f09f1680c8b6" dependencies = [ "clap", ] @@ -571,9 +571,9 @@ checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" [[package]] name = "cmake" -version = "0.1.51" +version = "0.1.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb1e43aa7fd152b1f968787f7dbcdeb306d1867ff373c69955211876c053f91a" +checksum = "e24a03c8b52922d68a1589ad61032f2c1aa5a8158d2aa0d93c6e9534944bbad6" dependencies = [ "cc", ] @@ -634,9 +634,9 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cpufeatures" -version = "0.2.14" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "608697df725056feaccfa42cffdaeeec3fccc4ffc38358ecd19b243e716a78e0" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" dependencies = [ "libc", ] @@ -712,9 +712,9 @@ dependencies = [ [[package]] name = "data-encoding" -version = "2.6.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2" +checksum = "0e60eed09d8c01d3cee5b7d30acb059b76614c918fa0f992e0dd6eeb10daad6f" [[package]] name = "deranged" @@ -804,7 +804,10 @@ checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2" [[package]] name = "ebml-struct" version = "0.1.0" -source = "git+https://codeberg.org/metamuffin/ebml-struct#5574bebaa4e2104a5fd8ce7eba410e8453b99cf7" +source = "git+https://codeberg.org/metamuffin/ebml-struct#baa1f77aea4accf7a6046bf6b60275e5d942d816" +dependencies = [ + "bincode", +] [[package]] name = "ebml_derive" @@ -866,12 +869,12 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.9" +version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" +checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -897,9 +900,9 @@ checksum = "9afc2bd4d5a73106dd53d10d73d3401c2f32730ba2c0b93ddb888a8983680471" [[package]] name = "fastrand" -version = "2.1.1" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "fdeflate" @@ -1289,9 +1292,9 @@ dependencies = [ [[package]] name = "httparse" -version = "1.9.5" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d71d3574edd2771538b901e6549113b4006ece66150fb69c0fb6d9a2adae946" +checksum = "f2d708df4e7140240a16cd6ab0ab65c972d7433ab77819ea693fde9c43811e2a" [[package]] name = "httpdate" @@ -1316,9 +1319,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.31" +version = "0.14.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c08302e8fa335b151b788c775ff56e7a03ae64ff85c548ee820fecb70356e85" +checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" dependencies = [ "bytes", "futures-channel", @@ -1340,9 +1343,9 @@ dependencies = [ [[package]] name = "hyper" -version = "1.5.2" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "256fb8d4bd6413123cc9d91832d78325c48ff41677595be797d90f42969beae0" +checksum = "cc2b571658e38e0c01b1fdca3bbbe93c00d3d71693ff2770043f8c29bc7d6f80" dependencies = [ "bytes", "futures-channel", @@ -1365,7 +1368,7 @@ checksum = "2d191583f3da1305256f22463b9bb0471acad48a4e534a5218b9963e9c1f59b2" dependencies = [ "futures-util", "http 1.2.0", - "hyper 1.5.2", + "hyper 1.6.0", "hyper-util", "rustls", "rustls-pki-types", @@ -1386,7 +1389,7 @@ dependencies = [ "futures-util", "http 1.2.0", "http-body 1.0.1", - "hyper 1.5.2", + "hyper 1.6.0", "pin-project-lite", "socket2", "tokio", @@ -1659,19 +1662,19 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.10.1" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddc24109865250148c2e0f3d25d4f0f479571723792d3802153c60922a4fb708" +checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" [[package]] name = "is-terminal" -version = "0.4.13" +version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "261f68e344040fbd0edea105bef17c66edf46f984ddb1115b775ce31be948f4b" +checksum = "e19b23d53f35ce9f56aebc7d1bb4e6ac1e9c0db7ac85c8d1760c04379edced37" dependencies = [ "hermit-abi 0.4.0", "libc", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -1971,9 +1974,9 @@ dependencies = [ [[package]] name = "libfuzzer-sys" -version = "0.4.8" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b9569d2f74e257076d8c6bfa73fb505b46b851e51ddaecc825944aa3bed17fa" +checksum = "cf78f52d400cf2d84a3a973a78a592b4adc535739e0a5597a0da6f0c357adc75" dependencies = [ "arbitrary", "cc", @@ -1987,9 +1990,9 @@ checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa" [[package]] name = "linux-raw-sys" -version = "0.4.14" +version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" [[package]] name = "litemap" @@ -2893,7 +2896,7 @@ dependencies = [ "http 1.2.0", "http-body 1.0.1", "http-body-util", - "hyper 1.5.2", + "hyper 1.6.0", "hyper-rustls", "hyper-util", "ipnet", @@ -3012,7 +3015,7 @@ dependencies = [ "either", "futures", "http 0.2.12", - "hyper 0.14.31", + "hyper 0.14.32", "indexmap", "log", "memchr", @@ -3069,22 +3072,22 @@ checksum = "c7fb8039b3032c191086b10f11f319a6e99e1e82889c5cc6046f515c9db1d497" [[package]] name = "rustix" -version = "0.38.38" +version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa260229e6538e52293eeb577aabd09945a09d6d9cc0fc550ed7529056c2e32a" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ "bitflags 2.8.0", "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] name = "rustls" -version = "0.23.20" +version = "0.23.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5065c3f250cbd332cd894be57c40fa52387247659b14a2d6041d121547903b1b" +checksum = "8f287924602bf649d949c63dc8ac8b235fa5387d394020705b80c4eb597ce5b8" dependencies = [ "once_cell", "ring", @@ -3105,9 +3108,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.10.1" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2bf47e6ff922db3825eb750c4e2ff784c6ff8fb9e13046ef6a1d1c5401b0b37" +checksum = "917ce264624a4b4db1c364dcc35bfca9ded014d0a958cd47ad3e960e988ea51c" dependencies = [ "web-time", ] @@ -3125,15 +3128,15 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.18" +version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e819f2bc632f285be6d7cd36e25940d45b2391dd6d9b939e79de557f7014248" +checksum = "f7c45b9784283f1b2e7fb61b42047c2fd678ef0960d4f6f1eba131594cc369d4" [[package]] name = "ryu" -version = "1.0.18" +version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" +checksum = "6ea1a2d0a644769cc99faa24c3ad26b379b786fe7c36fd3c546254801650e6dd" [[package]] name = "scoped-tls" @@ -3558,12 +3561,13 @@ checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" [[package]] name = "tempfile" -version = "3.13.0" +version = "3.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0f2c9fc62d0beef6951ccffd757e241266a2c833136efbe35af6cd2567dca5b" +checksum = "38c246215d7d24f48ae091a2902398798e05d978b24315d6efbc00ede9a8bb91" dependencies = [ "cfg-if", "fastrand", + "getrandom 0.3.1", "once_cell", "rustix", "windows-sys 0.59.0", @@ -3632,9 +3636,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.36" +version = "0.3.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" +checksum = "35e7868883861bd0e56d9ac6efcaaca0d6d5d82a2a7ec8209ff492c07cf37b21" dependencies = [ "deranged", "itoa", @@ -3653,9 +3657,9 @@ checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.18" +version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" +checksum = "2834e6017e3e5e4b9834939793b282bc03b37a3336245fa820e35e233e2a85de" dependencies = [ "num-conv", "time-core", @@ -3727,9 +3731,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.16" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f4e6ce100d0eb49a2734f8c0812bcd324cf357d21810932c5df6b96ef2b86f1" +checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047" dependencies = [ "futures-core", "pin-project-lite", @@ -3867,9 +3871,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.18" +version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" +checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008" dependencies = [ "matchers", "nu-ansi-term", @@ -3935,9 +3939,9 @@ dependencies = [ [[package]] name = "unicode-ident" -version = "1.0.15" +version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11cd88e12b17c6494200a9c1b683a04fcac9573ed74cd1b62aeb2727c5592243" +checksum = "a210d160f08b701c8721ba1c726c11662f877ea6b7094007e1ca9a1041945034" [[package]] name = "unicode-width" @@ -4022,9 +4026,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.11.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8c5f0a0af699448548ad1a2fbf920fb4bee257eae39953ba95cb84891a0446a" +checksum = "b3758f5e68192bb96cc8f9b7e2c2cfdabb435499a28499a42f8f984092adad4b" dependencies = [ "getrandom 0.2.15", "serde", @@ -4043,9 +4047,9 @@ dependencies = [ [[package]] name = "valuable" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" [[package]] name = "version-compare" @@ -4192,9 +4196,9 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.26.6" +version = "0.26.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "841c67bff177718f1d4dfefde8d8f0e78f9b6589319ba88312f567fc5841a958" +checksum = "5d642ff16b7e79272ae451b7322067cdc17cadf68c23264be9d94a32319efe7e" dependencies = [ "rustls-pki-types", ] @@ -4416,9 +4420,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" -version = "0.6.24" +version = "0.6.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8d71a593cc5c42ad7876e2c1fda56f314f3754c084128833e64f1345ff8a03a" +checksum = "ad699df48212c6cc6eb4435f35500ac6fd3b9913324f938aea302022ce19d310" dependencies = [ "memchr", ] diff --git a/base/src/database.rs b/base/src/database.rs index e9fe156..bec630a 100644 --- a/base/src/database.rs +++ b/base/src/database.rs @@ -10,7 +10,7 @@ use jellycommon::{ Node, NodeID, }; use log::info; -use redb::{ReadableTable, TableDefinition}; +use redb::{ReadableTable, StorageError, TableDefinition}; use std::{ fs::create_dir_all, path::Path, @@ -29,6 +29,8 @@ const T_USER_NODE: TableDefinition<(&str, [u8; 32]), Ser<NodeUserData>> = TableDefinition::new("user_node"); const T_INVITE: TableDefinition<&str, ()> = TableDefinition::new("invite"); const T_NODE: TableDefinition<[u8; 32], Ser<Node>> = TableDefinition::new("node"); +const T_NODE_CHILDREN: TableDefinition<([u8; 32], [u8; 32]), ()> = + TableDefinition::new("node_children"); const T_IMPORT_FILE_MTIME: TableDefinition<&[u8], u64> = TableDefinition::new("import_file_mtime"); #[derive(Clone)] @@ -75,11 +77,21 @@ impl Database { Ok(None) } } + pub fn get_node_children(&self, id: NodeID) -> Result<Vec<NodeID>> { + let txn = self.inner.begin_read()?; + let t_node_children = txn.open_table(T_NODE_CHILDREN)?; + Ok(t_node_children + .range((id.0, NodeID::MIN.0)..(id.0, NodeID::MAX.0))? + .map(|r| r.map(|r| NodeID(r.0.value().1))) + .collect::<Result<Vec<_>, StorageError>>()?) + } pub fn clear_nodes(&self) -> Result<()> { let txn = self.inner.begin_write()?; - let mut table = txn.open_table(T_NODE)?; - table.retain(|_, _| false)?; - drop(table); + let mut t_node = txn.open_table(T_NODE)?; + let mut t_node_children = txn.open_table(T_NODE_CHILDREN)?; + t_node.retain(|_, _| false)?; + t_node_children.retain(|_, _| false)?; + drop((t_node, t_node_children)); txn.commit()?; Ok(()) } @@ -209,7 +221,7 @@ impl Database { txn.commit()?; Ok(()) } - pub fn list_nodes_with_udata(&self, username: &str) -> Result<Vec<(Node, NodeUserData)>> { + pub fn list_nodes_with_udata(&self, username: &str) -> Result<Vec<(Arc<Node>, NodeUserData)>> { let txn = self.inner.begin_read()?; let nodes = txn.open_table(T_NODE)?; let node_users = txn.open_table(T_USER_NODE)?; @@ -217,7 +229,7 @@ impl Database { .iter()? .map(|a| { let (x, y) = a.unwrap(); - let (x, y) = (x.value().to_owned(), y.value().0); + let (x, y) = (x.value().to_owned(), Arc::new(y.value().0)); let z = node_users .get(&(username, x)) .unwrap() @@ -285,12 +297,15 @@ impl Database { ) -> Result<()> { let txn = self.inner.begin_write()?; let mut t_nodes = txn.open_table(T_NODE)?; + let mut t_node_children = txn.open_table(T_NODE_CHILDREN)?; let mut node = t_nodes.get(id.0)?.map(|v| v.value().0).unwrap_or_default(); update(&mut node)?; + for parent in &node.parents { + t_node_children.insert((parent.0, id.0), ())?; + } t_nodes.insert(&id.0, Ser(node))?; - drop(t_nodes); + drop((t_nodes, t_node_children)); txn.commit()?; - Ok(()) } pub fn get_import_file_mtime(&self, path: &Path) -> Result<Option<u64>> { diff --git a/common/src/impl.rs b/common/src/impl.rs index a98015a..3814b1d 100644 --- a/common/src/impl.rs +++ b/common/src/impl.rs @@ -166,3 +166,7 @@ impl Node { NodeID::from_node(self) } } +impl NodeID { + pub const MIN: NodeID = NodeID([0; 32]); + pub const MAX: NodeID = NodeID([255; 32]); +} diff --git a/import/Cargo.toml b/import/Cargo.toml index d0342df..988e626 100644 --- a/import/Cargo.toml +++ b/import/Cargo.toml @@ -8,7 +8,9 @@ jellycommon = { path = "../common" } jellybase = { path = "../base" } jellyclient = { path = "../client" } -ebml-struct = { git = "https://codeberg.org/metamuffin/ebml-struct" } +ebml-struct = { git = "https://codeberg.org/metamuffin/ebml-struct", features = [ + "bincode", +] } rayon = "1.10.0" crossbeam-channel = "0.5.14" diff --git a/import/src/infojson.rs b/import/src/infojson.rs index c2ae305..3e4667e 100644 --- a/import/src/infojson.rs +++ b/import/src/infojson.rs @@ -3,13 +3,13 @@ which is licensed under the GNU Affero General Public License (version 3); see /COPYING. Copyright (C) 2025 metamuffin <metamuffin.org> */ - use anyhow::Context; +use bincode::{Decode, Encode}; use jellycommon::chrono::{format::Parsed, Utc}; use serde::{Deserialize, Serialize}; use std::collections::HashMap; -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, Encode, Decode)] pub struct YVideo { pub id: String, pub title: String, @@ -63,7 +63,7 @@ pub struct YVideo { pub epoch: usize, } -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, Encode, Decode)] pub struct YCaption { pub url: Option<String>, pub ext: String, //"vtt" | "json3" | "srv1" | "srv2" | "srv3" | "ttml", @@ -71,7 +71,7 @@ pub struct YCaption { pub name: Option<String>, } -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, Encode, Decode)] pub struct YFormat { pub format_id: String, pub format_note: Option<String>, @@ -96,13 +96,13 @@ pub struct YFormat { pub format: String, } -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, Encode, Decode)] pub struct YFragment { pub url: Option<String>, pub duration: Option<f64>, } -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, Encode, Decode)] pub struct YThumbnail { pub url: String, pub preference: Option<i32>, @@ -112,14 +112,14 @@ pub struct YThumbnail { pub resolution: Option<String>, } -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, Encode, Decode)] pub struct YChapter { pub start_time: f64, pub end_time: f64, pub title: String, } -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, Encode, Decode)] pub struct YHeatmapSample { pub start_time: f64, pub end_time: f64, diff --git a/import/src/lib.rs b/import/src/lib.rs index add7e4d..10bd0ec 100644 --- a/import/src/lib.rs +++ b/import/src/lib.rs @@ -4,24 +4,19 @@ Copyright (C) 2025 metamuffin <metamuffin.org> */ use anyhow::{anyhow, Context, Result}; -use ebml_struct::{ - ids::*, - matroska::*, - read::{EbmlReadExt, TagRead}, -}; use infojson::YVideo; -use jellybase::{assetfed::AssetInner, cache::cache_file, database::Database, CONF, SECRETS}; +use jellybase::{assetfed::AssetInner, database::Database, CONF, SECRETS}; use jellycommon::{ Chapter, LocalTrack, MediaInfo, Node, NodeID, NodeKind, Rating, SourceTrack, SourceTrackKind, TrackSource, }; use log::info; +use matroska::matroska_metadata; use rayon::iter::{ParallelDrainRange, ParallelIterator}; -use regex::Regex; use std::{ collections::HashMap, fs::File, - io::{BufReader, ErrorKind, Read, Write}, + io::{BufReader, Read}, mem::swap, path::{Path, PathBuf}, sync::LazyLock, @@ -35,14 +30,15 @@ use tokio::{ use trakt::Trakt; pub mod infojson; +pub mod matroska; pub mod tmdb; pub mod trakt; static IMPORT_SEM: LazyLock<Semaphore> = LazyLock::new(|| Semaphore::new(1)); pub static IMPORT_ERRORS: RwLock<Vec<String>> = RwLock::const_new(Vec::new()); -static RE_EPISODE_FILENAME: LazyLock<Regex> = - LazyLock::new(|| Regex::new(r#"([sS](\d+))?([eE](\d+))( (.+))?"#).unwrap()); +// static RE_EPISODE_FILENAME: LazyLock<Regex> = +// LazyLock::new(|| Regex::new(r#"([sS](\d+))?([eE](\d+))( (.+))?"#).unwrap()); struct Apis { trakt: Option<Trakt>, @@ -57,6 +53,7 @@ pub async fn import_wrap(db: Database, incremental: bool) -> Result<()> { let _sem = IMPORT_SEM.try_acquire()?; let jh = spawn_blocking(move || { + *IMPORT_ERRORS.blocking_write() = Vec::new(); if let Err(e) = import(&db, incremental) { IMPORT_ERRORS.blocking_write().push(format!("{e:#}")); } @@ -121,25 +118,26 @@ fn import_iter_inner(path: &Path, db: &Database, incremental: bool) -> Result<Ve } fn import_file(db: &Database, path: &Path) -> Result<()> { - let parent = NodeID::from_slug( - &path - .parent() - .ok_or(anyhow!("no parent"))? - .file_name() - .ok_or(anyhow!("parent no filename"))? - .to_string_lossy(), - ); + let parent_slug = path + .parent() + .ok_or(anyhow!("no parent"))? + .file_name() + .ok_or(anyhow!("parent no filename"))? + .to_string_lossy(); + let parent = NodeID::from_slug(&parent_slug); let filename = path.file_name().unwrap().to_string_lossy(); match filename.as_ref() { "poster.jpeg" | "poster.webp" => { db.update_node_init(parent, |node| { + node.slug = parent_slug.to_string(); node.poster = Some(AssetInner::Media(path.to_owned()).ser()); Ok(()) })?; } "backdrop.jpeg" | "backdrop.webp" => { db.update_node_init(parent, |node| { + node.slug = parent_slug.to_string(); node.backdrop = Some(AssetInner::Media(path.to_owned()).ser()); Ok(()) })?; @@ -147,6 +145,7 @@ fn import_file(db: &Database, path: &Path) -> Result<()> { "info.json" | "info.yaml" => { let data = serde_yaml::from_reader::<_, Node>(BufReader::new(File::open(path)?))?; db.update_node_init(parent, |node| { + node.slug = parent_slug.to_string(); fn merge_option<T>(a: &mut Option<T>, b: Option<T>) { if b.is_some() { *a = b; @@ -161,6 +160,7 @@ fn import_file(db: &Database, path: &Path) -> Result<()> { "channel.info.json" => { let data = serde_json::from_reader::<_, YVideo>(BufReader::new(File::open(path)?))?; db.update_node_init(parent, |node| { + node.slug = parent_slug.to_string(); node.title = Some( data.title .strip_suffix(" - Videos") @@ -189,76 +189,14 @@ fn import_file(db: &Database, path: &Path) -> Result<()> { fn import_media_file(db: &Database, path: &Path, parent: NodeID) -> Result<()> { info!("reading media file {path:?}"); - let mut file = BufReader::new(File::open(path)?); - let mut file = file.by_ref().take(u64::MAX); - - let (x, mut ebml) = file.read_tag()?; - assert_eq!(x, EL_EBML); - let ebml = Ebml::read(&mut ebml).unwrap(); - assert!(ebml.doc_type == "matroska" || ebml.doc_type == "webm"); - let (x, mut segment) = file.read_tag()?; - assert_eq!(x, EL_SEGMENT); - let mut info = None; - let mut infojson = None; - let mut tracks = None; - let mut cover = None; - let mut chapters = None; - let mut tags = None; - loop { - let (x, mut seg) = match segment.read_tag() { - Ok(o) => o, - Err(e) if e.kind() == ErrorKind::UnexpectedEof => break, - Err(e) => return Err(e.into()), - }; - match x { - EL_INFO => info = Some(Info::read(&mut seg).context("info")?), - EL_TRACKS => tracks = Some(Tracks::read(&mut seg).context("tracks")?), - EL_CHAPTERS => chapters = Some(Chapters::read(&mut seg).context("chapters")?), - EL_TAGS => tags = Some(Tags::read(&mut seg).context("tags")?), - EL_ATTACHMENTS => { - let attachments = Attachments::read(&mut seg).context("attachments")?; - for f in attachments.files { - match f.name.as_str() { - "info.json" => { - infojson = Some( - serde_json::from_slice::<infojson::YVideo>(&f.data) - .context("infojson")?, - ); - } - "cover.webp" | "cover.png" | "cover.jpg" | "cover.jpeg" | "cover.avif" => { - cover = Some( - AssetInner::Cache(cache_file( - &["att-cover", path.to_string_lossy().as_ref()], - move |mut file| { - file.write_all(&f.data)?; - Ok(()) - }, - )?) - .ser(), - ) - } - a => println!("{a:?}"), - } - } - } - EL_VOID | EL_CRC32 | EL_CUES | EL_SEEKHEAD => { - seg.consume()?; - } - EL_CLUSTER => { - break; - } - id => { - eprintln!("unknown top-level element {id:x}"); - seg.consume()?; - } - } - } + let m = (*matroska_metadata(path)?).to_owned(); - let info = info.ok_or(anyhow!("no info"))?; - let tracks = tracks.ok_or(anyhow!("no tracks"))?; + let info = m.info.ok_or(anyhow!("no info"))?; + let tracks = m.tracks.ok_or(anyhow!("no tracks"))?; - let mut tags = tags + let mut tags = m + .tags .map(|tags| { tags.tags .into_iter() @@ -274,7 +212,8 @@ fn import_media_file(db: &Database, path: &Path, parent: NodeID) -> Result<()> { .to_string_lossy() .to_string(); - let slug = infojson + let slug = m + .infojson .as_ref() .map(|ij| format!("youtube-{}", ij.id)) .unwrap_or(make_kebab(&filepath_stem)); @@ -282,13 +221,13 @@ fn import_media_file(db: &Database, path: &Path, parent: NodeID) -> Result<()> { db.update_node_init(NodeID::from_slug(&slug), |node| { node.slug = slug; node.title = info.title; - node.poster = cover; + node.poster = m.cover.clone(); node.description = tags.remove("DESCRIPTION"); node.tagline = tags.remove("COMMENT"); if !node.parents.contains(&parent) { node.parents.push(parent) } - if let Some(infojson) = infojson { + if let Some(infojson) = m.infojson { node.kind = Some( if infojson.duration.unwrap_or(0.) < 600. && infojson.aspect_ratio.unwrap_or(2.) < 1. @@ -314,7 +253,9 @@ fn import_media_file(db: &Database, path: &Path, parent: NodeID) -> Result<()> { } } node.media = Some(MediaInfo { - chapters: chapters + chapters: m + .chapters + .clone() .map(|c| { let mut chaps = Vec::new(); if let Some(ee) = c.edition_entries.first() { diff --git a/import/src/matroska.rs b/import/src/matroska.rs new file mode 100644 index 0000000..bb8d927 --- /dev/null +++ b/import/src/matroska.rs @@ -0,0 +1,112 @@ +/* + This file is part of jellything (https://codeberg.org/metamuffin/jellything) + which is licensed under the GNU Affero General Public License (version 3); see /COPYING. + Copyright (C) 2025 metamuffin <metamuffin.org> +*/ +use crate::infojson::{self, YVideo}; +use anyhow::{Context, Result}; +use bincode::{Decode, Encode}; +use ebml_struct::{ + ids::*, + matroska::*, + read::{EbmlReadExt, TagRead}, +}; +use jellybase::{ + assetfed::AssetInner, + cache::{cache_file, cache_memory}, +}; +use jellycommon::Asset; +use std::{ + fs::File, + io::{BufReader, ErrorKind, Read, Write}, + path::Path, + sync::Arc, +}; + +#[derive(Encode, Decode, Clone)] +pub(crate) struct MatroskaMetadata { + pub info: Option<Info>, + pub tracks: Option<Tracks>, + pub cover: Option<Asset>, + pub chapters: Option<Chapters>, + pub tags: Option<Tags>, + pub infojson: Option<YVideo>, +} +pub(crate) fn matroska_metadata(path: &Path) -> Result<Arc<MatroskaMetadata>> { + cache_memory(&["mkmeta-v1", path.to_string_lossy().as_ref()], || { + let mut file = BufReader::new(File::open(path)?); + let mut file = file.by_ref().take(u64::MAX); + + let (x, mut ebml) = file.read_tag()?; + assert_eq!(x, EL_EBML); + let ebml = Ebml::read(&mut ebml).unwrap(); + assert!(ebml.doc_type == "matroska" || ebml.doc_type == "webm"); + let (x, mut segment) = file.read_tag()?; + assert_eq!(x, EL_SEGMENT); + + let mut info = None; + let mut infojson = None; + let mut tracks = None; + let mut cover = None; + let mut chapters = None; + let mut tags = None; + loop { + let (x, mut seg) = match segment.read_tag() { + Ok(o) => o, + Err(e) if e.kind() == ErrorKind::UnexpectedEof => break, + Err(e) => return Err(e.into()), + }; + match x { + EL_INFO => info = Some(Info::read(&mut seg).context("info")?), + EL_TRACKS => tracks = Some(Tracks::read(&mut seg).context("tracks")?), + EL_CHAPTERS => chapters = Some(Chapters::read(&mut seg).context("chapters")?), + EL_TAGS => tags = Some(Tags::read(&mut seg).context("tags")?), + EL_ATTACHMENTS => { + let attachments = Attachments::read(&mut seg).context("attachments")?; + for f in attachments.files { + match f.name.as_str() { + "info.json" => { + infojson = Some( + serde_json::from_slice::<infojson::YVideo>(&f.data) + .context("infojson")?, + ); + } + "cover.webp" | "cover.png" | "cover.jpg" | "cover.jpeg" + | "cover.avif" => { + cover = Some( + AssetInner::Cache(cache_file( + &["att-cover", path.to_string_lossy().as_ref()], + move |mut file| { + file.write_all(&f.data)?; + Ok(()) + }, + )?) + .ser(), + ) + } + a => println!("{a:?}"), + } + } + } + EL_VOID | EL_CRC32 | EL_CUES | EL_SEEKHEAD => { + seg.consume()?; + } + EL_CLUSTER => { + break; + } + id => { + eprintln!("unknown top-level element {id:x}"); + seg.consume()?; + } + } + } + Ok(MatroskaMetadata { + chapters, + cover, + info, + infojson, + tags, + tracks, + }) + }) +} diff --git a/server/src/routes/ui/admin/mod.rs b/server/src/routes/ui/admin/mod.rs index 5c2c48f..50faa2e 100644 --- a/server/src/routes/ui/admin/mod.rs +++ b/server/src/routes/ui/admin/mod.rs @@ -6,7 +6,10 @@ pub mod log; pub mod user; -use super::account::session::AdminSession; +use super::{ + account::session::AdminSession, + assets::{resolve_asset, AVIF_QUALITY, AVIF_SPEED}, +}; use crate::{ database::Database, routes::ui::{ @@ -17,7 +20,7 @@ use crate::{ uri, }; use anyhow::{anyhow, Context}; -use jellybase::{federation::Federation, CONF}; +use jellybase::{assetfed::AssetInner, federation::Federation, CONF}; use jellyimport::{import_wrap, is_importing, IMPORT_ERRORS}; use markup::DynRender; use rand::Rng; @@ -196,25 +199,22 @@ pub async fn r_admin_transcode_posters( let t = Instant::now(); - // TODO - // { - // let txn = database.begin_read()?; - // let nodes = txn.open_table(T_NODE)?; - // for node in nodes.iter()? { - // let (_, node) = node?; - // if let Some(poster) = &node.value().0.poster { - // let asset = AssetInner::deser(&poster.0)?; - // if asset.is_federated() { - // continue; - // } - // let source = resolve_asset(asset).await.context("resolving asset")?; - // jellytranscoder::image::transcode(source, AVIF_QUALITY, AVIF_SPEED, 1024) - // .await - // .context("transcoding asset")?; - // } - // } - // } - // drop(_permit); + { + let nodes = database.list_nodes_with_udata("")?; + for (node, _) in nodes { + if let Some(poster) = &node.poster { + let asset = AssetInner::deser(&poster.0)?; + if asset.is_federated() { + continue; + } + let source = resolve_asset(asset).await.context("resolving asset")?; + jellytranscoder::image::transcode(source, AVIF_QUALITY, AVIF_SPEED, 1024) + .await + .context("transcoding asset")?; + } + } + } + drop(_permit); admin_dashboard( database, diff --git a/server/src/routes/ui/node.rs b/server/src/routes/ui/node.rs index 3332483..121896e 100644 --- a/server/src/routes/ui/node.rs +++ b/server/src/routes/ui/node.rs @@ -1,5 +1,3 @@ -use std::sync::Arc; - /* This file is part of jellything (https://codeberg.org/metamuffin/jellything) which is licensed under the GNU Affero General Public License (version 3); see /COPYING. @@ -37,6 +35,7 @@ use jellycommon::{ Chapter, MediaInfo, Node, NodeID, NodeKind, PeopleGroup, Rating, SourceTrackKind, }; use rocket::{get, serde::json::Json, Either, State}; +use std::sync::Arc; /// This function is a stub and only useful for use in the uri! macro. #[get("/n/<id>")] @@ -44,43 +43,32 @@ pub fn r_library_node(id: String) { drop(id) } -#[get("/n/<id>?<filter..>")] +#[get("/n/<slug>?<filter..>")] pub async fn r_library_node_filter<'a>( session: Session, - id: &'a str, + slug: &'a str, db: &'a State<Database>, aj: AcceptJson, filter: NodeFilterSort, ) -> MyResult<Either<DynLayoutPage<'a>, Json<Node>>> { - let (node, udata) = db.get_node_with_userdata(NodeID::from_slug(id), &session)?; + let id = NodeID::from_slug(slug); + let (node, udata) = db.get_node_with_userdata(id, &session)?; if *aj { return Ok(Either::Right(Json((*node).clone()))); } - // let mut children = node - // .children - // .iter() - // .map(|c| db.get_node_with_userdata(c, &session)) - // .collect::<anyhow::Result<Vec<_>>>()? - // .into_iter() - // .collect(); + let mut children = db + .get_node_children(id)? + .into_iter() + .map(|c| db.get_node_with_userdata(c, &session)) + .collect::<anyhow::Result<Vec<_>>>()?; - // let path = node - // .path - // .iter() - // .map(|c| { - // Ok(( - // c.to_owned(), - // T_NODE - // .get(db, c.as_str())? - // .ok_or(anyhow!("parent node missing"))? - // .public, - // )) - // }) - // .collect::<anyhow::Result<Vec<_>>>()? - // .into_iter() - // .collect::<Vec<_>>(); + let parents = node + .parents + .iter() + .flat_map(|pid| db.get_node(*pid).transpose()) + .collect::<Result<Vec<_>, _>>()?; filter_and_sort_nodes( &filter, @@ -89,13 +77,13 @@ pub async fn r_library_node_filter<'a>( _ => (SortProperty::Title, SortOrder::Ascending), }, // TODO - &mut Vec::new(), + &mut children, ); Ok(Either::Left(LayoutPage { title: node.title.clone().unwrap_or_default(), content: markup::new! { - @NodePage { node: &node, id, udata: &udata, children: &[], path: &[], filter: &filter } + @NodePage { node: &node, id: slug, udata: &udata, children: &children, parents: &parents, filter: &filter } }, ..Default::default() })) @@ -128,7 +116,7 @@ markup::define! { } } } - NodePage<'a>(id: &'a str, node: &'a Node, udata: &'a NodeUserData, children: &'a [(Arc<Node>, NodeUserData)], path: &'a [(String, Node)], filter: &'a NodeFilterSort) { + NodePage<'a>(id: &'a str, node: &'a Node, udata: &'a NodeUserData, children: &'a [(Arc<Node>, NodeUserData)], parents: &'a [Arc<Node>], filter: &'a NodeFilterSort) { @if !matches!(node.kind.unwrap_or_default(), NodeKind::Collection) { img.backdrop[src=uri!(r_item_backdrop(id, Some(2048))), loading="lazy"]; } @@ -139,9 +127,9 @@ markup::define! { } .title { h1 { @node.title } - span.path { @for (cid, cnode) in *path { - " / " a.component[href=uri!(r_library_node(cid))] { @cnode.title } - }} + ul.parents { @for node in *parents { li { + a.component[href=uri!(r_library_node(&node.slug))] { @node.title } + }}} @if node.media.is_some() { a.play[href=&uri!(r_player(id, PlayerConfig::default()))] { "Watch now" }} @if !matches!(node.kind.unwrap_or_default(), NodeKind::Collection | NodeKind::Channel) { @if matches!(udata.watched, WatchedState::None | WatchedState::Pending | WatchedState::Progress(_)) { diff --git a/server/src/routes/ui/sort.rs b/server/src/routes/ui/sort.rs index 705b616..68bd588 100644 --- a/server/src/routes/ui/sort.rs +++ b/server/src/routes/ui/sort.rs @@ -8,6 +8,7 @@ use rocket::{ http::uri::fmt::{Query, UriDisplay}, FromForm, FromFormField, UriDisplayQuery, }; +use std::sync::Arc; #[derive(FromForm, UriDisplayQuery, Default, Clone)] pub struct NodeFilterSort { @@ -134,7 +135,7 @@ pub enum SortOrder { pub fn filter_and_sort_nodes( f: &NodeFilterSort, default_sort: (SortProperty, SortOrder), - nodes: &mut Vec<(Node, NodeUserData)>, + nodes: &mut Vec<(Arc<Node>, NodeUserData)>, ) { let sort_prop = f.sort_by.unwrap_or(default_sort.0); nodes.retain(|(node, udata)| { |