rework import system pt. 1

10 files changed, 57 insertions, 373 deletions

diff --git a/server/Cargo.toml b/server/Cargo.toml
index da8603d..68ae861 100644
--- a/server/Cargo.toml
+++ b/server/Cargo.toml
@@ -7,8 +7,8 @@ edition = "2021"
 jellycommon = { path = "../common", features = ["rocket"] }
 jellybase = { path = "../base" }
 jellystream = { path = "../stream" }
-jellyclient = { path = "../client" }
 jellytranscoder = { path = "../transcoder" }
+jellyimport = { path = "../import" }
 
 serde = { version = "1.0.193", features = ["derive"] }
 bincode = { version = "2.0.0-rc.3", features = ["serde", "derive"] }
diff --git a/server/src/federation.rs b/server/src/federation.rs
deleted file mode 100644
index eb2a1ac..0000000
--- a/server/src/federation.rs
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
-    This file is part of jellything (https://codeberg.org/metamuffin/jellything)
-    which is licensed under the GNU Affero General Public License (version 3); see /COPYING.
-    Copyright (C) 2023 metamuffin <metamuffin.org>
-*/
-use anyhow::anyhow;
-use jellybase::CONF;
-use jellyclient::{Instance, LoginDetails, Session};
-use std::{collections::HashMap, sync::Arc};
-use tokio::sync::RwLock;
-
-pub struct Federation {
-    instances: HashMap<String, Instance>,
-    sessions: RwLock<HashMap<String, Arc<Session>>>,
-}
-
-impl Federation {
-    pub fn initialize() -> Self {
-        let instances = CONF
-            .remote_credentials
-            .iter()
-            .map(|(k, (_, _, tls))| (k.to_owned(), Instance::new(k.to_owned(), *tls)))
-            .collect::<HashMap<_, _>>();
-
-        Self {
-            instances,
-            sessions: Default::default(),
-        }
-    }
-
-    pub fn get_instance(&self, host: &String) -> anyhow::Result<&Instance> {
-        Ok(self
-            .instances
-            .get(host)
-            .ok_or(anyhow!("unknown instance"))?)
-    }
-    pub async fn get_session(&self, host: &String) -> anyhow::Result<Arc<Session>> {
-        let mut w = self.sessions.write().await;
-        if let Some(s) = w.get(host) {
-            Ok(s.to_owned())
-        } else {
-            let (username, password, _) = CONF
-                .remote_credentials
-                .get(host)
-                .ok_or(anyhow!("no credentials of the remote server"))?;
-            let s = Arc::new(
-                self.get_instance(host)?
-                    .to_owned()
-                    .login(LoginDetails {
-                        username: username.to_owned(),
-                        password: password.to_owned(),
-                        expire: None,
-                        drop_permissions: None,
-                    })
-                    .await?,
-            );
-            w.insert(host.to_owned(), s.clone());
-            Ok(s)
-        }
-    }
-}
diff --git a/server/src/import.rs b/server/src/import.rs
deleted file mode 100644
index dc32fbf..0000000
--- a/server/src/import.rs
+++ /dev/null
@@ -1,254 +0,0 @@
-/*
-    This file is part of jellything (https://codeberg.org/metamuffin/jellything)
-    which is licensed under the GNU Affero General Public License (version 3); see /COPYING.
-    Copyright (C) 2023 metamuffin <metamuffin.org>
-*/
-use crate::{database::Database, federation::Federation, CONF};
-use anyhow::{anyhow, bail, Context, Ok};
-use async_recursion::async_recursion;
-use futures::{stream::FuturesUnordered, StreamExt, TryFutureExt};
-use jellybase::cache::async_cache_file;
-use jellyclient::Session;
-use jellycommon::{AssetLocation, AssetRole, MediaSource, Node, NodePrivate, RemoteImportOptions};
-use log::{debug, error, info};
-use std::{
-    ffi::OsStr,
-    fs::File,
-    os::unix::prelude::OsStrExt,
-    path::PathBuf,
-    sync::{Arc, LazyLock},
-};
-use tokio::sync::Semaphore;
-
-static IMPORT_SEM: LazyLock<Semaphore> = LazyLock::new(|| Semaphore::new(1));
-
-pub async fn import(db: &Database, fed: &Federation) -> anyhow::Result<()> {
-    info!("clearing node tree");
-    let permit = IMPORT_SEM.try_acquire()?;
-    db.node.clear()?;
-    info!("importing...");
-    let (_, errs) = import_path(CONF.library_path.clone(), db, fed, vec![])
-        .await
-        .context("indexing")?;
-    info!("import completed");
-    drop(permit);
-    if errs == 0 {
-        Ok(())
-    } else {
-        Err(anyhow!(
-            "partial import, {errs} errors occured; see server log"
-        ))
-    }
-}
-
-#[async_recursion]
-pub async fn import_path(
-    path: PathBuf,
-    db: &Database,
-    fed: &Federation,
-    mut node_path: Vec<String>,
-) -> anyhow::Result<(Vec<String>, usize)> {
-    if path.is_dir() {
-        let mpath = path.join("directory.json");
-        let children_paths = path.read_dir()?.map(Result::unwrap).filter_map(|e| {
-            if e.path().extension() == Some(&OsStr::from_bytes(b"jelly"))
-                || e.metadata().unwrap().is_dir()
-            {
-                Some(e.path())
-            } else {
-                None
-            }
-        });
-        let identifier = if mpath.exists() {
-            path.file_name().unwrap().to_str().unwrap().to_string()
-        } else {
-            node_path
-                .last()
-                .cloned()
-                .ok_or(anyhow!("non-root node requires parent"))?
-        };
-
-        node_path.push(identifier.clone());
-        let mut all: FuturesUnordered<_> = children_paths
-            .into_iter()
-            .map(|p| import_path(p.clone(), db, fed, node_path.clone()).map_err(|e| (p, e)))
-            .collect();
-        node_path.pop(); // we will set the dirs path later and need it to not be included
-
-        let mut children_ids = Vec::new();
-        let mut errs = 0;
-        while let Some(k) = all.next().await {
-            match k {
-                core::result::Result::Ok((els, errs2)) => {
-                    errs += errs2;
-                    children_ids.extend(els)
-                }
-                Err((p, e)) => {
-                    errs += 1;
-                    error!("import of {p:?} failed: {e:?}")
-                }
-            }
-        }
-        if mpath.exists() {
-            let mut node: Node =
-                serde_json::from_reader(File::open(mpath).context("metadata missing")?)?;
-
-            node.public.children = children_ids;
-            node.public.path = node_path;
-            node.public.id = Some(identifier.to_owned());
-            info!("adding {identifier}");
-            db.node.insert(&identifier, &node)?;
-            Ok((vec![identifier], errs))
-        } else {
-            Ok((children_ids, errs))
-        }
-    } else if path.is_file() {
-        info!("loading {path:?}");
-        let datafile = File::open(path.clone()).context("cant load metadata")?;
-        let mut node: Node = serde_json::from_reader(datafile).context("invalid metadata")?;
-        let identifier = node.private.id.clone().unwrap_or_else(|| {
-            path.file_name()
-                .unwrap()
-                .to_str()
-                .unwrap()
-                .strip_suffix(".jelly")
-                .unwrap()
-                .to_string()
-        });
-
-        let idents = if let Some(io) = node.private.import.take() {
-            let session = fed
-                .get_session(&io.host)
-                .await
-                .context("creating session")?;
-
-            import_remote(io, db, &session, identifier.clone(), node_path)
-                .await
-                .context("federated import")?
-        } else {
-            debug!("adding {identifier}");
-            node.public.path = node_path;
-            node.public.id = Some(identifier.to_owned());
-            let did_insert = db.node.insert(&identifier, &node)?.is_none();
-            if did_insert {
-                vec![identifier]
-            } else {
-                vec![]
-            }
-        };
-        Ok((idents, 0))
-    } else {
-        bail!("did somebody really put a fifo or socket in the library?!")
-    }
-}
-
-static SEM_REMOTE_IMPORT: LazyLock<Semaphore> = LazyLock::new(|| Semaphore::new(16));
-
-#[async_recursion]
-async fn import_remote(
-    mut opts: RemoteImportOptions,
-    db: &Database,
-    session: &Arc<Session>,
-    identifier: String,
-    mut node_path: Vec<String>,
-) -> anyhow::Result<Vec<String>> {
-    let _permit = SEM_REMOTE_IMPORT.acquire().await.unwrap();
-    info!("loading federated node {identifier:?}");
-
-    let flatten = opts.flatten;
-    opts.flatten = false;
-
-    let node = session
-        .node(&opts.id)
-        .await
-        .context("fetching remote node")?;
-
-    if node.federated.as_ref() == Some(&CONF.hostname) {
-        return Ok(vec![]); // node is federated from us, lets not import it
-    }
-
-    let poster =
-        cache_federation_asset(session.to_owned(), opts.id.clone(), AssetRole::Poster).await?;
-    let backdrop =
-        cache_federation_asset(session.to_owned(), opts.id.clone(), AssetRole::Backdrop).await?;
-
-    drop(_permit);
-
-    let mut did_insert = false;
-    if !flatten {
-        let mut node = Node {
-            public: node.clone(),
-            private: NodePrivate {
-                backdrop: Some(backdrop),
-                poster: Some(poster),
-                import: None,
-                id: None,
-                source: Some(MediaSource::Remote {
-                    host: opts.host.clone(),
-                    remote_id: opts.id.clone(),
-                }),
-            },
-        };
-        node.public.path = node_path.clone();
-        node.public.federated = Some(opts.host.clone());
-        node.public
-            .children
-            .iter_mut()
-            .for_each(|c| *c = format!("{}{c}", opts.prefix.clone().unwrap_or(String::new())));
-
-        debug!("adding {identifier}");
-        node.public.id = Some(identifier.to_owned());
-        did_insert = db
-            .node
-            .fetch_and_update(&identifier, |pnode| Some(pnode.unwrap_or(node.clone())))?
-            .is_none();
-        node_path.push(opts.id.clone());
-    }
-
-    let mut children: FuturesUnordered<_> = node
-        .children
-        .iter()
-        .map(|c| {
-            let prefixed = format!("{}{c}", opts.prefix.clone().unwrap_or(String::new()));
-            import_remote(
-                RemoteImportOptions {
-                    id: c.to_owned(),
-                    ..opts.clone()
-                },
-                db,
-                session,
-                prefixed,
-                node_path.clone(),
-            )
-        })
-        .collect();
-
-    let mut children_idents = Vec::new();
-    while let Some(r) = children.next().await {
-        children_idents.extend(r?);
-    }
-    Ok(if flatten {
-        children_idents
-    } else if did_insert {
-        vec![identifier]
-    } else {
-        vec![]
-    })
-}
-
-async fn cache_federation_asset(
-    session: Arc<Session>,
-    identifier: String,
-    role: AssetRole,
-) -> anyhow::Result<AssetLocation> {
-    async_cache_file(
-        &["federation-asset", role.as_str(), &identifier.clone()],
-        move |out| async move {
-            let session = session;
-            session
-                .node_asset(identifier.as_str(), role, 1024, out)
-                .await
-        },
-    )
-    .await
-}
diff --git a/server/src/main.rs b/server/src/main.rs
index 6e732bd..d9ddf8b 100644
--- a/server/src/main.rs
+++ b/server/src/main.rs
@@ -8,16 +8,13 @@
 
 use crate::routes::ui::{account::hash_password, admin::log::enable_logging};
 use database::Database;
-use federation::Federation;
-use jellybase::CONF;
+use jellybase::{CONF, federation::Federation};
 use jellycommon::user::{PermissionSet, Theme, User};
 use log::{error, warn};
 use routes::build_rocket;
 use tokio::fs::create_dir_all;
 
 pub use jellybase::database;
-pub mod federation;
-pub mod import;
 pub mod routes;
 
 #[rocket::main]
diff --git a/server/src/routes/mod.rs b/server/src/routes/mod.rs
index e95d714..4c7838d 100644
--- a/server/src/routes/mod.rs
+++ b/server/src/routes/mod.rs
@@ -3,10 +3,10 @@
     which is licensed under the GNU Affero General Public License (version 3); see /COPYING.
     Copyright (C) 2023 metamuffin <metamuffin.org>
 */
-use crate::{database::Database, federation::Federation, routes::ui::error::MyResult};
+use crate::{database::Database, routes::ui::error::MyResult};
 use api::{r_api_account_login, r_api_node_raw, r_api_root, r_api_version};
 use base64::Engine;
-use jellybase::CONF;
+use jellybase::{CONF, federation::Federation};
 use log::warn;
 use progress::{r_player_progress, r_player_watched};
 use rand::random;
diff --git a/server/src/routes/stream.rs b/server/src/routes/stream.rs
index e7b3d54..14db462 100644
--- a/server/src/routes/stream.rs
+++ b/server/src/routes/stream.rs
@@ -4,14 +4,14 @@
     Copyright (C) 2023 metamuffin <metamuffin.org>
 */
 use super::ui::{account::session::Session, error::MyError};
-use crate::{database::Database, federation::Federation};
+use crate::database::Database;
 use anyhow::{anyhow, Result};
 use jellybase::{
+    federation::Federation,
     permission::{NodePermissionExt, PermissionSetExt},
     CONF,
 };
-use jellyclient::LoginDetails;
-use jellycommon::{stream::StreamSpec, user::UserPermission, MediaSource};
+use jellycommon::{stream::StreamSpec, user::UserPermission, TrackSource};
 use log::{info, warn};
 use rocket::{
     get, head,
@@ -58,35 +58,36 @@ pub async fn r_stream(
         .as_ref()
         .ok_or(anyhow!("item does not contain media"))?;
 
-    if let MediaSource::Remote { host, remote_id } = source {
-        session
-            .user
-            .permissions
-            .assert(&UserPermission::FederatedContent)?;
+    // TODO federated streams
+    // if let MediaSource::Remote { host, remote_id } = source {
+    //     session
+    //         .user
+    //         .permissions
+    //         .assert(&UserPermission::FederatedContent)?;
 
-        let (username, password, _) = CONF
-            .remote_credentials
-            .get(host)
-            .ok_or(anyhow!("no credentials on the server-side"))?;
+    //     let (username, password, _) = CONF
+    //         .remote_credentials
+    //         .get(host)
+    //         .ok_or(anyhow!("no credentials on the server-side"))?;
 
-        info!("creating session on {host}");
-        let instance = federation.get_instance(&host)?.to_owned();
-        let session = instance
-            .login(LoginDetails {
-                username: username.to_owned(),
-                password: password.to_owned(),
-                expire: Some(60),
-                drop_permissions: Some(HashSet::from_iter([
-                    UserPermission::ManageSelf,
-                    UserPermission::Admin, // in case somebody federated the admin :)))
-                ])),
-            })
-            .await?;
+    //     info!("creating session on {host}");
+    //     let instance = federation.get_instance(&host)?.to_owned();
+    //     let session = instance
+    //         .login(LoginDetails {
+    //             username: username.to_owned(),
+    //             password: password.to_owned(),
+    //             expire: Some(60),
+    //             drop_permissions: Some(HashSet::from_iter([
+    //                 UserPermission::ManageSelf,
+    //                 UserPermission::Admin, // in case somebody federated the admin :)))
+    //             ])),
+    //         })
+    //         .await?;
 
-        let uri = session.stream(&remote_id, &spec);
-        info!("federation redirect");
-        return Ok(Either::Right(RedirectResponse(uri)));
-    }
+    //     let uri = session.stream(&remote_id, &spec);
+    //     info!("federation redirect");
+    //     return Ok(Either::Right(RedirectResponse(uri)));
+    // }
 
     info!(
         "stream request (range={})",
diff --git a/server/src/routes/ui/admin/mod.rs b/server/src/routes/ui/admin/mod.rs
index 0d1ee0a..b976192 100644
--- a/server/src/routes/ui/admin/mod.rs
+++ b/server/src/routes/ui/admin/mod.rs
@@ -9,8 +9,6 @@ pub mod user;
 use super::account::session::AdminSession;
 use crate::{
     database::Database,
-    federation::Federation,
-    import::import,
     routes::ui::{
         admin::log::rocket_uri_macro_r_admin_log,
         error::MyResult,
@@ -19,7 +17,8 @@ use crate::{
     uri,
 };
 use anyhow::anyhow;
-use jellybase::CONF;
+use jellybase::{federation::Federation, CONF};
+use jellyimport::import;
 use rand::Rng;
 use rocket::{form::Form, get, post, FromForm, State};
 use std::time::Instant;
diff --git a/server/src/routes/ui/node.rs b/server/src/routes/ui/node.rs
index 6e2f532..bcb7362 100644
--- a/server/src/routes/ui/node.rs
+++ b/server/src/routes/ui/node.rs
@@ -72,7 +72,7 @@ pub async fn r_library_node_filter<'a>(
     filter_and_sort_nodes(&filter, &mut children);
 
     Ok(Either::Left(LayoutPage {
-        title: node.title.to_string(),
+        title: node.title.clone().unwrap_or_default(),
         content: markup::new! {
             @NodePage { node: &node, id: &id, udata: &udata, children: &children, filter: &filter }
         },
@@ -82,14 +82,14 @@ pub async fn r_library_node_filter<'a>(
 
 markup::define! {
     NodeCard<'a>(id: &'a str, node: &'a NodePublic, udata: &'a NodeUserData) {
-        @let cls = format!("node card poster {}", match node.kind {NodeKind::Channel => "aspect-square", NodeKind::Video => "aspect-thumb", NodeKind::Collection => "aspect-land", _ => "aspect-port"});
+        @let cls = format!("node card poster {}", match node.kind.unwrap_or_default() {NodeKind::Channel => "aspect-square", NodeKind::Video => "aspect-thumb", NodeKind::Collection => "aspect-land", _ => "aspect-port"});
         div[class=cls] {
             .poster {
                 a[href=uri!(r_library_node(id))] {
                     img[src=uri!(r_item_assets(id, AssetRole::Poster, Some(1024)))];
                 }
                 .cardhover.item {
-                    @if !(matches!(node.kind, NodeKind::Collection | NodeKind::Channel)) {
+                    @if !(matches!(node.kind.unwrap_or_default(), NodeKind::Collection | NodeKind::Channel)) {
                         a.play.icon[href=&uri!(r_player(id, PlayerConfig::default()))] { "play_arrow" }
                     }
                     @Props { node, udata }
@@ -103,17 +103,17 @@ markup::define! {
         }
     }
     NodePage<'a>(id: &'a str, node: &'a NodePublic, udata: &'a NodeUserData, children: &'a Vec<(String, NodePublic, NodeUserData)>, filter: &'a NodeFilterSort) {
-        @if !matches!(node.kind, NodeKind::Collection) {
+        @if !matches!(node.kind.unwrap_or_default(), NodeKind::Collection) {
             img.backdrop[src=uri!(r_item_assets(id, AssetRole::Backdrop, Some(2048)))];
         }
         .page.node {
-            @if !matches!(node.kind, NodeKind::Collection) {
+            @if !matches!(node.kind.unwrap_or_default(), NodeKind::Collection) {
                 div.bigposter { img[src=uri!(r_item_assets(id, AssetRole::Poster, Some(2048)))]; }
             }
             .title {
                 h1 { @node.title }
                 @if node.media.is_some() { a.play[href=&uri!(r_player(id, PlayerConfig::default()))] { "Watch now" }}
-                @if !matches!(node.kind, NodeKind::Collection | NodeKind::Channel) {
+                @if !matches!(node.kind.unwrap_or_default(), NodeKind::Collection | NodeKind::Channel) {
                     @match udata.watched {
                         WatchedState::None |
                         WatchedState::Progress(_) => {
@@ -152,15 +152,15 @@ markup::define! {
                     }
                 }
             }
-            @if matches!(node.kind, NodeKind::Collection | NodeKind::Channel) {
-                @if matches!(node.kind, NodeKind::Collection) {
+            @if matches!(node.kind.unwrap_or_default(), NodeKind::Collection | NodeKind::Channel) {
+                @if matches!(node.kind.unwrap_or_default(), NodeKind::Collection) {
                     @if let Some(parent) = &node.path.last().cloned() {
                         a.dirup[href=uri!(r_library_node(parent))] { "Go up" }
                     }
                 }
                 @NodeFilterSortForm { f: filter }
             }
-            @match node.kind {
+            @match node.kind.unwrap_or_default() {
                 NodeKind::Collection | NodeKind::Channel => {
                     ul.children {@for (id, node, udata) in children.iter() {
                         li { @NodeCard { id, node, udata } }
@@ -261,7 +261,9 @@ impl MediaInfoExt for MediaInfo {
         let mut maxdim = 0;
         for t in &self.tracks {
             match &t.kind {
-                SourceTrackKind::Video { width, height, .. } => maxdim = maxdim.max(*width.max(height)),
+                SourceTrackKind::Video { width, height, .. } => {
+                    maxdim = maxdim.max(*width.max(height))
+                }
                 _ => (),
             }
         }
diff --git a/server/src/routes/ui/player.rs b/server/src/routes/ui/player.rs
index 177a5f6..8b8adf6 100644
--- a/server/src/routes/ui/player.rs
+++ b/server/src/routes/ui/player.rs
@@ -59,7 +59,7 @@ pub fn r_player<'a>(
 
     let conf = player_conf(item.clone(), playing)?;
     Ok(LayoutPage {
-        title: item.public.title.to_owned(),
+        title: item.public.title.to_owned().unwrap_or_default(),
         class: Some("player"),
         content: markup::new! {
             @if playing {
diff --git a/server/src/routes/ui/sort.rs b/server/src/routes/ui/sort.rs
index c7fbfc2..143a101 100644
--- a/server/src/routes/ui/sort.rs
+++ b/server/src/routes/ui/sort.rs
@@ -139,14 +139,14 @@ pub fn filter_and_sort_nodes(
                 o &= !match p {
                     FilterProperty::FederationLocal => node.federated.is_none(),
                     FilterProperty::FederationRemote => node.federated.is_some(),
-                    FilterProperty::KindMovie => node.kind == NodeKind::Movie,
-                    FilterProperty::KindVideo => node.kind == NodeKind::Video,
-                    FilterProperty::KindCollection => node.kind == NodeKind::Collection,
-                    FilterProperty::KindChannel => node.kind == NodeKind::Channel,
-                    FilterProperty::KindShow => node.kind == NodeKind::Show,
-                    FilterProperty::KindSeries => node.kind == NodeKind::Series,
-                    FilterProperty::KindSeason => node.kind == NodeKind::Season,
-                    FilterProperty::KindEpisode => node.kind == NodeKind::Episode,
+                    FilterProperty::KindMovie => node.kind == Some(NodeKind::Movie),
+                    FilterProperty::KindVideo => node.kind == Some(NodeKind::Video),
+                    FilterProperty::KindCollection => node.kind == Some(NodeKind::Collection),
+                    FilterProperty::KindChannel => node.kind == Some(NodeKind::Channel),
+                    FilterProperty::KindShow => node.kind == Some(NodeKind::Show),
+                    FilterProperty::KindSeries => node.kind == Some(NodeKind::Series),
+                    FilterProperty::KindSeason => node.kind == Some(NodeKind::Season),
+                    FilterProperty::KindEpisode => node.kind == Some(NodeKind::Episode),
                     FilterProperty::Watched => udata.watched == WatchedState::Watched,
                     FilterProperty::Unwatched => udata.watched == WatchedState::None,
                     FilterProperty::WatchProgress => {