/* This file is part of jellything (https://codeberg.org/metamuffin/jellything) which is licensed under the GNU Affero General Public License (version 3); see /COPYING. Copyright (C) 2023 metamuffin */ use crate::{database::Database, federation::Federation, CONF}; use anyhow::{anyhow, bail, Context, Ok}; use async_recursion::async_recursion; use base64::Engine; use futures::{stream::FuturesUnordered, StreamExt}; use jellyclient::Session; use jellycommon::{AssetLocation, MediaSource, Node, NodePrivate, RemoteImportOptions}; use log::{debug, error, info}; use std::{ ffi::OsStr, fs::File, os::unix::prelude::OsStrExt, path::PathBuf, str::FromStr, sync::LazyLock, }; use tokio::sync::Semaphore; static IMPORT_SEM: LazyLock = LazyLock::new(|| Semaphore::new(1)); pub async fn import(db: &Database, fed: &Federation) -> anyhow::Result<()> { info!("clearing node tree"); let permit = IMPORT_SEM.try_acquire()?; db.node.clear()?; info!("importing..."); let (_, errs) = import_path(CONF.library_path.clone(), db, fed, None) .await .context("indexing")?; info!("import completed"); drop(permit); if errs == 0 { Ok(()) } else { Err(anyhow!( "partial import, {errs} errors occured; see server log" )) } } #[async_recursion] pub async fn import_path( path: PathBuf, db: &Database, fed: &Federation, parent: Option, ) -> anyhow::Result<(Vec, usize)> { if path.is_dir() { let mpath = path.join("directory.json"); let children_paths = path.read_dir()?.map(Result::unwrap).filter_map(|e| { if e.path().extension() == Some(&OsStr::from_bytes(b"jelly")) || e.metadata().unwrap().is_dir() { Some(e.path()) } else { None } }); let identifier = if mpath.exists() { path.file_name().unwrap().to_str().unwrap().to_string() } else { parent .clone() .ok_or(anyhow!("non-root node requires parent"))? }; let mut all: FuturesUnordered<_> = children_paths .into_iter() .map(|p| import_path(p, db, fed, Some(identifier.clone()))) .collect(); let mut children_ids = Vec::new(); let mut errs = 0; while let Some(k) = all.next().await { match k { core::result::Result::Ok((els, errs2)) => { errs += errs2; children_ids.extend(els) } Err(e) => { errs += 1; error!("import failed: {e:?}") } } } if mpath.exists() { let mut data: Node = serde_json::from_reader(File::open(mpath).context("metadata missing")?)?; data.public.children = children_ids; data.public.parent = parent; info!("adding {identifier}"); db.node.insert(&identifier, &data)?; Ok((vec![identifier], errs)) } else { Ok((children_ids, errs)) } } else if path.is_file() { info!("loading {path:?}"); let datafile = File::open(path.clone()).context("cant load metadata")?; let mut data: Node = serde_json::from_reader(datafile).context("invalid metadata")?; let identifier = data.private.id.clone().unwrap_or_else(|| { path.file_name() .unwrap() .to_str() .unwrap() .strip_suffix(".jelly") .unwrap() .to_string() }); if let Some(io) = data.private.import.take() { let session = fed .get_session(&io.host) .await .context("creating session")?; import_remote(io, db, &session, &identifier, parent) .await .context("federated import")?; } else { debug!("adding {identifier}"); data.public.parent = parent; db.node.insert(&identifier, &data)?; } Ok((vec![identifier], 0)) } else { bail!("did somebody really put a fifo or socket in the library?!") } } #[async_recursion] async fn import_remote( opts: RemoteImportOptions, db: &Database, session: &Session, identifier: &String, parent: Option, ) -> anyhow::Result<()> { info!("loading federated node {identifier:?}"); let node = session .node(&opts.id) .await .context("fetching remote node")?; let poster = cache_federation_asset(session, &opts.id, "poster").await?; let backdrop = cache_federation_asset(session, &opts.id, "backdrop").await?; let mut node = Node { public: node, private: NodePrivate { backdrop: Some(AssetLocation::Cache(backdrop)), poster: Some(AssetLocation::Cache(poster)), import: None, id: None, source: Some(MediaSource::Remote { host: opts.host.clone(), remote_id: opts.id.clone(), }), }, }; for c in &mut node.public.children { let prefixed = format!("{}{c}", opts.prefix.clone().unwrap_or(String::new())); import_remote( RemoteImportOptions { id: c.to_owned(), ..opts.clone() }, db, session, &prefixed, Some(opts.id.clone()), ) .await?; *c = prefixed; } node.public.parent = parent; node.public.federated = Some(opts.host.clone()); debug!("adding {identifier}"); db.node.insert(identifier, &node)?; Ok(()) } async fn cache_federation_asset( session: &Session, identifier: &String, role: &str, ) -> anyhow::Result { let (poster, download) = cache_file(&["federation-asset", role, identifier]); if let Some(d) = download { session.node_asset(&identifier, role, d).await?; } Ok(poster) } fn cache_file(seed: &[&str]) -> (PathBuf, Option) { use sha2::Digest; let mut d = sha2::Sha512::new(); for s in seed { d.update(s.as_bytes()); d.update(b"\0"); } let d = d.finalize(); let fname = base64::engine::general_purpose::URL_SAFE.encode(d); let fname = &fname[..22]; // about 128 bits let fullpath = CONF.cache_path.join(fname); let cachepath = PathBuf::from_str(fname).unwrap(); let f = if !fullpath.exists() { Some(File::create(&fullpath).unwrap()) } else { None }; (cachepath, f) }