aboutsummaryrefslogtreecommitdiff
path: root/import/src
diff options
context:
space:
mode:
Diffstat (limited to 'import/src')
-rw-r--r--import/src/lib.rs105
-rw-r--r--import/src/plugins/acoustid.rs24
-rw-r--r--import/src/plugins/infojson.rs144
-rw-r--r--import/src/plugins/media_info.rs166
-rw-r--r--import/src/plugins/misc.rs58
-rw-r--r--import/src/plugins/mod.rs9
-rw-r--r--import/src/plugins/tags.rs4
-rw-r--r--import/src/plugins/tmdb.rs9
-rw-r--r--import/src/plugins/trakt.rs83
-rw-r--r--import/src/plugins/vgmdb.rs3
10 files changed, 359 insertions, 246 deletions
diff --git a/import/src/lib.rs b/import/src/lib.rs
index ad929fa..2d8d987 100644
--- a/import/src/lib.rs
+++ b/import/src/lib.rs
@@ -14,7 +14,7 @@ use crate::{
};
use anyhow::{Context, Result, anyhow};
use jellycache::{HashKey, cache_memory, cache_store};
-use jellycommon::jellyobject::{self, Object, ObjectBuffer, Tag, TypedTag};
+use jellycommon::jellyobject::{self, ObjectBuffer, Tag, TypedTag};
use jellydb::{
backends::Database,
query::{Filter, Query, Sort},
@@ -33,12 +33,14 @@ use serde::{Deserialize, Serialize};
use std::{
collections::HashSet,
fs::{File, read_to_string},
+ hash::{DefaultHasher, Hash},
marker::PhantomData,
mem::swap,
path::{Path, PathBuf},
sync::{Arc, LazyLock, Mutex},
time::UNIX_EPOCH,
};
+use std::{fmt::Display, hash::Hasher};
use tokio::{runtime::Handle, sync::Semaphore, task::spawn_blocking};
#[rustfmt::skip]
@@ -81,13 +83,20 @@ pub fn is_importing() -> bool {
IMPORT_SEM.available_permits() == 0
}
+#[derive(Debug, Clone)]
pub struct NodeID(pub String);
+impl Display for NodeID {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ f.write_str(&self.0)
+ }
+}
const NODE_ID: TypedTag<&str> = TypedTag(Tag(0x8123), PhantomData);
+#[derive(Clone)]
struct DatabaseTables {
db: Arc<dyn Database>,
- nodes: Table,
+ nodes: Arc<Table>,
}
fn node_id_query(node: &NodeID) -> Query {
@@ -101,10 +110,23 @@ fn node_id_query(node: &NodeID) -> Query {
}
impl DatabaseTables {
- pub fn update_node_init(
+ pub fn update_node(
+ &self,
+ node: RowNum,
+ mut update: impl FnMut(ObjectBuffer) -> ObjectBuffer,
+ ) -> Result<()> {
+ self.db.write_transaction(&mut |txn| {
+ let ob_before = self.nodes.get(txn, node)?.unwrap();
+ let ob_after = update(ob_before);
+ self.nodes.update(txn, node, ob_after)?;
+ Ok(())
+ })?;
+ Ok(())
+ }
+ pub fn update_node_by_nodeid(
&self,
node: NodeID,
- mut update: impl FnMut(Object) -> Option<ObjectBuffer>,
+ mut update: impl FnMut(ObjectBuffer) -> ObjectBuffer,
) -> Result<()> {
self.db.write_transaction(&mut |txn| {
let node = match self.nodes.query_single(txn, node_id_query(&node)) {
@@ -113,9 +135,17 @@ impl DatabaseTables {
.nodes
.insert(txn, ObjectBuffer::new(&mut [(NODE_ID.0, &node.0.as_str())]))?,
};
- let ob = self.nodes.get(txn, node)?.unwrap();
- if let Some(changed) = update(ob.as_object()) {
- self.nodes.update(txn, node, changed)?;
+
+ let ob_before = self.nodes.get(txn, node)?.unwrap();
+ let mut hash_before = DefaultHasher::new();
+ ob_before.hash(&mut hash_before);
+
+ let ob_after = update(ob_before);
+
+ let mut hash_after = DefaultHasher::new();
+ ob_after.hash(&mut hash_after);
+ if hash_before.finish() != hash_after.finish() {
+ self.nodes.update(txn, node, ob_after)?;
}
Ok(())
})?;
@@ -145,14 +175,14 @@ pub async fn import_wrap(db: DatabaseTables, incremental: bool) -> Result<()> {
Ok(())
}
-fn import(db: DatabaseTables, rt: &Handle, incremental: bool) -> Result<()> {
+fn import(dba: DatabaseTables, rt: &Handle, incremental: bool) -> Result<()> {
let plugins = init_plugins(&CONF.api);
let files = Mutex::new(Vec::new());
import_traverse(
&CONF.media_path,
- db,
+ &dba,
incremental,
- NodeID::MIN,
+ None,
InheritedFlags::default(),
&files,
)?;
@@ -162,7 +192,7 @@ fn import(db: DatabaseTables, rt: &Handle, incremental: bool) -> Result<()> {
files.into_par_iter().for_each(|(path, parent, iflags)| {
reporting::set_task(format!("unknown: {path:?}"));
- import_file(db, &rt, &nodes, &plugins, &path, parent, iflags);
+ import_file(&dba, &rt, &nodes, &plugins, &path, parent, iflags);
IMPORT_PROGRESS
.blocking_write()
.as_mut()
@@ -183,7 +213,7 @@ fn import(db: DatabaseTables, rt: &Handle, incremental: bool) -> Result<()> {
swap(nodes.get_mut().unwrap(), &mut cur_nodes);
cur_nodes.into_par_iter().for_each(|node| {
reporting::set_task(format!("unknown: {node}"));
- process_node(db, &rt, &plugins, &nodes, node);
+ process_node(&dba, &rt, &plugins, &nodes, node);
IMPORT_PROGRESS
.blocking_write()
.as_mut()
@@ -196,27 +226,20 @@ fn import(db: DatabaseTables, rt: &Handle, incremental: bool) -> Result<()> {
Ok(())
}
-#[derive(Debug, Clone, Copy)]
+#[derive(Debug, Clone, Copy, Default)]
pub struct InheritedFlags {
- visibility: Visibility,
+ hidden: bool,
+ reduced: bool,
use_acoustid: bool,
}
-impl Default for InheritedFlags {
- fn default() -> Self {
- Self {
- visibility: Visibility::Visible,
- use_acoustid: false,
- }
- }
-}
fn import_traverse(
path: &Path,
- db: DatabaseTables,
+ dba: &DatabaseTables,
incremental: bool,
- parent: Option<RowNum>,
+ parent: Option<NodeID>,
mut iflags: InheritedFlags,
- out: &Mutex<Vec<(PathBuf, RowNum, InheritedFlags)>>,
+ out: &Mutex<Vec<(PathBuf, NodeID, InheritedFlags)>>,
) -> Result<()> {
if path.is_dir() {
reporting::set_task(format!("indexing {path:?}"));
@@ -228,15 +251,15 @@ fn import_traverse(
if let Ok(content) = read_to_string(path.join("flags")) {
for flag in content.lines() {
match flag.trim() {
- "hidden" => iflags.visibility = iflags.visibility.min(Visibility::Hidden),
- "reduced" => iflags.visibility = iflags.visibility.min(Visibility::Reduced),
+ "hidden" => iflags.hidden = true,
+ "reduced" => iflags.reduced = true,
"use_acoustid" => iflags.use_acoustid = true,
_ => (),
}
}
}
- db.update_node_init(node, |n| {
+ dba.update_node_by_nodeid(node, |n| {
if parent != NodeID::MIN {
n.parents.insert(parent);
}
@@ -247,7 +270,7 @@ fn import_traverse(
path.read_dir()?.par_bridge().try_for_each(|e| {
let path = e?.path();
reporting::catch(
- import_traverse(&path, db, incremental, node, iflags, out)
+ import_traverse(&path, dba, incremental, node, iflags, out)
.context(anyhow!("index {:?}", path.file_name().unwrap())),
);
anyhow::Ok(())
@@ -260,7 +283,7 @@ fn import_traverse(
let mtime = meta.modified()?.duration_since(UNIX_EPOCH)?.as_secs();
if incremental {
- if let Some(last_mtime) = db.get_import_file_mtime(path)? {
+ if let Some(last_mtime) = dba.get_import_file_mtime(path)? {
if last_mtime >= mtime {
return Ok(());
}
@@ -278,9 +301,9 @@ fn import_traverse(
}
fn import_file(
- db: &Database,
+ dba: &DatabaseTables,
rt: &Handle,
- nodes: &Mutex<HashSet<NodeID>>,
+ pending_nodes: &Mutex<HashSet<NodeID>>,
plugins: &[Box<dyn ImportPlugin>],
path: &Path,
parent: NodeID,
@@ -288,10 +311,10 @@ fn import_file(
) {
let mut all_ok = true;
let ct = ImportContext {
- db,
+ dba,
rt,
iflags,
- pending_nodes: nodes,
+ pending_nodes,
};
let filename = path.file_name().unwrap().to_string_lossy();
if filename == "flags" {
@@ -300,7 +323,7 @@ fn import_file(
else {
return;
};
- nodes.lock().unwrap().insert(parent);
+ pending_nodes.lock().unwrap().insert(parent);
for line in content.lines() {
for p in plugins {
let inf = p.info();
@@ -320,7 +343,7 @@ fn import_file(
let slug = get_node_slug(path).unwrap();
let node = NodeID::from_slug(&slug);
- nodes.lock().unwrap().insert(node);
+ pending_nodes.lock().unwrap().insert(node);
all_ok &= reporting::catch(db.update_node_init(node, |node| {
node.slug = slug;
if parent != NodeID::MIN {
@@ -390,14 +413,14 @@ fn import_file(
}
fn process_node(
- dba: DatabaseTables,
+ dba: &DatabaseTables,
rt: &Handle,
plugins: &[Box<dyn ImportPlugin>],
- nodes: &Mutex<HashSet<NodeID>>,
+ pending_nodes: &Mutex<HashSet<NodeID>>,
node: NodeID,
) {
let Some(data) = reporting::catch(
- db.get_node(node)
+ dba.get_node(node)
.and_then(|e| e.ok_or(anyhow!("node missing"))),
) else {
return;
@@ -408,7 +431,7 @@ fn process_node(
if inf.handle_process {
reporting::set_task(format!("{}(proc): {slug}", inf.name));
let Some(data) = reporting::catch(
- db.get_node(node)
+ dba.get_node(node)
.and_then(|e| e.ok_or(anyhow!("node missing"))),
) else {
return;
@@ -419,7 +442,7 @@ fn process_node(
dba,
rt,
iflags: InheritedFlags::default(),
- pending_nodes: nodes,
+ pending_nodes,
},
node,
&data,
diff --git a/import/src/plugins/acoustid.rs b/import/src/plugins/acoustid.rs
index b93533a..9edcb63 100644
--- a/import/src/plugins/acoustid.rs
+++ b/import/src/plugins/acoustid.rs
@@ -9,6 +9,9 @@ use crate::{
};
use anyhow::{Context, Result};
use jellycache::{HashKey, cache_memory};
+use jellycommon::{
+ IDENT_ACOUST_ID_TRACK, IDENT_MUSICBRAINZ_RECORDING, NO_IDENTIFIERS, jellyobject::Object,
+};
use jellydb::table::RowNum;
use jellyremuxer::matroska::Segment;
use log::info;
@@ -171,12 +174,23 @@ impl ImportPlugin for AcoustID {
return Ok(());
}
let fp = acoustid_fingerprint(path)?;
+
if let Some((atid, mbid)) = self.get_atid_mbid(&fp, &ct.rt)? {
- ct.db.update_node_init(node, |n| {
- n.identifiers.insert(IdentifierType::AcoustIdTrack, atid);
- n.identifiers
- .insert(IdentifierType::MusicbrainzRecording, mbid);
- })?;
+ ct.dba.db.write_transaction(&mut |txn| {
+ let ob = ct.dba.nodes.get(txn, node)?.unwrap();
+ let ob = ob.as_object();
+ let ob = ob.insert(
+ NO_IDENTIFIERS,
+ ob.get(NO_IDENTIFIERS)
+ .unwrap_or(Object::EMPTY)
+ .insert(IDENT_ACOUST_ID_TRACK, &atid)
+ .as_object()
+ .insert(IDENT_MUSICBRAINZ_RECORDING, &mbid)
+ .as_object(),
+ );
+ ct.dba.nodes.update(txn, node, ob)?;
+ Ok(())
+ });
};
Ok(())
}
diff --git a/import/src/plugins/infojson.rs b/import/src/plugins/infojson.rs
index fd15e03..72dd1ab 100644
--- a/import/src/plugins/infojson.rs
+++ b/import/src/plugins/infojson.rs
@@ -3,17 +3,22 @@
which is licensed under the GNU Affero General Public License (version 3); see /COPYING.
Copyright (C) 2026 metamuffin <metamuffin.org>
*/
+use crate::plugins::{ImportContext, ImportPlugin, PluginInfo};
use anyhow::{Context, Result, anyhow};
use chrono::{Utc, format::Parsed};
use jellycache::cache_read;
+use jellycommon::{
+ IDENT_BANDCAMP, IDENT_YOUTUBE_CHANNEL, IDENT_YOUTUBE_CHANNEL_HANDLE, IDENT_YOUTUBE_VIDEO,
+ KIND_CHANNEL, KIND_MUSIC, KIND_SHORTFORMVIDEO, KIND_VIDEO, NO_DESCRIPTION, NO_IDENTIFIERS,
+ NO_KIND, NO_RATINGS, NO_RELEASEDATE, NO_SUBTITLE, NO_TAG, NO_TITLE, RTYP_YOUTUBE_FOLLOWERS,
+ RTYP_YOUTUBE_LIKES, RTYP_YOUTUBE_VIEWS,
+};
use jellydb::table::RowNum;
use jellyremuxer::matroska::{AttachedFile, Segment};
use log::info;
use serde::{Deserialize, Serialize};
use std::{collections::HashMap, fs::File, io::BufReader, path::Path};
-use crate::plugins::{ImportContext, ImportPlugin, PluginInfo};
-
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct YVideo {
pub album: Option<String>,
@@ -174,29 +179,38 @@ impl ImportPlugin for Infojson {
info!("import channel info.json at {path:?}");
let data = serde_json::from_reader::<_, YVideo>(BufReader::new(File::open(path)?))?;
- ct.db.update_node_init(parent, |node| {
- node.kind = NodeKind::Channel;
- node.title = Some(clean_uploader_name(&data.title).to_owned());
- if let Some(cid) = data.channel_id {
- node.identifiers.insert(IdentifierType::YoutubeChannel, cid);
+ let title = clean_uploader_name(&data.title);
+
+ ct.dba.db.write_transaction(&mut |txn| {
+ let mut node = ct.dba.nodes.get(txn, parent)?.unwrap();
+ node = node.as_object().insert(NO_KIND, KIND_CHANNEL);
+ node = node.as_object().insert(NO_TITLE, title);
+ if let Some(cid) = &data.channel_id {
+ node = node.as_object().update(NO_IDENTIFIERS, |ids| {
+ ids.insert(IDENT_YOUTUBE_CHANNEL, &cid)
+ });
}
- if let Some(uid) = data.uploader_id {
- node.identifiers
- .insert(IdentifierType::YoutubeChannelHandle, uid);
+ if let Some(uid) = &data.uploader_id {
+ node = node.as_object().update(NO_IDENTIFIERS, |ids| {
+ ids.insert(IDENT_YOUTUBE_CHANNEL_HANDLE, &uid)
+ })
}
- if let Some(desc) = data.description {
- node.description = Some(desc);
+ if let Some(desc) = &data.description {
+ node = node.as_object().insert(NO_DESCRIPTION, &desc);
}
if let Some(followers) = data.channel_follower_count {
- node.ratings
- .insert(RatingType::YoutubeFollowers, followers as f64);
+ node = node.as_object().update(NO_RATINGS, |rat| {
+ rat.insert(RTYP_YOUTUBE_FOLLOWERS, followers as f64)
+ });
}
- })?;
+
+ ct.dba.nodes.update(txn, parent, node)
+ });
Ok(())
}
- fn media(&self, ct: &ImportContext, node: NodeID, _path: &Path, seg: &Segment) -> Result<()> {
+ fn media(&self, ct: &ImportContext, row: RowNum, _path: &Path, seg: &Segment) -> Result<()> {
let infojson = seg
.attachments
.iter()
@@ -217,53 +231,71 @@ impl ImportPlugin for Infojson {
.map(|date| parse_upload_date(date).context("parsing upload date"))
.transpose()?;
- ct.db.update_node_init(node, |node| {
- node.kind = if let Some(ty) = &infojson.media_type
- && ty == "short"
+ let kind = if let Some(ty) = &infojson.media_type
+ && ty == "short"
+ {
+ KIND_SHORTFORMVIDEO
+ } else if infojson.album.is_some() {
+ KIND_MUSIC
+ } else {
+ KIND_VIDEO
+ };
+
+ ct.dba.db.write_transaction(&mut |txn| {
+ let mut node = ct.dba.nodes.get(txn, row)?.unwrap();
+ node = node.as_object().insert(NO_KIND, kind);
+ node = node.as_object().insert(NO_TITLE, &infojson.title);
+ if let Some(title) = &infojson.alt_title
+ && title != &infojson.title
+ && !node.as_object().has(NO_SUBTITLE.0)
{
- NodeKind::ShortFormVideo
- } else if infojson.album.is_some() {
- NodeKind::Music
- } else {
- NodeKind::Video
- };
- node.title = Some(infojson.title);
- node.subtitle = if infojson.alt_title != node.title {
- infojson.alt_title
- } else {
- None
+ node = node.as_object().insert(NO_SUBTITLE, &title);
}
- .or(infojson
- .uploader
- .as_ref()
- .map(|u| clean_uploader_name(u).to_owned()))
- .or(node.subtitle.clone());
-
- node.tags.extend(infojson.tags.unwrap_or_default());
-
- if let Some(desc) = infojson.description {
- node.description = Some(desc)
+ if let Some(up) = &infojson.uploader
+ && !node.as_object().has(NO_SUBTITLE.0)
+ {
+ node = node
+ .as_object()
+ .insert(NO_SUBTITLE, &clean_uploader_name(&up));
+ }
+ if let Some(desc) = &infojson.description {
+ node = node.as_object().insert(NO_DESCRIPTION, &desc);
+ }
+ if let Some(tag) = infojson.tags.clone() {
+ node = node
+ .as_object()
+ .extend(NO_TAG, tag.iter().map(String::as_str));
+ }
+ if let Some(rd) = release_date {
+ node = node.as_object().insert(NO_RELEASEDATE, rd);
}
- node.release_date = release_date.or(node.release_date);
- node.tagline = Some(infojson.webpage_url);
match infojson.extractor.as_str() {
"youtube" => {
- node.identifiers
- .insert(IdentifierType::YoutubeVideo, infojson.id);
- node.ratings.insert(
- RatingType::YoutubeViews,
- infojson.view_count.unwrap_or_default() as f64,
- );
- if let Some(lc) = infojson.like_count {
- node.ratings.insert(RatingType::YoutubeLikes, lc as f64);
- }
+ node = node.as_object().update(NO_IDENTIFIERS, |rat| {
+ rat.insert(IDENT_YOUTUBE_VIDEO, &infojson.id)
+ });
+ node = node.as_object().update(NO_RATINGS, |rat| {
+ rat.insert(
+ RTYP_YOUTUBE_VIEWS,
+ infojson.view_count.unwrap_or_default() as f64,
+ )
+ });
+ node = node.as_object().update(NO_RATINGS, |rat| {
+ rat.insert(
+ RTYP_YOUTUBE_LIKES,
+ infojson.like_count.unwrap_or_default() as f64,
+ )
+ });
+ }
+ "Bandcamp" => {
+ node = node.as_object().update(NO_IDENTIFIERS, |rat| {
+ rat.insert(IDENT_BANDCAMP, &infojson.id)
+ });
}
- "Bandcamp" => drop(
- node.identifiers
- .insert(IdentifierType::Bandcamp, infojson.id),
- ),
_ => (),
- }
+ };
+
+ ct.dba.nodes.update(txn, row, node)
})?;
}
Ok(())
diff --git a/import/src/plugins/media_info.rs b/import/src/plugins/media_info.rs
index da445a3..250e417 100644
--- a/import/src/plugins/media_info.rs
+++ b/import/src/plugins/media_info.rs
@@ -5,10 +5,25 @@
*/
use crate::plugins::{ImportContext, ImportPlugin, PluginInfo};
-use anyhow::{Result, anyhow};
-use jellyremuxer::matroska::Segment;
+use anyhow::Result;
+use jellycommon::{
+ jellyobject::{Object, ObjectBuffer, Tag},
+ *,
+};
+use jellydb::table::RowNum;
+use jellyremuxer::matroska::{Segment, TrackType};
use std::path::Path;
+fn lang_str_to_tag(s: &str) -> Tag {
+ use jellycommon::*;
+ match s {
+ "eng" => LANG_ENG.0,
+ "deu" => LANG_DEU.0,
+ "jpn" => LANG_JPN.0,
+ _ => LANG_UND.0,
+ }
+}
+
pub struct MediaInfo;
impl ImportPlugin for MediaInfo {
fn info(&self) -> PluginInfo {
@@ -18,71 +33,98 @@ impl ImportPlugin for MediaInfo {
..Default::default()
}
}
- fn media(&self, ct: &ImportContext, node: NodeID, path: &Path, seg: &Segment) -> Result<()> {
- let tracks = seg
- .tracks
- .as_ref()
- .ok_or(anyhow!("no tracks"))?
- .entries
- .iter()
- .map(|track| SourceTrack {
- codec: track.codec_id.clone(),
- language: track.language.clone(),
- name: track.name.clone().unwrap_or_default(),
- federated: Vec::new(),
- kind: if let Some(video) = &track.video {
- SourceTrackKind::Video {
- width: video.pixel_width,
- height: video.pixel_height,
- fps: video.frame_rate,
- }
- } else if let Some(audio) = &track.audio {
- SourceTrackKind::Audio {
- channels: audio.channels as usize,
- sample_rate: audio.sampling_frequency,
- bit_depth: audio.bit_depth.map(|r| r as usize),
- }
- } else {
- SourceTrackKind::Subtitle
- },
- source: TrackSource::Local(path.to_owned(), track.track_number),
- })
- .collect::<Vec<_>>();
-
+ fn media(&self, ct: &ImportContext, row: RowNum, path: &Path, seg: &Segment) -> Result<()> {
let size = path.metadata()?.len();
-
- ct.db.update_node_init(node, |node| {
- node.storage_size = size;
- node.media = Some(jellycommon::MediaInfo {
- chapters: seg
- .chapters
- .clone()
- .map(|c| {
- let mut chaps = Vec::new();
- if let Some(ee) = c.edition_entries.first() {
- for ca in &ee.chapter_atoms {
- let mut labels = Vec::new();
- for cd in &ca.displays {
- for lang in &cd.languages {
- labels.push((lang.to_owned(), cd.string.clone()))
- }
- }
- chaps.push(Chapter {
- labels,
- time_start: Some(ca.time_start as f64 * 1e-9),
- time_end: ca.time_end.map(|ts| ts as f64 * 1e-9),
- })
+ ct.dba.db.write_transaction(&mut |txn| {
+ let mut node = ct.dba.nodes.get(txn, row)?.unwrap();
+ if let Some(tracks) = &seg.tracks {
+ node = node.as_object().extend_object(
+ NO_TRACK,
+ TR_SOURCE.0,
+ tracks.entries.iter().map(|t| {
+ let mut track = ObjectBuffer::empty();
+ track = track.as_object().insert(TR_CODEC, &t.codec_id);
+ track = track
+ .as_object()
+ .insert(TR_LANGUAGE, lang_str_to_tag(&t.language));
+ if let Some(name) = &t.name {
+ track = track.as_object().insert(TR_NAME, name);
+ }
+ track = track.as_object().insert(
+ TR_KIND,
+ match t.track_type {
+ TrackType::Audio => TRKIND_AUDIO,
+ TrackType::Video => TRKIND_VIDEO,
+ TrackType::Subtitle => TRKIND_TEXT,
+ _ => TRKIND_UNKNOWN,
+ },
+ );
+ if let Some(v) = &t.video {
+ track = track
+ .as_object()
+ .insert(TR_PIXEL_WIDTH, v.pixel_width as u32);
+ track = track
+ .as_object()
+ .insert(TR_PIXEL_HEIGHT, v.pixel_height as u32);
+ if let Some(fr) = v.frame_rate {
+ track = track.as_object().insert(TR_RATE, fr);
+ }
+ }
+ if let Some(a) = &t.audio {
+ track = track.as_object().insert(TR_CHANNELS, a.channels as u32);
+ track = track.as_object().insert(TR_RATE, a.sampling_frequency);
+ if let Some(d) = a.bit_depth {
+ track = track.as_object().insert(TR_BIT_DEPTH, d as u32);
}
}
- chaps
- })
- .unwrap_or_default(),
- duration: fix_invalid_runtime(
+
+ let source = Object::EMPTY
+ .insert(TRSOURCE_LOCAL_PATH, &path.to_string_lossy())
+ .as_object()
+ .insert(TRSOURCE_LOCAL_TRACKNUM, t.track_number);
+ track = track.as_object().insert(TR_SOURCE, source.as_object());
+
+ track
+ }),
+ );
+ }
+
+ if let Some(chapters) = &seg.chapters {
+ node = node.as_object().extend_object(
+ NO_CHAPTER,
+ CH_NAME.0,
+ chapters
+ .edition_entries
+ .iter()
+ .flat_map(|e| &e.chapter_atoms)
+ .map(|cha| {
+ let mut chapter = ObjectBuffer::empty();
+ chapter = chapter
+ .as_object()
+ .insert(CH_START, cha.time_start as f64 * 1e-9);
+ if let Some(end) = cha.time_end {
+ chapter = chapter.as_object().insert(CH_END, end as f64 * 1e-9);
+ }
+ if let Some(display) = cha.displays.iter().next() {
+ chapter = chapter.as_object().insert(CH_NAME, &display.string);
+ }
+ chapter
+ }),
+ );
+ }
+
+ node = node.as_object().insert(
+ NO_DURATION,
+ fix_invalid_runtime(
seg.info.duration.unwrap_or_default() * seg.info.timestamp_scale as f64 * 1e-9,
),
- tracks,
- });
- })?;
+ );
+ node = node.as_object().insert(NO_STORAGE_SIZE, size);
+
+ ct.dba.nodes.update(txn, row, node);
+
+ Ok(())
+ });
Ok(())
}
diff --git a/import/src/plugins/misc.rs b/import/src/plugins/misc.rs
index 43bd118..97bb6a5 100644
--- a/import/src/plugins/misc.rs
+++ b/import/src/plugins/misc.rs
@@ -6,7 +6,7 @@
use crate::plugins::{ImportContext, ImportPlugin, PluginInfo};
use anyhow::{Context, Result, bail};
use jellycache::{HashKey, cache_store};
-use jellycommon::{PICT_BACKDROP, PICT_COVER};
+use jellycommon::{jellyobject::inspect::Inspector, *};
use jellydb::table::RowNum;
use jellyremuxer::matroska::{AttachedFile, Segment};
use log::info;
@@ -27,24 +27,33 @@ impl ImportPlugin for ImageFiles {
..Default::default()
}
}
- fn file(&self, ct: &ImportContext, parent: RowNum, path: &Path) -> Result<()> {
+ fn file(&self, ct: &ImportContext, row: RowNum, path: &Path) -> Result<()> {
let filename = path.file_name().unwrap().to_string_lossy();
let slot = match filename.as_ref() {
"poster.jpeg" | "poster.webp" | "poster.png" => PICT_COVER,
"backdrop.jpeg" | "backdrop.webp" | "backdrop.png" => PICT_BACKDROP,
_ => return Ok(()),
};
- info!("import {} at {path:?}", slot);
- let asset = Asset(cache_store(
- format!("media/literal/{}-{slot}.image", HashKey(path)),
+ info!("import {:?} at {path:?}", Inspector(&TAGREG, slot));
+ let asset = cache_store(
+ format!(
+ "media/literal/{}-{}.image",
+ HashKey(path),
+ TAGREG.name(slot.0)
+ ),
|| {
let mut data = Vec::new();
File::open(path)?.read_to_end(&mut data)?;
Ok(data)
},
- )?);
- ct.db.update_node_init(parent, |node| {
- node.pictures.insert(slot, asset);
+ )?;
+ ct.dba.db.write_transaction(&mut |txn| {
+ let mut node = ct.dba.nodes.get(txn, row)?.unwrap();
+ node = node
+ .as_object()
+ .update(NO_PICTURES, |picts| picts.insert(slot, &asset));
+ ct.dba.nodes.update(txn, row, node);
+ Ok(())
})?;
Ok(())
}
@@ -62,19 +71,20 @@ impl ImportPlugin for ImageAttachments {
..Default::default()
}
}
- fn media(&self, ct: &ImportContext, node: NodeID, _path: &Path, seg: &Segment) -> Result<()> {
+ fn media(&self, ct: &ImportContext, row: RowNum, _path: &Path, seg: &Segment) -> Result<()> {
let Some(cover) = seg
.attachments
.iter()
.flat_map(|a| &a.files)
.find(is_cover)
- .map(|att| Asset(att.data.clone().try_into().unwrap()))
+ .map(|att| String::from_utf8_lossy(&att.data))
else {
return Ok(());
};
- ct.db.update_node_init(node, |node| {
- node.pictures.insert(PictureSlot::Cover, cover);
+ ct.dba.update_node(row, |node| {
+ node.as_object()
+ .update(NO_PICTURES, |picts| picts.insert(PICT_COVER, &cover))
})?;
Ok(())
}
@@ -89,9 +99,9 @@ impl ImportPlugin for General {
..Default::default()
}
}
- fn instruction(&self, ct: &ImportContext, node: NodeID, line: &str) -> Result<()> {
+ fn instruction(&self, ct: &ImportContext, node: RowNum, line: &str) -> Result<()> {
if line == "hidden" {
- ct.db.update_node_init(node, |node| {
+ ct.dba.update_node(node, |node| {
node.visibility = node.visibility.min(Visibility::Hidden);
})?;
}
@@ -102,16 +112,16 @@ impl ImportPlugin for General {
}
if let Some(kind) = line.strip_prefix("kind-").or(line.strip_prefix("kind=")) {
let kind = match kind {
- "movie" => NodeKind::Movie,
- "video" => NodeKind::Video,
- "music" => NodeKind::Music,
- "short_form_video" => NodeKind::ShortFormVideo,
- "collection" => NodeKind::Collection,
- "channel" => NodeKind::Channel,
- "show" => NodeKind::Show,
- "series" => NodeKind::Series,
- "season" => NodeKind::Season,
- "episode" => NodeKind::Episode,
+ "movie" => KIND_MOVIE,
+ "video" => KIND_VIDEO,
+ "music" => KIND_MUSIC,
+ "short_form_video" => KIND_SHORTFORMVIDEO,
+ "collection" => KIND_COLLECTION,
+ "channel" => KIND_CHANNEL,
+ "show" => KIND_SHOW,
+ "series" => KIND_SERIES,
+ "season" => KIND_SEASON,
+ "episode" => KIND_EPISODE,
_ => bail!("unknown node kind"),
};
ct.db.update_node_init(node, |node| {
diff --git a/import/src/plugins/mod.rs b/import/src/plugins/mod.rs
index 095fd39..1402cf8 100644
--- a/import/src/plugins/mod.rs
+++ b/import/src/plugins/mod.rs
@@ -15,20 +15,19 @@ pub mod vgmdb;
pub mod wikidata;
pub mod wikimedia_commons;
-use crate::{ApiSecrets, DatabaseTables, InheritedFlags};
+use crate::{ApiSecrets, DatabaseTables, InheritedFlags, NodeID};
use anyhow::Result;
use jellycommon::jellyobject::Object;
-use jellydb::table::{RowNum, Table};
+use jellydb::table::RowNum;
use jellyremuxer::matroska::Segment;
use std::{collections::HashSet, path::Path, sync::Mutex};
use tokio::runtime::Handle;
pub struct ImportContext<'a> {
- pub dba: DatabaseTables,
- pub nodes: &'a Table,
+ pub dba: &'a DatabaseTables,
pub rt: &'a Handle,
pub iflags: InheritedFlags,
- pub pending_nodes: &'a Mutex<HashSet<RowNum>>,
+ pub pending_nodes: &'a Mutex<HashSet<NodeID>>,
}
#[derive(Default, Clone, Copy)]
diff --git a/import/src/plugins/tags.rs b/import/src/plugins/tags.rs
index 07e40cc..2257760 100644
--- a/import/src/plugins/tags.rs
+++ b/import/src/plugins/tags.rs
@@ -4,7 +4,7 @@
Copyright (C) 2026 metamuffin <metamuffin.org>
*/
-use crate::plugins::{ImportContext, ImportPlugin, PluginInfo};
+use crate::{NodeID, plugins::{ImportContext, ImportPlugin, PluginInfo}};
use anyhow::Result;
use jellyremuxer::matroska::Segment;
use std::{collections::HashMap, path::Path};
@@ -31,7 +31,7 @@ impl ImportPlugin for Tags {
})
.unwrap_or_default();
- ct.db.update_node_init(node, |node| {
+ ct.dba.update_node_by_nodeid(node, |node| {
node.title = seg.info.title.clone();
for (key, value) in tags {
match key.as_str() {
diff --git a/import/src/plugins/tmdb.rs b/import/src/plugins/tmdb.rs
index ce9ae59..206781b 100644
--- a/import/src/plugins/tmdb.rs
+++ b/import/src/plugins/tmdb.rs
@@ -4,11 +4,12 @@
Copyright (C) 2026 metamuffin <metamuffin.org>
*/
use crate::{
- USER_AGENT,
+ NodeID, USER_AGENT,
plugins::{ImportContext, ImportPlugin, PluginInfo},
};
use anyhow::{Context, Result, anyhow, bail};
use jellycache::{EscapeKey, HashKey, cache_memory, cache_store};
+use jellycommon::jellyobject::Object;
use log::info;
use reqwest::{
Client, ClientBuilder,
@@ -160,14 +161,14 @@ impl ImportPlugin for Tmdb {
..Default::default()
}
}
- fn process(&self, ct: &ImportContext, node: NodeID, data: &Node) -> Result<()> {
+ fn process(&self, ct: &ImportContext, node: NodeID, data: Object) -> Result<()> {
self.process_primary(ct, node, data)?;
self.process_episode(ct, node, data)?;
Ok(())
}
}
impl Tmdb {
- fn process_primary(&self, ct: &ImportContext, node: NodeID, data: &Node) -> Result<()> {
+ fn process_primary(&self, ct: &ImportContext, node: NodeID, data: Object) -> Result<()> {
let (tmdb_kind, tmdb_id): (_, u64) =
if let Some(id) = data.identifiers.get(&IdentifierType::TmdbSeries) {
(TmdbKind::Tv, id.parse()?)
@@ -199,7 +200,7 @@ impl Tmdb {
.transpose()?
.flatten();
- ct.db.update_node_init(node, |node| {
+ ct.dba.update_node_by_nodeid(node, |node| {
node.title = details.title.clone().or(node.title.clone());
node.tagline = details.tagline.clone().or(node.tagline.clone());
node.description = Some(details.overview.clone());
diff --git a/import/src/plugins/trakt.rs b/import/src/plugins/trakt.rs
index 7981713..5aee881 100644
--- a/import/src/plugins/trakt.rs
+++ b/import/src/plugins/trakt.rs
@@ -9,6 +9,8 @@ use crate::{
};
use anyhow::{Context, Result, anyhow, bail};
use jellycache::{HashKey, cache_memory};
+use jellycommon::jellyobject::{Object, Tag};
+use jellydb::table::RowNum;
use log::info;
use reqwest::{
Client, ClientBuilder,
@@ -248,33 +250,21 @@ pub enum TraktPeopleGroup {
CreatedBy,
}
impl TraktPeopleGroup {
- pub fn as_credit_category(self) -> CreditCategory {
+ pub fn as_credit_category(self) -> Tag {
+ use jellycommon::*;
match self {
- TraktPeopleGroup::Production => CreditCategory::Production,
- TraktPeopleGroup::Art => CreditCategory::Art,
- TraktPeopleGroup::Crew => CreditCategory::Crew,
- TraktPeopleGroup::CostumeMakeup => CreditCategory::CostumeMakeup,
- TraktPeopleGroup::Directing => CreditCategory::Directing,
- TraktPeopleGroup::Writing => CreditCategory::Writing,
- TraktPeopleGroup::Sound => CreditCategory::Sound,
- TraktPeopleGroup::Camera => CreditCategory::Camera,
- TraktPeopleGroup::VisualEffects => CreditCategory::Vfx,
- TraktPeopleGroup::Lighting => CreditCategory::Lighting,
- TraktPeopleGroup::Editing => CreditCategory::Editing,
- TraktPeopleGroup::CreatedBy => CreditCategory::CreatedBy,
- }
- }
-}
-impl TraktAppearance {
- pub fn a(&self) -> Appearance {
- Appearance {
- jobs: self.jobs.to_owned(),
- characters: self.characters.to_owned(),
- node: NodeID([0; 32]), // person: Person {
- // name: self.person.name.to_owned(),
- // headshot: None,
- // ids: self.person.ids.to_owned(),
- // },
+ TraktPeopleGroup::Production => CRCAT_PRODUCTION,
+ TraktPeopleGroup::Art => CRCAT_ART,
+ TraktPeopleGroup::Crew => CRCAT_CREW,
+ TraktPeopleGroup::CostumeMakeup => CRCAT_COSTUME_MAKEUP,
+ TraktPeopleGroup::Directing => CRCAT_DIRECTING,
+ TraktPeopleGroup::Writing => CRCAT_WRITING,
+ TraktPeopleGroup::Sound => CRCAT_SOUND,
+ TraktPeopleGroup::Camera => CRCAT_CAMERA,
+ TraktPeopleGroup::VisualEffects => CRCAT_VFX,
+ TraktPeopleGroup::Lighting => CRCAT_LIGHTING,
+ TraktPeopleGroup::Editing => CRCAT_EDITING,
+ TraktPeopleGroup::CreatedBy => CRCAT_CREATED_BY,
}
}
}
@@ -334,14 +324,15 @@ pub enum TraktKind {
}
impl TraktKind {
- pub fn as_node_kind(self) -> NodeKind {
+ pub fn as_node_kind(self) -> Tag {
+ use jellycommon::*;
match self {
- TraktKind::Movie => NodeKind::Movie,
- TraktKind::Show => NodeKind::Show,
- TraktKind::Season => NodeKind::Season,
- TraktKind::Episode => NodeKind::Episode,
- TraktKind::Person => NodeKind::Channel,
- TraktKind::User => NodeKind::Channel,
+ TraktKind::Movie => KIND_MOVIE,
+ TraktKind::Show => KIND_SHOW,
+ TraktKind::Season => KIND_SEASON,
+ TraktKind::Episode => KIND_EPISODE,
+ TraktKind::Person => KIND_CHANNEL,
+ TraktKind::User => KIND_CHANNEL,
}
}
}
@@ -390,31 +381,33 @@ impl ImportPlugin for Trakt {
..Default::default()
}
}
- fn instruction(&self, ct: &ImportContext, node: NodeID, line: &str) -> Result<()> {
+ fn instruction(&self, ct: &ImportContext, node: RowNum, line: &str) -> Result<()> {
+ use jellycommon::*;
if let Some(value) = line.strip_prefix("trakt-").or(line.strip_prefix("trakt=")) {
let (ty, id) = value.split_once(":").unwrap_or(("movie", value));
let ty = match ty {
- "movie" => IdentifierType::TraktMovie,
- "show" => IdentifierType::TraktShow,
- "season" => IdentifierType::TraktSeason,
- "episode" => IdentifierType::TraktEpisode,
+ "movie" => IDENT_TRAKT_MOVIE,
+ "show" => IDENT_TRAKT_SHOW,
+ "season" => IDENT_TRAKT_SEASON,
+ "episode" => IDENT_TRAKT_EPISODE,
_ => bail!("unknown trakt kind"),
};
- ct.db.update_node_init(node, |node| {
- node.identifiers.insert(ty, id.to_owned());
+ ct.dba.update_node(node, |node| {
+ node.as_object()
+ .update(NO_IDENTIFIERS, |idents| idents.insert(ty, id))
})?;
}
Ok(())
}
- fn process(&self, ct: &ImportContext, node: NodeID, data: &Node) -> Result<()> {
- self.process_primary(ct, node, data)?;
- self.process_episode(ct, node, data)?;
+ fn process(&self, ct: &ImportContext, node: RowNum, data: Object) -> Result<()> {
+ self.process_primary(ct, node.clone(), data)?;
+ self.process_episode(ct, node.clone(), data)?;
Ok(())
}
}
impl Trakt {
- fn process_primary(&self, ct: &ImportContext, node: NodeID, data: &Node) -> Result<()> {
+ fn process_primary(&self, ct: &ImportContext, node: RowNum, data: Object) -> Result<()> {
let (trakt_kind, trakt_id): (_, u64) =
if let Some(id) = data.identifiers.get(&IdentifierType::TraktShow) {
(TraktKind::Show, id.parse()?)
@@ -486,7 +479,7 @@ impl Trakt {
})?;
Ok(())
}
- fn process_episode(&self, ct: &ImportContext, node: NodeID, node_data: &Node) -> Result<()> {
+ fn process_episode(&self, ct: &ImportContext, node: RowNum, node_data: Object) -> Result<()> {
let (Some(episode), Some(season)) = (node_data.index, node_data.season_index) else {
return Ok(());
};
diff --git a/import/src/plugins/vgmdb.rs b/import/src/plugins/vgmdb.rs
index 734c7af..c62eb90 100644
--- a/import/src/plugins/vgmdb.rs
+++ b/import/src/plugins/vgmdb.rs
@@ -62,7 +62,7 @@ impl Vgmdb {
}
}
- pub fn get_artist_image(&self, id: u64, rt: &Handle) -> Result<Option<Asset>> {
+ pub fn get_artist_image(&self, id: u64, rt: &Handle) -> Result<Option<String>> {
if let Some(url) = self.get_artist_image_url(id, rt)? {
cache_store(
format!("ext/vgmdb/artist-image/{}.image", HashKey(&url)),
@@ -82,7 +82,6 @@ impl Vgmdb {
},
)
.context("vgmdb media download")
- .map(Asset)
.map(Some)
} else {
Ok(None)
generated by cgit v1.3 (git 2.53.0) at 2026-03-20 08:18:56 +0000