diff options
| -rw-r--r-- | base/src/database.rs | 8 | ||||
| -rw-r--r-- | import/src/infojson.rs | 28 | ||||
| -rw-r--r-- | import/src/lib.rs | 53 | ||||
| -rw-r--r-- | server/src/routes/ui/admin/mod.rs | 38 |
4 files changed, 78 insertions, 49 deletions
diff --git a/base/src/database.rs b/base/src/database.rs index a213a40..e9fe156 100644 --- a/base/src/database.rs +++ b/base/src/database.rs @@ -75,6 +75,14 @@ impl Database { Ok(None) } } + pub fn clear_nodes(&self) -> Result<()> { + let txn = self.inner.begin_write()?; + let mut table = txn.open_table(T_NODE)?; + table.retain(|_, _| false)?; + drop(table); + txn.commit()?; + Ok(()) + } pub fn get_node_udata(&self, id: NodeID, username: &str) -> Result<Option<NodeUserData>> { let txn = self.inner.begin_read()?; let t_node = txn.open_table(T_USER_NODE)?; diff --git a/import/src/infojson.rs b/import/src/infojson.rs index f4c028b..c2ae305 100644 --- a/import/src/infojson.rs +++ b/import/src/infojson.rs @@ -13,23 +13,23 @@ use std::collections::HashMap; pub struct YVideo { pub id: String, pub title: String, - pub formats: Vec<YFormat>, + pub formats: Option<Vec<YFormat>>, pub thumbnails: Vec<YThumbnail>, - pub thumbnail: String, + pub thumbnail: Option<String>, pub description: String, pub channel_id: String, pub duration: Option<f64>, - pub view_count: usize, + pub view_count: Option<usize>, pub average_rating: Option<String>, - pub age_limit: usize, + pub age_limit: Option<usize>, pub webpage_url: String, - pub categories: Vec<String>, + pub categories: Option<Vec<String>>, pub tags: Vec<String>, - pub playable_in_embed: bool, + pub playable_in_embed: Option<bool>, pub aspect_ratio: Option<f32>, pub width: Option<i32>, pub height: Option<i32>, - pub automatic_captions: HashMap<String, Vec<YCaption>>, + pub automatic_captions: Option<HashMap<String, Vec<YCaption>>>, pub comment_count: Option<usize>, pub chapters: Option<Vec<YChapter>>, pub heatmap: Option<Vec<YHeatmapSample>>, @@ -40,7 +40,7 @@ pub struct YVideo { pub uploader: Option<String>, pub uploader_id: Option<String>, pub uploader_url: Option<String>, - pub upload_date: String, + pub upload_date: Option<String>, pub availability: Option<String>, // "public" | "private" | "unlisted", pub original_url: Option<String>, pub webpage_url_basename: String, @@ -55,11 +55,11 @@ pub struct YVideo { pub playlist_uploader_id: Option<String>, pub n_entries: Option<usize>, pub playlist_index: Option<usize>, - pub display_id: String, - pub fulltitle: String, - pub duration_string: String, - pub is_live: bool, - pub was_live: bool, + pub display_id: Option<String>, + pub fulltitle: Option<String>, + pub duration_string: Option<String>, + pub is_live: Option<bool>, + pub was_live: Option<bool>, pub epoch: usize, } @@ -105,7 +105,7 @@ pub struct YFragment { #[derive(Debug, Serialize, Deserialize)] pub struct YThumbnail { pub url: String, - pub preference: i32, + pub preference: Option<i32>, pub id: String, pub height: Option<u32>, pub width: Option<u32>, diff --git a/import/src/lib.rs b/import/src/lib.rs index 243ac8a..add7e4d 100644 --- a/import/src/lib.rs +++ b/import/src/lib.rs @@ -9,27 +9,22 @@ use ebml_struct::{ matroska::*, read::{EbmlReadExt, TagRead}, }; +use infojson::YVideo; use jellybase::{assetfed::AssetInner, cache::cache_file, database::Database, CONF, SECRETS}; use jellycommon::{ Chapter, LocalTrack, MediaInfo, Node, NodeID, NodeKind, Rating, SourceTrack, SourceTrackKind, TrackSource, }; -use log::{info, warn}; -use rayon::iter::{ - IntoParallelIterator, IntoParallelRefIterator, ParallelBridge, ParallelDrainRange, - ParallelIterator, -}; +use log::info; +use rayon::iter::{ParallelDrainRange, ParallelIterator}; use regex::Regex; use std::{ - collections::{HashMap, VecDeque}, + collections::HashMap, fs::File, io::{BufReader, ErrorKind, Read, Write}, mem::swap, path::{Path, PathBuf}, - sync::{ - atomic::{AtomicUsize, Ordering}, - LazyLock, - }, + sync::LazyLock, time::UNIX_EPOCH, }; use tmdb::Tmdb; @@ -126,7 +121,6 @@ fn import_iter_inner(path: &Path, db: &Database, incremental: bool) -> Result<Ve } fn import_file(db: &Database, path: &Path) -> Result<()> { - let filename = path.file_stem().unwrap().to_string_lossy(); let parent = NodeID::from_slug( &path .parent() @@ -135,20 +129,22 @@ fn import_file(db: &Database, path: &Path) -> Result<()> { .ok_or(anyhow!("parent no filename"))? .to_string_lossy(), ); + + let filename = path.file_name().unwrap().to_string_lossy(); match filename.as_ref() { - "poster" => { + "poster.jpeg" | "poster.webp" => { db.update_node_init(parent, |node| { node.poster = Some(AssetInner::Media(path.to_owned()).ser()); Ok(()) })?; } - "backdrop" => { + "backdrop.jpeg" | "backdrop.webp" => { db.update_node_init(parent, |node| { node.backdrop = Some(AssetInner::Media(path.to_owned()).ser()); Ok(()) })?; } - "info" => { + "info.json" | "info.yaml" => { let data = serde_yaml::from_reader::<_, Node>(BufReader::new(File::open(path)?))?; db.update_node_init(parent, |node| { fn merge_option<T>(a: &mut Option<T>, b: Option<T>) { @@ -162,6 +158,23 @@ fn import_file(db: &Database, path: &Path) -> Result<()> { Ok(()) })?; } + "channel.info.json" => { + let data = serde_json::from_reader::<_, YVideo>(BufReader::new(File::open(path)?))?; + db.update_node_init(parent, |node| { + node.title = Some( + data.title + .strip_suffix(" - Videos") + .unwrap_or(&data.title) + .to_owned(), + ); + node.description = Some(data.description); + if let Some(followers) = data.channel_follower_count { + node.ratings + .insert(Rating::YoutubeFollowers, followers as f64); + } + Ok(()) + })?; + } _ => (), } @@ -288,12 +301,14 @@ fn import_media_file(db: &Database, path: &Path, parent: NodeID) -> Result<()> { node.title = Some(infojson.title); node.description = Some(infojson.description); node.tagline = Some(infojson.webpage_url); - node.release_date = Some( - infojson::parse_upload_date(&infojson.upload_date) - .context("parsing upload date")?, + if let Some(date) = &infojson.upload_date { + node.release_date = + Some(infojson::parse_upload_date(date).context("parsing upload date")?); + } + node.ratings.insert( + Rating::YoutubeViews, + infojson.view_count.unwrap_or_default() as f64, ); - node.ratings - .insert(Rating::YoutubeViews, infojson.view_count as f64); if let Some(lc) = infojson.like_count { node.ratings.insert(Rating::YoutubeLikes, lc as f64); } diff --git a/server/src/routes/ui/admin/mod.rs b/server/src/routes/ui/admin/mod.rs index 463319a..5c2c48f 100644 --- a/server/src/routes/ui/admin/mod.rs +++ b/server/src/routes/ui/admin/mod.rs @@ -51,10 +51,12 @@ pub async fn admin_dashboard<'a>( @FlashDisplay { flash: flash.clone() } @if !last_import_err.is_empty() { section.message.error { - p.error {"The last import resulted in at least one error:"} - ol { @for e in &last_import_err { - li.error { pre.error { @e } } - }} + details { + summary { p.error { @format!("The last import resulted in {} errors:", last_import_err.len()) } } + ol { @for e in &last_import_err { + li.error { pre.error { @e } } + }} + } } } ul { @@ -68,12 +70,15 @@ pub async fn admin_dashboard<'a>( @if is_transcoding() { section.message { p.warn { "Currently transcoding posters." } } } - form[method="POST", action=uri!(r_admin_import(true))] { + form[method="POST", action=uri!(r_admin_import(true, false))] { input[type="submit", disabled=is_importing(), value="Start incremental import"]; } - form[method="POST", action=uri!(r_admin_import(false))] { + form[method="POST", action=uri!(r_admin_import(false, false))] { input[type="submit", disabled=is_importing(), value="Start full import"]; } + form[method="POST", action=uri!(r_admin_import(false, true))] { + input[type="submit", disabled=is_importing(), value="Clear all nodes"]; + } form[method="POST", action=uri!(r_admin_transcode_posters())] { input[type="submit", disabled=is_transcoding(), value="Transcode all posters with low resolution"]; } @@ -134,24 +139,25 @@ pub async fn r_admin_remove_invite( admin_dashboard(database, Some(Ok("Invite invalidated".into()))).await } -#[post("/admin/import?<incremental>")] +#[post("/admin/import?<incremental>&<nuke>")] pub async fn r_admin_import( session: AdminSession, database: &State<Database>, _federation: &State<Federation>, incremental: bool, + nuke: bool, ) -> MyResult<DynLayoutPage<'static>> { drop(session); let t = Instant::now(); - let r = import_wrap((*database).clone(), incremental).await; - admin_dashboard( - database, - Some( - r.map_err(|e| e.into()) - .map(|_| format!("Import successful; took {:?}", t.elapsed())), - ), - ) - .await + let flash = if nuke { + database.clear_nodes()?; + Ok(format!("All nodes cleared.")) + } else { + let r = import_wrap((*database).clone(), incremental).await; + r.map_err(|e| e.into()) + .map(|_| format!("Import successful; took {:?}", t.elapsed())) + }; + admin_dashboard(database, Some(flash)).await } #[post("/admin/delete_cache")] |