diff options
| -rw-r--r-- | common/src/impl.rs | 6 | ||||
| -rw-r--r-- | import/src/acoustid.rs | 2 | ||||
| -rw-r--r-- | import/src/lib.rs | 30 | ||||
| -rw-r--r-- | import/src/musicbrainz.rs | 15 | ||||
| -rw-r--r-- | import/src/wikidata.rs | 120 | ||||
| -rw-r--r-- | import/src/wikimedia_commons.rs | 29 |
6 files changed, 201 insertions, 1 deletions
diff --git a/common/src/impl.rs b/common/src/impl.rs index 3fa8d80..5b35be3 100644 --- a/common/src/impl.rs +++ b/common/src/impl.rs @@ -153,6 +153,12 @@ impl FromStr for PeopleGroup { "Visual Effects" => PeopleGroup::Vfx, "Costume & Makeup" => PeopleGroup::CostumeMakeup, "Created by:" => PeopleGroup::CreatedBy, + "Performance" => PeopleGroup::Performance, + "Instrument" => PeopleGroup::Instrument, + "Vocal" => PeopleGroup::Vocal, + "Arranger" => PeopleGroup::Arranger, + "Producer" => PeopleGroup::Producer, + "Engineer" => PeopleGroup::Engineer, _ => return Err(()), }) } diff --git a/import/src/acoustid.rs b/import/src/acoustid.rs index e53bf13..8d33821 100644 --- a/import/src/acoustid.rs +++ b/import/src/acoustid.rs @@ -7,6 +7,7 @@ use crate::USER_AGENT; use anyhow::Result; use bincode::{Decode, Encode}; use jellybase::cache::async_cache_memory; +use log::info; use reqwest::{ header::{HeaderMap, HeaderName, HeaderValue}, Client, ClientBuilder, @@ -92,6 +93,7 @@ impl AcoustID { async_cache_memory("api-acoustid", fp.clone(), || async move { let _permit = self.rate_limit.clone().acquire_owned().await?; let permit_drop_ts = Instant::now() + Duration::SECOND; + info!("acoustid lookup"); let duration = fp.duration; let fingerprint = fp.fingerprint.replace("=", "%3D"); diff --git a/import/src/lib.rs b/import/src/lib.rs index f033171..2c1676c 100644 --- a/import/src/lib.rs +++ b/import/src/lib.rs @@ -14,7 +14,8 @@ use jellybase::{ CONF, SECRETS, }; use jellyclient::{ - Appearance, LocalTrack, PeopleGroup, TmdbKind, TrackSource, TraktKind, Visibility, + Appearance, LocalTrack, ObjectIds, PeopleGroup, Person, TmdbKind, TrackSource, TraktKind, + Visibility, }; use jellyremuxer::metadata::checked_matroska_metadata; use log::info; @@ -42,6 +43,8 @@ pub mod infojson; pub mod musicbrainz; pub mod tmdb; pub mod trakt; +pub mod wikidata; +pub mod wikimedia_commons; pub const USER_AGENT: &'static str = concat!( "jellything/", @@ -689,6 +692,31 @@ fn apply_musicbrainz_recording( .insert("musicbrainz.artist".to_string(), a.artist.id.to_string()); } + for rel in &rec.relations { + use musicbrainz::reltypes::*; + match rel.type_id.as_str() { + INSTRUMENT => { + node.people + .entry(PeopleGroup::Instrument) + .or_default() + .push(Appearance { + jobs: rel.attributes.clone(), + characters: vec![], + person: Person { + name: rel + .artist + .as_ref() + .map(|a| a.name.clone()) + .unwrap_or_default(), + headshot: None, + ids: ObjectIds::default(), + }, + }); + } + _ => (), + } + } + for isrc in &rec.isrcs { node.external_ids .insert("isrc".to_string(), isrc.to_string()); diff --git a/import/src/musicbrainz.rs b/import/src/musicbrainz.rs index 2d3d532..51573db 100644 --- a/import/src/musicbrainz.rs +++ b/import/src/musicbrainz.rs @@ -8,6 +8,7 @@ use crate::USER_AGENT; use anyhow::Result; use bincode::{Decode, Encode}; use jellybase::cache::async_cache_memory; +use log::info; use reqwest::{ header::{HeaderMap, HeaderName, HeaderValue}, Client, ClientBuilder, @@ -19,6 +20,19 @@ use tokio::{ time::{sleep_until, Instant}, }; +pub mod reltypes { + pub const MUSIC_VIDEO: &str = "ce3de655-7451-44d1-9224-87eb948c205d"; + pub const INSTRUMENTAL: &str = "9fc01a58-7801-4bd2-b07d-61cc7ffacf90"; + pub const VOCAL: &str = "0fdbe3c6-7700-4a31-ae54-b53f06ae1cfa"; + pub const RECORDING: &str = "a01ee869-80a8-45ef-9447-c59e91aa7926"; + pub const PROGRAMMING: &str = "36c50022-44e0-488d-994b-33f11d20301e"; + pub const PRODUCER: &str = "5c0ceac3-feb4-41f0-868d-dc06f6e27fc0"; + pub const ARTIST: &str = "5c0ceac3-feb4-41f0-868d-dc06f6e27fc0"; + pub const PHONOGRAPHIC_COPYRIGHT: &str = "7fd5fbc0-fbf4-4d04-be23-417d50a4dc30"; + pub const MIX: &str = "3e3102e1-1896-4f50-b5b2-dd9824e46efe"; + pub const INSTRUMENT: &str = "59054b12-01ac-43ee-a618-285fd397e461"; +} + pub struct MusicBrainz { client: Client, rate_limit: Arc<Semaphore>, @@ -150,6 +164,7 @@ impl MusicBrainz { async_cache_memory("api-musicbrainz-recording", id.clone(), || async move { let _permit = self.rate_limit.clone().acquire_owned().await?; let permit_drop_ts = Instant::now() + Duration::from_secs(10); + info!("recording lookup: {id}"); let inc = [ "isrcs", diff --git a/import/src/wikidata.rs b/import/src/wikidata.rs new file mode 100644 index 0000000..267899f --- /dev/null +++ b/import/src/wikidata.rs @@ -0,0 +1,120 @@ +/* + This file is part of jellything (https://codeberg.org/metamuffin/jellything) + which is licensed under the GNU Affero General Public License (version 3); see /COPYING. + Copyright (C) 2025 metamuffin <metamuffin.org> +*/ + +use crate::USER_AGENT; +use anyhow::{bail, Result}; +use jellybase::cache::async_cache_memory; +use log::info; +use reqwest::{ + header::{HeaderMap, HeaderName, HeaderValue}, + Client, ClientBuilder, +}; +use serde::Deserialize; +use serde_json::Value; +use std::collections::BTreeMap; + +pub struct Wikidata { + client: Client, +} + +#[derive(Debug, Deserialize, Clone)] +pub struct WikidataResponse { + entities: BTreeMap<String, WikidataEntity>, +} + +#[derive(Debug, Deserialize, Clone)] +pub struct WikidataEntity { + pub pageid: u64, + pub ns: u64, + pub title: String, + pub lastrevid: u64, + pub modified: String, + pub r#type: String, + pub id: String, + pub claims: BTreeMap<String, Vec<WikidataClaim>>, +} +#[derive(Debug, Deserialize, Clone)] +pub struct WikidataClaim { + pub r#type: String, + pub id: String, + pub rank: String, + pub mainsnak: WikidataSnak, +} + +#[derive(Debug, Deserialize, Clone)] +pub struct WikidataSnak { + pub snaktype: String, + pub property: String, + pub hash: String, + pub datavalue: WikidataValue, + pub datatype: String, +} + +#[derive(Debug, Deserialize, Clone)] +pub struct WikidataValue { + pub value: Value, + pub r#type: String, +} + +pub mod properties { + pub static IMAGE: &str = "P18"; +} + +impl Wikidata { + pub fn new() -> Self { + let client = ClientBuilder::new() + .default_headers(HeaderMap::from_iter([ + ( + HeaderName::from_static("accept"), + HeaderValue::from_static("application/json"), + ), + ( + HeaderName::from_static("user-agent"), + HeaderValue::from_static(USER_AGENT), + ), + ])) + .build() + .unwrap(); + Self { client } + } + + pub async fn query_image_path(&self, id: String) -> Result<Option<String>> { + let response = self.query(id.clone()).await?; + if let Some(entity) = response.entities.get(&id) { + if let Some(images) = entity.claims.get(properties::IMAGE) { + for image in images { + if image.mainsnak.datatype != "commonsMedia" { + bail!("image is of type {:?}", image.mainsnak.datatype); + } + if let Value::String(filename) = &image.mainsnak.datavalue.value { + return Ok(Some(filename.to_owned())); + } + } + } + } + Ok(None) + } + + pub async fn query(&self, id: String) -> Result<WikidataResponse> { + let json = async_cache_memory("api-wikidata", id.clone(), || async move { + info!("entity query: {id}"); + + let resp = self + .client + .get(format!("https://www.wikidata.org/entity/{id}")) + .send() + .await? + .error_for_status()? + .text() + .await?; + + Ok(resp) + }) + .await?; + + Ok(serde_json::from_str(&json)?) + } +} diff --git a/import/src/wikimedia_commons.rs b/import/src/wikimedia_commons.rs new file mode 100644 index 0000000..081aa5a --- /dev/null +++ b/import/src/wikimedia_commons.rs @@ -0,0 +1,29 @@ +/* + This file is part of jellything (https://codeberg.org/metamuffin/jellything) + which is licensed under the GNU Affero General Public License (version 3); see /COPYING. + Copyright (C) 2025 metamuffin <metamuffin.org> +*/ + +use crate::USER_AGENT; +use reqwest::{ + header::{HeaderMap, HeaderName, HeaderValue}, + redirect::Policy, + Client, ClientBuilder, +}; + +struct WikimediaCommons { + client: Client, +} +impl WikimediaCommons { + pub fn new() -> Self { + let client = ClientBuilder::new() + .default_headers(HeaderMap::from_iter([( + HeaderName::from_static("user-agent"), + HeaderValue::from_static(USER_AGENT), + )])) + .redirect(Policy::limited(5)) + .build() + .unwrap(); + Self { client } + } +} |