diff options
| -rw-r--r-- | scripts/cli.ts | 13 | ||||
| -rw-r--r-- | scripts/youtube_atom_compare.md | 6 | ||||
| -rw-r--r-- | scripts/youtube_atom_compare.ts | 61 | ||||
| -rw-r--r-- | scripts/youtube_atom_flatten.ts (renamed from scripts/youtube_xml_feed.ts) | 4 |
4 files changed, 79 insertions, 5 deletions
diff --git a/scripts/cli.ts b/scripts/cli.ts index bbb9b4e..bf3e44a 100644 --- a/scripts/cli.ts +++ b/scripts/cli.ts @@ -1,14 +1,21 @@ - const ws = new WebSocket(Deno.args[0]) - -function do_stuff() { +async function do_stuff() { switch (Deno.args[1]) { case "enqueue": if (Deno.args.length >= 4) ws.send(JSON.stringify({ t: "metadata", key: Deno.args[2], data: JSON.parse(Deno.args[3]) })) ws.send(JSON.stringify({ t: "enqueue", key: Deno.args[2], ignore_complete: true })) break; + // deno-lint-ignore no-case-declarations + case "enqueue_batch": + const a = await new Response(Deno.stdin.readable).text() + for (let key of a.split("\n")) { + key = key.trim() + if (!key) continue + ws.send(JSON.stringify({ t: "enqueue", key, ignore_complete: true })) + } + break; case "retry_all": ws.send(JSON.stringify({ t: "query", diff --git a/scripts/youtube_atom_compare.md b/scripts/youtube_atom_compare.md new file mode 100644 index 0000000..d898dd9 --- /dev/null +++ b/scripts/youtube_atom_compare.md @@ -0,0 +1,6 @@ +# Youtube Atom feed compare + +Processes task with key `youtube-channel-atom-compare:<channel id>`, checks the +respective youtube channel's Atom feed and enqueues the corresponding the +`youtube-channel:<channel id>` task if the atom feed has new entries. All +metadata is copied in that process. diff --git a/scripts/youtube_atom_compare.ts b/scripts/youtube_atom_compare.ts new file mode 100644 index 0000000..353be14 --- /dev/null +++ b/scripts/youtube_atom_compare.ts @@ -0,0 +1,61 @@ +import { Config } from "./config.ts"; + +const ws = new WebSocket(Deno.args[0]) +// deno-lint-ignore no-unused-vars +let config: Config = {} as unknown as Config + +async function flat_feed(cid: string, data: { [key: string]: unknown }) { + const feedurl = `https://www.youtube.com/feeds/videos.xml?channel_id=${cid}` + console.log(`load feed ${feedurl}`); + const res = await fetch(feedurl) + if (!res.ok) throw new Error("feed download error:" + await res.text()); + const feed = await res.text() + + let last_seen_ids = "" + for (const entry of feed.split("<entry>")) { + const id_match = entry.match(/\<yt:videoId\>([0-9A-Za-z-_]{11})\<\/yt:videoId\>/) + if (!id_match) continue + last_seen_ids += id_match[1] + "|" + } + + if (data.last_seen_ids && data.last_seen_ids == last_seen_ids) { + console.log("feed identical"); + return + } else { + console.log("feed changed"); + ws.send(JSON.stringify({ + t: "metadata", + key: `youtube-channel-atom-compare:${cid}`, + data: { last_seen_ids } + })) + ws.send(JSON.stringify({ + t: "metadata", + key: `youtube-channel:${cid}`, + data: { ...data, last_seen_ids: undefined } + })) + ws.send(JSON.stringify({ t: "enqueue", key: `youtube-channel:${cid}`, ignore_complete: true })) + } + console.log("done"); +} + +ws.onerror = () => console.error("ws error") +ws.onclose = () => { + console.error("ws closed") + Deno.exit(1) +} +ws.onopen = () => { + console.log("ws open"); + ws.send(JSON.stringify({ t: "register", name: "YouTube Atom feed checker", task_kinds: ["youtube-channel-atom-compare"] })) + ws.send(JSON.stringify({ t: "accept" })) +} +ws.onmessage = async ev => { + if (typeof ev.data != "string") return + const p = JSON.parse(ev.data) + if (p.t == "config") config = p.config + if (p.t == "error") console.error(`error: ${p.message}`); + if (p.t == "work") { + await flat_feed(p.key.replace("youtube-channel-atom-compare:", ""), p.data) + ws.send(JSON.stringify({ t: "complete", key: p.key })) + ws.send(JSON.stringify({ t: "accept" })) + } +} diff --git a/scripts/youtube_xml_feed.ts b/scripts/youtube_atom_flatten.ts index 5d21542..7bddced 100644 --- a/scripts/youtube_xml_feed.ts +++ b/scripts/youtube_atom_flatten.ts @@ -41,7 +41,7 @@ ws.onclose = () => { } ws.onopen = () => { console.log("ws open"); - ws.send(JSON.stringify({ t: "register", name: "YouTube XML feed flattener", task_kinds: ["youtube-channel-xmlfeed"] })) + ws.send(JSON.stringify({ t: "register", name: "YouTube XML feed flattener", task_kinds: ["youtube-channel-atom"] })) ws.send(JSON.stringify({ t: "accept" })) } ws.onmessage = async ev => { @@ -50,7 +50,7 @@ ws.onmessage = async ev => { if (p.t == "config") config = p.config if (p.t == "error") console.error(`error: ${p.message}`); if (p.t == "work") { - await flat_feed(p.key.replace("youtube-channel-xmlfeed:", ""), p.data) + await flat_feed(p.key.replace("youtube-channel-atom:", ""), p.data) ws.send(JSON.stringify({ t: "complete", key: p.key })) ws.send(JSON.stringify({ t: "save" })) ws.send(JSON.stringify({ t: "accept" })) |