downloader script and status reporting works

3 files changed, 99 insertions, 6 deletions

diff --git a/scripts/enqueue.ts b/scripts/enqueue.ts
index 934f49d..8d19e53 100644
--- a/scripts/enqueue.ts
+++ b/scripts/enqueue.ts
@@ -1,9 +1,8 @@
 
-const file = await Deno.readTextFile(Deno.args[1])
-const note_filter = Deno.args.length >= 3 ? Deno.args[2] : ""
-
 const ws = new WebSocket(Deno.args[0])
-
+const file = await Deno.readTextFile(Deno.args[1])
+const outdir = Deno.args.length >= 3 ? Deno.args[2] : "."
+const note_filter = Deno.args.length >= 4 ? Deno.args[3] : ""
 const requeue = Deno.env.has("REQUEUE")
 
 function run_enqueue() {
@@ -17,7 +16,7 @@ function run_enqueue() {
             const [id, note] = rest.split(";", 2)
             if (note_filter.length && note != note_filter) continue
             const key = `${kind}:${id}`;
-            ws.send(JSON.stringify({ t: "metadata", key, data: { output: name, title: name } }))
+            ws.send(JSON.stringify({ t: "metadata", key, data: { output: outdir + "/" + name, title: name } }))
             ws.send(JSON.stringify({ t: "enqueue", key, ignore_complete: requeue }))
         }
     }
diff --git a/scripts/ytdlp_download.ts b/scripts/ytdlp_download.ts
index e69de29..cc45367 100644
--- a/scripts/ytdlp_download.ts
+++ b/scripts/ytdlp_download.ts
@@ -0,0 +1,92 @@
+
+const ws = new WebSocket(Deno.args[0])
+
+export class TextLineStream extends TransformStream<string, string> {
+    line = "";
+    constructor() {
+        super({
+            transform: (chunk, controller) => {
+                this.line += chunk;
+                while (true) {
+                    const newline = this.line.indexOf("\n");
+                    if (newline === -1) break;
+                    controller.enqueue(this.line.slice(0, newline));
+                    this.line = this.line.slice(newline + 1);
+                }
+            },
+            flush: (controller) => {
+                if (this.line === "") return;
+                controller.enqueue(this.line);
+            },
+        });
+    }
+}
+
+
+function key_to_url(key: string): string {
+    const [kind, id] = key.split(":", 2)
+    if (kind == "youtube") return `https://youtube.com/watch?v=${id}`
+    throw new Error("unknown kind");
+}
+
+async function do_download(key: string, output: string) {
+    const url = key_to_url(key)
+    await Deno.mkdir(output, { recursive: true })
+    const child = new Deno.Command("yt-dlp", {
+        args: [
+            "--quiet",
+            "--progress",
+            "--progress-template", "%(progress)j",
+            "--newline",
+            "--download-archive", "archive",
+            "-f", "bestvideo+bestaudio",
+            "--embed-metadata",
+            "--embed-thumbnail",
+            "--embed-info-json",
+            "--embed-subs",
+            "--embed-chapters",
+            "--ignore-no-formats",
+            "--remux", "mkv",
+            "-o", "%(id)s",
+            url
+        ],
+        stdout: "piped",
+        stderr: "inherit",
+        cwd: output
+    }).spawn()
+
+    const lines = child.stdout.pipeThrough(new TextDecoderStream()).pipeThrough(new TextLineStream())
+    for await (const line of lines) {
+        if (!line.length) continue
+        console.log(JSON.stringify(line));
+        const k = JSON.parse(line)
+        ws.send(JSON.stringify({
+            t: "metadata", key, data: {
+                progress: (k._percent ?? 0) / 100,
+                status: k._default_template?.trim() ?? ""
+            }
+        }))
+    }
+    const status = await child.status
+    if (!status.success) throw new Error("download failed");
+
+    ws.send(JSON.stringify({ t: "complete", key }))
+}
+
+ws.onerror = () => console.error("ws error")
+ws.onclose = () => console.error("ws closed")
+ws.onopen = () => {
+    console.log("ws open");
+    ws.send(JSON.stringify({ t: "register", name: "dummy worker", task_kinds: ["youtube"] }))
+    ws.send(JSON.stringify({ t: "accept" }))
+}
+ws.onmessage = async ev => {
+    if (typeof ev.data != "string") return
+    const p = JSON.parse(ev.data)
+    if (p.t == "error") console.error(`error: ${p.message}`);
+    if (p.t == "work") {
+        if (!p.data.output) throw new Error("no output");
+        await do_download(p.key, p.data.output)
+        ws.send(JSON.stringify({ t: "accept" }))
+    }
+}
diff --git a/scripts/ytdlp_flatten.ts b/scripts/ytdlp_flatten.ts
index 7cd9da1..84bbc8f 100644
--- a/scripts/ytdlp_flatten.ts
+++ b/scripts/ytdlp_flatten.ts
@@ -16,7 +16,9 @@ async function flat_playlist(url: string, kind: string, output: string) {
             "--print-json",
             "--match-filter", "availability=public & live_status=not_live",
             url
-        ]
+        ],
+        stdout: "piped",
+        stderr: "inherit",
     }).output()
     const otext = new TextDecoder().decode(o.stdout)
     for (const line of otext.split("\n")) {