prototype stream previews

1 files changed, 52 insertions, 13 deletions

diff --git a/client-web/source/resource/track.ts b/client-web/source/resource/track.ts
index 46440ec..2cf01f1 100644
--- a/client-web/source/resource/track.ts
+++ b/client-web/source/resource/track.ts
@@ -10,6 +10,7 @@ import { PO } from "../locale/mod.ts";
 import { log } from "../logger.ts";
 import { on_pref_changed, PREFS } from "../preferences/mod.ts";
 import { get_rnnoise_node } from "../rnnoise.ts";
+import { Room } from "../room.ts";
 import { LocalResource, ResourceHandlerDecl } from "./mod.ts";
 
 export const resource_track: ResourceHandlerDecl = {
@@ -29,6 +30,17 @@ export const resource_track: ResourceHandlerDecl = {
             info,
             el: e("div", { class: [`media-${info.track_kind}`] }, enable_button),
             on_statechange() { },
+            on_preview(preview) {
+                if (this.el.querySelector("audio, video")) return
+                let pi = this.el.querySelector(".preview") as HTMLImageElement
+                if (!pi) {
+                    pi = document.createElement("img")
+                    pi.classList.add("preview")
+                    this.el.prepend(pi)
+                }
+                if (!preview.startsWith("data:")) return
+                pi.src = preview
+            },
             on_enable(stream, disable) {
                 this.el.removeChild(enable_button)
                 if (!(stream instanceof MediaStream)) return console.warn("expected mediastream");
@@ -51,17 +63,42 @@ export const resource_track: ResourceHandlerDecl = {
 
 export function new_local_track(info: ProvideInfo, stream: MediaStream, ...extra_controls: HTMLElement[]): LocalResource {
     let destroy: () => void;
+    let room: Room;
+
+    const el = e("div", { class: `media-${stream.getVideoTracks().length > 0 ? "video" : "audio"}` },
+        e("button", { class: ["abort", "topright"], onclick: () => destroy() }, PO.stop_sharing),
+        ...extra_controls
+    );
+
+    const generate_previews = (video: HTMLVideoElement) => {
+        const canvas = document.createElement("canvas")
+        const context = canvas.getContext("2d")!
+        context.fillStyle = "#ff00ff"
+        setInterval(() => {
+            context.fillRect(0, 0, video.videoWidth, video.videoHeight)
+            const res = PREFS.preview_resolution
+            canvas.width = res
+            canvas.height = res
+            context.drawImage(video, 0, 0, res, res)
+            canvas.toDataURL()
+            canvas.toBlob(blob => {
+                if (!blob) return log({ error: true, scope: "media" }, "Failed to encode stream preview");
+                const reader = new FileReader();
+                reader.addEventListener("load", ev => {
+                    const data_url = ev.target!.result as string;
+                    room.signaling.send_relay({ preview: { id: info.id, data: data_url } })
+                })
+                reader.readAsDataURL(blob)
+
+            }, "image/webp", PREFS.preview_encoding_quality * 0.01)
+        }, 1000 * PREFS.preview_rate)
+    }
+    create_track_display(el, stream, true, generate_previews)
     return {
+        set_room(r) { room = r },
         set_destroy(cb) { destroy = cb },
         info,
-        el: create_track_display(
-            e("div", { class: `media-${stream.getVideoTracks().length > 0 ? "video" : "audio"}` },
-                e("button", { class: ["abort", "topright"], onclick: () => destroy() }, PO.stop_sharing),
-                ...extra_controls
-            ),
-            stream,
-            true
-        ),
+        el,
         destroy() {
             stream.dispatchEvent(new Event("ended"));
             stream.getTracks().forEach(t => t.stop())
@@ -72,7 +109,7 @@ export function new_local_track(info: ProvideInfo, stream: MediaStream, ...extra
     }
 }
 
-function create_track_display(target: HTMLElement, stream: MediaStream, local: boolean): HTMLElement {
+function create_track_display(target: HTMLElement, stream: MediaStream, local: boolean, preview_callback?: (v: HTMLVideoElement) => void): HTMLElement {
     const is_video = stream.getVideoTracks().length > 0
     const is_audio = stream.getAudioTracks().length > 0
 
@@ -88,16 +125,18 @@ function create_track_display(target: HTMLElement, stream: MediaStream, local: b
 
     if (local) media_el.muted = true
 
-    target.querySelectorAll("video, audio").forEach(e => e.remove())
+    target.querySelectorAll("video, audio, .preview").forEach(e => e.remove())
     target.prepend(media_el)
 
     console.log(stream.getTracks());
     const master = stream.getTracks()[0]
     master.addEventListener("ended", () => {
-        if (is_video) media_el.controls = false
-        media_el.classList.add("media-freeze")
+        // if (is_video) media_el.controls = false
+        // media_el.classList.add("media-freeze")
+        media_el.remove()
     })
 
+    if (is_video && PREFS.send_previews && local && preview_callback) preview_callback(media_el as HTMLVideoElement)
     if (is_audio && PREFS.audio_activity_threshold !== undefined) check_volume(stream, vol => {
         const active = vol > PREFS.audio_activity_threshold
         if (active != target.classList.contains("audio-active")) {
@@ -106,7 +145,7 @@ function create_track_display(target: HTMLElement, stream: MediaStream, local: b
         }
     })
 
-    return target
+    return media_el
 }
 
 function check_volume(stream: MediaStream, cb: (vol: number) => void) {