stream previews on requestv1.1.0

3 files changed, 56 insertions, 24 deletions

diff --git a/client-web/source/resource/file.ts b/client-web/source/resource/file.ts
index 7b63d84..0554e2f 100644
--- a/client-web/source/resource/file.ts
+++ b/client-web/source/resource/file.ts
@@ -26,7 +26,6 @@ export const resource_file: ResourceHandlerDecl = {
         }, PO.download)
         return {
             info,
-            on_preview(_) { },
             el: e("div", {},
                 e("span", {}, `${PO.file}: ${JSON.stringify(info.label)} (${display_filesize(info.size!)})`),
                 download_button,
diff --git a/client-web/source/resource/mod.ts b/client-web/source/resource/mod.ts
index 5c76f41..d5f6a7b 100644
--- a/client-web/source/resource/mod.ts
+++ b/client-web/source/resource/mod.ts
@@ -6,7 +6,6 @@
 /// <reference lib="dom" />
 
 import { ProvideInfo } from "../../../common/packets.d.ts"
-import { Room } from "../room.ts";
 import { RemoteUser } from "../user/remote.ts"
 import { resource_file } from "./file.ts";
 import { resource_track } from "./track.ts";
@@ -15,23 +14,23 @@ export type TransportMethod = "data-channel" | "track"
 export type RemoteResourceState = "connected" | "disconnected" | "await_connect" | "await_disconnect"
 export interface ResourceHandlerDecl {
     kind: string
-    new_remote(info: ProvideInfo, user: RemoteUser, enable: () => void): RemoteResource
+    new_remote(info: ProvideInfo, user: RemoteUser, enable: () => void, request_preview: () => void): RemoteResource
 }
 export interface RemoteResource {
     el: HTMLElement
     info: ProvideInfo,
     on_statechange(state: RemoteResourceState): void
+    on_preview_response?: (data: string, expire: number) => void,
     on_enable(t: MediaStream | RTCDataChannel, disable: () => void): void,
-    on_preview(p: string): void,
     stream?: MediaStream
 }
 export interface LocalResource {
     el: HTMLElement
     info: ProvideInfo,
     destroy(): void
+    on_preview_request?: (user: RemoteUser) => Promise<{ data?: string, expire: number }>,
     on_request(user: RemoteUser, create_channel: () => RTCDataChannel): MediaStream | RTCDataChannel,
     set_destroy(cb: () => void): void
-    set_room?: (room: Room) => void
 }
 
 const RESOURCE_HANDLERS: ResourceHandlerDecl[] = [resource_file, resource_track]
@@ -41,6 +40,8 @@ export function new_remote_resource(user: RemoteUser, info: ProvideInfo): Remote
     if (!h) return undefined
     const res = h.new_remote(info, user, () => {
         user.request_resource(res)
+    }, () => {
+        user.request_preview(res)
     })
     return res
 }
diff --git a/client-web/source/resource/track.ts b/client-web/source/resource/track.ts
index 75208d0..56c2b30 100644
--- a/client-web/source/resource/track.ts
+++ b/client-web/source/resource/track.ts
@@ -10,12 +10,16 @@ import { PO } from "../locale/mod.ts";
 import { log } from "../logger.ts";
 import { on_pref_changed, PREFS } from "../preferences/mod.ts";
 import { get_rnnoise_node } from "../rnnoise.ts";
-import { Room } from "../room.ts";
 import { LocalResource, ResourceHandlerDecl } from "./mod.ts";
 
 export const resource_track: ResourceHandlerDecl = {
     kind: "track",
-    new_remote: (info, _user, enable) => {
+    new_remote: (info, user, enable) => {
+        const can_preview = info.track_kind == "video"
+        let preview_enabled = can_preview
+        if (preview_enabled) user.send_to({ preview_request: { id: info.id } })
+
+        let preview_request_timeout: number | undefined;
         const enable_label = PO.enable(`"${info.label ?? info.kind}"`)
         const enable_button = e("button", {
             class: "center",
@@ -30,7 +34,12 @@ export const resource_track: ResourceHandlerDecl = {
             info,
             el: e("div", { class: [`media-${info.track_kind}`] }, enable_button),
             on_statechange() { },
-            on_preview(preview) {
+            on_preview_response(preview, expire) {
+                if (!preview_enabled) return
+                preview_request_timeout = setTimeout(() => {
+                    user.request_preview(this)
+                }, Math.max(expire, 500))
+
                 if (this.el.querySelector("audio, video")) return
                 let pi = this.el.querySelector(".preview") as HTMLImageElement
                 if (!pi) {
@@ -42,12 +51,19 @@ export const resource_track: ResourceHandlerDecl = {
                 pi.src = preview
             },
             on_enable(stream, disable) {
+                preview_enabled = false;
+                clearTimeout(preview_request_timeout);
+
                 this.el.removeChild(enable_button)
                 if (!(stream instanceof MediaStream)) return console.warn("expected mediastream");
                 this.el.append(e("button", {
                     icon: "close",
                     class: ["topleft", "abort"],
                     onclick: (self) => {
+                        if (can_preview) {
+                            preview_enabled = true;
+                            user.request_preview(this)
+                        }
                         disable()
                         this.el.appendChild(enable_button)
                         self.disabled = true
@@ -64,17 +80,18 @@ export const resource_track: ResourceHandlerDecl = {
 
 export function new_local_track(info: ProvideInfo, stream: MediaStream, ...extra_controls: HTMLElement[]): LocalResource {
     let destroy: () => void;
-    let room: Room;
-
     const el = e("div", { class: `media-${stream.getVideoTracks().length > 0 ? "video" : "audio"}` },
         e("button", { icon: "stop", class: ["abort", "topleft"], onclick: () => destroy() }, PO.stop_sharing),
         ...extra_controls
     );
 
+    let preview: string | undefined;
+    let preview_on_ready: (() => void)[] | undefined = []
+    let last_preview_ts: number = Date.now();
     const generate_previews = (video: HTMLVideoElement) => {
         const canvas = document.createElement("canvas")
         const context = canvas.getContext("2d")!
-        setInterval(() => {
+        const update_preview = () => {
             const res = PREFS.preview_resolution
             canvas.width = res
             canvas.height = res * video.videoHeight / video.videoWidth
@@ -86,18 +103,31 @@ export function new_local_track(info: ProvideInfo, stream: MediaStream, ...extra
                 if (!blob) return log({ error: true, scope: "media" }, "Failed to encode stream preview");
                 const reader = new FileReader();
                 reader.addEventListener("load", ev => {
-                    const data_url = ev.target!.result as string;
-                    room.signaling.send_relay({ preview: { id: info.id, data: data_url } })
+                    preview = ev.target!.result as string;
+                    preview_on_ready?.forEach(f => f())
+                    preview_on_ready = undefined
+                    last_preview_ts = Date.now()
                 })
                 reader.readAsDataURL(blob)
 
             }, "image/webp", PREFS.preview_encoding_quality * 0.01)
-        }, 1000 * PREFS.preview_rate)
+        };
+        setTimeout(update_preview, 1000)
+        const interval = setInterval(update_preview, 1000 * PREFS.preview_rate)
+        return () => clearInterval(interval)
     }
     create_track_display(el, stream, true, generate_previews)
     return {
-        set_room(r) { room = r },
         set_destroy(cb) { destroy = cb },
+        async on_preview_request(_user) {
+            await new Promise<void>(done => {
+                if (preview_on_ready) {
+                    log("media", "preview delayed")
+                    preview_on_ready.push(done)
+                } else done()
+            })
+            return { data: preview, expire: (1000 * PREFS.preview_rate) - (Date.now() - last_preview_ts) + 500 };
+        },
         info,
         el,
         destroy() {
@@ -110,7 +140,7 @@ export function new_local_track(info: ProvideInfo, stream: MediaStream, ...extra
     }
 }
 
-function create_track_display(target: HTMLElement, stream: MediaStream, local: boolean, preview_callback?: (v: HTMLVideoElement) => void): HTMLElement {
+function create_track_display(target: HTMLElement, stream: MediaStream, local: boolean, preview_callback?: (v: HTMLVideoElement) => (() => void)): HTMLElement {
     const is_video = stream.getVideoTracks().length > 0
     const is_audio = stream.getAudioTracks().length > 0
 
@@ -137,15 +167,17 @@ function create_track_display(target: HTMLElement, stream: MediaStream, local: b
             else target.classList.remove("audio-active")
         }
     })
-    let fullscreen
+    let fullscreen: HTMLButtonElement | undefined;
     if (is_video) {
-        fullscreen = e("button", { icon: "fullscreen", class: ["topright", "fullscreen"],
-        onclick() { 
-            if (document.fullscreenElement && document.fullscreenElement !== null) 
-                document.exitFullscreen()
-            else
-                media_el.requestFullscreen()
-        }}, PO.fullscreen)
+        fullscreen = e("button", {
+            icon: "fullscreen", class: ["topright", "fullscreen"],
+            onclick() {
+                if (document.fullscreenElement && document.fullscreenElement !== null)
+                    document.exitFullscreen()
+                else
+                    media_el.requestFullscreen()
+            }
+        }, PO.fullscreen)
         target.prepend(fullscreen)
     }