stream previews on requestv1.1.0

7 files changed, 76 insertions, 28 deletions

diff --git a/client-web/source/protocol/mod.ts b/client-web/source/protocol/mod.ts
index 805600d..d554820 100644
--- a/client-web/source/protocol/mod.ts
+++ b/client-web/source/protocol/mod.ts
@@ -87,6 +87,7 @@ export class SignalingConnection {
         this.websocket.send(JSON.stringify(data))
     }
     async send_relay(data: RelayMessage, recipient?: number | null) {
+        log("ws", "->", data, recipient ? recipient : " ")
         recipient ??= undefined // null -> undefined
         const packet: RelayMessageWrapper = { inner: data, sender: this.my_id! }
         const message = await encrypt(this.key!, JSON.stringify(packet))
diff --git a/client-web/source/resource/file.ts b/client-web/source/resource/file.ts
index 7b63d84..0554e2f 100644
--- a/client-web/source/resource/file.ts
+++ b/client-web/source/resource/file.ts
@@ -26,7 +26,6 @@ export const resource_file: ResourceHandlerDecl = {
         }, PO.download)
         return {
             info,
-            on_preview(_) { },
             el: e("div", {},
                 e("span", {}, `${PO.file}: ${JSON.stringify(info.label)} (${display_filesize(info.size!)})`),
                 download_button,
diff --git a/client-web/source/resource/mod.ts b/client-web/source/resource/mod.ts
index 5c76f41..d5f6a7b 100644
--- a/client-web/source/resource/mod.ts
+++ b/client-web/source/resource/mod.ts
@@ -6,7 +6,6 @@
 /// <reference lib="dom" />
 
 import { ProvideInfo } from "../../../common/packets.d.ts"
-import { Room } from "../room.ts";
 import { RemoteUser } from "../user/remote.ts"
 import { resource_file } from "./file.ts";
 import { resource_track } from "./track.ts";
@@ -15,23 +14,23 @@ export type TransportMethod = "data-channel" | "track"
 export type RemoteResourceState = "connected" | "disconnected" | "await_connect" | "await_disconnect"
 export interface ResourceHandlerDecl {
     kind: string
-    new_remote(info: ProvideInfo, user: RemoteUser, enable: () => void): RemoteResource
+    new_remote(info: ProvideInfo, user: RemoteUser, enable: () => void, request_preview: () => void): RemoteResource
 }
 export interface RemoteResource {
     el: HTMLElement
     info: ProvideInfo,
     on_statechange(state: RemoteResourceState): void
+    on_preview_response?: (data: string, expire: number) => void,
     on_enable(t: MediaStream | RTCDataChannel, disable: () => void): void,
-    on_preview(p: string): void,
     stream?: MediaStream
 }
 export interface LocalResource {
     el: HTMLElement
     info: ProvideInfo,
     destroy(): void
+    on_preview_request?: (user: RemoteUser) => Promise<{ data?: string, expire: number }>,
     on_request(user: RemoteUser, create_channel: () => RTCDataChannel): MediaStream | RTCDataChannel,
     set_destroy(cb: () => void): void
-    set_room?: (room: Room) => void
 }
 
 const RESOURCE_HANDLERS: ResourceHandlerDecl[] = [resource_file, resource_track]
@@ -41,6 +40,8 @@ export function new_remote_resource(user: RemoteUser, info: ProvideInfo): Remote
     if (!h) return undefined
     const res = h.new_remote(info, user, () => {
         user.request_resource(res)
+    }, () => {
+        user.request_preview(res)
     })
     return res
 }
diff --git a/client-web/source/resource/track.ts b/client-web/source/resource/track.ts
index 75208d0..56c2b30 100644
--- a/client-web/source/resource/track.ts
+++ b/client-web/source/resource/track.ts
@@ -10,12 +10,16 @@ import { PO } from "../locale/mod.ts";
 import { log } from "../logger.ts";
 import { on_pref_changed, PREFS } from "../preferences/mod.ts";
 import { get_rnnoise_node } from "../rnnoise.ts";
-import { Room } from "../room.ts";
 import { LocalResource, ResourceHandlerDecl } from "./mod.ts";
 
 export const resource_track: ResourceHandlerDecl = {
     kind: "track",
-    new_remote: (info, _user, enable) => {
+    new_remote: (info, user, enable) => {
+        const can_preview = info.track_kind == "video"
+        let preview_enabled = can_preview
+        if (preview_enabled) user.send_to({ preview_request: { id: info.id } })
+
+        let preview_request_timeout: number | undefined;
         const enable_label = PO.enable(`"${info.label ?? info.kind}"`)
         const enable_button = e("button", {
             class: "center",
@@ -30,7 +34,12 @@ export const resource_track: ResourceHandlerDecl = {
             info,
             el: e("div", { class: [`media-${info.track_kind}`] }, enable_button),
             on_statechange() { },
-            on_preview(preview) {
+            on_preview_response(preview, expire) {
+                if (!preview_enabled) return
+                preview_request_timeout = setTimeout(() => {
+                    user.request_preview(this)
+                }, Math.max(expire, 500))
+
                 if (this.el.querySelector("audio, video")) return
                 let pi = this.el.querySelector(".preview") as HTMLImageElement
                 if (!pi) {
@@ -42,12 +51,19 @@ export const resource_track: ResourceHandlerDecl = {
                 pi.src = preview
             },
             on_enable(stream, disable) {
+                preview_enabled = false;
+                clearTimeout(preview_request_timeout);
+
                 this.el.removeChild(enable_button)
                 if (!(stream instanceof MediaStream)) return console.warn("expected mediastream");
                 this.el.append(e("button", {
                     icon: "close",
                     class: ["topleft", "abort"],
                     onclick: (self) => {
+                        if (can_preview) {
+                            preview_enabled = true;
+                            user.request_preview(this)
+                        }
                         disable()
                         this.el.appendChild(enable_button)
                         self.disabled = true
@@ -64,17 +80,18 @@ export const resource_track: ResourceHandlerDecl = {
 
 export function new_local_track(info: ProvideInfo, stream: MediaStream, ...extra_controls: HTMLElement[]): LocalResource {
     let destroy: () => void;
-    let room: Room;
-
     const el = e("div", { class: `media-${stream.getVideoTracks().length > 0 ? "video" : "audio"}` },
         e("button", { icon: "stop", class: ["abort", "topleft"], onclick: () => destroy() }, PO.stop_sharing),
         ...extra_controls
     );
 
+    let preview: string | undefined;
+    let preview_on_ready: (() => void)[] | undefined = []
+    let last_preview_ts: number = Date.now();
     const generate_previews = (video: HTMLVideoElement) => {
         const canvas = document.createElement("canvas")
         const context = canvas.getContext("2d")!
-        setInterval(() => {
+        const update_preview = () => {
             const res = PREFS.preview_resolution
             canvas.width = res
             canvas.height = res * video.videoHeight / video.videoWidth
@@ -86,18 +103,31 @@ export function new_local_track(info: ProvideInfo, stream: MediaStream, ...extra
                 if (!blob) return log({ error: true, scope: "media" }, "Failed to encode stream preview");
                 const reader = new FileReader();
                 reader.addEventListener("load", ev => {
-                    const data_url = ev.target!.result as string;
-                    room.signaling.send_relay({ preview: { id: info.id, data: data_url } })
+                    preview = ev.target!.result as string;
+                    preview_on_ready?.forEach(f => f())
+                    preview_on_ready = undefined
+                    last_preview_ts = Date.now()
                 })
                 reader.readAsDataURL(blob)
 
             }, "image/webp", PREFS.preview_encoding_quality * 0.01)
-        }, 1000 * PREFS.preview_rate)
+        };
+        setTimeout(update_preview, 1000)
+        const interval = setInterval(update_preview, 1000 * PREFS.preview_rate)
+        return () => clearInterval(interval)
     }
     create_track_display(el, stream, true, generate_previews)
     return {
-        set_room(r) { room = r },
         set_destroy(cb) { destroy = cb },
+        async on_preview_request(_user) {
+            await new Promise<void>(done => {
+                if (preview_on_ready) {
+                    log("media", "preview delayed")
+                    preview_on_ready.push(done)
+                } else done()
+            })
+            return { data: preview, expire: (1000 * PREFS.preview_rate) - (Date.now() - last_preview_ts) + 500 };
+        },
         info,
         el,
         destroy() {
@@ -110,7 +140,7 @@ export function new_local_track(info: ProvideInfo, stream: MediaStream, ...extra
     }
 }
 
-function create_track_display(target: HTMLElement, stream: MediaStream, local: boolean, preview_callback?: (v: HTMLVideoElement) => void): HTMLElement {
+function create_track_display(target: HTMLElement, stream: MediaStream, local: boolean, preview_callback?: (v: HTMLVideoElement) => (() => void)): HTMLElement {
     const is_video = stream.getVideoTracks().length > 0
     const is_audio = stream.getAudioTracks().length > 0
 
@@ -137,15 +167,17 @@ function create_track_display(target: HTMLElement, stream: MediaStream, local: b
             else target.classList.remove("audio-active")
         }
     })
-    let fullscreen
+    let fullscreen: HTMLButtonElement | undefined;
     if (is_video) {
-        fullscreen = e("button", { icon: "fullscreen", class: ["topright", "fullscreen"],
-        onclick() { 
-            if (document.fullscreenElement && document.fullscreenElement !== null) 
-                document.exitFullscreen()
-            else
-                media_el.requestFullscreen()
-        }}, PO.fullscreen)
+        fullscreen = e("button", {
+            icon: "fullscreen", class: ["topright", "fullscreen"],
+            onclick() {
+                if (document.fullscreenElement && document.fullscreenElement !== null)
+                    document.exitFullscreen()
+                else
+                    media_el.requestFullscreen()
+            }
+        }, PO.fullscreen)
         target.prepend(fullscreen)
     }
 
diff --git a/client-web/source/user/local.ts b/client-web/source/user/local.ts
index 88e0852..a90a6f2 100644
--- a/client-web/source/user/local.ts
+++ b/client-web/source/user/local.ts
@@ -64,7 +64,6 @@ export class LocalUser extends User {
         this.el.append(r.el)
         this.room.signaling.send_relay({ provide })
 
-        if (r.set_room) r.set_room(this.room)
         r.set_destroy(() => {
             r.destroy()
             this.el.removeChild(r.el);
diff --git a/client-web/source/user/remote.ts b/client-web/source/user/remote.ts
index 1b46b91..4cb31a2 100644
--- a/client-web/source/user/remote.ts
+++ b/client-web/source/user/remote.ts
@@ -92,8 +92,22 @@ export class RemoteUser extends User {
             if (PREFS.notify_join) notify(PO.join_message(this.display_name).join(""))
             this.room.chat.add_control_message({ join: this })
         }
-        if (message.preview)
-            this.resources.get(message.preview.id)?.on_preview(message.preview.data)
+        if (message.preview_response) {
+            const res =
+                this.resources.get(message.preview_response.id);
+            if (!res) return log({ scope: "media", warn: true }, "unexpected preview response")
+            if (!res.on_preview_response) return log({ scope: "media", warn: true }, "unsupported preview response")
+            if (!message.preview_response.data) return // remote does not want go generate previews
+            res.on_preview_response(message.preview_response.data, message.preview_response.expire);
+        }
+        if (message.preview_request) {
+            const res = this.room.local_user.resources.get(message.preview_request.id)
+            if (!res) return log({ scope: "media" }, "unexpected preview request");
+            if (!res.on_preview_request) return log({ scope: "media", warn: true }, "unsupported preview request");
+            res.on_preview_request(this).then(({ expire, data }) => {
+                this.send_to({ preview_response: { id: res.info.id, data, expire } })
+            });
+        }
         if (message.provide) {
             const d = new_remote_resource(this, message.provide)
             if (!d) return
@@ -134,6 +148,7 @@ export class RemoteUser extends User {
     }
     request_resource(r: RemoteResource) { this.send_to({ request: { id: r.info.id } }) }
     request_resource_stop(r: RemoteResource) { this.send_to({ request_stop: { id: r.info.id } }) }
+    request_preview(r: RemoteResource) { this.send_to({ preview_request: { id: r.info.id } }) }
 
     add_ice_candidate(candidate: RTCIceCandidateInit) {
         this.pc.addIceCandidate(new RTCIceCandidate(candidate))
diff --git a/common/packets.d.ts b/common/packets.d.ts
index bf8fe9f..35972bb 100644
--- a/common/packets.d.ts
+++ b/common/packets.d.ts
@@ -43,7 +43,8 @@ export interface RelayMessage {
     offer?: Sdp
     answer?: Sdp
     ice_candidate?: F_RTCIceCandidateInit
-    preview?: { id: string, data: string }
+    preview_response?: { id: string, data?: string, expire: number }
+    preview_request?: { id: string }
 }
 
 export interface ChatMessage { text?: string, image?: string }