From 8b11f29a5e83a8bd6d9111e7b2d871eaab5536eb Mon Sep 17 00:00:00 2001 From: metamuffin Date: Sat, 27 Apr 2024 20:22:56 +0200 Subject: prototype stream previews --- client-web/source/locale/de.ts | 8 +++-- client-web/source/locale/en.ts | 6 +++- client-web/source/locale/es.ts | 6 +++- client-web/source/locale/ja.ts | 6 +++- client-web/source/preferences/decl.ts | 8 +++-- client-web/source/preferences/mod.ts | 2 ++ client-web/source/preferences/ui.ts | 2 ++ client-web/source/resource/file.ts | 1 + client-web/source/resource/mod.ts | 6 ++-- client-web/source/resource/track.ts | 65 ++++++++++++++++++++++++++++------- client-web/source/user/local.ts | 1 + client-web/source/user/remote.ts | 2 ++ client-web/style/room.sass | 14 +++++--- common/packets.d.ts | 1 + 14 files changed, 102 insertions(+), 26 deletions(-) diff --git a/client-web/source/locale/de.ts b/client-web/source/locale/de.ts index 0bfae8f..a0831d0 100644 --- a/client-web/source/locale/de.ts +++ b/client-web/source/locale/de.ts @@ -60,7 +60,7 @@ export const PO_DE: LanguageStrings = { video_stream: "Videoübertragung", audio_stream: "Audioübertragung", disable: "Deaktivieren", - enable: thing => `${thing} aktivieren`, + enable: thing => `${thing} aktivieren`, status_await_stream: "Übertragung startet…", notification_perm_explain: "Um Benarchichtigungen zu erhalten, musst du keks-meet die Berechtigung dafür geben. ", grant: "Berechtigen", @@ -91,6 +91,10 @@ export const PO_DE: LanguageStrings = { enable_onbeforeunload: "Frage nach Bestätigung beim Verlassen der Seite, wenn Spuren geteilt sind", room_watches: "Bekannte Räume (Als semikolongetrennte Liste von name=geheimnis Paaren)", username: "Benutzername", - show_log: "Zeige ausführlichen log" + show_log: "Zeige ausführlichen log", + preview_rate: "Preview rate", + send_previews: "Send video previews", + preview_resolution: "Preview resolution", + preview_encoding_quality: "Preview encoding quality (0 - 100)", } } diff --git a/client-web/source/locale/en.ts b/client-web/source/locale/en.ts index f829364..8ebbeb3 100644 --- a/client-web/source/locale/en.ts +++ b/client-web/source/locale/en.ts @@ -91,6 +91,10 @@ export const PO_EN: LanguageStrings = { enable_onbeforeunload: "Prompt for confirmation when leaving the site while local resources are shared", room_watches: "Known rooms (as semicolon seperated list of name=secret pairs)", username: "Username", - show_log: "Show extended log" + show_log: "Show extended log", + preview_rate: "Preview rate", + send_previews: "Send video previews", + preview_resolution: "Preview resolution", + preview_encoding_quality: "Preview encoding quality (0 - 100)", } } diff --git a/client-web/source/locale/es.ts b/client-web/source/locale/es.ts index 6367313..7c0b6dc 100644 --- a/client-web/source/locale/es.ts +++ b/client-web/source/locale/es.ts @@ -92,6 +92,10 @@ export const PO_ES: LanguageStrings = { enable_onbeforeunload: "Prompt for confirmation when leaving the site while local resources are shared", room_watches: "Habitaciones conocidas (como semicolon seperated list of name=secret pairs)", username: "Nombre de usuario", - show_log: "Mostrar registro extendido." + show_log: "Mostrar registro extendido.", + preview_rate: "Preview rate", + send_previews: "Send video previews", + preview_resolution: "Preview resolution", + preview_encoding_quality: "Preview encoding quality (0 - 100)", } } diff --git a/client-web/source/locale/ja.ts b/client-web/source/locale/ja.ts index 09e0375..559d7ca 100644 --- a/client-web/source/locale/ja.ts +++ b/client-web/source/locale/ja.ts @@ -92,6 +92,10 @@ export const PO_JA: LanguageStrings = { enable_onbeforeunload: "ローカルリソースが共有されている間、サイトを離れるときに確認のためのプロンプト", room_watches: "既知の客室(セミコロンは、name=secretペアの区切りリストとして)", username: "ユーザ名", - show_log: "拡張ログを表示します。" + show_log: "拡張ログを表示します。", + preview_rate: "Preview rate", + send_previews: "Send video previews", + preview_resolution: "Preview resolution", + preview_encoding_quality: "Preview encoding quality (0 - 100)", }, } diff --git a/client-web/source/preferences/decl.ts b/client-web/source/preferences/decl.ts index 7d5c8b6..536c5c7 100644 --- a/client-web/source/preferences/decl.ts +++ b/client-web/source/preferences/decl.ts @@ -19,7 +19,7 @@ const optional = (a: T): T | undefined => a export const PREF_DECLS = { username: { type: string, default: "guest-" + hex_id(), allow_url: true }, language: { type: string, possible_values: ["system", ...Object.keys(LOCALES)], default: "system", allow_url: true }, - + /* MEDIA */ rnnoise: { type: bool, default: true, allow_url: true }, native_noise_suppression: { type: bool, default: false }, @@ -35,7 +35,11 @@ export const PREF_DECLS = { microphone_enabled: { type: bool, default: false }, screencast_enabled: { type: bool, default: false }, camera_enabled: { type: bool, default: false }, - + send_previews: { type: bool, default: true }, + preview_resolution: { type: number, default: 256, min: 16, max: 512 }, + preview_rate: { type: number, default: 8, min: 1, max: 60 }, + preview_encoding_quality: { type: number, default: 80, min: 10, max: 100 }, + // TODO differenciate between mic, cam and screen optional_audio_default_enable: { type: bool, default: true }, optional_video_default_enable: { type: bool, default: false }, diff --git a/client-web/source/preferences/mod.ts b/client-web/source/preferences/mod.ts index 1df57b1..bbf9bfb 100644 --- a/client-web/source/preferences/mod.ts +++ b/client-web/source/preferences/mod.ts @@ -15,6 +15,8 @@ export interface PrefDecl { hidden?: boolean allow_url?: boolean require_reload?: boolean, + min?: number, + max?: number, } type Type = "string" | "number" | "bigint" | "boolean" | "symbol" | "undefined" | "object" | "function"; diff --git a/client-web/source/preferences/ui.ts b/client-web/source/preferences/ui.ts index 9cb52fd..c710f54 100644 --- a/client-web/source/preferences/ui.ts +++ b/client-web/source/preferences/ui.ts @@ -57,6 +57,8 @@ export function ui_preferences(): HTMLElement { textbox.type = "number" textbox.id = id textbox.value = PREFS[key] as string + if (decl.min) textbox.min = "" + decl.min + if (decl.max) textbox.max = "" + decl.max textbox.onchange = () => { change_pref(key, parseFloat(textbox.value)) } diff --git a/client-web/source/resource/file.ts b/client-web/source/resource/file.ts index 0554e2f..7b63d84 100644 --- a/client-web/source/resource/file.ts +++ b/client-web/source/resource/file.ts @@ -26,6 +26,7 @@ export const resource_file: ResourceHandlerDecl = { }, PO.download) return { info, + on_preview(_) { }, el: e("div", {}, e("span", {}, `${PO.file}: ${JSON.stringify(info.label)} (${display_filesize(info.size!)})`), download_button, diff --git a/client-web/source/resource/mod.ts b/client-web/source/resource/mod.ts index 5ffbd67..5c76f41 100644 --- a/client-web/source/resource/mod.ts +++ b/client-web/source/resource/mod.ts @@ -6,6 +6,7 @@ /// import { ProvideInfo } from "../../../common/packets.d.ts" +import { Room } from "../room.ts"; import { RemoteUser } from "../user/remote.ts" import { resource_file } from "./file.ts"; import { resource_track } from "./track.ts"; @@ -21,7 +22,7 @@ export interface RemoteResource { info: ProvideInfo, on_statechange(state: RemoteResourceState): void on_enable(t: MediaStream | RTCDataChannel, disable: () => void): void, - + on_preview(p: string): void, stream?: MediaStream } export interface LocalResource { @@ -30,6 +31,7 @@ export interface LocalResource { destroy(): void on_request(user: RemoteUser, create_channel: () => RTCDataChannel): MediaStream | RTCDataChannel, set_destroy(cb: () => void): void + set_room?: (room: Room) => void } const RESOURCE_HANDLERS: ResourceHandlerDecl[] = [resource_file, resource_track] @@ -41,4 +43,4 @@ export function new_remote_resource(user: RemoteUser, info: ProvideInfo): Remote user.request_resource(res) }) return res -} \ No newline at end of file +} diff --git a/client-web/source/resource/track.ts b/client-web/source/resource/track.ts index 46440ec..2cf01f1 100644 --- a/client-web/source/resource/track.ts +++ b/client-web/source/resource/track.ts @@ -10,6 +10,7 @@ import { PO } from "../locale/mod.ts"; import { log } from "../logger.ts"; import { on_pref_changed, PREFS } from "../preferences/mod.ts"; import { get_rnnoise_node } from "../rnnoise.ts"; +import { Room } from "../room.ts"; import { LocalResource, ResourceHandlerDecl } from "./mod.ts"; export const resource_track: ResourceHandlerDecl = { @@ -29,6 +30,17 @@ export const resource_track: ResourceHandlerDecl = { info, el: e("div", { class: [`media-${info.track_kind}`] }, enable_button), on_statechange() { }, + on_preview(preview) { + if (this.el.querySelector("audio, video")) return + let pi = this.el.querySelector(".preview") as HTMLImageElement + if (!pi) { + pi = document.createElement("img") + pi.classList.add("preview") + this.el.prepend(pi) + } + if (!preview.startsWith("data:")) return + pi.src = preview + }, on_enable(stream, disable) { this.el.removeChild(enable_button) if (!(stream instanceof MediaStream)) return console.warn("expected mediastream"); @@ -51,17 +63,42 @@ export const resource_track: ResourceHandlerDecl = { export function new_local_track(info: ProvideInfo, stream: MediaStream, ...extra_controls: HTMLElement[]): LocalResource { let destroy: () => void; + let room: Room; + + const el = e("div", { class: `media-${stream.getVideoTracks().length > 0 ? "video" : "audio"}` }, + e("button", { class: ["abort", "topright"], onclick: () => destroy() }, PO.stop_sharing), + ...extra_controls + ); + + const generate_previews = (video: HTMLVideoElement) => { + const canvas = document.createElement("canvas") + const context = canvas.getContext("2d")! + context.fillStyle = "#ff00ff" + setInterval(() => { + context.fillRect(0, 0, video.videoWidth, video.videoHeight) + const res = PREFS.preview_resolution + canvas.width = res + canvas.height = res + context.drawImage(video, 0, 0, res, res) + canvas.toDataURL() + canvas.toBlob(blob => { + if (!blob) return log({ error: true, scope: "media" }, "Failed to encode stream preview"); + const reader = new FileReader(); + reader.addEventListener("load", ev => { + const data_url = ev.target!.result as string; + room.signaling.send_relay({ preview: { id: info.id, data: data_url } }) + }) + reader.readAsDataURL(blob) + + }, "image/webp", PREFS.preview_encoding_quality * 0.01) + }, 1000 * PREFS.preview_rate) + } + create_track_display(el, stream, true, generate_previews) return { + set_room(r) { room = r }, set_destroy(cb) { destroy = cb }, info, - el: create_track_display( - e("div", { class: `media-${stream.getVideoTracks().length > 0 ? "video" : "audio"}` }, - e("button", { class: ["abort", "topright"], onclick: () => destroy() }, PO.stop_sharing), - ...extra_controls - ), - stream, - true - ), + el, destroy() { stream.dispatchEvent(new Event("ended")); stream.getTracks().forEach(t => t.stop()) @@ -72,7 +109,7 @@ export function new_local_track(info: ProvideInfo, stream: MediaStream, ...extra } } -function create_track_display(target: HTMLElement, stream: MediaStream, local: boolean): HTMLElement { +function create_track_display(target: HTMLElement, stream: MediaStream, local: boolean, preview_callback?: (v: HTMLVideoElement) => void): HTMLElement { const is_video = stream.getVideoTracks().length > 0 const is_audio = stream.getAudioTracks().length > 0 @@ -88,16 +125,18 @@ function create_track_display(target: HTMLElement, stream: MediaStream, local: b if (local) media_el.muted = true - target.querySelectorAll("video, audio").forEach(e => e.remove()) + target.querySelectorAll("video, audio, .preview").forEach(e => e.remove()) target.prepend(media_el) console.log(stream.getTracks()); const master = stream.getTracks()[0] master.addEventListener("ended", () => { - if (is_video) media_el.controls = false - media_el.classList.add("media-freeze") + // if (is_video) media_el.controls = false + // media_el.classList.add("media-freeze") + media_el.remove() }) + if (is_video && PREFS.send_previews && local && preview_callback) preview_callback(media_el as HTMLVideoElement) if (is_audio && PREFS.audio_activity_threshold !== undefined) check_volume(stream, vol => { const active = vol > PREFS.audio_activity_threshold if (active != target.classList.contains("audio-active")) { @@ -106,7 +145,7 @@ function create_track_display(target: HTMLElement, stream: MediaStream, local: b } }) - return target + return media_el } function check_volume(stream: MediaStream, cb: (vol: number) => void) { diff --git a/client-web/source/user/local.ts b/client-web/source/user/local.ts index a90a6f2..88e0852 100644 --- a/client-web/source/user/local.ts +++ b/client-web/source/user/local.ts @@ -64,6 +64,7 @@ export class LocalUser extends User { this.el.append(r.el) this.room.signaling.send_relay({ provide }) + if (r.set_room) r.set_room(this.room) r.set_destroy(() => { r.destroy() this.el.removeChild(r.el); diff --git a/client-web/source/user/remote.ts b/client-web/source/user/remote.ts index ad891bd..1b46b91 100644 --- a/client-web/source/user/remote.ts +++ b/client-web/source/user/remote.ts @@ -92,6 +92,8 @@ export class RemoteUser extends User { if (PREFS.notify_join) notify(PO.join_message(this.display_name).join("")) this.room.chat.add_control_message({ join: this }) } + if (message.preview) + this.resources.get(message.preview.id)?.on_preview(message.preview.data) if (message.provide) { const d = new_remote_resource(this, message.provide) if (!d) return diff --git a/client-web/style/room.sass b/client-web/style/room.sass index 5c41171..8a72bf8 100644 --- a/client-web/style/room.sass +++ b/client-web/style/room.sass @@ -102,10 +102,16 @@ .media-freeze filter: saturate(0%) -video + +video, .preview position: absolute - max-width: 100% - max-height: 100% left: 50% top: 50% - transform: translate(-50%, -50%) \ No newline at end of file + transform: translate(-50%, -50%) + +video + max-width: 100% + max-height: 100% +.preview + width: 100% + height: 100% diff --git a/common/packets.d.ts b/common/packets.d.ts index 662c1fe..bf8fe9f 100644 --- a/common/packets.d.ts +++ b/common/packets.d.ts @@ -43,6 +43,7 @@ export interface RelayMessage { offer?: Sdp answer?: Sdp ice_candidate?: F_RTCIceCandidateInit + preview?: { id: string, data: string } } export interface ChatMessage { text?: string, image?: string } -- cgit v1.2.3-70-g09d2