get rid of trackhandle and use mediastream everwhere

10 files changed, 88 insertions, 97 deletions

diff --git a/client-web/source/locale/de.ts b/client-web/source/locale/de.ts
index eee093c..83ce075 100644
--- a/client-web/source/locale/de.ts
+++ b/client-web/source/locale/de.ts
@@ -1,3 +1,8 @@
+/*
+    This file is part of keks-meet (https://codeberg.org/metamuffin/keks-meet)
+    which is licensed under the GNU Affero General Public License (version 3); see /COPYING.
+    Copyright (C) 2024 metamuffin <metamuffin.org>
+*/
 import { LanguageStrings } from "./mod.ts";
 
 export const PO_DE_DE: LanguageStrings = {
@@ -32,6 +37,7 @@ export const PO_DE_DE: LanguageStrings = {
     settings: "Einstellungen",
     edit: "Bearbeiten",
     finish_edit: "Fertig",
+    local: "Lokal",
     add_current_room: "Aktuellen Raum hinzufügen",
     add: "Hinzufügen",
     move_down: "Runter",
@@ -55,7 +61,7 @@ export const PO_DE_DE: LanguageStrings = {
     audio_stream: "Audioübertragung",
     disable: "Deaktivieren",
     enable: "Aktivieren",
-    status_await_track: "Spur wird erwartet…",
+    status_await_stream: "Übertragung startet…",
     notification_perm_explain: "Um Benarchichtigungen zu erhalten, musst du die keks-meet die Berechtigung dafür geben. ",
     grant: "Berechtigen",
     clear_prefs: "Du willst alle Einstellungen löschen? Nimm den hier: ",
diff --git a/client-web/source/locale/en.ts b/client-web/source/locale/en.ts
index 897ac15..db16fe7 100644
--- a/client-web/source/locale/en.ts
+++ b/client-web/source/locale/en.ts
@@ -1,3 +1,8 @@
+/*
+    This file is part of keks-meet (https://codeberg.org/metamuffin/keks-meet)
+    which is licensed under the GNU Affero General Public License (version 3); see /COPYING.
+    Copyright (C) 2024 metamuffin <metamuffin.org>
+*/
 import { LanguageStrings } from "./mod.ts";
 
 export const PO_EN_US: LanguageStrings = {
@@ -53,9 +58,10 @@ export const PO_EN_US: LanguageStrings = {
     mute: "Mute",
     video_stream: "video stream",
     audio_stream: "audio stream",
+    local: "Local",
     disable: "Disable",
     enable: "Enable",
-    status_await_track: "Awaiting track…",
+    status_await_stream: "Awaiting stream…",
     notification_perm_explain: "For keks-meet to send notifications, it needs you to grant permission: ",
     grant: "Grant",
     clear_prefs: "Want to clear all settings? Use this:",
diff --git a/client-web/source/locale/mod.ts b/client-web/source/locale/mod.ts
index 18f845f..4a570ce 100644
--- a/client-web/source/locale/mod.ts
+++ b/client-web/source/locale/mod.ts
@@ -22,6 +22,7 @@ export interface LanguageStrings {
     camera: string,
     screen: string,
     file: string,
+    local: string,
     warn_short_secret: string,
     warn_no_webrtc: string,
     warn_secure_context: string,
@@ -74,7 +75,7 @@ export interface LanguageStrings {
     disable: string,
     notification_perm_explain: string,
     grant: string,
-    status_await_track: string,
+    status_await_stream: string,
     clear_prefs: string,
     setting_descs: { [key in keyof typeof PREF_DECLS]: string },
 }
diff --git a/client-web/source/preferences/decl.ts b/client-web/source/preferences/decl.ts
index 269e247..35762d3 100644
--- a/client-web/source/preferences/decl.ts
+++ b/client-web/source/preferences/decl.ts
@@ -1,7 +1,7 @@
 /*
     This file is part of keks-meet (https://codeberg.org/metamuffin/keks-meet)
     which is licensed under the GNU Affero General Public License (version 3); see /COPYING.
-    Copyright (C) 2023 metamuffin <metamuffin.org>
+    Copyright (C) 2024 metamuffin <metamuffin.org>
 */
 // there should be no deps to dom APIs in this file for the tablegen to work
 
diff --git a/client-web/source/preferences/ui.ts b/client-web/source/preferences/ui.ts
index 252cba8..9cb52fd 100644
--- a/client-web/source/preferences/ui.ts
+++ b/client-web/source/preferences/ui.ts
@@ -1,7 +1,7 @@
 /*
     This file is part of keks-meet (https://codeberg.org/metamuffin/keks-meet)
     which is licensed under the GNU Affero General Public License (version 3); see /COPYING.
-    Copyright (C) 2023 metamuffin <metamuffin.org>
+    Copyright (C) 2024 metamuffin <metamuffin.org>
 */
 /// <reference lib="dom" />
 
diff --git a/client-web/source/resource/mod.ts b/client-web/source/resource/mod.ts
index 57c80d2..5ffbd67 100644
--- a/client-web/source/resource/mod.ts
+++ b/client-web/source/resource/mod.ts
@@ -1,12 +1,11 @@
 /*
     This file is part of keks-meet (https://codeberg.org/metamuffin/keks-meet)
     which is licensed under the GNU Affero General Public License (version 3); see /COPYING.
-    Copyright (C) 2023 metamuffin <metamuffin.org>
+    Copyright (C) 2024 metamuffin <metamuffin.org>
 */
 /// <reference lib="dom" />
 
 import { ProvideInfo } from "../../../common/packets.d.ts"
-import { TrackHandle } from "../track_handle.ts";
 import { RemoteUser } from "../user/remote.ts"
 import { resource_file } from "./file.ts";
 import { resource_track } from "./track.ts";
@@ -21,13 +20,15 @@ export interface RemoteResource {
     el: HTMLElement
     info: ProvideInfo,
     on_statechange(state: RemoteResourceState): void
-    on_enable(t: TrackHandle | RTCDataChannel, disable: () => void): void
+    on_enable(t: MediaStream | RTCDataChannel, disable: () => void): void,
+
+    stream?: MediaStream
 }
 export interface LocalResource {
     el: HTMLElement
     info: ProvideInfo,
     destroy(): void
-    on_request(user: RemoteUser, create_channel: () => RTCDataChannel): TrackHandle | RTCDataChannel,
+    on_request(user: RemoteUser, create_channel: () => RTCDataChannel): MediaStream | RTCDataChannel,
     set_destroy(cb: () => void): void
 }
 
diff --git a/client-web/source/resource/track.ts b/client-web/source/resource/track.ts
index 5db07dc..6d6c34f 100644
--- a/client-web/source/resource/track.ts
+++ b/client-web/source/resource/track.ts
@@ -1,7 +1,7 @@
 /*
     This file is part of keks-meet (https://codeberg.org/metamuffin/keks-meet)
     which is licensed under the GNU Affero General Public License (version 3); see /COPYING.
-    Copyright (C) 2023 metamuffin <metamuffin.org>
+    Copyright (C) 2024 metamuffin <metamuffin.org>
 */
 /// <reference lib="dom" />
 import { ProvideInfo } from "../../../common/packets.d.ts";
@@ -10,7 +10,6 @@ import { PO } from "../locale/mod.ts";
 import { log } from "../logger.ts";
 import { on_pref_changed, PREFS } from "../preferences/mod.ts";
 import { get_rnnoise_node } from "../rnnoise.ts";
-import { TrackHandle } from "../track_handle.ts";
 import { LocalResource, ResourceHandlerDecl } from "./mod.ts";
 
 export const resource_track: ResourceHandlerDecl = {
@@ -23,17 +22,18 @@ export const resource_track: ResourceHandlerDecl = {
             class: "center",
             onclick: self => {
                 self.disabled = true;
-                self.textContent = PO.status_await_track;
+                self.textContent = PO.status_await_stream;
                 enable()
             }
         }, enable_label)
+
         return {
             info,
             el: e("div", { class: [`media-${info.track_kind}`] }, enable_button),
             on_statechange() { },
-            on_enable(track, disable) {
+            on_enable(stream, disable) {
                 this.el.removeChild(enable_button)
-                if (!(track instanceof TrackHandle)) return console.warn("aservuoivasretuoip");
+                if (!(stream instanceof MediaStream)) return console.warn("expected mediastream");
                 this.el.append(e("button", {
                     class: ["topright", "abort"],
                     onclick: (self) => {
@@ -45,36 +45,39 @@ export const resource_track: ResourceHandlerDecl = {
                         self.remove()
                     }
                 }, PO.disable))
-                create_track_display(this.el, track)
-            }
+                create_track_display(this.el, stream, false)
+            },
         }
     }
 }
 
-export function new_local_track(info: ProvideInfo, track: TrackHandle, ...extra_controls: HTMLElement[]): LocalResource {
+export function new_local_track(info: ProvideInfo, stream: MediaStream, ...extra_controls: HTMLElement[]): LocalResource {
     let destroy: () => void;
     return {
         set_destroy(cb) { destroy = cb },
         info,
         el: create_track_display(
-            e("div", { class: `media-${track.kind}` },
+            e("div", { class: `media-${stream.getVideoTracks().length > 0 ? "video" : "audio"}` },
                 e("button", { class: ["abort", "topright"], onclick: () => destroy() }, PO.stop_sharing),
                 ...extra_controls
             ),
-            track
+            stream,
+            true
         ),
-        destroy() { track.end() },
+        destroy() {
+            stream.dispatchEvent(new Event("ended"));
+            stream.getTracks().forEach(t => t.stop())
+        },
         on_request(_user, _create_channel) {
-            return track
+            return stream
         }
     }
 }
 
-function create_track_display(target: HTMLElement, track: TrackHandle): HTMLElement {
-    const is_video = track.kind == "video"
-    const is_audio = track.kind == "audio"
+function create_track_display(target: HTMLElement, stream: MediaStream, local: boolean): HTMLElement {
+    const is_video = stream.getVideoTracks().length > 0
+    const is_audio = stream.getAudioTracks().length > 0
 
-    const stream = new MediaStream([track.track])
     const media_el = is_video
         ? document.createElement("video")
         : document.createElement("audio")
@@ -85,11 +88,16 @@ function create_track_display(target: HTMLElement, track: TrackHandle): HTMLElem
     media_el.ariaLabel = is_video ? PO.video_stream : PO.audio_stream
     media_el.addEventListener("pause", () => media_el.play())
 
-    if (track.local) media_el.muted = true
+    if (local) media_el.muted = true
+
+    target.querySelectorAll("video, audio").forEach(e => e.remove())
     target.prepend(media_el)
-    track.addEventListener("ended", () => {
-        media_el.srcObject = null // TODO // TODO figure out why i wrote todo here
-        media_el.remove()
+
+    console.log(stream.getTracks());
+    const master = stream.getTracks()[0]
+    master.addEventListener("ended", () => {
+        if (is_video) media_el.controls = false
+        media_el.classList.add("media-freeze")
     })
 
     if (is_audio && PREFS.audio_activity_threshold !== undefined) check_volume(stream, vol => {
@@ -103,18 +111,21 @@ function create_track_display(target: HTMLElement, track: TrackHandle): HTMLElem
     return target
 }
 
-function check_volume(track: MediaStream, cb: (vol: number) => void) {
+function check_volume(stream: MediaStream, cb: (vol: number) => void) {
     const ctx = new AudioContext();
-    const s = ctx.createMediaStreamSource(track)
+    const s = ctx.createMediaStreamSource(stream)
     const a = ctx.createAnalyser()
     s.connect(a)
     const samples = new Float32Array(a.fftSize);
-    setInterval(() => {
+    const interval = setInterval(() => {
         a.getFloatTimeDomainData(samples);
         let sum = 0.0;
         for (const amplitude of samples) { sum += amplitude * amplitude; }
         cb(Math.sqrt(sum / samples.length))
     }, 1000 / 15)
+    stream.addEventListener("ended", () => {
+        clearInterval(interval)
+    })
 }
 
 export async function create_camera_res() {
@@ -124,10 +135,9 @@ export async function create_camera_res() {
             facingMode: { ideal: PREFS.camera_facing_mode },
             frameRate: { ideal: PREFS.video_fps },
             width: { ideal: PREFS.video_resolution }
-        }
+        },
     })
-    const t = new TrackHandle(user_media.getVideoTracks()[0], true)
-    return new_local_track({ id: t.id, kind: "track", track_kind: "video", label: "Camera" }, t)
+    return new_local_track({ id: user_media.id, kind: "track", track_kind: "video", label: "Camera" }, user_media)
 }
 
 export async function create_screencast_res() {
@@ -137,9 +147,9 @@ export async function create_screencast_res() {
             frameRate: { ideal: PREFS.video_fps },
             width: { ideal: PREFS.video_resolution }
         },
+        audio: PREFS.screencast_audio
     })
-    const t = new TrackHandle(user_media.getVideoTracks()[0], true)
-    return new_local_track({ id: t.id, kind: "track", track_kind: "video", label: "Screen" }, t)
+    return new_local_track({ id: user_media.id, kind: "track", track_kind: "video", label: "Screen" }, user_media)
 }
 
 export async function create_mic_res() {
@@ -169,28 +179,30 @@ export async function create_mic_res() {
     }
     gain.connect(destination)
 
-    const t = new TrackHandle(destination.stream.getAudioTracks()[0], true)
-    t.addEventListener("ended", () => {
-        user_media.getTracks().forEach(t => t.stop())
-        source.disconnect()
-        if (rnnoise) rnnoise.disconnect()
-        gain.disconnect()
-        clear_gain_cb()
-        destination.disconnect()
-    })
-
     const mute = document.createElement("input")
     mute.type = "checkbox"
 
     const mute_label = e("label", { class: "check-button" }, PO.mute)
     mute_label.prepend(mute)
 
-    const res = new_local_track({ id: t.id, kind: "track", track_kind: "audio", label: "Microphone" }, t, mute_label)
+    const res = new_local_track({ id: destination.stream.id, kind: "track", track_kind: "audio", label: "Microphone" }, destination.stream, mute_label)
     mute.onchange = () => {
         log("media", mute.checked ? "muted" : "unmuted")
         gain.gain.value = mute.checked ? Number.MIN_VALUE : PREFS.microphone_gain
         if (mute.checked) res.el.classList.add("audio-mute")
         else res.el.classList.remove("audio-mute")
     }
+
+    const old_destroy = res.destroy
+    res.destroy = () => {
+        user_media.getTracks().forEach(t => t.stop())
+        source.disconnect()
+        if (rnnoise) rnnoise.disconnect()
+        gain.disconnect()
+        clear_gain_cb()
+        destination.disconnect()
+        old_destroy()
+    }
+
     return res
 }
diff --git a/client-web/source/track_handle.ts b/client-web/source/track_handle.ts
deleted file mode 100644
index b680949..0000000
--- a/client-web/source/track_handle.ts
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
-    This file is part of keks-meet (https://codeberg.org/metamuffin/keks-meet)
-    which is licensed under the GNU Affero General Public License (version 3); see /COPYING.
-    Copyright (C) 2023 metamuffin <metamuffin.org>
-*/
-/// <reference lib="dom" />
-
-/// We need this to adjust the way events are fired
-export class TrackHandle extends EventTarget {
-    stream: MediaStream // this is used to create an id that is persistent across clients
-
-    constructor(
-        public track: MediaStreamTrack,
-        public local = false
-    ) {
-        super()
-        track.onended = () => this.dispatchEvent(new CustomEvent("ended"))
-        // TODO research how onmute and onunmute behave
-        track.onmute = () => this.dispatchEvent(new CustomEvent("ended")) // onmute seems to be called when the remote ends the track
-        track.onunmute = () => this.dispatchEvent(new CustomEvent("started"))
-
-        this.addEventListener("ended", () => {
-            // drop all references to help gc
-            track.onunmute = track.onmute = track.onended = null
-        })
-
-        this.stream = new MediaStream([track])
-    }
-
-    get kind() { return this.track.kind }
-    get label() { return this.track.label }
-    get muted() { return this.track.muted }
-    get id() { return this.stream.id } //!!
-
-    end() { this.track.stop(); this.dispatchEvent(new CustomEvent("ended")) }
-}
diff --git a/client-web/source/user/local.ts b/client-web/source/user/local.ts
index dae01b2..633ccaf 100644
--- a/client-web/source/user/local.ts
+++ b/client-web/source/user/local.ts
@@ -1,7 +1,7 @@
 /*
     This file is part of keks-meet (https://codeberg.org/metamuffin/keks-meet)
     which is licensed under the GNU Affero General Public License (version 3); see /COPYING.
-    Copyright (C) 2023 metamuffin <metamuffin.org>
+    Copyright (C) 2024 metamuffin <metamuffin.org>
 */
 /// <reference lib="dom" />
 
@@ -13,6 +13,7 @@ import { User } from "./mod.ts";
 import { create_camera_res, create_mic_res, create_screencast_res } from "../resource/track.ts";
 import { LocalResource } from "../resource/mod.ts";
 import { PREFS } from "../preferences/mod.ts";
+import { PO } from "../locale/mod.ts";
 
 export class LocalUser extends User {
     resources: Map<string, LocalResource> = new Map()
@@ -20,7 +21,7 @@ export class LocalUser extends User {
     constructor(room: Room, id: number) {
         super(room, id)
         this.el.classList.add("local")
-        this.status_el.textContent = "Local"
+        this.status_el.textContent = PO.local
         this.name = PREFS.username
         log("users", `added local user: ${this.display_name}`)
         this.add_initial_tracks()
diff --git a/client-web/source/user/remote.ts b/client-web/source/user/remote.ts
index 313ddc2..ad891bd 100644
--- a/client-web/source/user/remote.ts
+++ b/client-web/source/user/remote.ts
@@ -1,7 +1,7 @@
 /*
     This file is part of keks-meet (https://codeberg.org/metamuffin/keks-meet)
     which is licensed under the GNU Affero General Public License (version 3); see /COPYING.
-    Copyright (C) 2023 metamuffin <metamuffin.org>
+    Copyright (C) 2024 metamuffin <metamuffin.org>
 */
 /// <reference lib="dom" />
 
@@ -12,12 +12,11 @@ import { log } from "../logger.ts"
 import { PREFS } from "../preferences/mod.ts";
 import { new_remote_resource, RemoteResource } from "../resource/mod.ts";
 import { Room } from "../room.ts"
-import { TrackHandle } from "../track_handle.ts";
 import { User } from "./mod.ts";
 
 export class RemoteUser extends User {
     pc: RTCPeerConnection
-    senders: Map<string, RTCRtpSender> = new Map()
+    senders: Map<string, RTCRtpSender[]> = new Map()
     data_channels: Map<string, RTCDataChannel> = new Map()
     resources: Map<string, RemoteResource> = new Map()
 
@@ -36,16 +35,16 @@ export class RemoteUser extends User {
             this.update_status()
         }
         this.pc.ontrack = ev => {
-            const t = ev.track
             const id = ev.streams[0]?.id
             if (!id) { ev.transceiver.stop(); return log({ scope: "media", warn: true }, "got a track without stream") }
             const r = this.resources.get(id)
             if (!r) { ev.transceiver.stop(); return log({ scope: "media", warn: true }, "got an unassociated track") }
-            r.on_enable(new TrackHandle(t), () => {
+            r.stream = ev.streams[0]
+            r.on_enable(ev.streams[0], () => {
                 this.request_resource_stop(r)
                 ev.transceiver.stop()
             })
-            log("media", `remote track: ${this.display_name}`, t)
+            log("media", `remote stream: ${id}`, ev.streams[0])
             this.update_status()
         }
         this.pc.ondatachannel = ({ channel }) => {
@@ -104,6 +103,7 @@ export class RemoteUser extends User {
             this.resources.set(message.provide.id, d)
         }
         if (message.provide_stop) {
+            this.resources.get(message.provide_stop.id)?.stream?.dispatchEvent(new Event("ended"))
             this.resources.get(message.provide_stop.id)?.el.remove()
             this.resources.delete(message.provide_stop.id)
         }
@@ -111,9 +111,9 @@ export class RemoteUser extends User {
             const r = this.room.local_user.resources.get(message.request.id)
             if (!r) return log({ scope: "*", warn: true }, "somebody requested an unknown resource")
             const channel = r.on_request(this, () => this.pc.createDataChannel(r.info.id))
-            if (channel instanceof TrackHandle) {
-                const sender = this.pc.addTrack(channel.track, channel.stream)
-                this.senders.set(channel.id, sender)
+            if (channel instanceof MediaStream) {
+                const senders = channel.getTracks().map(t => this.pc.addTrack(t, channel))
+                this.senders.set(channel.id, senders)
                 channel.addEventListener("end", () => { this.senders.delete(r.info.id) })
             } else if (channel instanceof RTCDataChannel) {
                 this.data_channels.set(r.info.id, channel)
@@ -121,8 +121,8 @@ export class RemoteUser extends User {
             } else throw new Error("unreachable");
         }
         if (message.request_stop) {
-            const sender = this.senders.get(message.request_stop.id)
-            if (sender) this.pc.removeTrack(sender)
+            const sender = this.senders.get(message.request_stop.id) ?? []
+            sender.forEach(s => this.pc.removeTrack(s))
             const dc = this.data_channels.get(message.request_stop.id)
             if (dc) dc.close()
         }