fix a bunch of stupid things regarding subtitles. still doesnt work

13 files changed, 209 insertions, 87 deletions

diff --git a/common/src/jhls.rs b/common/src/jhls.rs
index b365b3f..f4448b6 100644
--- a/common/src/jhls.rs
+++ b/common/src/jhls.rs
@@ -31,3 +31,10 @@ pub enum EncodingProfile {
         codec: String,
     },
 }
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct SubtitleCue {
+    pub start: f64,
+    pub end: f64,
+    pub content: String,
+}
diff --git a/common/src/stream.rs b/common/src/stream.rs
index 151d497..aa7195e 100644
--- a/common/src/stream.rs
+++ b/common/src/stream.rs
@@ -30,6 +30,7 @@ pub enum StreamFormat {
     #[cfg_attr(feature = "rocket", field(value = "jhlsi"))] JhlsIndex,
     #[cfg_attr(feature = "rocket", field(value = "snippet"))] Snippet,
     #[cfg_attr(feature = "rocket", field(value = "webvtt"))] Webvtt,
+    #[cfg_attr(feature = "rocket", field(value = "jvtt"))] Jvtt,
 }
 
 impl Default for StreamSpec {
@@ -78,6 +79,7 @@ impl StreamSpec {
 impl StreamFormat {
     pub fn ident(&self) -> &'static str {
         match self {
+            StreamFormat::Jvtt => "jvtt",
             StreamFormat::Original => "original",
             StreamFormat::Matroska => "matroska",
             StreamFormat::HlsMaster => "hlsmaster",
diff --git a/remuxer/src/lib.rs b/remuxer/src/lib.rs
index ffefcae..96aeca1 100644
--- a/remuxer/src/lib.rs
+++ b/remuxer/src/lib.rs
@@ -82,7 +82,7 @@ pub fn ebml_track_entry(
             els.push(MatroskaTag::BitDepth(bit_depth.try_into().unwrap()));
         }
         SourceTrackKind::Subtitles => {
-            els.push(MatroskaTag::TrackType(19));
+            els.push(MatroskaTag::TrackType(17));
         }
     }
     if let Some(d) = &codec_private {
diff --git a/remuxer/src/snippet.rs b/remuxer/src/snippet.rs
index 4c3c47f..5b271e4 100644
--- a/remuxer/src/snippet.rs
+++ b/remuxer/src/snippet.rs
@@ -9,7 +9,7 @@ use crate::{
     segment_extractor::SegmentExtractIter,
 };
 use anyhow::{anyhow, Context, Result};
-use jellycommon::{LocalTrack, NodePublic};
+use jellycommon::{LocalTrack, NodePublic, SourceTrackKind};
 use jellymatroska::{read::EbmlReader, write::EbmlWriter, Master, MatroskaTag};
 use log::{debug, info};
 use std::{
@@ -25,6 +25,7 @@ pub fn snippet_index(
     path_base: &Path,
     item: &NodePublic,
     local_track: &LocalTrack,
+    track_index: usize,
 ) -> Result<Vec<Range<f64>>> {
     let media_info = item.media.as_ref().unwrap();
     let source_path = path_base.join(&local_track.path);
@@ -32,20 +33,46 @@ pub fn snippet_index(
     let index = index
         .get(&(local_track.track as u64))
         .ok_or(anyhow!("seek index track missing"))?;
-    let average_kf_interval = media_info.duration / index.keyframes.len() as f64;
+
+    // everything is a keyframe (even though nothing is...)
+    let force_kf = matches!(
+        media_info.tracks[track_index].kind,
+        SourceTrackKind::Subtitles { .. }
+    );
+
+    let n_kf = if force_kf {
+        index.blocks.len()
+    } else {
+        index.keyframes.len()
+    };
+
+    let average_kf_interval = media_info.duration / n_kf as f64;
     let kf_per_snip = (SNIPPET_LENGTH / average_kf_interval).ceil() as usize;
     debug!("average keyframe interval: {average_kf_interval}");
     debug!(" => keyframes per snippet {kf_per_snip}");
 
-    let n_snips = index.keyframes.len().div_ceil(kf_per_snip);
+    let n_snips = n_kf.div_ceil(kf_per_snip);
     Ok((0..n_snips)
         .map(|i| {
-            let start = index.blocks[index.keyframes[i * kf_per_snip]].pts as f64 / 1000.;
-            let end = index
-                .keyframes
-                .get((i + 1) * kf_per_snip)
-                .map(|i| index.blocks[*i].pts as f64 / 1000.)
-                .unwrap_or(media_info.duration);
+            let start = index.blocks[if force_kf {
+                i * kf_per_snip
+            } else {
+                index.keyframes[i * kf_per_snip]
+            }]
+            .pts as f64
+                / 1000.;
+            let end = if force_kf {
+                let n = (i + 1) * kf_per_snip;
+                if n >= index.blocks.len() {
+                    None
+                } else {
+                    Some(n)
+                }
+            } else {
+                index.keyframes.get((i + 1) * kf_per_snip).copied()
+            }
+            .map(|i| index.blocks[i].pts as f64 / 1000.)
+            .unwrap_or(media_info.duration);
             start..end
         })
         .collect())
@@ -62,7 +89,6 @@ pub fn write_snippet_into(
 ) -> anyhow::Result<()> {
     info!("writing snippet {n} of {:?} (track {track})", item.title);
     let mut output = EbmlWriter::new(BufWriter::new(writer), 0);
-
     let media_info = item.media.as_ref().unwrap();
     let info = media_info
         .tracks
@@ -85,19 +111,33 @@ pub fn write_snippet_into(
     debug!("\t    seek index: {} blocks loaded", index.blocks.len());
     let mut reader = EbmlReader::new(file);
 
-    let average_kf_interval = media_info.duration / index.keyframes.len() as f64;
+    let force_kf = matches!(info.kind, SourceTrackKind::Subtitles { .. });
+    let n_kf = if force_kf {
+        index.blocks.len()
+    } else {
+        index.keyframes.len()
+    };
+
+    let average_kf_interval = media_info.duration / n_kf as f64;
     let kf_per_snip = (SNIPPET_LENGTH / average_kf_interval).ceil() as usize;
     debug!("average keyframe interval: {average_kf_interval}");
     debug!(" => keyframes per snippet {kf_per_snip}");
 
-    let start_block_index = *index
-        .keyframes
-        .get(n * kf_per_snip)
-        .ok_or(anyhow!("snippet index out of range"))?;
-    let end_block_index = *index
-        .keyframes
-        .get((n + 1) * kf_per_snip)
-        .unwrap_or(&index.blocks.len());
+    let (start_block_index, end_block_index) = if force_kf {
+        (n * kf_per_snip, (n + 1) * kf_per_snip)
+    } else {
+        (
+            *index
+                .keyframes
+                .get(n * kf_per_snip)
+                .ok_or(anyhow!("snippet index out of range"))?,
+            *index
+                .keyframes
+                .get((n + 1) * kf_per_snip)
+                .unwrap_or(&index.blocks.len()),
+        )
+    };
+
     let start_block = &index.blocks[start_block_index];
     let last_block = &index.blocks[end_block_index - 1];
 
@@ -152,5 +192,7 @@ pub fn write_snippet_into(
     }
 
     output.write_tag(&MatroskaTag::Segment(Master::End))?;
+
+    debug!("wrote {} bytes", output.position());
     Ok(())
 }
diff --git a/stream/src/hls.rs b/stream/src/hls.rs
index c036e98..4f83577 100644
--- a/stream/src/hls.rs
+++ b/stream/src/hls.rs
@@ -49,11 +49,14 @@ pub async fn hls_variant_stream(
     mut spec: StreamSpec,
     mut b: DuplexStream,
 ) -> Result<()> {
+    let local_track = local_tracks.get(0).ok_or(anyhow!("no track"))?.to_owned();
+    let track_index = spec.tracks[0];
     let snips = spawn_blocking(move || {
         jellyremuxer::snippet::snippet_index(
             &CONF.media_path,
             &node.public,
-            local_tracks.get(0).ok_or(anyhow!("no track"))?,
+            &local_track,
+            track_index,
         )
     })
     .await??;
diff --git a/stream/src/jhls.rs b/stream/src/jhls.rs
index d5bb575..e26df1b 100644
--- a/stream/src/jhls.rs
+++ b/stream/src/jhls.rs
@@ -16,7 +16,7 @@ use tokio::io::{AsyncWriteExt, DuplexStream};
 pub async fn jhls_index(
     node: Node,
     local_tracks: &[LocalTrack],
-    _spec: StreamSpec,
+    spec: StreamSpec,
     mut b: DuplexStream,
     perms: &PermissionSet,
 ) -> Result<()> {
@@ -26,7 +26,12 @@ pub async fn jhls_index(
         .to_owned();
 
     let segments = tokio::task::spawn_blocking(move || {
-        jellyremuxer::snippet::snippet_index(&CONF.media_path, &node.public, &local_track)
+        jellyremuxer::snippet::snippet_index(
+            &CONF.media_path,
+            &node.public,
+            &local_track,
+            spec.tracks[0],
+        )
     })
     .await??;
 
diff --git a/stream/src/lib.rs b/stream/src/lib.rs
index ee47857..833af3e 100644
--- a/stream/src/lib.rs
+++ b/stream/src/lib.rs
@@ -25,7 +25,7 @@ use tokio::{
     io::{duplex, AsyncReadExt, AsyncSeekExt, AsyncWriteExt, DuplexStream},
 };
 use tokio_util::io::SyncIoBridge;
-use webvtt::webvtt_stream;
+use webvtt::vtt_stream;
 
 pub struct StreamHead {
     pub content_type: &'static str,
@@ -42,6 +42,7 @@ pub fn stream_head(spec: &StreamSpec) -> StreamHead {
         StreamFormat::JhlsIndex => StreamHead { content_type: "application/jellything-seekindex+json", range_supported: false },
         StreamFormat::Webvtt => StreamHead { content_type: "text/vtt", range_supported: false },
         StreamFormat::Snippet => StreamHead { content_type: webm_or_mkv, range_supported: false },
+        StreamFormat::Jvtt => StreamHead { content_type: "applcation/jellything-vtt+json", range_supported: false },
     }
 }
 
@@ -87,7 +88,8 @@ pub async fn stream(
         StreamFormat::HlsVariant => hls_variant_stream(node, local_tracks, spec, b).await?,
         StreamFormat::JhlsIndex => jhls_index(node, &local_tracks, spec, b, perms).await?,
         StreamFormat::Snippet => segment_stream(node, local_tracks, spec, b, perms).await?,
-        StreamFormat::Webvtt => webvtt_stream(node, local_tracks, spec, b).await?,
+        StreamFormat::Webvtt => vtt_stream(false, node, local_tracks, spec, b).await?,
+        StreamFormat::Jvtt => vtt_stream(true, node, local_tracks, spec, b).await?,
     }
 
     Ok(a)
diff --git a/stream/src/webvtt.rs b/stream/src/webvtt.rs
index 48cf8d5..2de9835 100644
--- a/stream/src/webvtt.rs
+++ b/stream/src/webvtt.rs
@@ -5,12 +5,13 @@
 */
 use anyhow::{anyhow, bail, Context, Result};
 use jellybase::CONF;
-use jellycommon::{stream::StreamSpec, LocalTrack, Node};
+use jellycommon::{jhls::SubtitleCue, stream::StreamSpec, LocalTrack, Node};
 use jellyremuxer::extract::extract_track;
-use jellytranscoder::subtitles::webvtt_from_ass_blocks;
+use jellytranscoder::subtitles::{parse_ass_blocks, parse_webvtt_blocks, write_webvtt};
 use tokio::io::{AsyncWriteExt, DuplexStream};
 
-pub async fn webvtt_stream(
+pub async fn vtt_stream(
+    json: bool,
     node: Node,
     local_tracks: Vec<LocalTrack>,
     spec: StreamSpec,
@@ -24,7 +25,29 @@ pub async fn webvtt_stream(
     let local_track = local_tracks.get(0).ok_or(anyhow!("no tracks"))?.clone();
     let track = &node.public.media.unwrap().tracks[tracki];
 
+    let write = |blocks: Vec<SubtitleCue>| -> anyhow::Result<()> {
+        let output = if json {
+            serde_json::to_string(&blocks)?
+        } else {
+            write_webvtt(node.public.title.clone().unwrap_or_default(), blocks)
+                .context("writing webvtt")?
+        };
+        tokio::task::spawn(async move {
+            let _ = b.write_all(output.as_bytes()).await;
+        });
+        Ok(())
+    };
+
     match track.codec.as_str() {
+        "D_WEBVTT/SUBTITLES" => {
+            let webvtt_blocks = tokio::task::spawn_blocking(move || {
+                extract_track(CONF.media_path.clone(), local_track)
+            })
+            .await??;
+
+            let subtitles = parse_webvtt_blocks(webvtt_blocks).context("parsing subtitles")?;
+            write(subtitles)?;
+        }
         "S_TEXT/UTF8" => bail!("no subrip yet"),
         "S_VOBSUB" => bail!("no vobsub yet"),
         "S_TEXT/ASS" => {
@@ -34,23 +57,16 @@ pub async fn webvtt_stream(
                 .ok_or(anyhow!("ASS is missing required codec private data"))?;
 
             let ass_blocks = tokio::task::spawn_blocking(move || {
-                extract_track(CONF.library_path.clone(), local_track)
+                extract_track(CONF.media_path.clone(), local_track)
             })
             .await??;
 
-            let webvtt = webvtt_from_ass_blocks(
-                node.public.title.clone().unwrap_or_default(),
-                codec_private,
-                ass_blocks,
-            )
-            .context("transcoding subtitles")?;
-
-            tokio::task::spawn(async move {
-                let _ = b.write_all(webvtt.as_bytes()).await;
-            });
+            let subtitles =
+                parse_ass_blocks(codec_private, ass_blocks).context("parsing subtitles")?;
+            write(subtitles)?;
         }
 
-        _ => bail!("unknown sub codec"),
+        x => bail!("unknown sub codec {x:?}"),
     };
     Ok(())
 }
diff --git a/transcoder/src/subtitles.rs b/transcoder/src/subtitles.rs
index a2546f4..1fea3cf 100644
--- a/transcoder/src/subtitles.rs
+++ b/transcoder/src/subtitles.rs
@@ -4,52 +4,87 @@
     Copyright (C) 2024 metamuffin <metamuffin.org>
 */
 use anyhow::anyhow;
+use jellycommon::jhls::SubtitleCue;
 use std::fmt::Write;
 
 // TODO discontinued for now. since this should be snippetized aswell.
 
-pub fn webvtt_from_ass_blocks(
-    title: String,
-    _codec_private: Vec<u8>,
-    blocks: Vec<(u64, u64, Vec<u8>)>,
-) -> anyhow::Result<String> {
+pub fn write_webvtt(title: String, subtitles: Vec<SubtitleCue>) -> anyhow::Result<String> {
     let mut out = String::new();
 
     writeln!(out, "WEBVTT - {title}")?; // TODO ensure title does not contain "-->"
     writeln!(out)?;
-
-    for (pts, dur, block) in blocks {
-        let block = String::from_utf8(block)?;
-        let text = convert_block(&block).ok_or(anyhow!("bad ass xD"))?;
-        writeln!(out, "{} --> {}", format_time(pts), format_time(pts + dur))?;
-        writeln!(out, "- {text}")?;
+    for SubtitleCue {
+        start,
+        end,
+        content,
+    } in subtitles
+    {
+        writeln!(out, "{} --> {}", format_time(start), format_time(end))?;
+        writeln!(out, "- {content}")?;
         writeln!(out)?;
     }
+    Ok(out)
+}
 
+pub fn parse_webvtt_blocks(blocks: Vec<(u64, u64, Vec<u8>)>) -> anyhow::Result<Vec<SubtitleCue>> {
+    let mut out = Vec::new();
+    for (pts, dur, block) in blocks {
+        let content = String::from_utf8_lossy(&block).trim().to_string();
+        out.push(SubtitleCue {
+            content,
+            start: pts as f64 / 1000.,
+            end: (pts + dur) as f64 / 1000.,
+        })
+    }
     Ok(out)
 }
 
-fn format_time(t: u64) -> String {
-    let t = t as f64 / 1_000_000.;
+pub fn parse_ass_blocks(
+    _codec_private: Vec<u8>,
+    blocks: Vec<(u64, u64, Vec<u8>)>,
+) -> anyhow::Result<Vec<SubtitleCue>> {
+    // TODO dont ignore codec_private
 
-    // let mmm = (t.fract() * 1000.).floor();
-    // let mmm = ((t * 60.).fract() * 60.).floor();
-    // let mmm = ((t / 60).fract() * 60.).floor();
+    fn convert_block(s: &str) -> Option<&str> {
+        // ReadOrder, Layer, Style, Name, MarginL, MarginR, MarginV, Effect, Text
+        let (_read_order, s) = s.split_once(',')?;
+        let (_layer, s) = s.split_once(',')?;
+        let (_style, s) = s.split_once(',')?;
+        let (_name, s) = s.split_once(',')?;
+        let (_marginl, s) = s.split_once(',')?;
+        let (_marginr, s) = s.split_once(',')?;
+        let (_marginv, s) = s.split_once(',')?;
+        let (_effect, text) = s.split_once(',')?;
 
-    // format!("{hh:4}:{mm:02}:{ss:02}.{mmm:03}")
-    format!("{t}")
+        Some(text)
+    }
+
+    let mut out = Vec::new();
+    for (pts, dur, block) in blocks {
+        let block = String::from_utf8(block)?;
+        let text = convert_block(&block).ok_or(anyhow!("bad ass xD"))?;
+        out.push(SubtitleCue {
+            content: text.to_owned(),
+            start: pts as f64 / 1000.,
+            end: (pts + dur) as f64 / 1000.,
+        })
+    }
+    Ok(out)
 }
 
-fn convert_block(s: &str) -> Option<&str> {
-    // ReadOrder, Layer, Style, Name, MarginL, MarginR, MarginV, Effect, Text
-    let (_read_order, s) = s.split_once(',')?;
-    let (_layer, s) = s.split_once(',')?;
-    let (_style, s) = s.split_once(',')?;
-    let (_name, s) = s.split_once(',')?;
-    let (_marginl, s) = s.split_once(',')?;
-    let (_marginr, s) = s.split_once(',')?;
-    let (_marginv, s) = s.split_once(',')?;
-    let (_effect, text) = s.split_once(',')?;
+fn format_time(t: f64) -> String {
+    const SECOND: u64 = 1000;
+    const MINUTE: u64 = SECOND * 60;
+    const HOUR: u64 = MINUTE * 60;
+
+    let t = (t * 1000.) as u64;
+    let hh = t / HOUR;
+    let t = t % HOUR;
+    let mm = t / MINUTE;
+    let t = t % MINUTE;
+    let ss = t / SECOND;
+    let mmm = t % SECOND;
 
-    Some(text)
+    format!("{hh:04}:{mm:02}:{ss:02}.{mmm:03}")
 }
diff --git a/web/script/player/jhls.d.ts b/web/script/player/jhls.d.ts
index 9030a88..9642c66 100644
--- a/web/script/player/jhls.d.ts
+++ b/web/script/player/jhls.d.ts
@@ -61,19 +61,21 @@ export interface SourceTrack {
     codec: string,
     language: string,
 }
-export interface SourceTrackKind {
-    video?: {
+export type SourceTrackKind = {
+    video: {
         width: number,
         height: number,
         fps: number,
-    },
-    audio?: {
-        channels: number,
-        sample_rate: number,
-        bit_depth: number,
-    },
-    subtitles?: boolean, // incorrect but lazy rn
+    }
 }
+    | {
+        audio: {
+            channels: number,
+            sample_rate: number,
+            bit_depth: number,
+        }
+    } | "subtitles";
+
 export interface EncodingProfile {
     video?: {
         codec: string,
diff --git a/web/script/player/mediacaps.ts b/web/script/player/mediacaps.ts
index 77a210e..ad1a037 100644
--- a/web/script/player/mediacaps.ts
+++ b/web/script/player/mediacaps.ts
@@ -21,13 +21,15 @@ export async function test_media_capability(track: SourceTrack): Promise<boolean
     return r
 }
 async function test_media_capability_inner(track: SourceTrack) {
-    if (track.kind.subtitles) {
-        return track.codec == "V_TEXT/WEBVTT" // TODO: actually implement it
+    console.log(track);
+    if (track.kind == "subtitles") {
+        // TODO do we need to check this?
+        return track.codec == "V_TEXT/WEBVTT" || track.codec == "D_WEBVTT/SUBTITLES"
     }
     let res;
     const codec = MASTROSKA_CODEC_MAP[track.codec]
     if (!codec) return console.warn(`unknown codec: ${track.codec}`), false
-    if (track.kind.audio) {
+    if ("audio" in track.kind) {
         res = await navigator.mediaCapabilities.decodingInfo({
             type: "media-source",
             audio: {
@@ -38,7 +40,7 @@ async function test_media_capability_inner(track: SourceTrack) {
             }
         })
     }
-    if (track.kind.video) {
+    if ("video" in track.kind) {
         res = await navigator.mediaCapabilities.decodingInfo({
             type: "media-source",
             video: {
@@ -54,6 +56,7 @@ async function test_media_capability_inner(track: SourceTrack) {
 }
 
 export function track_to_content_type(track: SourceTrack): string | undefined {
+    if (track.kind == "subtitles") return "video/webm"
     const codec = MASTROSKA_CODEC_MAP[track.codec]
     if (!codec) return
     return `${get_track_kind(track.kind)}/webm; codecs="${codec}"`
@@ -76,7 +79,7 @@ export function profile_to_partial_track(profile: EncodingProfile): SourceTrack
     } else if (profile.subtitles) {
         return {
             codec: FFMPEG_ENCODER_CODEC_MAP[profile.subtitles.codec],
-            kind: { subtitles: true },
+            kind: "subtitles",
             language: "en",
             name: "test subtitle"
         }
@@ -92,6 +95,7 @@ const MASTROSKA_CODEC_MAP: { [key: string]: string } = {
     "A_OPUS": "opus",
     "A_VORBIS": "vorbis",
     "S_TEXT/WEBVTT": "webvtt",
+    "D_WEBVTT/SUBTITLES": "webvtt",
 }
 
 const FFMPEG_ENCODER_CODEC_MAP: { [key: string]: string } = {
@@ -104,10 +108,10 @@ const FFMPEG_ENCODER_CODEC_MAP: { [key: string]: string } = {
 
 export type TrackKind = "audio" | "video" | "subtitles"
 export function get_track_kind(track: SourceTrackKind): TrackKind {
-    if (track.audio) return "audio"
-    if (track.video) return "video"
     //@ts-ignore // TODO clean this mess up please
     // TODO why is the subtitle encoded diffenrently sometimes?!
     if (track == "subtitles" || track["subtitles"]) return "subtitles"
+    if ("audio" in track) return "audio"
+    if ("video" in track) return "video"
     throw new Error("invalid track");
 }
diff --git a/web/style/forms.css b/web/style/forms.css
index fca6648..f020eb6 100644
--- a/web/style/forms.css
+++ b/web/style/forms.css
@@ -35,6 +35,7 @@ fieldset {
 }
 
 input[type="submit"],
+.play,
 button {
     color: var(--font-highlight);
     padding: 0.5em;
@@ -46,10 +47,12 @@ button {
     cursor: pointer;
 }
 input[type="submit"]:disabled,
+.play,
 button:disabled {
     background-color: var(--background-disable);
 }
 input[type="submit"]:hover,
+.play,
 button:hover {
     filter: brightness(150%);
 }
diff --git a/web/style/nodepage.css b/web/style/nodepage.css
index 67642e8..2b76ede 100644
--- a/web/style/nodepage.css
+++ b/web/style/nodepage.css
@@ -47,7 +47,8 @@
 }
 .page.node .title .play {
     display: inline-block;
-    font-stretch: 200%;
+    font-size: small;
+    background-color: #52b83340;
 }
 .page.node .title .play::before {
     content: "play_arrow";
@@ -89,4 +90,4 @@
     .page.node .bigposter {
         margin: 1.5em;
     }
-}
\ No newline at end of file
+}