generate seek index

15 files changed, 155 insertions, 97 deletions

diff --git a/Cargo.lock b/Cargo.lock
index 0af1785..8422aa7 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -998,6 +998,7 @@ checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440"
 name = "jellycommon"
 version = "0.1.0"
 dependencies = [
+ "bincode 2.0.0-rc.2",
  "serde",
 ]
 
diff --git a/Cargo.toml b/Cargo.toml
index a4027f5..709b88f 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,3 +1,2 @@
 [workspace]
 members = ["server", "remuxer", "common", "tools", "matroska", "ebml_derive"]
-default-members = ["server"]
diff --git a/common/Cargo.toml b/common/Cargo.toml
index de089d6..0557282 100644
--- a/common/Cargo.toml
+++ b/common/Cargo.toml
@@ -5,3 +5,4 @@ edition = "2021"
 
 [dependencies]
 serde = { version = "1.0.152", features = ["derive"] }
+bincode = { version = "2.0.0-rc.2", features = ["derive"] }
diff --git a/common/src/lib.rs b/common/src/lib.rs
index 0587613..08517c3 100644
--- a/common/src/lib.rs
+++ b/common/src/lib.rs
@@ -1,7 +1,11 @@
 pub mod r#impl;
 
+use bincode::{Decode, Encode};
 use serde::{Deserialize, Serialize};
-use std::{collections::BTreeMap, path::PathBuf};
+use std::{
+    collections::{BTreeMap, HashMap},
+    path::PathBuf,
+};
 
 #[derive(Debug, Clone, Deserialize, Serialize)]
 pub struct DirectoryInfo {
@@ -45,3 +49,15 @@ pub enum SourceTrackKind {
     },
     Subtitles,
 }
+
+#[derive(Debug, Clone, Decode, Encode)]
+pub struct SeekIndex {
+    pub blocks: Vec<BlockIndex>,
+}
+
+#[derive(Debug, Clone, Decode, Encode)]
+pub struct BlockIndex {
+    pub pts: u64,
+    pub source_off: usize,
+    pub size: usize,
+}
diff --git a/matroska/src/bin/experiment.rs b/matroska/src/bin/experiment.rs
index 9bc667c..7787f4f 100644
--- a/matroska/src/bin/experiment.rs
+++ b/matroska/src/bin/experiment.rs
@@ -3,7 +3,7 @@
     which is licensed under the GNU Affero General Public License (version 3); see /COPYING.
     Copyright (C) 2023 metamuffin <metamuffin.org>
 */
-use jellymatroska::{matroska::MatroskaTag, read::EbmlReader, write::EbmlWriter};
+use jellymatroska::{matroska::MatroskaTag, read::EbmlReader, write::EbmlWriter, unflatten::IterWithPos};
 use std::{
     fs::File,
     io::{stdout, BufReader, BufWriter},
diff --git a/matroska/src/bin/mkvdump.rs b/matroska/src/bin/mkvdump.rs
index 6f2e063..10c697a 100644
--- a/matroska/src/bin/mkvdump.rs
+++ b/matroska/src/bin/mkvdump.rs
@@ -3,7 +3,7 @@
     which is licensed under the GNU Affero General Public License (version 3); see /COPYING.
     Copyright (C) 2023 metamuffin <metamuffin.org>
 */
-use jellymatroska::{matroska::MatroskaTag, read::EbmlReader};
+use jellymatroska::{matroska::MatroskaTag, read::EbmlReader, unflatten::IterWithPos};
 use std::{fs::File, io::BufReader};
 
 fn main() -> anyhow::Result<()> {
diff --git a/matroska/src/read.rs b/matroska/src/read.rs
index 1211351..c99f304 100644
--- a/matroska/src/read.rs
+++ b/matroska/src/read.rs
@@ -3,7 +3,7 @@
     which is licensed under the GNU Affero General Public License (version 3); see /COPYING.
     Copyright (C) 2023 metamuffin <metamuffin.org>
 */
-use crate::{matroska::MatroskaTag, size::EbmlSize, Master};
+use crate::{matroska::MatroskaTag, size::EbmlSize, unflatten::IterWithPos, Master};
 use anyhow::{anyhow, bail, Result};
 use log::{debug, warn};
 use std::{
@@ -159,9 +159,13 @@ impl EbmlReader {
     }
 }
 
-impl Iterator for EbmlReader {
+impl IterWithPos for EbmlReader {
     type Item = Result<MatroskaTag>;
 
+    fn position(&self) -> usize {
+        self.position
+    }
+
     fn next(&mut self) -> Option<Self::Item> {
         if let Some(t) = self.queue.pop_front() {
             // match t {
diff --git a/matroska/src/unflatten.rs b/matroska/src/unflatten.rs
index 5e0ba31..663eebc 100644
--- a/matroska/src/unflatten.rs
+++ b/matroska/src/unflatten.rs
@@ -6,19 +6,25 @@
 use crate::{matroska::MatroskaTag, Master};
 use anyhow::Result;
 
+pub trait IterWithPos {
+    type Item;
+    fn next(&mut self) -> Option<Self::Item>;
+    fn position(&self) -> usize;
+}
+
 pub struct Unflat<'a> {
     pub item: MatroskaTag,
     pub children: Option<Unflatten<'a>>,
 }
 
 pub struct Unflatten<'a> {
-    inner: &'a mut dyn Iterator<Item = Result<MatroskaTag>>,
+    inner: &'a mut dyn IterWithPos<Item = Result<MatroskaTag>>,
     stop: bool,
     end: Option<MatroskaTag>,
 }
 
 impl<'a> Unflatten<'a> {
-    pub fn new(inner: &'a mut dyn Iterator<Item = Result<MatroskaTag>>) -> Self {
+    pub fn new(inner: &'a mut dyn IterWithPos<Item = Result<MatroskaTag>>) -> Self {
         Self {
             inner,
             stop: false,
@@ -26,7 +32,7 @@ impl<'a> Unflatten<'a> {
         }
     }
     pub fn new_with_end(
-        inner: &'a mut dyn Iterator<Item = Result<MatroskaTag>>,
+        inner: &'a mut dyn IterWithPos<Item = Result<MatroskaTag>>,
         start: MatroskaTag,
     ) -> Self {
         Self {
@@ -36,6 +42,10 @@ impl<'a> Unflatten<'a> {
         }
     }
 
+    pub fn position(&self) -> usize {
+        self.inner.position()
+    }
+
     pub fn next(&mut self) -> Option<Result<Unflat>> {
         if self.stop {
             return None;
diff --git a/matroska/src/write.rs b/matroska/src/write.rs
index 70ee06c..ee1c44c 100644
--- a/matroska/src/write.rs
+++ b/matroska/src/write.rs
@@ -26,6 +26,31 @@ impl EbmlWriter {
         Ok(())
     }
 
+    pub fn write_padding(&mut self, position: usize) -> Result<()> {
+        let mut size = position - self.position;
+        match size {
+            0 => return Ok(()),
+            1 => bail!("this is sadly not possible"),
+            _ => (),
+        }
+        size -= 1; // subtract tag size
+        size -= 4; // subtract vint size
+
+        // match size {
+        //     _ if size < (1 << 7) => size -= 1,
+        //     _ if size < (1 << 14) => size -= 2,
+        //     _ if size < (1 << 21) => size -= 3,
+        //     _ if size < (1 << 28) => size -= 4,
+        //     _ if size < (1 << 35) => size -= 5,
+        //     _ => bail!("padding to large"),
+        // }
+
+        self.write(&[0xec])?;
+        self.write_vint_len(size.try_into().unwrap(), 4)?;
+        self.write(&vec![0; size])?;
+        Ok(())
+    }
+
     pub fn write_tag(&mut self, tag: &MatroskaTag) -> Result<()> {
         let mut buf = vec![];
         tag.write_full(&mut buf)?;
@@ -44,6 +69,9 @@ impl EbmlWriter {
             }
             len += 1;
         }
+        self.write_vint_len(i, len)
+    }
+    pub fn write_vint_len(&mut self, i: u64, len: usize) -> Result<()> {
         let mut bytes = i.to_be_bytes();
         let trunc = &mut bytes[(8 - len)..];
         trunc[0] |= 1 << (8 - len);
diff --git a/remuxer/Cargo.toml b/remuxer/Cargo.toml
index b4ee07e..f60340b 100644
--- a/remuxer/Cargo.toml
+++ b/remuxer/Cargo.toml
@@ -12,4 +12,4 @@ anyhow = "1.0.68"
 log = "0.4.17"
 
 serde = { version = "1.0.152", features = ["derive"] }
-bincode = "2.0.0-rc.2"
\ No newline at end of file
+bincode = { version = "2.0.0-rc.2", features = ["serde"] }
diff --git a/remuxer/src/format.rs b/remuxer/src/format.rs
deleted file mode 100644
index 22527f7..0000000
--- a/remuxer/src/format.rs
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
-    This file is part of jellything (https://codeberg.org/metamuffin/jellything)
-    which is licensed under the GNU Affero General Public License (version 3); see /COPYING.
-    Copyright (C) 2023 metamuffin <metamuffin.org>
-*/
-use serde::{Deserialize, Serialize};
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct TrackIndex {
-    pub clusters: Vec<ClusterIndex>,
-}
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct ClusterIndex {
-    pub timestamp: usize,
-    pub blocks: BlockIndex,
-}
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct BlockIndex {
-    pub timestamp_off: usize,
-    pub offset: usize,
-}
-
-pub struct FBlock {}
diff --git a/remuxer/src/import/mod.rs b/remuxer/src/import/mod.rs
index 65fc455..04fb88c 100644
--- a/remuxer/src/import/mod.rs
+++ b/remuxer/src/import/mod.rs
@@ -4,15 +4,15 @@
     Copyright (C) 2023 metamuffin <metamuffin.org>
 */
 use anyhow::{anyhow, bail, Result};
-use jellycommon::{ItemInfo, SourceTrack, SourceTrackKind};
+use jellycommon::{BlockIndex, ItemInfo, SeekIndex, SourceTrack, SourceTrackKind};
 use jellymatroska::{
     block::Block,
     matroska::MatroskaTag,
     read::EbmlReader,
-    unflatten::{Unflat, Unflatten},
+    unflatten::{IterWithPos, Unflat, Unflatten},
 };
 use log::{debug, error, info, trace, warn};
-use std::path::PathBuf;
+use std::{collections::HashMap, fs::File, path::PathBuf};
 
 pub fn import_read(path: &PathBuf, input: &mut EbmlReader, iteminfo: &mut ItemInfo) -> Result<()> {
     // TODO dont traverse the entire file, if the tracks are listed at the end
@@ -57,6 +57,8 @@ fn import_read_segment(
     iteminfo: &mut ItemInfo,
 ) -> Result<()> {
     let (mut timestamp_scale, mut duration) = (None, None);
+    let mut seek_index = HashMap::new();
+
     while let Some(Ok(Unflat { children, item })) = children.next() {
         match item {
             MatroskaTag::SeekHead(_) => {}
@@ -70,57 +72,8 @@ fn import_read_segment(
                     }
                 }
             }
-            MatroskaTag::Cluster(_) => {
-                info!("start of cluster found");
-                let mut children = children.unwrap();
-                while let Some(Ok(Unflat { children, item })) = children.next() {
-                    match item {
-                        MatroskaTag::BlockGroup(_) => {
-                            debug!("group");
-                            let mut children = children.unwrap();
-                            while let Some(Ok(Unflat { children: _, item })) = children.next() {
-                                match item {
-                                    MatroskaTag::Block(buf) => {
-                                        let block = Block::parse(&buf)?;
-                                        debug!(
-                                            "block: track={} tso={}",
-                                            block.track, block.timestamp_off
-                                        )
-                                    }
-                                    _ => trace!("{item:?}"),
-                                }
-                            }
-                        }
-                        MatroskaTag::SimpleBlock(buf) => {
-                            let block = Block::parse(&buf)?;
-                            debug!(
-                                "simple block: track={} tso={}",
-                                block.track, block.timestamp_off
-                            )
-                        }
-                        _ => debug!("(rsc) tag ignored: {item:?}"),
-                    }
-                }
-            }
             MatroskaTag::Tags(_) => {}
-            MatroskaTag::Cues(_) => {
-                let mut children = children.unwrap();
-                while let Some(Ok(Unflat { children, item })) = children.next() {
-                    match item {
-                        MatroskaTag::CuePoint(_) => {
-                            let mut children = children.unwrap();
-                            while let Some(Ok(Unflat {
-                                children: _,
-                                item: _,
-                            })) = children.next()
-                            {
-                                // error!("{item:?}")
-                            }
-                        }
-                        _ => (),
-                    }
-                }
-            }
+            MatroskaTag::Cues(_) => {}
             MatroskaTag::Chapters(_) => {}
             MatroskaTag::Tracks(_) => {
                 let mut children = children.unwrap();
@@ -218,9 +171,72 @@ fn import_read_segment(
                     }
                 }
             }
+            MatroskaTag::Cluster(_) => {
+                let mut children = children.unwrap();
+                let mut pts = 0;
+
+                while let Some(Ok(Unflat { children, item })) = children.next() {
+                    match item {
+                        MatroskaTag::Timestamp(ts) => pts = ts,
+                        MatroskaTag::BlockGroup(_) => {
+                            debug!("group");
+                            let mut children = children.unwrap();
+                            let pos = children.position();
+                            while let Some(Ok(Unflat { children: _, item })) = children.next() {
+                                match item {
+                                    MatroskaTag::Block(ref buf) => {
+                                        let block = Block::parse(buf)?;
+                                        debug!(
+                                            "block: track={} tso={}",
+                                            block.track, block.timestamp_off
+                                        );
+                                        seek_index
+                                            .entry(block.track)
+                                            .or_insert(SeekIndex { blocks: vec![] })
+                                            .blocks
+                                            .push(BlockIndex {
+                                                pts: pts + block.timestamp_off as u64,
+                                                source_off: pos,
+                                                size: block.data.len(),
+                                            });
+                                    }
+                                    _ => trace!("{item:?}"),
+                                }
+                            }
+                        }
+                        MatroskaTag::SimpleBlock(buf) => {
+                            let block = Block::parse(&buf)?;
+                            debug!(
+                                "simple block: track={} tso={}",
+                                block.track, block.timestamp_off
+                            );
+                            seek_index
+                                .entry(block.track)
+                                .or_insert(SeekIndex { blocks: vec![] })
+                                .blocks
+                                .push(BlockIndex {
+                                    pts: pts + block.timestamp_off as u64,
+                                    source_off: 0,
+                                    size: block.data.len(),
+                                });
+                        }
+                        _ => debug!("(rsc) tag ignored: {item:?}"),
+                    }
+                }
+            }
+
             _ => debug!("(rs) tag ignored: {item:?}"),
         }
     }
+
+    for (tn, index) in seek_index {
+        bincode::encode_into_std_write(
+            index,
+            &mut File::create(path.with_extension(&format!("si.{}", tn)))?,
+            bincode::config::standard(),
+        )?;
+    }
+
     iteminfo.duration =
         (duration.unwrap() * timestamp_scale.unwrap() as f64) as f64 / 1_000_000_000 as f64;
     Ok(())
diff --git a/remuxer/src/lib.rs b/remuxer/src/lib.rs
index be1ecc0..2c2e6d0 100644
--- a/remuxer/src/lib.rs
+++ b/remuxer/src/lib.rs
@@ -3,15 +3,14 @@
     which is licensed under the GNU Affero General Public License (version 3); see /COPYING.
     Copyright (C) 2023 metamuffin <metamuffin.org>
 */
-pub mod format;
 pub mod import;
 
-use anyhow::{anyhow, Result};
-use jellycommon::{ItemInfo, SourceTrack, SourceTrackKind};
+use anyhow::{anyhow, Context, Result};
+use jellycommon::{ItemInfo, SeekIndex, SourceTrack, SourceTrackKind};
 use jellymatroska::{
     block::Block,
     read::EbmlReader,
-    unflatten::{Unflat, Unflatten},
+    unflatten::{IterWithPos, Unflat, Unflatten},
     write::EbmlWriter,
     Master, MatroskaTag,
 };
@@ -42,6 +41,7 @@ impl RemuxerContext {
             info: SourceTrack,
             reader: EbmlReader,
             mapped: u64,
+            index: SeekIndex,
         }
 
         let mut inputs = selection
@@ -60,9 +60,16 @@ impl RemuxerContext {
                     info.track_number
                 );
                 info!("\t    {}", info);
-                let file = File::open(source_path)?;
+                let file = File::open(&source_path).context("opening source file")?;
+                let mut index = File::open(source_path.with_extension(&format!("si.{}", info.track_number)))
+                    .context("opening seek index file")?;
+                let index = bincode::decode_from_std_read::<SeekIndex, _, _>(
+                    &mut index,
+                    bincode::config::standard(),
+                )?;
                 let reader = EbmlReader::new(file);
                 Ok(ReaderC {
+                    index,
                     reader,
                     info,
                     mapped,
diff --git a/server/src/routes/stream.rs b/server/src/routes/stream.rs
index 1b2c741..4fbc990 100644
--- a/server/src/routes/stream.rs
+++ b/server/src/routes/stream.rs
@@ -15,15 +15,16 @@ use std::path::PathBuf;
 use tokio::io::{duplex, DuplexStream};
 use tokio_util::io::SyncIoBridge;
 
-pub fn stream_uri(path: &PathBuf, tracks: &Vec<u64>) -> String {
+pub fn stream_uri(path: &PathBuf, tracks: &Vec<u64>, webm: bool) -> String {
     format!(
-        "/stream/{}?tracks={}",
+        "/stream/{}?tracks={}&webm={}",
         path.to_str().unwrap().to_string(),
         tracks
             .iter()
             .map(|v| format!("{v}"))
             .collect::<Vec<_>>()
-            .join(",")
+            .join(","),
+        if webm { "1" } else { "0" }
     )
 }
 
diff --git a/server/src/routes/ui/player.rs b/server/src/routes/ui/player.rs
index 4b66f04..ed7e4f7 100644
--- a/server/src/routes/ui/player.rs
+++ b/server/src/routes/ui/player.rs
@@ -49,7 +49,7 @@ pub fn r_player(
     Ok(LayoutPage {
         title: item.info.title.to_owned(),
         content: markup::new! {
-            video[src=stream_uri(&item.lib_path, &tracks), controls];
+            video[src=stream_uri(&item.lib_path, &tracks, true), controls];
         },
     })
 }