diff options
| author | metamuffin <metamuffin@disroot.org> | 2026-02-06 01:35:44 +0100 |
|---|---|---|
| committer | metamuffin <metamuffin@disroot.org> | 2026-02-06 01:35:44 +0100 |
| commit | c914529348b8aa74a142b12f2a3b7532350d3f04 (patch) | |
| tree | 643f6b9f51835212355f0662d8aa9312896e97c2 | |
| parent | 17be68281eae0be371be63db4c59d4ecaf2f1ba4 (diff) | |
| download | jellything-c914529348b8aa74a142b12f2a3b7532350d3f04.tar jellything-c914529348b8aa74a142b12f2a3b7532350d3f04.tar.bz2 jellything-c914529348b8aa74a142b12f2a3b7532350d3f04.tar.zst | |
index key serialization
| -rw-r--r-- | database/src/kv/binning.rs | 17 | ||||
| -rw-r--r-- | database/src/kv/counters.rs | 14 | ||||
| -rw-r--r-- | database/src/kv/index_key.rs | 137 | ||||
| -rw-r--r-- | database/src/kv/mod.rs | 296 | ||||
| -rw-r--r-- | database/src/kv/sort/mod.rs | 20 | ||||
| -rw-r--r-- | database/src/kv/sort/none.rs | 6 | ||||
| -rw-r--r-- | database/src/kv/sort/value.rs | 6 | ||||
| -rw-r--r-- | database/src/lib.rs | 6 | ||||
| -rw-r--r-- | server/src/main.rs | 6 |
9 files changed, 292 insertions, 216 deletions
diff --git a/database/src/kv/binning.rs b/database/src/kv/binning.rs index 7bec294..16c88d4 100644 --- a/database/src/kv/binning.rs +++ b/database/src/kv/binning.rs @@ -4,13 +4,12 @@ Copyright (C) 2026 metamuffin <metamuffin.org> */ -use jellyobject::{Object, Path}; - use crate::Filter; +use jellyobject::{Object, Path}; -/// Sorted list of components to bin objects by filterable values. +/// Sorted list of components to bin objects by filtered values. #[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct Binning(Vec<BinningComponent>); +pub struct Binning(pub Vec<BinningComponent>); #[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)] pub enum BinningComponent { @@ -33,9 +32,8 @@ impl BinningComponent { pub fn apply(&self, ob: Object<'_>, keys: &mut Vec<Vec<u8>>) { match self { BinningComponent::Has(path) => { - let has = path.get_matching_value(ob).is_some(); - for co in keys { - co.push(has as u8) + if path.get_matching_value(ob).is_none() { + keys.clear(); } } BinningComponent::Match(path) => { @@ -85,7 +83,10 @@ impl Filter { mod test { use jellyobject::{Path, Tag}; - use crate::{Filter, kv::binning::{Binning, BinningComponent}}; + use crate::{ + Filter, + kv::binning::{Binning, BinningComponent}, + }; #[test] fn all() { diff --git a/database/src/kv/counters.rs b/database/src/kv/counters.rs index fae7b42..bcbe6a0 100644 --- a/database/src/kv/counters.rs +++ b/database/src/kv/counters.rs @@ -5,13 +5,13 @@ */ use crate::{ Query, RowNum, - kv::{self, TableNum, binning::Binning}, + kv::{SubtreeNum, binning::Binning}, }; use anyhow::Result; use jellyobject::Object; use std::collections::HashMap; -pub(crate) struct Counters(pub HashMap<Binning, TableNum>); +pub(crate) struct Counters(pub HashMap<Binning, SubtreeNum>); impl Counters { pub fn update( &self, @@ -25,7 +25,7 @@ impl Counters { if k.is_empty() { continue; } - let mut counter = read_counter(txn, table)?; + let mut counter = read_counter(txn, table, 0)?; if remove { counter += k.len() as u64; } else { @@ -41,19 +41,19 @@ impl Counters { let Some(b) = self.0.get(&binning) else { return Ok(None); }; - total += read_counter(txn, *b)?; + total += read_counter(txn, *b, 0)?; } Ok(Some(total)) } } -pub fn write_counter(txn: &mut dyn jellykv::Transaction, t: TableNum, value: u64) -> Result<()> { +pub fn write_counter(txn: &mut dyn jellykv::Transaction, t: SubtreeNum, value: u64) -> Result<()> { txn.set(&t.to_be_bytes(), &value.to_be_bytes()) } -pub fn read_counter(txn: &dyn jellykv::Transaction, t: TableNum) -> Result<u64> { +pub fn read_counter(txn: &dyn jellykv::Transaction, t: SubtreeNum, default: u64) -> Result<u64> { Ok(txn .get(&t.to_be_bytes())? .map(|k| k.as_slice().try_into().map(RowNum::from_be_bytes).ok()) .flatten() - .unwrap_or(0)) + .unwrap_or(default)) } diff --git a/database/src/kv/index_key.rs b/database/src/kv/index_key.rs new file mode 100644 index 0000000..55aa5e8 --- /dev/null +++ b/database/src/kv/index_key.rs @@ -0,0 +1,137 @@ +/* + This file is part of jellything (https://codeberg.org/metamuffin/jellything) + which is licensed under the GNU Affero General Public License (version 3); see /COPYING. + Copyright (C) 2026 metamuffin <metamuffin.org> +*/ +use crate::{ + MultiBehaviour, Sort, + kv::binning::{Binning, BinningComponent}, +}; +use jellyobject::{Path, Tag}; + +pub struct IndexKey(Binning, SortKey); + +#[derive(Hash, PartialEq, Eq)] +pub enum SortKey { + None, + Count, + Value(Path, MultiBehaviour), + Text(Path), +} + +impl Sort { + pub fn key(&self) -> SortKey { + match self { + Sort::None => SortKey::None, + Sort::Value(vs) => SortKey::Value(vs.path.clone(), vs.multi), + Sort::TextSearch(p, _) => SortKey::Text(p.to_owned()), + } + } +} + +impl IndexKey { + pub fn to_bytes(&self) -> Vec<u8> { + let mut out = Vec::new(); + self.0.write(&mut out); + self.1.write(&mut out); + out + } + pub fn from_bytes(mut b: &[u8]) -> Self { + let binning = Binning::read(&mut b); + let sort = SortKey::read(&mut b); + assert!(b.is_empty()); + Self(binning, sort) + } +} +impl Binning { + fn read(b: &mut &[u8]) -> Self { + let len = b[0]; + *b = &b[1..]; + let mut o = Vec::new(); + for _ in 0..len { + let ty = b[0]; + *b = &b[1..]; + o.push(match ty { + 0 => BinningComponent::Has(read_path(b)), + 1 => BinningComponent::Match(read_path(b)), + _ => unreachable!(), + }); + } + Self(o) + } + fn write(&self, out: &mut Vec<u8>) { + assert!(out.len() < 256); + out.push(self.0.len() as u8); + for b in &self.0 { + match b { + BinningComponent::Has(path) => { + out.push(0); + write_path(path, out); + } + BinningComponent::Match(path) => { + out.push(1); + write_path(path, out); + } + } + } + } +} +impl SortKey { + fn read(b: &mut &[u8]) -> Self { + let ty = b[0]; + match ty { + 0 => SortKey::None, + 1 => SortKey::Count, + 2 => SortKey::Value( + read_path(b), + match b[0] { + 0 => MultiBehaviour::First, + 1 => MultiBehaviour::ForEach, + 2 => MultiBehaviour::Max, + 3 => MultiBehaviour::Min, + 4 => MultiBehaviour::Count, + _ => unreachable!(), + }, + ), + 3 => SortKey::Text(read_path(b)), + _ => unreachable!(), + } + } + fn write(&self, out: &mut Vec<u8>) { + match self { + SortKey::None => out.push(0), + SortKey::Count => out.push(1), + SortKey::Value(path, multi_behaviour) => { + write_path(path, out); + out.push(match multi_behaviour { + MultiBehaviour::First => 0, + MultiBehaviour::ForEach => 1, + MultiBehaviour::Max => 2, + MultiBehaviour::Min => 3, + MultiBehaviour::Count => 4, + }); + } + SortKey::Text(path) => { + write_path(path, out); + } + } + } +} + +fn write_path(path: &Path, out: &mut Vec<u8>) { + assert!(path.0.len() < 256); + out.push(path.0.len() as u8); + for c in &path.0 { + out.extend(c.0.to_be_bytes()); + } +} +fn read_path(b: &mut &[u8]) -> Path { + let len = b[0]; + *b = &b[1..]; + let mut o = Vec::new(); + for _ in 0..len { + o.push(Tag(u32::from_be_bytes([b[0], b[1], b[2], b[3]]))); + *b = &b[4..]; + } + Path(o) +} diff --git a/database/src/kv/mod.rs b/database/src/kv/mod.rs index b4bd000..9408916 100644 --- a/database/src/kv/mod.rs +++ b/database/src/kv/mod.rs @@ -6,212 +6,168 @@ pub mod binning; pub mod counters; +pub mod index_key; pub mod prefix_iterator; pub mod sort; use crate::{ Database, Query, RowIter, RowNum, Transaction, - kv::{binning::Binning, sort::SortKey}, + kv::counters::{read_counter, write_counter}, }; -use anyhow::Result; +use anyhow::{Result, anyhow}; use jellyobject::ObjectBuffer; -pub type IndexKey = (Binning, SortKey); -pub type TableNum = u32; +pub type SubtreeNum = u32; + +const T_ROW_COUNTER: SubtreeNum = 0; +const T_ROWS: SubtreeNum = 1; +const T_INDEX_COUNTER: SubtreeNum = 2; +const T_INDICES: SubtreeNum = 3; + +const INDEX_TNUM_OFFSET: SubtreeNum = 64; impl<T: jellykv::Store> Database for T { fn transaction(&self, f: &mut dyn FnMut(&mut dyn Transaction) -> Result<()>) -> Result<()> { - todo!() + jellykv::Store::transaction(self, &mut |mut txn| f(&mut txn)) } } -impl<T: jellykv::Transaction> Transaction for T { - fn insert(&self, entry: ObjectBuffer) -> Result<RowNum> { - todo!() - } +impl Transaction for &mut dyn jellykv::Transaction { + fn insert(&mut self, entry: ObjectBuffer) -> Result<RowNum> { + let mut id_counter = read_counter(*self, T_ROW_COUNTER, 0)?; + let row = id_counter; + id_counter += 1; + write_counter(*self, T_ROW_COUNTER, id_counter)?; - fn get(&self, row: RowNum) -> Result<Option<ObjectBuffer>> { - todo!() + jellykv::Transaction::set( + *self, + &row_key(row), + bytemuck::cast_slice(entry.0.as_slice()), + )?; + + let ob = entry.as_object(); + // for idx in self.indices.values() { + // idx.add(txn, row, ob)?; + // } + + Ok(row) + } + fn remove(&mut self, row: RowNum) -> Result<()> { + let entry = self.get(row)?.ok_or(anyhow!("row did not exist"))?; + let ob = entry.as_object(); + // for index in self.indices.values() { + // index.remove(db, row, ob)?; + // } + jellykv::Transaction::del(*self, &row_key(row)) } + fn update(&mut self, row: RowNum, entry: ObjectBuffer) -> Result<()> { + let before = self.get(row)?.ok_or(anyhow!("row to update missing"))?; + let before = before.as_object(); + let after = entry.as_object(); - fn remove(&self, row: RowNum) -> Result<()> { - todo!() + jellykv::Transaction::set( + *self, + &row_key(row), + bytemuck::cast_slice(entry.0.as_slice()), + )?; + + // for index in self.indices.values() { + // if !index.compare(before, after) { + // index.remove(txn, row, before)?; + // index.add(txn, row, after)?; + // } + // } + + Ok(()) } - fn update(&self, row: RowNum, entry: ObjectBuffer) -> Result<()> { - todo!() + fn get(&self, row: RowNum) -> Result<Option<ObjectBuffer>> { + Ok(jellykv::Transaction::get(*self, &row_key(row))?.map(ObjectBuffer::from)) } fn query(&self, query: Query) -> Result<RowIter> { + // query + // .filter + // .get_bins() + // .into_iter() + // .flat_map(|b| { + // let ikey = (b, query.sort.key()); + // self.indices.get(&ikey) + // }) + // .map(|i| i.query(txn, &query.sort)) todo!() } - fn query_single(&self, query: Query) -> Result<Option<RowNum>> { - todo!() + self.query(query)?.next().transpose() } - fn count(&self, query: Query) -> Result<u64> { - todo!() + let mut total = 0; + // for b in query.filter.get_bins() { + // if let Some(&c) = self.counters.get(&b) { + // total += read_counter(txn, c)?; + // } + // } + Ok(total) } } -// pub struct Table { -// id: TableNum, -// pub(crate) counters: HashMap<Binning, TableNum>, -// pub(crate) indices: HashMap<IndexKey, TableNum>, -// } -// impl Table { -// pub fn new(id: u32) -> Self { -// Self { -// id, -// counters: HashMap::new(), -// indices: HashMap::new(), -// } -// } -// fn key(&self, row: RowNum) -> Vec<u8> { -// let mut key = Vec::new(); -// key.extend(self.id.to_be_bytes()); -// key.extend(row.to_be_bytes()); -// key -// } -// pub fn iter(&self, txn: &dyn ReadTransaction) -> Result<impl Iterator<Item = Result<RowNum>>> { -// Ok(PrefixIterator { -// inner: txn.iter(&self.id.to_be_bytes(), false)?, -// prefix: self.id.to_be_bytes().to_vec().into(), -// } -// .map(|r| r.map(|r| RowNum::from_be_bytes(r.try_into().unwrap())))) -// } -// pub fn insert(&self, txn: &mut dyn WriteTransaction, entry: ObjectBuffer) -> Result<RowNum> { -// let mut id_counter = read_counter(txn, self.id)?; -// let row = id_counter; -// id_counter += 1; -// write_counter(txn, self.id, id_counter)?; - -// txn.set(&self.key(row), bytemuck::cast_slice(entry.0.as_slice()))?; - -// let ob = entry.as_object(); -// // for idx in self.indices.values() { -// // idx.add(txn, row, ob)?; -// // } - -// Ok(row) -// } -// pub fn get(&self, txn: &dyn ReadTransaction, row: RowNum) -> Result<Option<ObjectBuffer>> { -// Ok(txn.get(&self.key(row))?.map(ObjectBuffer::from)) -// } -// pub fn remove(&self, db: &mut dyn WriteTransaction, row: RowNum) -> Result<bool> { -// let Some(entry) = self.get(db, row)? else { -// return Ok(false); -// }; -// let ob = entry.as_object(); -// // for index in self.indices.values() { -// // index.remove(db, row, ob)?; -// // } -// db.del(&self.key(row))?; -// Ok(true) -// } -// pub fn update( -// &self, -// txn: &mut dyn WriteTransaction, -// row: RowNum, -// entry: ObjectBuffer, -// ) -> Result<()> { -// let before = self -// .get(txn, row)? -// .ok_or(anyhow!("row to update missing"))?; -// let before = before.as_object(); -// let after = entry.as_object(); - -// txn.set(&self.key(row), bytemuck::cast_slice(entry.0.as_slice()))?; - -// // for index in self.indices.values() { -// // if !index.compare(before, after) { -// // index.remove(txn, row, before)?; -// // index.add(txn, row, after)?; -// // } -// // } - -// Ok(()) -// } -// pub fn query(&self, txn: &dyn ReadTransaction, query: Query) -> Result<RowIter> { -// // query -// // .filter -// // .get_bins() -// // .into_iter() -// // .flat_map(|b| { -// // let ikey = (b, query.sort.key()); -// // self.indices.get(&ikey) -// // }) -// // .map(|i| i.query(txn, &query.sort)) -// todo!() -// } -// pub fn query_single(&self, txn: &dyn ReadTransaction, query: Query) -> Result<Option<RowNum>> { -// self.query(txn, query)?.next().transpose() -// } -// pub fn count(&self, txn: &dyn ReadTransaction, query: Query) -> Result<u64> { -// let mut total = 0; -// for b in query.filter.get_bins() { -// if let Some(&c) = self.counters.get(&b) { -// total += read_counter(txn, c)?; -// } -// } -// Ok(total) -// } -// } +fn row_key(row: RowNum) -> Vec<u8> { + let mut key = Vec::new(); + key.extend(T_ROWS.to_be_bytes()); + key.extend(row.to_be_bytes()); + key +} -// #[cfg(test)] -// mod test { -// use { -// table::Table, -// test_shared::{NAME, new_bob}, -// }; -// use anyhow::Result; -// use jellykv::Database; +#[cfg(test)] +mod test { + use crate::{ + Database, + test_shared::{NAME, new_bob}, + }; + use anyhow::Result; -// #[test] -// pub fn insert_get() -> Result<()> { -// let db = jellykv::memory::new(); -// let table = Table::new(5); + #[test] + pub fn insert_get() -> Result<()> { + let db = jellykv::memory::new(); -// let mut bob_row = 0; -// db.write_transaction(&mut |txn| { -// bob_row = table.insert(txn, new_bob())?; -// Ok(()) -// })?; + let mut bob_row = 0; + db.transaction(&mut |txn| { + bob_row = txn.insert(new_bob())?; + Ok(()) + })?; -// let mut bob = None; -// db.read_transaction(&mut |txn| { -// bob = table.get(txn, bob_row)?; -// Ok(()) -// })?; + let mut bob = None; + db.transaction(&mut |txn| { + bob = txn.get(bob_row)?; + Ok(()) + })?; -// assert_eq!(bob.unwrap().as_object().get(NAME).unwrap(), "Bob"); -// Ok(()) -// } + assert_eq!(bob.unwrap().as_object().get(NAME).unwrap(), "Bob"); + Ok(()) + } -// #[test] -// pub fn update() -> Result<()> { -// let db = jellykv::memory::new(); -// let table = Table::new(5); + #[test] + pub fn update() -> Result<()> { + let db = jellykv::memory::new(); -// let mut bob_row = 0; -// let mut bob = None; + let mut bob_row = 0; + let mut bob = None; -// db.write_transaction(&mut |txn| { -// bob_row = table.insert(txn, new_bob())?; -// Ok(()) -// })?; -// db.write_transaction(&mut |txn| { -// let better_bob = new_bob().as_object().insert(NAME, "Better Bob"); -// table.update(txn, bob_row, better_bob)?; -// Ok(()) -// })?; -// db.read_transaction(&mut |txn| { -// bob = table.get(txn, bob_row)?; -// Ok(()) -// })?; + db.transaction(&mut |txn| { + bob_row = txn.insert(new_bob())?; + Ok(()) + })?; + db.transaction(&mut |txn| { + let better_bob = new_bob().as_object().insert(NAME, "Better Bob"); + txn.update(bob_row, better_bob)?; + Ok(()) + })?; + db.transaction(&mut |txn| { + bob = txn.get(bob_row)?; + Ok(()) + })?; -// assert_eq!(bob.unwrap().as_object().get(NAME).unwrap(), "Better Bob"); -// Ok(()) -// } -// } + assert_eq!(bob.unwrap().as_object().get(NAME).unwrap(), "Better Bob"); + Ok(()) + } +} diff --git a/database/src/kv/sort/mod.rs b/database/src/kv/sort/mod.rs index 403ba73..8b814cd 100644 --- a/database/src/kv/sort/mod.rs +++ b/database/src/kv/sort/mod.rs @@ -4,26 +4,6 @@ Copyright (C) 2026 metamuffin <metamuffin.org> */ -use jellyobject::Path; - -use crate::{MultiBehaviour, Sort}; pub mod none; pub mod value; - -#[derive(Hash, PartialEq, Eq)] -pub enum SortKey { - None, - Value(Path, MultiBehaviour), - Text(Path), -} - -impl Sort { - pub fn key(&self) -> SortKey { - match self { - Sort::None => SortKey::None, - Sort::Value(vs) => SortKey::Value(vs.path.clone(), vs.multi), - Sort::TextSearch(p, _) => SortKey::Text(p.to_owned()), - } - } -} diff --git a/database/src/kv/sort/none.rs b/database/src/kv/sort/none.rs index b4d5db2..4044919 100644 --- a/database/src/kv/sort/none.rs +++ b/database/src/kv/sort/none.rs @@ -6,17 +6,17 @@ use crate::{ RowNum, - kv::{TableNum, binning::Binning}, + kv::{SubtreeNum, binning::Binning}, }; use jellyobject::Object; pub struct UnsortedIndex { - id: TableNum, + id: SubtreeNum, binning: Binning, } impl UnsortedIndex { - pub fn new(id: TableNum, binning: Binning) -> Self { + pub fn new(id: SubtreeNum, binning: Binning) -> Self { Self { id, binning } } fn keys(&self, id: RowNum, ob: Object) -> Vec<Vec<u8>> { diff --git a/database/src/kv/sort/value.rs b/database/src/kv/sort/value.rs index 0d4ceb7..5637ac7 100644 --- a/database/src/kv/sort/value.rs +++ b/database/src/kv/sort/value.rs @@ -8,17 +8,17 @@ use jellyobject::Object; use crate::{ MultiBehaviour, RowNum, ValueSort, - kv::{TableNum, binning::Binning}, + kv::{SubtreeNum, binning::Binning}, }; pub struct ValueIndex { - id: TableNum, + id: SubtreeNum, binning: Binning, sort: ValueSort, } impl ValueIndex { - pub fn new(id: TableNum, binning: Binning, sort: ValueSort) -> Self { + pub fn new(id: SubtreeNum, binning: Binning, sort: ValueSort) -> Self { Self { id, binning, sort } } fn keys(&self, id: RowNum, ob: Object) -> Vec<Vec<u8>> { diff --git a/database/src/lib.rs b/database/src/lib.rs index 8e13b1b..da15d86 100644 --- a/database/src/lib.rs +++ b/database/src/lib.rs @@ -18,10 +18,10 @@ pub trait Database: Send + Sync { } pub trait Transaction { - fn insert(&self, entry: ObjectBuffer) -> Result<RowNum>; + fn insert(&mut self, entry: ObjectBuffer) -> Result<RowNum>; + fn remove(&mut self, row: RowNum) -> Result<()>; + fn update(&mut self, row: RowNum, entry: ObjectBuffer) -> Result<()>; fn get(&self, row: RowNum) -> Result<Option<ObjectBuffer>>; - fn remove(&self, row: RowNum) -> Result<()>; - fn update(&self, row: RowNum, entry: ObjectBuffer) -> Result<()>; fn query(&self, query: Query) -> Result<RowIter>; fn query_single(&self, query: Query) -> Result<Option<RowNum>>; fn count(&self, query: Query) -> Result<u64>; diff --git a/server/src/main.rs b/server/src/main.rs index 37d71f2..db02c29 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -8,7 +8,7 @@ #![recursion_limit = "4096"] use crate::{auth::token::SessionKey, logger::setup_logger}; -use anyhow::Result; +use anyhow::{Result, anyhow}; use jellycache::Cache; use jellydb::Database; use log::{error, info}; @@ -67,7 +67,9 @@ pub struct Config { } pub fn create_state() -> Result<Arc<State>> { - let config_path = args().nth(1).unwrap(); + let config_path = args() + .nth(1) + .ok_or(anyhow!("first argument (config path) missing"))?; let config: Config = serde_yaml_ng::from_str(&read_to_string(config_path)?)?; let cache_storage = jellykv::rocksdb::new(&config.cache_path)?; |