1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
|
use anyhow::Result;
use indicatif::{ProgressBar, ProgressIterator, ProgressStyle};
use serde::{Deserialize, Serialize};
use std::{io::{BufRead, BufReader, copy, Cursor}, path::PathBuf, process::{Command, Stdio}};
use crate::{Config, BatchEmbedder, MetricElem};
#[repr(transparent)]
#[derive(Serialize, Deserialize)]
pub(crate) struct Imgbedding (Vec<f32>); // TODO das hier zu einem const size slice machen
impl MetricElem for Imgbedding {
fn dist(&self, other: &Self) -> f64 {
self.0.iter().zip(other.0.iter())
.map(|(a, b)| (a - b).powf(2.))
.sum::<f32>()
.sqrt() as f64
}
}
pub(crate) struct ContentEmbedder<'a> {
cfg: &'a Config,
}
impl<'a> ContentEmbedder<'a> {
pub(crate) fn new(cfg: &'a Config) -> Self {
ContentEmbedder { cfg: cfg }
}
}
impl<'a> Drop for ContentEmbedder<'a> {
fn drop(&mut self) {
self.cfg.base_dirs.place_runtime_file("imgbeddings-api.py")
.iter()
.for_each(|p| { let _ = std::fs::remove_file(&p); });
}
}
impl BatchEmbedder for ContentEmbedder<'_> {
type Embedding = Imgbedding;
const NAME: &'static str = "imgbeddings";
fn embeds(&mut self, paths: &[PathBuf]) -> Result<Vec<Self::Embedding>> {
let venv_dir = self.cfg.base_dirs
.create_data_directory("imgbeddings-venv")?;
let script_file = self.cfg.base_dirs
.place_runtime_file("imgbeddings-api.py")?;
let api_prog = include_bytes!("imgbeddings-api.py");
copy(&mut Cursor::new(api_prog), &mut std::fs::File::create(&script_file)?)?;
let bar = ProgressBar::new_spinner();
bar.set_style(ProgressStyle::with_template("{spinner} {msg}")?);
bar.enable_steady_tick(std::time::Duration::from_millis(100));
bar.set_message("Creating venv...");
Command::new("python3")
.args(["-m", "venv", venv_dir.to_str().unwrap()])
.stdout(Stdio::null())
.spawn()?
.wait()?;
bar.set_message("Installing/checking packages...");
Command::new(venv_dir.join("bin/pip3"))
.args(["install", "imgbeddings"])
.stdout(Stdio::null())
.spawn()?
.wait()?;
bar.finish();
let child = Command::new(venv_dir.join("bin/python3"))
.arg(script_file)
.args(paths)
.stderr(Stdio::null())
.stdout(Stdio::piped())
.spawn()?;
Ok(BufReader::new(child.stdout.unwrap())
.lines()
.progress_count(paths.len().try_into().unwrap())
.map(|l| Ok::<_, anyhow::Error>(serde_json::from_str(&l?)?))
.try_collect()?)
}
}
|
