parallel import indexing

1 files changed, 9 insertions, 7 deletions

diff --git a/import/src/lib.rs b/import/src/lib.rs
index 561a5c9..3156b5a 100644
--- a/import/src/lib.rs
+++ b/import/src/lib.rs
@@ -23,7 +23,7 @@ use jellyremuxer::{
     demuxers::create_demuxer_autodetect,
     matroska::{self, AttachedFile, Segment},
 };
-use rayon::iter::{IntoParallelIterator, ParallelIterator};
+use rayon::iter::{IntoParallelIterator, ParallelBridge, ParallelIterator};
 use serde::{Deserialize, Serialize};
 use std::{
     fs::{File, read_to_string},
@@ -98,15 +98,16 @@ pub async fn import_wrap(db: Database, incremental: bool) -> Result<()> {
 fn import(db: &Database, incremental: bool) -> Result<()> {
     let plugins = init_plugins(&CONF.api);
 
-    let mut files = Vec::new();
+    let files = Mutex::new(Vec::new());
     import_traverse(
         &CONF.media_path,
         db,
         incremental,
         NodeID::MIN,
         InheritedFlags::default(),
-        &mut files,
+        &files,
     )?;
+    let files = files.into_inner().unwrap();
 
     let rt = Handle::current();
 
@@ -142,7 +143,7 @@ fn import_traverse(
     incremental: bool,
     parent: NodeID,
     mut iflags: InheritedFlags,
-    out: &mut Vec<(PathBuf, NodeID, InheritedFlags)>,
+    out: &Mutex<Vec<(PathBuf, NodeID, InheritedFlags)>>,
 ) -> Result<()> {
     if path.is_dir() {
         reporting::set_task(format!("indexing {path:?}"));
@@ -171,13 +172,14 @@ fn import_traverse(
             Ok(())
         })?;
 
-        for e in path.read_dir()? {
+        path.read_dir()?.par_bridge().try_for_each(|e| {
             let path = e?.path();
             reporting::catch(
                 import_traverse(&path, db, incremental, node, iflags, out)
                     .context(anyhow!("index {:?}", path.file_name().unwrap())),
             );
-        }
+            anyhow::Ok(())
+        })?;
         return Ok(());
     }
 
@@ -198,7 +200,7 @@ fn import_traverse(
             .as_mut()
             .unwrap()
             .total_items += 1;
-        out.push((path.to_owned(), parent, iflags));
+        out.lock().unwrap().push((path.to_owned(), parent, iflags));
     }
     Ok(())
 }