feat: --counting-threads flag to configure amount of threads when cou…

…nting (#67)

The efficiency of multi-threaded counting is low per core, and despite
some speedups might be desirable, one might not want to commit all cores
to this amount of waste.

gitoxide-core/src/pack/create.rs

@@ -66,12 +66,11 @@ impl From<ObjectExpansion> for pack::data::output::count::objects::ObjectExpansi
 pub struct Context<W> {
     /// The way input objects should be handled
     pub expansion: ObjectExpansion,
-    /// If set, use `tread_limit` to accelerate the counting phase at the cost of loosing determinism as the order of objects
-    /// during expansion changes with multiple threads unless no expansion is performed. In the latter case, this flag
-    /// has no effect.
-    /// If unset, counting will only use one thread and thus yield the same sequence of objects in any case.
-    /// If the `thread_limit` is 1, the count is always deterministic.
-    pub nondeterministic_count: bool,
+    /// If `Some(threads)`, use this amount of `threads` to accelerate the counting phase at the cost of loosing
+    /// determinism as the order of objects during expansion changes with multiple threads unless no expansion is performed.
+    /// In the latter case, this flag has no effect.
+    /// If `None`, counting will only use one thread and thus yield the same sequence of objects in any case.
+    pub nondeterministic_thread_count: Option<usize>,
     /// If true, delta objects may refer to their base as reference, allowing it not to be included in the created back.
     /// Otherwise these have to be recompressed in order to make the pack self-contained.
     pub thin: bool,
@@ -105,7 +104,7 @@ pub fn create<W>(
     mut progress: impl Progress,
     Context {
         expansion,
-        nondeterministic_count,
+        nondeterministic_thread_count,
         thin,
         thread_limit,
         statistics,
@@ -176,13 +175,14 @@ where
     let counts = {
         let mut progress = progress.add_child("counting");
         progress.init(None, progress::count("objects"));
-        let may_use_multiple_threads = nondeterministic_count || matches!(expansion, ObjectExpansion::None);
+        let may_use_multiple_threads =
+            nondeterministic_thread_count.is_some() || matches!(expansion, ObjectExpansion::None);
         let thread_limit = if may_use_multiple_threads {
-            thread_limit
+            nondeterministic_thread_count.or(thread_limit)
         } else {
             Some(1)
         };
-        if nondeterministic_count && !may_use_multiple_threads {
+        if nondeterministic_thread_count.is_some() && !may_use_multiple_threads {
             progress.fail("Cannot use multi-threaded counting in tree-diff object expansion mode as it may yield way too many objects.");
         }
         let (_, _, thread_count) = git::parallel::optimize_chunk_size_and_thread_limit(50, None, thread_limit, None);

src/plumbing/main.rs

@@ -158,6 +158,7 @@ pub fn main() -> Result<()> {
                 nondeterministic_count,
                 tips,
                 pack_cache_size_mb,
+                counting_threads,
                 object_cache_size_mb,
                 output_directory,
             } => {
@@ -183,7 +184,7 @@ pub fn main() -> Result<()> {
                         let context = core::pack::create::Context {
                             thread_limit,
                             thin,
-                            nondeterministic_count,
+                            nondeterministic_thread_count: nondeterministic_count.then(|| counting_threads),
                             pack_cache_size_in_bytes: pack_cache_size_mb.unwrap_or(0) * 1_000_000,
                             object_cache_size_in_bytes: object_cache_size_mb.unwrap_or(0) * 1_000_000,
                             statistics: if statistics { Some(format) } else { None },

src/plumbing/options.rs

@@ -89,6 +89,14 @@ pub mod pack {
             /// Possible values are "none" and "tree-traversal". Default is "none".
             expansion: Option<core::pack::create::ObjectExpansion>,
 
+            #[clap(long, default_value_t = 3, requires = "nondeterministic-count")]
+            /// The amount of threads to use when counting and the `--nondeterminisitc-count` flag is set, defaulting
+            /// to the globally configured threads.
+            ///
+            /// Use it to have different trade-offs between counting performance and cost in terms of CPU, as the scaling
+            /// here is everything but linear. The effectiveness of each core seems to be no more than 30%.
+            counting_threads: usize,
+
             #[clap(long)]
             /// if set, the counting phase may be accelerated using multithreading.
             ///