Fast-path multi-pack index verification in the CLI (#279)

GitoxideLabs · Jan 1, 2022 · bcde935 · bcde935
1 parent 7517482
commit bcde935
Show file tree

Hide file tree

Showing 5 changed files with 50 additions and 18 deletions.
diff --git a/gitoxide-core/src/pack/multi_index.rs b/gitoxide-core/src/pack/multi_index.rs
@@ -7,6 +7,11 @@ use git_repository as git;
 
 pub const PROGRESS_RANGE: std::ops::RangeInclusive<u8> = 1..=3;
 
+pub fn verify(multi_index_path: PathBuf, progress: impl Progress, should_interrupt: &AtomicBool) -> anyhow::Result<()> {
+    git::odb::pack::multi_index::File::at(multi_index_path)?.verify_integrity_fast(progress, should_interrupt)?;
+    Ok(())
+}
+
 pub fn create(
     index_paths: Vec<PathBuf>,
     output_path: PathBuf,

diff --git a/src/plumbing/main.rs b/src/plumbing/main.rs
@@ -243,6 +243,20 @@ pub fn main() -> Result<()> {
             )
             .map(|_| ()),
             pack::Subcommands::MultiIndex(subcommands) => match subcommands {
+                pack::multi_index::Subcommands::Verify { multi_index_path } => prepare_and_run(
+                    "pack-multi-index-verify",
+                    verbose,
+                    progress,
+                    progress_keep_open,
+                    core::pack::multi_index::PROGRESS_RANGE,
+                    move |progress, _out, _err| {
+                        core::pack::multi_index::verify(
+                            multi_index_path,
+                            progress,
+                            &git_repository::interrupt::IS_INTERRUPTED,
+                        )
+                    },
+                ),
                 pack::multi_index::Subcommands::Create {
                     output_path,
                     index_paths,

diff --git a/src/plumbing/options.rs b/src/plumbing/options.rs
@@ -248,7 +248,13 @@ pub mod pack {
 
         #[derive(Debug, clap::Parser)]
         pub enum Subcommands {
-            /// create a multi-pack index from one or more pack index files
+            /// Verify a multi-index quickly without inspecting objects themselves
+            #[clap(setting = AppSettings::DisableVersionFlag)]
+            Verify {
+                /// The path to the multi-pack-index to verify.
+                multi_index_path: PathBuf,
+            },
+            /// Create a multi-pack index from one or more pack index files
             #[clap(setting = AppSettings::DisableVersionFlag)]
             Create {
                 /// The path to which the multi-index file should be written, overwriting any possibly existing file.

diff --git a/tests/journey/gix.sh b/tests/journey/gix.sh
@@ -384,99 +384,106 @@ title "gix pack"
       }
     )
     (with "a valid pack INDEX file"
-      PACK_INDEX_FILE="$fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx"
+      MULTI_PACK_INDEX="$fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx"
       (with "no statistics"
         it "verifies the pack index successfully and with desired output" && {
           WITH_SNAPSHOT="$snapshot/index-success" \
-          expect_run $SUCCESSFULLY "$exe_plumbing" pack verify "$PACK_INDEX_FILE"
+          expect_run $SUCCESSFULLY "$exe_plumbing" pack verify "$MULTI_PACK_INDEX"
         }
       )
       (with "statistics"
         it "verifies the pack index successfully and with desired output" && {
           WITH_SNAPSHOT="$snapshot/index-with-statistics-success" \
-          expect_run $SUCCESSFULLY "$exe_plumbing" pack verify --statistics "$PACK_INDEX_FILE"
+          expect_run $SUCCESSFULLY "$exe_plumbing" pack verify --statistics "$MULTI_PACK_INDEX"
         }
 
         (with "and the less-memory algorithm"
           it "verifies the pack index successfully and with desired output" && {
             WITH_SNAPSHOT="$snapshot/index-with-statistics-success" \
-            expect_run $SUCCESSFULLY "$exe_plumbing" pack verify --algorithm less-memory --statistics "$PACK_INDEX_FILE"
+            expect_run $SUCCESSFULLY "$exe_plumbing" pack verify --algorithm less-memory --statistics "$MULTI_PACK_INDEX"
           }
         )
       )
       (with "decode"
         it "verifies the pack index successfully and with desired output, and decodes all objects" && {
           WITH_SNAPSHOT="$snapshot/index-success" \
-          expect_run $SUCCESSFULLY "$exe_plumbing" pack verify --algorithm less-memory --decode "$PACK_INDEX_FILE"
+          expect_run $SUCCESSFULLY "$exe_plumbing" pack verify --algorithm less-memory --decode "$MULTI_PACK_INDEX"
         }
       )
       (with "re-encode"
         it "verifies the pack index successfully and with desired output, and re-encodes all objects" && {
           WITH_SNAPSHOT="$snapshot/index-success" \
-          expect_run $SUCCESSFULLY "$exe_plumbing" pack verify --algorithm less-time --re-encode "$PACK_INDEX_FILE"
+          expect_run $SUCCESSFULLY "$exe_plumbing" pack verify --algorithm less-time --re-encode "$MULTI_PACK_INDEX"
         }
       )
       if test "$kind" = "max"; then
       (with "statistics (JSON)"
         it "verifies the pack index successfully and with desired output" && {
           WITH_SNAPSHOT="$snapshot/index-with-statistics-json-success" \
-          expect_run $SUCCESSFULLY "$exe_plumbing" --format json --threads 1 pack verify --statistics "$PACK_INDEX_FILE"
+          expect_run $SUCCESSFULLY "$exe_plumbing" --format json --threads 1 pack verify --statistics "$MULTI_PACK_INDEX"
         }
       )
       fi
     )
     (with "a valid multi-pack index"
       snapshot="$snapshot/multi-index"
       (sandbox
-        PACK_INDEX_FILE=multi-pack-index
+        MULTI_PACK_INDEX=multi-pack-index
         cp $fixtures/packs/pack-* .
-        $exe_plumbing pack multi-index create *.idx -o $PACK_INDEX_FILE
+        $exe_plumbing pack multi-index create *.idx -o $MULTI_PACK_INDEX
+
+        (when "using fast validation via 'pack multi-index verify'"
+          it "verifies the pack index successfully and with desired output" && {
+            WITH_SNAPSHOT="$snapshot/fast-index-success" \
+            expect_run $SUCCESSFULLY "$exe_plumbing" pack multi-index verify "$MULTI_PACK_INDEX"
+          }
+        )
 
         (with "no statistics"
           it "verifies the pack index successfully and with desired output" && {
             WITH_SNAPSHOT="$snapshot/index-success" \
-            expect_run $SUCCESSFULLY "$exe_plumbing" pack verify "$PACK_INDEX_FILE"
+            expect_run $SUCCESSFULLY "$exe_plumbing" pack verify "$MULTI_PACK_INDEX"
           }
         )
         (with "statistics"
           it "verifies the pack index successfully and with desired output" && {
             WITH_SNAPSHOT="$snapshot/index-with-statistics-success" \
-            expect_run $SUCCESSFULLY "$exe_plumbing" pack verify --statistics "$PACK_INDEX_FILE"
+            expect_run $SUCCESSFULLY "$exe_plumbing" pack verify --statistics "$MULTI_PACK_INDEX"
           }
 
           (with "and the less-memory algorithm"
             it "verifies the pack index successfully and with desired output" && {
               WITH_SNAPSHOT="$snapshot/index-with-statistics-success" \
-              expect_run $SUCCESSFULLY "$exe_plumbing" pack verify --algorithm less-memory --statistics "$PACK_INDEX_FILE"
+              expect_run $SUCCESSFULLY "$exe_plumbing" pack verify --algorithm less-memory --statistics "$MULTI_PACK_INDEX"
             }
           )
         )
         (with "decode"
           it "verifies the pack index successfully and with desired output, and decodes all objects" && {
             WITH_SNAPSHOT="$snapshot/index-success" \
-            expect_run $SUCCESSFULLY "$exe_plumbing" pack verify --algorithm less-memory --decode "$PACK_INDEX_FILE"
+            expect_run $SUCCESSFULLY "$exe_plumbing" pack verify --algorithm less-memory --decode "$MULTI_PACK_INDEX"
           }
         )
         (with "re-encode"
           it "verifies the pack index successfully and with desired output, and re-encodes all objects" && {
             WITH_SNAPSHOT="$snapshot/index-success" \
-            expect_run $SUCCESSFULLY "$exe_plumbing" pack verify --algorithm less-time --re-encode "$PACK_INDEX_FILE"
+            expect_run $SUCCESSFULLY "$exe_plumbing" pack verify --algorithm less-time --re-encode "$MULTI_PACK_INDEX"
           }
         )
         if test "$kind" = "max"; then
         (with "statistics (JSON)"
           it "verifies the pack index successfully and with desired output" && {
             WITH_SNAPSHOT="$snapshot/index-with-statistics-json-success" \
-            expect_run $SUCCESSFULLY "$exe_plumbing" --format json --threads 1 pack verify --statistics "$PACK_INDEX_FILE"
+            expect_run $SUCCESSFULLY "$exe_plumbing" --format json --threads 1 pack verify --statistics "$MULTI_PACK_INDEX"
           }
         )
         fi
       )
     )
     (sandbox
       (with "an INvalid pack INDEX file"
-        PACK_INDEX_FILE="$fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx"
-        cp $PACK_INDEX_FILE index.idx
+        MULTI_PACK_INDEX="$fixtures/packs/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.idx"
+        cp $MULTI_PACK_INDEX index.idx
         echo $'\0' >> index.idx
         it "fails to verify the pack index and with desired output" && {
           WITH_SNAPSHOT="$snapshot/index-failure" \

diff --git a/tests/snapshots/plumbing/pack/verify/multi-index/fast-index-success b/tests/snapshots/plumbing/pack/verify/multi-index/fast-index-success