The first 'explode' implementation…

…done entirely in gitoxide core based on quite powerful building blocks
from the subcrates.

Neat!

git-odb/src/pack/bundle.rs

@@ -52,15 +52,15 @@ impl Bundle {
         out: &'a mut Vec<u8>,
         cache: &mut impl pack::cache::DecodeEntry,
     ) -> Option<Result<Object<'a>, Error>> {
-        let idx = self.index.lookup_index(id)?;
+        let idx = self.index.lookup(id)?;
         let ofs = self.index.pack_offset_at_index(idx);
         let entry = self.pack.entry(ofs);
         self.pack
             .decode_entry(
                 entry,
                 out,
                 |id, _out| {
-                    self.index.lookup_index(id).map(|idx| {
+                    self.index.lookup(id).map(|idx| {
                         pack::data::decode::ResolvedBase::InPack(self.pack.entry(self.index.pack_offset_at_index(idx)))
                     })
                 },

git-odb/src/pack/index/access.rs

@@ -103,7 +103,7 @@ impl index::File {
     }
 
     /// Returns the offset of the given SHA1 for use with the `(oid|pack_offset|crc32)_at_index()`
-    pub fn lookup_index(&self, id: borrowed::Id) -> Option<u32> {
+    pub fn lookup(&self, id: borrowed::Id) -> Option<u32> {
         let first_byte = id.first_byte() as usize;
         let mut upper_bound = self.fan[first_byte];
         let mut lower_bound = if first_byte != 0 { self.fan[first_byte - 1] } else { 0 };

git-odb/src/pack/index/init.rs

@@ -36,7 +36,7 @@ impl TryFrom<&Path> for index::File {
     type Error = Error;
 
     fn try_from(path: &Path) -> Result<Self, Self::Error> {
-        let data = FileBuffer::open(path).map_err(|e| Error::Io(e, path.to_owned()))?;
+        let data = FileBuffer::open(&path).map_err(|e| Error::Io(e, path.to_owned()))?;
         let idx_len = data.len();
         if idx_len < FAN_LEN * N32_SIZE + FOOTER_SIZE {
             return Err(Error::Corrupt(format!(
@@ -73,6 +73,7 @@ impl TryFrom<&Path> for index::File {
         };
         Ok(index::File {
             data,
+            path: path.to_owned(),
             kind,
             num_objects,
             version,

git-odb/src/pack/index/mod.rs

@@ -90,6 +90,7 @@ const FAN_LEN: usize = 256;
 
 pub struct File {
     pub(crate) data: FileBuffer,
+    path: std::path::PathBuf,
     kind: Kind,
     version: u32,
     num_objects: u32,
@@ -100,6 +101,9 @@ impl File {
     pub fn kind(&self) -> Kind {
         self.kind
     }
+    pub fn path(&self) -> &std::path::Path {
+        &self.path
+    }
     pub fn num_objects(&self) -> u32 {
         self.num_objects
     }

git-odb/src/pack/index/traverse/indexed.rs

@@ -37,7 +37,7 @@ impl index::File {
             sorted_entries.iter().map(|e| e.pack_offset),
             pack.path(),
             root.add_child("indexing"),
-            |id| self.lookup_index(id).map(|idx| self.pack_offset_at_index(idx)),
+            |id| self.lookup(id).map(|idx| self.pack_offset_at_index(idx)),
         )?;
         let if_there_are_enough_objects = || self.num_objects > 10_000;
 

git-odb/src/pack/index/traverse/mod.rs

@@ -153,7 +153,7 @@ impl Default for Context {
 
 /// Verify and validate the content of the index file
 impl index::File {
-    pub fn traverse_index<P, C, Processor>(
+    pub fn traverse<P, C, Processor>(
         &self,
         pack: &pack::data::File,
         Context {
@@ -243,7 +243,7 @@ impl index::File {
                 pack_entry,
                 buf,
                 |id, _| {
-                    self.lookup_index(id).map(|index| {
+                    self.lookup(id).map(|index| {
                         pack::data::decode::ResolvedBase::InPack(pack.entry(self.pack_offset_at_index(index)))
                     })
                 },

git-odb/src/pack/index/verify.rs

@@ -88,7 +88,7 @@ impl index::File {
         match pack {
             None => verify_self().map_err(Into::into).map(|id| (id, None)),
             Some((pack, mode, algorithm)) => self
-                .traverse_index(
+                .traverse(
                     pack,
                     index::traverse::Context {
                         algorithm,

git-odb/tests/pack/index.rs

@@ -16,22 +16,22 @@ mod method {
         use git_odb::pack::index;
 
         #[test]
-        fn lookup_index() {
+        fn lookup() {
             let idx = index::File::at(&fixture_path(INDEX_V1)).unwrap();
             for (id, desired_index, assertion) in &[
                 (&b"036bd66fe9b6591e959e6df51160e636ab1a682e"[..], Some(0), "first"),
                 (b"f7f791d96b9a34ef0f08db4b007c5309b9adc3d6", Some(65), "close to last"),
                 (b"ffffffffffffffffffffffffffffffffffffffff", None, "not in pack"),
             ] {
                 assert_eq!(
-                    idx.lookup_index(owned::Id::from_40_bytes_in_hex(*id).unwrap().to_borrowed()),
+                    idx.lookup(owned::Id::from_40_bytes_in_hex(*id).unwrap().to_borrowed()),
                     *desired_index,
                     "{}",
                     assertion
                 );
             }
             for entry in idx.iter() {
-                let index = idx.lookup_index(entry.oid.to_borrowed()).unwrap();
+                let index = idx.lookup(entry.oid.to_borrowed()).unwrap();
                 assert_eq!(entry.oid.to_borrowed(), idx.oid_at_index(index));
                 assert_eq!(entry.pack_offset, idx.pack_offset_at_index(index));
                 assert_eq!(entry.crc32, idx.crc32_at_index(index));
@@ -45,22 +45,22 @@ mod method {
         use git_odb::pack::index;
 
         #[test]
-        fn lookup_index() {
+        fn lookup() {
             let idx = index::File::at(&fixture_path(INDEX_V2)).unwrap();
             for (id, desired_index, assertion) in &[
                 (&b"0ead45fc727edcf5cadca25ef922284f32bb6fc1"[..], Some(0), "first"),
                 (b"e800b9c207e17f9b11e321cc1fba5dfe08af4222", Some(29), "last"),
                 (b"ffffffffffffffffffffffffffffffffffffffff", None, "not in pack"),
             ] {
                 assert_eq!(
-                    idx.lookup_index(owned::Id::from_40_bytes_in_hex(*id).unwrap().to_borrowed()),
+                    idx.lookup(owned::Id::from_40_bytes_in_hex(*id).unwrap().to_borrowed()),
                     *desired_index,
                     "{}",
                     assertion
                 );
             }
             for entry in idx.iter() {
-                let index = idx.lookup_index(entry.oid.to_borrowed()).unwrap();
+                let index = idx.lookup(entry.oid.to_borrowed()).unwrap();
                 assert_eq!(entry.oid.to_borrowed(), idx.oid_at_index(index));
                 assert_eq!(entry.pack_offset, idx.pack_offset_at_index(index));
                 assert_eq!(entry.crc32, idx.crc32_at_index(index), "{} {:?}", index, entry);

gitoxide-core/src/pack/explode.rs

@@ -1,9 +1,12 @@
 use anyhow::{Context, Result};
 use git_features::progress::Progress;
 use git_object::{owned, HashKind};
-use git_odb::{loose, pack};
-use std::io::Read;
-use std::path::Path;
+use git_odb::{loose, pack, Write};
+use std::{
+    fs,
+    io::{self, Read},
+    path::{Path, PathBuf},
+};
 
 #[derive(PartialEq, Debug)]
 pub enum SafetyCheck {
@@ -67,6 +70,17 @@ quick_error! {
             source(err)
             from()
         }
+        Write(err: Box<dyn std::error::Error + Send + Sync>, kind: git_object::Kind, id: owned::Id) {
+            display("Failed to write {} object {}", kind, id)
+            source(&**err)
+        }
+        ObjectEncodeMismatch(kind: git_object::Kind, actual: owned::Id, expected: owned::Id) {
+            display("{} object {} wasn't re-encoded without change - new hash is {}", kind, expected, actual)
+        }
+        RemoveFile(err: io::Error, index: PathBuf, data: PathBuf) {
+            display("Failed to delete pack index file at '{} or data file at '{}'", index.display(), data.display())
+            source(err)
+        }
     }
 }
 
@@ -98,21 +112,56 @@ impl git_odb::Write for OutputWriter {
 pub fn pack_or_pack_index<P>(
     pack_path: impl AsRef<Path>,
     object_path: Option<impl AsRef<Path>>,
-    _check: SafetyCheck,
-    _progress: Option<P>,
+    check: SafetyCheck,
+    thread_limit: Option<usize>,
+    progress: Option<P>,
     _delete_pack: bool,
 ) -> Result<()>
 where
     P: Progress,
+    <P as Progress>::SubProgress: Send,
 {
     let path = pack_path.as_ref();
-    let _bundle = pack::Bundle::at(path).with_context(|| {
+    let bundle = pack::Bundle::at(path).with_context(|| {
         format!(
             "Could not find .idx or .pack file from given file at '{}'",
             path.display()
         )
     })?;
 
-    let _out = OutputWriter(object_path.map(|path| loose::Db::at(path.as_ref())));
-    Ok(())
+    let out = OutputWriter(object_path.map(|path| loose::Db::at(path.as_ref())));
+    bundle.index.traverse(
+        &bundle.pack,
+        pack::index::traverse::Context {
+            algorithm: pack::index::traverse::Algorithm::Lookup,
+            thread_limit,
+            check: check.into(),
+        },
+        progress,
+        || {
+            |object_kind, buf, index_entry, _entry_stats, progress| {
+                let written_id = out
+                    .write_buf(object_kind, buf, HashKind::Sha1)
+                    .map_err(|err| Error::Write(Box::new(err) as Box<dyn std::error::Error + Send + Sync>, object_kind, index_entry.oid))
+                    .map_err(|err| Box::new(err) as Box<dyn std::error::Error + Send + Sync>)?;
+                if written_id != index_entry.oid {
+                   if let git_object::Kind::Tree = object_kind {
+                       progress.info(format!("The tree in pack named {} was written as {} due to modes 100664 and 100640 rewritten as 100644.", index_entry.oid, written_id));
+                   } else {
+                       return Err(Box::new(Error::ObjectEncodeMismatch(object_kind, index_entry.oid, written_id)))
+                   }
+                }
+                Ok(())
+            }
+        },
+        pack::cache::DecodeEntryLRU::default,
+    ).with_context(|| "Some loose objects could not be extracted")?;
+
+    let (index_path, data_path) = (bundle.index.path().to_owned(), bundle.pack.path().to_owned());
+    drop(bundle);
+
+    fs::remove_file(&index_path)
+        .and_then(|_| fs::remove_file(&data_path))
+        .map_err(|err| Error::RemoveFile(err, index_path, data_path))
+        .map_err(Into::into)
 }

src/plumbing/lean.rs

@@ -146,6 +146,7 @@ pub fn main() -> Result<()> {
                 pack_path,
                 object_path,
                 check.unwrap_or(core::pack::explode::SafetyCheck::All),
+                thread_limit,
                 progress,
                 delete_pack,
             )

src/plumbing/pretty.rs

@@ -233,7 +233,14 @@ pub fn main() -> Result<()> {
             progress,
             progress_keep_open,
             move |progress, _out, _err| {
-                core::pack::explode::pack_or_pack_index(pack_path, object_path, check, progress, delete_pack)
+                core::pack::explode::pack_or_pack_index(
+                    pack_path,
+                    object_path,
+                    check,
+                    thread_limit,
+                    progress,
+                    delete_pack,
+                )
             },
         ),
         Subcommands::PackVerify {