Read mandatory extensions

.github/workflows/ci.yml

@@ -24,7 +24,7 @@ jobs:
     - name: Setup dependencies (macos)
       if: startsWith(matrix.os, 'macos')
       run:
-        brew install tree
+        brew install tree openssl
     - name: test
       env:
         CI: true

Cargo.toml

@@ -115,6 +115,7 @@ members = [
     "git-diff",
     "git-traverse",
     "git-index",
+    "git-bitmap",
     "git-worktree",
     "git-packetline",
     "git-transport",

README.md

@@ -84,6 +84,7 @@ Follow linked crate name for detailed status. Please note that all crates follow
   * `gitoxide-core`
 * **very early**    
   * [git-index](https://github.com/Byron/gitoxide/blob/main/crate-status.md#git-index)
+  * [git-bitmap](https://github.com/Byron/gitoxide/blob/main/crate-status.md#git-bitmap)
 * **idea**
   * [git-worktree](https://github.com/Byron/gitoxide/blob/main/crate-status.md#git-worktree)
   * [git-tui](https://github.com/Byron/gitoxide/blob/main/crate-status.md#git-tui)

cargo-smart-release/src/git/history.rs

@@ -29,12 +29,12 @@ pub enum SegmentScope {
 }
 
 pub fn collect(handle: &git::easy::Handle) -> anyhow::Result<Option<commit::History>> {
-    let mut repo = handle.clone();
-    repo.object_cache_size(64 * 1024);
-    let reference = match repo.head()?.peeled()?.kind {
+    let mut handle = handle.clone();
+    handle.object_cache_size(64 * 1024);
+    let reference = match handle.head()?.peeled()?.kind {
         head::Kind::Detached { .. } => bail!("Refusing to operate on a detached head."),
         head::Kind::Unborn { .. } => return Ok(None),
-        head::Kind::Symbolic(r) => r.attach(&repo),
+        head::Kind::Symbolic(r) => r.attach(&handle),
     };
 
     let mut items = Vec::new();
@@ -60,7 +60,7 @@ pub fn collect(handle: &git::easy::Handle) -> anyhow::Result<Option<commit::Hist
             (
                 message,
                 tree_id,
-                parent_commit_id.map(|id| id.attach(&repo).object().expect("present").to_commit_ref().tree()),
+                parent_commit_id.map(|id| id.attach(&handle).object().expect("present").to_commit_ref().tree()),
                 commit_time,
             )
         };
@@ -75,9 +75,9 @@ pub fn collect(handle: &git::easy::Handle) -> anyhow::Result<Option<commit::Hist
             }
             Ok(m) => m,
         };
-        data_by_tree_id.insert(tree_id, repo.find_object(tree_id)?.data.to_owned());
+        data_by_tree_id.insert(tree_id, handle.find_object(tree_id)?.data.to_owned());
         if let Some(tree_id) = parent_tree_id {
-            data_by_tree_id.insert(tree_id, repo.find_object(tree_id)?.data.to_owned());
+            data_by_tree_id.insert(tree_id, handle.find_object(tree_id)?.data.to_owned());
         }
         items.push(commit::history::Item {
             id: commit_id.detach(),

crate-status.md

@@ -211,28 +211,47 @@ Check out the [performance discussion][git-traverse-performance] as well.
 * manage multiple worktrees
 * deal with exclude specifications, like .gitignore and other exclude files.
 
+### git-bitmap
+
+A plumbing crate with shared functionality regarding EWAH compressed bitmaps, as well as other kinds of bitmap implementations.
+
+* **EWAH**
+  * [ ] `Array` type to read and write bits
+  * [x] decode on-disk representation
+  * [ ] encode on-disk representation
+
 ### git-index
+
+The git staging area.
+
 * read 
-  * [ ] V2
-  * [ ] V3
-  * [ ] V4
+  * [x] V2
+  * [x] V3
+  * [x] V4
   * optional threading
-    * [ ] concurrent loading of index extensions
-    * [ ] threaded cache entry reading
+    * [x] concurrent loading of index extensions
+    * [x] threaded entry reading
+  * extensions
+    * [x] TREE for speeding up tree generation
+    * [ ] REUC resolving undo
+    * [ ] UNTR untracked cache
+    * [ ] FSMN file system monitor cache V1 and V2
+    * [x] 'link' base indices to take information from, split index
+    * [ ] 'sdir' sparse directory entries
 * `stat` update
     * [ ] optional threaded `stat` based on thread_cost (aka preload)
 * [ ] handling of `.gitignore` and system file exclude configuration
 * [ ] handle potential races
-* extensions
+* maintain extensions when altering the cache
     * [ ] TREE for speeding up tree generation
     * [ ] REUC resolving undo
     * [ ] UNTR untracked cache
     * [ ] FSMN file system monitor cache V1 and V2
     * [ ] EOIE end of index entry
     * [ ] IEOT index entry offset table
-    * [ ] link base indices to take information from, split index
-    * [ ] sdir sparse directory entries
-* additinoal support
+    * [ ] 'link' base indices to take information from, split index
+    * [ ] 'sdir' sparse directory entries
+* additional support
     * [ ] non-sparse
     * [ ] sparse (search for [`sparse index` here](https://github.blog/2021-08-16-highlights-from-git-2-33/))
 * add and remove entries

etc/check-package-size.sh

@@ -19,6 +19,7 @@ echo "in root: gitoxide CLI"
 (enter cargo-smart-release && indent cargo diet -n --package-size-limit 85KB)
 (enter git-actor && indent cargo diet -n --package-size-limit 5KB)
 (enter git-index && indent cargo diet -n --package-size-limit 15KB)
+(enter git-bitmap && indent cargo diet -n --package-size-limit 5KB)
 (enter git-tempfile && indent cargo diet -n --package-size-limit 25KB)
 (enter git-lock && indent cargo diet -n --package-size-limit 15KB)
 (enter git-config && indent cargo diet -n --package-size-limit 65KB)

git-bitmap/CHANGELOG.md

@@ -0,0 +1,30 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## 0.0.0 (2022-01-12)
+
+Initial release, entirely empty.
+
+### Commit Statistics
+
+<csr-read-only-do-not-edit/>
+
+ - 2 commits contributed to the release.
+ - 0 commits where understood as [conventional](https://www.conventionalcommits.org).
+ - 1 unique issue was worked on: [#293](https://github.com/Byron/gitoxide/issues/293)
+
+### Commit Details
+
+<csr-read-only-do-not-edit/>
+
+<details><summary>view details</summary>
+
+ * **[#293](https://github.com/Byron/gitoxide/issues/293)**
+    - git-bitmap - changelog ([`339318c`](https://github.com/Byron/gitoxide/commit/339318c072928b0d683a3746ea9e5c18e485dbbd))
+    - Add git-bitmap crate for use in git-index ([`a517f26`](https://github.com/Byron/gitoxide/commit/a517f2697678f31e29ec9982d02ccfec6a777bbf))
+</details>
+

git-bitmap/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "git-bitmap"
+version = "0.0.0"
+repository = "https://github.com/Byron/gitoxide"
+license = "MIT/Apache-2.0"
+description = "A WIP crate of the gitoxide project dedicated implementing the standard git bitmap format"
+authors = ["Sebastian Thiel <[email protected]>"]
+edition = "2021"
+
+[lib]
+doctest = false
+test = true
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+quick-error = "2.0.0"
+
+[dev-dependencies]
+git-testtools = { path = "../tests/tools"}

git-bitmap/src/lib.rs

@@ -0,0 +1,73 @@
+#![deny(unsafe_code, missing_docs, rust_2018_idioms)]
+#![allow(missing_docs)]
+//! An implementation of the shared parts of git bitmaps used in `git-pack`, `git-index` and `git-worktree`.
+//!
+//! Note that many tests are performed indirectly by tests in the aforementioned consumer crates.
+
+/// Bitmap utilities for the advanced word-aligned hybrid bitmap
+pub mod ewah {
+    pub mod decode {
+        use quick_error::quick_error;
+
+        quick_error! {
+            #[derive(Debug)]
+            pub enum Error {
+                Corrupt(message: &'static str) {
+                    display("{}", message)
+                }
+            }
+        }
+
+        #[inline]
+        pub(crate) fn split_at_pos(data: &[u8], pos: usize) -> Option<(&[u8], &[u8])> {
+            if data.len() < pos {
+                return None;
+            }
+            data.split_at(pos).into()
+        }
+
+        #[inline]
+        pub(crate) fn u32(data: &[u8]) -> Option<(u32, &[u8])> {
+            split_at_pos(data, 4).map(|(num, data)| (u32::from_be_bytes(num.try_into().unwrap()), data))
+        }
+    }
+
+    pub fn decode(data: &[u8]) -> Result<(Vec, &[u8]), decode::Error> {
+        let (num_bits, data) = decode::u32(data).ok_or(decode::Error::Corrupt("eof reading amount of bits"))?;
+        let (len, data) = decode::u32(data).ok_or(decode::Error::Corrupt("eof reading chunk length"))?;
+        let len = len as usize;
+
+        // NOTE: git does this by copying all bytes first, and then it will change the endianess in a separate loop.
+        //       Maybe it's faster, but we can't do it without unsafe. Let's leave it to the optimizer and maybe
+        //       one day somebody will find out that it's worth it to use unsafe here.
+        let (mut bits, data) = decode::split_at_pos(data, len * std::mem::size_of::<u64>())
+            .ok_or(decode::Error::Corrupt("eof while reading bit data"))?;
+        let mut buf = std::vec::Vec::<u64>::with_capacity(len);
+        for _ in 0..len {
+            let (bit_num, rest) = bits.split_at(std::mem::size_of::<u64>());
+            bits = rest;
+            buf.push(u64::from_be_bytes(bit_num.try_into().unwrap()))
+        }
+
+        let (rlw, data) = decode::u32(data).ok_or(decode::Error::Corrupt("eof while reading run length width"))?;
+        dbg!(rlw);
+
+        Ok((
+            Vec {
+                num_bits,
+                bits: buf,
+                rlw: rlw as usize,
+            },
+            data,
+        ))
+    }
+
+    /// A growable collection of u64 that are seen as stream of individual bits.
+    #[allow(dead_code)]
+    pub struct Vec {
+        num_bits: u32,
+        bits: std::vec::Vec<u64>,
+        /// RLW is an offset into the `bits` buffer, so `1` translates into &bits[1] essentially.
+        rlw: usize,
+    }
+}

git-features/src/parallel/in_order.rs

@@ -4,9 +4,6 @@ use std::{cmp::Ordering, collections::BTreeMap};
 pub type SequenceId = usize;
 
 /// An iterator which olds iterated items with a **sequential** ID starting at 0 long enough to dispense them in order.
-///
-/// Note that this iterator is made specifically to support the signature of the iterator returned
-/// by [from_counts_iter(…)][super::entry::iter_from_counts()].
 pub struct InOrderIter<T, I> {
     /// The iterator yielding the out-of-order elements we are to yield in order.
     pub inner: I,

git-hash/src/borrowed.rs

@@ -107,7 +107,6 @@ impl oid {
         self.bytes[0]
     }
 
-    #[inline]
     /// Interpret this object id as raw byte slice.
     #[inline]
     pub fn as_bytes(&self) -> &[u8] {

git-index/Cargo.toml

@@ -32,6 +32,7 @@ internal-testing-to-avoid-being-run-by-cargo-test-all = []
 [dependencies]
 git-features = { version ="^0.18.0", path = "../git-features", features = ["rustsha1"] }
 git-hash = { version ="^0.8.0", path = "../git-hash" }
+git-bitmap = { version ="^0.0.0", path = "../git-bitmap" }
 
 quick-error = "2.0.0"
 memmap2 = "0.5.0"

git-index/src/decode/mod.rs

@@ -8,7 +8,7 @@ pub mod header;
 mod error {
     use quick_error::quick_error;
 
-    use crate::decode;
+    use crate::{decode, extension};
 
     quick_error! {
         #[derive(Debug)]
@@ -21,6 +21,11 @@ mod error {
             Entry(index: u32) {
                 display("Could not parse entry at index {}", index)
             }
+            Extension(err: extension::decode::Error) {
+                display("Mandatory extension wasn't implemented or malformed.")
+                source(err)
+                from()
+            }
             UnexpectedTrailerLength { expected: usize, actual: usize } {
                 display("Index trailer should have been {} bytes long, but was {}", expected, actual)
             }
@@ -69,7 +74,7 @@ impl State {
             Some(offset) if num_threads > 1 => {
                 let extensions_data = &data[offset..];
                 let index_offsets_table = extension::index_entry_offset_table::find(extensions_data, object_hash);
-                let (entries_res, (ext, data)) = git_features::parallel::threads(|scope| {
+                let (entries_res, ext_res) = git_features::parallel::threads(|scope| {
                     let extension_loading =
                         (extensions_data.len() > min_extension_block_in_bytes_for_threading).then({
                             num_threads -= 1;
@@ -166,6 +171,7 @@ impl State {
                     (entries_res, ext_res)
                 })
                 .unwrap(); // this unwrap is for panics - if these happened we are done anyway.
+                let (ext, data) = ext_res?;
                 (entries_res?.0, ext, data)
             }
             None | Some(_) => {
@@ -176,7 +182,7 @@ impl State {
                     object_hash,
                     version,
                 )?;
-                let (ext, data) = extension::decode::all(data, object_hash);
+                let (ext, data) = extension::decode::all(data, object_hash)?;
                 (entries, ext, data)
             }
         };
@@ -194,16 +200,18 @@ impl State {
             path_backing,
             is_sparse,
         } = entries;
-        let extension::decode::Outcome { cache_tree } = ext;
+        let extension::decode::Outcome { tree, link } = ext;
 
         Ok((
             State {
                 timestamp,
                 version,
-                cache_tree,
                 entries,
                 path_backing,
                 is_sparse,
+
+                tree,
+                link,
             },
             checksum,
         ))