fetchGit/fetchTree: add shallowRev option

src/libexpr/primops/fetchTree.cc

@@ -423,9 +423,16 @@ static RegisterPrimOp primop_fetchGit({
 
         A Boolean parameter that specifies whether submodules should be checked out.
 
+      - `shallowRev` (default: `false`)
+
+        A Boolean parameter that specifies whether to perform a shallow clone.
+        This reduces the amount of data transmitted while fetching.
+
       - `shallow` (default: `false`)
 
-        A Boolean parameter that specifies whether fetching a shallow clone is allowed.
+        A Boolean parameter that specifies whether fetching from a shallow remote repository is allowed.
+        This still performs a full clone of what is available on the remote.
+        For most cases 'shallowRev = true' should be used instead.
 
       - `allRefs`
 

src/libfetchers/git.cc

@@ -362,6 +362,7 @@ struct GitInputScheme : InputScheme
             "ref",
             "rev",
             "shallow",
+            "shallowRev",
             "submodules",
             "lastModified",
             "revCount",
@@ -387,6 +388,7 @@ struct GitInputScheme : InputScheme
                 experimentalFeatureSettings.require(Xp::VerifiedFetches);
 
         maybeGetBoolAttr(attrs, "shallow");
+        maybeGetBoolAttr(attrs, "shallowRev");
         maybeGetBoolAttr(attrs, "submodules");
         maybeGetBoolAttr(attrs, "allRefs");
         maybeGetBoolAttr(attrs, "verifyCommit");
@@ -412,6 +414,8 @@ struct GitInputScheme : InputScheme
         if (auto ref = input.getRef()) url.query.insert_or_assign("ref", *ref);
         if (maybeGetBoolAttr(input.attrs, "shallow").value_or(false))
             url.query.insert_or_assign("shallow", "1");
+        if (maybeGetBoolAttr(input.attrs, "shallowRev").value_or(false))
+            url.query.insert_or_assign("shallowRev", "1");
         if (maybeGetBoolAttr(input.attrs, "verifyCommit").value_or(false))
             url.query.insert_or_assign("verifyCommit", "1");
         auto publicKeys = getPublicKeys(input.attrs);
@@ -500,6 +504,80 @@ struct GitInputScheme : InputScheme
         return {isLocal, isLocal ? url.path : url.base};
     }
 
+
+    std::pair<StorePath, Input> fetchShallowRev(
+        ref<Store> store,
+        Input & input,
+        const Path & actualUrl,
+        const bool verifyCommit,
+        std::vector<PublicKey> publicKeys,
+        const Attrs & inAttrs) const
+    {
+        auto gitDir = ".git";
+        std::string name = input.getName();
+
+        Path repoDir = createTempDir();
+        AutoDelete delTmpRepoDir(repoDir, true);
+        gitDir = ".";
+        createDirs(dirOf(repoDir));
+        PathLocks repoDirLock({repoDir + ".lock"});
+        runProgram("git", true, { "init", "--bare", repoDir });
+
+        Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", actualUrl));
+
+        // FIXME: git stderr messes up our progress indicator, so
+        // we're using --quiet for now. Should process its stderr.
+        try {
+            runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "fetch", "--quiet", "--force", "--depth", "1", "--", actualUrl, input.getRev()->gitRev()}, {}, true);
+        } catch (Error & e) {
+            throw Error(
+                "Could not fetch single revision of Git repository '%s'. "
+                "One reason could be that the remote does not support or allow shallow revision fetching. "
+                "In this case, try removing 'shallowRev = true' from fetchGit",
+                actualUrl
+            );
+        }
+
+        if (verifyCommit)
+            doCommitVerification(repoDir, gitDir, input.getRev()->gitRev(), publicKeys);
+
+        // FIXME: should pipe this, or find some better way to extract a
+        // revision.
+        auto source = sinkToSource([&](Sink & sink) {
+            runProgram2({
+                .program = "git",
+                .args = { "-C", repoDir, "--git-dir", gitDir, "archive", input.getRev()->gitRev() },
+                .standardOut = &sink
+            });
+        });
+
+        Path tmpDir = createTempDir();
+        AutoDelete delTmpDir(tmpDir, true);
+        PathFilter filter = defaultPathFilter;
+
+        unpackTarfile(*source, tmpDir);
+
+        auto storePath = store->addToStore(name, tmpDir, FileIngestionMethod::Recursive, htSHA256, filter);
+
+        auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "log", "-1", "--format=%ct", "--no-show-signature", input.getRev()->gitRev() }));
+
+        Attrs infoAttrs({
+            {"rev", input.getRev()->gitRev()},
+            {"lastModified", lastModified},
+        });
+
+        getCache()->add(
+            store,
+            inAttrs,
+            infoAttrs,
+            storePath,
+            true);
+
+        input.attrs.insert_or_assign("lastModified", getIntAttr(infoAttrs, "lastModified"));
+        return {std::move(storePath), input};
+    }
+
+
     std::pair<StorePath, Input> fetch(ref<Store> store, const Input & _input) override
     {
         Input input(_input);
@@ -512,9 +590,11 @@ struct GitInputScheme : InputScheme
         bool allRefs = maybeGetBoolAttr(input.attrs, "allRefs").value_or(false);
         std::vector<PublicKey> publicKeys = getPublicKeys(input.attrs);
         bool verifyCommit = maybeGetBoolAttr(input.attrs, "verifyCommit").value_or(!publicKeys.empty());
+        bool shallowRev = maybeGetBoolAttr(input.attrs, "shallowRev").value_or(false);
 
         std::string cacheType = "git";
         if (shallow) cacheType += "-shallow";
+        if (shallowRev) cacheType += "-shallowRev";
         if (submodules) cacheType += "-submodules";
         if (allRefs) cacheType += "-all-refs";
 
@@ -542,7 +622,7 @@ struct GitInputScheme : InputScheme
         {
             assert(input.getRev());
             assert(!_input.getRev() || _input.getRev() == input.getRev());
-            if (!shallow)
+            if (!shallow && !shallowRev)
                 input.attrs.insert_or_assign("revCount", getIntAttr(infoAttrs, "revCount"));
             input.attrs.insert_or_assign("lastModified", getIntAttr(infoAttrs, "lastModified"));
             return {std::move(storePath), input};
@@ -578,7 +658,20 @@ struct GitInputScheme : InputScheme
 
         Path repoDir;
 
-        if (isLocal) {
+        if (shallowRev) {
+            // sanity checks
+            if (!input.getRev())
+                throw Error("Error fetching git repo '%s'. 'rev' must be specified if 'shallowRev = true' is used", actualUrl);
+            // ensure shallow is not used when shallowRev is used
+            if (shallow)
+                throw Error("Error fetching git repo '%s'. Set either 'shallow' or 'shallowRev', not both", actualUrl);
+            // TODO: add support for submodules
+            // For the purpose of this POC implementation omitting submodules is fine
+            if (submodules)
+                throw Error("Error fetching git repo '%s'. 'submodules' cannot be used if 'shallowRev = true' is used", actualUrl);
+            return fetchShallowRev(store, input, actualUrl, verifyCommit, publicKeys, getLockedAttrs());
+
+        } else if (isLocal) {
             if (!input.getRef()) {
                 auto head = readHead(actualUrl);
                 if (!head) {

tests/functional/fetchGit-shallowRev.sh

@@ -0,0 +1,50 @@
+source common.sh
+
+requireGit
+
+clearStore
+
+# Intentionally not in a canonical form
+# See https://github.com/NixOS/nix/issues/6195
+repo=$TEST_ROOT/./git
+
+export _NIX_FORCE_HTTP=1
+
+rm -rf $TEST_HOME/.cache/nix $TEST_ROOT/shallow
+
+git init $repo
+git -C $repo config user.email "[email protected]"
+git -C $repo config user.name "Foobar"
+
+echo utrecht > $repo/hello
+touch $repo/.gitignore
+touch $repo/not-exported-file
+echo "/not-exported-file export-ignore" >> $repo/.gitattributes
+git -C $repo add hello not-exported-file .gitignore .gitattributes
+git -C $repo commit -m 'Bla1'
+rev1=$(git -C $repo rev-parse HEAD)
+
+echo world > $repo/hello
+git -C $repo commit -m 'Bla2' -a
+rev2=$(git -C $repo rev-parse HEAD)
+
+
+# Fetch local repo using shallowRev
+path0=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = file://$repo; rev = \"$rev1\"; shallowRev = true; name = \"test1\"; }).outPath")
+[[ $(cat $path0/hello) = utrecht ]]
+[[ $(nix eval --impure --raw --expr "(builtins.fetchGit { url = file://$repo; rev = \"$rev1\"; shallowRev = true; name = \"test1\"; }).rev") = $rev1 ]]
+
+# Ensure .gitattributes is respected
+[[ ! -e $path0/not-exported-file ]]
+
+# Fetch local shallow repo using shallowRev
+git clone --depth 1 file://$repo $TEST_ROOT/shallow
+path1=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = file://$TEST_ROOT/shallow; rev = \"$rev2\"; shallowRev = true; name = \"test2\"; }).outPath")
+[[ $(cat $path1/hello) = world ]]
+
+# Ensure that fetching a non-existing revision fails
+# (since /shallow is a truncated repo, it does not contain $rev1)
+out=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = file://$TEST_ROOT/shallow; rev = \"$rev1\"; shallowRev = true; name = \"test3\"; }).outPath" 2>&1) || status=$?
+[[ $status == 1 ]]
+# The error message is not very spcific in this case, but what matters is that it fails properly
+[[ $out =~ 'Could not fetch single revision of Git repository' ]]

tests/functional/fetchGit.sh

@@ -235,6 +235,15 @@ rev_tag2=$(git -C $repo rev-parse refs/tags/tag2)
 [[ $rev_tag2_nix = $rev_tag2 ]]
 unset _NIX_FORCE_HTTP
 
+# Ensure .gitattributes is respected
+touch $repo/not-exported-file
+echo "/not-exported-file export-ignore" >> $repo/.gitattributes
+git -C $repo add not-exported-file .gitattributes
+git -C $repo commit -m 'Bla6'
+rev5=$(git -C $repo rev-parse HEAD)
+path12=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = file://$repo; rev = \"$rev5\"; }).outPath")
+[[ ! -e $path12/not-exported-file ]]
+
 # should fail if there is no repo
 rm -rf $repo/.git
 (! nix eval --impure --raw --expr "(builtins.fetchGit \"file://$repo\").outPath")

tests/functional/local.mk

@@ -38,6 +38,7 @@ nix_tests = \
   gc-runtime.sh \
   tarball.sh \
   fetchGit.sh \
+  fetchGit-shallowRev.sh \
   fetchurl.sh \
   fetchPath.sh \
   fetchTree-file.sh \