fetchTree/fetchGit: re-enable shallow fetching

src/libfetchers/git-utils.cc

@@ -387,13 +387,20 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
         //       then use code that was removed in this commit (see blame)
 
         auto dir = this->path;
+        Strings gitArgs;
+        if (shallow) {
+            gitArgs = { "-C", dir.abs(), "fetch", "--quiet", "--force", "--depth", "1", "--", url, refspec };
+        }
+        else {
+            gitArgs = { "-C", dir.abs(), "fetch", "--quiet", "--force", "--", url, refspec };
+        }
 
         runProgram(RunOptions {
             .program = "git",
             .searchPath = true,
             // FIXME: git stderr messes up our progress indicator, so
             // we're using --quiet for now. Should process its stderr.
-            .args = { "-C", path.abs(), "fetch", "--quiet", "--force", "--", url, refspec },
+            .args = gitArgs,
             .input = {},
             .isInteractive = true
         });

src/libfetchers/git.cc

@@ -50,10 +50,12 @@ bool touchCacheFile(const Path & path, time_t touch_time)
     return lutimes(path.c_str(), times) == 0;
 }
 
-Path getCachePath(std::string_view key)
+Path getCachePath(std::string_view key, bool shallow)
 {
-    return getCacheDir() + "/nix/gitv3/" +
-        hashString(HashAlgorithm::SHA256, key).to_string(HashFormat::Nix32, false);
+    return getCacheDir()
+    + "/nix/gitv3/"
+    + hashString(HashAlgorithm::SHA256, key).to_string(HashFormat::Nix32, false)
+    + (shallow ? "-shallow" : "");
 }
 
 // Returns the name of the HEAD branch.
@@ -92,7 +94,8 @@ std::optional<std::string> readHead(const Path & path)
 // Persist the HEAD ref from the remote repo in the local cached repo.
 bool storeCachedHead(const std::string & actualUrl, const std::string & headRef)
 {
-    Path cacheDir = getCachePath(actualUrl);
+    // set shallow=false as HEAD will never be queried for a shallow repo
+    Path cacheDir = getCachePath(actualUrl, false);
     try {
         runProgram("git", true, { "-C", cacheDir, "--git-dir", ".", "symbolic-ref", "--", "HEAD", headRef });
     } catch (ExecError &e) {
@@ -107,7 +110,8 @@ std::optional<std::string> readHeadCached(const std::string & actualUrl)
 {
     // Create a cache path to store the branch of the HEAD ref. Append something
     // in front of the URL to prevent collision with the repository itself.
-    Path cacheDir = getCachePath(actualUrl);
+    // set shallow=false as HEAD will never be queried for a shallow repo
+    Path cacheDir = getCachePath(actualUrl, false);
     Path headRefFile = cacheDir + "/HEAD";
 
     time_t now = time(0);
@@ -508,7 +512,7 @@ struct GitInputScheme : InputScheme
             if (!input.getRev())
                 input.attrs.insert_or_assign("rev", GitRepo::openRepo(CanonPath(repoDir))->resolveRef(ref).gitRev());
         } else {
-            Path cacheDir = getCachePath(repoInfo.url);
+            Path cacheDir = getCachePath(repoInfo.url, getShallowAttr(input));
             repoDir = cacheDir;
             repoInfo.gitDir = ".";
 

tests/nixos/fetch-git/test-cases/shallow-cache-separation/default.nix

@@ -0,0 +1,57 @@
+{
+  description = "can fetch the same repo shallowly and non-shallowly";
+  script = ''
+    # create branch1 off of main
+    client.succeed(f"""
+      echo chiang-mai > {repo.path}/thailand \
+      && {repo.git} add thailand \
+      && {repo.git} commit -m 'commit1' \
+      \
+      && {repo.git} push origin --all
+    """)
+
+    # save the revision
+    mainRev = client.succeed(f"""
+      {repo.git} rev-parse main
+    """).strip()
+
+    # fetch shallowly
+    revCountShallow = client.succeed(f"""
+      nix eval --impure --expr '
+        (builtins.fetchGit {{
+          url = "{repo.remote}";
+          rev = "{mainRev}";
+          shallow = true;
+        }}).revCount
+      '
+    """).strip()
+    # ensure the revCount is 0
+    assert revCountShallow == "0", f"revCountShallow should be 0, but is {revCountShallow}"
+
+    # fetch non-shallowly
+    revCountNonShallow = client.succeed(f"""
+      nix eval --impure --expr '
+        (builtins.fetchGit {{
+          url = "{repo.remote}";
+          rev = "{mainRev}";
+          shallow = false;
+        }}).revCount
+      '
+    """).strip()
+    # ensure the revCount is 1
+    assert revCountNonShallow == "1", f"revCountNonShallow should be 1, but is {revCountNonShallow}"
+
+    # fetch shallowly again
+    revCountShallow2 = client.succeed(f"""
+      nix eval --impure --expr '
+        (builtins.fetchGit {{
+          url = "{repo.remote}";
+          rev = "{mainRev}";
+          shallow = true;
+        }}).revCount
+      '
+    """).strip()
+    # ensure the revCount is 0
+    assert revCountShallow2 == "0", f"revCountShallow2 should be 0, but is {revCountShallow2}"
+  '';
+}

tests/nixos/fetch-git/test-cases/shallow-ignore-ref/default.nix

@@ -0,0 +1,40 @@
+{
+  description = "ensure that ref gets ignored when shallow=true is set";
+  script = ''
+    # create branch1 off of main
+    client.succeed(f"""
+      echo chiang-mai > {repo.path}/thailand \
+      && {repo.git} add thailand \
+      && {repo.git} commit -m 'commit1' \
+      \
+      && {repo.git} checkout -b branch1 main \
+      && echo bangkok > {repo.path}/thailand \
+      && {repo.git} add thailand \
+      && {repo.git} commit -m 'commit2' \
+      \
+      && {repo.git} push origin --all
+    """)
+
+    # save the revisions
+    mainRev = client.succeed(f"""
+      {repo.git} rev-parse main
+    """).strip()
+    branch1Rev = client.succeed(f"""
+      {repo.git} rev-parse branch1
+    """).strip()
+
+    # Ensure that ref gets ignored when fetching shallowly.
+    # This would fail if the ref was respected, as branch1Rev is not on main.
+    client.succeed(f"""
+      nix eval --impure --raw --expr '
+        (builtins.fetchGit {{
+          url = "{repo.remote}";
+          rev = "{branch1Rev}";
+          ref = "main";
+          shallow = true;
+        }})
+      '
+    """)
+
+  '';
+}

tests/nixos/fetch-git/test-cases/ssh-shallow/default.nix

@@ -0,0 +1,52 @@
+{
+  description = "can fetch a git repo via ssh using shallow=1";
+  script = ''
+    # add a file to the repo
+    client.succeed(f"""
+      echo chiang-mai > {repo.path}/thailand \
+      && {repo.git} add thailand \
+      && {repo.git} commit -m 'commit1'
+    """)
+
+    # memoize the revision
+    rev1 = client.succeed(f"""
+      {repo.git} rev-parse HEAD
+    """).strip()
+
+    # push to the server
+    client.succeed(f"""
+      {repo.git} push origin-ssh main
+    """)
+
+    fetchGit_expr = f"""
+      builtins.fetchGit {{
+        url = "{repo.remote_ssh}";
+        rev = "{rev1}";
+        shallow = true;
+      }}
+    """
+
+    # fetch the repo via nix
+    fetched1 = client.succeed(f"""
+      nix eval --impure --raw --expr '({fetchGit_expr}).outPath'
+    """)
+
+    # check if the committed file is there
+    client.succeed(f"""
+      test -f {fetched1}/thailand
+    """)
+
+    # check if the revision is the same
+    rev1_fetched = client.succeed(f"""
+      nix eval --impure --raw --expr '({fetchGit_expr}).rev'
+    """).strip()
+    assert rev1 == rev1_fetched, f"rev1: {rev1} != rev1_fetched: {rev1_fetched}"
+
+    # check if revCount is 1
+    revCount1 = client.succeed(f"""
+      nix eval --impure --expr '({fetchGit_expr}).revCount'
+    """).strip()
+    print(f"revCount1: {revCount1}")
+    assert revCount1 == '0', f"rev count is not 0 but {revCount1}"
+  '';
+}

tests/nixos/fetch-git/testsupport/gitea-repo.nix

@@ -72,4 +72,4 @@ in
           """)
     '';
   };
-}
+}