Add option to include untracked files in git and mercurial fetchers

src/libfetchers/fetch-settings.hh

@@ -70,6 +70,9 @@ struct FetchSettings : public Config
     Setting<bool> warnDirty{this, true, "warn-dirty",
         "Whether to warn about dirty Git/Mercurial trees."};
 
+    Setting<bool> includeUntrackedFiles{this, false, "include-untracked-files",
+        "Whether to include untracked files in Git/Mercurial trees."};
+
     Setting<std::string> flakeRegistry{this, "https://channels.nixos.org/flake-registry.json", "flake-registry",
         R"(
           Path or URI of the global flake registry.

src/libfetchers/git-utils.cc

@@ -257,7 +257,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
         return (*((std::function<int(const char * path, unsigned int statusFlags)> *) payload))(path, statusFlags);
     }
 
-    WorkdirInfo getWorkdirInfo() override
+    WorkdirInfo getWorkdirInfo(bool includeUntrackedFiles) override
     {
         WorkdirInfo info;
 
@@ -284,6 +284,10 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
         git_status_options options = GIT_STATUS_OPTIONS_INIT;
         options.flags |= GIT_STATUS_OPT_INCLUDE_UNMODIFIED;
         options.flags |= GIT_STATUS_OPT_EXCLUDE_SUBMODULES;
+        if (includeUntrackedFiles) {
+            options.flags |=  GIT_STATUS_OPT_INCLUDE_UNTRACKED |
+                              GIT_STATUS_OPT_RECURSE_UNTRACKED_DIRS;
+        }
         if (git_status_foreach_ext(*this, &options, &statusCallbackTrampoline, &statusCallback))
             throw Error("getting working directory status: %s", git_error_last()->message);
 

src/libfetchers/git-utils.hh

@@ -48,7 +48,7 @@ struct GitRepo
         std::vector<Submodule> submodules;
     };
 
-    virtual WorkdirInfo getWorkdirInfo() = 0;
+    virtual WorkdirInfo getWorkdirInfo(bool includeUntrackedFiles) = 0;
 
     /* Get the ref that HEAD points to. */
     virtual std::optional<std::string> getWorkdirRef() = 0;

src/libfetchers/git.cc

@@ -386,7 +386,7 @@ struct GitInputScheme : InputScheme
         // If this is a local directory and no ref or revision is
         // given, then allow the use of an unclean working tree.
         if (!input.getRef() && !input.getRev() && repoInfo.isLocal)
-            repoInfo.workdirInfo = GitRepo::openRepo(CanonPath(repoInfo.url))->getWorkdirInfo();
+            repoInfo.workdirInfo = GitRepo::openRepo(CanonPath(repoInfo.url))->getWorkdirInfo(fetchSettings.includeUntrackedFiles);
 
         return repoInfo;
     }

src/libfetchers/mercurial.cc

@@ -175,7 +175,10 @@ struct MercurialInputScheme : InputScheme
 
         if (!input.getRef() && !input.getRev() && isLocal && pathExists(actualUrl + "/.hg")) {
 
-            bool clean = runHg({ "status", "-R", actualUrl, "--modified", "--added", "--removed" }) == "";
+            Strings hgStatusOptions = { "status", "-R", actualUrl, "--modified", "--added", "--removed" };
+            if (fetchSettings.includeUntrackedFiles)
+                hgStatusOptions.push_back("--unknown");
+            bool clean = runHg(hgStatusOptions) == "";
 
             if (!clean) {
 
@@ -190,8 +193,11 @@ struct MercurialInputScheme : InputScheme
 
                 input.attrs.insert_or_assign("ref", chomp(runHg({ "branch", "-R", actualUrl })));
 
+                hgStatusOptions = { "status", "-R", actualUrl, "--clean", "--modified", "--added", "--no-status", "--print0" };
+                if (fetchSettings.includeUntrackedFiles)
+                    hgStatusOptions.push_back("--unknown");
                 auto files = tokenizeString<std::set<std::string>>(
-                    runHg({ "status", "-R", actualUrl, "--clean", "--modified", "--added", "--no-status", "--print0" }), "\0"s);
+                    runHg(hgStatusOptions), "\0"s);
 
                 Path actualPath(absPath(actualUrl));
 

tests/functional/fetchGit.sh

@@ -106,6 +106,25 @@ path2=$(nix eval --impure --raw --expr "(builtins.fetchGit $repo).outPath")
 [ ! -e $path2/.git ]
 [[ $(cat $path2/dir1/foo) = foo ]]
 
+# Using an unclean tree with --include-git-untracked-files option should yield the tracked and uncommitted changes
+path21=$(nix eval --include-untracked-files --impure --raw --expr "(builtins.fetchGit $repo).outPath")
+[ ! -e $path21/hello ]
+[ -e $path21/bar ]
+[ -e $path21/dir2/bar ]
+[ ! -e $path21/.git ]
+[[ $(cat $path21/dir1/foo) = foo ]]
+
+# Using an unclean tree with --include-git-untracked-files option should take into account .gitignore files
+echo "/bar" >> $repo/.gitignore
+echo "bar" >> $repo/dir2/.gitignore
+path22=$(nix eval --include-untracked-files --impure --raw --expr "(builtins.fetchGit $repo).outPath")
+[ ! -e $path22/hello ]
+[ ! -e $path22/bar ]
+[ ! -e $path22/dir2/bar ]
+[ ! -e $path22/.git ]
+git -C $repo checkout -- $repo/.gitignore
+git -C $repo clean -fd
+
 [[ $(nix eval --impure --raw --expr "(builtins.fetchGit $repo).rev") = 0000000000000000000000000000000000000000 ]]
 [[ $(nix eval --impure --raw --expr "(builtins.fetchGit $repo).dirtyRev") = "${rev2}-dirty" ]]
 [[ $(nix eval --impure --raw --expr "(builtins.fetchGit $repo).dirtyShortRev") = "${rev2:0:7}-dirty" ]]

tests/functional/fetchMercurial.sh

@@ -1,6 +1,6 @@
 source common.sh
 
-[[ $(type -p hq) ]] || skipTest "Mercurial not installed"
+[[ $(type -p hg) ]] || skipTest "Mercurial not installed"
 
 clearStore
 
@@ -88,6 +88,22 @@ path2=$(nix eval --impure --raw --expr "(builtins.fetchMercurial $repo).outPath"
 [ ! -e $path2/.hg ]
 [[ $(cat $path2/dir1/foo) = foo ]]
 
+path21=$(nix eval --include-untracked-files --impure --raw --expr "(builtins.fetchMercurial $repo).outPath")
+[ ! -e $path21/hello ]
+[ -e $path21/bar ]
+[ -e $path21/dir2/bar ]
+[ ! -e $path21/.hg ]
+[[ $(cat $path21/dir1/foo) = foo ]]
+
+echo "bar" >> $repo/.hgignore
+path22=$(nix eval --include-untracked-files --impure --raw --expr "(builtins.fetchMercurial $repo).outPath")
+[ ! -e $path22/hello ]
+[ ! -e $path22/bar ]
+[ ! -e $path22/dir2/bar ]
+[ ! -e $path22/.git ]
+sed -i '/bar/d' $repo/.hgignore
+hg status --cwd $repo
+
 [[ $(nix eval --impure --raw --expr "(builtins.fetchMercurial $repo).rev") = 0000000000000000000000000000000000000000 ]]
 
 # ... unless we're using an explicit ref.

tests/functional/flakes/mercurial.sh

@@ -1,6 +1,6 @@
 source ./common.sh
 
-[[ $(type -p hq) ]] || skipTest "Mercurial not installed"
+[[ $(type -p hg) ]] || skipTest "Mercurial not installed"
 
 flake1Dir=$TEST_ROOT/flake-hg1
 mkdir -p $flake1Dir