From fc295b1e286c79e011bba5c9fa253c62ec32a658 Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Thu, 27 Nov 2025 12:41:28 +0800 Subject: [PATCH 01/13] fetchgit: take postCheckout to allow collecting revision without leaving .git --- pkgs/build-support/fetchgit/default.nix | 7 +++++++ pkgs/build-support/fetchgit/tests.nix | 10 ++++++++++ 2 files changed, 17 insertions(+) diff --git a/pkgs/build-support/fetchgit/default.nix b/pkgs/build-support/fetchgit/default.nix index 7a8f689df4398..6bd8b632f2f8a 100644 --- a/pkgs/build-support/fetchgit/default.nix +++ b/pkgs/build-support/fetchgit/default.nix @@ -83,6 +83,8 @@ lib.makeOverridable ( # run operations between the checkout completing and deleting the .git # directory. preFetch ? "", + # Shell code executed after `git checkout` and before .git directory removal/sanitization. + postCheckout ? "", # Shell code executed after the file has been fetched # successfully. This can do things like check or transform the file. postFetch ? "", @@ -171,6 +173,7 @@ lib.makeOverridable ( deepClone branchName preFetch + postCheckout postFetch fetchTags rootDir @@ -227,6 +230,10 @@ lib.makeOverridable ( inherit preferLocalBuild meta allowedRequisites; + env = { + NIX_PREFETCH_GIT_CHECKOUT_HOOK = finalAttrs.postCheckout; + }; + passthru = { gitRepoUrl = url; } diff --git a/pkgs/build-support/fetchgit/tests.nix b/pkgs/build-support/fetchgit/tests.nix index 9ccb3ff3058b4..c836f830545dc 100644 --- a/pkgs/build-support/fetchgit/tests.nix +++ b/pkgs/build-support/fetchgit/tests.nix @@ -17,6 +17,16 @@ sha256 = "sha256-7DszvbCNTjpzGRmpIVAWXk20P0/XTrWZ79KSOGLrUWY="; }; + collect-rev = testers.invalidateFetcherByDrvHash fetchgit { + name = "collect-rev-nix-source"; + url = "https://github.com/NixOS/nix"; + rev = "9d9dbe6ed05854e03811c361a3380e09183f4f4a"; + hash = "sha256-AUTX1K7J5+fojvKYJacXYVV5kio3hrWYz5MCekO6h68="; + postCheckout = '' + git -C "$out" rev-parse HEAD | tee "$out/revision.txt" + ''; + }; + sparseCheckout = testers.invalidateFetcherByDrvHash fetchgit { name = "sparse-checkout-nix-source"; url = "https://github.com/NixOS/nix"; From cbd24f4a8a18f881ed9e468a434cf4127618ba09 Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Sat, 29 Nov 2025 20:40:32 +0800 Subject: [PATCH 02/13] tests.fetchgit: add simple-tag --- pkgs/build-support/fetchgit/tests.nix | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pkgs/build-support/fetchgit/tests.nix b/pkgs/build-support/fetchgit/tests.nix index c836f830545dc..e1d2ad2c2bedf 100644 --- a/pkgs/build-support/fetchgit/tests.nix +++ b/pkgs/build-support/fetchgit/tests.nix @@ -27,6 +27,13 @@ ''; }; + simple-tag = testers.invalidateFetcherByDrvHash fetchgit { + name = "simple-tag-nix-source"; + url = "https://github.com/NixOS/nix"; + tag = "2.3.15"; + hash = "sha256-7DszvbCNTjpzGRmpIVAWXk20P0/XTrWZ79KSOGLrUWY="; + }; + sparseCheckout = testers.invalidateFetcherByDrvHash fetchgit { name = "sparse-checkout-nix-source"; url = "https://github.com/NixOS/nix"; From 7fe783f1871b3f12f56702b6197780da297452a8 Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Sun, 30 Nov 2025 15:13:43 +0800 Subject: [PATCH 03/13] nix-prefetch-git: fetch the fetching tag for NIX_PREFETCH_GIT_CHECKOUT_HOOK --- pkgs/build-support/fetchgit/nix-prefetch-git | 7 +++++++ pkgs/build-support/fetchgit/tests.nix | 10 ++++++++++ 2 files changed, 17 insertions(+) diff --git a/pkgs/build-support/fetchgit/nix-prefetch-git b/pkgs/build-support/fetchgit/nix-prefetch-git index f9c8af8428fe3..c3df5fc502b1d 100755 --- a/pkgs/build-support/fetchgit/nix-prefetch-git +++ b/pkgs/build-support/fetchgit/nix-prefetch-git @@ -258,6 +258,13 @@ clone(){ clean_git fetch origin 'refs/tags/*:refs/tags/*' || echo "warning: failed to fetch some tags" >&2 fi + # Name "$ref" to make `git describe` work reproducibly in `NIX_PREFETCH_GIT_CHECKOUT_HOOK`. + # Name only when not leaving `.git` for compatibility purposes. + if [[ -n "$ref" ]] && [[ -z "$leaveDotGit" ]]; then + echo "refer to FETCH_HEAD as its original name $ref" + clean_git update-ref "$ref" FETCH_HEAD + fi + # Checkout linked sources. if test -n "$fetchSubmodules"; then init_submodules diff --git a/pkgs/build-support/fetchgit/tests.nix b/pkgs/build-support/fetchgit/tests.nix index e1d2ad2c2bedf..261330564c433 100644 --- a/pkgs/build-support/fetchgit/tests.nix +++ b/pkgs/build-support/fetchgit/tests.nix @@ -34,6 +34,16 @@ hash = "sha256-7DszvbCNTjpzGRmpIVAWXk20P0/XTrWZ79KSOGLrUWY="; }; + describe-tag = testers.invalidateFetcherByDrvHash fetchgit { + name = "describe-tag-nix-source"; + url = "https://github.com/NixOS/nix"; + tag = "2.3.15"; + hash = "sha256-7DszvbCNTjpzGRmpIVAWXk20P0/XTrWZ79KSOGLrUWY="; + postCheckout = '' + { git -C "$out" describe || echo "git describe failed"; } | tee describe-output.txt + ''; + }; + sparseCheckout = testers.invalidateFetcherByDrvHash fetchgit { name = "sparse-checkout-nix-source"; url = "https://github.com/NixOS/nix"; From e03674923594275aa27e7cb40d892684cde244af Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Wed, 10 Dec 2025 02:04:28 +0800 Subject: [PATCH 04/13] nix-prefetch-git: dont't fetch tags when deep clone unless leaving .git --- pkgs/build-support/fetchgit/nix-prefetch-git | 31 +++++++++++++++++--- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/pkgs/build-support/fetchgit/nix-prefetch-git b/pkgs/build-support/fetchgit/nix-prefetch-git index c3df5fc502b1d..e0406b49530b9 100755 --- a/pkgs/build-support/fetchgit/nix-prefetch-git +++ b/pkgs/build-support/fetchgit/nix-prefetch-git @@ -181,9 +181,26 @@ checkout_hash(){ hash=$(hash_from_ref "$ref") fi - [[ -z "$deepClone" ]] && \ - clean_git fetch ${builder:+--progress} --depth=1 origin "$hash" || \ - clean_git fetch -t ${builder:+--progress} origin || return 1 + fetchTagsFlags=("--tags") + if [[ -z "$fetchTags" ]]; then + fetchTagsFlags=("--no-tags") + fi + { + if [[ -z "$deepClone" ]]; then + clean_git fetch "${fetchTagsFlags[@]}" ${builder:+--progress} --depth=1 origin "$hash" + else + clean_git fetch "${fetchTagsFlags[@]}" ${builder:+--progress} origin + fi + } || { + echo "ERROR: \`git fetch' failed." >&2 + # Git remotes using the "dumb" protocol does not support shallow fetch; + # fall back to deep fetch if shallow fetch failed. + # TODO(@ShamrockLee): Determine whether the transfer protocol is smart reliably. + if [[ -z "$deepClone" ]] || [[ -z "$fetchTags" ]]; then + echo "This might be due to the dumb transfer protocol not supporting shallow fetch or no-tag cloning. Trying with \`--deep-clone' and \`--fetch-tags'..." >&2 + clean_git fetch --tags ${builder:+--progress} origin || return 1 + fi + } || return 1 local object_type=$(git cat-file -t "$hash") if [[ "$object_type" == "commit" || "$object_type" == "tag" ]]; then @@ -253,7 +270,13 @@ clone(){ ) # Fetch all tags if requested - if test -n "$fetchTags"; then + # The fetched tags are potentially non-reproducible, as tags are mutable parts of the Git tree. + # + # `deepClone` used to effectively imply `fetchTags`. + # We avoid such behaviour to enhance the `postCheckout` reproducibility, + # while keeping the old behaviour for `.git` for backward compatibility purposes. + # In bash, `&&` doesn't take precedence over `||``, and they are evaluated left-to-right. + if [[ -n "$deepClone" ]] && [[ -n "$leaveDotGit" ]] || [[ -n "$fetchTags" ]]; then echo "fetching all tags..." >&2 clean_git fetch origin 'refs/tags/*:refs/tags/*' || echo "warning: failed to fetch some tags" >&2 fi From 29458868f15aa1d1227e931f2c9f07690ba2fc5e Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Sun, 30 Nov 2025 19:35:37 +0800 Subject: [PATCH 05/13] tests.fetchgit: add submodule-revision-count --- pkgs/build-support/fetchgit/tests.nix | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/pkgs/build-support/fetchgit/tests.nix b/pkgs/build-support/fetchgit/tests.nix index 261330564c433..46a1e8173bdfc 100644 --- a/pkgs/build-support/fetchgit/tests.nix +++ b/pkgs/build-support/fetchgit/tests.nix @@ -104,6 +104,20 @@ postFetch = "rm -r $out/.git"; }; + submodule-revision-count = testers.invalidateFetcherByDrvHash fetchgit { + name = "submodule-revision-count-source"; + url = "https://github.com/pineapplehunter/nix-test-repo-with-submodule"; + rev = "26473335b84ead88ee0a3b649b1c7fa4a91cfd4a"; + hash = "sha256-ok1e6Pb0fII5TF8HXF8DXaRGSoq7kgRCoXqSEauh1wk="; + fetchSubmodules = true; + deepClone = true; + leaveDotGit = false; + postCheckout = '' + { git -C "$out" rev-list --count HEAD || echo "git rev-list failed"; } | tee "$out/revision_count.txt" + { git -C "$out/nix-test-repo-submodule" rev-list --count HEAD || echo "git rev-list failed"; } | tee "$out/nix-test-repo-submodule/revision_count.txt" + ''; + }; + submodule-leave-git-deep = testers.invalidateFetcherByDrvHash fetchgit { name = "submodule-leave-git-deep-source"; url = "https://github.com/pineapplehunter/nix-test-repo-with-submodule"; From 1beaa0517efb370ef91b8b54341d708ecc8d3f89 Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Mon, 24 Nov 2025 05:11:05 +0800 Subject: [PATCH 06/13] fetchurl: sort inherited variables --- pkgs/build-support/fetchurl/default.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkgs/build-support/fetchurl/default.nix b/pkgs/build-support/fetchurl/default.nix index 5ffb70e3a9f8a..fda7c95d3cf06 100644 --- a/pkgs/build-support/fetchurl/default.nix +++ b/pkgs/build-support/fetchurl/default.nix @@ -300,11 +300,11 @@ lib.extendMkDerivation { curlOptsList = lib.escapeShellArgs curlOptsList; inherit - showURLs - mirrorsFile - postFetch downloadToTemp executable + mirrorsFile + postFetch + showURLs ; impureEnvVars = impureEnvVars ++ netrcImpureEnvVars; From a49d1dc937e58605d52bc83f3dc6fcdbb09e006c Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Mon, 24 Nov 2025 06:11:56 +0800 Subject: [PATCH 07/13] fetchurl: use __structuredAttrs = true and pass curlOptsList directly --- pkgs/build-support/fetchurl/builder.sh | 5 +++-- pkgs/build-support/fetchurl/default.nix | 6 ++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/pkgs/build-support/fetchurl/builder.sh b/pkgs/build-support/fetchurl/builder.sh index 560b912d414f2..44ac80737bc07 100644 --- a/pkgs/build-support/fetchurl/builder.sh +++ b/pkgs/build-support/fetchurl/builder.sh @@ -1,3 +1,4 @@ +source "$NIX_ATTRS_SH_FILE" source $mirrorsFile curlVersion=$(curl -V | head -1 | cut -d' ' -f2) @@ -22,10 +23,10 @@ if ! [ -f "$SSL_CERT_FILE" ]; then curl+=(--insecure) fi -eval "curl+=($curlOptsList)" +curl+=("${curlOptsList[@]}") curl+=( - $curlOpts + ${curlOpts[*]} $NIX_CURL_FLAGS ) diff --git a/pkgs/build-support/fetchurl/default.nix b/pkgs/build-support/fetchurl/default.nix index fda7c95d3cf06..26aaa90e1bccb 100644 --- a/pkgs/build-support/fetchurl/default.nix +++ b/pkgs/build-support/fetchurl/default.nix @@ -136,6 +136,7 @@ lib.extendMkDerivation { # Passthru information, if any. passthru ? { }, + # Doing the download on a remote machine just duplicates network # traffic, so don't do that by default preferLocalBuild ? true, @@ -238,6 +239,8 @@ lib.extendMkDerivation { derivationArgs // { + __structuredAttrs = true; + name = if finalAttrs.pname or null != null && finalAttrs.version or null != null then "${finalAttrs.pname}-${finalAttrs.version}" @@ -297,9 +300,8 @@ lib.extendMkDerivation { '' ) curlOpts; - curlOptsList = lib.escapeShellArgs curlOptsList; - inherit + curlOptsList downloadToTemp executable mirrorsFile From 19e721ad3bd1221863d43e10ef31f14deb07398c Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Wed, 26 Nov 2025 10:23:20 +0800 Subject: [PATCH 08/13] fetchgit: use __structuredAttrs = true --- pkgs/build-support/fetchgit/builder.sh | 2 ++ pkgs/build-support/fetchgit/default.nix | 8 +++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/pkgs/build-support/fetchgit/builder.sh b/pkgs/build-support/fetchgit/builder.sh index 704f14598deae..b4c2f67830b17 100644 --- a/pkgs/build-support/fetchgit/builder.sh +++ b/pkgs/build-support/fetchgit/builder.sh @@ -4,6 +4,8 @@ # - revision specified and remote without HEAD # +source "$NIX_ATTRS_SH_FILE" + echo "exporting $url (rev $rev) into $out" runHook preFetch diff --git a/pkgs/build-support/fetchgit/default.nix b/pkgs/build-support/fetchgit/default.nix index 6bd8b632f2f8a..26b73ba1f5407 100644 --- a/pkgs/build-support/fetchgit/default.nix +++ b/pkgs/build-support/fetchgit/default.nix @@ -135,6 +135,8 @@ lib.makeOverridable ( derivationArgs // { + __structuredAttrs = true; + inherit name; builder = ./builder.sh; @@ -228,7 +230,11 @@ lib.makeOverridable ( "FETCHGIT_HTTP_PROXIES" ]; - inherit preferLocalBuild meta allowedRequisites; + outputChecks.out = { + ${if allowedRequisites != null then "allowedRequisites" else null} = allowedRequisites; + }; + + inherit preferLocalBuild meta; env = { NIX_PREFETCH_GIT_CHECKOUT_HOOK = finalAttrs.postCheckout; From 5a44c12604249f007af2e0d8dea717144b996457 Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Wed, 29 Oct 2025 01:42:15 +0800 Subject: [PATCH 09/13] fetchFromGitHub: explicitly support addditional fetchgit args --- pkgs/build-support/fetchgithub/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/build-support/fetchgithub/default.nix b/pkgs/build-support/fetchgithub/default.nix index 1283dcd7d25ad..74659e559d496 100644 --- a/pkgs/build-support/fetchgithub/default.nix +++ b/pkgs/build-support/fetchgithub/default.nix @@ -27,7 +27,7 @@ lib.makeOverridable ( varPrefix ? null, passthru ? { }, meta ? { }, - ... # For hash agility + ... # For hash agility and additional fetchgit arguments }@args: assert ( From d7d156079b076167d5269c286221ba4207774542 Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Sun, 16 Nov 2025 15:16:54 +0800 Subject: [PATCH 10/13] fetchFromGitHub: converge arguments that determines useFetchGit --- pkgs/build-support/fetchgithub/default.nix | 97 +++++++++++++--------- 1 file changed, 58 insertions(+), 39 deletions(-) diff --git a/pkgs/build-support/fetchgithub/default.nix b/pkgs/build-support/fetchgithub/default.nix index 74659e559d496..7f6cb26e9387f 100644 --- a/pkgs/build-support/fetchgithub/default.nix +++ b/pkgs/build-support/fetchgithub/default.nix @@ -4,8 +4,40 @@ fetchgit, fetchzip, }: +let + # Here defines fetchFromGitHub arguments that determines useFetchGit, + # The attribute value is their default values. + # As fetchFromGitHub prefers fetchzip for hash stability, + # `defaultFetchGitArgs` attributes should lead to `useFetchGit = false`. + useFetchGitArgsDefault = { + deepClone = false; + fetchSubmodules = false; # This differs from fetchgit's default + fetchLFS = false; + forceFetchGit = false; + leaveDotGit = null; + rootDir = ""; + sparseCheckout = null; + }; + useFetchGitArgsDefaultNullable = { + leaveDotGit = false; + sparseCheckout = [ ]; + }; -lib.makeOverridable ( + useFetchGitargsDefaultNonNull = useFetchGitArgsDefault // useFetchGitArgsDefaultNullable; + + # useFetchGitArgsWD to exclude from automatic passing. + # Other useFetchGitArgsWD will pass down to fetchgit. + excludeUseFetchGitArgNames = [ + "forceFetchGit" + ]; + + faUseFetchGit = lib.mapAttrs (_: _: true) useFetchGitArgsDefault; + + adjustFunctionArgs = f: lib.setFunctionArgs f (faUseFetchGit // lib.functionArgs f); + + decorate = f: lib.makeOverridable (adjustFunctionArgs f); +in +decorate ( { owner, repo, @@ -13,16 +45,7 @@ lib.makeOverridable ( rev ? null, # TODO(@ShamrockLee): Add back after reconstruction with lib.extendMkDerivation # name ? repoRevToNameMaybe finalAttrs.repo (lib.revOrTag finalAttrs.revCustom finalAttrs.tag) "github", - # `fetchFromGitHub` defaults to use `fetchzip` for better hash stability. - # We default not to fetch submodules, which is contrary to `fetchgit`'s default. - fetchSubmodules ? false, - leaveDotGit ? null, - deepClone ? false, private ? false, - forceFetchGit ? false, - fetchLFS ? false, - rootDir ? "", - sparseCheckout ? null, githubBase ? "github.com", varPrefix ? null, passthru ? { }, @@ -37,6 +60,16 @@ lib.makeOverridable ( ); let + useFetchGit = useFetchGitArgsWDNonNull != useFetchGitargsDefaultNonNull; + + useFetchGitArgs = lib.intersectAttrs useFetchGitArgsDefault args; + useFetchGitArgsWD = useFetchGitArgsDefault // useFetchGitArgs; + useFetchGitArgsWDPassing = removeAttrs useFetchGitArgsWD excludeUseFetchGitArgNames; + useFetchGitArgsWDNonNull = + useFetchGitArgsWD + // lib.mapAttrs ( + name: nonNullDefault: lib.defaultTo nonNullDefault useFetchGitArgsWD.${name} + ) useFetchGitArgsDefaultNullable; position = ( if args.meta.description or null != null then @@ -56,26 +89,19 @@ lib.makeOverridable ( # to indicate where derivation originates, similar to make-derivation.nix's mkDerivation position = "${position.file}:${toString position.line}"; }; - passthruAttrs = removeAttrs args [ - "owner" - "repo" - "tag" - "rev" - "fetchSubmodules" - "forceFetchGit" - "private" - "githubBase" - "varPrefix" - ]; + passthruAttrs = removeAttrs args ( + [ + "owner" + "repo" + "tag" + "rev" + "private" + "githubBase" + "varPrefix" + ] + ++ (if useFetchGit then excludeUseFetchGitArgNames else lib.attrNames faUseFetchGit) + ); varBase = "NIX${lib.optionalString (varPrefix != null) "_${varPrefix}"}_GITHUB_PRIVATE_"; - useFetchGit = - fetchSubmodules - || lib.defaultTo false leaveDotGit == true - || deepClone - || forceFetchGit - || fetchLFS - || (rootDir != "") - || lib.defaultTo [ ] sparseCheckout != [ ]; # We prefer fetchzip in cases we don't need submodules as the hash # is more stable in that case. fetcher = @@ -118,16 +144,9 @@ lib.makeOverridable ( passthruAttrs // ( if useFetchGit then - { - inherit - tag - rev - deepClone - fetchSubmodules - leaveDotGit - sparseCheckout - fetchLFS - ; + useFetchGitArgsWDPassing + // { + inherit tag rev; url = gitRepoUrl; inherit passthru; derivationArgs = { From 7148c756f38c32543d6b3dcb133f5c2153792a29 Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Mon, 1 Dec 2025 11:32:05 +0800 Subject: [PATCH 11/13] fetchgit: allow and encourage fetchTags as an attribute set --- pkgs/build-support/fetchgit/builder.sh | 2 +- pkgs/build-support/fetchgit/default.nix | 35 ++++++++-- pkgs/build-support/fetchgit/nix-prefetch-git | 68 +++++++++++++++++++- 3 files changed, 95 insertions(+), 10 deletions(-) diff --git a/pkgs/build-support/fetchgit/builder.sh b/pkgs/build-support/fetchgit/builder.sh index b4c2f67830b17..acff3690b0bff 100644 --- a/pkgs/build-support/fetchgit/builder.sh +++ b/pkgs/build-support/fetchgit/builder.sh @@ -20,7 +20,7 @@ $SHELL $fetcher --builder --url "$url" --out "$out" --rev "$rev" --name "$name" ${fetchLFS:+--fetch-lfs} \ ${deepClone:+--deepClone} \ ${fetchSubmodules:+--fetch-submodules} \ - ${fetchTags:+--fetch-tags} \ + "${fetchTagFlags[@]}" \ ${sparseCheckoutText:+--sparse-checkout "$sparseCheckoutText"} \ ${nonConeMode:+--non-cone-mode} \ ${branchName:+--branch-name "$branchName"} \ diff --git a/pkgs/build-support/fetchgit/default.nix b/pkgs/build-support/fetchgit/default.nix index 26b73ba1f5407..d84ecbcec4f9d 100644 --- a/pkgs/build-support/fetchgit/default.nix +++ b/pkgs/build-support/fetchgit/default.nix @@ -66,7 +66,7 @@ lib.makeOverridable ( # when rootDir is specified, avoid invalidating the result when rev changes append = if rootDir != "" then "-${lib.strings.sanitizeDerivationName rootDir}" else ""; }, - # When null, will default to: `deepClone || fetchTags` + # When null, will default to: `deepClone || fetchTags == true` for backward compatibility. leaveDotGit ? null, outputHash ? lib.fakeHash, outputHashAlgo ? null, @@ -98,8 +98,12 @@ lib.makeOverridable ( passthru ? { }, meta ? { }, allowedRequisites ? null, - # fetch all tags after tree (useful for git describe) - fetchTags ? false, + # Additional tags to fetch after tree (useful for git describe) + # Specify as `{ ${""} = [ "" "" ]; }`, + # where subpath begins with "/" and is relative to the fetching project root. + # The `subPath` of the main module is `"/"`. + # If specified as `true`, fetch all tags (with potential non-reproducibility). + fetchTags ? { }, # make this subdirectory the root of the result rootDir ? "", # GIT_CONFIG_GLOBAL (as a file) @@ -177,17 +181,36 @@ lib.makeOverridable ( preFetch postCheckout postFetch - fetchTags rootDir gitConfigFile ; + fetchTags = if fetchTags == false then { } else fetchTags; + fetchTagFlags = + if lib.isAttrs finalAttrs.fetchTags then + lib.concatLists ( + lib.attrValues ( + lib.mapAttrs ( + subPath: + lib.concatMap (tag: [ + "--fetch-submodule-tag" + subPath + tag + ]) + ) finalAttrs.fetchTags + ) + ) + else if finalAttrs.fetchTags == true then + [ "--fetch-tags" ] + else if finalAttrs.fetchTags == false then + [ ] + else + throw "fetchgit: unsupported fetchTags value, expecting either attribute sets of subpaths and tags, or boolean `true'"; leaveDotGit = if leaveDotGit != null then - assert fetchTags -> leaveDotGit; assert rootDir != "" -> !leaveDotGit; leaveDotGit else - deepClone || fetchTags; + deepClone || fetchTags == true; nonConeMode = lib.defaultTo (finalAttrs.rootDir != "") nonConeMode; inherit tag; revCustom = rev; diff --git a/pkgs/build-support/fetchgit/nix-prefetch-git b/pkgs/build-support/fetchgit/nix-prefetch-git index e0406b49530b9..92b9b6230acd7 100755 --- a/pkgs/build-support/fetchgit/nix-prefetch-git +++ b/pkgs/build-support/fetchgit/nix-prefetch-git @@ -12,6 +12,7 @@ fetchSubmodules= fetchLFS= builder= fetchTags= +declare -A tagsToFetch=() branchName=$NIX_PREFETCH_GIT_BRANCH_NAME # ENV params @@ -57,7 +58,10 @@ Options: --leave-dotGit Keep the .git directories. --fetch-lfs Fetch git Large File Storage (LFS) files. --fetch-submodules Fetch submodules. - --fetch-tags Fetch all tags (useful for git describe). + --fetch-submodule-tag subpath tag + Fetch a tag for a submodule or the main repo, useful for reproducible \`git describe'. + --fetch-tag Fetch the specified tag for the main repo, equivalent to \`--fetch-submodule-tag "" tag'. + --fetch-tags Fetch all tags. This option is less reproducible than \`--fetch-submodule-tag'. --builder Clone as fetchgit does, but url, rev, and out option are mandatory. --no-add-path Do not actually add the contents of the git repo to the store. --root-dir dir Directory in the repository that will be copied to the output instead of the full repository. @@ -73,6 +77,7 @@ clean_git(){ argi=0 argfun="" +preserve_argfun="" for arg; do if test -z "$argfun"; then case $arg in @@ -90,6 +95,8 @@ for arg; do --leave-dotGit) leaveDotGit=true;; --fetch-lfs) fetchLFS=true;; --fetch-submodules) fetchSubmodules=true;; + --fetch-submodule-tag) argfun=add_submodule_tag_key;; + --fetch-tag) argfun=add_main_tag;; --fetch-tags) fetchTags=true;; --builder) builder=true;; --no-add-path) noAddPath=true;; @@ -111,8 +118,24 @@ for arg; do var=${argfun#set_} eval "$var=$(printf %q "$arg")" ;; + add_submodule_tag_key) + argfun=add_submodule_tag_value_$arg + preserve_argfun=1 + ;; + add_submodule_tag_value_*) + key=${argfun#add_submodule_tag_value_} + key="/${key#/}" + tagsToFetch[$key]="${tagsToFetch[$key]-}${tagsToFetch[$key]:+ } $arg" + ;; + add_main_tag) + key="/" + tagsToFetch[$key]="${tagsToFetch[$key]-}${tagsToFetch[$key]:+ } $arg" + ;; esac - argfun="" + if [[ -z "$preserve_argfun" ]]; then + argfun="" + fi + preserve_argfun="" fi done @@ -293,6 +316,14 @@ clone(){ init_submodules fi + for key in "${!tagsToFetch[@]}"; do + subpath="${key#/}" + echo "fetching specified tags${subpath:+ at submodule $subpath}..." >&2 + for tagToFetch in ${tagsToFetch[$key]}; do + clean_git -C "$subpath" fetch origin "refs/tags/$tagToFetch:refs/tags/$tagToFetch" || echo "warning: failed to fetch tag $tagToFetch" >&2 + done + done + if [ -z "$builder" ] && [ -f .topdeps ]; then if tg help &>/dev/null; then echo "populating TopGit branches..." @@ -448,6 +479,21 @@ json_escape() { echo "$s" } +json_list() { + local result= + local arg + for arg; do + if [[ -n "$isFirst" ]]; then + result="$result, " + else + result="[" + fi + result="$result\"$(json_escape "$arg")\"" + done + result="$result]" + echo "$result" +} + print_results() { hash="$1" if ! test -n "$QUIET"; then @@ -474,7 +520,23 @@ print_results() { "fetchLFS": $([[ -n "$fetchLFS" ]] && echo true || echo false), "fetchSubmodules": $([[ -n "$fetchSubmodules" ]] && echo true || echo false), "deepClone": $([[ -n "$deepClone" ]] && echo true || echo false), - "fetchTags": $([[ -n "$fetchTags" ]] && echo true || echo false), + "fetchTags": $( + if [[ -n "$fetchTags" ]]; then + echo true + elif ((${#tagsToFetch[@]})); then + keys=("${!tagsToFetch[@]}") + keysWithComma=("${keys[@]:0:${#keys[@]-1}}") + keyLast="${keys[@]:${#keys[@]-1}:1}" + echo "{" + for key in "${keysWithComma[@]}"; do + echo " \"$(json_escape "${key#/}")\": $(json_list ${tagsToFetch[$key]})," + done + echo " \"$(json_escape "${key#/}")\": $(json_list ${tagsToFetch[$keyLast]})" + echo " }" + else + echo "{}" + fi + ), "leaveDotGit": $([[ -n "$leaveDotGit" ]] && echo true || echo false), "rootDir": "$(json_escape "$rootDir")" } From e0d1ed1d309a654651d41810036f7f08f81873ad Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Mon, 1 Dec 2025 12:45:55 +0800 Subject: [PATCH 12/13] fetchgit: support specifying both rev and tags for unstable version git-describe --- pkgs/build-support/fetchgit/default.nix | 21 ++++++++++++++++----- pkgs/build-support/fetchgit/tests.nix | 12 ++++++++++++ 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/pkgs/build-support/fetchgit/default.nix b/pkgs/build-support/fetchgit/default.nix index d84ecbcec4f9d..d550684010589 100644 --- a/pkgs/build-support/fetchgit/default.nix +++ b/pkgs/build-support/fetchgit/default.nix @@ -26,12 +26,10 @@ let rev ? null, tag ? null, }: - if tag != null && rev != null then - throw "fetchgit requires one of either `rev` or `tag` to be provided (not both)." + if rev != null then + rev else if tag != null then "refs/tags/${tag}" - else if rev != null then - rev else # FIXME fetching HEAD if no rev or tag is provided is problematic at best "HEAD"; @@ -184,7 +182,20 @@ lib.makeOverridable ( rootDir gitConfigFile ; - fetchTags = if fetchTags == false then { } else fetchTags; + fetchTags = + let + addAdditionalTag = finalAttrs.revCustom != null && finalAttrs.tag != null; + additionalTags = [ finalAttrs.tag ]; + in + if lib.isAttrs fetchTags then + fetchTags + // { + ${if addAdditionalTag then "" else null} = fetchTags."" or [ ] ++ additionalTags; + } + else if fetchTags == false then + { ${if addAdditionalTag then "" else null} = additionalTags; } + else + fetchTags; fetchTagFlags = if lib.isAttrs finalAttrs.fetchTags then lib.concatLists ( diff --git a/pkgs/build-support/fetchgit/tests.nix b/pkgs/build-support/fetchgit/tests.nix index 46a1e8173bdfc..9380f74543a9e 100644 --- a/pkgs/build-support/fetchgit/tests.nix +++ b/pkgs/build-support/fetchgit/tests.nix @@ -44,6 +44,18 @@ ''; }; + describe-tag-unstable-version = testers.invalidateFetcherByDrvHash fetchgit { + name = "describe-tag-nix-source"; + url = "https://github.com/NixOS/nix"; + rev = "9d9dbe6ed05854e03811c361a3380e09183f4f4a"; + # for `git describe` + tag = "2.3.15"; + hash = "sha256-7DszvbCNTjpzGRmpIVAWXk20P0/XTrWZ79KSOGLrUWY="; + postCheckout = '' + { git -C "$out" describe || echo "git describe failed"; } | tee describe-output.txt + ''; + }; + sparseCheckout = testers.invalidateFetcherByDrvHash fetchgit { name = "sparse-checkout-nix-source"; url = "https://github.com/NixOS/nix"; From 5625e5da638308f5d15681f492735ebaded59932 Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Tue, 9 Dec 2025 22:55:00 +0800 Subject: [PATCH 13/13] fetchFromGitHub: adopt fetchgit's postCheckout --- pkgs/build-support/fetchgithub/default.nix | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pkgs/build-support/fetchgithub/default.nix b/pkgs/build-support/fetchgithub/default.nix index 7f6cb26e9387f..ca5aa39906f4c 100644 --- a/pkgs/build-support/fetchgithub/default.nix +++ b/pkgs/build-support/fetchgithub/default.nix @@ -15,6 +15,7 @@ let fetchLFS = false; forceFetchGit = false; leaveDotGit = null; + postCheckout = ""; rootDir = ""; sparseCheckout = null; }; @@ -54,9 +55,9 @@ decorate ( }@args: assert ( - lib.assertMsg (lib.xor (tag == null) ( - rev == null - )) "fetchFromGitHub requires one of either `rev` or `tag` to be provided (not both)." + lib.assertMsg ( + tag != null || rev != null + ) "fetchFromGitHub requires at least one of `rev` or `tag` to be provided." ); let