From 1d6c7d66505debebd6bdc5877d45f8833c4616b5 Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Fri, 14 Nov 2025 13:27:51 +0800 Subject: [PATCH 01/26] fetchgit: move assertions down to argument values --- pkgs/build-support/fetchgit/default.nix | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/pkgs/build-support/fetchgit/default.nix b/pkgs/build-support/fetchgit/default.nix index 5a8ea3cf04880..4d0c2c7af8b1a 100644 --- a/pkgs/build-support/fetchgit/default.nix +++ b/pkgs/build-support/fetchgit/default.nix @@ -128,10 +128,6 @@ lib.makeOverridable ( server admins start using the new version? */ - assert nonConeMode -> (sparseCheckout != [ ]); - assert fetchTags -> leaveDotGit; - assert rootDir != "" -> !leaveDotGit; - if builtins.isString sparseCheckout then # Changed to throw on 2023-06-04 throw @@ -154,14 +150,15 @@ lib.makeOverridable ( inherit outputHash outputHashAlgo; outputHashMode = "recursive"; - # git-sparse-checkout(1) says: - # > When the --stdin option is provided, the directories or patterns are read - # > from standard in as a newline-delimited list instead of from the arguments. - sparseCheckout = builtins.concatStringsSep "\n" sparseCheckout; + sparseCheckout = + assert nonConeMode -> (sparseCheckout != [ ]); + # git-sparse-checkout(1) says: + # > When the --stdin option is provided, the directories or patterns are read + # > from standard in as a newline-delimited list instead of from the arguments. + builtins.concatStringsSep "\n" sparseCheckout; inherit url - leaveDotGit fetchLFS fetchSubmodules deepClone @@ -173,6 +170,10 @@ lib.makeOverridable ( rootDir gitConfigFile ; + leaveDotGit = + assert fetchTags -> leaveDotGit; + assert rootDir != "" -> !leaveDotGit; + leaveDotGit; inherit tag; revCustom = rev; rev = getRevWithTag { From 9fe4e65f3cbf46dbdf575b63a04df96acf27874a Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Sun, 16 Nov 2025 01:03:28 +0800 Subject: [PATCH 02/26] fetchgit: default argument leaveDotGit to null --- pkgs/build-support/fetchgit/default.nix | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/pkgs/build-support/fetchgit/default.nix b/pkgs/build-support/fetchgit/default.nix index 4d0c2c7af8b1a..ef2c2cd43093d 100644 --- a/pkgs/build-support/fetchgit/default.nix +++ b/pkgs/build-support/fetchgit/default.nix @@ -66,7 +66,8 @@ lib.makeOverridable ( # when rootDir is specified, avoid invalidating the result when rev changes append = if rootDir != "" then "-${lib.strings.sanitizeDerivationName rootDir}" else ""; }, - leaveDotGit ? deepClone || fetchTags, + # When null, will default to: `deepClone || fetchTags` + leaveDotGit ? null, outputHash ? lib.fakeHash, outputHashAlgo ? null, fetchSubmodules ? true, @@ -171,9 +172,12 @@ lib.makeOverridable ( gitConfigFile ; leaveDotGit = - assert fetchTags -> leaveDotGit; - assert rootDir != "" -> !leaveDotGit; - leaveDotGit; + if leaveDotGit != null then + assert fetchTags -> leaveDotGit; + assert rootDir != "" -> !leaveDotGit; + leaveDotGit + else + deepClone || fetchTags; inherit tag; revCustom = rev; rev = getRevWithTag { From d461ad81b21a1253a92dac886d5ee89fc849217d Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Sun, 16 Nov 2025 03:53:47 +0800 Subject: [PATCH 03/26] fetchFromGitHub: pass leaveDotGit unconditionally using the null default --- pkgs/build-support/fetchgithub/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/build-support/fetchgithub/default.nix b/pkgs/build-support/fetchgithub/default.nix index ae5ff338264a2..46fed2304871e 100644 --- a/pkgs/build-support/fetchgithub/default.nix +++ b/pkgs/build-support/fetchgithub/default.nix @@ -68,7 +68,7 @@ lib.makeOverridable ( varBase = "NIX${lib.optionalString (varPrefix != null) "_${varPrefix}"}_GITHUB_PRIVATE_"; useFetchGit = fetchSubmodules - || (leaveDotGit == true) + || lib.defaultTo false leaveDotGit == true || deepClone || forceFetchGit || fetchLFS @@ -122,6 +122,7 @@ lib.makeOverridable ( rev deepClone fetchSubmodules + leaveDotGit sparseCheckout fetchLFS ; @@ -135,7 +136,6 @@ lib.makeOverridable ( ; }; } - // lib.optionalAttrs (leaveDotGit != null) { inherit leaveDotGit; } else let revWithTag = finalAttrs.rev; From 59168687f99b8c1eadba9edb814d95200e99e53a Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Mon, 17 Nov 2025 03:34:58 +0800 Subject: [PATCH 04/26] fetchFromGitHub: elaborate the fetchSubmodules default choice --- pkgs/build-support/fetchgithub/default.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkgs/build-support/fetchgithub/default.nix b/pkgs/build-support/fetchgithub/default.nix index 46fed2304871e..c02732a90e96d 100644 --- a/pkgs/build-support/fetchgithub/default.nix +++ b/pkgs/build-support/fetchgithub/default.nix @@ -13,6 +13,8 @@ lib.makeOverridable ( rev ? null, # TODO(@ShamrockLee): Add back after reconstruction with lib.extendMkDerivation # name ? repoRevToNameMaybe finalAttrs.repo (lib.revOrTag finalAttrs.revCustom finalAttrs.tag) "github", + # `fetchFromGitHub` defaults to use `fetchzip` for better hash stability. + # We default not to fetch submodules, which is contrary to `fetchgit`'s default. fetchSubmodules ? false, leaveDotGit ? null, deepClone ? false, From 6c5dc5d57f446a9c65d01699a9b16963d54d4e06 Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Mon, 17 Nov 2025 03:39:53 +0800 Subject: [PATCH 05/26] fetchgit: default nonConeMode to null and manage its default internally --- pkgs/build-support/fetchgit/default.nix | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkgs/build-support/fetchgit/default.nix b/pkgs/build-support/fetchgit/default.nix index ef2c2cd43093d..eac73f713c74f 100644 --- a/pkgs/build-support/fetchgit/default.nix +++ b/pkgs/build-support/fetchgit/default.nix @@ -74,7 +74,8 @@ lib.makeOverridable ( deepClone ? false, branchName ? null, sparseCheckout ? lib.optional (rootDir != "") rootDir, - nonConeMode ? rootDir != "", + # When null, will default to: `rootDir != ""` + nonConeMode ? null, nativeBuildInputs ? [ ], # Shell code executed before the file has been fetched. This, in # particular, can do things like set NIX_PREFETCH_GIT_CHECKOUT_HOOK to @@ -164,7 +165,6 @@ lib.makeOverridable ( fetchSubmodules deepClone branchName - nonConeMode preFetch postFetch fetchTags @@ -178,6 +178,7 @@ lib.makeOverridable ( leaveDotGit else deepClone || fetchTags; + nonConeMode = lib.defaultTo (rootDir != "") nonConeMode; inherit tag; revCustom = rev; rev = getRevWithTag { From ea5b3c6d5d2de8241377d31292b20c21f53af488 Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Thu, 4 Dec 2025 15:25:35 +0800 Subject: [PATCH 06/26] fetchgit: nonConeMode: reference depending attributes from finalAttrs --- pkgs/build-support/fetchgit/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/build-support/fetchgit/default.nix b/pkgs/build-support/fetchgit/default.nix index eac73f713c74f..64a51e2a8059d 100644 --- a/pkgs/build-support/fetchgit/default.nix +++ b/pkgs/build-support/fetchgit/default.nix @@ -178,7 +178,7 @@ lib.makeOverridable ( leaveDotGit else deepClone || fetchTags; - nonConeMode = lib.defaultTo (rootDir != "") nonConeMode; + nonConeMode = lib.defaultTo (finalAttrs.rootDir != "") nonConeMode; inherit tag; revCustom = rev; rev = getRevWithTag { From 83b42e34eca9347f410f3b41c71e50d3c2536f81 Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Mon, 17 Nov 2025 01:26:38 +0800 Subject: [PATCH 07/26] fetchgit: keep sparseCheckout a list and stringify as sparseCheckoutText --- pkgs/build-support/fetchgit/builder.sh | 2 +- pkgs/build-support/fetchgit/default.nix | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pkgs/build-support/fetchgit/builder.sh b/pkgs/build-support/fetchgit/builder.sh index 8f72881d3b911..704f14598deae 100644 --- a/pkgs/build-support/fetchgit/builder.sh +++ b/pkgs/build-support/fetchgit/builder.sh @@ -19,7 +19,7 @@ $SHELL $fetcher --builder --url "$url" --out "$out" --rev "$rev" --name "$name" ${deepClone:+--deepClone} \ ${fetchSubmodules:+--fetch-submodules} \ ${fetchTags:+--fetch-tags} \ - ${sparseCheckout:+--sparse-checkout "$sparseCheckout"} \ + ${sparseCheckoutText:+--sparse-checkout "$sparseCheckoutText"} \ ${nonConeMode:+--non-cone-mode} \ ${branchName:+--branch-name "$branchName"} \ ${rootDir:+--root-dir "$rootDir"} diff --git a/pkgs/build-support/fetchgit/default.nix b/pkgs/build-support/fetchgit/default.nix index 64a51e2a8059d..7d634f684fa1e 100644 --- a/pkgs/build-support/fetchgit/default.nix +++ b/pkgs/build-support/fetchgit/default.nix @@ -152,8 +152,9 @@ lib.makeOverridable ( inherit outputHash outputHashAlgo; outputHashMode = "recursive"; - sparseCheckout = - assert nonConeMode -> (sparseCheckout != [ ]); + inherit sparseCheckout; + sparseCheckoutText = + assert finalAttrs.nonConeMode -> (finalAttrs.sparseCheckout != [ ]); # git-sparse-checkout(1) says: # > When the --stdin option is provided, the directories or patterns are read # > from standard in as a newline-delimited list instead of from the arguments. From dd264228ff306e3cce538e4281f834476483393d Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Mon, 17 Nov 2025 01:44:17 +0800 Subject: [PATCH 08/26] fetchgit: default sparseCheckout to null and handle the default internally --- pkgs/build-support/fetchgit/default.nix | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkgs/build-support/fetchgit/default.nix b/pkgs/build-support/fetchgit/default.nix index 7d634f684fa1e..76539d2131835 100644 --- a/pkgs/build-support/fetchgit/default.nix +++ b/pkgs/build-support/fetchgit/default.nix @@ -73,7 +73,8 @@ lib.makeOverridable ( fetchSubmodules ? true, deepClone ? false, branchName ? null, - sparseCheckout ? lib.optional (rootDir != "") rootDir, + # When null, will default to: `lib.optional (rootdir != "") rootdir` + sparseCheckout ? null, # When null, will default to: `rootDir != ""` nonConeMode ? null, nativeBuildInputs ? [ ], @@ -152,7 +153,7 @@ lib.makeOverridable ( inherit outputHash outputHashAlgo; outputHashMode = "recursive"; - inherit sparseCheckout; + sparseCheckout = lib.defaultTo (lib.optional (rootDir != "") rootDir) sparseCheckout; sparseCheckoutText = assert finalAttrs.nonConeMode -> (finalAttrs.sparseCheckout != [ ]); # git-sparse-checkout(1) says: From d1275c4ff19cd51361d9474a306c85bb6b4b251e Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Mon, 17 Nov 2025 01:36:04 +0800 Subject: [PATCH 09/26] fetchgit: move sparseCheckout type check down to sparseCheckoutText --- pkgs/build-support/fetchgit/default.nix | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/pkgs/build-support/fetchgit/default.nix b/pkgs/build-support/fetchgit/default.nix index 76539d2131835..450103027b6bd 100644 --- a/pkgs/build-support/fetchgit/default.nix +++ b/pkgs/build-support/fetchgit/default.nix @@ -131,11 +131,6 @@ lib.makeOverridable ( server admins start using the new version? */ - if builtins.isString sparseCheckout then - # Changed to throw on 2023-06-04 - throw - "Please provide directories/patterns for sparse checkout as a list of strings. Passing a (multi-line) string is not supported any more." - else derivationArgs // { inherit name; @@ -155,11 +150,15 @@ lib.makeOverridable ( sparseCheckout = lib.defaultTo (lib.optional (rootDir != "") rootDir) sparseCheckout; sparseCheckoutText = + # Changed to throw on 2023-06-04 + assert ( + lib.assertMsg (lib.isList finalAttrs.sparseCheckout) "Please provide directories/patterns for sparse checkout as a list of strings. Passing a (multi-line) string is not supported any more." + ); assert finalAttrs.nonConeMode -> (finalAttrs.sparseCheckout != [ ]); # git-sparse-checkout(1) says: # > When the --stdin option is provided, the directories or patterns are read # > from standard in as a newline-delimited list instead of from the arguments. - builtins.concatStringsSep "\n" sparseCheckout; + builtins.concatStringsSep "\n" finalAttrs.sparseCheckout; inherit url From 8c8557860213f05a58dbbc6b2dba80cef31d96b0 Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Mon, 17 Nov 2025 01:52:24 +0800 Subject: [PATCH 10/26] fetchgit: format expression after eliminating global assertions --- pkgs/build-support/fetchgit/default.nix | 192 ++++++++++++------------ 1 file changed, 96 insertions(+), 96 deletions(-) diff --git a/pkgs/build-support/fetchgit/default.nix b/pkgs/build-support/fetchgit/default.nix index 450103027b6bd..3bcda3af1c597 100644 --- a/pkgs/build-support/fetchgit/default.nix +++ b/pkgs/build-support/fetchgit/default.nix @@ -131,103 +131,103 @@ lib.makeOverridable ( server admins start using the new version? */ - derivationArgs - // { - inherit name; - - builder = ./builder.sh; - fetcher = ./nix-prefetch-git; - - nativeBuildInputs = [ - git - cacert - ] - ++ lib.optionals fetchLFS [ git-lfs ] - ++ nativeBuildInputs; - - inherit outputHash outputHashAlgo; - outputHashMode = "recursive"; - - sparseCheckout = lib.defaultTo (lib.optional (rootDir != "") rootDir) sparseCheckout; - sparseCheckoutText = - # Changed to throw on 2023-06-04 - assert ( - lib.assertMsg (lib.isList finalAttrs.sparseCheckout) "Please provide directories/patterns for sparse checkout as a list of strings. Passing a (multi-line) string is not supported any more." - ); - assert finalAttrs.nonConeMode -> (finalAttrs.sparseCheckout != [ ]); - # git-sparse-checkout(1) says: - # > When the --stdin option is provided, the directories or patterns are read - # > from standard in as a newline-delimited list instead of from the arguments. - builtins.concatStringsSep "\n" finalAttrs.sparseCheckout; - - inherit - url - fetchLFS - fetchSubmodules - deepClone - branchName - preFetch - postFetch - fetchTags - rootDir - gitConfigFile - ; - leaveDotGit = - if leaveDotGit != null then - assert fetchTags -> leaveDotGit; - assert rootDir != "" -> !leaveDotGit; - leaveDotGit - else - deepClone || fetchTags; - nonConeMode = lib.defaultTo (finalAttrs.rootDir != "") nonConeMode; - inherit tag; - revCustom = rev; - rev = getRevWithTag { - inherit (finalAttrs) tag; - rev = finalAttrs.revCustom; - }; - - postHook = - if netrcPhase == null then - null - else - '' - ${netrcPhase} - # required that git uses the netrc file - mv {,.}netrc - export NETRC=$PWD/.netrc - export HOME=$PWD - ''; - - impureEnvVars = - lib.fetchers.proxyImpureEnvVars - ++ netrcImpureEnvVars - ++ [ - "GIT_PROXY_COMMAND" - "NIX_GIT_SSL_CAINFO" - "SOCKS_SERVER" - - # This is a parameter intended to be set by setup hooks or preFetch - # scripts that want per-URL control over HTTP proxies used by Git - # (if per-URL control isn't needed, `http_proxy` etc. will - # suffice). It must be a whitespace-separated (with backslash as an - # escape character) list of pairs like this: - # - # http://domain1/path1 proxy1 https://domain2/path2 proxy2 - # - # where the URLs are as documented in the `git-config` manual page - # under `http..*`, and the proxies are as documented on the - # same page under `http.proxy`. - "FETCHGIT_HTTP_PROXIES" - ]; - - inherit preferLocalBuild meta allowedRequisites; - - passthru = { - gitRepoUrl = url; - } - // passthru; + derivationArgs + // { + inherit name; + + builder = ./builder.sh; + fetcher = ./nix-prefetch-git; + + nativeBuildInputs = [ + git + cacert + ] + ++ lib.optionals fetchLFS [ git-lfs ] + ++ nativeBuildInputs; + + inherit outputHash outputHashAlgo; + outputHashMode = "recursive"; + + sparseCheckout = lib.defaultTo (lib.optional (rootDir != "") rootDir) sparseCheckout; + sparseCheckoutText = + # Changed to throw on 2023-06-04 + assert ( + lib.assertMsg (lib.isList finalAttrs.sparseCheckout) "Please provide directories/patterns for sparse checkout as a list of strings. Passing a (multi-line) string is not supported any more." + ); + assert finalAttrs.nonConeMode -> (finalAttrs.sparseCheckout != [ ]); + # git-sparse-checkout(1) says: + # > When the --stdin option is provided, the directories or patterns are read + # > from standard in as a newline-delimited list instead of from the arguments. + builtins.concatStringsSep "\n" finalAttrs.sparseCheckout; + + inherit + url + fetchLFS + fetchSubmodules + deepClone + branchName + preFetch + postFetch + fetchTags + rootDir + gitConfigFile + ; + leaveDotGit = + if leaveDotGit != null then + assert fetchTags -> leaveDotGit; + assert rootDir != "" -> !leaveDotGit; + leaveDotGit + else + deepClone || fetchTags; + nonConeMode = lib.defaultTo (finalAttrs.rootDir != "") nonConeMode; + inherit tag; + revCustom = rev; + rev = getRevWithTag { + inherit (finalAttrs) tag; + rev = finalAttrs.revCustom; + }; + + postHook = + if netrcPhase == null then + null + else + '' + ${netrcPhase} + # required that git uses the netrc file + mv {,.}netrc + export NETRC=$PWD/.netrc + export HOME=$PWD + ''; + + impureEnvVars = + lib.fetchers.proxyImpureEnvVars + ++ netrcImpureEnvVars + ++ [ + "GIT_PROXY_COMMAND" + "NIX_GIT_SSL_CAINFO" + "SOCKS_SERVER" + + # This is a parameter intended to be set by setup hooks or preFetch + # scripts that want per-URL control over HTTP proxies used by Git + # (if per-URL control isn't needed, `http_proxy` etc. will + # suffice). It must be a whitespace-separated (with backslash as an + # escape character) list of pairs like this: + # + # http://domain1/path1 proxy1 https://domain2/path2 proxy2 + # + # where the URLs are as documented in the `git-config` manual page + # under `http..*`, and the proxies are as documented on the + # same page under `http.proxy`. + "FETCHGIT_HTTP_PROXIES" + ]; + + inherit preferLocalBuild meta allowedRequisites; + + passthru = { + gitRepoUrl = url; } + // passthru; + } ); # No ellipsis. From 4c681c7ce093f04f7a12cc948fe16a055dc2ae59 Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Mon, 17 Nov 2025 03:29:12 +0800 Subject: [PATCH 11/26] fetchFromGitHub: default sparseCheckout to null --- pkgs/build-support/fetchgithub/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/build-support/fetchgithub/default.nix b/pkgs/build-support/fetchgithub/default.nix index c02732a90e96d..1283dcd7d25ad 100644 --- a/pkgs/build-support/fetchgithub/default.nix +++ b/pkgs/build-support/fetchgithub/default.nix @@ -22,7 +22,7 @@ lib.makeOverridable ( forceFetchGit ? false, fetchLFS ? false, rootDir ? "", - sparseCheckout ? lib.optional (rootDir != "") rootDir, + sparseCheckout ? null, githubBase ? "github.com", varPrefix ? null, passthru ? { }, @@ -75,7 +75,7 @@ lib.makeOverridable ( || forceFetchGit || fetchLFS || (rootDir != "") - || (sparseCheckout != [ ]); + || lib.defaultTo [ ] sparseCheckout != [ ]; # We prefer fetchzip in cases we don't need submodules as the hash # is more stable in that case. fetcher = From 5c51547b2287da9504b125b804ad4829ac0fa031 Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Thu, 4 Dec 2025 15:28:38 +0800 Subject: [PATCH 12/26] fetchgit: sparseCheckout: reference depending attributes from finalAttrs --- pkgs/build-support/fetchgit/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/build-support/fetchgit/default.nix b/pkgs/build-support/fetchgit/default.nix index 3bcda3af1c597..6c5abd592e559 100644 --- a/pkgs/build-support/fetchgit/default.nix +++ b/pkgs/build-support/fetchgit/default.nix @@ -148,7 +148,7 @@ lib.makeOverridable ( inherit outputHash outputHashAlgo; outputHashMode = "recursive"; - sparseCheckout = lib.defaultTo (lib.optional (rootDir != "") rootDir) sparseCheckout; + sparseCheckout = lib.defaultTo (lib.optional (finalAttrs.rootDir != "") finalAttrs.rootDir) sparseCheckout; sparseCheckoutText = # Changed to throw on 2023-06-04 assert ( From 239a6f82c13711847939e88a579f4bf004e08f13 Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Wed, 29 Oct 2025 01:42:15 +0800 Subject: [PATCH 13/26] fetchFromGitHub: explicitly support addditional fetchgit args --- pkgs/build-support/fetchgithub/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/build-support/fetchgithub/default.nix b/pkgs/build-support/fetchgithub/default.nix index 1283dcd7d25ad..74659e559d496 100644 --- a/pkgs/build-support/fetchgithub/default.nix +++ b/pkgs/build-support/fetchgithub/default.nix @@ -27,7 +27,7 @@ lib.makeOverridable ( varPrefix ? null, passthru ? { }, meta ? { }, - ... # For hash agility + ... # For hash agility and additional fetchgit arguments }@args: assert ( From 746e44320f0f972e609480dc3cdf1bc3364806c1 Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Sun, 16 Nov 2025 15:16:54 +0800 Subject: [PATCH 14/26] fetchFromGitHub: converge arguments that determines useFetchGit --- pkgs/build-support/fetchgithub/default.nix | 97 +++++++++++++--------- 1 file changed, 58 insertions(+), 39 deletions(-) diff --git a/pkgs/build-support/fetchgithub/default.nix b/pkgs/build-support/fetchgithub/default.nix index 74659e559d496..7f6cb26e9387f 100644 --- a/pkgs/build-support/fetchgithub/default.nix +++ b/pkgs/build-support/fetchgithub/default.nix @@ -4,8 +4,40 @@ fetchgit, fetchzip, }: +let + # Here defines fetchFromGitHub arguments that determines useFetchGit, + # The attribute value is their default values. + # As fetchFromGitHub prefers fetchzip for hash stability, + # `defaultFetchGitArgs` attributes should lead to `useFetchGit = false`. + useFetchGitArgsDefault = { + deepClone = false; + fetchSubmodules = false; # This differs from fetchgit's default + fetchLFS = false; + forceFetchGit = false; + leaveDotGit = null; + rootDir = ""; + sparseCheckout = null; + }; + useFetchGitArgsDefaultNullable = { + leaveDotGit = false; + sparseCheckout = [ ]; + }; -lib.makeOverridable ( + useFetchGitargsDefaultNonNull = useFetchGitArgsDefault // useFetchGitArgsDefaultNullable; + + # useFetchGitArgsWD to exclude from automatic passing. + # Other useFetchGitArgsWD will pass down to fetchgit. + excludeUseFetchGitArgNames = [ + "forceFetchGit" + ]; + + faUseFetchGit = lib.mapAttrs (_: _: true) useFetchGitArgsDefault; + + adjustFunctionArgs = f: lib.setFunctionArgs f (faUseFetchGit // lib.functionArgs f); + + decorate = f: lib.makeOverridable (adjustFunctionArgs f); +in +decorate ( { owner, repo, @@ -13,16 +45,7 @@ lib.makeOverridable ( rev ? null, # TODO(@ShamrockLee): Add back after reconstruction with lib.extendMkDerivation # name ? repoRevToNameMaybe finalAttrs.repo (lib.revOrTag finalAttrs.revCustom finalAttrs.tag) "github", - # `fetchFromGitHub` defaults to use `fetchzip` for better hash stability. - # We default not to fetch submodules, which is contrary to `fetchgit`'s default. - fetchSubmodules ? false, - leaveDotGit ? null, - deepClone ? false, private ? false, - forceFetchGit ? false, - fetchLFS ? false, - rootDir ? "", - sparseCheckout ? null, githubBase ? "github.com", varPrefix ? null, passthru ? { }, @@ -37,6 +60,16 @@ lib.makeOverridable ( ); let + useFetchGit = useFetchGitArgsWDNonNull != useFetchGitargsDefaultNonNull; + + useFetchGitArgs = lib.intersectAttrs useFetchGitArgsDefault args; + useFetchGitArgsWD = useFetchGitArgsDefault // useFetchGitArgs; + useFetchGitArgsWDPassing = removeAttrs useFetchGitArgsWD excludeUseFetchGitArgNames; + useFetchGitArgsWDNonNull = + useFetchGitArgsWD + // lib.mapAttrs ( + name: nonNullDefault: lib.defaultTo nonNullDefault useFetchGitArgsWD.${name} + ) useFetchGitArgsDefaultNullable; position = ( if args.meta.description or null != null then @@ -56,26 +89,19 @@ lib.makeOverridable ( # to indicate where derivation originates, similar to make-derivation.nix's mkDerivation position = "${position.file}:${toString position.line}"; }; - passthruAttrs = removeAttrs args [ - "owner" - "repo" - "tag" - "rev" - "fetchSubmodules" - "forceFetchGit" - "private" - "githubBase" - "varPrefix" - ]; + passthruAttrs = removeAttrs args ( + [ + "owner" + "repo" + "tag" + "rev" + "private" + "githubBase" + "varPrefix" + ] + ++ (if useFetchGit then excludeUseFetchGitArgNames else lib.attrNames faUseFetchGit) + ); varBase = "NIX${lib.optionalString (varPrefix != null) "_${varPrefix}"}_GITHUB_PRIVATE_"; - useFetchGit = - fetchSubmodules - || lib.defaultTo false leaveDotGit == true - || deepClone - || forceFetchGit - || fetchLFS - || (rootDir != "") - || lib.defaultTo [ ] sparseCheckout != [ ]; # We prefer fetchzip in cases we don't need submodules as the hash # is more stable in that case. fetcher = @@ -118,16 +144,9 @@ lib.makeOverridable ( passthruAttrs // ( if useFetchGit then - { - inherit - tag - rev - deepClone - fetchSubmodules - leaveDotGit - sparseCheckout - fetchLFS - ; + useFetchGitArgsWDPassing + // { + inherit tag rev; url = gitRepoUrl; inherit passthru; derivationArgs = { From 7f9a1c88bd229082a5409f7f3b6abcfcbcb28332 Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Thu, 13 Nov 2025 01:28:42 +0800 Subject: [PATCH 15/26] lib.makeOverrdable: preserve constructor override and metadata attributes --- lib/customisation.nix | 19 ++++++++++- lib/tests/misc.nix | 79 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+), 1 deletion(-) diff --git a/lib/customisation.nix b/lib/customisation.nix index 6159ce8ea27a4..63bea15755aa5 100644 --- a/lib/customisation.nix +++ b/lib/customisation.nix @@ -156,8 +156,25 @@ rec { let # Creates a functor with the same arguments as f mirrorArgs = mirrorFunctionArgs f; + # Recover overrider and additional attributes for f + # When f is a callable attribute set, + # it may contain its own `f.override` and additional attributes. + # This helper function recovers those attributes and decorate the overrider. + recoverMetadata = + if isAttrs f then + fDecorated: + # Preserve additional attributes for f + f + // fDecorated + # Decorate f.override if presented + // lib.optionalAttrs (f ? override) { + override = fdrv: makeOverridable (f.override fdrv); + } + else + id; + decorate = f': recoverMetadata (mirrorArgs f'); in - mirrorArgs ( + decorate ( origArgs: let result = f origArgs; diff --git a/lib/tests/misc.nix b/lib/tests/misc.nix index e4f2222b76850..4bf2a215129a7 100644 --- a/lib/tests/misc.nix +++ b/lib/tests/misc.nix @@ -202,6 +202,85 @@ runTests { }; }; + testOverridePreserveFunctionMetadata = + let + toCallableAttrs = f: setFunctionArgs f (functionArgs f); + constructDefinition = + { + a ? 3, + }: + toCallableAttrs ( + { + b ? 5, + }: + { + inherit a b; + } + ) + // { + inherit a; + c = 7; + }; + construct0 = makeOverridable constructDefinition { }; + construct1 = makeOverridable construct0; + construct0p = construct0.override { a = 11; }; + construct1p = construct1.override { a = 11; }; + in + { + expr = { + construct-metadata = { + inherit (construct1) a c; + }; + construct-overridden-metadata = { + v = construct0p.a; + inherit (construct1p) a c; + }; + construct-overridden-result-overrider = { + result-overriders-exist = mapAttrs (_: f: (f { }) ? override) { + inherit construct1 construct1p; + }; + result-overrider-functionality = { + overridden = { + inherit ((construct1p { }).override { b = 13; }) a b; + }; + direct = { + inherit (construct1p { b = 13; }) a b; + }; + v = { + inherit (construct0p { b = 13; }) a b; + }; + }; + }; + }; + expected = { + construct-metadata = { + inherit (construct0) a c; + }; + construct-overridden-metadata = { + v = 11; + inherit (construct0p) a c; + }; + construct-overridden-result-overrider = { + result-overriders-exist = { + construct1 = true; + construct1p = true; + }; + result-overrider-functionality = { + overridden = { + inherit (construct0p { b = 13; }) a b; + }; + direct = { + inherit (construct0p { b = 13; }) a b; + }; + v = { + a = 11; + b = 13; + }; + }; + }; + }; + }; + testCallPackageWithOverridePreservesArguments = let f = From 185b80bee56e788bdcdc50358fe76f5416936f33 Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Fri, 14 Nov 2025 01:02:22 +0800 Subject: [PATCH 16/26] fetchFromGitProvider: init --- .../fetchgitprovider/default.nix | 283 ++++++++++++++++++ pkgs/top-level/all-packages.nix | 2 + 2 files changed, 285 insertions(+) create mode 100644 pkgs/build-support/fetchgitprovider/default.nix diff --git a/pkgs/build-support/fetchgitprovider/default.nix b/pkgs/build-support/fetchgitprovider/default.nix new file mode 100644 index 0000000000000..b157738809987 --- /dev/null +++ b/pkgs/build-support/fetchgitprovider/default.nix @@ -0,0 +1,283 @@ +{ + lib, + repoRevToNameMaybe, + stdenvNoCC, + fetchgit, + fetchzip, + # Disable referencing `finalAttrs.fetchGit` by default + # due to performance overhead (+4% CPU time for `pkgs` evaluation if enabled globally). + # Switching this on enables aggressive dynamic switching between backends after `.overrideAttrs` by default, + # and also enable manual specifying useFetchGit via `overrideAttrs`. + enableUseFetchGitFinal ? false, +}: +let + accumulateConstructorMetadata = + constructDrv: + let + f = + cM1: + { + excludeDrvArgNames, + extendDrvArgs, + transformDrv, + ... + }@cfg: + if cM1 ? constructDrv then + cfg + // f cM1.constructDrv { + excludeDrvArgNames = excludeDrvArgNames ++ cM1.excludeDrvArgNames; + extendDrvArgs = + finalAttrs: args: + let + args' = extendDrvArgs finalAttrs args; + in + # Also let extendDrvArgs complete the original work of excludeDrvArgNames, + # making the performance close to direct-composition. + removeAttrs args' cM1.excludeDrvArgNames // cM1.extendDrvArgs finalAttrs args'; + transformDrv = drv: transformDrv (cM1.transformDrv drv); + } + else + cfg; + in + f constructDrv.constructDrv ( + constructDrv + // { + extendDrvArgs = + finalAttrs: args: + removeAttrs args constructDrv.excludeDrvArgNames // constructDrv.extendDrvArgs finalAttrs args; + } + ); + + # Here defines fetchFromGitHub arguments that determines useFetchGit, + # The attribute value is their default values. + # As fetchFromGitHub prefers fetchzip for hash stability, + # `defaultFetchGitArgs` attributes should lead to `useFetchGit = false`. + useFetchGitArgsDefault = { + fetchSubmodules = false; # This differs from fetchgit's default + leaveDotGit = null; + deepClone = false; + forceFetchGit = false; + fetchLFS = false; + rootDir = ""; + sparseCheckout = null; + }; + useFetchGitArgsDefaultNullable = { + leaveDotGit = false; + sparseCheckout = [ ]; + }; + + useFetchGitArgsDefaultNonNull = useFetchGitArgsDefault // useFetchGitArgsDefaultNullable; + useFetchGitArgsDefaultPassing = removeAttrs useFetchGitArgsDefault excludeUseFetchGitArgNames; + + excludeUseFetchGitArgNames = [ + "forceFetchGit" + ]; + + faUseFetchGit = lib.mapAttrs (_: _: true) useFetchGitArgsDefault; + + faFetchGitSpecific = removeAttrs (lib.functionArgs fetchgit) ( + lib.attrNames (faUseFetchGit // lib.functionArgs fetchzip) + ++ [ + "sha256" + "outputHash" + "outputHashAlgo" + ] + ); + faFetchZipSpecific = removeAttrs (lib.functionArgs fetchzip) ( + lib.attrNames (faUseFetchGit // lib.functionArgs fetchgit) + ++ [ + "sha256" + "outputHash" + "outputHashAlgo" + ] + ); + + excludeDrvArgNamesShared = [ + "providerName" + "repo" + "rev" + "tag" + "gitRepoUrl" + "useFetchGit" + "fetchgitArgs" + "fetchzipArgs" + ]; + + mirrorArgs = f: lib.setFunctionArgs f (lib.functionArgs (extendDrvArgsShared { }) // faUseFetchGit); + + # We prefer fetchzip in cases we don't need submodules as the hash + # is more stable in that case. + choices = { + fetchgit = accumulateConstructorMetadata ( + lib.extendMkDerivation { + constructDrv = fetchgit; + excludeDrvArgNames = + excludeUseFetchGitArgNames ++ lib.attrNames faFetchZipSpecific ++ excludeDrvArgNamesShared; + extendDrvArgs = finalAttrs: args: (extendDrvArgsShared finalAttrs args).fetchgitArgs; + } + ); + fetchzip = accumulateConstructorMetadata ( + lib.extendMkDerivation { + constructDrv = fetchzip; + excludeDrvArgNames = + lib.attrNames (useFetchGitArgsDefault // faFetchGitSpecific) ++ excludeDrvArgNamesShared; + extendDrvArgs = finalAttrs: args: (extendDrvArgsShared finalAttrs args).fetchzipArgs; + } + ); + }; + + excludeDrvArgNamesUnion = lib.uniqueStrings ( + choices.fetchgit.excludeDrvArgNames ++ choices.fetchzip.excludeDrvArgNames + ); + + extendDrvArgsShared = + finalAttrs: + { + name ? + repoRevToNameMaybe finalAttrs.repo (lib.revOrTag finalAttrs.revCustom finalAttrs.tag) + providerName, + derivationArgs ? { }, + passthru ? { }, + meta ? { }, + providerName ? "git-provider", + repo, + rev ? null, + tag ? null, + gitRepoUrl, + fetchgitArgs ? { }, + fetchzipArgs, + ... + }@args: + { + fetchgitArgs = + fetchgitArgs + // { + name = fetchgitArgs.name or name; + inherit tag rev; + url = gitRepoUrl; + derivationArgs = { + inherit repo; + } + // derivationArgs + // fetchgitArgs.derivationArgs or { }; + passthru = passthru // fetchgitArgs.passthru or { }; + meta = meta // fetchgitArgs.meta or { }; + } + // lib.overrideExisting useFetchGitArgsDefaultPassing args; + fetchzipArgs = fetchzipArgs // { + name = fetchzipArgs.name or name; + derivationArgs = { + inherit + repo + tag + ; + rev = fetchgit.getRevWithTag { + inherit (finalAttrs) tag; + rev = finalAttrs.revCustom; + }; + revCustom = rev; + } + // derivationArgs + // fetchzipArgs.derivationArgs or { }; + passthru = { + inherit gitRepoUrl; + } + // passthru + // fetchzipArgs.passthru or { }; + meta = meta // fetchzipArgs.meta or { }; + }; + }; +in +lib.extendMkDerivation { + constructDrv = stdenvNoCC.mkDerivation; + + excludeDrvArgNames = excludeDrvArgNamesUnion; + + extendDrvArgs = + finalAttrs: + mirrorArgs ( + args: + let + useFetchGitCurrent = + lib.mapAttrs ( + n: default: + if !(args ? ${n}) || (useFetchGitArgsDefaultNullable ? ${n} && args.${n} == null) then + default + else + args.${n} + ) useFetchGitArgsDefaultNonNull != useFetchGitArgsDefaultNonNull; + # accumulateConstructorMetadata make `extendDrvArgs` also produce + # the `removeAttrs args excludeDrvArgNames` result, + # so we don't need to do it again. + fetchgitDerivationArgs = choices.fetchgit.extendDrvArgs finalAttrs args; + fetchzipDerivationArgs = choices.fetchzip.extendDrvArgs finalAttrs args; + useFetchGitFinal = if enableUseFetchGitFinal then finalAttrs.useFetchGit else useFetchGitCurrent; + newDerivationArgsChosen = + if useFetchGitFinal then fetchgitDerivationArgs else fetchzipDerivationArgs; + newDerivationArgsMixed = fetchgitDerivationArgs // fetchzipDerivationArgs; + faForbidden = removeAttrs ( + if useFetchGitCurrent then faFetchZipSpecific else faFetchGitSpecific + ) excludeDrvArgNamesShared; + forbiddenArgs = lib.filterAttrs (n: v: args.${n} or null != null) faForbidden; + forbiddenArgsThrown = + let + forbiddenArgNamesConcatenated = lib.concatStringsSep ", " (lib.attrNames forbiddenArgs); + thisBackendName = if useFetchGitCurrent then "fetchgit" else "fetchzip"; + otherBackendName = if useFetchGitCurrent then "fetchzip" else "fetchgit"; + message = '' + fetchGitProvider: Unexpected arguments for ${thisBackendName}: ${forbiddenArgNamesConcatenated} + Adjust arguments to use the ${otherBackendName} backend instead. + ''; + in + lib.mapAttrs (n: throw message) forbiddenArgs; + in + forbiddenArgsThrown + // ( + if enableUseFetchGitFinal then + # The final useFetchGitArgs extracted from fetchgitDerivationArgs + # causes FODs that uses the `fetchzip` backend + # to allocate both fetchgitDerivationArgs and fetchzipDerivationArgs + # due to the attribute names being always stict. + # This is why statically determining newDerivationArgsMixed does not optimise the evaluation. + # + # If we land the Nix features proposed in PR NixOS/nix#4090, + # that unstrict `a` in `(a // b).`, + # we could adjust `fetchgit` to group the attributes needed by `useFetchGitArgs` after the rest of attributes. + # Together with the static determination of newDerivationArgsMixed, + # we could make the performance overhead of aggressive dynamic switching negligible + # and enable it by default. + lib.mapAttrs ( + n: _: + if useFetchGitArgsDefault ? ${n} then + # The final useFetchGitArgs produced by fetchgit. + fetchgitDerivationArgs.${n} + else + newDerivationArgsChosen.${n} or null + ) newDerivationArgsMixed + // lib.overrideExisting (lib.getAttrs excludeUseFetchGitArgNames useFetchGitArgsDefault) args + // { + useFetchGit = + (lib.mapAttrs (n: _: finalAttrs.${n}) useFetchGitArgsDefaultNonNull) + != useFetchGitArgsDefaultNonNull; + } + else + newDerivationArgsChosen // { useFetchGit = useFetchGitCurrent; } + ) + ); + + transformDrv = + drv: + drv.overrideAttrs ( + finalAttrs: previousAttrs: { + # Add rev xor tag assertion + # This rev is revOrTag + rev = + lib.throwIfNot (lib.xor (finalAttrs.tag or null == null) (finalAttrs.revCustom or null == null)) + '' + fetchFromGitProvider requires one of either `rev` or `tag` to be provided (not both). + For `rev` overriding, override `revCustom` in `.overrideAttrs`. + '' + previousAttrs.rev; + } + ); +} diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 870df36051cad..064e2536d4697 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -659,6 +659,8 @@ with pkgs; fetchCrate = callPackage ../build-support/rust/fetchcrate.nix { }; + fetchFromGitProvider = callPackage ../build-support/fetchgitprovider { }; + fetchFromGitea = callPackage ../build-support/fetchgitea { }; fetchFromGitHub = callPackage ../build-support/fetchgithub { }; From a514aaf611250d9fd6e1b19bea3a4c1a9e2ade93 Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Tue, 2 Dec 2025 16:45:11 +0800 Subject: [PATCH 17/26] fetchFromGitHub: base on fetchFromGitProvider and lib.extendMkDerivation --- pkgs/build-support/fetchgithub/default.nix | 208 +++++++-------------- 1 file changed, 66 insertions(+), 142 deletions(-) diff --git a/pkgs/build-support/fetchgithub/default.nix b/pkgs/build-support/fetchgithub/default.nix index 7f6cb26e9387f..65e16dba1d980 100644 --- a/pkgs/build-support/fetchgithub/default.nix +++ b/pkgs/build-support/fetchgithub/default.nix @@ -1,51 +1,41 @@ { lib, - repoRevToNameMaybe, - fetchgit, - fetchzip, + fetchFromGitProvider, }: -let - # Here defines fetchFromGitHub arguments that determines useFetchGit, - # The attribute value is their default values. - # As fetchFromGitHub prefers fetchzip for hash stability, - # `defaultFetchGitArgs` attributes should lead to `useFetchGit = false`. - useFetchGitArgsDefault = { - deepClone = false; - fetchSubmodules = false; # This differs from fetchgit's default - fetchLFS = false; - forceFetchGit = false; - leaveDotGit = null; - rootDir = ""; - sparseCheckout = null; - }; - useFetchGitArgsDefaultNullable = { - leaveDotGit = false; - sparseCheckout = [ ]; - }; - useFetchGitargsDefaultNonNull = useFetchGitArgsDefault // useFetchGitArgsDefaultNullable; - - # useFetchGitArgsWD to exclude from automatic passing. - # Other useFetchGitArgsWD will pass down to fetchgit. - excludeUseFetchGitArgNames = [ - "forceFetchGit" - ]; +lib.makeOverridable ( + lib.extendMkDerivation { + constructDrv = fetchFromGitProvider.override (previousArgs: { + fetchzip = + # fetchzip may not be overridable when using external tools, for example nix-prefetch + if previousArgs.fetchzip ? override then + previousArgs.fetchzip.override { + withUnzip = false; + } + else + previousArgs.fetchzip; + }); - faUseFetchGit = lib.mapAttrs (_: _: true) useFetchGitArgsDefault; + excludeDrvArgNames = [ + # Pass via derivationArgs + "githubBase" + "owner" - adjustFunctionArgs = f: lib.setFunctionArgs f (faUseFetchGit // lib.functionArgs f); + # Private attributes + # TODO(@ShamrockLee): check if those are still functional. + "private" + "varPrefix" + ]; - decorate = f: lib.makeOverridable (adjustFunctionArgs f); -in -decorate ( + extendDrvArgs = + finalAttrs: { owner, repo, tag ? null, rev ? null, - # TODO(@ShamrockLee): Add back after reconstruction with lib.extendMkDerivation - # name ? repoRevToNameMaybe finalAttrs.repo (lib.revOrTag finalAttrs.revCustom finalAttrs.tag) "github", private ? false, + rootDir ? "", githubBase ? "github.com", varPrefix ? null, passthru ? { }, @@ -53,23 +43,7 @@ decorate ( ... # For hash agility and additional fetchgit arguments }@args: - assert ( - lib.assertMsg (lib.xor (tag == null) ( - rev == null - )) "fetchFromGitHub requires one of either `rev` or `tag` to be provided (not both)." - ); - let - useFetchGit = useFetchGitArgsWDNonNull != useFetchGitargsDefaultNonNull; - - useFetchGitArgs = lib.intersectAttrs useFetchGitArgsDefault args; - useFetchGitArgsWD = useFetchGitArgsDefault // useFetchGitArgs; - useFetchGitArgsWDPassing = removeAttrs useFetchGitArgsWD excludeUseFetchGitArgNames; - useFetchGitArgsWDNonNull = - useFetchGitArgsWD - // lib.mapAttrs ( - name: nonNullDefault: lib.defaultTo nonNullDefault useFetchGitArgsWD.${name} - ) useFetchGitArgsDefaultNullable; position = ( if args.meta.description or null != null then @@ -89,30 +63,8 @@ decorate ( # to indicate where derivation originates, similar to make-derivation.nix's mkDerivation position = "${position.file}:${toString position.line}"; }; - passthruAttrs = removeAttrs args ( - [ - "owner" - "repo" - "tag" - "rev" - "private" - "githubBase" - "varPrefix" - ] - ++ (if useFetchGit then excludeUseFetchGitArgNames else lib.attrNames faUseFetchGit) - ); varBase = "NIX${lib.optionalString (varPrefix != null) "_${varPrefix}"}_GITHUB_PRIVATE_"; - # We prefer fetchzip in cases we don't need submodules as the hash - # is more stable in that case. - fetcher = - if useFetchGit then - fetchgit - # fetchzip may not be overridable when using external tools, for example nix-prefetch - else if fetchzip ? override then - fetchzip.override { withUnzip = false; } - else - fetchzip; - privateAttrs = lib.optionalAttrs private { + getPrivateAttrs = useFetchGit: lib.optionalAttrs private { netrcPhase = # When using private repos: # - Fetching with git works using https://github.com but not with the GitHub API endpoint @@ -136,77 +88,49 @@ decorate ( "${varBase}PASSWORD" ]; }; + in + { + providerName = "github"; + + derivationArgs = { + inherit + githubBase + owner + ; + }; + + inherit repo; gitRepoUrl = "${baseUrl}.git"; - fetcherArgs = - finalAttrs: - passthruAttrs - // ( - if useFetchGit then - useFetchGitArgsWDPassing - // { - inherit tag rev; - url = gitRepoUrl; - inherit passthru; - derivationArgs = { - inherit - githubBase - owner - repo - ; - }; - } - else - let - revWithTag = finalAttrs.rev; - in - { - # Use the API endpoint for private repos, as the archive URI doesn't - # support access with GitHub's fine-grained access tokens. - # - # Use the archive URI for non-private repos, as the API endpoint has - # relatively restrictive rate limits for unauthenticated users. - url = - if private then - let - endpoint = "/repos/${finalAttrs.owner}/${finalAttrs.repo}/tarball/${revWithTag}"; - in - if githubBase == "github.com" then - "https://api.github.com${endpoint}" - else - "https://${githubBase}/api/v3${endpoint}" - else - "${baseUrl}/archive/${revWithTag}.tar.gz"; + fetchgitArgs = getPrivateAttrs true; + + fetchzipArgs = + let + revWithTag = finalAttrs.rev; + in + { + # Use the API endpoint for private repos, as the archive URI doesn't + # support access with GitHub's fine-grained access tokens. + # + # Use the archive URI for non-private repos, as the API endpoint has + # relatively restrictive rate limits for unauthenticated users. + url = + if private then + let + endpoint = "/repos/${finalAttrs.owner}/${finalAttrs.repo}/tarball/${revWithTag}"; + in + if githubBase == "github.com" then + "https://api.github.com${endpoint}" + else + "https://${githubBase}/api/v3${endpoint}" + else + "${baseUrl}/archive/${revWithTag}.tar.gz"; extension = "tar.gz"; - derivationArgs = { - inherit - githubBase - owner - repo - tag - ; - rev = fetchgit.getRevWithTag { - inherit (finalAttrs) tag; - rev = finalAttrs.revCustom; - }; - revCustom = rev; - }; - passthru = { - inherit gitRepoUrl; - } - // passthru; - } - ) - // privateAttrs - // { - # TODO(@ShamrockLee): Change back to `inherit name;` after reconstruction with lib.extendMkDerivation - name = - args.name - or (repoRevToNameMaybe finalAttrs.repo (lib.revOrTag finalAttrs.revCustom finalAttrs.tag) "github"); - meta = newMeta; - }; - in + } + // getPrivateAttrs false; - fetcher fetcherArgs + meta = newMeta; + }; + } ) From 351c1829c00e10d7324f6206df55cd32d6c16830 Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Fri, 14 Nov 2025 01:11:42 +0800 Subject: [PATCH 18/26] fetchFromGitHub: format expression after lib.extendMkDerivation reconstruction --- pkgs/build-support/fetchgithub/default.nix | 190 +++++++++++---------- 1 file changed, 96 insertions(+), 94 deletions(-) diff --git a/pkgs/build-support/fetchgithub/default.nix b/pkgs/build-support/fetchgithub/default.nix index 65e16dba1d980..73841d0054f9a 100644 --- a/pkgs/build-support/fetchgithub/default.nix +++ b/pkgs/build-support/fetchgithub/default.nix @@ -29,108 +29,110 @@ lib.makeOverridable ( extendDrvArgs = finalAttrs: - { - owner, - repo, - tag ? null, - rev ? null, - private ? false, - rootDir ? "", - githubBase ? "github.com", - varPrefix ? null, - passthru ? { }, - meta ? { }, - ... # For hash agility and additional fetchgit arguments - }@args: + { + owner, + repo, + tag ? null, + rev ? null, + private ? false, + rootDir ? "", + githubBase ? "github.com", + varPrefix ? null, + passthru ? { }, + meta ? { }, + ... # For hash agility and additional fetchgit arguments + }@args: - let + let - position = ( - if args.meta.description or null != null then - builtins.unsafeGetAttrPos "description" args.meta - else if tag != null then - builtins.unsafeGetAttrPos "tag" args - else - builtins.unsafeGetAttrPos "rev" args - ); - baseUrl = "https://${githubBase}/${owner}/${repo}"; - newMeta = - meta - // { - homepage = meta.homepage or baseUrl; - } - // lib.optionalAttrs (position != null) { - # to indicate where derivation originates, similar to make-derivation.nix's mkDerivation - position = "${position.file}:${toString position.line}"; - }; - varBase = "NIX${lib.optionalString (varPrefix != null) "_${varPrefix}"}_GITHUB_PRIVATE_"; - getPrivateAttrs = useFetchGit: lib.optionalAttrs private { - netrcPhase = - # When using private repos: - # - Fetching with git works using https://github.com but not with the GitHub API endpoint - # - Fetching a tarball from a private repo requires to use the GitHub API endpoint - let - machineName = if githubBase == "github.com" && !useFetchGit then "api.github.com" else githubBase; - in - '' - if [ -z "''$${varBase}USERNAME" -o -z "''$${varBase}PASSWORD" ]; then - echo "Error: Private fetchFromGitHub requires the nix building process (nix-daemon in multi user mode) to have the ${varBase}USERNAME and ${varBase}PASSWORD env vars set." >&2 - exit 1 - fi - cat > netrc <&2 + exit 1 + fi + cat > netrc < Date: Mon, 1 Dec 2025 21:19:00 +0800 Subject: [PATCH 19/26] fetchurl: provide fetchurl.extendDrvArgs --- pkgs/build-support/fetchurl/default.nix | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/pkgs/build-support/fetchurl/default.nix b/pkgs/build-support/fetchurl/default.nix index 5ffb70e3a9f8a..d0e1007b76017 100644 --- a/pkgs/build-support/fetchurl/default.nix +++ b/pkgs/build-support/fetchurl/default.nix @@ -51,11 +51,6 @@ let ] ++ (map (site: "NIX_MIRRORS_${site}") sites); -in - -lib.extendMkDerivation { - constructDrv = stdenvNoCC.mkDerivation; - excludeDrvArgNames = [ # Passed via passthru "url" @@ -329,6 +324,26 @@ lib.extendMkDerivation { // passthru; }; +in + +lib.extendMkDerivation { + constructDrv = stdenvNoCC.mkDerivation; + + inherit + excludeDrvArgNames + extendDrvArgs + ; + # No ellipsis inheritFunctionArgs = false; } +// { + expectDrvArgs = + let + faRaw = lib.functionArgs (extendDrvArgs { }); + in + lib.zipAttrsWith (n: lib.any lib.id) [ + (lib.mapAttrs (n: v: !v) (removeAttrs faRaw excludeDrvArgNames)) + (lib.mapAttrs (n: v: true) (extendDrvArgs { } (faRaw // { derivationArgs = { }; }))) + ]; +} From e08b2c3e334bdf5bbaeed5c981a47a721b034ac4 Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Sun, 16 Nov 2025 02:07:51 +0800 Subject: [PATCH 20/26] fetchzip: provide fetchzip.expectDrvArgs --- pkgs/build-support/fetchzip/default.nix | 33 ++++++++++++++++--------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/pkgs/build-support/fetchzip/default.nix b/pkgs/build-support/fetchzip/default.nix index 4376fef410854..7f2657b95ee2e 100644 --- a/pkgs/build-support/fetchzip/default.nix +++ b/pkgs/build-support/fetchzip/default.nix @@ -14,17 +14,7 @@ glibcLocalesUtf8, }: -lib.extendMkDerivation { - constructDrv = fetchurl; - - excludeDrvArgNames = [ - "extraPostFetch" - - # Pass via derivationArgs - "extension" - "stripRoot" - ]; - +let extendDrvArgs = finalAttrs: { @@ -122,4 +112,25 @@ lib.extendMkDerivation { ; }; }; +in +lib.extendMkDerivation { + constructDrv = fetchurl; + + excludeDrvArgNames = [ + "extraPostFetch" + + # Pass via derivationArgs + "extension" + "stripRoot" + ]; + + inherit extendDrvArgs; +} +// { + expectDrvArgs = lib.zipAttrsWith (_: lib.any lib.id) [ + (lib.mapAttrs (n: _: true) + (extendDrvArgs { } (lib.functionArgs extendDrvArgs // { derivationArgs = { }; })).derivationArgs + ) + fetchurl.expectDrvArgs + ]; } From 1bafb302f6cc22e163cdc90690707a1613f9781c Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Sun, 16 Nov 2025 06:01:30 +0800 Subject: [PATCH 21/26] mkDerivation: make allowedReference/allowedRequisites nullable when __structuedAttrs is true --- pkgs/stdenv/generic/make-derivation.nix | 9 +++++++-- pkgs/test/overriding.nix | 10 ++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/pkgs/stdenv/generic/make-derivation.nix b/pkgs/stdenv/generic/make-derivation.nix index d00df6ff4139c..cbd3dfe49bf69 100644 --- a/pkgs/stdenv/generic/make-derivation.nix +++ b/pkgs/stdenv/generic/make-derivation.nix @@ -17,6 +17,7 @@ let elemAt extendDerivation filter + filterAttrs findFirst getDev head @@ -715,8 +716,12 @@ let ]; } // ( + let + attrsOutputChecks = makeOutputChecks attrs; + attrsOutputChecksFiltered = filterAttrs (_: v: v != null) attrsOutputChecks; + in if !__structuredAttrs then - makeOutputChecks attrs + attrsOutputChecks else { outputChecks = builtins.listToAttrs ( @@ -725,7 +730,7 @@ let value = let raw = zipAttrsWith (_: builtins.concatLists) [ - (makeOutputChecks attrs) + attrsOutputChecksFiltered (makeOutputChecks attrs.outputChecks.${name} or { }) ]; in diff --git a/pkgs/test/overriding.nix b/pkgs/test/overriding.nix index a2221183a8a6f..d0ca30dec2cc4 100644 --- a/pkgs/test/overriding.nix +++ b/pkgs/test/overriding.nix @@ -62,6 +62,16 @@ let }).pname; expected = "hello-no-final-attrs-overridden"; }; + structuredAttrs-allowedRequisites-nullablility = { + expr = + lib.hasPrefix builtins.storeDir + (pkgs.stdenv.mkDerivation { + __structuredAttrs = true; + inherit (pkgs.hello) pname version src; + allowedRequisites = null; + }).drvPath; + expected = true; + }; }; test-extendMkDerivation = From a04426d1904a0e601c48881d8a4a95b2f5a13289 Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Sun, 16 Nov 2025 02:09:27 +0800 Subject: [PATCH 22/26] fetchgit: provide fetchgit.expectDrvArgs --- pkgs/build-support/fetchgit/default.nix | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/pkgs/build-support/fetchgit/default.nix b/pkgs/build-support/fetchgit/default.nix index 6c5abd592e559..ff0dec603e84d 100644 --- a/pkgs/build-support/fetchgit/default.nix +++ b/pkgs/build-support/fetchgit/default.nix @@ -35,11 +35,6 @@ let else # FIXME fetching HEAD if no rev or tag is provided is problematic at best "HEAD"; -in - -lib.makeOverridable ( - lib.extendMkDerivation { - constructDrv = stdenvNoCC.mkDerivation; excludeDrvArgNames = [ # Additional stdenv.mkDerivation arguments from derived fetchers. @@ -229,6 +224,13 @@ lib.makeOverridable ( // passthru; } ); +in + +lib.makeOverridable ( + lib.extendMkDerivation { + constructDrv = stdenvNoCC.mkDerivation; + + inherit excludeDrvArgNames extendDrvArgs; # No ellipsis. inheritFunctionArgs = false; @@ -236,4 +238,12 @@ lib.makeOverridable ( ) // { inherit getRevWithTag; + expectDrvArgs = + let + faRaw = lib.functionArgs (extendDrvArgs { }); + in + lib.zipAttrsWith (_: lib.any lib.id) [ + (lib.mapAttrs (_: v: !v) (removeAttrs faRaw excludeDrvArgNames)) + (lib.mapAttrs (_: _: true) (extendDrvArgs { } (faRaw // { derivationArgs = { }; }))) + ]; } From c54b74265dacef3ce18851e2e96571a49d8b43a7 Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Mon, 17 Nov 2025 20:51:41 +0800 Subject: [PATCH 23/26] fetchFromGitProvider, fetchFromGitHub: use and provide expectDrvArgs --- pkgs/build-support/fetchgithub/default.nix | 4 ++++ .../fetchgitprovider/default.nix | 22 +++++++++++++++++-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/pkgs/build-support/fetchgithub/default.nix b/pkgs/build-support/fetchgithub/default.nix index 73841d0054f9a..3654a4326aeb9 100644 --- a/pkgs/build-support/fetchgithub/default.nix +++ b/pkgs/build-support/fetchgithub/default.nix @@ -14,6 +14,10 @@ lib.makeOverridable ( } else previousArgs.fetchzip; + expectDrvArgsExtra = { + githubBase = true; + owner = true; + }; }); excludeDrvArgNames = [ diff --git a/pkgs/build-support/fetchgitprovider/default.nix b/pkgs/build-support/fetchgitprovider/default.nix index b157738809987..87fbdd79fa4a4 100644 --- a/pkgs/build-support/fetchgitprovider/default.nix +++ b/pkgs/build-support/fetchgitprovider/default.nix @@ -9,6 +9,7 @@ # Switching this on enables aggressive dynamic switching between backends after `.overrideAttrs` by default, # and also enable manual specifying useFetchGit via `overrideAttrs`. enableUseFetchGitFinal ? false, + expectDrvArgsExtra ? { }, }: let accumulateConstructorMetadata = @@ -48,6 +49,21 @@ let } ); + expectDrvArgs = lib.zipAttrsWith (_: lib.any lib.id) [ + expectDrvArgsExtra + { + repo = true; + rev = true; + tag = true; + useFetchGit = true; + } + faUseFetchGit + (lib.zipAttrsWith (_: lib.all lib.id) [ + fetchgit.expectDrvArgs + fetchzip.expectDrvArgs + ]) + ]; + # Here defines fetchFromGitHub arguments that determines useFetchGit, # The attribute value is their default values. # As fetchFromGitHub prefers fetchzip for hash stability, @@ -214,7 +230,6 @@ lib.extendMkDerivation { useFetchGitFinal = if enableUseFetchGitFinal then finalAttrs.useFetchGit else useFetchGitCurrent; newDerivationArgsChosen = if useFetchGitFinal then fetchgitDerivationArgs else fetchzipDerivationArgs; - newDerivationArgsMixed = fetchgitDerivationArgs // fetchzipDerivationArgs; faForbidden = removeAttrs ( if useFetchGitCurrent then faFetchZipSpecific else faFetchGitSpecific ) excludeDrvArgNamesShared; @@ -253,7 +268,7 @@ lib.extendMkDerivation { fetchgitDerivationArgs.${n} else newDerivationArgsChosen.${n} or null - ) newDerivationArgsMixed + ) expectDrvArgs // lib.overrideExisting (lib.getAttrs excludeUseFetchGitArgNames useFetchGitArgsDefault) args // { useFetchGit = @@ -281,3 +296,6 @@ lib.extendMkDerivation { } ); } +// { + inherit expectDrvArgs; +} From d27d2a389551c490254a33834db9509f4651a232 Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Thu, 4 Dec 2025 01:22:32 +0800 Subject: [PATCH 24/26] lib.fetchers.normalizeHash: move assertions down to outputHash and outputHashAlgo values --- lib/fetchers.nix | 37 ++++++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/lib/fetchers.nix b/lib/fetchers.nix index c09dbc7991032..cf9a8033a6580 100644 --- a/lib/fetchers.nix +++ b/lib/fetchers.nix @@ -117,24 +117,31 @@ rec { # All hashes passed in arguments (possibly 0 or >1) as a list of {name, value} pairs let hashesAsNVPairs = attrsToList (intersectAttrs hashSet args); + unspecifiedHash = hashesAsNVPairs == [ ]; + multipleHash = tail hashesAsNVPairs != [ ]; in - if hashesAsNVPairs == [ ] then - throwIf required "fetcher called without `hash`" null - else if tail hashesAsNVPairs != [ ] then - throw "fetcher called with mutually-incompatible arguments: ${ - concatMapStringsSep ", " (a: a.name) hashesAsNVPairs - }" - else - head hashesAsNVPairs; + { + errorMessage = + if unspecifiedHash && required then + "fetcher called without `hash`" + else if multipleHash then + "fetcher called with mutually-incompatible arguments: ${ + concatMapStringsSep ", " (a: a.name) hashesAsNVPairs + }" + else + null; + inherit (head hashesAsNVPairs) name value; + inherit unspecifiedHash; + }; in removeAttrs args hashNames - // (optionalAttrs (h != null) { - outputHashAlgo = if h.name == "hash" then null else h.name; - outputHash = - if h.value == "" then - fakeH.${h.name} or (throw "no “fake hash” defined for ${h.name}") - else - h.value; + // (optionalAttrs (required || !h.unspecifiedHash) { + outputHashAlgo = lib.throwIf (h.errorMessage != null) h.errorMessage ( + if h.name == "hash" then null else h.name + ); + outputHash = lib.throwIf (h.errorMessage != null) h.errorMessage ( + if h.value == "" then fakeH.${h.name} or (throw "no “fake hash” defined for ${h.name}") else h.value + ); }); /** From e28bd42873232ed9d57ecf81d83f15a5b64a2ca2 Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Fri, 21 Nov 2025 15:51:51 +0800 Subject: [PATCH 25/26] fetchGitProvider: default enableUseFetchGitFinal true --- pkgs/build-support/fetchgitprovider/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/build-support/fetchgitprovider/default.nix b/pkgs/build-support/fetchgitprovider/default.nix index 87fbdd79fa4a4..471279027c746 100644 --- a/pkgs/build-support/fetchgitprovider/default.nix +++ b/pkgs/build-support/fetchgitprovider/default.nix @@ -8,7 +8,7 @@ # due to performance overhead (+4% CPU time for `pkgs` evaluation if enabled globally). # Switching this on enables aggressive dynamic switching between backends after `.overrideAttrs` by default, # and also enable manual specifying useFetchGit via `overrideAttrs`. - enableUseFetchGitFinal ? false, + enableUseFetchGitFinal ? true, expectDrvArgsExtra ? { }, }: let From cc205a2c6733bd5877776152d0d146e12de275bd Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Wed, 3 Dec 2025 00:51:55 +0800 Subject: [PATCH 26/26] fetchGitProvider: reduce enableUseFetchGitFinal RAM consumption with fetchgitSpecialArgs --- pkgs/build-support/fetchgit/default.nix | 33 ++++++++++++++++++- .../fetchgitprovider/default.nix | 3 +- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/pkgs/build-support/fetchgit/default.nix b/pkgs/build-support/fetchgit/default.nix index ff0dec603e84d..a07af2edf4183 100644 --- a/pkgs/build-support/fetchgit/default.nix +++ b/pkgs/build-support/fetchgit/default.nix @@ -224,6 +224,34 @@ let // passthru; } ); + + resolveSpecialArgs = + finalAttrs: + { + deepClone ? false, + fetchTags ? false, + leaveDotGit ? null, + nonConeMode ? null, + rootDir ? "", + sparseCheckout ? null, + ... + }: + { + inherit + deepClone + fetchTags + rootDir + ; + leaveDotGit = + if leaveDotGit != null then + assert fetchTags -> leaveDotGit; + assert rootDir != "" -> !leaveDotGit; + leaveDotGit + else + deepClone || fetchTags; + nonConeMode = lib.defaultTo (finalAttrs.rootDir != "") nonConeMode; + sparseCheckout = lib.defaultTo (lib.optional (finalAttrs.rootDir != "") finalAttrs.rootDir) sparseCheckout; + }; in lib.makeOverridable ( @@ -237,7 +265,10 @@ lib.makeOverridable ( } ) // { - inherit getRevWithTag; + inherit + getRevWithTag + resolveSpecialArgs + ; expectDrvArgs = let faRaw = lib.functionArgs (extendDrvArgs { }); diff --git a/pkgs/build-support/fetchgitprovider/default.nix b/pkgs/build-support/fetchgitprovider/default.nix index 471279027c746..4088eea766289 100644 --- a/pkgs/build-support/fetchgitprovider/default.nix +++ b/pkgs/build-support/fetchgitprovider/default.nix @@ -227,6 +227,7 @@ lib.extendMkDerivation { # so we don't need to do it again. fetchgitDerivationArgs = choices.fetchgit.extendDrvArgs finalAttrs args; fetchzipDerivationArgs = choices.fetchzip.extendDrvArgs finalAttrs args; + fetchgitSpecialArgs = fetchgit.resolveSpecialArgs finalAttrs args; useFetchGitFinal = if enableUseFetchGitFinal then finalAttrs.useFetchGit else useFetchGitCurrent; newDerivationArgsChosen = if useFetchGitFinal then fetchgitDerivationArgs else fetchzipDerivationArgs; @@ -265,7 +266,7 @@ lib.extendMkDerivation { n: _: if useFetchGitArgsDefault ? ${n} then # The final useFetchGitArgs produced by fetchgit. - fetchgitDerivationArgs.${n} + fetchgitSpecialArgs.${n} or useFetchGitArgsDefaultNonNull.${n} else newDerivationArgsChosen.${n} or null ) expectDrvArgs