From 270ee9e19b1cc1daf31a0aed0d9fc6c765d4c1fd Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Thu, 18 Dec 2025 16:50:37 +0800 Subject: [PATCH 01/10] fetchurl: mirrorList: simplify expression Co-authored-by: Matt Sturgeon --- pkgs/build-support/fetchurl/default.nix | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/pkgs/build-support/fetchurl/default.nix b/pkgs/build-support/fetchurl/default.nix index 7747a3a8f0fe3..3e16efd619a28 100644 --- a/pkgs/build-support/fetchurl/default.nix +++ b/pkgs/build-support/fetchurl/default.nix @@ -227,11 +227,7 @@ lib.extendMkDerivation { let mirrorSplit = lib.match "mirror://([[:alpha:]]+)/(.+)" url; mirrorName = lib.head mirrorSplit; - mirrorList = - if lib.hasAttr mirrorName mirrors then - mirrors."${mirrorName}" - else - throw "unknown mirror:// site ${mirrorName}"; + mirrorList = mirrors."${mirrorName}" or (throw "unknown mirror:// site ${mirrorName}"); in if mirrorSplit == null || mirrorName == null then url From 7c4673bbb8483c711489ec71e1a7e1bfddd0e81a Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Thu, 18 Dec 2025 08:41:28 +0800 Subject: [PATCH 02/10] tests.fetchurl: add urls-simple --- pkgs/build-support/fetchurl/tests.nix | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pkgs/build-support/fetchurl/tests.nix b/pkgs/build-support/fetchurl/tests.nix index c89368a6d2094..0610aaf805a0a 100644 --- a/pkgs/build-support/fetchurl/tests.nix +++ b/pkgs/build-support/fetchurl/tests.nix @@ -6,6 +6,7 @@ jq, moreutils, emptyFile, + hello, ... }: let @@ -138,4 +139,13 @@ in # have to fallback to fetching the previously-built derivation from # tarballs.nixos.org, which provides pre-built derivation outputs. }; + + urls-simple = testers.invalidateFetcherByDrvHash fetchurl { + name = "test-fetchurl-urls-simple"; + urls = [ + "http://broken" + hello.src.resolvedUrl + ]; + hash = hello.src.outputHash; + }; } From ca050c53b2016a41d2a3107799615020b2094f7d Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Thu, 18 Dec 2025 08:03:43 +0800 Subject: [PATCH 03/10] fetchurl: provide fetchurl.resolveUrl --- pkgs/build-support/fetchurl/default.nix | 26 +++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/pkgs/build-support/fetchurl/default.nix b/pkgs/build-support/fetchurl/default.nix index 3e16efd619a28..95fac7219d522 100644 --- a/pkgs/build-support/fetchurl/default.nix +++ b/pkgs/build-support/fetchurl/default.nix @@ -35,6 +35,18 @@ let # "gnu", etc.). sites = builtins.attrNames mirrors; + resolveUrl = + url: + let + mirrorSplit = lib.match "mirror://([[:alpha:]]+)/(.+)" url; + mirrorName = lib.head mirrorSplit; + mirrorList = mirrors."${mirrorName}" or (throw "unknown mirror:// site ${mirrorName}"); + in + if mirrorSplit == null || mirrorName == null then + [ url ] + else + map (mirror: mirror + lib.elemAt mirrorSplit 1) mirrorList; + impureEnvVars = lib.fetchers.proxyImpureEnvVars ++ [ @@ -223,16 +235,7 @@ lib.extendMkDerivation { finalHashHasColon = lib.hasInfix ":" finalAttrs.hash; finalHashColonMatch = lib.match "([^:]+)[:](.*)" finalAttrs.hash; - resolvedUrl = - let - mirrorSplit = lib.match "mirror://([[:alpha:]]+)/(.+)" url; - mirrorName = lib.head mirrorSplit; - mirrorList = mirrors."${mirrorName}" or (throw "unknown mirror:// site ${mirrorName}"); - in - if mirrorSplit == null || mirrorName == null then - url - else - "${lib.head mirrorList}${lib.elemAt mirrorSplit 1}"; + resolvedUrl = lib.head (resolveUrl url); in derivationArgs @@ -339,3 +342,6 @@ lib.extendMkDerivation { # No ellipsis inheritFunctionArgs = false; } +// { + inherit resolveUrl; +} From 1e5ca4eadb86cc483ad239163354ee3723e24818 Mon Sep 17 00:00:00 2001 From: Matt Sturgeon Date: Thu, 18 Dec 2025 12:23:01 +0000 Subject: [PATCH 04/10] fetchurl: add doc-comment for fetchurl.resolveUrl --- pkgs/build-support/fetchurl/default.nix | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/pkgs/build-support/fetchurl/default.nix b/pkgs/build-support/fetchurl/default.nix index 95fac7219d522..ddd3075633679 100644 --- a/pkgs/build-support/fetchurl/default.nix +++ b/pkgs/build-support/fetchurl/default.nix @@ -35,6 +35,28 @@ let # "gnu", etc.). sites = builtins.attrNames mirrors; + /** + Resolve a URL against the available mirrors. + + If the input is a `"mirror://"` URL, it is normalized. + Otherwise, the URL is returned unmodified in a singleton list. + + Mirror URLs should be formatted as: + ``` + mirror://{mirror_name}/{path} + ``` + + The specified `mirror_name` must correspond to an entry in `pkgs/build-support/fetchurl/mirrors.nix`, otherwise an error is thrown. + + # Inputs + + `url` (String) + : A (possibly `"mirror://"`) URL to resolve. + + # Output + + A list of resolved URLs. + */ resolveUrl = url: let From fc180191def1822535c9d54e71f56574b1964fe8 Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Thu, 18 Dec 2025 06:20:24 +0800 Subject: [PATCH 05/10] fetchurl: builder.sh: handle `urls` as a Bash array Clean up leftover for commit cd13136f036d ("fetchurl: use __structuredAttrs = true and pass curlOptsList directly") Continue the work of commit 23236b331d06 ("fetchurl: fix handling of fallback URLs"), addressing a Bash array re-assignment quirk: when assigning a Bash array variable as if it were a plain variable, the value goes to the first element, and the rest of the array stays the same. ```console $ foo=(a b) $ declare -p foo declare -a foo=([0]="a" [1]="b") $ foo="c d" $ declare -p foo declare -a foo=([0]="c d" [1]="b") ``` Don't rewrite the `${urls[@]}` with resolved URLs, but hold them with `${resolvedUrls[@]}` instead. Co-authored-by: Matt Sturgeon Co-authored-by: Wolfgang Walther --- pkgs/build-support/fetchurl/builder.sh | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/pkgs/build-support/fetchurl/builder.sh b/pkgs/build-support/fetchurl/builder.sh index 9add2b27a0727..228fcd2ed185a 100644 --- a/pkgs/build-support/fetchurl/builder.sh +++ b/pkgs/build-support/fetchurl/builder.sh @@ -124,10 +124,10 @@ tryHashedMirrors() { # URL list may contain ?. No glob expansion for that, please set -o noglob -urls2= +resolvedUrls=() for url in "${urls[@]}"; do if test "${url:0:9}" != "mirror://"; then - urls2="$urls2 $url" + resolvedUrls+=("$url") else url2="${url:9}"; echo "${url2/\// }" > split; read site fileName < split #varName="mirror_$site" @@ -142,18 +142,17 @@ for url in "${urls[@]}"; do if test -n "${!varName}"; then mirrors="${!varName}"; fi for url3 in $mirrors; do - urls2="$urls2 $url3$fileName"; + resolvedUrls+=("$url3$fileName"); done fi fi done -urls="$urls2" # Restore globbing settings set +o noglob if test -n "$showURLs"; then - echo "$urls" > $out + echo "${resolvedUrls[*]}" > $out exit 0 fi @@ -165,7 +164,7 @@ fi set -o noglob success= -for url in $urls; do +for url in "${resolvedUrls[@]}"; do if [ -z "$postFetch" ]; then case "$url" in https://github.com/*/archive/*) From d8ac71ce95703c9dff55abba05850108cbc1707b Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Thu, 18 Dec 2025 08:41:40 +0800 Subject: [PATCH 06/10] tests.fetchurl: add urls-mirrors --- pkgs/build-support/fetchurl/tests.nix | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/pkgs/build-support/fetchurl/tests.nix b/pkgs/build-support/fetchurl/tests.nix index 0610aaf805a0a..ab8a3acefe27e 100644 --- a/pkgs/build-support/fetchurl/tests.nix +++ b/pkgs/build-support/fetchurl/tests.nix @@ -3,6 +3,7 @@ testers, fetchurl, writeShellScriptBin, + writeText, jq, moreutils, emptyFile, @@ -148,4 +149,23 @@ in ]; hash = hello.src.outputHash; }; + + urls-mirrors = testers.invalidateFetcherByDrvHash fetchurl rec { + name = "test-fetchurl-urls-simple"; + urls = [ + "http://broken" + ] + ++ hello.src.urls; + hash = hello.src.outputHash; + postFetch = hello.postFetch or "" + '' + if ! diff -u ${ + builtins.toFile "urls-resolved-by-eval" ( + lib.concatStringsSep "\n" (lib.concatMap fetchurl.resolveUrl urls) + "\n" + ) + } <(printf '%s\n' "''${resolvedUrls[@]}"); then + echo "ERROR: fetchurl: build-time-resolved URLs \`urls' differ from the evaluation-resolved URLs." >&2 + exit 1 + fi + ''; + }; } From cb39a53e9f424742e4ed816d0f7c428f0859bf7b Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Fri, 19 Dec 2025 21:56:01 +0800 Subject: [PATCH 07/10] testers.invalidateFetcherByDrvHash: salt pname if presented --- pkgs/build-support/testers/default.nix | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pkgs/build-support/testers/default.nix b/pkgs/build-support/testers/default.nix index ac31c22ab4a2c..f82c41c7fe5bc 100644 --- a/pkgs/build-support/testers/default.nix +++ b/pkgs/build-support/testers/default.nix @@ -135,8 +135,15 @@ drvPath = (f args).drvPath; # It's safe to discard the context, because we don't access the path. salt = builtins.unsafeDiscardStringContext (lib.substring 0 12 (baseNameOf drvPath)); + saltName = name: "${name}-salted-${salt}"; + getSaltedNames = + args: + if args.pname or null != null then + { pname = saltName args.pname; } + else + { name = saltName args.name or "source"; }; # New derivation incorporating the original drv hash in the name - salted = f (args // { name = "${args.name or "source"}-salted-${salt}"; }); + salted = f (args // getSaltedNames args); # Make sure we did change the derivation. If the fetcher ignores `name`, # `invalidateFetcherByDrvHash` doesn't work. checked = From 7ba63c64a628e99893dedf837f6291cecc5798bb Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Fri, 19 Dec 2025 22:23:47 +0800 Subject: [PATCH 08/10] testers.invalidateFetcherByDrvHash: use overrideAttrs if fetcher ignore name arguments --- pkgs/build-support/testers/default.nix | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/pkgs/build-support/testers/default.nix b/pkgs/build-support/testers/default.nix index f82c41c7fe5bc..6b33c4d2077eb 100644 --- a/pkgs/build-support/testers/default.nix +++ b/pkgs/build-support/testers/default.nix @@ -132,7 +132,8 @@ invalidateFetcherByDrvHash = f: args: let - drvPath = (f args).drvPath; + unsalted = f args; + drvPath = unsalted.drvPath; # It's safe to discard the context, because we don't access the path. salt = builtins.unsafeDiscardStringContext (lib.substring 0 12 (baseNameOf drvPath)); saltName = name: "${name}-salted-${salt}"; @@ -143,14 +144,23 @@ else { name = saltName args.name or "source"; }; # New derivation incorporating the original drv hash in the name - salted = f (args // getSaltedNames args); - # Make sure we did change the derivation. If the fetcher ignores `name`, + saltedByArgs = f (args // getSaltedNames args); + saltedByOverrideAttrs = unsalted.overrideAttrs (previousAttrs: getSaltedNames previousAttrs); + saltedByOverrideAttrsForced = unsalted.overrideAttrs (previousAttrs: { + name = saltName unsalted.name; + }); + # Make sure we did change the derivation. + # If the fetcher ignores `pname` and `name` and provide a broken `overrideAttrs`, # `invalidateFetcherByDrvHash` doesn't work. checked = - if salted.drvPath == drvPath then - throw "invalidateFetcherByDrvHash: Adding the derivation hash to the fixed-output derivation name had no effect. Make sure the fetcher's name argument ends up in the derivation name. Otherwise, the fetcher will not be re-run when its implementation changes. This is important for testing." + if saltedByArgs.drvPath != drvPath then + saltedByArgs + else if saltedByOverrideAttrs.drvPath != drvPath then + saltedByOverrideAttrs + else if saltedByOverrideAttrsForced.drvPath != drvPath then + saltedByOverrideAttrsForced else - salted; + throw "invalidateFetcherByDrvHash: Neither adding pname/name to the fetcher args nor overriding with overrideAttrs change the result drvPath."; in checked; From 1548251cfab11a247501eb4e89d1f5b78e785c4b Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Fri, 19 Dec 2025 22:27:51 +0800 Subject: [PATCH 09/10] testers.invalidateFetcherByDrvHash: take fixed-point arguments args --- pkgs/build-support/testers/default.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkgs/build-support/testers/default.nix b/pkgs/build-support/testers/default.nix index 6b33c4d2077eb..4dca262747472 100644 --- a/pkgs/build-support/testers/default.nix +++ b/pkgs/build-support/testers/default.nix @@ -132,6 +132,7 @@ invalidateFetcherByDrvHash = f: args: let + optionalFix = if lib.isFunction args then lib.id else lib.fix; unsalted = f args; drvPath = unsalted.drvPath; # It's safe to discard the context, because we don't access the path. @@ -144,7 +145,7 @@ else { name = saltName args.name or "source"; }; # New derivation incorporating the original drv hash in the name - saltedByArgs = f (args // getSaltedNames args); + saltedByArgs = f (optionalFix (lib.extends (lib.toExtension getSaltedNames) (lib.toFunction args))); saltedByOverrideAttrs = unsalted.overrideAttrs (previousAttrs: getSaltedNames previousAttrs); saltedByOverrideAttrsForced = unsalted.overrideAttrs (previousAttrs: { name = saltName unsalted.name; From 0b271455ecfd4db8990eda6b8e96e8047ba61286 Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Fri, 19 Dec 2025 22:35:49 +0800 Subject: [PATCH 10/10] tests.fetchurl: add showURLs-urls-mirrors --- pkgs/build-support/fetchurl/tests.nix | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/pkgs/build-support/fetchurl/tests.nix b/pkgs/build-support/fetchurl/tests.nix index ab8a3acefe27e..c745ec79e75ff 100644 --- a/pkgs/build-support/fetchurl/tests.nix +++ b/pkgs/build-support/fetchurl/tests.nix @@ -141,6 +141,24 @@ in # tarballs.nixos.org, which provides pre-built derivation outputs. }; + showURLs-urls-mirrors = testers.invalidateFetcherByDrvHash fetchurl (finalAttrs: { + name = "test-fetchurl-showURLs-urls-mirrors"; + showURLs = true; + urls = [ + "http://broken" + ] + ++ hello.src.urls; + hash = + let + hashAlgo = lib.head (lib.splitString "-" lib.fakeHash); + in + hashAlgo + + ":" + + builtins.hashString hashAlgo ( + lib.concatStringsSep " " (lib.concatMap fetchurl.resolveUrl finalAttrs.urls) + "\n" + ); + }); + urls-simple = testers.invalidateFetcherByDrvHash fetchurl { name = "test-fetchurl-urls-simple"; urls = [