From 2cdd102a1579b90dbe4c8b4ce300cff2d55ef401 Mon Sep 17 00:00:00 2001 From: OPNA2608 Date: Thu, 3 Jul 2025 14:15:07 +0200 Subject: [PATCH] heads: Improve updateDepsScript - Fix leftover ftpmirror.gnu.org URLs We still can't rely on all the mirrors in there giving correct results - Exclude acpica-unix2-20220331.tar.gz from being fetched by the script Original source is dead, unsure which mirror might be correct & safe due to hash differences - Don't throw away everything if a single URL fails to fetch Just give an empty hash and move on. A human can manually re-attempt this later. --- pkgs/by-name/heads/deps.nix | 24 +++++----- pkgs/by-name/heads/package.nix | 1 + pkgs/by-name/heads/update.sh | 88 ++++++++++++++++++++++++++++------ 3 files changed, 86 insertions(+), 27 deletions(-) diff --git a/pkgs/by-name/heads/deps.nix b/pkgs/by-name/heads/deps.nix index f03ffe639..b0eb52066 100644 --- a/pkgs/by-name/heads/deps.nix +++ b/pkgs/by-name/heads/deps.nix @@ -109,7 +109,7 @@ pkgs = [ { name = "bash-5.1.16.tar.gz"; - url = "https://ftpmirror.gnu.org/bash/bash-5.1.16.tar.gz"; + url = "https://ftp.gnu.org/gnu/bash/bash-5.1.16.tar.gz"; hash = "sha256-W6wXIY05EYNFINrRPNH4WrlE4cCa4aulWQa+H4GS9Vg="; } { @@ -666,13 +666,11 @@ url = "https://ftp.gnu.org/gnu/binutils/binutils-2.37.tar.xz"; hash = "sha256-gg2XJPAgo+acszeJOgtjwtsWHa3LDgb8Edwp6x6Eoyw="; } - # URL seems long-term broken, needs #1256 to get fixed - # Short-term, commenting this package out makes boards that don't need this particular coreboot version build again - #{ - # name = "coreboot-crossgcc-acpica-unix2-20220331.tar.gz"; - # url = "https://acpica.org/sites/acpica/files/acpica-unix2-20220331.tar.gz"; - # hash = "sha256-CG1rZYX1Zndob+2K9rC1upOuwYC0HSKFobdM0CtY8ko="; - #} + + # Skipping acpica-unix2-20220331.tar.gz because we don't have a known-good mirror + # Candidate (Heads explicitly *doesn't* use this one for this version): https://mirror.math.princeton.edu/pub/libreboot/misc/acpica/acpica-unix2-20220331.tar.gz + # Candidate (involves an archive rename): https://distfiles.macports.org/acpica/acpica-unix-20220331.tar.gz + { name = "coreboot-crossgcc-llvm-14.0.6.src.tar.xz"; url = "https://github.com/llvm/llvm-project/releases/download/llvmorg-14.0.6/llvm-14.0.6.src.tar.xz"; @@ -914,17 +912,17 @@ } { name = "binutils-2.33.1.tar.xz"; - url = "https://ftpmirror.gnu.org/gnu/binutils/binutils-2.33.1.tar.xz"; + url = "https://ftp.gnu.org/gnu/binutils/binutils-2.33.1.tar.xz"; hash = "sha256-q2b8LRw+wDWbjgiEPJ8ztj6HB+/f9eTMXCAOriRyLL8="; } { name = "gcc-9.4.0.tar.xz"; - url = "https://ftpmirror.gnu.org/gnu/gcc/gcc-9.4.0/gcc-9.4.0.tar.xz"; + url = "https://ftp.gnu.org/gnu/gcc/gcc-9.4.0/gcc-9.4.0.tar.xz"; hash = "sha256-yV2jL0QDeNd1HdlVMxhvf8Bc60+2XrW4UjTmKZ65g44="; } { name = "gmp-6.1.2.tar.bz2"; - url = "https://ftpmirror.gnu.org/gnu/gmp/gmp-6.1.2.tar.bz2"; + url = "https://ftp.gnu.org/gnu/gmp/gmp-6.1.2.tar.bz2"; hash = "sha256-UnW7BPSGOhNRay85OSrF4nL14buAV7GK7Bybedc9j7I="; } { @@ -934,12 +932,12 @@ } { name = "mpc-1.1.0.tar.gz"; - url = "https://ftpmirror.gnu.org/gnu/mpc/mpc-1.1.0.tar.gz"; + url = "https://ftp.gnu.org/gnu/mpc/mpc-1.1.0.tar.gz"; hash = "sha256-aYXFOBQ8EgjcsaxCztrW/1LiZ7R+X5cBg6PnUSW0PC4="; } { name = "mpfr-4.0.2.tar.bz2"; - url = "https://ftpmirror.gnu.org/gnu/mpfr/mpfr-4.0.2.tar.bz2"; + url = "https://ftp.gnu.org/gnu/mpfr/mpfr-4.0.2.tar.bz2"; hash = "sha256-wF4/AtCeDpAZOEzdWODxnGTm2x/W9ez3e0scYcolOsw="; } { diff --git a/pkgs/by-name/heads/package.nix b/pkgs/by-name/heads/package.nix index 64f0df2fc..65b886fa6 100644 --- a/pkgs/by-name/heads/package.nix +++ b/pkgs/by-name/heads/package.nix @@ -386,6 +386,7 @@ let runtimeInputs = [ envsubst getopt # running crossgcc's script to get list of archives + git gnumake jq nix diff --git a/pkgs/by-name/heads/update.sh b/pkgs/by-name/heads/update.sh index 90cafd5e3..3f5bb176e 100644 --- a/pkgs/by-name/heads/update.sh +++ b/pkgs/by-name/heads/update.sh @@ -72,6 +72,30 @@ checkArgCount() { fi } +# Certain URLs just don't work (inconsistent results, permanently dead) +# This will apply some replacements to such URLs, in an effort to make fetching results more consistent +fixUrl() { + checkArgCount "${FUNCNAME[0]}" 1 "$#" + + fixedUrl="$1" + + # ftpmirror.gnu.org sends us through mirror roulette, but some mirrors seem misconfigured and cause nix-prefetch-url + # to save a .tar file instead of the original .tar.gz + # Current hypothesis: headers content-type=application/x-gzip + content-encoding=x-gzip make Nix unzip the archive + # For example: mirror.checkdomain.de + # For this reason, we can't rely on it, and need to use GNU's main server - response times be damned + # + # Some URLs have /gnu/ already included, others need it added in. + fixedUrl="${fixedUrl/#"https://ftpmirror.gnu.org/gnu"/"https://ftp.gnu.org/gnu"}" + fixedUrl="${fixedUrl/#"https://ftpmirror.gnu.org"/"https://ftp.gnu.org/gnu"}" + + # Old acpica sources are painful, Intel downloadmirror gives access denied for acpica-unix2-20220331 + # This source mirror is used for other versions, don't know why upstream isn't using it for this one as well + fixedUrl="${fixedUrl/#"https://downloadmirror.intel.com/774879"/"https://mirror.math.princeton.edu/pub/libreboot/misc/acpica"}" + + echo "$fixedUrl" +} + # nix-prefetch-url doesn't give us a nice SRI hash # This will run the resulting hash through nix hash (convert) to print an SRI one getSriHash() { @@ -79,8 +103,11 @@ getSriHash() { url="$1" - hashRaw="$(nix-prefetch-url "$url" --type sha256)" - nix --extra-experimental-features nix-command hash convert --from nix32 --hash-algo sha256 --to sri "$hashRaw" + # In case of temporary gnu.org failure, let's not loose tons of progress - a human can re-attempt this manually + hashRaw="$(nix-prefetch-url "$url" --type sha256 || true)" + if [ "x" != "x${hashRaw}" ]; then + nix --extra-experimental-features nix-command hash convert --from nix32 --hash-algo sha256 --to sri "$hashRaw" + fi } # Given a package name, url & hash, substitute them into a template for a downloaded file and add the resulting @@ -110,25 +137,48 @@ collectCorebootCrossgccDeps() { corebootSrc="$2" appendToFile="$3" + # If there are patches available, apply them. They may affect download URLs + corebootPatchesDir="${srcDir}/patches/${corebootName}" + if [ -d "${corebootPatchesDir}" ]; then + # Need to apply changes to src, so need modifiable version + modifiableCorebootSrc="${tmpDir}/${corebootName}" + cp -r "$corebootSrc" "$modifiableCorebootSrc" + chmod -R +w "$modifiableCorebootSrc" + corebootSrc="$modifiableCorebootSrc" + + # Apply all found patch files + pushd "$corebootSrc" + for patch in "${corebootPatchesDir}"/*; do + git apply --verbose --reject --binary < "$patch" + done + popd + fi + crossgccSrc="$corebootSrc"/util/crossgcc/buildgcc echo "\"${corebootName}\" = [" >> "$appendToFile" while IFS= read -r -d ' ' crossgccDepUrl; do crossgccDepArchive="$(basename "$crossgccDepUrl")" - echo "Handling ${corebootName} crossgcc package: ${crossgccDepArchive}" - # ftpmirror.gnu.org sends us through mirror roulette, but some mirrors seem misconfigured and cause nix-prefetch-url - # to save a .tar file instead of the original .tar.gz - # Current hypothesis: headers content-type=application/x-gzip + content-encoding=x-gzip make Nix unzip the archive - # For example: mirror.checkdomain.de - # For this reason, we can't rely on it, and need to use GNU's main server - response times be damned - crossGccDepUrl="${crossgccDepUrl/#"https://ftpmirror.gnu.org"/"https://ftp.gnu.org/gnu"}" + if [ "$crossgccDepArchive" == "acpica-unix2-20220331.tar.gz" ]; then + echo "Skipping due to no known safe mirror: ${crossgccDepArchive}" + echo " + # Skipping ${crossgccDepArchive} because we don't have a known-good mirror + # Candidate (Heads explicitly *doesn't* use this one for this version): https://mirror.math.princeton.edu/pub/libreboot/misc/acpica/acpica-unix2-20220331.tar.gz + # Candidate (involves an archive rename): https://distfiles.macports.org/acpica/acpica-unix-20220331.tar.gz + " >> "$appendToFile" + else + echo "Handling ${corebootName} crossgcc package: ${crossgccDepArchive}" - # nix-prefetch-url doesn't give the hash in SRI format :( - crossgccDepHash="$(getSriHash "$crossGccDepUrl")" + # Apply some common URL fixes + crossGccDepUrl="$(fixUrl "$crossgccDepUrl")" + + # nix-prefetch-url doesn't give the hash in SRI format :( + crossgccDepHash="$(getSriHash "$crossGccDepUrl")" - addPackageDefinition "coreboot-crossgcc-${crossgccDepArchive}" "$crossGccDepUrl" "$crossgccDepHash" "$appendToFile" + addPackageDefinition "coreboot-crossgcc-${crossgccDepArchive}" "$crossGccDepUrl" "$crossgccDepHash" "$appendToFile" + fi done < <(echo "$(env CROSSGCC_VERSION="$corebootName" "$crossgccSrc" --urls) " | tr -d '\t') # CROSSGCC_VERSION so git isn't invoked, trailing space for read to get last entry echo "];" >> "$appendToFile" @@ -229,6 +279,9 @@ while IFS= read -r packageVersion; do archiveUrl="https://web.archive.org/web/20240910005455/https://fukuchi.org/works/qrencode/qrencode-${archiveVersion}.tar.gz" fi + # Apply some common URL fixes + archiveUrl="$(fixUrl "$archiveUrl")" + # nix-prefetch-url doesn't give the hash in SRI format :( archiveHash="$(getSriHash "$archiveUrl")" @@ -268,8 +321,12 @@ while IFS= read -r packageVersion; do # nix-prefetch-url doesn't give the hash in SRI format :( # Need to override name, contains illegal ";" - configSubHashRaw="$(nix-prefetch-url "$configSubUrl" --type sha256 --name "config.sub")" - configSubHash="$(nix --extra-experimental-features nix-command hash convert --from nix32 --hash-algo sha256 --to sri "$configSubHashRaw")" + # In case of temporary gnu.org failure, let's not loose tons of progress - a human can re-attempt this manually + configSubHashRaw="$(nix-prefetch-url "$configSubUrl" --type sha256 --name "config.sub" || true)" + configSubHash="" + if [ "x" != "x${configSubHashRaw}" ]; then + configSubHash="$(nix --extra-experimental-features nix-command hash convert --from nix32 --hash-algo sha256 --to sri "$configSubHashRaw")" + fi addPackageDefinition "config.sub" "$configSubUrl" "$configSubHash" "$muslCrossMakeTmpFile" } @@ -293,6 +350,9 @@ while IFS= read -r packageVersion; do muslCrossMakeDepUrl="${muslCrossMakeDepSite}/${muslCrossMakeDepArchive}" + # Apply some common URL fixes + muslCrossMakeDepUrl="$(fixUrl "$muslCrossMakeDepUrl")" + # nix-prefetch-url doesn't give the hash in SRI format :( muslCrossMakeDepHash="$(getSriHash "$muslCrossMakeDepUrl")"