heads: Improve updateDepsScript

pkgs/by-name/heads/deps.nix

@@ -109,7 +109,7 @@
   pkgs = [
     {
       name = "bash-5.1.16.tar.gz";
-      url = "https://ftpmirror.gnu.org/bash/bash-5.1.16.tar.gz";
+      url = "https://ftp.gnu.org/gnu/bash/bash-5.1.16.tar.gz";
       hash = "sha256-W6wXIY05EYNFINrRPNH4WrlE4cCa4aulWQa+H4GS9Vg=";
     }
     {
@@ -666,13 +666,11 @@
         url = "https://ftp.gnu.org/gnu/binutils/binutils-2.37.tar.xz";
         hash = "sha256-gg2XJPAgo+acszeJOgtjwtsWHa3LDgb8Edwp6x6Eoyw=";
       }
-      # URL seems long-term broken, needs #1256 to get fixed
-      # Short-term, commenting this package out makes boards that don't need this particular coreboot version build again
-      #{
-      #  name = "coreboot-crossgcc-acpica-unix2-20220331.tar.gz";
-      #  url = "https://acpica.org/sites/acpica/files/acpica-unix2-20220331.tar.gz";
-      #  hash = "sha256-CG1rZYX1Zndob+2K9rC1upOuwYC0HSKFobdM0CtY8ko=";
-      #}
+
+      # Skipping acpica-unix2-20220331.tar.gz because we don't have a known-good mirror
+      # Candidate (Heads explicitly *doesn't* use this one for this version): https://mirror.math.princeton.edu/pub/libreboot/misc/acpica/acpica-unix2-20220331.tar.gz
+      # Candidate (involves an archive rename): https://distfiles.macports.org/acpica/acpica-unix-20220331.tar.gz
+
       {
         name = "coreboot-crossgcc-llvm-14.0.6.src.tar.xz";
         url = "https://github.com/llvm/llvm-project/releases/download/llvmorg-14.0.6/llvm-14.0.6.src.tar.xz";
@@ -914,17 +912,17 @@
     }
     {
       name = "binutils-2.33.1.tar.xz";
-      url = "https://ftpmirror.gnu.org/gnu/binutils/binutils-2.33.1.tar.xz";
+      url = "https://ftp.gnu.org/gnu/binutils/binutils-2.33.1.tar.xz";
       hash = "sha256-q2b8LRw+wDWbjgiEPJ8ztj6HB+/f9eTMXCAOriRyLL8=";
     }
     {
       name = "gcc-9.4.0.tar.xz";
-      url = "https://ftpmirror.gnu.org/gnu/gcc/gcc-9.4.0/gcc-9.4.0.tar.xz";
+      url = "https://ftp.gnu.org/gnu/gcc/gcc-9.4.0/gcc-9.4.0.tar.xz";
       hash = "sha256-yV2jL0QDeNd1HdlVMxhvf8Bc60+2XrW4UjTmKZ65g44=";
     }
     {
       name = "gmp-6.1.2.tar.bz2";
-      url = "https://ftpmirror.gnu.org/gnu/gmp/gmp-6.1.2.tar.bz2";
+      url = "https://ftp.gnu.org/gnu/gmp/gmp-6.1.2.tar.bz2";
       hash = "sha256-UnW7BPSGOhNRay85OSrF4nL14buAV7GK7Bybedc9j7I=";
     }
     {
@@ -934,12 +932,12 @@
     }
     {
       name = "mpc-1.1.0.tar.gz";
-      url = "https://ftpmirror.gnu.org/gnu/mpc/mpc-1.1.0.tar.gz";
+      url = "https://ftp.gnu.org/gnu/mpc/mpc-1.1.0.tar.gz";
       hash = "sha256-aYXFOBQ8EgjcsaxCztrW/1LiZ7R+X5cBg6PnUSW0PC4=";
     }
     {
       name = "mpfr-4.0.2.tar.bz2";
-      url = "https://ftpmirror.gnu.org/gnu/mpfr/mpfr-4.0.2.tar.bz2";
+      url = "https://ftp.gnu.org/gnu/mpfr/mpfr-4.0.2.tar.bz2";
       hash = "sha256-wF4/AtCeDpAZOEzdWODxnGTm2x/W9ez3e0scYcolOsw=";
     }
     {

pkgs/by-name/heads/package.nix

@@ -386,6 +386,7 @@ let
           runtimeInputs = [
             envsubst
             getopt # running crossgcc's script to get list of archives
+            git
             gnumake
             jq
             nix

pkgs/by-name/heads/update.sh

@@ -72,15 +72,42 @@ checkArgCount() {
   fi
 }
 
+# Certain URLs just don't work (inconsistent results, permanently dead)
+# This will apply some replacements to such URLs, in an effort to make fetching results more consistent
+fixUrl() {
+  checkArgCount "${FUNCNAME[0]}" 1 "$#"
+
+  fixedUrl="$1"
+
+  # ftpmirror.gnu.org sends us through mirror roulette, but some mirrors seem misconfigured and cause nix-prefetch-url
+  # to save a .tar file instead of the original .tar.gz
+  # Current hypothesis: headers content-type=application/x-gzip + content-encoding=x-gzip make Nix unzip the archive
+  # For example: mirror.checkdomain.de
+  # For this reason, we can't rely on it, and need to use GNU's main server - response times be damned
+  #
+  # Some URLs have /gnu/ already included, others need it added in.
+  fixedUrl="${fixedUrl/#"https://ftpmirror.gnu.org/gnu"/"https://ftp.gnu.org/gnu"}"
+  fixedUrl="${fixedUrl/#"https://ftpmirror.gnu.org"/"https://ftp.gnu.org/gnu"}"
+
+  # Old acpica sources are painful, Intel downloadmirror gives access denied for acpica-unix2-20220331
+  # This source mirror is used for other versions, don't know why upstream isn't using it for this one as well
+  fixedUrl="${fixedUrl/#"https://downloadmirror.intel.com/774879"/"https://mirror.math.princeton.edu/pub/libreboot/misc/acpica"}"
+
+  echo "$fixedUrl"
+}
+
 # nix-prefetch-url doesn't give us a nice SRI hash
 # This will run the resulting hash through nix hash (convert) to print an SRI one
 getSriHash() {
   checkArgCount "${FUNCNAME[0]}" 1 "$#"
 
   url="$1"
 
-  hashRaw="$(nix-prefetch-url "$url" --type sha256)"
-  nix --extra-experimental-features nix-command hash convert --from nix32 --hash-algo sha256 --to sri "$hashRaw"
+  # In case of temporary gnu.org failure, let's not loose tons of progress - a human can re-attempt this manually
+  hashRaw="$(nix-prefetch-url "$url" --type sha256 || true)"
+  if [ "x" != "x${hashRaw}" ]; then
+    nix --extra-experimental-features nix-command hash convert --from nix32 --hash-algo sha256 --to sri "$hashRaw"
+  fi
 }
 
 # Given a package name, url & hash, substitute them into a template for a downloaded file and add the resulting
@@ -110,25 +137,48 @@ collectCorebootCrossgccDeps() {
   corebootSrc="$2"
   appendToFile="$3"
 
+  # If there are patches available, apply them. They may affect download URLs
+  corebootPatchesDir="${srcDir}/patches/${corebootName}"
+  if [ -d "${corebootPatchesDir}" ]; then
+    # Need to apply changes to src, so need modifiable version
+    modifiableCorebootSrc="${tmpDir}/${corebootName}"
+    cp -r "$corebootSrc" "$modifiableCorebootSrc"
+    chmod -R +w "$modifiableCorebootSrc"
+    corebootSrc="$modifiableCorebootSrc"
+
+    # Apply all found patch files
+    pushd "$corebootSrc"
+    for patch in "${corebootPatchesDir}"/*; do
+      git apply --verbose --reject --binary < "$patch"
+    done
+    popd
+  fi
+
   crossgccSrc="$corebootSrc"/util/crossgcc/buildgcc
 
   echo "\"${corebootName}\" = [" >> "$appendToFile"
 
   while IFS= read -r -d ' ' crossgccDepUrl; do
     crossgccDepArchive="$(basename "$crossgccDepUrl")"
-    echo "Handling ${corebootName} crossgcc package: ${crossgccDepArchive}"
 
-    # ftpmirror.gnu.org sends us through mirror roulette, but some mirrors seem misconfigured and cause nix-prefetch-url
-    # to save a .tar file instead of the original .tar.gz
-    # Current hypothesis: headers content-type=application/x-gzip + content-encoding=x-gzip make Nix unzip the archive
-    # For example: mirror.checkdomain.de
-    # For this reason, we can't rely on it, and need to use GNU's main server - response times be damned
-    crossGccDepUrl="${crossgccDepUrl/#"https://ftpmirror.gnu.org"/"https://ftp.gnu.org/gnu"}"
+    if [ "$crossgccDepArchive" == "acpica-unix2-20220331.tar.gz" ]; then
+      echo "Skipping due to no known safe mirror: ${crossgccDepArchive}"
+      echo "
+        # Skipping ${crossgccDepArchive} because we don't have a known-good mirror
+        # Candidate (Heads explicitly *doesn't* use this one for this version): https://mirror.math.princeton.edu/pub/libreboot/misc/acpica/acpica-unix2-20220331.tar.gz
+        # Candidate (involves an archive rename): https://distfiles.macports.org/acpica/acpica-unix-20220331.tar.gz
+      " >> "$appendToFile"
+    else
+      echo "Handling ${corebootName} crossgcc package: ${crossgccDepArchive}"
 
-    # nix-prefetch-url doesn't give the hash in SRI format :(
-    crossgccDepHash="$(getSriHash "$crossGccDepUrl")"
+      # Apply some common URL fixes
+      crossGccDepUrl="$(fixUrl "$crossgccDepUrl")"
+
+      # nix-prefetch-url doesn't give the hash in SRI format :(
+      crossgccDepHash="$(getSriHash "$crossGccDepUrl")"
 
-    addPackageDefinition "coreboot-crossgcc-${crossgccDepArchive}" "$crossGccDepUrl" "$crossgccDepHash" "$appendToFile"
+      addPackageDefinition "coreboot-crossgcc-${crossgccDepArchive}" "$crossGccDepUrl" "$crossgccDepHash" "$appendToFile"
+    fi
   done < <(echo "$(env CROSSGCC_VERSION="$corebootName" "$crossgccSrc" --urls) " | tr -d '\t') # CROSSGCC_VERSION so git isn't invoked, trailing space for read to get last entry
 
   echo "];" >> "$appendToFile"
@@ -229,6 +279,9 @@ while IFS= read -r packageVersion; do
       archiveUrl="https://web.archive.org/web/20240910005455/https://fukuchi.org/works/qrencode/qrencode-${archiveVersion}.tar.gz"
     fi
 
+    # Apply some common URL fixes
+    archiveUrl="$(fixUrl "$archiveUrl")"
+
     # nix-prefetch-url doesn't give the hash in SRI format :(
     archiveHash="$(getSriHash "$archiveUrl")"
 
@@ -268,8 +321,12 @@ while IFS= read -r packageVersion; do
 
         # nix-prefetch-url doesn't give the hash in SRI format :(
         # Need to override name, contains illegal ";"
-        configSubHashRaw="$(nix-prefetch-url "$configSubUrl" --type sha256 --name "config.sub")"
-        configSubHash="$(nix --extra-experimental-features nix-command hash convert --from nix32 --hash-algo sha256 --to sri "$configSubHashRaw")"
+        # In case of temporary gnu.org failure, let's not loose tons of progress - a human can re-attempt this manually
+        configSubHashRaw="$(nix-prefetch-url "$configSubUrl" --type sha256 --name "config.sub" || true)"
+        configSubHash=""
+        if [ "x" != "x${configSubHashRaw}" ]; then
+          configSubHash="$(nix --extra-experimental-features nix-command hash convert --from nix32 --hash-algo sha256 --to sri "$configSubHashRaw")"
+        fi
 
         addPackageDefinition "config.sub" "$configSubUrl" "$configSubHash" "$muslCrossMakeTmpFile"
       }
@@ -293,6 +350,9 @@ while IFS= read -r packageVersion; do
 
         muslCrossMakeDepUrl="${muslCrossMakeDepSite}/${muslCrossMakeDepArchive}"
 
+        # Apply some common URL fixes
+        muslCrossMakeDepUrl="$(fixUrl "$muslCrossMakeDepUrl")"
+
         # nix-prefetch-url doesn't give the hash in SRI format :(
         muslCrossMakeDepHash="$(getSriHash "$muslCrossMakeDepUrl")"