diff --git a/modules/tarball-mirror.patch b/modules/tarball-mirror.patch index 4848eb23..0eaeb515 100644 --- a/modules/tarball-mirror.patch +++ b/modules/tarball-mirror.patch @@ -1,11 +1,155 @@ +From 89093ba05e6f9710aa0dcb500f6226f1be80cc86 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= +Date: Wed, 4 Dec 2024 09:39:04 +0100 +Subject: [PATCH] copy-tarballs: drop perl bindings + +This hopefully makes it easier to re-write this script in a language +that people understand. Because it's shelling out, it's likely slower +but hopefully still fast enough for our purposes. +--- + maintainers/scripts/copy-tarballs.pl | 78 +++++++++++++++++++++------- + 1 file changed, 58 insertions(+), 20 deletions(-) + diff --git a/maintainers/scripts/copy-tarballs.pl b/maintainers/scripts/copy-tarballs.pl -index 30fbac6f002d..71569966d51b 100755 +index 30fbac6f002d90..cb117ad2be0762 100755 --- a/maintainers/scripts/copy-tarballs.pl +++ b/maintainers/scripts/copy-tarballs.pl @@ -1,5 +1,5 @@ #! /usr/bin/env nix-shell -#! nix-shell -i perl -p perl perlPackages.NetAmazonS3 perlPackages.FileSlurp perlPackages.JSON perlPackages.LWPProtocolHttps nix nix.perl-bindings -+#! nix-shell -i perl -p perl perlPackages.NetAmazonS3 perlPackages.FileSlurp perlPackages.JSON perlPackages.LWPProtocolHttps nixVersions.nix_2_18 nixVersions.nix_2_18.perl-bindings ++#! nix-shell -i perl -p perl perlPackages.NetAmazonS3 perlPackages.FileSlurp perlPackages.JSON perlPackages.LWPProtocolHttps nix # This command uploads tarballs to tarballs.nixos.org, the # content-addressed cache used by fetchurl as a fallback for when +@@ -20,14 +20,51 @@ + use File::Slurp; + use JSON; + use Net::Amazon::S3; +-use Nix::Store; +- +-isValidPath("/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo"); # FIXME: forces Nix::Store initialisation + + sub usage { + die "Syntax: $0 [--dry-run] [--exclude REGEXP] [--expr EXPR | --file FILES...]\n"; + } + ++sub computeFixedOutputPath { ++ my ($name, $algo, $hash) = @_; ++ my $expr = <<'EXPR'; ++{ name, outputHashAlgo, outputHash }: ++builtins.toString (derivation { ++ inherit name outputHashAlgo outputHash; ++ builder = "false"; ++ system = "dontcare"; ++ outputHashMode = "flat"; ++}) ++EXPR ++ open(my $fh, "-|", ++ "nix-instantiate", ++ "--eval", ++ "--strict", ++ "-E", $expr, ++ "--argstr", "name", $name, ++ "--argstr", "outputHashAlgo", $algo, ++ "--argstr", "outputHash", $hash) or die "Failed to run nix-instantiate: $!"; ++ ++ my $storePathJson = <$fh>; ++ chomp $storePathJson; ++ my $storePath = decode_json($storePathJson); ++ close $fh; ++ return $storePath; ++} ++ ++sub nixHash { ++ my ($algo, $base16, $path) = @_; ++ open(my $fh, "-|", ++ "nix-hash", ++ "--type", $algo, ++ "--flat", ++ ($base16 ? "--base16" : ()), ++ $path) or die "Failed to run nix-hash: $!"; ++ my $hash = <$fh>; ++ chomp $hash; ++ return $hash; ++} ++ + my $dryRun = 0; + my $expr; + my @fileNames; +@@ -90,12 +127,12 @@ sub alreadyMirrored { + sub uploadFile { + my ($fn, $name) = @_; + +- my $md5_16 = hashFile("md5", 0, $fn) or die; +- my $sha1_16 = hashFile("sha1", 0, $fn) or die; +- my $sha256_32 = hashFile("sha256", 1, $fn) or die; +- my $sha256_16 = hashFile("sha256", 0, $fn) or die; +- my $sha512_32 = hashFile("sha512", 1, $fn) or die; +- my $sha512_16 = hashFile("sha512", 0, $fn) or die; ++ my $md5_16 = nixHash("md5", 0, $fn) or die; ++ my $sha1_16 = nixHash("sha1", 0, $fn) or die; ++ my $sha256_32 = nixHash("sha256", 1, $fn) or die; ++ my $sha256_16 = nixHash("sha256", 0, $fn) or die; ++ my $sha512_32 = nixHash("sha512", 1, $fn) or die; ++ my $sha512_16 = nixHash("sha512", 0, $fn) or die; + + my $mainKey = "sha512/$sha512_16"; + +@@ -130,7 +167,7 @@ sub uploadFile { + my $res = 0; + foreach my $fn (@fileNames) { + eval { +- if (alreadyMirrored("sha512", hashFile("sha512", 0, $fn))) { ++ if (alreadyMirrored("sha512", nixHash("sha512", 0, $fn))) { + print STDERR "$fn is already mirrored\n"; + } else { + uploadFile($fn, basename $fn); +@@ -176,7 +213,9 @@ sub uploadFile { + + if ($hash =~ /^([a-z0-9]+)-([A-Za-z0-9+\/=]+)$/) { + $algo = $1; +- $hash = `nix hash to-base16 $hash` or die; ++ open(my $fh, "-|", "nix", "--extra-experimental-features", "nix-command", "hash", "convert", "--to", "base16", $hash) or die; ++ $hash = <$fh>; ++ close $fh; + chomp $hash; + } + +@@ -184,11 +223,13 @@ sub uploadFile { + + # Convert non-SRI base-64 to base-16. + if ($hash =~ /^[A-Za-z0-9+\/=]+$/) { +- $hash = `nix hash to-base16 --type '$algo' $hash` or die; ++ open(my $fh, "-|", "nix", "--extra-experimental-features", "nix-command", "hash", "convert", "--to", "base16", "--hash-algo", $algo, $hash) or die; ++ $hash = <$fh>; ++ close $fh; + chomp $hash; + } + +- my $storePath = makeFixedOutputPath(0, $algo, $hash, $name); ++ my $storePath = computeFixedOutputPath($name, $algo, $hash); + + for my $url (@$urls) { + if (defined $ENV{DEBUG}) { +@@ -210,18 +251,15 @@ sub uploadFile { + + print STDERR "mirroring $url ($storePath, $algo, $hash)...\n"; + ++ + if ($dryRun) { + $mirrored++; + last; + } +- +- # Substitute the output. +- if (!isValidPath($storePath)) { +- system("nix-store", "-r", $storePath); +- } ++ my $isValidPath = system("nix-store", "-r", $storePath) == 0; + + # Otherwise download the file using nix-prefetch-url. +- if (!isValidPath($storePath)) { ++ if (!$isValidPath) { + $ENV{QUIET} = 1; + $ENV{PRINT_PATH} = 1; + my $fh;