From 6b9fa3f1fa2a5266c4062022c175ff19a4ac1391 Mon Sep 17 00:00:00 2001 From: Yuriy Taraday Date: Wed, 18 Jun 2025 14:38:18 +0200 Subject: [PATCH 1/6] fixVersioneerSourcesHook: init Add a hook that would fix a common issue with projects using Versioneer on GitHub: when GitHub runs analog of `git archive` to generate an archive from a tag, it subsitutes `$Format:%d$` with `(HEAD -> main, tag: the_tag)` when the tag points to the latest commit, but when more commits are added, it becomes `(tag: the_tag)`, which changes source hash. Packages that are either automatically or manually updated early, get source with this `HEAD -> main` part and so their sources are not reproducible more often. Adding this hook to projects that use Versioneer will remove this `HEAD -> main` part of the description string and make the source reproducible. Here's how it would look, for example: src = fetchFromGitHub { owner = "ANCPLabOldenburg"; repo = "ancp-bids"; tag = version; hash = "sha256-vmw8SAikvbaHnPOthBQxTbyvDwnnZwCOV97aUogIgxw="; nativeBuildInputs = [ fixVersioneerSourcesHook ]; }; I ran into this issue when preparing https://github.com/NixOS/nixpkgs/pull/416464, the general case of this issue is described in https://github.com/NixOS/nixpkgs/issues/84312. --- .../fi/fixVersioneerSourcesHook/package.nix | 59 +++++++++++++++++++ .../fi/fixVersioneerSourcesHook/setup-hook.sh | 18 ++++++ 2 files changed, 77 insertions(+) create mode 100644 pkgs/by-name/fi/fixVersioneerSourcesHook/package.nix create mode 100644 pkgs/by-name/fi/fixVersioneerSourcesHook/setup-hook.sh diff --git a/pkgs/by-name/fi/fixVersioneerSourcesHook/package.nix b/pkgs/by-name/fi/fixVersioneerSourcesHook/package.nix new file mode 100644 index 0000000000000..962051c34dfc5 --- /dev/null +++ b/pkgs/by-name/fi/fixVersioneerSourcesHook/package.nix @@ -0,0 +1,59 @@ +{ + lib, + makeSetupHook, + fetchFromGitHub, + fetchzip, + fixVersioneerSourcesHook, + runCommand, +}: +makeSetupHook { + name = "fixVersioneerSourcesHook"; + + passthru.tests.test = + let + # An example of a project that uses versioneer + src = fetchFromGitHub { + owner = "jcrist"; + repo = "msgspec"; + tag = "0.19.0"; + hash = "sha256-CajdPNAkssriY/sie5gR+4k31b3Wd7WzqcsFmrlSoPY="; + }; + # Make a clean tarball out of the source to be fetched by fetchzip later + srcClean = runCommand "source-clean.tar.gz" { } '' + tar -cf $out -C ${src} . + ''; + # Patch tarball adding a common error produced in GitHub source archives + srcDirty = runCommand "source-dirty.tar.gz" { } '' + unpackFile ${src} + chmod -R u+w source + cd source + sed -i 's/git_refnames = " (\(.*\))"/git_refnames = " (HEAD -> main, \1)"/' msgspec/_version.py + tar -cf $out . + ''; + # Make the result of fetchzip not fixed output to run it with and without hook + doFetch = + name: src: withHook: + (fetchzip { + inherit name; + url = "file://${src}"; + stripRoot = false; + nativeBuildInputs = lib.optionals withHook [ fixVersioneerSourcesHook ]; + }).overrideAttrs + { + outputHashAlgo = null; + outputHash = null; + }; + in + # Now verify that all these produce the result equal to the clean one + runCommand "test-fixVersioneerSourcesHook" + { + clean = doFetch "clean" srcClean false; + cleanWithHook = doFetch "cleanWithHook" srcClean true; + dirtyWithHook = doFetch "dirtyWithHook" srcDirty true; + } + '' + diff -ru $clean $cleanWithHook + diff -ru $clean $dirtyWithHook + touch $out + ''; +} ./setup-hook.sh diff --git a/pkgs/by-name/fi/fixVersioneerSourcesHook/setup-hook.sh b/pkgs/by-name/fi/fixVersioneerSourcesHook/setup-hook.sh new file mode 100644 index 0000000000000..829025f5ace52 --- /dev/null +++ b/pkgs/by-name/fi/fixVersioneerSourcesHook/setup-hook.sh @@ -0,0 +1,18 @@ +appendToVar postFetchHooks fixVersioneerSource + +fixVersioneerSource() { + local versionfile + # try pyproject.toml + versionfile="$([ -f "$out/pyproject.toml" ] && awk 'match($0, /^versionfile_source ?= ?['"'"'"](.*)['"'"'"]/, m) { print m[1] }' "$out/pyproject.toml" || true)" + if [ -z "$versionfile" ]; then + echo "'versionfile_source' not found in 'pyproject.toml', trying 'setup.cfg'" + # try setup.cfg + versionfile="$([ -f "$out/pyproject.toml" ] && awk 'match($0, /^versionfile_source ?= ?(.*)/, m) { print m[1] }' "$out/setup.cfg" || true)" + if [ -z "$versionfile" ]; then + echo "'versionfile_source' not found in 'setup.cfg'" + return + fi + fi + echo "found versionfile_source = $versionfile" + sed -i 's/git_refnames = " (.*\(tag:[^,)]*\).*)"/git_refnames = " (\1)"/' "$out/$versionfile" +} From 80e1dc85a4f005dcfba792b7ec081f31ea855a0d Mon Sep 17 00:00:00 2001 From: Yuriy Taraday Date: Wed, 18 Jun 2025 14:49:53 +0200 Subject: [PATCH 2/6] python3Packages.ancp-bids: enable fixVersioneerSourcesHook Also fix resulting hash, which now matches the current tarball. --- pkgs/development/python-modules/ancp-bids/default.nix | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkgs/development/python-modules/ancp-bids/default.nix b/pkgs/development/python-modules/ancp-bids/default.nix index 10a184ca67cd7..b4d9c92a23a12 100644 --- a/pkgs/development/python-modules/ancp-bids/default.nix +++ b/pkgs/development/python-modules/ancp-bids/default.nix @@ -7,6 +7,7 @@ setuptools, numpy, pandas, + fixVersioneerSourcesHook, }: buildPythonPackage rec { @@ -21,7 +22,8 @@ buildPythonPackage rec { owner = "ANCPLabOldenburg"; repo = "ancp-bids"; tag = version; - hash = "sha256-n8QfQ2PGdAO6kTfkbFpj3f2gYa3vwuYg+vPpZlGNpb0="; + hash = "sha256-dfk7AILVWXCvA12l6RrrUIR93A8nT0kn/tZT/cN63ZU="; + nativeBuildInputs = [ fixVersioneerSourcesHook ]; }; build-system = [ setuptools ]; From de0a13b67b75064c7c12139b19bf9243eebce796 Mon Sep 17 00:00:00 2001 From: Yuriy Taraday Date: Wed, 18 Jun 2025 15:17:17 +0200 Subject: [PATCH 3/6] python3Packages.bayespy: enable fixVersioneerSourcesHook Also fix resulting hash, which now matches the current tarball. --- pkgs/development/python-modules/bayespy/default.nix | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkgs/development/python-modules/bayespy/default.nix b/pkgs/development/python-modules/bayespy/default.nix index f5cfd044e850e..8720e495c81cc 100644 --- a/pkgs/development/python-modules/bayespy/default.nix +++ b/pkgs/development/python-modules/bayespy/default.nix @@ -8,6 +8,7 @@ h5py, truncnorm, pytestCheckHook, + fixVersioneerSourcesHook, }: buildPythonPackage rec { @@ -19,7 +20,8 @@ buildPythonPackage rec { owner = "bayespy"; repo = "bayespy"; tag = version; - hash = "sha256-X7CwJBrKHlU1jqMkt/7XEzaiwul1Yzkb/V64lXG4Aqo="; + hash = "sha256-kx87XY4GCL1PQIeZyovEbrPyCC/EVA6Hdvt+3P/D6VI="; + nativeBuildInputs = [ fixVersioneerSourcesHook ]; }; postPatch = '' From fb340ffeeb67c75b6b92bc0ed449f0790e1c1584 Mon Sep 17 00:00:00 2001 From: Yuriy Taraday Date: Wed, 18 Jun 2025 15:32:31 +0200 Subject: [PATCH 4/6] python3Packages.birch: enable fixVersioneerSourcesHook --- pkgs/development/python-modules/birch/default.nix | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkgs/development/python-modules/birch/default.nix b/pkgs/development/python-modules/birch/default.nix index 772ab25109e4c..3ef64390d04a9 100644 --- a/pkgs/development/python-modules/birch/default.nix +++ b/pkgs/development/python-modules/birch/default.nix @@ -8,6 +8,7 @@ pytestCheckHook, pytest-cov-stub, pyyaml, + fixVersioneerSourcesHook, }: buildPythonPackage rec { @@ -18,8 +19,9 @@ buildPythonPackage rec { src = fetchFromGitHub { owner = "shaypal5"; repo = "birch"; - rev = "v${version}"; + tag = "v${version}"; hash = "sha256-KdQZzQJvJ+logpcLQfaqqEEZJ/9VmNTQX/a4v0oBC98="; + nativeBuildInputs = [ fixVersioneerSourcesHook ]; }; patches = [ From cf592a6a05537315c6cce86755f0886dc9bd66b1 Mon Sep 17 00:00:00 2001 From: Yuriy Taraday Date: Wed, 18 Jun 2025 15:42:25 +0200 Subject: [PATCH 5/6] python3Packages.dask-jobqueue: enable fixVersioneerSourcesHook --- pkgs/development/python-modules/dask-jobqueue/default.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkgs/development/python-modules/dask-jobqueue/default.nix b/pkgs/development/python-modules/dask-jobqueue/default.nix index db3e8a4876b37..3efe7bf02a063 100644 --- a/pkgs/development/python-modules/dask-jobqueue/default.nix +++ b/pkgs/development/python-modules/dask-jobqueue/default.nix @@ -3,6 +3,7 @@ stdenv, buildPythonPackage, fetchFromGitHub, + fixVersioneerSourcesHook, # build-system setuptools, @@ -27,6 +28,7 @@ buildPythonPackage rec { repo = "dask-jobqueue"; tag = version; hash = "sha256-YujfhjOJzl4xsjjsyrQkEu/CBR04RwJ79c1iSTcMIgw="; + nativeBuildInputs = [ fixVersioneerSourcesHook ]; }; build-system = [ setuptools ]; From 4d6204ade1a06836d4159dbf20003d0d79f2c36d Mon Sep 17 00:00:00 2001 From: Yuriy Taraday Date: Wed, 18 Jun 2025 15:46:25 +0200 Subject: [PATCH 6/6] python3Packages.coredis: enable fixVersioneerSourcesHook --- pkgs/development/python-modules/coredis/default.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkgs/development/python-modules/coredis/default.nix b/pkgs/development/python-modules/coredis/default.nix index 8a955a192973c..f9ed6ede8b555 100644 --- a/pkgs/development/python-modules/coredis/default.nix +++ b/pkgs/development/python-modules/coredis/default.nix @@ -14,6 +14,7 @@ redis, typing-extensions, wrapt, + fixVersioneerSourcesHook, }: buildPythonPackage rec { @@ -26,6 +27,7 @@ buildPythonPackage rec { repo = "coredis"; tag = version; hash = "sha256-5Ho9X2VCOwKo079M2ReJ93jqEpG2ZV6vKM5/qrgzjxM="; + nativeBuildInputs = [ fixVersioneerSourcesHook ]; }; postPatch = ''