diff --git a/pkgs/build-support/fetchgit/builder.sh b/pkgs/build-support/fetchgit/builder.sh index 704f14598deae..acff3690b0bff 100644 --- a/pkgs/build-support/fetchgit/builder.sh +++ b/pkgs/build-support/fetchgit/builder.sh @@ -4,6 +4,8 @@ # - revision specified and remote without HEAD # +source "$NIX_ATTRS_SH_FILE" + echo "exporting $url (rev $rev) into $out" runHook preFetch @@ -18,7 +20,7 @@ $SHELL $fetcher --builder --url "$url" --out "$out" --rev "$rev" --name "$name" ${fetchLFS:+--fetch-lfs} \ ${deepClone:+--deepClone} \ ${fetchSubmodules:+--fetch-submodules} \ - ${fetchTags:+--fetch-tags} \ + "${fetchTagFlags[@]}" \ ${sparseCheckoutText:+--sparse-checkout "$sparseCheckoutText"} \ ${nonConeMode:+--non-cone-mode} \ ${branchName:+--branch-name "$branchName"} \ diff --git a/pkgs/build-support/fetchgit/default.nix b/pkgs/build-support/fetchgit/default.nix index 7a8f689df4398..d550684010589 100644 --- a/pkgs/build-support/fetchgit/default.nix +++ b/pkgs/build-support/fetchgit/default.nix @@ -26,12 +26,10 @@ let rev ? null, tag ? null, }: - if tag != null && rev != null then - throw "fetchgit requires one of either `rev` or `tag` to be provided (not both)." + if rev != null then + rev else if tag != null then "refs/tags/${tag}" - else if rev != null then - rev else # FIXME fetching HEAD if no rev or tag is provided is problematic at best "HEAD"; @@ -66,7 +64,7 @@ lib.makeOverridable ( # when rootDir is specified, avoid invalidating the result when rev changes append = if rootDir != "" then "-${lib.strings.sanitizeDerivationName rootDir}" else ""; }, - # When null, will default to: `deepClone || fetchTags` + # When null, will default to: `deepClone || fetchTags == true` for backward compatibility. leaveDotGit ? null, outputHash ? lib.fakeHash, outputHashAlgo ? null, @@ -83,6 +81,8 @@ lib.makeOverridable ( # run operations between the checkout completing and deleting the .git # directory. preFetch ? "", + # Shell code executed after `git checkout` and before .git directory removal/sanitization. + postCheckout ? "", # Shell code executed after the file has been fetched # successfully. This can do things like check or transform the file. postFetch ? "", @@ -96,8 +96,12 @@ lib.makeOverridable ( passthru ? { }, meta ? { }, allowedRequisites ? null, - # fetch all tags after tree (useful for git describe) - fetchTags ? false, + # Additional tags to fetch after tree (useful for git describe) + # Specify as `{ ${""} = [ "" "" ]; }`, + # where subpath begins with "/" and is relative to the fetching project root. + # The `subPath` of the main module is `"/"`. + # If specified as `true`, fetch all tags (with potential non-reproducibility). + fetchTags ? { }, # make this subdirectory the root of the result rootDir ? "", # GIT_CONFIG_GLOBAL (as a file) @@ -133,6 +137,8 @@ lib.makeOverridable ( derivationArgs // { + __structuredAttrs = true; + inherit name; builder = ./builder.sh; @@ -171,18 +177,51 @@ lib.makeOverridable ( deepClone branchName preFetch + postCheckout postFetch - fetchTags rootDir gitConfigFile ; + fetchTags = + let + addAdditionalTag = finalAttrs.revCustom != null && finalAttrs.tag != null; + additionalTags = [ finalAttrs.tag ]; + in + if lib.isAttrs fetchTags then + fetchTags + // { + ${if addAdditionalTag then "" else null} = fetchTags."" or [ ] ++ additionalTags; + } + else if fetchTags == false then + { ${if addAdditionalTag then "" else null} = additionalTags; } + else + fetchTags; + fetchTagFlags = + if lib.isAttrs finalAttrs.fetchTags then + lib.concatLists ( + lib.attrValues ( + lib.mapAttrs ( + subPath: + lib.concatMap (tag: [ + "--fetch-submodule-tag" + subPath + tag + ]) + ) finalAttrs.fetchTags + ) + ) + else if finalAttrs.fetchTags == true then + [ "--fetch-tags" ] + else if finalAttrs.fetchTags == false then + [ ] + else + throw "fetchgit: unsupported fetchTags value, expecting either attribute sets of subpaths and tags, or boolean `true'"; leaveDotGit = if leaveDotGit != null then - assert fetchTags -> leaveDotGit; assert rootDir != "" -> !leaveDotGit; leaveDotGit else - deepClone || fetchTags; + deepClone || fetchTags == true; nonConeMode = lib.defaultTo (finalAttrs.rootDir != "") nonConeMode; inherit tag; revCustom = rev; @@ -225,7 +264,15 @@ lib.makeOverridable ( "FETCHGIT_HTTP_PROXIES" ]; - inherit preferLocalBuild meta allowedRequisites; + outputChecks.out = { + ${if allowedRequisites != null then "allowedRequisites" else null} = allowedRequisites; + }; + + inherit preferLocalBuild meta; + + env = { + NIX_PREFETCH_GIT_CHECKOUT_HOOK = finalAttrs.postCheckout; + }; passthru = { gitRepoUrl = url; diff --git a/pkgs/build-support/fetchgit/nix-prefetch-git b/pkgs/build-support/fetchgit/nix-prefetch-git index f9c8af8428fe3..a5b942ab9be35 100755 --- a/pkgs/build-support/fetchgit/nix-prefetch-git +++ b/pkgs/build-support/fetchgit/nix-prefetch-git @@ -12,6 +12,8 @@ fetchSubmodules= fetchLFS= builder= fetchTags= +declare -A fetchSubmoduleTags=() +declare -A tagsToFetch=() branchName=$NIX_PREFETCH_GIT_BRANCH_NAME # ENV params @@ -57,7 +59,13 @@ Options: --leave-dotGit Keep the .git directories. --fetch-lfs Fetch git Large File Storage (LFS) files. --fetch-submodules Fetch submodules. - --fetch-tags Fetch all tags (useful for git describe). + --fetch-submodule-tag subpath tag + Fetch a tag for a submodule or the main repo, useful for reproducible \`git describe'. + --fetch-submodule-tags subpath + Fetch all tags for a submodule, useful for update scripts. + This is less reproducible than \`--fetch-submodule-tag' and is not recommended for typical fetcher calls. + --fetch-tag Fetch the specified tag for the main repo, equivalent to \`--fetch-submodule-tag "" tag'. + --fetch-tags Fetch all tags for the main repo, equivalent to \`--fetch-submodule-tags ""'. --builder Clone as fetchgit does, but url, rev, and out option are mandatory. --no-add-path Do not actually add the contents of the git repo to the store. --root-dir dir Directory in the repository that will be copied to the output instead of the full repository. @@ -73,6 +81,7 @@ clean_git(){ argi=0 argfun="" +preserve_argfun="" for arg; do if test -z "$argfun"; then case $arg in @@ -90,7 +99,10 @@ for arg; do --leave-dotGit) leaveDotGit=true;; --fetch-lfs) fetchLFS=true;; --fetch-submodules) fetchSubmodules=true;; - --fetch-tags) fetchTags=true;; + --fetch-submodule-tag) argfun=add_submodule_tag_key;; + --fetch-submodule-tags) argfun=add_submodule_tags;; + --fetch-tag) argfun=add_main_tag;; + --fetch-tags) fetchSubmoduleTags[/]=true;; --builder) builder=true;; --no-add-path) noAddPath=true;; --root-dir) argfun=set_rootDir;; @@ -111,11 +123,33 @@ for arg; do var=${argfun#set_} eval "$var=$(printf %q "$arg")" ;; + add_submodule_tag_key) + argfun=add_submodule_tag_value_$arg + preserve_argfun=1 + ;; + add_submodule_tag_value_*) + key=${argfun#add_submodule_tag_value_} + key="/${key#/}" + tagsToFetch[$key]="${tagsToFetch[$key]-}${tagsToFetch[$key]:+ } $arg" + ;; + add_submodule_tags) + fetchSubmoduleTags[/${arg#/}]=true + ;; + add_main_tag) + key="/" + tagsToFetch[$key]="${tagsToFetch[$key]-}${tagsToFetch[$key]:+ } $arg" + ;; esac - argfun="" + if [[ -z "$preserve_argfun" ]]; then + argfun="" + fi + preserve_argfun="" fi done +fetchTags="${fetchSubmoduleTags[/]-}" +unset fetchSubmoduleTags[/] + if test -z "$url"; then usage fi @@ -181,9 +215,26 @@ checkout_hash(){ hash=$(hash_from_ref "$ref") fi - [[ -z "$deepClone" ]] && \ - clean_git fetch ${builder:+--progress} --depth=1 origin "$hash" || \ - clean_git fetch -t ${builder:+--progress} origin || return 1 + fetchTagsFlags=("--tags") + if [[ -z "$fetchTags" ]]; then + fetchTagsFlags=("--no-tags") + fi + { + if [[ -z "$deepClone" ]]; then + clean_git fetch "${fetchTagsFlags[@]}" ${builder:+--progress} --depth=1 origin "$hash" + else + clean_git fetch "${fetchTagsFlags[@]}" ${builder:+--progress} origin + fi + } || { + echo "ERROR: \`git fetch' failed." >&2 + # Git remotes using the "dumb" protocol does not support shallow fetch; + # fall back to deep fetch if shallow fetch failed. + # TODO(@ShamrockLee): Determine whether the transfer protocol is smart reliably. + if [[ -z "$deepClone" ]] || [[ -z "$fetchTags" ]]; then + echo "This might be due to the dumb transfer protocol not supporting shallow fetch or no-tag cloning. Trying with \`--deep-clone' and \`--fetch-tags'..." >&2 + clean_git fetch --tags ${builder:+--progress} origin || return 1 + fi + } || return 1 local object_type=$(git cat-file -t "$hash") if [[ "$object_type" == "commit" || "$object_type" == "tag" ]]; then @@ -253,16 +304,46 @@ clone(){ ) # Fetch all tags if requested - if test -n "$fetchTags"; then + # The fetched tags are potentially non-reproducible, as tags are mutable parts of the Git tree. + # + # `deepClone` used to effectively imply `fetchTags`. + # We avoid such behaviour to enhance the `postCheckout` reproducibility, + # while keeping the old behaviour for `.git` for backward compatibility purposes. + # In bash, `&&` doesn't take precedence over `||``, and they are evaluated left-to-right. + if [[ -n "$deepClone" ]] && [[ -n "$leaveDotGit" ]] || [[ -n "$fetchTags" ]]; then echo "fetching all tags..." >&2 clean_git fetch origin 'refs/tags/*:refs/tags/*' || echo "warning: failed to fetch some tags" >&2 fi + # Name "$ref" to make `git describe` work reproducibly in `NIX_PREFETCH_GIT_CHECKOUT_HOOK`. + # Name only when not leaving `.git` for compatibility purposes. + if [[ -n "$ref" ]] && [[ -z "$leaveDotGit" ]]; then + echo "refer to FETCH_HEAD as its original name $ref" + clean_git update-ref "$ref" FETCH_HEAD + fi + # Checkout linked sources. if test -n "$fetchSubmodules"; then init_submodules fi + for key in "${!fetchSubmoduleTags[@]}"; do + subpath="${key#/}" + echo "fetching all tags for submodule $subpath..." >&2 + clean_git -C "$subpath" fetch origin 'refs/tags/*:refs/tags/*' || echo "warning: failed to fetch some tags" >&2 + done + + for key in "${!tagsToFetch[@]}"; do + if [[ -n "${fetchSubmoduleTags[$key]-}" ]]; then + continue + fi + subpath="${key#/}" + echo "fetching specified tags${subpath:+ at submodule $subpath}..." >&2 + for tagToFetch in ${tagsToFetch[$key]}; do + clean_git -C "$subpath" fetch origin "refs/tags/$tagToFetch:refs/tags/$tagToFetch" || echo "warning: failed to fetch tag $tagToFetch" >&2 + done + done + if [ -z "$builder" ] && [ -f .topdeps ]; then if tg help &>/dev/null; then echo "populating TopGit branches..." @@ -418,6 +499,21 @@ json_escape() { echo "$s" } +json_list() { + local result= + local arg + for arg; do + if [[ -n "$isFirst" ]]; then + result="$result, " + else + result="[" + fi + result="$result\"$(json_escape "$arg")\"" + done + result="$result]" + echo "$result" +} + print_results() { hash="$1" if ! test -n "$QUIET"; then @@ -444,7 +540,23 @@ print_results() { "fetchLFS": $([[ -n "$fetchLFS" ]] && echo true || echo false), "fetchSubmodules": $([[ -n "$fetchSubmodules" ]] && echo true || echo false), "deepClone": $([[ -n "$deepClone" ]] && echo true || echo false), - "fetchTags": $([[ -n "$fetchTags" ]] && echo true || echo false), + "fetchTags": $( + if [[ -n "$fetchTags" ]]; then + echo true + elif ((${#tagsToFetch[@]})); then + keys=("${!tagsToFetch[@]}") + keysWithComma=("${keys[@]:0:${#keys[@]-1}}") + keyLast="${keys[@]:${#keys[@]-1}:1}" + echo "{" + for key in "${keysWithComma[@]}"; do + echo " \"$(json_escape "${key#/}")\": $(json_list ${tagsToFetch[$key]})," + done + echo " \"$(json_escape "${key#/}")\": $(json_list ${tagsToFetch[$keyLast]})" + echo " }" + else + echo "{}" + fi + ), "leaveDotGit": $([[ -n "$leaveDotGit" ]] && echo true || echo false), "rootDir": "$(json_escape "$rootDir")" } diff --git a/pkgs/build-support/fetchgit/tests.nix b/pkgs/build-support/fetchgit/tests.nix index 9ccb3ff3058b4..9380f74543a9e 100644 --- a/pkgs/build-support/fetchgit/tests.nix +++ b/pkgs/build-support/fetchgit/tests.nix @@ -17,6 +17,45 @@ sha256 = "sha256-7DszvbCNTjpzGRmpIVAWXk20P0/XTrWZ79KSOGLrUWY="; }; + collect-rev = testers.invalidateFetcherByDrvHash fetchgit { + name = "collect-rev-nix-source"; + url = "https://github.com/NixOS/nix"; + rev = "9d9dbe6ed05854e03811c361a3380e09183f4f4a"; + hash = "sha256-AUTX1K7J5+fojvKYJacXYVV5kio3hrWYz5MCekO6h68="; + postCheckout = '' + git -C "$out" rev-parse HEAD | tee "$out/revision.txt" + ''; + }; + + simple-tag = testers.invalidateFetcherByDrvHash fetchgit { + name = "simple-tag-nix-source"; + url = "https://github.com/NixOS/nix"; + tag = "2.3.15"; + hash = "sha256-7DszvbCNTjpzGRmpIVAWXk20P0/XTrWZ79KSOGLrUWY="; + }; + + describe-tag = testers.invalidateFetcherByDrvHash fetchgit { + name = "describe-tag-nix-source"; + url = "https://github.com/NixOS/nix"; + tag = "2.3.15"; + hash = "sha256-7DszvbCNTjpzGRmpIVAWXk20P0/XTrWZ79KSOGLrUWY="; + postCheckout = '' + { git -C "$out" describe || echo "git describe failed"; } | tee describe-output.txt + ''; + }; + + describe-tag-unstable-version = testers.invalidateFetcherByDrvHash fetchgit { + name = "describe-tag-nix-source"; + url = "https://github.com/NixOS/nix"; + rev = "9d9dbe6ed05854e03811c361a3380e09183f4f4a"; + # for `git describe` + tag = "2.3.15"; + hash = "sha256-7DszvbCNTjpzGRmpIVAWXk20P0/XTrWZ79KSOGLrUWY="; + postCheckout = '' + { git -C "$out" describe || echo "git describe failed"; } | tee describe-output.txt + ''; + }; + sparseCheckout = testers.invalidateFetcherByDrvHash fetchgit { name = "sparse-checkout-nix-source"; url = "https://github.com/NixOS/nix"; @@ -77,6 +116,20 @@ postFetch = "rm -r $out/.git"; }; + submodule-revision-count = testers.invalidateFetcherByDrvHash fetchgit { + name = "submodule-revision-count-source"; + url = "https://github.com/pineapplehunter/nix-test-repo-with-submodule"; + rev = "26473335b84ead88ee0a3b649b1c7fa4a91cfd4a"; + hash = "sha256-ok1e6Pb0fII5TF8HXF8DXaRGSoq7kgRCoXqSEauh1wk="; + fetchSubmodules = true; + deepClone = true; + leaveDotGit = false; + postCheckout = '' + { git -C "$out" rev-list --count HEAD || echo "git rev-list failed"; } | tee "$out/revision_count.txt" + { git -C "$out/nix-test-repo-submodule" rev-list --count HEAD || echo "git rev-list failed"; } | tee "$out/nix-test-repo-submodule/revision_count.txt" + ''; + }; + submodule-leave-git-deep = testers.invalidateFetcherByDrvHash fetchgit { name = "submodule-leave-git-deep-source"; url = "https://github.com/pineapplehunter/nix-test-repo-with-submodule"; diff --git a/pkgs/build-support/fetchgithub/default.nix b/pkgs/build-support/fetchgithub/default.nix index 1283dcd7d25ad..ca5aa39906f4c 100644 --- a/pkgs/build-support/fetchgithub/default.nix +++ b/pkgs/build-support/fetchgithub/default.nix @@ -4,8 +4,41 @@ fetchgit, fetchzip, }: +let + # Here defines fetchFromGitHub arguments that determines useFetchGit, + # The attribute value is their default values. + # As fetchFromGitHub prefers fetchzip for hash stability, + # `defaultFetchGitArgs` attributes should lead to `useFetchGit = false`. + useFetchGitArgsDefault = { + deepClone = false; + fetchSubmodules = false; # This differs from fetchgit's default + fetchLFS = false; + forceFetchGit = false; + leaveDotGit = null; + postCheckout = ""; + rootDir = ""; + sparseCheckout = null; + }; + useFetchGitArgsDefaultNullable = { + leaveDotGit = false; + sparseCheckout = [ ]; + }; -lib.makeOverridable ( + useFetchGitargsDefaultNonNull = useFetchGitArgsDefault // useFetchGitArgsDefaultNullable; + + # useFetchGitArgsWD to exclude from automatic passing. + # Other useFetchGitArgsWD will pass down to fetchgit. + excludeUseFetchGitArgNames = [ + "forceFetchGit" + ]; + + faUseFetchGit = lib.mapAttrs (_: _: true) useFetchGitArgsDefault; + + adjustFunctionArgs = f: lib.setFunctionArgs f (faUseFetchGit // lib.functionArgs f); + + decorate = f: lib.makeOverridable (adjustFunctionArgs f); +in +decorate ( { owner, repo, @@ -13,30 +46,31 @@ lib.makeOverridable ( rev ? null, # TODO(@ShamrockLee): Add back after reconstruction with lib.extendMkDerivation # name ? repoRevToNameMaybe finalAttrs.repo (lib.revOrTag finalAttrs.revCustom finalAttrs.tag) "github", - # `fetchFromGitHub` defaults to use `fetchzip` for better hash stability. - # We default not to fetch submodules, which is contrary to `fetchgit`'s default. - fetchSubmodules ? false, - leaveDotGit ? null, - deepClone ? false, private ? false, - forceFetchGit ? false, - fetchLFS ? false, - rootDir ? "", - sparseCheckout ? null, githubBase ? "github.com", varPrefix ? null, passthru ? { }, meta ? { }, - ... # For hash agility + ... # For hash agility and additional fetchgit arguments }@args: assert ( - lib.assertMsg (lib.xor (tag == null) ( - rev == null - )) "fetchFromGitHub requires one of either `rev` or `tag` to be provided (not both)." + lib.assertMsg ( + tag != null || rev != null + ) "fetchFromGitHub requires at least one of `rev` or `tag` to be provided." ); let + useFetchGit = useFetchGitArgsWDNonNull != useFetchGitargsDefaultNonNull; + + useFetchGitArgs = lib.intersectAttrs useFetchGitArgsDefault args; + useFetchGitArgsWD = useFetchGitArgsDefault // useFetchGitArgs; + useFetchGitArgsWDPassing = removeAttrs useFetchGitArgsWD excludeUseFetchGitArgNames; + useFetchGitArgsWDNonNull = + useFetchGitArgsWD + // lib.mapAttrs ( + name: nonNullDefault: lib.defaultTo nonNullDefault useFetchGitArgsWD.${name} + ) useFetchGitArgsDefaultNullable; position = ( if args.meta.description or null != null then @@ -56,26 +90,19 @@ lib.makeOverridable ( # to indicate where derivation originates, similar to make-derivation.nix's mkDerivation position = "${position.file}:${toString position.line}"; }; - passthruAttrs = removeAttrs args [ - "owner" - "repo" - "tag" - "rev" - "fetchSubmodules" - "forceFetchGit" - "private" - "githubBase" - "varPrefix" - ]; + passthruAttrs = removeAttrs args ( + [ + "owner" + "repo" + "tag" + "rev" + "private" + "githubBase" + "varPrefix" + ] + ++ (if useFetchGit then excludeUseFetchGitArgNames else lib.attrNames faUseFetchGit) + ); varBase = "NIX${lib.optionalString (varPrefix != null) "_${varPrefix}"}_GITHUB_PRIVATE_"; - useFetchGit = - fetchSubmodules - || lib.defaultTo false leaveDotGit == true - || deepClone - || forceFetchGit - || fetchLFS - || (rootDir != "") - || lib.defaultTo [ ] sparseCheckout != [ ]; # We prefer fetchzip in cases we don't need submodules as the hash # is more stable in that case. fetcher = @@ -118,16 +145,9 @@ lib.makeOverridable ( passthruAttrs // ( if useFetchGit then - { - inherit - tag - rev - deepClone - fetchSubmodules - leaveDotGit - sparseCheckout - fetchLFS - ; + useFetchGitArgsWDPassing + // { + inherit tag rev; url = gitRepoUrl; inherit passthru; derivationArgs = { diff --git a/pkgs/build-support/fetchurl/builder.sh b/pkgs/build-support/fetchurl/builder.sh index 560b912d414f2..44ac80737bc07 100644 --- a/pkgs/build-support/fetchurl/builder.sh +++ b/pkgs/build-support/fetchurl/builder.sh @@ -1,3 +1,4 @@ +source "$NIX_ATTRS_SH_FILE" source $mirrorsFile curlVersion=$(curl -V | head -1 | cut -d' ' -f2) @@ -22,10 +23,10 @@ if ! [ -f "$SSL_CERT_FILE" ]; then curl+=(--insecure) fi -eval "curl+=($curlOptsList)" +curl+=("${curlOptsList[@]}") curl+=( - $curlOpts + ${curlOpts[*]} $NIX_CURL_FLAGS ) diff --git a/pkgs/build-support/fetchurl/default.nix b/pkgs/build-support/fetchurl/default.nix index 5ffb70e3a9f8a..26aaa90e1bccb 100644 --- a/pkgs/build-support/fetchurl/default.nix +++ b/pkgs/build-support/fetchurl/default.nix @@ -136,6 +136,7 @@ lib.extendMkDerivation { # Passthru information, if any. passthru ? { }, + # Doing the download on a remote machine just duplicates network # traffic, so don't do that by default preferLocalBuild ? true, @@ -238,6 +239,8 @@ lib.extendMkDerivation { derivationArgs // { + __structuredAttrs = true; + name = if finalAttrs.pname or null != null && finalAttrs.version or null != null then "${finalAttrs.pname}-${finalAttrs.version}" @@ -297,14 +300,13 @@ lib.extendMkDerivation { '' ) curlOpts; - curlOptsList = lib.escapeShellArgs curlOptsList; - inherit - showURLs - mirrorsFile - postFetch + curlOptsList downloadToTemp executable + mirrorsFile + postFetch + showURLs ; impureEnvVars = impureEnvVars ++ netrcImpureEnvVars;