[WIP] nix-prefetch-git: facilitate submodule tag discovery for update scripts

pkgs/build-support/fetchgit/builder.sh

@@ -4,6 +4,8 @@
 # - revision specified and remote without HEAD
 #
 
+source "$NIX_ATTRS_SH_FILE"
+
 echo "exporting $url (rev $rev) into $out"
 
 runHook preFetch
@@ -18,7 +20,7 @@ $SHELL $fetcher --builder --url "$url" --out "$out" --rev "$rev" --name "$name"
   ${fetchLFS:+--fetch-lfs} \
   ${deepClone:+--deepClone} \
   ${fetchSubmodules:+--fetch-submodules} \
-  ${fetchTags:+--fetch-tags} \
+  "${fetchTagFlags[@]}" \
   ${sparseCheckoutText:+--sparse-checkout "$sparseCheckoutText"} \
   ${nonConeMode:+--non-cone-mode} \
   ${branchName:+--branch-name "$branchName"} \

pkgs/build-support/fetchgit/default.nix

@@ -26,12 +26,10 @@ let
       rev ? null,
       tag ? null,
     }:
-    if tag != null && rev != null then
-      throw "fetchgit requires one of either `rev` or `tag` to be provided (not both)."
+    if rev != null then
+      rev
     else if tag != null then
       "refs/tags/${tag}"
-    else if rev != null then
-      rev
     else
       # FIXME fetching HEAD if no rev or tag is provided is problematic at best
       "HEAD";
@@ -66,7 +64,7 @@ lib.makeOverridable (
             # when rootDir is specified, avoid invalidating the result when rev changes
             append = if rootDir != "" then "-${lib.strings.sanitizeDerivationName rootDir}" else "";
           },
-          # When null, will default to: `deepClone || fetchTags`
+          # When null, will default to: `deepClone || fetchTags == true` for backward compatibility.
           leaveDotGit ? null,
           outputHash ? lib.fakeHash,
           outputHashAlgo ? null,
@@ -83,6 +81,8 @@ lib.makeOverridable (
           # run operations between the checkout completing and deleting the .git
           # directory.
           preFetch ? "",
+          # Shell code executed after `git checkout` and before .git directory removal/sanitization.
+          postCheckout ? "",
           # Shell code executed after the file has been fetched
           # successfully. This can do things like check or transform the file.
           postFetch ? "",
@@ -96,8 +96,12 @@ lib.makeOverridable (
           passthru ? { },
           meta ? { },
           allowedRequisites ? null,
-          # fetch all tags after tree (useful for git describe)
-          fetchTags ? false,
+          # Additional tags to fetch after tree (useful for git describe)
+          # Specify as `{ ${"<subPath>"} = [ "<tag1>" "<tag2>" ]; }`,
+          # where subpath begins with "/" and is relative to the fetching project root.
+          # The `subPath` of the main module is `"/"`.
+          # If specified as `true`, fetch all tags (with potential non-reproducibility).
+          fetchTags ? { },
           # make this subdirectory the root of the result
           rootDir ? "",
           # GIT_CONFIG_GLOBAL (as a file)
@@ -133,6 +137,8 @@ lib.makeOverridable (
 
         derivationArgs
         // {
+          __structuredAttrs = true;
+
           inherit name;
 
           builder = ./builder.sh;
@@ -171,18 +177,51 @@ lib.makeOverridable (
             deepClone
             branchName
             preFetch
+            postCheckout
             postFetch
-            fetchTags
             rootDir
             gitConfigFile
             ;
+          fetchTags =
+            let
+              addAdditionalTag = finalAttrs.revCustom != null && finalAttrs.tag != null;
+              additionalTags = [ finalAttrs.tag ];
+            in
+            if lib.isAttrs fetchTags then
+              fetchTags
+              // {
+                ${if addAdditionalTag then "" else null} = fetchTags."" or [ ] ++ additionalTags;
+              }
+            else if fetchTags == false then
+              { ${if addAdditionalTag then "" else null} = additionalTags; }
+            else
+              fetchTags;
+          fetchTagFlags =
+            if lib.isAttrs finalAttrs.fetchTags then
+              lib.concatLists (
+                lib.attrValues (
+                  lib.mapAttrs (
+                    subPath:
+                    lib.concatMap (tag: [
+                      "--fetch-submodule-tag"
+                      subPath
+                      tag
+                    ])
+                  ) finalAttrs.fetchTags
+                )
+              )
+            else if finalAttrs.fetchTags == true then
+              [ "--fetch-tags" ]
+            else if finalAttrs.fetchTags == false then
+              [ ]
+            else
+              throw "fetchgit: unsupported fetchTags value, expecting either attribute sets of subpaths and tags, or boolean `true'";
           leaveDotGit =
             if leaveDotGit != null then
-              assert fetchTags -> leaveDotGit;
               assert rootDir != "" -> !leaveDotGit;
               leaveDotGit
             else
-              deepClone || fetchTags;
+              deepClone || fetchTags == true;
           nonConeMode = lib.defaultTo (finalAttrs.rootDir != "") nonConeMode;
           inherit tag;
           revCustom = rev;
@@ -225,7 +264,15 @@ lib.makeOverridable (
               "FETCHGIT_HTTP_PROXIES"
             ];
 
-          inherit preferLocalBuild meta allowedRequisites;
+          outputChecks.out = {
+            ${if allowedRequisites != null then "allowedRequisites" else null} = allowedRequisites;
+          };
+
+          inherit preferLocalBuild meta;
+
+          env = {
+            NIX_PREFETCH_GIT_CHECKOUT_HOOK = finalAttrs.postCheckout;
+          };
 
           passthru = {
             gitRepoUrl = url;

pkgs/build-support/fetchgit/nix-prefetch-git

@@ -12,6 +12,8 @@ fetchSubmodules=
 fetchLFS=
 builder=
 fetchTags=
+declare -A fetchSubmoduleTags=()
+declare -A tagsToFetch=()
 branchName=$NIX_PREFETCH_GIT_BRANCH_NAME
 
 # ENV params
@@ -57,7 +59,13 @@ Options:
       --leave-dotGit  Keep the .git directories.
       --fetch-lfs     Fetch git Large File Storage (LFS) files.
       --fetch-submodules Fetch submodules.
-      --fetch-tags    Fetch all tags (useful for git describe).
+      --fetch-submodule-tag subpath tag
+          Fetch a tag for a submodule or the main repo, useful for reproducible \`git describe'.
+      --fetch-submodule-tags subpath
+          Fetch all tags for a submodule, useful for update scripts.
+          This is less reproducible than \`--fetch-submodule-tag' and is not recommended for typical fetcher calls.
+      --fetch-tag     Fetch the specified tag for the main repo, equivalent to \`--fetch-submodule-tag "" tag'.
+      --fetch-tags    Fetch all tags for the main repo, equivalent to \`--fetch-submodule-tags ""'.
       --builder       Clone as fetchgit does, but url, rev, and out option are mandatory.
       --no-add-path   Do not actually add the contents of the git repo to the store.
       --root-dir dir  Directory in the repository that will be copied to the output instead of the full repository.
@@ -73,6 +81,7 @@ clean_git(){
 
 argi=0
 argfun=""
+preserve_argfun=""
 for arg; do
     if test -z "$argfun"; then
         case $arg in
@@ -90,7 +99,10 @@ for arg; do
             --leave-dotGit) leaveDotGit=true;;
             --fetch-lfs) fetchLFS=true;;
             --fetch-submodules) fetchSubmodules=true;;
-            --fetch-tags) fetchTags=true;;
+            --fetch-submodule-tag) argfun=add_submodule_tag_key;;
+            --fetch-submodule-tags) argfun=add_submodule_tags;;
+            --fetch-tag) argfun=add_main_tag;;
+            --fetch-tags) fetchSubmoduleTags[/]=true;;
             --builder) builder=true;;
             --no-add-path) noAddPath=true;;
             --root-dir) argfun=set_rootDir;;
@@ -111,11 +123,33 @@ for arg; do
                 var=${argfun#set_}
                 eval "$var=$(printf %q "$arg")"
                 ;;
+            add_submodule_tag_key)
+                argfun=add_submodule_tag_value_$arg
+                preserve_argfun=1
+                ;;
+            add_submodule_tag_value_*)
+                key=${argfun#add_submodule_tag_value_}
+                key="/${key#/}"
+                tagsToFetch[$key]="${tagsToFetch[$key]-}${tagsToFetch[$key]:+ } $arg"
+                ;;
+            add_submodule_tags)
+                fetchSubmoduleTags[/${arg#/}]=true
+                ;;
+            add_main_tag)
+                key="/"
+                tagsToFetch[$key]="${tagsToFetch[$key]-}${tagsToFetch[$key]:+ } $arg"
+                ;;
         esac
-        argfun=""
+        if [[ -z "$preserve_argfun" ]]; then
+            argfun=""
+        fi
+        preserve_argfun=""
     fi
 done
 
+fetchTags="${fetchSubmoduleTags[/]-}"
+unset fetchSubmoduleTags[/]
+
 if test -z "$url"; then
     usage
 fi
@@ -181,9 +215,26 @@ checkout_hash(){
         hash=$(hash_from_ref "$ref")
     fi
 
-    [[ -z "$deepClone" ]] && \
-    clean_git fetch ${builder:+--progress} --depth=1 origin "$hash" || \
-    clean_git fetch -t ${builder:+--progress} origin || return 1
+    fetchTagsFlags=("--tags")
+    if [[ -z "$fetchTags" ]]; then
+        fetchTagsFlags=("--no-tags")
+    fi
+    {
+        if [[ -z "$deepClone" ]]; then
+            clean_git fetch "${fetchTagsFlags[@]}" ${builder:+--progress} --depth=1 origin "$hash"
+        else
+            clean_git fetch "${fetchTagsFlags[@]}" ${builder:+--progress} origin
+        fi
+    } || {
+        echo "ERROR: \`git fetch' failed." >&2
+        # Git remotes using the "dumb" protocol does not support shallow fetch;
+        # fall back to deep fetch if shallow fetch failed.
+        # TODO(@ShamrockLee): Determine whether the transfer protocol is smart reliably.
+        if [[ -z "$deepClone" ]] || [[ -z "$fetchTags" ]]; then
+            echo "This might be due to the dumb transfer protocol not supporting shallow fetch or no-tag cloning. Trying with \`--deep-clone' and \`--fetch-tags'..." >&2
+            clean_git fetch --tags ${builder:+--progress} origin || return 1
+        fi
+    } || return 1
 
     local object_type=$(git cat-file -t "$hash")
     if [[ "$object_type" == "commit" || "$object_type" == "tag" ]]; then
@@ -253,16 +304,46 @@ clone(){
     )
 
     # Fetch all tags if requested
-    if test -n "$fetchTags"; then
+    # The fetched tags are potentially non-reproducible, as tags are mutable parts of the Git tree.
+    #
+    # `deepClone` used to effectively imply `fetchTags`.
+    # We avoid such behaviour to enhance the `postCheckout` reproducibility,
+    # while keeping the old behaviour for `.git` for backward compatibility purposes.
+    # In bash, `&&` doesn't take precedence over `||``, and they are evaluated left-to-right.
+    if [[ -n "$deepClone" ]] && [[ -n "$leaveDotGit" ]] || [[ -n "$fetchTags" ]]; then
         echo "fetching all tags..." >&2
         clean_git fetch origin 'refs/tags/*:refs/tags/*' || echo "warning: failed to fetch some tags" >&2
     fi
 
+    # Name "$ref" to make `git describe` work reproducibly in `NIX_PREFETCH_GIT_CHECKOUT_HOOK`.
+    # Name only when not leaving `.git` for compatibility purposes.
+    if [[ -n "$ref" ]] && [[ -z "$leaveDotGit" ]]; then
+        echo "refer to FETCH_HEAD as its original name $ref"
+        clean_git update-ref "$ref" FETCH_HEAD
+    fi
+
     # Checkout linked sources.
     if test -n "$fetchSubmodules"; then
         init_submodules
     fi
 
+    for key in "${!fetchSubmoduleTags[@]}"; do
+        subpath="${key#/}"
+        echo "fetching all tags for submodule $subpath..." >&2
+        clean_git -C "$subpath" fetch origin 'refs/tags/*:refs/tags/*' || echo "warning: failed to fetch some tags" >&2
+    done
+
+    for key in "${!tagsToFetch[@]}"; do
+        if [[ -n "${fetchSubmoduleTags[$key]-}" ]]; then
+            continue
+        fi
+        subpath="${key#/}"
+        echo "fetching specified tags${subpath:+ at submodule $subpath}..." >&2
+        for tagToFetch in ${tagsToFetch[$key]}; do
+            clean_git -C "$subpath" fetch origin "refs/tags/$tagToFetch:refs/tags/$tagToFetch" || echo "warning: failed to fetch tag $tagToFetch" >&2
+        done
+    done
+
     if [ -z "$builder" ] && [ -f .topdeps ]; then
         if tg help &>/dev/null; then
             echo "populating TopGit branches..."
@@ -418,6 +499,21 @@ json_escape() {
     echo "$s"
 }
 
+json_list() {
+    local result=
+    local arg
+    for arg; do
+        if [[ -n "$isFirst" ]]; then
+            result="$result, "
+        else
+            result="["
+        fi
+        result="$result\"$(json_escape "$arg")\""
+    done
+    result="$result]"
+    echo "$result"
+}
+
 print_results() {
     hash="$1"
     if ! test -n "$QUIET"; then
@@ -444,7 +540,23 @@ print_results() {
   "fetchLFS": $([[ -n "$fetchLFS" ]] && echo true || echo false),
   "fetchSubmodules": $([[ -n "$fetchSubmodules" ]] && echo true || echo false),
   "deepClone": $([[ -n "$deepClone" ]] && echo true || echo false),
-  "fetchTags": $([[ -n "$fetchTags" ]] && echo true || echo false),
+  "fetchTags": $(
+    if [[ -n "$fetchTags" ]]; then
+        echo true
+    elif ((${#tagsToFetch[@]})); then
+        keys=("${!tagsToFetch[@]}")
+        keysWithComma=("${keys[@]:0:${#keys[@]-1}}")
+        keyLast="${keys[@]:${#keys[@]-1}:1}"
+        echo "{"
+        for key in "${keysWithComma[@]}"; do
+            echo "    \"$(json_escape "${key#/}")\": $(json_list ${tagsToFetch[$key]}),"
+        done
+        echo "    \"$(json_escape "${key#/}")\": $(json_list ${tagsToFetch[$keyLast]})"
+        echo "  }"
+    else
+        echo "{}"
+    fi
+  ),
   "leaveDotGit": $([[ -n "$leaveDotGit" ]] && echo true || echo false),
   "rootDir": "$(json_escape "$rootDir")"
 }