WIP: experiment with a hook based alternative to buildGoModule

pkgs/build-support/go/build-hook.sh

@@ -0,0 +1,140 @@
+
+goBuildHook() {
+
+  runHook preBuild
+
+  export GOFLAGS
+
+  # currently pie is only enabled by default in pkgsMusl
+  # this will respect the `hardening{Disable,Enable}` flags if set
+  if [[ $NIX_HARDENING_ENABLE =~ "pie" ]]; then
+    prependToVar GOFLAGS "-buildmode=pie"
+  fi
+
+  if [ -z "${allowGoReference-}" ]; then
+    appendToVar GOFLAGS "-trimpath"
+  fi
+
+  if [ -z "${proxyVendor-}" ]; then
+    appendToVar GOFLAGS "-mod=vendor"
+  fi
+
+  # TODO: conditionalize
+  appendToVar ldflags "-buildid="
+
+  exclude='\(/_\|examples\|Godeps\|testdata'
+  if [[ -n "$excludedPackages" ]]; then
+    IFS=' ' read -r -a excludedArr <<<$excludedPackages
+    printf -v excludedAlternates '%s\\|' "${excludedArr[@]}"
+    excludedAlternates=${excludedAlternates%\\|} # drop final \| added by printf
+    exclude+='\|'"$excludedAlternates"
+  fi
+  exclude+='\)'
+
+  buildGoDir() {
+    local cmd="$1" dir="$2"
+
+    declare -ga buildFlagsArray
+    declare -a flags
+    flags+=($buildFlags "${buildFlagsArray[@]}")
+    flags+=(${tags:+-tags=${tags// /,}})
+    flags+=(${ldflags:+-ldflags="$ldflags"})
+    flags+=("-p" "$NIX_BUILD_CORES")
+
+    if [ "$cmd" = "test" ]; then
+      flags+=(-vet=off)
+      flags+=($checkFlags)
+    fi
+
+    local OUT
+    if ! OUT="$(go $cmd "${flags[@]}" $dir 2>&1)"; then
+      if ! echo "$OUT" | grep -qE '(no( buildable| non-test)?|build constraints exclude all) Go (source )?files'; then
+        echo "$OUT" >&2
+        return 1
+      fi
+    fi
+    if [ -n "$OUT" ]; then
+      echo "$OUT" >&2
+    fi
+    return 0
+  }
+
+  getGoDirs() {
+    local type;
+    type="$1"
+    if [ -n "$subPackages" ]; then
+      echo "$subPackages" | sed "s,\(^\| \),\1./,g"
+    else
+      find . -type f -name \*$type.go -exec dirname {} \; | grep -v "/vendor/" | sort --unique | grep -v "$exclude"
+    fi
+  }
+
+  if (( "${NIX_DEBUG:-0}" >= 1 )); then
+    buildFlagsArray+=(-x)
+  fi
+
+  if [ -z "$enableParallelBuilding" ]; then
+      export NIX_BUILD_CORES=1
+  fi
+
+  if [ -n "${modRoot}" ]; then
+    pushd "$modRoot"
+  fi
+
+  for pkg in $(getGoDirs ""); do
+    echo "Building subPackage $pkg"
+    buildGoDir install "$pkg"
+  done
+
+  if [ -n "${modRoot}" ]; then
+    popd
+  fi
+
+  # TODO: maybe conditionalize: only run if actually cross
+
+  # normalize cross-compiled builds w.r.t. native builds
+  (
+    dir="$GOPATH"/bin/"$GOOS"_"$GOARCH"
+    if [[ -n "$(shopt -s nullglob; echo "$dir"/*)" ]]; then
+      mv "$dir"/* "$dir"/..
+    fi
+    if [[ -d "$dir" ]]; then
+      rmdir "$dir"
+    fi
+  )
+
+  runHook postBuild
+}
+
+goCheckHook() {
+  runHook preCheck
+
+  # We do not set trimpath for tests, in case they reference test assets
+  export GOFLAGS="${GOFLAGS//-trimpath/}"
+
+  if [ -n "${modRoot}" ]; then
+    pushd "$modRoot"
+  fi
+
+  for pkg in $(getGoDirs test); do
+    buildGoDir test "$pkg"
+  done
+
+  if [ -n "${modRoot}" ]; then
+    popd
+  fi
+
+  runHook postCheck
+}
+
+
+if [ -z "${dontGoBuild-}" ] && [ -z "${buildPhase-}" ]; then
+  buildPhase=goBuildHook
+fi
+
+
+if [ -z "${dontGoCheck-}" ] && [ -z "${checkPhase-}" ]; then
+  checkPhase=goCheckHook
+fi
+
+

pkgs/build-support/go/config-hook.sh

@@ -0,0 +1,28 @@
+goConfigHook() {
+  export GOCACHE=$TMPDIR/go-cache
+  export GOPATH="$TMPDIR/go"
+  export GOPROXY=off
+  export GOSUMDB=off
+
+  if [ -d "${goModules-}" ]; then
+    if [ -n "${proxyVendor-}" ]; then
+      export GOPROXY="file://$goModules"
+    else
+
+      if [ -n "${modRoot}" ]; then
+        pushd "$modRoot"
+      fi
+
+      rm -rf vendor
+      cp -r --reflink=auto "$goModules" vendor
+
+      if [ -n "${modRoot}" ]; then
+        popd
+      fi
+    fi
+  else
+    echo "goModules is not set or incorrect"
+  fi
+}
+
+postConfigureHooks+=(goConfigHook)

pkgs/build-support/go/install-hook.sh

@@ -0,0 +1,14 @@
+goInstallHook() {
+  runHook preInstall
+
+  mkdir -p "$out"
+  dir="$GOPATH/bin"
+  [ -e "$dir" ] && cp -r "$dir" "$out"
+
+  runHook postInstall
+}
+
+
+if [ -z "${dontGoInstall-}" ] && [ -z "${installPhase-}" ]; then
+    installPhase=goInstallHook
+fi

pkgs/build-support/go/module2.nix

@@ -0,0 +1,140 @@
+{
+  fetchGoModVendor,
+  makeSetupHook,
+  go,
+  cacert,
+  git,
+  lib,
+  stdenv,
+}:
+
+argsOrFun:
+
+let
+
+  # inherit given names from an attrset, but only if the name actually exists
+  takeAttrs = names: lib.filterAttrs (n: _: lib.elem n names);
+
+  # these are already handled manually
+  namesToRemove = [
+    "overrideModAttrs"
+    "nativeBuildInputs"
+    "passthru"
+    "meta"
+  ];
+
+  configHook = makeSetupHook {
+    name = "go-config-hook";
+  } ./config-hook.sh;
+
+  buildHook = makeSetupHook {
+    name = "go-build-hook";
+  } ./build-hook.sh;
+
+  installHook = makeSetupHook {
+    name = "go-install-hook";
+  } ./install-hook.sh;
+
+in
+
+(stdenv.mkDerivation (
+  finalAttrs:
+
+  let
+    args = if lib.isFunction argsOrFun then argsOrFun finalAttrs else argsOrFun;
+  in
+  {
+
+    # The SRI hash of the vendored dependencies.
+    # If `vendorHash` is `null`, no dependencies are fetched and
+    # the build relies on the vendor folder within the source.
+    vendorHash = throw "buildGoModule: vendorHash needs to be set";
+
+    # Directory to the `go.mod` and `go.sum` relative to the `src`.
+    modRoot = "./";
+
+    # Whether to delete the vendor folder supplied with the source.
+    deleteVendor = false;
+
+    # Whether to fetch (go mod download) and proxy the vendor directory.
+    # This is useful if your code depends on c code and go mod tidy does not
+    # include the needed sources to build or if any dependency has case-insensitive
+    # conflicts which will produce platform dependant `vendorHash` checksums.
+    proxyVendor = false;
+
+    # Do not enable this without good reason
+    # IE: programs coupled with the compiler.
+    allowGoReference = false;
+
+    goModules =
+      if (finalAttrs.vendorHash == null) then
+        null
+      else
+        (fetchGoModVendor (
+          (takeAttrs [
+            "pname"
+            "version"
+            "name"
+            "src"
+            "srcs"
+            "sourceRoot"
+
+            "modRoot"
+            "deleteVendor"
+            "proxyVendor"
+            "enableParallelBuilding"
+          ] finalAttrs)
+          // {
+            hash = finalAttrs.vendorHash;
+          }
+        )).overrideAttrs
+          finalAttrs.passthru.overrideModAttrs;
+
+    # We want parallel builds by default.
+    enableParallelBuilding = true;
+
+    strictDeps = true;
+
+    nativeBuildInputs = [
+      go
+      configHook
+      buildHook
+      installHook
+    ] ++ args.nativeBuildInputs or [ ];
+
+    # TODO: should probably be moved to the hooks
+    inherit (go) GOOS GOARCH CGO_ENABLED;
+
+    GO111MODULE = "on";
+    GOTOOLCHAIN = "local";
+
+    doCheck = true;
+
+    disallowedReferences = lib.optional (!finalAttrs.allowGoReference) go;
+
+    passthru = {
+      inherit go;
+      overrideModAttrs = args.overrideModAttrs or (finalAttrs: previousAttrs: { });
+    } // args.passthru or { };
+
+    meta = {
+      # Add default meta information.
+      platforms = go.meta.platforms or lib.platforms.all;
+    } // args.meta or { };
+
+    # TODO: move warnings to bash or remove them
+    __warnings = lib.pipe null [
+      (lib.warnIf (lib.any (lib.hasPrefix "-mod=")
+        args.GOFLAGS or [ ]
+      ) "use `proxyVendor` to control Go module/vendor behavior instead of setting `-mod=` in GOFLAGS")
+      (lib.warnIf (builtins.elem "-trimpath" args.GOFLAGS or [ ])
+        "`-trimpath` is added by default to GOFLAGS by buildGoModule when allowGoReference isn't set to true"
+      )
+      (lib.warnIf (builtins.elem "-buildid="
+        args.ldflags or [ ]
+      ) "`-buildid=` is set by default as ldflag by buildGoModule")
+    ];
+
+  }
+  // lib.removeAttrs args namesToRemove
+))