diff --git a/pkgs/build-support/go/build-hook.sh b/pkgs/build-support/go/build-hook.sh new file mode 100644 index 0000000000000..6e2eab74842be --- /dev/null +++ b/pkgs/build-support/go/build-hook.sh @@ -0,0 +1,140 @@ + +goBuildHook() { + + runHook preBuild + + export GOFLAGS + + # currently pie is only enabled by default in pkgsMusl + # this will respect the `hardening{Disable,Enable}` flags if set + if [[ $NIX_HARDENING_ENABLE =~ "pie" ]]; then + prependToVar GOFLAGS "-buildmode=pie" + fi + + if [ -z "${allowGoReference-}" ]; then + appendToVar GOFLAGS "-trimpath" + fi + + if [ -z "${proxyVendor-}" ]; then + appendToVar GOFLAGS "-mod=vendor" + fi + + # TODO: conditionalize + appendToVar ldflags "-buildid=" + + exclude='\(/_\|examples\|Godeps\|testdata' + if [[ -n "$excludedPackages" ]]; then + IFS=' ' read -r -a excludedArr <<<$excludedPackages + printf -v excludedAlternates '%s\\|' "${excludedArr[@]}" + excludedAlternates=${excludedAlternates%\\|} # drop final \| added by printf + exclude+='\|'"$excludedAlternates" + fi + exclude+='\)' + + buildGoDir() { + local cmd="$1" dir="$2" + + declare -ga buildFlagsArray + declare -a flags + flags+=($buildFlags "${buildFlagsArray[@]}") + flags+=(${tags:+-tags=${tags// /,}}) + flags+=(${ldflags:+-ldflags="$ldflags"}) + flags+=("-p" "$NIX_BUILD_CORES") + + if [ "$cmd" = "test" ]; then + flags+=(-vet=off) + flags+=($checkFlags) + fi + + local OUT + if ! OUT="$(go $cmd "${flags[@]}" $dir 2>&1)"; then + if ! echo "$OUT" | grep -qE '(no( buildable| non-test)?|build constraints exclude all) Go (source )?files'; then + echo "$OUT" >&2 + return 1 + fi + fi + if [ -n "$OUT" ]; then + echo "$OUT" >&2 + fi + return 0 + } + + getGoDirs() { + local type; + type="$1" + if [ -n "$subPackages" ]; then + echo "$subPackages" | sed "s,\(^\| \),\1./,g" + else + find . -type f -name \*$type.go -exec dirname {} \; | grep -v "/vendor/" | sort --unique | grep -v "$exclude" + fi + } + + if (( "${NIX_DEBUG:-0}" >= 1 )); then + buildFlagsArray+=(-x) + fi + + if [ -z "$enableParallelBuilding" ]; then + export NIX_BUILD_CORES=1 + fi + + if [ -n "${modRoot}" ]; then + pushd "$modRoot" + fi + + for pkg in $(getGoDirs ""); do + echo "Building subPackage $pkg" + buildGoDir install "$pkg" + done + + if [ -n "${modRoot}" ]; then + popd + fi + + # TODO: maybe conditionalize: only run if actually cross + + # normalize cross-compiled builds w.r.t. native builds + ( + dir="$GOPATH"/bin/"$GOOS"_"$GOARCH" + if [[ -n "$(shopt -s nullglob; echo "$dir"/*)" ]]; then + mv "$dir"/* "$dir"/.. + fi + if [[ -d "$dir" ]]; then + rmdir "$dir" + fi + ) + + runHook postBuild +} + +goCheckHook() { + runHook preCheck + + # We do not set trimpath for tests, in case they reference test assets + export GOFLAGS="${GOFLAGS//-trimpath/}" + + if [ -n "${modRoot}" ]; then + pushd "$modRoot" + fi + + for pkg in $(getGoDirs test); do + buildGoDir test "$pkg" + done + + if [ -n "${modRoot}" ]; then + popd + fi + + runHook postCheck +} + + +if [ -z "${dontGoBuild-}" ] && [ -z "${buildPhase-}" ]; then + buildPhase=goBuildHook +fi + + +if [ -z "${dontGoCheck-}" ] && [ -z "${checkPhase-}" ]; then + checkPhase=goCheckHook +fi + + diff --git a/pkgs/build-support/go/config-hook.sh b/pkgs/build-support/go/config-hook.sh new file mode 100644 index 0000000000000..40ecadc0fb2b5 --- /dev/null +++ b/pkgs/build-support/go/config-hook.sh @@ -0,0 +1,28 @@ +goConfigHook() { + export GOCACHE=$TMPDIR/go-cache + export GOPATH="$TMPDIR/go" + export GOPROXY=off + export GOSUMDB=off + + if [ -d "${goModules-}" ]; then + if [ -n "${proxyVendor-}" ]; then + export GOPROXY="file://$goModules" + else + + if [ -n "${modRoot}" ]; then + pushd "$modRoot" + fi + + rm -rf vendor + cp -r --reflink=auto "$goModules" vendor + + if [ -n "${modRoot}" ]; then + popd + fi + fi + else + echo "goModules is not set or incorrect" + fi +} + +postConfigureHooks+=(goConfigHook) diff --git a/pkgs/build-support/go/install-hook.sh b/pkgs/build-support/go/install-hook.sh new file mode 100644 index 0000000000000..391966ac58ffc --- /dev/null +++ b/pkgs/build-support/go/install-hook.sh @@ -0,0 +1,14 @@ +goInstallHook() { + runHook preInstall + + mkdir -p "$out" + dir="$GOPATH/bin" + [ -e "$dir" ] && cp -r "$dir" "$out" + + runHook postInstall +} + + +if [ -z "${dontGoInstall-}" ] && [ -z "${installPhase-}" ]; then + installPhase=goInstallHook +fi diff --git a/pkgs/build-support/go/module2.nix b/pkgs/build-support/go/module2.nix new file mode 100644 index 0000000000000..b7efd261bf1b5 --- /dev/null +++ b/pkgs/build-support/go/module2.nix @@ -0,0 +1,140 @@ +{ + fetchGoModVendor, + makeSetupHook, + go, + cacert, + git, + lib, + stdenv, +}: + +argsOrFun: + +let + + # inherit given names from an attrset, but only if the name actually exists + takeAttrs = names: lib.filterAttrs (n: _: lib.elem n names); + + # these are already handled manually + namesToRemove = [ + "overrideModAttrs" + "nativeBuildInputs" + "passthru" + "meta" + ]; + + configHook = makeSetupHook { + name = "go-config-hook"; + } ./config-hook.sh; + + buildHook = makeSetupHook { + name = "go-build-hook"; + } ./build-hook.sh; + + installHook = makeSetupHook { + name = "go-install-hook"; + } ./install-hook.sh; + +in + +(stdenv.mkDerivation ( + finalAttrs: + + let + args = if lib.isFunction argsOrFun then argsOrFun finalAttrs else argsOrFun; + in + { + + # The SRI hash of the vendored dependencies. + # If `vendorHash` is `null`, no dependencies are fetched and + # the build relies on the vendor folder within the source. + vendorHash = throw "buildGoModule: vendorHash needs to be set"; + + # Directory to the `go.mod` and `go.sum` relative to the `src`. + modRoot = "./"; + + # Whether to delete the vendor folder supplied with the source. + deleteVendor = false; + + # Whether to fetch (go mod download) and proxy the vendor directory. + # This is useful if your code depends on c code and go mod tidy does not + # include the needed sources to build or if any dependency has case-insensitive + # conflicts which will produce platform dependant `vendorHash` checksums. + proxyVendor = false; + + # Do not enable this without good reason + # IE: programs coupled with the compiler. + allowGoReference = false; + + goModules = + if (finalAttrs.vendorHash == null) then + null + else + (fetchGoModVendor ( + (takeAttrs [ + "pname" + "version" + "name" + "src" + "srcs" + "sourceRoot" + + "modRoot" + "deleteVendor" + "proxyVendor" + "enableParallelBuilding" + ] finalAttrs) + // { + hash = finalAttrs.vendorHash; + } + )).overrideAttrs + finalAttrs.passthru.overrideModAttrs; + + # We want parallel builds by default. + enableParallelBuilding = true; + + strictDeps = true; + + nativeBuildInputs = [ + go + configHook + buildHook + installHook + ] ++ args.nativeBuildInputs or [ ]; + + # TODO: should probably be moved to the hooks + inherit (go) GOOS GOARCH CGO_ENABLED; + + GO111MODULE = "on"; + GOTOOLCHAIN = "local"; + + doCheck = true; + + disallowedReferences = lib.optional (!finalAttrs.allowGoReference) go; + + passthru = { + inherit go; + overrideModAttrs = args.overrideModAttrs or (finalAttrs: previousAttrs: { }); + } // args.passthru or { }; + + meta = { + # Add default meta information. + platforms = go.meta.platforms or lib.platforms.all; + } // args.meta or { }; + + # TODO: move warnings to bash or remove them + __warnings = lib.pipe null [ + (lib.warnIf (lib.any (lib.hasPrefix "-mod=") + args.GOFLAGS or [ ] + ) "use `proxyVendor` to control Go module/vendor behavior instead of setting `-mod=` in GOFLAGS") + (lib.warnIf (builtins.elem "-trimpath" args.GOFLAGS or [ ]) + "`-trimpath` is added by default to GOFLAGS by buildGoModule when allowGoReference isn't set to true" + ) + (lib.warnIf (builtins.elem "-buildid=" + args.ldflags or [ ] + ) "`-buildid=` is set by default as ldflag by buildGoModule") + ]; + + } + // lib.removeAttrs args namesToRemove +)) diff --git a/pkgs/build-support/go/vendor.nix b/pkgs/build-support/go/vendor.nix new file mode 100644 index 0000000000000..3c963e364000e --- /dev/null +++ b/pkgs/build-support/go/vendor.nix @@ -0,0 +1,113 @@ +{ + lib, + stdenv, + go, + git, + cacert, +}: + +args: + +let + removedArgs = [ + "name" + "pname" + "version" + "nativeBuildInputs" + "hash" + ]; +in +stdenv.mkDerivation ( + { + name = "${args.name or "${args.pname}-${args.version}"}-go-modules"; + + nativeBuildInputs = (args.nativeBuildInputs or [ ]) ++ [ + go + git + cacert + ]; + + inherit (go) GOOS GOARCH; + GO111MODULE = "on"; + GOTOOLCHAIN = "local"; + + impureEnvVars = lib.fetchers.proxyImpureEnvVars ++ [ + "GIT_PROXY_COMMAND" + "SOCKS_SERVER" + "GOPROXY" + ]; + + configurePhase = '' + runHook preConfigure + + export GOCACHE=$TMPDIR/go-cache + export GOPATH="$TMPDIR/go" + cd "$modRoot" + + runHook postConfigure + ''; + + buildPhase = '' + runHook preBuild + + ${lib.optionalString args.deleteVendor '' + if [ ! -d vendor ]; then + echo "vendor folder does not exist, 'deleteVendor' is not needed" + exit 10 + else + rm -rf vendor + fi + ''} + + if [ -d vendor ]; then + echo "vendor folder exists, please set 'vendorHash = null;' in your expression" + exit 10 + fi + + export GIT_SSL_CAINFO=$NIX_SSL_CERT_FILE + + ${lib.optionalString args.proxyVendor '' + mkdir -p "''${GOPATH}/pkg/mod/cache/download" + go mod download + ''} + + ${lib.optionalString (!args.proxyVendor) '' + if (( "''${NIX_DEBUG:-0}" >= 1 )); then + goModVendorFlags+=(-v) + fi + go mod vendor "''${goModVendorFlags[@]}" + ''} + + mkdir -p vendor + + runHook postBuild + ''; + + installPhase = '' + runHook preInstall + + ${lib.optionalString args.proxyVendor '' + rm -rf "''${GOPATH}/pkg/mod/cache/download/sumdb" + cp -r --reflink=auto "''${GOPATH}/pkg/mod/cache/download" $out + ''} + + ${lib.optionalString (!args.proxyVendor) '' + cp -r --reflink=auto vendor $out + ''} + + if ! [ "$(ls -A $out)" ]; then + echo "vendor folder is empty, please set 'vendorHash = null;' in your expression" + exit 10 + fi + + runHook postInstall + ''; + + dontFixup = true; + + outputHashMode = "recursive"; + outputHash = args.hash; + outputHashAlgo = if args.hash == "" then "sha256" else null; + } + // (lib.removeAttrs args removedArgs) +) diff --git a/pkgs/by-name/co/cook-framework/package.nix b/pkgs/by-name/co/cook-framework/package.nix index 8d8591750945f..8a1364819af5a 100644 --- a/pkgs/by-name/co/cook-framework/package.nix +++ b/pkgs/by-name/co/cook-framework/package.nix @@ -1,21 +1,21 @@ { lib, - buildGoModule, + buildGoModule2, fetchFromGitHub, }: -buildGoModule rec { +buildGoModule2 (finalAttrs: { pname = "cook-framework"; version = "2.2.1"; src = fetchFromGitHub { owner = "glitchedgitz"; repo = "cook"; - rev = "refs/tags/v${version}"; + rev = "refs/tags/v${finalAttrs.version}"; hash = "sha256-DK0kbvM11t64nGkrzThZgSruHTCHAPP374YPWmoM50g="; }; - sourceRoot = "${src.name}/v2"; + sourceRoot = "${finalAttrs.src.name}/v2"; vendorHash = "sha256-VpNr06IiVKpMsJXzcKCuNfJ+T+zeA9dMBMp6jeCRgn8="; @@ -24,9 +24,9 @@ buildGoModule rec { meta = { description = "Wordlist generator, splitter, merger, finder, saver for security researchers, bug bounty and hackers"; homepage = "https://github.com/glitchedgitz/cook"; - changelog = "https://github.com/glitchedgitz/cook/releases/tag/v${version}"; + changelog = "https://github.com/glitchedgitz/cook/releases/tag/v${finalAttrs.version}"; license = lib.licenses.mit; mainProgram = "cook"; maintainers = with lib.maintainers; [ tomasajt ]; }; -} +}) diff --git a/pkgs/by-name/gc/gcs/package.nix b/pkgs/by-name/gc/gcs/package.nix index e6c5ee6c54862..df99a19218c7a 100644 --- a/pkgs/by-name/gc/gcs/package.nix +++ b/pkgs/by-name/gc/gcs/package.nix @@ -1,8 +1,10 @@ { lib, stdenv, - buildGoModule, - buildNpmPackage, + buildGoModule2, + fetchNpmDeps, + npmHooks, + nodejs, fetchFromGitHub, cacert, unzip, @@ -20,14 +22,14 @@ apple-sdk_11, }: -buildGoModule rec { +buildGoModule2 (finalAttrs: { pname = "gcs"; version = "5.28.1"; src = fetchFromGitHub { owner = "richardwilkes"; repo = "gcs"; - rev = "refs/tags/v${version}"; + rev = "refs/tags/v${finalAttrs.version}"; nativeBuildInputs = [ cacert @@ -46,34 +48,36 @@ buildGoModule rec { hash = "sha256-ArJ+GveG2Y1PYeCuIFJoQ3eVyqvAi4HEeAEd4X03yu4="; }; - modPostBuild = '' - chmod +w vendor/github.com/richardwilkes/pdf - sed -i 's|-lmupdf[^ ]* |-lmupdf |g' vendor/github.com/richardwilkes/pdf/pdf.go - ''; + overrideModAttrs = { + postBuild = '' + chmod +w vendor/github.com/richardwilkes/pdf + sed -i 's|-lmupdf[^ ]* |-lmupdf |g' vendor/github.com/richardwilkes/pdf/pdf.go + ''; + }; vendorHash = "sha256-EmAGkQ+GHzVbSq/nPu0awL79jRmZuMHheBWwanfEgGI="; - frontend = buildNpmPackage { - name = "${pname}-${version}-frontend"; + nativeBuildInputs = [ + pkg-config + npmHooks.npmConfigHook + nodejs + ]; - inherit src; - sourceRoot = "${src.name}/server/frontend"; - npmDepsHash = "sha256-LqOH3jhp4Mx7JGYSjF29kVUny3xNn7oX0qCYi79SH4w="; + npmRoot = "server/frontend"; - installPhase = '' - runHook preInstall - mkdir -p $out - cp -r dist $out/dist - runHook postInstall - ''; + npmDeps = fetchNpmDeps { + name = "gcs-${finalAttrs.version}-frontend-npm-deps"; + inherit (finalAttrs) src; + sourceRoot = "${finalAttrs.src.name}/${finalAttrs.npmRoot}"; + hash = "sha256-LqOH3jhp4Mx7JGYSjF29kVUny3xNn7oX0qCYi79SH4w="; }; - postPatch = '' - cp -r ${frontend}/dist server/frontend/dist + preBuild = '' + pushd "$npmRoot" + npm run build + popd ''; - nativeBuildInputs = [ pkg-config ]; - buildInputs = [ mupdf @@ -98,7 +102,7 @@ buildGoModule rec { ldflags = [ "-s" "-w" - "-X github.com/richardwilkes/toolbox/cmdline.AppVersion=${version}" + "-X github.com/richardwilkes/toolbox/cmdline.AppVersion=${finalAttrs.version}" ]; installPhase = '' @@ -108,7 +112,7 @@ buildGoModule rec { ''; meta = { - changelog = "https://github.com/richardwilkes/gcs/releases/tag/v${version}"; + changelog = "https://github.com/richardwilkes/gcs/releases/tag/v${finalAttrs.version}"; description = "Stand-alone, interactive, character sheet editor for the GURPS 4th Edition roleplaying game system"; homepage = "https://gurpscharactersheet.com/"; license = lib.licenses.mpl20; @@ -118,4 +122,4 @@ buildGoModule rec { # incompatible vendor/github.com/richardwilkes/unison/internal/skia/libskia_linux.a broken = stdenv.hostPlatform.isLinux && stdenv.hostPlatform.isAarch64; }; -} +}) diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index ee4c200056ff6..cc3300a96b934 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -11425,6 +11425,14 @@ with pkgs; go = buildPackages.go_1_23; }; + fetchGoModVendor = callPackage ../build-support/go/vendor.nix { + go = buildPackages.go; + }; + + buildGoModule2 = callPackage ../build-support/go/module2.nix { + go = buildPackages.go; + }; + ### DEVELOPMENT / HARE hareHook = callPackage ../by-name/ha/hare/hook.nix { };