Merge pull request #18017 from MadhavJivrajani/go-version-setup-35

[release-3.5] Setup a way to consistently manage go versions across scripts and go.mods

.github/workflows/static-analysis.yaml

@@ -0,0 +1,32 @@
+---
+name: Static Analysis
+on: [push, pull_request]
+permissions: read-all
+jobs:
+  run:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5
+      - id: goversion
+        run: echo "goversion=$(cat .go-version)" >> "$GITHUB_OUTPUT"
+      - uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5.0.1
+        with:
+          go-version: ${{ steps.goversion.outputs.goversion }}
+      - run: |
+          set -euo pipefail
+
+          make verify
+      - run: |
+          set -euo pipefail
+
+          make fix
+
+          DIFF=$(git status --porcelain)
+
+          if [ -n "$DIFF" ]; then
+            echo "These files were modified:"
+            echo
+            echo "$DIFF"
+            echo
+            exit 1
+          fi

Makefile

@@ -580,3 +580,18 @@ gofail-enable: install-gofail
 .PHONY: gofail-disable
 gofail-disable: install-gofail
 	PASSES="toggle_failpoints" ./test.sh
+
+.PHONY: verify
+verify: verify-go-versions
+
+.PHONY: verify-go-versions
+verify-go-versions:
+	./scripts/verify_go_versions.sh
+
+.PHONY: fix
+fix: sync-toolchain-directive
+	./scripts/fix.sh
+
+.PHONY: sync-toolchain-directive
+sync-toolchain-directive:
+	./scripts/sync_go_toolchain_directive.sh

api/go.mod

@@ -2,6 +2,8 @@ module go.etcd.io/etcd/api/v3
 
 go 1.21
 
+toolchain go1.21.10
+
 require (
 	github.com/coreos/go-semver v0.3.0
 	github.com/gogo/protobuf v1.3.2

client/pkg/go.mod

@@ -2,6 +2,8 @@ module go.etcd.io/etcd/client/pkg/v3
 
 go 1.21
 
+toolchain go1.21.10
+
 require (
 	github.com/coreos/go-systemd/v22 v22.3.2
 	github.com/stretchr/testify v1.8.4

client/v2/go.mod

@@ -2,6 +2,8 @@ module go.etcd.io/etcd/client/v2
 
 go 1.21
 
+toolchain go1.21.10
+
 require (
 	github.com/json-iterator/go v1.1.11
 	github.com/modern-go/reflect2 v1.0.1

client/v3/go.mod

@@ -2,6 +2,8 @@ module go.etcd.io/etcd/client/v3
 
 go 1.21
 
+toolchain go1.21.10
+
 require (
 	github.com/dustin/go-humanize v1.0.0
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0

etcdctl/go.mod

@@ -2,6 +2,8 @@ module go.etcd.io/etcd/etcdctl/v3
 
 go 1.21
 
+toolchain go1.21.10
+
 require (
 	github.com/bgentry/speakeasy v0.1.0
 	github.com/dustin/go-humanize v1.0.0

etcdutl/go.mod

@@ -2,6 +2,8 @@ module go.etcd.io/etcd/etcdutl/v3
 
 go 1.21
 
+toolchain go1.21.10
+
 replace (
 	go.etcd.io/etcd/api/v3 => ../api
 	go.etcd.io/etcd/client/pkg/v3 => ../client/pkg

go.mod

@@ -2,6 +2,8 @@ module go.etcd.io/etcd/v3
 
 go 1.21
 
+toolchain go1.21.10
+
 replace (
 	go.etcd.io/etcd/api/v3 => ./api
 	go.etcd.io/etcd/client/pkg/v3 => ./client/pkg

pkg/go.mod

@@ -2,6 +2,8 @@ module go.etcd.io/etcd/pkg/v3
 
 go 1.21
 
+toolchain go1.21.10
+
 require (
 	github.com/creack/pty v1.1.11
 	github.com/dustin/go-humanize v1.0.0

raft/go.mod

@@ -2,6 +2,8 @@ module go.etcd.io/etcd/raft/v3
 
 go 1.21
 
+toolchain go1.21.10
+
 require (
 	github.com/cockroachdb/datadriven v1.0.2
 	github.com/gogo/protobuf v1.3.2

scripts/sync_go_toolchain_directive.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+# This script looks at the version present in the .go-version file and treats
+# that to be the value of the toolchain directive that go should use. It then
+# updates the toolchain directives of all go.mod files to reflect this version.
+#
+# We do this to ensure that .go-version acts as the source of truth for go versions.
+
+set -euo pipefail
+
+source ./scripts/test_lib.sh
+
+TARGET_GO_VERSION="${TARGET_GO_VERSION:-"$(cat "${ETCD_ROOT_DIR}/.go-version")"}"
+find . -name 'go.mod' -exec go mod edit -toolchain=go"${TARGET_GO_VERSION}" {} \;

scripts/test_lib.sh

@@ -384,3 +384,27 @@ function git_assert_branch_in_sync {
     log_warning "Cannot verify consistency with the origin, as git is on detached branch."
   fi
 }
+
+# The version present in the .go-verion is the default version that test and build scripts will use.
+# However, it is possible to control the version that should be used with the help of env vars:
+# - FORCE_HOST_GO: if set to a non-empty value, use the version of go installed in system's $PATH.
+# - GO_VERSION: desired version of go to be used, might differ from what is present in .go-version.
+#               If empty, the value defaults to the version in .go-version.
+function determine_go_version {
+  # Borrowing from how Kubernetes does this:
+  #  https://github.com/kubernetes/kubernetes/blob/17854f0e0a153b06f9d0db096e2cd8ab2fa89c11/hack/lib/golang.sh#L510-L520
+  #
+  # default GO_VERSION to content of .go-version
+  GO_VERSION="${GO_VERSION:-"$(cat "${ETCD_ROOT_DIR}/.go-version")"}"
+  if [ "${GOTOOLCHAIN:-auto}" != 'auto' ]; then
+    # no-op, just respect GOTOOLCHAIN
+    :
+  elif [ -n "${FORCE_HOST_GO:-}" ]; then
+    export GOTOOLCHAIN='local'
+  else
+    GOTOOLCHAIN="go${GO_VERSION}"
+    export GOTOOLCHAIN
+  fi
+}
+
+determine_go_version

scripts/verify_go_versions.sh

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+
+# This script verifies that the value of the toolchain directive in the
+# go.mod files always match that of the .go-version file to ensure that
+# we accidentally don't test and release with differing versions of Go.
+
+set -euo pipefail
+
+source ./scripts/test_lib.sh
+
+target_go_version="${target_go_version:-"$(cat "${ETCD_ROOT_DIR}/.go-version")"}"
+log_info "expected go toolchain directive: go${target_go_version}"
+log_info
+
+toolchain_out_of_sync="false"
+go_line_violation="false"
+
+# verify_go_versions takes a go.mod filepath as an argument
+# and checks if:
+#  (1) go directive <= version in .go-version
+#  (2) toolchain directive == version in .go-version
+function verify_go_versions() {
+    # shellcheck disable=SC2086
+    toolchain_version="$(go mod edit -json $1 | jq -r .Toolchain)"
+    # shellcheck disable=SC2086
+    go_line_version="$(go mod edit -json $1 | jq -r .Go)"
+    if [[ "go${target_go_version}" != "${toolchain_version}" ]]; then
+        log_error "go toolchain directive out of sync for $1, got: ${toolchain_version}"
+        toolchain_out_of_sync="true"
+    fi
+    if ! printf '%s\n' "${go_line_version}" "${target_go_version}" | sort --check=silent --version-sort; then
+        log_error "go directive in $1 is greater than maximum allowed: go${target_go_version}"
+        go_line_violation="true"
+    fi
+}
+
+while read -r mod; do
+    verify_go_versions "${mod}";
+done < <(find . -name 'go.mod')
+
+if [[ "${toolchain_out_of_sync}" == "true" ]]; then
+    log_error
+    log_error "Please run scripts/sync_go_toolchain_directive.sh or update .go-version to rectify this error"
+fi
+
+if [[ "${go_line_violation}" == "true" ]]; then
+    log_error
+    log_error "Please update .go-version to rectify this error, any go directive should be <= .go-version"
+fi
+
+if [[ "${go_line_violation}" == "true" ]] || [[ "${toolchain_out_of_sync}" == "true" ]]; then
+    exit 1
+fi

server/go.mod

@@ -2,6 +2,8 @@ module go.etcd.io/etcd/server/v3
 
 go 1.21
 
+toolchain go1.21.10
+
 require (
 	github.com/coreos/go-semver v0.3.0
 	github.com/coreos/go-systemd/v22 v22.3.2

tests/go.mod

@@ -2,6 +2,8 @@ module go.etcd.io/etcd/tests/v3
 
 go 1.21
 
+toolchain go1.21.10
+
 replace (
 	go.etcd.io/etcd/api/v3 => ../api
 	go.etcd.io/etcd/client/pkg/v3 => ../client/pkg

tools/mod/go.mod

@@ -2,6 +2,8 @@ module go.etcd.io/etcd/tools/v3
 
 go 1.21
 
+toolchain go1.21.10
+
 require (
 	github.com/alexkohler/nakedret v1.0.0
 	github.com/chzchzchz/goword v0.0.0-20170907005317-a9744cb52b03