Update release tools

release-tools/SIDECAR_RELEASE_PROCESS.md

@@ -84,7 +84,7 @@ naming convention `<hostpath-deployment-version>-on-<kubernetes-version>`.
     1. Compare the generated output to the new commits for the release to check if
        any notable change missed a release note.
     1. Reword release notes as needed, ideally in the original PRs so that the
-       release notes can be regnerated. Make sure to check notes for breaking
+       release notes can be regenerated. Make sure to check notes for breaking
        changes and deprecations.
     1. If release is a new major/minor version, create a new `CHANGELOG-<major>.<minor>.md`
        file.

release-tools/build.make

@@ -45,9 +45,10 @@ REV=$(shell git describe --long --tags --match='v*' --dirty 2>/dev/null || git r
 # Determined dynamically.
 IMAGE_TAGS=
 
-# A "canary" image gets built if the current commit is the head of the remote "master" branch.
+# A "canary" image gets built if the current commit is the head of the remote "master" or "main" branch.
 # That branch does not exist when building some other branch in TravisCI.
 IMAGE_TAGS+=$(shell if [ "$$(git rev-list -n1 HEAD)" = "$$(git rev-list -n1 origin/master 2>/dev/null)" ]; then echo "canary"; fi)
+IMAGE_TAGS+=$(shell if [ "$$(git rev-list -n1 HEAD)" = "$$(git rev-list -n1 origin/main 2>/dev/null)" ]; then echo "canary"; fi)
 
 # A "X.Y.Z-canary" image gets built if the current commit is the head of a "origin/release-X.Y.Z" branch.
 # The actual suffix does not matter, only the "release-" prefix is checked.
@@ -62,9 +63,9 @@ IMAGE_NAME=$(REGISTRY_NAME)/$*
 
 ifdef V
 # Adding "-alsologtostderr" assumes that all test binaries contain glog. This is not guaranteed.
-TESTARGS = -v -args -alsologtostderr -v 5
+TESTARGS = -race -v -args -alsologtostderr -v 5
 else
-TESTARGS =
+TESTARGS = -race
 endif
 
 # Specific packages can be excluded from each of the tests below by setting the *_FILTER_CMD variables
@@ -143,7 +144,7 @@ DOCKER_BUILDX_CREATE_ARGS ?=
 # Windows binaries can be built before adding a Dockerfile for it.
 #
 # BUILD_PLATFORMS determines which individual images are included in the multiarch image.
-# PULL_BASE_REF must be set to 'master', 'release-x.y', or a tag name, and determines
+# PULL_BASE_REF must be set to 'master', 'main', 'release-x.y', or a tag name, and determines
 # the tag for the resulting multiarch image.
 $(CMDS:%=push-multiarch-%): push-multiarch-%: check-pull-base-ref build-%
 	set -ex; \
@@ -191,7 +192,7 @@ $(CMDS:%=push-multiarch-%): push-multiarch-%: check-pull-base-ref build-%
 		done; \
 		docker manifest push -p $(IMAGE_NAME):$$tag; \
 	}; \
-	if [ $(PULL_BASE_REF) = "master" ]; then \
+	if [ $(PULL_BASE_REF) = "master" ] || [ $(PULL_BASE_REF) = "main" ]; then \
 			: "creating or overwriting canary image"; \
 			pushMultiArch canary; \
 	elif echo $(PULL_BASE_REF) | grep -q -e 'release-*' ; then \
@@ -209,7 +210,7 @@ $(CMDS:%=push-multiarch-%): push-multiarch-%: check-pull-base-ref build-%
 .PHONY: check-pull-base-ref
 check-pull-base-ref:
 	if ! [ "$(PULL_BASE_REF)" ]; then \
-		echo >&2 "ERROR: PULL_BASE_REF must be set to 'master', 'release-x.y', or a tag name."; \
+		echo >&2 "ERROR: PULL_BASE_REF must be set to 'master', 'main', 'release-x.y', or a tag name."; \
 		exit 1; \
 	fi
 

release-tools/cloudbuild.yaml

@@ -26,7 +26,7 @@ steps:
   # The image must contain bash and curl. Ideally it should also contain
   # the desired version of Go (currently defined in release-tools/prow.sh),
   # but that just speeds up the build and is not required.
-  - name: 'gcr.io/k8s-testimages/gcb-docker-gcloud:v20230623-56e06d7c18'
+  - name: 'gcr.io/k8s-testimages/gcb-docker-gcloud:v20240718-5ef92b5c36'
     entrypoint: ./.cloudbuild.sh
     env:
     - GIT_TAG=${_GIT_TAG}

release-tools/generate-patch-release-notes.sh

@@ -23,13 +23,17 @@
 # CSI_RELEASE_TOKEN: Github token needed for generating release notes
 # GITHUB_USER: Github username to create PRs with
 #
+# Required tools:
+# - gh
+# - release-notes (https://github.com/kubernetes/release/blob/master/cmd/release-notes/README.md)
+# 
 # Instructions:
-# 1. Login with "gh auth login"
-# 2. Copy this script to the kubernetes-csi directory (one directory above the
-# repos)
-# 3. Update the repos and versions in the $releases array
-# 4. Set environment variables
-# 5. Run script from the kubernetes-csi directory
+# 1. Install the required tools
+# 2. Login with "gh auth login"
+# 3. Copy this script to the kubernetes-csi directory (one directory above the repos)
+# 4. Update the repos and versions in the $releases array
+# 5. Set environment variables
+# 6. Run script from the kubernetes-csi directory
 #
 # Caveats:
 # - This script doesn't handle regenerating and updating existing PRs yet.

release-tools/prow.sh

@@ -86,7 +86,7 @@ configvar CSI_PROW_BUILD_PLATFORMS "linux amd64 amd64; linux ppc64le ppc64le -pp
 # which is disabled with GOFLAGS=-mod=vendor).
 configvar GOFLAGS_VENDOR "$( [ -d vendor ] && echo '-mod=vendor' )" "Go flags for using the vendor directory"
 
-configvar CSI_PROW_GO_VERSION_BUILD "1.22.3" "Go version for building the component" # depends on component's source code
+configvar CSI_PROW_GO_VERSION_BUILD "1.23.1" "Go version for building the component" # depends on component's source code
 configvar CSI_PROW_GO_VERSION_E2E "" "override Go version for building the Kubernetes E2E test suite" # normally doesn't need to be set, see install_e2e
 configvar CSI_PROW_GO_VERSION_SANITY "${CSI_PROW_GO_VERSION_BUILD}" "Go version for building the csi-sanity test suite" # depends on CSI_PROW_SANITY settings below
 configvar CSI_PROW_GO_VERSION_KIND "${CSI_PROW_GO_VERSION_BUILD}" "Go version for building 'kind'" # depends on CSI_PROW_KIND_VERSION below
@@ -199,7 +199,7 @@ kindest/node:v1.18.20@sha256:738cdc23ed4be6cc0b7ea277a2ebcc454c8373d7d8fb991a7fc
 # If the deployment script is called with CSI_PROW_TEST_DRIVER=<file name> as
 # environment variable, then it must write a suitable test driver configuration
 # into that file in addition to installing the driver.
-configvar CSI_PROW_DRIVER_VERSION "v1.12.0" "CSI driver version"
+configvar CSI_PROW_DRIVER_VERSION "v1.15.0" "CSI driver version"
 configvar CSI_PROW_DRIVER_REPO https://github.com/kubernetes-csi/csi-driver-host-path "CSI driver repo"
 configvar CSI_PROW_DEPLOYMENT "" "deployment"
 configvar CSI_PROW_DEPLOYMENT_SUFFIX "" "additional suffix in kubernetes-x.yy[suffix].yaml files"
@@ -231,16 +231,19 @@ configvar CSI_PROW_E2E_VERSION "$(version_to_git "${CSI_PROW_KUBERNETES_VERSION}
 configvar CSI_PROW_E2E_REPO "https://github.com/kubernetes/kubernetes" "E2E repo"
 configvar CSI_PROW_E2E_IMPORT_PATH "k8s.io/kubernetes" "E2E package"
 
-# Local path for e2e tests. Set to "none" to disable.
-configvar CSI_PROW_SIDECAR_E2E_IMPORT_PATH "none" "CSI Sidecar E2E package"
+# Local path & package path for e2e tests. Set to "none" to disable.
+# When using versioned go modules, the import path is the module path whereas the path
+# should not contain the version and be the directory where the module is checked out.
+configvar CSI_PROW_SIDECAR_E2E_IMPORT_PATH "none" "CSI Sidecar E2E package (go import path)"
+configvar CSI_PROW_SIDECAR_E2E_PATH "${CSI_PROW_SIDECAR_E2E_IMPORT_PATH}" "CSI Sidecar E2E path (directory)"
 
 # csi-sanity testing from the csi-test repo can be run against the installed
 # CSI driver. For this to work, deploying the driver must expose the Unix domain
 # csi.sock as a TCP service for use by the csi-sanity command, which runs outside
 # of the cluster. The alternative would have been to (cross-)compile csi-sanity
 # and install it inside the cluster, which is not necessarily easier.
 configvar CSI_PROW_SANITY_REPO https://github.com/kubernetes-csi/csi-test "csi-test repo"
-configvar CSI_PROW_SANITY_VERSION v5.2.0 "csi-test version"
+configvar CSI_PROW_SANITY_VERSION v5.3.1 "csi-test version"
 configvar CSI_PROW_SANITY_PACKAGE_PATH github.com/kubernetes-csi/csi-test "csi-test package"
 configvar CSI_PROW_SANITY_SERVICE "hostpath-service" "Kubernetes TCP service name that exposes csi.sock"
 configvar CSI_PROW_SANITY_POD "csi-hostpathplugin-0" "Kubernetes pod with CSI driver"
@@ -422,23 +425,24 @@ die () {
     exit 1
 }
 
-# Ensure that PATH has the desired version of the Go tools, then run command given as argument.
+# Ensure we use the desired version of the Go tools, then run command given as argument.
 # Empty parameter uses the already installed Go. In Prow, that version is kept up-to-date by
 # bumping the container image regularly.
 run_with_go () {
     local version
     version="$1"
     shift
 
-    if ! [ "$version" ] || go version 2>/dev/null | grep -q "go$version"; then
-        run "$@"
-    else
-        if ! [ -d "${CSI_PROW_WORK}/go-$version" ];  then
-            run curl --fail --location "https://dl.google.com/go/go$version.linux-amd64.tar.gz" | tar -C "${CSI_PROW_WORK}" -zxf - || die "installation of Go $version failed"
-            mv "${CSI_PROW_WORK}/go" "${CSI_PROW_WORK}/go-$version"
+    if [ "$version" ]; then
+        version=go$version
+        if [ "$(GOTOOLCHAIN=$version go version | cut -d' ' -f3)" != "$version" ]; then
+            die "Please install Go 1.21+"
         fi
-        PATH="${CSI_PROW_WORK}/go-$version/bin:$PATH" run "$@"
+    else
+        version=local
     fi
+    # Set GOMODCACHE to make sure Kubernetes does not need to download again.
+    GOTOOLCHAIN=$version GOMODCACHE="$(go env GOMODCACHE)" run "$@"
 }
 
 # Ensure that we have the desired version of kind.
@@ -621,7 +625,7 @@ start_cluster () {
             go_version="$(go_version_for_kubernetes "${CSI_PROW_WORK}/src/kubernetes" "$version")" || die "cannot proceed without knowing Go version for Kubernetes"
             # Changing into the Kubernetes source code directory is a workaround for https://github.com/kubernetes-sigs/kind/issues/1910
             # shellcheck disable=SC2046
-            (cd "${CSI_PROW_WORK}/src/kubernetes" && run_with_go "$go_version" kind build node-image --image csiprow/node:latest --kube-root "${CSI_PROW_WORK}/src/kubernetes") || die "'kind build node-image' failed"
+            (cd "${CSI_PROW_WORK}/src/kubernetes" && run_with_go "$go_version" kind build node-image "${CSI_PROW_WORK}/src/kubernetes" --image csiprow/node:latest) || die "'kind build node-image' failed"
             csi_prow_kind_have_kubernetes=true
         fi
         image="csiprow/node:latest"
@@ -1035,7 +1039,7 @@ run_e2e () (
     trap move_junit EXIT
 
     if [ "${name}" == "local" ]; then
-        cd "${GOPATH}/src/${CSI_PROW_SIDECAR_E2E_IMPORT_PATH}" &&
+        cd "${GOPATH}/src/${CSI_PROW_SIDECAR_E2E_PATH}" &&
         run_with_loggers env KUBECONFIG="$KUBECONFIG" KUBE_TEST_REPO_LIST="$(if [ -e "${CSI_PROW_WORK}/e2e-repo-list" ]; then echo "${CSI_PROW_WORK}/e2e-repo-list"; fi)" ginkgo --timeout="${CSI_PROW_GINKGO_TIMEOUT}" -v "$@" "${CSI_PROW_WORK}/e2e-local.test" -- -report-dir "${ARTIFACTS}" -report-prefix local
     else
         cd "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" &&

release-tools/pull-test.sh

@@ -20,6 +20,11 @@
 
 set -ex
 
+# Prow checks out repos with --filter=blob:none. This breaks
+# "git subtree pull" unless we enable fetching missing file content.
+GIT_NO_LAZY_FETCH=0
+export GIT_NO_LAZY_FETCH
+
 # It must be called inside the updated csi-release-tools repo.
 CSI_RELEASE_TOOLS_DIR="$(pwd)"