Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

master: update release-tools #64

Merged
merged 2 commits into from
Oct 28, 2020
Merged

master: update release-tools #64

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
bde2250
Squashed 'release-tools/' changes from 3041b8a..4aff857
pohly Oct 27, 2020
bb4918c
Merge commit 'bde2250b70b8f7e061a5003618b9dc02c62763ca' into prow-upd…
pohly Oct 27, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion release-tools/build.make
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,12 +69,17 @@ endif
# toolchain.
BUILD_PLATFORMS =

# Add go ldflags using LDFLAGS at the time of compilation.
IMPORTPATH_LDFLAGS = -X main.version=$(REV)
EXT_LDFLAGS = -extldflags "-static"
LDFLAGS =
FULL_LDFLAGS = $(LDFLAGS) $(IMPORTPATH_LDFLAGS) $(EXT_LDFLAGS)
# This builds each command (= the sub-directories of ./cmd) for the target platform(s)
# defined by BUILD_PLATFORMS.
$(CMDS:%=build-%): build-%: check-go-version-go
mkdir -p bin
echo '$(BUILD_PLATFORMS)' | tr ';' '\n' | while read -r os arch suffix; do \
if ! (set -x; CGO_ENABLED=0 GOOS="$$os" GOARCH="$$arch" go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o "./bin/$*$$suffix" ./cmd/$*); then \
if ! (set -x; CGO_ENABLED=0 GOOS="$$os" GOARCH="$$arch" go build $(GOFLAGS_VENDOR) -a -ldflags '$(FULL_LDFLAGS)' -o "./bin/$*$$suffix" ./cmd/$*); then \
echo "Building $* for GOOS=$$os GOARCH=$$arch failed, see error(s) above."; \
exit 1; \
fi; \
Expand Down
200 changes: 81 additions & 119 deletions release-tools/prow.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,9 +33,9 @@
# The expected environment is:
# - $GOPATH/src/<import path> for the repository that is to be tested,
# with PR branch merged (when testing a PR)
# - optional: bazel installed (when testing against Kubernetes master),
# must be recent enough for Kubernetes master
# - running on linux-amd64
# - bazel installed (when testing against Kubernetes master), must be recent
# enough for Kubernetes master
# - kind (https://github.com/kubernetes-sigs/kind) installed
# - optional: Go already installed

Expand All @@ -52,26 +52,6 @@ configvar () {
eval echo "\$3:" "$1=\${$1}"
}

# Takes the minor version of $CSI_PROW_KUBERNETES_VERSION and overrides it to
# $1 if they are equal minor versions. Ignores versions that begin with
# "release-".
override_k8s_version () {
local current_minor_version
local override_minor_version

# Ignore: See if you can use ${variable//search/replace} instead.
# shellcheck disable=SC2001
current_minor_version="$(echo "${CSI_PROW_KUBERNETES_VERSION}" | sed -e 's/\([0-9]*\)\.\([0-9]*\).*/\1\.\2/')"

# Ignore: See if you can use ${variable//search/replace} instead.
# shellcheck disable=SC2001
override_minor_version="$(echo "${1}" | sed -e 's/\([0-9]*\)\.\([0-9]*\).*/\1\.\2/')"
if [ "${current_minor_version}" == "${override_minor_version}" ]; then
CSI_PROW_KUBERNETES_VERSION="$1"
echo "Overriding CSI_PROW_KUBERNETES_VERSION with $1: $CSI_PROW_KUBERNETES_VERSION"
fi
}

# Prints the value of a variable + version suffix, falling back to variable + "LATEST".
get_versioned_variable () {
local var="$1"
Expand Down Expand Up @@ -107,9 +87,22 @@ configvar CSI_PROW_GO_VERSION_KIND "${CSI_PROW_GO_VERSION_BUILD}" "Go version fo
configvar CSI_PROW_GO_VERSION_GINKGO "${CSI_PROW_GO_VERSION_BUILD}" "Go version for building ginkgo" # depends on CSI_PROW_GINKGO_VERSION below

# kind version to use. If the pre-installed version is different,
# the desired version is downloaded from https://github.com/kubernetes-sigs/kind/releases/download/
# the desired version is downloaded from https://github.com/kubernetes-sigs/kind/releases
# (if available), otherwise it is built from source.
configvar CSI_PROW_KIND_VERSION "v0.6.0" "kind"
configvar CSI_PROW_KIND_VERSION "v0.9.0" "kind"

# kind images to use. Must match the kind version.
# The release notes of each kind release list the supported images.
configvar CSI_PROW_KIND_IMAGES "kindest/node:v1.19.1@sha256:98cf5288864662e37115e362b23e4369c8c4a408f99cbc06e58ac30ddc721600
kindest/node:v1.18.8@sha256:f4bcc97a0ad6e7abaf3f643d890add7efe6ee4ab90baeb374b4f41a4c95567eb
kindest/node:v1.17.11@sha256:5240a7a2c34bf241afb54ac05669f8a46661912eab05705d660971eeb12f6555
kindest/node:v1.16.15@sha256:a89c771f7de234e6547d43695c7ab047809ffc71a0c3b65aa54eda051c45ed20
kindest/node:v1.15.12@sha256:d9b939055c1e852fe3d86955ee24976cab46cba518abcb8b13ba70917e6547a6
kindest/node:v1.14.10@sha256:ce4355398a704fca68006f8a29f37aafb49f8fc2f64ede3ccd0d9198da910146
kindest/node:v1.13.12@sha256:1c1a48c2bfcbae4d5f4fa4310b5ed10756facad0b7a2ca93c7a4b5bae5db29f5" "kind images"

# Use kind node-image --type=bazel by default, but allow to disable that.
configvar CSI_PROW_USE_BAZEL true "use Bazel during 'kind node-image' invocation"

# ginkgo test runner version to use. If the pre-installed version is
# different, the desired version is built from source.
Expand All @@ -124,10 +117,13 @@ configvar CSI_PROW_GINKO_PARALLEL "-p" "Ginko parallelism parameter(s)"
configvar CSI_PROW_BUILD_JOB true "building code in repo enabled"

# Kubernetes version to test against. This must be a version number
# (like 1.13.3) for which there is a pre-built kind image (see
# https://hub.docker.com/r/kindest/node/tags), "latest" (builds
# Kubernetes from the master branch) or "release-x.yy" (builds
# Kubernetes from a release branch).
# (like 1.13.3), "latest" (builds Kubernetes from the master branch)
# or "release-x.yy" (builds Kubernetes from a release branch).
#
# The patch version is only relevant for picking the E2E test suite
# that is used for testing. The script automatically picks
# the kind images for the major/minor version of Kubernetes
# that the kind release supports.
#
# This can also be a version that was not released yet at the time
# that the settings below were chose. The script will then
Expand All @@ -136,16 +132,6 @@ configvar CSI_PROW_BUILD_JOB true "building code in repo enabled"
# deprecating or changing the implementation of an alpha feature.
configvar CSI_PROW_KUBERNETES_VERSION 1.17.0 "Kubernetes"

# This is a hack to workaround the issue that each version
# of kind currently only supports specific patch versions of
# Kubernetes. We need to override CSI_PROW_KUBERNETES_VERSION
# passed in by our CI/pull jobs to the versions that
# kind v0.5.0 supports.
#
# If the version is prefixed with "release-", then nothing
# is overridden.
override_k8s_version "1.15.3"

# CSI_PROW_KUBERNETES_VERSION reduced to first two version numbers and
# with underscore (1_13 instead of 1.13.3) and in uppercase (LATEST
# instead of latest).
Expand Down Expand Up @@ -262,11 +248,16 @@ configvar CSI_PROW_DEP_VERSION v0.5.1 "golang dep version to be used for vendor
#
# Unknown or unsupported entries are ignored.
#
# Testing of alpha features is only supported for CSI_PROW_KUBERNETES_VERSION=latest
# because CSI_PROW_E2E_ALPHA and CSI_PROW_E2E_ALPHA_GATES are not set for
# older Kubernetes releases. The script supports that, it just isn't done because
# it is not needed and would cause additional maintenance effort.
#
# Sanity testing with csi-sanity only covers the CSI driver itself and
# thus only makes sense in repos which provide their own CSI
# driver. Repos can enable sanity testing by setting
# CSI_PROW_TESTS_SANITY=sanity.
configvar CSI_PROW_TESTS "unit parallel serial parallel-alpha serial-alpha sanity" "tests to run"
configvar CSI_PROW_TESTS "unit parallel serial $(if [ "${CSI_PROW_KUBERNETES_VERSION}" = "latest" ]; then echo parallel-alpha serial-alpha; fi) sanity" "tests to run"
tests_enabled () {
local t1 t2
# We want word-splitting here, so ignore: Quote to prevent word splitting, or split robustly with mapfile or read -a.
Expand Down Expand Up @@ -298,11 +289,7 @@ tests_need_alpha_cluster () {

# Regex for non-alpha, feature-tagged tests that should be run.
#
# Starting with 1.17, snapshots is beta, but the E2E tests still have the
# [Feature:] tag. They need to be explicitly enabled.
configvar CSI_PROW_E2E_FOCUS_1_15 '^' "non-alpha, feature-tagged tests for Kubernetes = 1.15" # no tests to run, match nothing
configvar CSI_PROW_E2E_FOCUS_1_16 '^' "non-alpha, feature-tagged tests for Kubernetes = 1.16" # no tests to run, match nothing
configvar CSI_PROW_E2E_FOCUS_LATEST '\[Feature:VolumeSnapshotDataSource\]' "non-alpha, feature-tagged tests for Kubernetes >= 1.17"
configvar CSI_PROW_E2E_FOCUS_LATEST '\[Feature:VolumeSnapshotDataSource\]' "non-alpha, feature-tagged tests for latest Kubernetes version"
configvar CSI_PROW_E2E_FOCUS "$(get_versioned_variable CSI_PROW_E2E_FOCUS "${csi_prow_kubernetes_version_suffix}")" "non-alpha, feature-tagged tests"

# Serial vs. parallel is always determined by these regular expressions.
Expand All @@ -324,7 +311,7 @@ regex_join () {
# alpha in previous Kubernetes releases. This was considered too
# error prone. Therefore we use E2E tests that match the Kubernetes
# version that is getting tested.
configvar CSI_PROW_E2E_ALPHA_LATEST '\[Feature:' "alpha tests for Kubernetes >= 1.14" # there's no need to update this, adding a new case for CSI_PROW_E2E for a new Kubernetes is enough
configvar CSI_PROW_E2E_ALPHA_LATEST '\[Feature:' "alpha tests for latest Kubernetes version" # there's no need to update this, adding a new case for CSI_PROW_E2E for a new Kubernetes is enough
configvar CSI_PROW_E2E_ALPHA "$(get_versioned_variable CSI_PROW_E2E_ALPHA "${csi_prow_kubernetes_version_suffix}")" "alpha tests"

# After the parallel E2E test without alpha features, a test cluster
Expand All @@ -339,15 +326,11 @@ configvar CSI_PROW_E2E_ALPHA "$(get_versioned_variable CSI_PROW_E2E_ALPHA "${csi
# kubernetes-csi components must be updated, either by disabling
# the failing test for "latest" or by updating the test and not running
# it anymore for older releases.
configvar CSI_PROW_E2E_ALPHA_GATES_1_15 'VolumeSnapshotDataSource=true,ExpandCSIVolumes=true' "alpha feature gates for Kubernetes 1.15"
configvar CSI_PROW_E2E_ALPHA_GATES_1_16 'VolumeSnapshotDataSource=true' "alpha feature gates for Kubernetes 1.16"
# TODO: add new CSI_PROW_ALPHA_GATES_xxx entry for future Kubernetes releases and
# add new gates to CSI_PROW_E2E_ALPHA_GATES_LATEST.
configvar CSI_PROW_E2E_ALPHA_GATES_LATEST '' "alpha feature gates for latest Kubernetes"
configvar CSI_PROW_E2E_ALPHA_GATES_LATEST 'GenericEphemeralVolume=true,CSIStorageCapacity=true' "alpha feature gates for latest Kubernetes"
configvar CSI_PROW_E2E_ALPHA_GATES "$(get_versioned_variable CSI_PROW_E2E_ALPHA_GATES "${csi_prow_kubernetes_version_suffix}")" "alpha E2E feature gates"

# Which external-snapshotter tag to use for the snapshotter CRD and snapshot-controller deployment
configvar CSI_SNAPSHOTTER_VERSION 'v2.0.1' "external-snapshotter version tag"
configvar CSI_SNAPSHOTTER_VERSION 'v3.0.0' "external-snapshotter version tag"

# Some tests are known to be unusable in a KinD cluster. For example,
# stopping kubelet with "ssh <node IP> systemctl stop kubelet" simply
Expand Down Expand Up @@ -507,6 +490,22 @@ list_gates () (
done
)

# Turn feature gates in the format foo=true,bar=false into
# a YAML map with the corresponding API groups for use
# with https://kind.sigs.k8s.io/docs/user/configuration/#runtime-config
list_api_groups () (
set -f; IFS=','
# Ignore: Double quote to prevent globbing and word splitting.
# shellcheck disable=SC2086
set -- $1
while [ "$1" ]; do
if [ "$1" = 'CSIStorageCapacity=true' ]; then
echo ' "storage.k8s.io/v1alpha1": "true"'
fi
shift
done
)

go_version_for_kubernetes () (
local path="$1"
local version="$2"
Expand Down Expand Up @@ -536,77 +535,52 @@ start_cluster () {
run kind delete cluster --name=csi-prow || die "kind delete failed"
fi

# Build from source?
if [[ "${CSI_PROW_KUBERNETES_VERSION}" =~ ^release-|^latest$ ]]; then
# Try to find a pre-built kind image if asked to use a specific version.
if ! [[ "${CSI_PROW_KUBERNETES_VERSION}" =~ ^release-|^latest$ ]]; then
# Ignore: See if you can use ${variable//search/replace} instead.
# shellcheck disable=SC2001
major_minor=$(echo "${CSI_PROW_KUBERNETES_VERSION}" | sed -e 's/^\([0-9]*\)\.\([0-9]*\).*/\1.\2/')
for i in ${CSI_PROW_KIND_IMAGES}; do
if echo "$i" | grep -q "kindest/node:v${major_minor}"; then
image="$i"
break
fi
done
fi

# Need to build from source?
if ! [ "$image" ]; then
if ! ${csi_prow_kind_have_kubernetes}; then
local version="${CSI_PROW_KUBERNETES_VERSION}"
if [ "$version" = "latest" ]; then
version=master
fi
if ${CSI_PROW_USE_BAZEL}; then
type="bazel"
else
type="docker"
fi
git_clone_branch https://github.com/kubernetes/kubernetes "${CSI_PROW_WORK}/src/kubernetes" "$version" || die "checking out Kubernetes $version failed"

go_version="$(go_version_for_kubernetes "${CSI_PROW_WORK}/src/kubernetes" "$version")" || die "cannot proceed without knowing Go version for Kubernetes"
run_with_go "$go_version" kind build node-image --type bazel --image csiprow/node:latest --kube-root "${CSI_PROW_WORK}/src/kubernetes" || die "'kind build node-image' failed"
# Changing into the Kubernetes source code directory is a workaround for https://github.com/kubernetes-sigs/kind/issues/1910
(cd "${CSI_PROW_WORK}/src/kubernetes" && run_with_go "$go_version" kind build node-image --image csiprow/node:latest --type="$type" --kube-root "${CSI_PROW_WORK}/src/kubernetes") || die "'kind build node-image' failed"
csi_prow_kind_have_kubernetes=true
fi
image="csiprow/node:latest"
else
image="kindest/node:v${CSI_PROW_KUBERNETES_VERSION}"
fi
cat >"${CSI_PROW_WORK}/kind-config.yaml" <<EOF
kind: Cluster
apiVersion: kind.sigs.k8s.io/v1alpha3
apiVersion: kind.x-k8s.io/v1alpha4
nodes:
- role: control-plane
- role: worker
- role: worker
EOF

# kubeadm has API dependencies between apiVersion and Kubernetes version
# 1.15+ requires kubeadm.k8s.io/v1beta2
# We only run alpha tests against master so we don't need to maintain
# different patches for different Kubernetes releases.
if [[ -n "$gates" ]]; then
cat >>"${CSI_PROW_WORK}/kind-config.yaml" <<EOF
kubeadmConfigPatches:
- |
apiVersion: kubeadm.k8s.io/v1beta2
kind: ClusterConfiguration
metadata:
name: config
apiServer:
extraArgs:
"feature-gates": "$gates"
controllerManager:
extraArgs:
"feature-gates": "$gates"
scheduler:
extraArgs:
"feature-gates": "$gates"
- |
apiVersion: kubeadm.k8s.io/v1beta2
kind: InitConfiguration
metadata:
name: config
nodeRegistration:
kubeletExtraArgs:
"feature-gates": "$gates"
- |
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
metadata:
name: config
featureGates:
$(list_gates "$gates")
- |
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
metadata:
name: config
featureGates:
featureGates:
$(list_gates "$gates")
runtimeConfig:
$(list_api_groups "$gates")
EOF
fi

info "kind-config.yaml:"
cat "${CSI_PROW_WORK}/kind-config.yaml"
Expand Down Expand Up @@ -718,7 +692,7 @@ install_csi_driver () {
# Installs all nessesary snapshotter CRDs
install_snapshot_crds() {
# Wait until volumesnapshot CRDs are in place.
CRD_BASE_DIR="https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/${CSI_SNAPSHOTTER_VERSION}/config/crd"
CRD_BASE_DIR="https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/${CSI_SNAPSHOTTER_VERSION}/client/config/crd"
kubectl apply -f "${CRD_BASE_DIR}/snapshot.storage.k8s.io_volumesnapshotclasses.yaml" --validate=false
kubectl apply -f "${CRD_BASE_DIR}/snapshot.storage.k8s.io_volumesnapshots.yaml" --validate=false
kubectl apply -f "${CRD_BASE_DIR}/snapshot.storage.k8s.io_volumesnapshotcontents.yaml" --validate=false
Expand Down Expand Up @@ -1101,14 +1075,8 @@ main () {
start_cluster || die "starting the non-alpha cluster failed"

# Install necessary snapshot CRDs and snapshot controller
# For Kubernetes 1.17+, we will install the CRDs and snapshot controller.
if version_gt "${CSI_PROW_KUBERNETES_VERSION}" "1.16.255" || "${CSI_PROW_KUBERNETES_VERSION}" == "latest"; then
info "Version ${CSI_PROW_KUBERNETES_VERSION}, installing CRDs and snapshot controller"
install_snapshot_crds
install_snapshot_controller
else
info "Version ${CSI_PROW_KUBERNETES_VERSION}, skipping CRDs and snapshot controller"
fi
install_snapshot_crds
install_snapshot_controller

# Installing the driver might be disabled.
if ${CSI_PROW_DRIVER_INSTALL} "$images"; then
Expand Down Expand Up @@ -1158,14 +1126,8 @@ main () {
start_cluster "${CSI_PROW_E2E_ALPHA_GATES}" || die "starting alpha cluster failed"

# Install necessary snapshot CRDs and snapshot controller
# For Kubernetes 1.17+, we will install the CRDs and snapshot controller.
if version_gt "${CSI_PROW_KUBERNETES_VERSION}" "1.16.255" || "${CSI_PROW_KUBERNETES_VERSION}" == "latest"; then
info "Version ${CSI_PROW_KUBERNETES_VERSION}, installing CRDs and snapshot controller"
install_snapshot_crds
install_snapshot_controller
else
info "Version ${CSI_PROW_KUBERNETES_VERSION}, skipping CRDs and snapshot controller"
fi
install_snapshot_crds
install_snapshot_controller

# Installing the driver might be disabled.
if ${CSI_PROW_DRIVER_INSTALL} "$images"; then
Expand Down