From 6477e4505478d4f935a93dce0b919a965840e5eb Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Fri, 21 May 2021 09:26:09 +0200 Subject: [PATCH] Squashed 'release-tools/' changes from 7b96bea..f325590 https://github.com/kubernetes-csi/csi-release-tools/commit/f325590 Merge https://github.com/kubernetes-csi/csi-release-tools/pull/149 from pohly/cluster-logs https://github.com/kubernetes-csi/csi-release-tools/commit/4b03b30 Merge https://github.com/kubernetes-csi/csi-release-tools/pull/155 from pohly/owners https://github.com/kubernetes-csi/csi-release-tools/commit/a6453c8 owners: introduce aliases https://github.com/kubernetes-csi/csi-release-tools/commit/ad83def Merge https://github.com/kubernetes-csi/csi-release-tools/pull/153 from pohly/fix-image-builds https://github.com/kubernetes-csi/csi-release-tools/commit/5561780 build.make: fix image publishng https://github.com/kubernetes-csi/csi-release-tools/commit/29bd39b Merge https://github.com/kubernetes-csi/csi-release-tools/pull/152 from pohly/bump-csi-test https://github.com/kubernetes-csi/csi-release-tools/commit/bc42793 prow.sh: use csi-test v4.2.0 https://github.com/kubernetes-csi/csi-release-tools/commit/b546baa Merge https://github.com/kubernetes-csi/csi-release-tools/pull/150 from mauriciopoppe/windows-multiarch-args https://github.com/kubernetes-csi/csi-release-tools/commit/bfbb6f3 add parameter base_image and addon_image to BUILD_PARAMETERS https://github.com/kubernetes-csi/csi-release-tools/commit/2d61d3b Merge https://github.com/kubernetes-csi/csi-release-tools/pull/151 from humblec/cm https://github.com/kubernetes-csi/csi-release-tools/commit/48e71f0 Replace `which` command ( non standard) with `command -v` builtin https://github.com/kubernetes-csi/csi-release-tools/commit/feb20e2 prow.sh: collect cluster logs git-subtree-dir: release-tools git-subtree-split: f3255906abf2e4134cb3db4ff79ddfb5ce1e91aa --- KUBERNETES_CSI_OWNERS_ALIASES | 44 +++++++++++++++++++++++++++++++ OWNERS | 7 ++--- OWNERS_ALIASES | 1 + build.make | 49 +++++++++++++++++++++++++++-------- cloudbuild.yaml | 2 +- prow.sh | 28 +++++++++++++------- verify-shellcheck.sh | 2 +- 7 files changed, 105 insertions(+), 28 deletions(-) create mode 100644 KUBERNETES_CSI_OWNERS_ALIASES create mode 120000 OWNERS_ALIASES diff --git a/KUBERNETES_CSI_OWNERS_ALIASES b/KUBERNETES_CSI_OWNERS_ALIASES new file mode 100644 index 00000000..6fb858aa --- /dev/null +++ b/KUBERNETES_CSI_OWNERS_ALIASES @@ -0,0 +1,44 @@ +# See the OWNERS docs: https://git.k8s.io/community/contributors/guide/owners.md + +aliases: + + # SIG-Storage chairs and leads should always have approval rights in all repos. + # Others may be added as needed here or in each repo. + kubernetes-csi-approvers: + - jsafrane + - msau42 + - saad-ali + - xing-yang + + # Reviewers are automatically assigned to new PRs. The following + # reviewers will be active in all repos. Other reviewers can be + # added in each repo. + # + # Reviewers are encouraged to set the "Busy" flag in their GitHub status + # when they are temporarily unable to review PRs. + kubernetes-csi-reviewers: + - andyzhangx + - chrishenzie + - ggriffiths + - gnufied + - j-griffith + - Jiawei0227 + - jingxu97 + - jsafrane + - pohly + - xing-yang + +# This documents who previously contributed to Kubernetes-CSI +# as approver. +emeritus_approver: +- lpabon +- sbezverk +- vladimirvivien + +# This documents who previously contributed to Kubernetes-CSI +# as reviewer. +emeritus_reviewer: +- lpabon +- saad-ali +- sbezverk +- vladimirvivien diff --git a/OWNERS b/OWNERS index 6d2f474e..1fb74587 100644 --- a/OWNERS +++ b/OWNERS @@ -1,11 +1,8 @@ # See the OWNERS docs: https://git.k8s.io/community/contributors/guide/owners.md approvers: -- saad-ali -- msau42 +- kubernetes-csi-approvers - pohly reviewers: -- saad-ali -- msau42 -- pohly +- kubernetes-csi-reviewers diff --git a/OWNERS_ALIASES b/OWNERS_ALIASES new file mode 120000 index 00000000..7ec6c034 --- /dev/null +++ b/OWNERS_ALIASES @@ -0,0 +1 @@ +KUBERNETES_CSI_OWNERS_ALIASES \ No newline at end of file diff --git a/build.make b/build.make index 1faaf3b9..0ed51e80 100644 --- a/build.make +++ b/build.make @@ -12,6 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +# force the usage of /bin/bash instead of /bin/sh +SHELL := /bin/bash + .PHONY: build-% build container-% container push-% push clean test # A space-separated list of all commands in the repository, must be @@ -63,26 +66,35 @@ endif # Specific packages can be excluded from each of the tests below by setting the *_FILTER_CMD variables # to something like "| grep -v 'github.com/kubernetes-csi/project/pkg/foobar'". See usage below. -# BUILD_PLATFORMS contains a set of triplets, +# BUILD_PLATFORMS contains a set of tuples [os arch suffix base_image addon_image] # separated by semicolon. An empty variable or empty entry (= just a # semicolon) builds for the default platform of the current Go # toolchain. BUILD_PLATFORMS = # Add go ldflags using LDFLAGS at the time of compilation. -IMPORTPATH_LDFLAGS = -X main.version=$(REV) +IMPORTPATH_LDFLAGS = -X main.version=$(REV) EXT_LDFLAGS = -extldflags "-static" -LDFLAGS = +LDFLAGS = FULL_LDFLAGS = $(LDFLAGS) $(IMPORTPATH_LDFLAGS) $(EXT_LDFLAGS) # This builds each command (= the sub-directories of ./cmd) for the target platform(s) # defined by BUILD_PLATFORMS. $(CMDS:%=build-%): build-%: check-go-version-go mkdir -p bin - echo '$(BUILD_PLATFORMS)' | tr ';' '\n' | while read -r os arch suffix; do \ + # os_arch_seen captures all of the $$os-$$arch seen for the current binary + # that we want to build, if we've seen an $$os-$$arch before it means that + # we don't need to build it again, this is done to avoid building + # the windows binary multiple times (see the default value of $$BUILD_PLATFORMS) + export os_arch_seen="" && echo '$(BUILD_PLATFORMS)' | tr ';' '\n' | while read -r os arch suffix base_image addon_image; do \ + os_arch_seen_pre=$${os_arch_seen%%$$os-$$arch*}; \ + if ! [ $${#os_arch_seen_pre} = $${#os_arch_seen} ]; then \ + continue; \ + fi; \ if ! (set -x; CGO_ENABLED=0 GOOS="$$os" GOARCH="$$arch" go build $(GOFLAGS_VENDOR) -a -ldflags '$(FULL_LDFLAGS)' -o "./bin/$*$$suffix" ./cmd/$*); then \ echo "Building $* for GOOS=$$os GOARCH=$$arch failed, see error(s) above."; \ exit 1; \ fi; \ + os_arch_seen+=";$$os-$$arch"; \ done $(CMDS:%=container-%): container-%: build-% @@ -131,30 +143,46 @@ DOCKER_BUILDX_CREATE_ARGS ?= # the tag for the resulting multiarch image. $(CMDS:%=push-multiarch-%): push-multiarch-%: check-pull-base-ref build-% set -ex; \ - DOCKER_CLI_EXPERIMENTAL=enabled; \ - export DOCKER_CLI_EXPERIMENTAL; \ + export DOCKER_CLI_EXPERIMENTAL=enabled; \ docker buildx create $(DOCKER_BUILDX_CREATE_ARGS) --use --name multiarchimage-buildertest; \ trap "docker buildx rm multiarchimage-buildertest" EXIT; \ dockerfile_linux=$$(if [ -e ./cmd/$*/Dockerfile ]; then echo ./cmd/$*/Dockerfile; else echo Dockerfile; fi); \ dockerfile_windows=$$(if [ -e ./cmd/$*/Dockerfile.Windows ]; then echo ./cmd/$*/Dockerfile.Windows; else echo Dockerfile.Windows; fi); \ if [ '$(BUILD_PLATFORMS)' ]; then build_platforms='$(BUILD_PLATFORMS)'; else build_platforms="linux amd64"; fi; \ if ! [ -f "$$dockerfile_windows" ]; then \ - build_platforms="$$(echo "$$build_platforms" | sed -e 's/windows *[^ ]* *.exe//g' -e 's/; *;/;/g')"; \ + build_platforms="$$(echo "$$build_platforms" | sed -e 's/windows *[^ ]* *.exe *[^ ]* *[^ ]*//g' -e 's/; *;/;/g' -e 's/;[ ]*$$//')"; \ fi; \ pushMultiArch () { \ tag=$$1; \ - echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix; do \ + echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix base_image addon_image; do \ + escaped_base_image=$${base_image/:/-}; \ + if ! [ -z $$escaped_base_image ]; then escaped_base_image+="-"; fi; \ docker buildx build --push \ - --tag $(IMAGE_NAME):$$arch-$$os-$$tag \ + --tag $(IMAGE_NAME):$$arch-$$os-$$escaped_base_image$$tag \ --platform=$$os/$$arch \ --file $$(eval echo \$${dockerfile_$$os}) \ --build-arg binary=./bin/$*$$suffix \ --build-arg ARCH=$$arch \ + --build-arg BASE_IMAGE=$$base_image \ + --build-arg ADDON_IMAGE=$$addon_image \ --label revision=$(REV) \ .; \ done; \ - images=$$(echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix; do echo $(IMAGE_NAME):$$arch-$$os-$$tag; done); \ + images=$$(echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix base_image addon_image; do \ + escaped_base_image=$${base_image/:/-}; \ + if ! [ -z $$escaped_base_image ]; then escaped_base_image+="-"; fi; \ + echo $(IMAGE_NAME):$$arch-$$os-$$escaped_base_image$$tag; \ + done); \ docker manifest create --amend $(IMAGE_NAME):$$tag $$images; \ + echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix base_image addon_image; do \ + if [ $$os = "windows" ]; then \ + escaped_base_image=$${base_image/:/-}; \ + if ! [ -z $$escaped_base_image ]; then escaped_base_image+="-"; fi; \ + image=$(IMAGE_NAME):$$arch-$$os-$$escaped_base_image$$tag; \ + os_version=$$(docker manifest inspect mcr.microsoft.com/windows/$${base_image} | grep "os.version" | head -n 1 | awk '{print $$2}' | sed -e 's/"//g') || true; \ + docker manifest annotate --os-version $$os_version $(IMAGE_NAME):$$tag $$image; \ + fi; \ + done; \ docker manifest push -p $(IMAGE_NAME):$$tag; \ }; \ if [ $(PULL_BASE_REF) = "master" ]; then \ @@ -288,4 +316,3 @@ test-spelling: test-boilerplate: @ echo; echo "### $@:" @ ./release-tools/verify-boilerplate.sh "$(pwd)" - diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 1e02ba6c..b39a6db3 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -27,7 +27,7 @@ steps: # The image must contain bash and curl. Ideally it should also contain # the desired version of Go (currently defined in release-tools/prow.sh), # but that just speeds up the build and is not required. - - name: 'gcr.io/k8s-testimages/gcb-docker-gcloud:v20200421-a2bf5f8' + - name: 'gcr.io/k8s-testimages/gcb-docker-gcloud:v20210331-c732583' entrypoint: ./.cloudbuild.sh env: - GIT_TAG=${_GIT_TAG} diff --git a/prow.sh b/prow.sh index 952b0513..55f2a91c 100755 --- a/prow.sh +++ b/prow.sh @@ -77,7 +77,10 @@ version_to_git () { esac } -configvar CSI_PROW_BUILD_PLATFORMS "linux amd64; windows amd64 .exe; linux ppc64le -ppc64le; linux s390x -s390x; linux arm64 -arm64" "Go target platforms (= GOOS + GOARCH) and file suffix of the resulting binaries" +# the list of windows versions was matched from: +# - https://hub.docker.com/_/microsoft-windows-nanoserver +# - https://hub.docker.com/_/microsoft-windows-servercore +configvar CSI_PROW_BUILD_PLATFORMS "linux amd64; linux ppc64le -ppc64le; linux s390x -s390x; linux arm64 -arm64; windows amd64 .exe nanoserver:1809 servercore:ltsc2019; windows amd64 .exe nanoserver:1909 servercore:1909; windows amd64 .exe nanoserver:2004 servercore:2004; windows amd64 .exe nanoserver:20H2 servercore:20H2" "Go target platforms (= GOOS + GOARCH) and file suffix of the resulting binaries" # If we have a vendor directory, then use it. We must be careful to only # use this for "make" invocations inside the project's repo itself because @@ -236,7 +239,7 @@ configvar CSI_PROW_E2E_IMPORT_PATH "k8s.io/kubernetes" "E2E package" # of the cluster. The alternative would have been to (cross-)compile csi-sanity # and install it inside the cluster, which is not necessarily easier. configvar CSI_PROW_SANITY_REPO https://github.com/kubernetes-csi/csi-test "csi-test repo" -configvar CSI_PROW_SANITY_VERSION v4.0.2 "csi-test version" # v4.0.2 +configvar CSI_PROW_SANITY_VERSION v4.2.0 "csi-test version" configvar CSI_PROW_SANITY_PACKAGE_PATH github.com/kubernetes-csi/csi-test "csi-test package" configvar CSI_PROW_SANITY_SERVICE "hostpath-service" "Kubernetes TCP service name that exposes csi.sock" configvar CSI_PROW_SANITY_POD "csi-hostpathplugin-0" "Kubernetes pod with CSI driver" @@ -632,11 +635,16 @@ EOF # Deletes kind cluster inside a prow job delete_cluster_inside_prow_job() { + local name="$1" + # Inside a real Prow job it is better to clean up at runtime # instead of leaving that to the Prow job cleanup code # because the later sometimes times out (https://github.com/kubernetes-csi/csi-release-tools/issues/24#issuecomment-554765872). + # + # This is also a good time to collect logs. if [ "$JOB_NAME" ]; then if kind get clusters | grep -q csi-prow; then + run kind export logs --name=csi-prow "${ARTIFACTS}/cluster-logs/$name" run kind delete cluster --name=csi-prow || die "kind delete failed" fi unset KUBECONFIG @@ -723,7 +731,7 @@ install_csi_driver () { fi } -# Installs all nessesary snapshotter CRDs +# Installs all nessesary snapshotter CRDs install_snapshot_crds() { # Wait until volumesnapshot CRDs are in place. CRD_BASE_DIR="https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/${CSI_SNAPSHOTTER_VERSION}/client/config/crd" @@ -770,7 +778,7 @@ install_snapshot_controller() { fi echo "$(date +%H:%M:%S)" "waiting for snapshot RBAC setup complete, attempt #$cnt" cnt=$((cnt + 1)) - sleep 10 + sleep 10 done SNAPSHOT_CONTROLLER_YAML="${CONTROLLER_DIR}/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml" @@ -839,7 +847,7 @@ install_snapshot_controller() { fi echo "$(date +%H:%M:%S)" "waiting for snapshot controller deployment to complete, attempt #$cnt" cnt=$((cnt + 1)) - sleep 10 + sleep 10 done } @@ -979,7 +987,7 @@ run_sanity () ( kubectl exec "${CSI_PROW_SANITY_POD}" -c "${CSI_PROW_SANITY_CONTAINER}" -- mkdir "\$@" && echo "\$@" EOF # Using "rm -rf" as fallback for "rmdir" is a workaround for: - # Node Service + # Node Service # should work # /nvme/gopath.tmp/src/github.com/kubernetes-csi/csi-test/pkg/sanity/node.go:624 # STEP: reusing connection to CSI driver at dns:///172.17.0.2:30896 @@ -1143,7 +1151,7 @@ make_test_to_junit () { # version_gt 1.3.1 v1.2.0 (returns true) # version_gt 1.1.1 release-1.2.0 (returns false) # version_gt 1.2.0 1.2.2 (returns false) -function version_gt() { +function version_gt() { versions=$(for ver in "$@"; do ver=${ver#release-}; ver=${ver#kubernetes-}; echo "${ver#v}"; done) greaterVersion=${1#"release-"}; greaterVersion=${greaterVersion#"kubernetes-"}; @@ -1206,7 +1214,7 @@ main () { if [ "$rbac_file_path" == "" ]; then rbac_file_path="$(pwd)/deploy/kubernetes/rbac.yaml" fi - + if [ -e "$rbac_file_path" ]; then # This is one of those components which has its own RBAC rules (like external-provisioner). # We are testing a locally built image and also want to test with the the current, @@ -1271,7 +1279,7 @@ main () { fi fi fi - delete_cluster_inside_prow_job + delete_cluster_inside_prow_job non-alpha fi if tests_need_alpha_cluster && [ "${CSI_PROW_E2E_ALPHA_GATES}" ]; then @@ -1306,7 +1314,7 @@ main () { fi fi fi - delete_cluster_inside_prow_job + delete_cluster_inside_prow_job alpha fi fi diff --git a/verify-shellcheck.sh b/verify-shellcheck.sh index fd28021a..664b919a 100755 --- a/verify-shellcheck.sh +++ b/verify-shellcheck.sh @@ -84,7 +84,7 @@ done < <(find . -name "*.sh" \ # detect if the host machine has the required shellcheck version installed # if so, we will use that instead. HAVE_SHELLCHECK=false -if which shellcheck &>/dev/null; then +if command -v shellcheck &>/dev/null; then detected_version="$(shellcheck --version | grep 'version: .*')" if [[ "${detected_version}" = "version: ${SHELLCHECK_VERSION}" ]]; then HAVE_SHELLCHECK=true