From 3fced5b112d17f17c1151118064c7937bdfac596 Mon Sep 17 00:00:00 2001 From: Stefan Bueringer Date: Fri, 26 Mar 2021 16:53:35 +0100 Subject: [PATCH] test: cleanup the way we patch our cluster templates for ci artifacts --- .../ci-artifacts-platform-kustomization.yaml | 286 ------------------ ...ster-template-external-cloud-provider.yaml | 11 + .../cluster-template.yaml | 12 - ...injection_script_control_plane.envsubst.sh | 115 +++++++ ...debian_injection_script_worker.envsubst.sh | 115 +++++++ .../data/kustomization.yaml.tpl | 20 ++ .../e2e/shared/kubernetesversions/template.go | 224 ++++++++++++++ test/e2e/shared/suite.go | 14 +- test/e2e/shared/tmp_template.go | 109 ------- 9 files changed, 489 insertions(+), 417 deletions(-) create mode 100644 test/e2e/shared/kubernetesversions/data/debian_injection_script_control_plane.envsubst.sh create mode 100644 test/e2e/shared/kubernetesversions/data/debian_injection_script_worker.envsubst.sh create mode 100644 test/e2e/shared/kubernetesversions/data/kustomization.yaml.tpl create mode 100644 test/e2e/shared/kubernetesversions/template.go delete mode 100644 test/e2e/shared/tmp_template.go diff --git a/test/e2e/data/ci-artifacts-platform-kustomization.yaml b/test/e2e/data/ci-artifacts-platform-kustomization.yaml index 797deadb6f..6cf5a8331a 100644 --- a/test/e2e/data/ci-artifacts-platform-kustomization.yaml +++ b/test/e2e/data/ci-artifacts-platform-kustomization.yaml @@ -7,149 +7,6 @@ metadata: spec: infrastructureTemplate: {} kubeadmConfigSpec: - files: - # We need to add these files again, because kustomize overrides the whole array - # See: https://github.com/kubernetes-sigs/kustomize/issues/2825 - - content: ${OPENSTACK_CLOUD_PROVIDER_CONF_B64} - encoding: base64 - owner: root - path: /etc/kubernetes/cloud.conf - permissions: "0600" - - content: ${OPENSTACK_CLOUD_CACERT_B64} - encoding: base64 - owner: root - path: /etc/certs/cacert - permissions: "0600" - - content: | - #!/bin/bash - - ## Please note that this file needs to be escaped for envsubst to function - - function retry { - attempt=0 - max_attempts=$${1} - interval=$${2} - shift; shift - until [[ $${attempt} -ge "$${max_attempts}" ]] ; do - attempt=$((attempt+1)) - set +e - eval "$*" && return || echo "failed $${attempt} times: $*" - set -e - sleep "$${interval}" - done - echo "error: reached max attempts at retry($*)" - return 1 - } - retry 10 5 "apt-get update" - - owner: root:root - path: /usr/local/bin/retry-apt.sh - permissions: "0750" - - content: | - #!/bin/bash - - # Copyright 2020 The Kubernetes Authors. - # - # Licensed under the Apache License, Version 2.0 (the "License"); - # you may not use this file except in compliance with the License. - # You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - - ## Please note that this file needs to be escaped for envsubst to function - - # shellcheck disable=SC1083,SC2034,SC2066,SC2193 - - set -o nounset - set -o pipefail - set -o errexit - - [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" - - USE_CI_ARTIFACTS=${USE_CI_ARTIFACTS:=false} - - if [ ! "${USE_CI_ARTIFACTS}" = true ]; then - echo "No CI Artifacts installation, exiting" - exit 0 - fi - - # This test installs release packages or binaries that are a result of the CI and release builds. - # It runs '... --version' commands to verify that the binaries are correctly installed - # and finally uninstalls the packages. - # For the release packages it tests all versions in the support skew. - LINE_SEPARATOR="*************************************************" - echo "$${LINE_SEPARATOR}" - - ## Clusterctl set variables - ## - # $${KUBERNETES_VERSION} will be replaced by clusterctl - KUBERNETES_VERSION=${KUBERNETES_VERSION} - ## - ## End clusterctl set variables - - if [[ "$${KUBERNETES_VERSION}" != "" ]]; then - CI_DIR=/tmp/k8s-ci - mkdir -p "$${CI_DIR}" - declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm") - declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler") - CONTAINER_EXT="tar" - echo "* testing CI version $${KUBERNETES_VERSION}" - # Check for semver - if [[ "$${KUBERNETES_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - CI_URL="https://storage.googleapis.com/kubernetes-release/release/$${KUBERNETES_VERSION}/bin/linux/amd64" - VERSION_WITHOUT_PREFIX="$${KUBERNETES_VERSION#v}" - DEBIAN_FRONTEND=noninteractive apt-get update - DEBIAN_FRONTEND=noninteractive apt-get install -y apt-transport-https ca-certificates curl - curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - - echo 'deb https://apt.kubernetes.io/ kubernetes-xenial main' >/etc/apt/sources.list.d/kubernetes.list - apt-get update - # replace . with \. - VERSION_REGEX="$${VERSION_WITHOUT_PREFIX//./\\.}" - PACKAGE_VERSION="$(apt-cache madison kubelet | grep "$${VERSION_REGEX}-" | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')" - for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do - echo "* installing package: $${CI_PACKAGE} $${PACKAGE_VERSION}" - DEBIAN_FRONTEND=noninteractive apt-get install -y "$${CI_PACKAGE}=$${PACKAGE_VERSION}" - done - else - CI_URL="https://storage.googleapis.com/kubernetes-release-dev/ci/$${KUBERNETES_VERSION}/bin/linux/amd64" - for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do - # Browser: https://console.cloud.google.com/storage/browser/kubernetes-release-dev?project=kubernetes-release-dev - # e.g.: https://storage.googleapis.com/kubernetes-release-dev/ci/v1.21.0-beta.1.378+cf3374e43491c5/bin/linux/amd64/kubectl - echo "* downloading binary: $${CI_URL}/$${CI_PACKAGE}" - wget "$${CI_URL}/$${CI_PACKAGE}" -O "$${CI_DIR}/$${CI_PACKAGE}" - chmod +x "$${CI_DIR}/$${CI_PACKAGE}" - mv "$${CI_DIR}/$${CI_PACKAGE}" "/usr/bin/$${CI_PACKAGE}" - done - systemctl restart kubelet - fi - for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do - # Browser: https://console.cloud.google.com/storage/browser/kubernetes-release?project=kubernetes-release - # e.g.: https://storage.googleapis.com/kubernetes-release/release/v1.20.4/bin/linux/amd64/kube-apiserver.tar - echo "* downloading package: $${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}" - wget "$${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}" -O "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}" - $${SUDO} ctr -n k8s.io images import "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}" || echo "* ignoring expected 'ctr images import' result" - $${SUDO} ctr -n k8s.io images tag "k8s.gcr.io/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "k8s.gcr.io/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}" - $${SUDO} ctr -n k8s.io images tag "k8s.gcr.io/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "gcr.io/kubernetes-ci-images/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}" - done - fi - echo "* checking binary versions" - echo "ctr version: " "$(ctr version)" - echo "kubeadm version: " "$(kubeadm version -o=short)" - echo "kubectl version: " "$(kubectl version --client=true --short=true)" - echo "kubelet version: " "$(kubelet --version)" - echo "$${LINE_SEPARATOR}" - owner: root:root - path: /usr/local/bin/ci-artifacts.sh - permissions: "0750" - preKubeadmCommands: - - /usr/local/bin/retry-apt.sh - - /usr/local/bin/ci-artifacts.sh users: - name: "capi" sudo: "ALL=(ALL) NOPASSWD:ALL" @@ -169,149 +26,6 @@ metadata: spec: template: spec: - files: - # We need to add these files again, because kustomize overrides the whole array - # See: https://github.com/kubernetes-sigs/kustomize/issues/2825 - - path: /etc/kubernetes/cloud.conf - owner: root - permissions: "0600" - content: ${OPENSTACK_CLOUD_PROVIDER_CONF_B64} - encoding: base64 - - path: /etc/certs/cacert - owner: root - permissions: "0600" - content: ${OPENSTACK_CLOUD_CACERT_B64} - encoding: base64 - - content: | - #!/bin/bash - - ## Please note that this file needs to be escaped for envsubst to function - - function retry { - attempt=0 - max_attempts=$${1} - interval=$${2} - shift; shift - until [[ $${attempt} -ge "$${max_attempts}" ]] ; do - attempt=$((attempt+1)) - set +e - eval "$*" && return || echo "failed $${attempt} times: $*" - set -e - sleep "$${interval}" - done - echo "error: reached max attempts at retry($*)" - return 1 - } - retry 10 5 "apt-get update" - - owner: root:root - path: /usr/local/bin/retry-apt.sh - permissions: "0750" - - content: | - #!/bin/bash - - # Copyright 2020 The Kubernetes Authors. - # - # Licensed under the Apache License, Version 2.0 (the "License"); - # you may not use this file except in compliance with the License. - # You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - - ## Please note that this file needs to be escaped for envsubst to function - - # shellcheck disable=SC1083,SC2034,SC2066,SC2193 - - set -o nounset - set -o pipefail - set -o errexit - - [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" - - USE_CI_ARTIFACTS=${USE_CI_ARTIFACTS:=false} - - if [ ! "${USE_CI_ARTIFACTS}" = true ]; then - echo "No CI Artifacts installation, exiting" - exit 0 - fi - - # This test installs release packages or binaries that are a result of the CI and release builds. - # It runs '... --version' commands to verify that the binaries are correctly installed - # and finally uninstalls the packages. - # For the release packages it tests all versions in the support skew. - LINE_SEPARATOR="*************************************************" - echo "$${LINE_SEPARATOR}" - - ## Clusterctl set variables - ## - # $${KUBERNETES_VERSION} will be replaced by clusterctl - KUBERNETES_VERSION=${KUBERNETES_VERSION} - ## - ## End clusterctl set variables - - if [[ "$${KUBERNETES_VERSION}" != "" ]]; then - CI_DIR=/tmp/k8s-ci - mkdir -p "$${CI_DIR}" - declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm") - declare -a CONTAINERS_TO_TEST=("kube-proxy") - CONTAINER_EXT="tar" - echo "* testing CI version $${KUBERNETES_VERSION}" - # Check for semver - if [[ "$${KUBERNETES_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - CI_URL="https://storage.googleapis.com/kubernetes-release/release/$${KUBERNETES_VERSION}/bin/linux/amd64" - VERSION_WITHOUT_PREFIX="$${KUBERNETES_VERSION#v}" - DEBIAN_FRONTEND=noninteractive apt-get update - DEBIAN_FRONTEND=noninteractive apt-get install -y apt-transport-https ca-certificates curl - curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - - echo 'deb https://apt.kubernetes.io/ kubernetes-xenial main' >/etc/apt/sources.list.d/kubernetes.list - apt-get update - # replace . with \. - VERSION_REGEX="$${VERSION_WITHOUT_PREFIX//./\\.}" - PACKAGE_VERSION="$(apt-cache madison kubelet | grep "$${VERSION_REGEX}-" | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')" - for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do - echo "* installing package: $${CI_PACKAGE} $${PACKAGE_VERSION}" - DEBIAN_FRONTEND=noninteractive apt-get install -y "$${CI_PACKAGE}=$${PACKAGE_VERSION}" - done - else - CI_URL="https://storage.googleapis.com/kubernetes-release-dev/ci/$${KUBERNETES_VERSION}/bin/linux/amd64" - for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do - # Browser: https://console.cloud.google.com/storage/browser/kubernetes-release-dev?project=kubernetes-release-dev - # e.g.: https://storage.googleapis.com/kubernetes-release-dev/ci/v1.21.0-beta.1.378+cf3374e43491c5/bin/linux/amd64/kubectl - echo "* downloading binary: $${CI_URL}/$${CI_PACKAGE}" - wget "$${CI_URL}/$${CI_PACKAGE}" -O "$${CI_DIR}/$${CI_PACKAGE}" - chmod +x "$${CI_DIR}/$${CI_PACKAGE}" - mv "$${CI_DIR}/$${CI_PACKAGE}" "/usr/bin/$${CI_PACKAGE}" - done - systemctl restart kubelet - fi - for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do - # Browser: https://console.cloud.google.com/storage/browser/kubernetes-release?project=kubernetes-release - # e.g.: https://storage.googleapis.com/kubernetes-release/release/v1.20.4/bin/linux/amd64/kube-apiserver.tar - echo "* downloading package: $${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}" - wget "$${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}" -O "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}" - $${SUDO} ctr -n k8s.io images import "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}" || echo "* ignoring expected 'ctr images import' result" - $${SUDO} ctr -n k8s.io images tag "k8s.gcr.io/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "k8s.gcr.io/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}" - $${SUDO} ctr -n k8s.io images tag "k8s.gcr.io/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "gcr.io/kubernetes-ci-images/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}" - done - fi - echo "* checking binary versions" - echo "ctr version: " "$(ctr version)" - echo "kubeadm version: " "$(kubeadm version -o=short)" - echo "kubectl version: " "$(kubectl version --client=true --short=true)" - echo "kubelet version: " "$(kubelet --version)" - echo "$${LINE_SEPARATOR}" - owner: root:root - path: /usr/local/bin/ci-artifacts.sh - permissions: "0750" - preKubeadmCommands: - - /usr/local/bin/retry-apt.sh - - /usr/local/bin/ci-artifacts.sh users: - name: "capi" sudo: "ALL=(ALL) NOPASSWD:ALL" diff --git a/test/e2e/data/infrastructure-openstack/cluster-template-external-cloud-provider.yaml b/test/e2e/data/infrastructure-openstack/cluster-template-external-cloud-provider.yaml index e439055d3d..af28062389 100644 --- a/test/e2e/data/infrastructure-openstack/cluster-template-external-cloud-provider.yaml +++ b/test/e2e/data/infrastructure-openstack/cluster-template-external-cloud-provider.yaml @@ -70,6 +70,17 @@ spec: name: '{{ local_hostname }}' kubeletExtraArgs: cloud-provider: external + files: + - path: /etc/kubernetes/cloud.conf + owner: root + permissions: "0600" + content: ${OPENSTACK_CLOUD_PROVIDER_CONF_B64} + encoding: base64 + - path: /etc/certs/cacert + owner: root + permissions: "0600" + content: ${OPENSTACK_CLOUD_CACERT_B64} + encoding: base64 version: "${KUBERNETES_VERSION}" --- apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4 diff --git a/test/e2e/data/infrastructure-openstack/cluster-template.yaml b/test/e2e/data/infrastructure-openstack/cluster-template.yaml index cf5d795457..77c0c8f9d8 100644 --- a/test/e2e/data/infrastructure-openstack/cluster-template.yaml +++ b/test/e2e/data/infrastructure-openstack/cluster-template.yaml @@ -98,12 +98,6 @@ spec: permissions: "0600" content: ${OPENSTACK_CLOUD_CACERT_B64} encoding: base64 - users: - - name: "capi" - sudo: "ALL=(ALL) NOPASSWD:ALL" - # user: capi, passwd: capi - passwd: "$6$rounds=4096$yKTFKL6RmN128$a7cGMiNjeTSd091s6QzZcUNrMTgm3HhML5rVmpDFlCfgD7scTW7ZHr0OChcXCaeiO/kbhdn0XzIzWk63nSqRH1" - lockPassword: false version: "${KUBERNETES_VERSION}" --- apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4 @@ -182,12 +176,6 @@ spec: kubeletExtraArgs: cloud-config: /etc/kubernetes/cloud.conf cloud-provider: openstack - users: - - name: "capi" - sudo: "ALL=(ALL) NOPASSWD:ALL" - # user: capi, passwd: capi - passwd: "$6$rounds=4096$yKTFKL6RmN128$a7cGMiNjeTSd091s6QzZcUNrMTgm3HhML5rVmpDFlCfgD7scTW7ZHr0OChcXCaeiO/kbhdn0XzIzWk63nSqRH1" - lockPassword: false --- apiVersion: v1 kind: Secret diff --git a/test/e2e/shared/kubernetesversions/data/debian_injection_script_control_plane.envsubst.sh b/test/e2e/shared/kubernetesversions/data/debian_injection_script_control_plane.envsubst.sh new file mode 100644 index 0000000000..d69566a8e4 --- /dev/null +++ b/test/e2e/shared/kubernetesversions/data/debian_injection_script_control_plane.envsubst.sh @@ -0,0 +1,115 @@ +#!/bin/bash + +# Copyright 2020 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## Please note that this file needs to be escaped for envsubst to function + +# shellcheck disable=SC1083,SC2034,SC2066,SC2193 + +set -o nounset +set -o pipefail +set -o errexit + +function retry { + attempt=0 + max_attempts=$${1} + interval=$${2} + shift; shift + until [[ $${attempt} -ge "$${max_attempts}" ]] ; do + attempt=$((attempt+1)) + set +e + eval "$*" && return || echo "failed $${attempt} times: $*" + set -e + sleep "$${interval}" + done + echo "error: reached max attempts at retry($*)" + return 1 +} + +[[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + +USE_CI_ARTIFACTS=${USE_CI_ARTIFACTS:=false} + +if [ ! "${USE_CI_ARTIFACTS}" = true ]; then + echo "No CI Artifacts installation, exiting" + exit 0 +fi + +# This test installs release packages or binaries that are a result of the CI and release builds. +# It runs '... --version' commands to verify that the binaries are correctly installed +# and finally uninstalls the packages. +# For the release packages it tests all versions in the support skew. +LINE_SEPARATOR="*************************************************" +echo "$${LINE_SEPARATOR}" + +## Clusterctl set variables +## +# $${KUBERNETES_VERSION} will be replaced by clusterctl +KUBERNETES_VERSION=${KUBERNETES_VERSION} +## +## End clusterctl set variables + +if [[ "$${KUBERNETES_VERSION}" != "" ]]; then + CI_DIR=/tmp/k8s-ci + mkdir -p "$${CI_DIR}" + declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm") + declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler") + CONTAINER_EXT="tar" + echo "* testing CI version $${KUBERNETES_VERSION}" + # Check for semver + if [[ "$${KUBERNETES_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + CI_URL="https://storage.googleapis.com/kubernetes-release/release/$${KUBERNETES_VERSION}/bin/linux/amd64" + VERSION_WITHOUT_PREFIX="$${KUBERNETES_VERSION#v}" + export DEBIAN_FRONTEND=noninteractive + retry 10 5 "apt-get update" + apt-get install -y apt-transport-https ca-certificates curl + curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - + echo 'deb https://apt.kubernetes.io/ kubernetes-xenial main' >/etc/apt/sources.list.d/kubernetes.list + apt-get update + # replace . with \. + VERSION_REGEX="$${VERSION_WITHOUT_PREFIX//./\\.}" + PACKAGE_VERSION="$(apt-cache madison kubelet | grep "$${VERSION_REGEX}-" | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')" + for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do + echo "* installing package: $${CI_PACKAGE} $${PACKAGE_VERSION}" + apt-get install -y "$${CI_PACKAGE}=$${PACKAGE_VERSION}" + done + else + CI_URL="https://storage.googleapis.com/kubernetes-release-dev/ci/$${KUBERNETES_VERSION}/bin/linux/amd64" + for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do + # Browser: https://console.cloud.google.com/storage/browser/kubernetes-release-dev?project=kubernetes-release-dev + # e.g.: https://storage.googleapis.com/kubernetes-release-dev/ci/v1.21.0-beta.1.378+cf3374e43491c5/bin/linux/amd64/kubectl + echo "* downloading binary: $${CI_URL}/$${CI_PACKAGE}" + wget "$${CI_URL}/$${CI_PACKAGE}" -O "$${CI_DIR}/$${CI_PACKAGE}" + chmod +x "$${CI_DIR}/$${CI_PACKAGE}" + mv "$${CI_DIR}/$${CI_PACKAGE}" "/usr/bin/$${CI_PACKAGE}" + done + systemctl restart kubelet + fi + for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do + # Browser: https://console.cloud.google.com/storage/browser/kubernetes-release?project=kubernetes-release + # e.g.: https://storage.googleapis.com/kubernetes-release/release/v1.20.4/bin/linux/amd64/kube-apiserver.tar + echo "* downloading package: $${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}" + wget "$${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}" -O "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}" + $${SUDO} ctr -n k8s.io images import "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}" || echo "* ignoring expected 'ctr images import' result" + $${SUDO} ctr -n k8s.io images tag "k8s.gcr.io/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "k8s.gcr.io/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}" + $${SUDO} ctr -n k8s.io images tag "k8s.gcr.io/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "gcr.io/kubernetes-ci-images/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}" + done +fi +echo "* checking binary versions" +echo "ctr version: " "$(ctr version)" +echo "kubeadm version: " "$(kubeadm version -o=short)" +echo "kubectl version: " "$(kubectl version --client=true --short=true)" +echo "kubelet version: " "$(kubelet --version)" +echo "$${LINE_SEPARATOR}" diff --git a/test/e2e/shared/kubernetesversions/data/debian_injection_script_worker.envsubst.sh b/test/e2e/shared/kubernetesversions/data/debian_injection_script_worker.envsubst.sh new file mode 100644 index 0000000000..e71e585c76 --- /dev/null +++ b/test/e2e/shared/kubernetesversions/data/debian_injection_script_worker.envsubst.sh @@ -0,0 +1,115 @@ +#!/bin/bash + +# Copyright 2020 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## Please note that this file needs to be escaped for envsubst to function + +# shellcheck disable=SC1083,SC2034,SC2066,SC2193 + +set -o nounset +set -o pipefail +set -o errexit + +function retry { + attempt=0 + max_attempts=$${1} + interval=$${2} + shift; shift + until [[ $${attempt} -ge "$${max_attempts}" ]] ; do + attempt=$((attempt+1)) + set +e + eval "$*" && return || echo "failed $${attempt} times: $*" + set -e + sleep "$${interval}" + done + echo "error: reached max attempts at retry($*)" + return 1 +} + +[[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + +USE_CI_ARTIFACTS=${USE_CI_ARTIFACTS:=false} + +if [ ! "${USE_CI_ARTIFACTS}" = true ]; then + echo "No CI Artifacts installation, exiting" + exit 0 +fi + +# This test installs release packages or binaries that are a result of the CI and release builds. +# It runs '... --version' commands to verify that the binaries are correctly installed +# and finally uninstalls the packages. +# For the release packages it tests all versions in the support skew. +LINE_SEPARATOR="*************************************************" +echo "$${LINE_SEPARATOR}" + +## Clusterctl set variables +## +# $${KUBERNETES_VERSION} will be replaced by clusterctl +KUBERNETES_VERSION=${KUBERNETES_VERSION} +## +## End clusterctl set variables + +if [[ "$${KUBERNETES_VERSION}" != "" ]]; then + CI_DIR=/tmp/k8s-ci + mkdir -p "$${CI_DIR}" + declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm") + declare -a CONTAINERS_TO_TEST=("kube-proxy") + CONTAINER_EXT="tar" + echo "* testing CI version $${KUBERNETES_VERSION}" + # Check for semver + if [[ "$${KUBERNETES_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + CI_URL="https://storage.googleapis.com/kubernetes-release/release/$${KUBERNETES_VERSION}/bin/linux/amd64" + VERSION_WITHOUT_PREFIX="$${KUBERNETES_VERSION#v}" + export DEBIAN_FRONTEND=noninteractive + retry 10 5 "apt-get update" + apt-get install -y apt-transport-https ca-certificates curl + curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - + echo 'deb https://apt.kubernetes.io/ kubernetes-xenial main' >/etc/apt/sources.list.d/kubernetes.list + apt-get update + # replace . with \. + VERSION_REGEX="$${VERSION_WITHOUT_PREFIX//./\\.}" + PACKAGE_VERSION="$(apt-cache madison kubelet | grep "$${VERSION_REGEX}-" | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')" + for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do + echo "* installing package: $${CI_PACKAGE} $${PACKAGE_VERSION}" + apt-get install -y "$${CI_PACKAGE}=$${PACKAGE_VERSION}" + done + else + CI_URL="https://storage.googleapis.com/kubernetes-release-dev/ci/$${KUBERNETES_VERSION}/bin/linux/amd64" + for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do + # Browser: https://console.cloud.google.com/storage/browser/kubernetes-release-dev?project=kubernetes-release-dev + # e.g.: https://storage.googleapis.com/kubernetes-release-dev/ci/v1.21.0-beta.1.378+cf3374e43491c5/bin/linux/amd64/kubectl + echo "* downloading binary: $${CI_URL}/$${CI_PACKAGE}" + wget "$${CI_URL}/$${CI_PACKAGE}" -O "$${CI_DIR}/$${CI_PACKAGE}" + chmod +x "$${CI_DIR}/$${CI_PACKAGE}" + mv "$${CI_DIR}/$${CI_PACKAGE}" "/usr/bin/$${CI_PACKAGE}" + done + systemctl restart kubelet + fi + for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do + # Browser: https://console.cloud.google.com/storage/browser/kubernetes-release?project=kubernetes-release + # e.g.: https://storage.googleapis.com/kubernetes-release/release/v1.20.4/bin/linux/amd64/kube-apiserver.tar + echo "* downloading package: $${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}" + wget "$${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}" -O "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}" + $${SUDO} ctr -n k8s.io images import "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}" || echo "* ignoring expected 'ctr images import' result" + $${SUDO} ctr -n k8s.io images tag "k8s.gcr.io/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "k8s.gcr.io/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}" + $${SUDO} ctr -n k8s.io images tag "k8s.gcr.io/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "gcr.io/kubernetes-ci-images/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}" + done +fi +echo "* checking binary versions" +echo "ctr version: " "$(ctr version)" +echo "kubeadm version: " "$(kubeadm version -o=short)" +echo "kubectl version: " "$(kubectl version --client=true --short=true)" +echo "kubelet version: " "$(kubelet --version)" +echo "$${LINE_SEPARATOR}" diff --git a/test/e2e/shared/kubernetesversions/data/kustomization.yaml.tpl b/test/e2e/shared/kubernetesversions/data/kustomization.yaml.tpl new file mode 100644 index 0000000000..55c4dbba79 --- /dev/null +++ b/test/e2e/shared/kubernetesversions/data/kustomization.yaml.tpl @@ -0,0 +1,20 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: default +resources: +- ci-artifacts-source-template.yaml +patchesStrategicMerge: +- platform-kustomization.yaml +patchesJson6902: +- path: kubeadmcontrolplane-patch.yaml + target: + group: controlplane.cluster.x-k8s.io + kind: KubeadmControlPlane + name: "{{ .KubeadmControlPlaneName }}" + version: v1alpha4 +- path: kubeadmconfigtemplate-patch.yaml + target: + group: bootstrap.cluster.x-k8s.io + kind: KubeadmConfigTemplate + name: "{{ .KubeadmConfigTemplateName }}" + version: v1alpha4 diff --git a/test/e2e/shared/kubernetesversions/template.go b/test/e2e/shared/kubernetesversions/template.go new file mode 100644 index 0000000000..fa71fc28e1 --- /dev/null +++ b/test/e2e/shared/kubernetesversions/template.go @@ -0,0 +1,224 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubernetesversions + +import ( + "bytes" + _ "embed" + "errors" + "fmt" + "io/ioutil" + "os" + "os/exec" + "path" + "strings" + "text/template" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "sigs.k8s.io/cluster-api/test/framework" + "sigs.k8s.io/yaml" +) + +var ( + //go:embed data/debian_injection_script_control_plane.envsubst.sh + debianInjectionScriptControlPlaneBytes string + + //go:embed data/debian_injection_script_worker.envsubst.sh + debianInjectionScriptWorkerBytes string + + //go:embed data/kustomization.yaml.tpl + kustomizationYAMLBytes string + + kustomizationTemplate = template.Must(template.New("kustomization").Parse(kustomizationYAMLBytes)) +) + +type GenerateCIArtifactsInjectedTemplateForDebianInput struct { + // ArtifactsDirectory is where conformance suite output will go. Defaults to _artifacts + ArtifactsDirectory string + // SourceTemplate is an input YAML clusterctl template which is to have + // the CI artifact script injection + SourceTemplate []byte + // PlatformKustomization is an SMP (strategic-merge-style) patch for adding + // platform specific kustomizations required for use with CI, such as + // referencing a specific image + PlatformKustomization []byte + // KubeadmConfigTemplateName is the name of the KubeadmConfigTemplate resource + // that needs to have the Debian install script injected. Defaults to "${CLUSTER_NAME}-md-0". + KubeadmConfigTemplateName string + // KubeadmControlPlaneName is the name of the KubeadmControlPlane resource + // that needs to have the Debian install script injected. Defaults to "${CLUSTER_NAME}-control-plane". + KubeadmControlPlaneName string + // KubeadmConfigName is the name of a KubeadmConfig that needs kustomizing. To be used in conjunction with MachinePools. Optional. + KubeadmConfigName string +} + +// GenerateCIArtifactsInjectedTemplateForDebian takes a source clusterctl template +// and a platform-specific Kustomize SMP patch and injects a bash script to download +// and install the debian packages for the given Kubernetes version, returning the +// location of the outputted file. +func GenerateCIArtifactsInjectedTemplateForDebian(input GenerateCIArtifactsInjectedTemplateForDebianInput) (string, error) { + if input.SourceTemplate == nil { + return "", errors.New("SourceTemplate must be provided") + } + input.ArtifactsDirectory = framework.ResolveArtifactsDirectory(input.ArtifactsDirectory) + if input.KubeadmConfigTemplateName == "" { + input.KubeadmConfigTemplateName = "${CLUSTER_NAME}-md-0" + } + if input.KubeadmControlPlaneName == "" { + input.KubeadmControlPlaneName = "${CLUSTER_NAME}-control-plane" + } + templateDir := path.Join(input.ArtifactsDirectory, "templates") + overlayDir := path.Join(input.ArtifactsDirectory, "overlay") + + if err := os.MkdirAll(templateDir, 0o750); err != nil { + return "", err + } + if err := os.MkdirAll(overlayDir, 0o750); err != nil { + return "", err + } + + kustomizedTemplate := path.Join(templateDir, "cluster-template-conformance-ci-artifacts.yaml") + + kustomizationYamlBytes, err := generateKustomizationYAML(input) + if err != nil { + return "", err + } + + if err := ioutil.WriteFile(path.Join(overlayDir, "kustomization.yaml"), kustomizationYamlBytes, 0o600); err != nil { + return "", err + } + + patch, err := generateDebianInjectionScriptJSONPatch(input.SourceTemplate, "KubeadmControlPlane", input.KubeadmControlPlaneName, "/spec/kubeadmConfigSpec", debianInjectionScriptControlPlaneBytes) + if err != nil { + return "", err + } + if err := ioutil.WriteFile(path.Join(overlayDir, "kubeadmcontrolplane-patch.yaml"), patch, 0o600); err != nil { + return "", err + } + + patch, err = generateDebianInjectionScriptJSONPatch(input.SourceTemplate, "KubeadmConfigTemplate", input.KubeadmConfigTemplateName, "/spec/template/spec", debianInjectionScriptWorkerBytes) + if err != nil { + return "", err + } + + if err := ioutil.WriteFile(path.Join(overlayDir, "kubeadmconfigtemplate-patch.yaml"), patch, 0o600); err != nil { + return "", err + } + if err := ioutil.WriteFile(path.Join(overlayDir, "ci-artifacts-source-template.yaml"), input.SourceTemplate, 0o600); err != nil { + return "", err + } + if err := ioutil.WriteFile(path.Join(overlayDir, "platform-kustomization.yaml"), input.PlatformKustomization, 0o600); err != nil { + return "", err + } + cmd := exec.Command("kustomize", "build", overlayDir) + data, err := cmd.CombinedOutput() + if err != nil { + return "", err + } + if err := ioutil.WriteFile(kustomizedTemplate, data, 0o600); err != nil { + return "", err + } + return kustomizedTemplate, nil +} + +func generateKustomizationYAML(input GenerateCIArtifactsInjectedTemplateForDebianInput) ([]byte, error) { + var kustomizationYamlBytes bytes.Buffer + if err := kustomizationTemplate.Execute(&kustomizationYamlBytes, input); err != nil { + return nil, err + } + return kustomizationYamlBytes.Bytes(), nil +} + +type jsonPatch struct { + Op string `json:"op"` + Path string `json:"path"` + Value interface{} `json:"value"` +} + +func generateDebianInjectionScriptJSONPatch(sourceTemplate []byte, kind, name, path, content string) ([]byte, error) { + filesPathExists, preKubeadmCommandsPathExists, err := checkExistingPaths(sourceTemplate, kind, name, path) + if err != nil { + return nil, err + } + + var patches []jsonPatch + if !filesPathExists { + patches = append(patches, jsonPatch{ + Op: "add", + Path: fmt.Sprintf("%s/files", path), + Value: []interface{}{}, + }) + } + patches = append(patches, jsonPatch{ + Op: "add", + Path: fmt.Sprintf("%s/files/0", path), + Value: map[string]string{ + "content": content, + "owner": "root:root", + "path": "/usr/local/bin/ci-artifacts.sh", + "permissions": "0750", + }, + }) + if !preKubeadmCommandsPathExists { + patches = append(patches, jsonPatch{ + Op: "add", + Path: fmt.Sprintf("%s/preKubeadmCommands", path), + Value: []string{}, + }) + } + patches = append(patches, jsonPatch{ + Op: "add", + Path: fmt.Sprintf("%s/preKubeadmCommands/0", path), + Value: "/usr/local/bin/ci-artifacts.sh", + }) + + return yaml.Marshal(patches) +} + +func checkExistingPaths(sourceTemplate []byte, kind, name, path string) (bool, bool, error) { + yamlDocs := strings.Split(string(sourceTemplate), "---") + for _, yamlDoc := range yamlDocs { + if yamlDoc == "" { + continue + } + var obj unstructured.Unstructured + if err := yaml.Unmarshal([]byte(yamlDoc), &obj); err != nil { + return false, false, err + } + + if obj.GetKind() != kind { + continue + } + if obj.GetName() != name { + continue + } + + pathSplit := strings.Split(strings.TrimPrefix(path, "/"), "/") + filesPath := append(pathSplit, "files") + preKubeadmCommandsPath := append(pathSplit, "preKubeadmCommands") + _, filesPathExists, err := unstructured.NestedFieldCopy(obj.Object, filesPath...) + if err != nil { + return false, false, err + } + _, preKubeadmCommandsPathExists, err := unstructured.NestedFieldCopy(obj.Object, preKubeadmCommandsPath...) + if err != nil { + return false, false, err + } + return filesPathExists, preKubeadmCommandsPathExists, nil + } + return false, false, fmt.Errorf("could not find document with kind %q and name %q", kind, name) +} diff --git a/test/e2e/shared/suite.go b/test/e2e/shared/suite.go index f1f111f92b..0876705fd6 100644 --- a/test/e2e/shared/suite.go +++ b/test/e2e/shared/suite.go @@ -35,6 +35,8 @@ import ( "sigs.k8s.io/cluster-api/test/framework" "sigs.k8s.io/cluster-api/test/framework/clusterctl" "sigs.k8s.io/yaml" + + "sigs.k8s.io/cluster-api-provider-openstack/test/e2e/shared/kubernetesversions" ) type synchronizedBeforeTestSuiteConfig struct { @@ -77,16 +79,8 @@ func Node1BeforeSuite(e2eCtx *E2EContext) []byte { platformKustomization, err := ioutil.ReadFile(filepath.Join(e2eCtx.Settings.DataFolder, "ci-artifacts-platform-kustomization.yaml")) Expect(err).NotTo(HaveOccurred()) - // TODO(sbuerin): should be removed after: https://github.com/kubernetes-sigs/kustomize/issues/2825 is fixed - //ciTemplate, err := kubernetesversions.GenerateCIArtifactsInjectedTemplateForDebian( - // kubernetesversions.GenerateCIArtifactsInjectedTemplateForDebianInput{ - // ArtifactsDirectory: e2eCtx.Settings.ArtifactFolder, - // SourceTemplate: sourceTemplate, - // PlatformKustomization: platformKustomization, - // }, - //) - ciTemplate, err := GenerateCIArtifactsInjectedTemplateForDebian( - GenerateCIArtifactsInjectedTemplateForDebianInput{ + ciTemplate, err := kubernetesversions.GenerateCIArtifactsInjectedTemplateForDebian( + kubernetesversions.GenerateCIArtifactsInjectedTemplateForDebianInput{ ArtifactsDirectory: e2eCtx.Settings.ArtifactFolder, SourceTemplate: sourceTemplate, PlatformKustomization: platformKustomization, diff --git a/test/e2e/shared/tmp_template.go b/test/e2e/shared/tmp_template.go deleted file mode 100644 index 72807f907c..0000000000 --- a/test/e2e/shared/tmp_template.go +++ /dev/null @@ -1,109 +0,0 @@ -/* -Copyright 2021 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// TODO(sbuerin): should be removed after: https://github.com/kubernetes-sigs/kustomize/issues/2825 is fixed -// copied from: cluster-api/test/framework/kubernetesversions/template.go -// currently we have to skip the CAPI generated debian patch because it would overwrite our files - -package shared - -import ( - "errors" - "io/ioutil" - "os" - "os/exec" - "path" - - "sigs.k8s.io/cluster-api/test/framework" -) - -type GenerateCIArtifactsInjectedTemplateForDebianInput struct { - // ArtifactsDirectory is where conformance suite output will go. Defaults to _artifacts - ArtifactsDirectory string - // SourceTemplate is an input YAML clusterctl template which is to have - // the CI artifact script injection - SourceTemplate []byte - // PlatformKustomization is an SMP (strategic-merge-style) patch for adding - // platform specific kustomizations required for use with CI, such as - // referencing a specific image - PlatformKustomization []byte - // KubeadmConfigTemplateName is the name of the KubeadmConfigTemplate resource - // that needs to have the Debian install script injected. Defaults to "${CLUSTER_NAME}-md-0". - KubeadmConfigTemplateName string - // KubeadmControlPlaneName is the name of the KubeadmControlPlane resource - // that needs to have the Debian install script injected. Defaults to "${CLUSTER_NAME}-control-plane". - KubeadmControlPlaneName string - // KubeadmConfigName is the name of a KubeadmConfig that needs kustomizing. To be used in conjunction with MachinePools. Optional. - KubeadmConfigName string -} - -// GenerateCIArtifactsInjectedTemplateForDebian takes a source clusterctl template -// and a platform-specific Kustomize SMP patch and injects a bash script to download -// and install the debian packages for the given Kubernetes version, returning the -// location of the outputted file. -func GenerateCIArtifactsInjectedTemplateForDebian(input GenerateCIArtifactsInjectedTemplateForDebianInput) (string, error) { - if input.SourceTemplate == nil { - return "", errors.New("SourceTemplate must be provided") - } - input.ArtifactsDirectory = framework.ResolveArtifactsDirectory(input.ArtifactsDirectory) - if input.KubeadmConfigTemplateName == "" { - input.KubeadmConfigTemplateName = "${CLUSTER_NAME}-md-0" - } - if input.KubeadmControlPlaneName == "" { - input.KubeadmControlPlaneName = "${CLUSTER_NAME}-control-plane" - } - templateDir := path.Join(input.ArtifactsDirectory, "templates") - overlayDir := path.Join(input.ArtifactsDirectory, "overlay") - - if err := os.MkdirAll(templateDir, 0o750); err != nil { - return "", err - } - if err := os.MkdirAll(overlayDir, 0o750); err != nil { - return "", err - } - - kustomizedTemplate := path.Join(templateDir, "cluster-template-conformance-ci-artifacts.yaml") - - kustomization := []byte(` -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: default -resources: - - ci-artifacts-source-template.yaml -patchesStrategicMerge: - - platform-kustomization.yaml -`) - - if err := ioutil.WriteFile(path.Join(overlayDir, "kustomization.yaml"), kustomization, 0o600); err != nil { - return "", err - } - - if err := ioutil.WriteFile(path.Join(overlayDir, "ci-artifacts-source-template.yaml"), input.SourceTemplate, 0o600); err != nil { - return "", err - } - if err := ioutil.WriteFile(path.Join(overlayDir, "platform-kustomization.yaml"), input.PlatformKustomization, 0o600); err != nil { - return "", err - } - cmd := exec.Command("kustomize", "build", overlayDir) - data, err := cmd.CombinedOutput() - if err != nil { - return "", err - } - if err := ioutil.WriteFile(kustomizedTemplate, data, 0o600); err != nil { - return "", err - } - return kustomizedTemplate, nil -}