From 2e0c1d68f70efa2390d486c2728452dfaeda1b79 Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Tue, 16 Mar 2021 16:42:27 -0700 Subject: [PATCH] test: cluster template for building from custom images --- .../src/developers/kubernetes-developers.md | 54 ++++ hack/gen-flavors.sh | 2 + templates/test/ci/README.md | 5 + templates/test/dev/README.md | 5 + .../dev/cluster-template-custom-builds.yaml | 306 ++++++++++++++++++ .../test/dev/custom-builds/kustomization.yaml | 5 + .../custom-builds/patches/custom-builds.yaml | 142 ++++++++ 7 files changed, 519 insertions(+) create mode 100644 templates/test/ci/README.md create mode 100644 templates/test/dev/README.md create mode 100644 templates/test/dev/cluster-template-custom-builds.yaml create mode 100644 templates/test/dev/custom-builds/kustomization.yaml create mode 100644 templates/test/dev/custom-builds/patches/custom-builds.yaml diff --git a/docs/book/src/developers/kubernetes-developers.md b/docs/book/src/developers/kubernetes-developers.md index 22e131d2422..fd7a45df2d1 100644 --- a/docs/book/src/developers/kubernetes-developers.md +++ b/docs/book/src/developers/kubernetes-developers.md @@ -123,3 +123,57 @@ AZURE_CLOUD_NODE_MANAGER_IMG=myrepo/my-cnm:v0.0.1 \ CLUSTER_TEMPLATE=cluster-template-external-cloud-provider.yaml \ make create-workload-cluster ``` + +## Testing clusters built from Kubernetes source + +[A template is provided](../../templates/test/dev/cluster-template-custom-builds.yaml) that enables building clusters from custom built Kubernetes components. To quickly build a cluster using this template, export the following environment variables with values that declare the relevant custom-built Kubernetes component references: + +- `$KUBE_BINARY_URL` + - URL to .tar.gz file containing Kubernetes source-built artifacts +- `$KUBE_APISERVER_IMAGE_URL` + - URL to source-built Kubernetes apiserver container image +- `$KUBE_CONTROLLER_MANAGER_IMAGE_URL` + - URL to source-built Kubernetes controller-manager container image +- `$KUBE_SCHEDULER_IMAGE_URL` + - URL to source-built Kubernetes scheduler container image +- `$KUBE_PROXY_IMAGE_URL` + - URL to source-built Kubernetes kube-proxy container image + +In addition, ensure that [these environment variables described in the capz documentation](https://capz.sigs.k8s.io/developers/development.html#customizing-the-cluster-deployment) are also declared and exported. + +After that, you can run `make create-workload-cluster` from the git root of the `cluster-api-provider-azure` repository to create a new cluster running your specified custom Kubernetes build components. For example: + +```sh +$ make create-workload-cluster +# Create workload Cluster. +/Users/jackfrancis/work/src/sigs.k8s.io/cluster-api-provider-azure/hack/tools/bin/envsubst-drone < /Users/jackfrancis/work/src/sigs.k8s.io/cluster-api-provider-azure/templates/test/dev/cluster-template-custom-builds.yaml | kubectl apply -f - +cluster.cluster.x-k8s.io/capzcustom created +azurecluster.infrastructure.cluster.x-k8s.io/capzcustom created +kubeadmcontrolplane.controlplane.cluster.x-k8s.io/capzcustom-control-plane created +azuremachinetemplate.infrastructure.cluster.x-k8s.io/capzcustom-control-plane created +machinedeployment.cluster.x-k8s.io/capzcustom-md-0 created +azuremachinetemplate.infrastructure.cluster.x-k8s.io/capzcustom-md-0 created +kubeadmconfigtemplate.bootstrap.cluster.x-k8s.io/capzcustom-md-0 created +# Wait for the kubeconfig to become available. +timeout --foreground 300 bash -c "while ! kubectl get secrets | grep capzcustom-kubeconfig; do sleep 1; done" +capzcustom-kubeconfig cluster.x-k8s.io/secret 1 1s +# Get kubeconfig and store it locally. +kubectl get secrets capzcustom-kubeconfig -o json | jq -r .data.value | base64 --decode > ./kubeconfig +timeout --foreground 600 bash -c "while ! kubectl --kubeconfig=./kubeconfig get nodes | grep master; do sleep 1; done" +Unable to connect to the server: dial tcp 20.190.10.231:6443: i/o timeout +capzcustom-control-plane-gmvnc NotReady control-plane,master 8s v1.20.5 +run "kubectl --kubeconfig=./kubeconfig ..." to work with the new target cluster +$ echo $? +0 +``` + +The above `make create-workload-cluster` workflow assumes that your kubeconfig context points to your cluster-api management cluster. As the stdout reports, you'll be able to connect to your newly built cluster by referring to a newly generated kubeconfig file in the current working directory. + +A few notes: + +- The URL referenced in a `KUBE_BINARY_URL` environment variable should be a tar'd + gzip'd file that contains files at these relative filepaths in the archive: + - `kubernetes/node/bin/kubelet` + - `kubernetes/node/bin/kubectl` +- It's not strictly required to include a reference to every component (for example, if you just want to test a custom build of kube-proxy, you may just define the `"KUBE_PROXY_IMAGE_URL"` environment variable), but we do assume that any referenced custom components passed to the cluster template are all built from the same source. +- Specify a value of `KUBERNETES_VERSION` that shares a common minor version heritage with the source commit, if possible. If you are testing against very recent main branch upstream Kubernetes commits, the guidance is to use the most recent version of Kubernetes that your capz workflow supports. The latest stable release can always be found [here](https://storage.googleapis.com/kubernetes-release/release/stable.txt). Additionally you can use the URL template to get the latest 1.XX release: + - https://storage.googleapis.com/kubernetes-release/release/stable-1.XX.txt diff --git a/hack/gen-flavors.sh b/hack/gen-flavors.sh index da033febfb6..2774cd6fc63 100755 --- a/hack/gen-flavors.sh +++ b/hack/gen-flavors.sh @@ -23,9 +23,11 @@ root=$(dirname "${BASH_SOURCE[0]}")/.. kustomize="${root}/hack/tools/bin/kustomize" flavors_dir="${root}/templates/flavors/" ci_dir="${root}/templates/test/ci/" +dev_dir="${root}/templates/test/dev/" find "${flavors_dir}"* -maxdepth 0 -type d -print0 | xargs -0 -I {} basename {} | grep -v base | xargs -I {} sh -c "${kustomize} build --reorder none ${flavors_dir}{} > ${root}/templates/cluster-template-{}.yaml" # move the default template to the default file expected by clusterctl mv "${root}/templates/cluster-template-default.yaml" "${root}/templates/cluster-template.yaml" find "${ci_dir}"* -maxdepth 0 -type d -print0 | xargs -0 -I {} basename {} | grep -v patches | xargs -I {} sh -c "${kustomize} build --load_restrictor none --reorder none ${ci_dir}{} > ${ci_dir}cluster-template-{}.yaml" +find "${dev_dir}"* -maxdepth 0 -type d -print0 | xargs -0 -I {} basename {} | grep -v patches | xargs -I {} sh -c "${kustomize} build --load_restrictor none --reorder none ${dev_dir}{} > ${dev_dir}cluster-template-{}.yaml" diff --git a/templates/test/ci/README.md b/templates/test/ci/README.md new file mode 100644 index 00000000000..014d9784e18 --- /dev/null +++ b/templates/test/ci/README.md @@ -0,0 +1,5 @@ +# CI test templates + +The template yaml specs in this directory are intended for regular, automated testing by CI jobs. + +The set of cluster configurations in `/templates/test/ci/` should be considered as the set of "known-working" cluster configurations, and continually validated against relevant code changes. diff --git a/templates/test/dev/README.md b/templates/test/dev/README.md new file mode 100644 index 00000000000..39f81b02905 --- /dev/null +++ b/templates/test/dev/README.md @@ -0,0 +1,5 @@ +# capz developer test templates + +The template yaml specs in this directory are intended for aiding and accelerating capz and/or upstream Kubernetes feature development. + +The set of cluster configurations in `/templates/test/dev/` do not necessarily reflect production-ready cluster configurations. For a set of cluster configurations that are regularly tested against production scenarios, see the template yaml specs under `/templates/test/ci/`. diff --git a/templates/test/dev/cluster-template-custom-builds.yaml b/templates/test/dev/cluster-template-custom-builds.yaml new file mode 100644 index 00000000000..cd1a4fec2bf --- /dev/null +++ b/templates/test/dev/cluster-template-custom-builds.yaml @@ -0,0 +1,306 @@ +apiVersion: cluster.x-k8s.io/v1alpha4 +kind: Cluster +metadata: + labels: + cni: calico + name: ${CLUSTER_NAME} + namespace: default +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1alpha4 + kind: KubeadmControlPlane + name: ${CLUSTER_NAME}-control-plane + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4 + kind: AzureCluster + name: ${CLUSTER_NAME} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4 +kind: AzureCluster +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + location: ${AZURE_LOCATION} + networkSpec: + vnet: + name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} + resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} + subscriptionID: ${AZURE_SUBSCRIPTION_ID} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1alpha4 +kind: KubeadmControlPlane +metadata: + annotations: + controlplane.cluster.x-k8s.io/skip-kube-proxy: "true" + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + infrastructureTemplate: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-control-plane + kubeadmConfigSpec: + clusterConfiguration: + apiServer: + extraArgs: + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + extraVolumes: + - hostPath: /etc/kubernetes/azure.json + mountPath: /etc/kubernetes/azure.json + name: cloud-config + readOnly: true + timeoutForControlPlane: 20m + controllerManager: + extraArgs: + allocate-node-cidrs: "false" + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + cluster-name: ${CLUSTER_NAME} + extraVolumes: + - hostPath: /etc/kubernetes/azure.json + mountPath: /etc/kubernetes/azure.json + name: cloud-config + readOnly: true + etcd: + local: + dataDir: /var/lib/etcddisk/etcd + kubernetesVersion: ${KUBERNETES_VERSION} + diskSetup: + filesystems: + - device: /dev/disk/azure/scsi1/lun0 + extraOpts: + - -E + - lazy_itable_init=1,lazy_journal_init=1 + filesystem: ext4 + label: etcd_disk + - device: ephemeral0.1 + filesystem: ext4 + label: ephemeral0 + replaceFS: ntfs + partitions: + - device: /dev/disk/azure/scsi1/lun0 + layout: true + overwrite: false + tableType: gpt + files: + - content: | + #!/bin/bash + + retrycmd() { + retries=$1; wait_sleep=$2; timeout=$3; shift && shift && shift + for i in $(seq 1 $retries); do + timeout $timeout $@ && break || + if [ $i -eq $retries ]; then + return 1 + else + sleep $wait_sleep + fi + done + echo Executed $i times + } + retrycmd_get_tarball() { + tar_retries=$1; wait_sleep=$2; tarball=$3; url=$4 + for i in $(seq 1 $tar_retries); do + tar -tzf $tarball && break || + if [ $i -eq $tar_retries ]; then + return 1 + else + timeout 60 curl -fsSL $url -o $tarball + sleep $wait_sleep + fi + done + } + + mkdir -p $HOME/.kube || exit 1 + cp -i /etc/kubernetes/admin.conf $HOME/.kube/config || exit 1 + chown $(id -u):$(id -g) $HOME/.kube/config || exit 1 + # wait for the cluster to come online based on the default kubeadm configuration + retrycmd 120 5 10 /usr/bin/kubectl 2>/dev/null cluster-info || exit 1 + if [ -n "${KUBE_BINARY_URL}" ]; then + # stop the kubelet on this node so we can replace it + retrycmd 10 5 10 systemctl stop kubelet || exit 1 + retrycmd_get_tarball 120 5 /tmp/kube-binary.tar.gz ${KUBE_BINARY_URL} || exit 1 + tar -xzvf /tmp/kube-binary.tar.gz --strip-components=3 -C /usr/bin kubernetes/node/bin/kubelet || exit 1 + tar -xzvf /tmp/kube-binary.tar.gz --strip-components=3 -C /usr/bin kubernetes/node/bin/kubectl || exit 1 + chmod +x /usr/bin/kubelet /usr/bin/kubectl || exit 1 + retrycmd 10 5 10 systemctl start kubelet || exit 1 + # wait for the cluster to come back online + retrycmd 120 5 10 /usr/bin/kubectl 2>/dev/null cluster-info || exit 1 + fi + if [ -n "${KUBE_PROXY_IMAGE_URL}" ]; then + retrycmd 10 5 10 kubectl -n kube-system set image daemonset/kube-proxy kube-proxy=${KUBE_PROXY_IMAGE_URL} || exit 1 + fi + if [ -n "${KUBE_APISERVER_IMAGE_URL}" ] || [ -n "${KUBE_CONTROLLER_MANAGER_IMAGE_URL}" ] || [ -n "${KUBE_SCHEDULER_IMAGE_URL}" ]; then + retrycmd_get_tarball 120 5 /tmp/yq_linux_amd64.tar.gz https://github.com/mikefarah/yq/releases/download/v4.6.1/yq_linux_amd64.tar.gz || exit 1 + tar -xzvf /tmp/yq_linux_amd64.tar.gz -C /tmp && mv /tmp/yq_linux_amd64 /usr/bin/yq || exit 1 + fi + if [ -n "${KUBE_APISERVER_IMAGE_URL}" ]; then + yq e '.spec.containers[0].image = "${KUBE_APISERVER_IMAGE_URL}"' -i /etc/kubernetes/manifests/kube-apiserver.yaml || exit 1 + fi + if [ -n "${KUBE_CONTROLLER_MANAGER_IMAGE_URL}" ]; then + yq e '.spec.containers[0].image = "${KUBE_CONTROLLER_MANAGER_IMAGE_URL}"' -i /etc/kubernetes/manifests/kube-controller-manager.yaml || exit 1 + fi + if [ -n "${KUBE_SCHEDULER_IMAGE_URL}" ]; then + yq e '.spec.containers[0].image = "${KUBE_SCHEDULER_IMAGE_URL}"' -i /etc/kubernetes/manifests/kube-scheduler.yaml || exit 1 + fi + owner: root:root + path: /tmp/replace-k8s-components.sh + permissions: "0744" + - contentFrom: + secret: + key: control-plane-azure.json + name: ${CLUSTER_NAME}-control-plane-azure-json + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + name: '{{ ds.meta_data["local_hostname"] }}' + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + name: '{{ ds.meta_data["local_hostname"] }}' + mounts: + - - LABEL=etcd_disk + - /var/lib/etcddisk + postKubeadmCommands: + - bash -c /tmp/replace-k8s-components.sh + useExperimentalRetryJoin: true + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + template: + spec: + dataDisks: + - diskSizeGB: 256 + lun: 0 + nameSuffix: etcddisk + osDisk: + diskSizeGB: 128 + managedDisk: + storageAccountType: Premium_LRS + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} +--- +apiVersion: cluster.x-k8s.io/v1alpha4 +kind: MachineDeployment +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT} + selector: + matchLabels: null + template: + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1alpha4 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-md-0 + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-md-0 + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + template: + spec: + osDisk: + diskSizeGB: 128 + managedDisk: + storageAccountType: Premium_LRS + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + vmSize: ${AZURE_NODE_MACHINE_TYPE} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1alpha4 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + template: + spec: + files: + - content: | + #!/bin/bash + + retrycmd() { + retries=$1; wait_sleep=$2; timeout=$3; shift && shift && shift + for i in $(seq 1 $retries); do + timeout $timeout $@ && break || + if [ $i -eq $retries ]; then + return 1 + else + sleep $wait_sleep + fi + done + echo Executed $i times + } + retrycmd_get_tarball() { + tar_retries=$1; wait_sleep=$2; tarball=$3; url=$4 + echo "$tar_retries retries" + for i in $(seq 1 $tar_retries); do + tar -tzf $tarball && break || + if [ $i -eq $tar_retries ]; then + return 1 + else + timeout 60 curl -fsSL $url -o $tarball + sleep $wait_sleep + fi + done + } + + if [ -n "${KUBE_BINARY_URL}" ]; then + retrycmd_get_tarball 120 5 /tmp/kube-binary.tar.gz ${KUBE_BINARY_URL} || exit 1 + # stop the kubelet on this node so we can replace it + retrycmd 10 5 10 systemctl stop kubelet || exit 1 + tar -xzvf /tmp/kube-binary.tar.gz --strip-components=3 -C /usr/bin kubernetes/node/bin/kubelet || exit 1 + chmod +x /usr/bin/kubelet || exit 1 + retrycmd 10 5 10 systemctl start kubelet || exit 1 + fi + owner: root:root + path: /tmp/replace-k8s-components.sh + permissions: "0744" + - contentFrom: + secret: + key: control-plane-azure.json + name: ${CLUSTER_NAME}-control-plane-azure-json + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + name: '{{ ds.meta_data["local_hostname"] }}' + postKubeadmCommands: + - bash -c /tmp/replace-k8s-components.sh diff --git a/templates/test/dev/custom-builds/kustomization.yaml b/templates/test/dev/custom-builds/kustomization.yaml new file mode 100644 index 00000000000..9187d069653 --- /dev/null +++ b/templates/test/dev/custom-builds/kustomization.yaml @@ -0,0 +1,5 @@ +namespace: default +resources: + - ../../../flavors/default +patchesStrategicMerge: + - patches/custom-builds.yaml diff --git a/templates/test/dev/custom-builds/patches/custom-builds.yaml b/templates/test/dev/custom-builds/patches/custom-builds.yaml new file mode 100644 index 00000000000..38e529f7c21 --- /dev/null +++ b/templates/test/dev/custom-builds/patches/custom-builds.yaml @@ -0,0 +1,142 @@ +apiVersion: controlplane.cluster.x-k8s.io/v1alpha4 +kind: KubeadmControlPlane +metadata: + name: "${CLUSTER_NAME}-control-plane" + annotations: + controlplane.cluster.x-k8s.io/skip-kube-proxy: "true" +spec: + kubeadmConfigSpec: + useExperimentalRetryJoin: true + clusterConfiguration: + kubernetesVersion: "${KUBERNETES_VERSION}" + postKubeadmCommands: + - bash -c /tmp/replace-k8s-components.sh + files: + - path: /tmp/replace-k8s-components.sh + owner: "root:root" + permissions: "0744" + content: | + #!/bin/bash + + retrycmd() { + retries=$1; wait_sleep=$2; timeout=$3; shift && shift && shift + for i in $(seq 1 $retries); do + timeout $timeout $@ && break || + if [ $i -eq $retries ]; then + return 1 + else + sleep $wait_sleep + fi + done + echo Executed $i times + } + retrycmd_get_tarball() { + tar_retries=$1; wait_sleep=$2; tarball=$3; url=$4 + for i in $(seq 1 $tar_retries); do + tar -tzf $tarball && break || + if [ $i -eq $tar_retries ]; then + return 1 + else + timeout 60 curl -fsSL $url -o $tarball + sleep $wait_sleep + fi + done + } + + mkdir -p $HOME/.kube || exit 1 + cp -i /etc/kubernetes/admin.conf $HOME/.kube/config || exit 1 + chown $(id -u):$(id -g) $HOME/.kube/config || exit 1 + # wait for the cluster to come online based on the default kubeadm configuration + retrycmd 120 5 10 /usr/bin/kubectl 2>/dev/null cluster-info || exit 1 + if [ -n "${KUBE_BINARY_URL}" ]; then + # stop the kubelet on this node so we can replace it + retrycmd 10 5 10 systemctl stop kubelet || exit 1 + retrycmd_get_tarball 120 5 /tmp/kube-binary.tar.gz ${KUBE_BINARY_URL} || exit 1 + tar -xzvf /tmp/kube-binary.tar.gz --strip-components=3 -C /usr/bin kubernetes/node/bin/kubelet || exit 1 + tar -xzvf /tmp/kube-binary.tar.gz --strip-components=3 -C /usr/bin kubernetes/node/bin/kubectl || exit 1 + chmod +x /usr/bin/kubelet /usr/bin/kubectl || exit 1 + retrycmd 10 5 10 systemctl start kubelet || exit 1 + # wait for the cluster to come back online + retrycmd 120 5 10 /usr/bin/kubectl 2>/dev/null cluster-info || exit 1 + fi + if [ -n "${KUBE_PROXY_IMAGE_URL}" ]; then + retrycmd 10 5 10 kubectl -n kube-system set image daemonset/kube-proxy kube-proxy=${KUBE_PROXY_IMAGE_URL} || exit 1 + fi + if [ -n "${KUBE_APISERVER_IMAGE_URL}" ] || [ -n "${KUBE_CONTROLLER_MANAGER_IMAGE_URL}" ] || [ -n "${KUBE_SCHEDULER_IMAGE_URL}" ]; then + retrycmd_get_tarball 120 5 /tmp/yq_linux_amd64.tar.gz https://github.com/mikefarah/yq/releases/download/v4.6.1/yq_linux_amd64.tar.gz || exit 1 + tar -xzvf /tmp/yq_linux_amd64.tar.gz -C /tmp && mv /tmp/yq_linux_amd64 /usr/bin/yq || exit 1 + fi + if [ -n "${KUBE_APISERVER_IMAGE_URL}" ]; then + yq e '.spec.containers[0].image = "${KUBE_APISERVER_IMAGE_URL}"' -i /etc/kubernetes/manifests/kube-apiserver.yaml || exit 1 + fi + if [ -n "${KUBE_CONTROLLER_MANAGER_IMAGE_URL}" ]; then + yq e '.spec.containers[0].image = "${KUBE_CONTROLLER_MANAGER_IMAGE_URL}"' -i /etc/kubernetes/manifests/kube-controller-manager.yaml || exit 1 + fi + if [ -n "${KUBE_SCHEDULER_IMAGE_URL}" ]; then + yq e '.spec.containers[0].image = "${KUBE_SCHEDULER_IMAGE_URL}"' -i /etc/kubernetes/manifests/kube-scheduler.yaml || exit 1 + fi + - path: /etc/kubernetes/azure.json + owner: "root:root" + permissions: "0644" + contentFrom: + secret: + key: control-plane-azure.json + name: ${CLUSTER_NAME}-control-plane-azure-json +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1alpha4 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + template: + spec: + postKubeadmCommands: + - bash -c /tmp/replace-k8s-components.sh + files: + - path: /tmp/replace-k8s-components.sh + owner: "root:root" + permissions: "0744" + content: | + #!/bin/bash + + retrycmd() { + retries=$1; wait_sleep=$2; timeout=$3; shift && shift && shift + for i in $(seq 1 $retries); do + timeout $timeout $@ && break || + if [ $i -eq $retries ]; then + return 1 + else + sleep $wait_sleep + fi + done + echo Executed $i times + } + retrycmd_get_tarball() { + tar_retries=$1; wait_sleep=$2; tarball=$3; url=$4 + echo "$tar_retries retries" + for i in $(seq 1 $tar_retries); do + tar -tzf $tarball && break || + if [ $i -eq $tar_retries ]; then + return 1 + else + timeout 60 curl -fsSL $url -o $tarball + sleep $wait_sleep + fi + done + } + + if [ -n "${KUBE_BINARY_URL}" ]; then + retrycmd_get_tarball 120 5 /tmp/kube-binary.tar.gz ${KUBE_BINARY_URL} || exit 1 + # stop the kubelet on this node so we can replace it + retrycmd 10 5 10 systemctl stop kubelet || exit 1 + tar -xzvf /tmp/kube-binary.tar.gz --strip-components=3 -C /usr/bin kubernetes/node/bin/kubelet || exit 1 + chmod +x /usr/bin/kubelet || exit 1 + retrycmd 10 5 10 systemctl start kubelet || exit 1 + fi + - path: /etc/kubernetes/azure.json + owner: "root:root" + permissions: "0644" + contentFrom: + secret: + key: control-plane-azure.json + name: ${CLUSTER_NAME}-control-plane-azure-json