From 3e4da2823246799bb77ee2be0d1e12bc4f6b1cfc Mon Sep 17 00:00:00 2001 From: Cecile Robert-Michon Date: Thu, 26 Jan 2023 18:44:09 +0000 Subject: [PATCH] Convert flavor and test ci templates to external cloud-provider --- Tiltfile | 2 +- .../src/developers/kubernetes-developers.md | 5 +- docs/book/src/topics/addons.md | 6 +- docs/book/src/topics/machinepools.md | 5 +- hack/util.sh | 10 + scripts/ci-conformance.sh | 7 + scripts/ci-entrypoint.sh | 37 +- templates/cluster-template-aad.yaml | 25 +- templates/cluster-template-azure-bastion.yaml | 25 +- templates/cluster-template-clusterclass.yaml | 25 +- templates/cluster-template-dual-stack.yaml | 25 +- templates/cluster-template-edgezone.yaml | 20 - templates/cluster-template-ephemeral.yaml | 25 +- ...e-external-cloud-provider-machinepool.yaml | 233 - templates/cluster-template-flatcar.yaml | 25 +- ...uster-template-intree-cloud-provider.yaml} | 10 +- templates/cluster-template-ipv6.yaml | 2 - .../cluster-template-machinepool-windows.yaml | 28 +- templates/cluster-template-machinepool.yaml | 25 +- templates/cluster-template-nvidia-gpu.yaml | 25 +- templates/cluster-template-private.yaml | 25 +- templates/cluster-template-windows.yaml | 28 +- templates/cluster-template.yaml | 25 +- templates/flavors/README.md | 6 +- templates/flavors/aad/machine-deployment.yaml | 3 +- .../aad/patches/kubeadm-controlplane.yaml | 14 +- templates/flavors/base/cluster-template.yaml | 22 +- .../clusterclass/kubeadm-config-template.yaml | 3 +- .../kubeadm-controlplane-template.yaml | 22 +- .../flavors/default/machine-deployment.yaml | 3 +- .../dual-stack/machine-deployment.yaml | 3 +- .../patches/kubeadm-controlplane.yaml | 12 +- templates/flavors/edgezone/kustomization.yaml | 1 - .../kustomization.yaml | 7 - .../machine-pool-deployment.yaml | 62 - .../patches/external-cloud-provider.yaml | 41 - .../kustomization.yaml | 6 - .../patches/external-cloud-provider.yaml | 44 - .../flavors/flatcar/machine-deployment.yaml | 3 +- templates/flavors/ipv6/kustomization.yaml | 4 +- .../flavors/ipv6/machine-deployment.yaml | 2 - templates/flavors/ipv6/patches/ipv6.yaml | 2 - .../ipv6/patches/kubeadm-controlplane.yaml | 10 - .../machine-pool-deployment-windows.yaml | 3 +- .../machinepool/machine-pool-deployment.yaml | 3 +- .../nvidia-gpu/machine-deployment.yaml | 3 +- .../windows/machine-deployment-windows.yaml | 3 +- ...plate-prow-azurediskcsi-migration-off.yaml | 244 - ...ow-ci-version-windows-containerd-2022.yaml | 28 +- .../ci/cluster-template-prow-ci-version.yaml | 28 +- ...template-prow-clusterclass-ci-default.yaml | 28 +- .../ci/cluster-template-prow-custom-vnet.yaml | 25 +- .../ci/cluster-template-prow-dual-stack.yaml | 25 +- .../ci/cluster-template-prow-edgezone.yaml | 20 - ...ate-prow-external-azurediskcsi-driver.yaml | 244 - ...row-external-cloud-provider-vmss-flex.yaml | 245 - ...template-prow-external-cloud-provider.yaml | 242 - .../ci/cluster-template-prow-flatcar.yaml | 25 +- ...uster-template-prow-identity-from-env.yaml | 4393 ----------------- ...ow-intree-cloud-provider-machine-pool.yaml | 689 +++ ...-template-prow-intree-cloud-provider.yaml} | 440 +- .../test/ci/cluster-template-prow-ipv6.yaml | 2 - ...template-prow-machine-pool-ci-version.yaml | 28 +- ...uster-template-prow-machine-pool-flex.yaml | 679 +++ .../cluster-template-prow-machine-pool.yaml | 28 +- .../ci/cluster-template-prow-nvidia-gpu.yaml | 25 +- ...e-prow-user-assigned-managed-identity.yaml | 237 - templates/test/ci/cluster-template-prow.yaml | 28 +- .../kubeadm-config-template.yaml | 3 +- .../prow-clusterclass-ci-default/windows.yaml | 3 +- .../patches/external-azurediskcsi-driver.yaml | 45 - .../kustomization.yaml | 8 - .../external-cloud-provider-windows.yaml | 13 - .../kustomization.yaml | 11 - .../kustomization.yaml | 11 - .../kustomization.yaml | 8 + .../patches/intree-mp.yaml | 28 + .../kustomization.yaml | 6 +- .../patches/intree-cp.yaml | 40 + .../patches/intree-md-0.yaml | 14 + .../patches/intree-md-win.yaml | 18 + .../kustomization.yaml | 5 +- .../patches/vmss-flex.yaml | 14 +- .../test/ci/prow-private/kustomization.yaml | 2 + .../kustomization.yaml | 12 - .../kustomization.yaml | 2 +- .../test/dev/custom-builds/kustomization.yaml | 2 +- test/e2e/azure_privatecluster.go | 2 +- test/e2e/azure_test.go | 15 +- test/e2e/cloud-provider-azure.go | 14 +- test/e2e/common.go | 2 +- test/e2e/config/azure-dev.yaml | 14 +- .../v1beta1/bases/cluster-with-kcp.yaml | 14 +- .../v1beta1/bases/md.yaml | 3 +- .../v1beta1/bases/mp.yaml | 5 +- .../cluster-template-kcp-remediation.yaml | 15 +- .../cluster-template-kcp-scale-in.yaml | 15 +- .../cluster-template-machine-pool.yaml | 15 +- .../cluster-template-md-remediation.yaml | 15 +- .../v1beta1/cluster-template-node-drain.yaml | 15 +- .../v1beta1/cluster-template-upgrades.yaml | 18 +- .../v1beta1/cluster-template.yaml | 15 +- 102 files changed, 1779 insertions(+), 7271 deletions(-) delete mode 100644 templates/cluster-template-external-cloud-provider-machinepool.yaml rename templates/{cluster-template-external-cloud-provider.yaml => cluster-template-intree-cloud-provider.yaml} (97%) delete mode 100644 templates/flavors/external-cloud-provider-machinepool/kustomization.yaml delete mode 100644 templates/flavors/external-cloud-provider-machinepool/machine-pool-deployment.yaml delete mode 100644 templates/flavors/external-cloud-provider-machinepool/patches/external-cloud-provider.yaml delete mode 100644 templates/flavors/external-cloud-provider/kustomization.yaml delete mode 100644 templates/flavors/external-cloud-provider/patches/external-cloud-provider.yaml delete mode 100644 templates/test/ci/cluster-template-prow-azurediskcsi-migration-off.yaml delete mode 100644 templates/test/ci/cluster-template-prow-external-azurediskcsi-driver.yaml delete mode 100644 templates/test/ci/cluster-template-prow-external-cloud-provider-vmss-flex.yaml delete mode 100644 templates/test/ci/cluster-template-prow-external-cloud-provider.yaml delete mode 100644 templates/test/ci/cluster-template-prow-identity-from-env.yaml create mode 100644 templates/test/ci/cluster-template-prow-intree-cloud-provider-machine-pool.yaml rename templates/test/ci/{cluster-template-prow-external-cloud-provider-ci-version.yaml => cluster-template-prow-intree-cloud-provider.yaml} (63%) create mode 100644 templates/test/ci/cluster-template-prow-machine-pool-flex.yaml delete mode 100644 templates/test/ci/cluster-template-prow-user-assigned-managed-identity.yaml delete mode 100644 templates/test/ci/prow-external-azurediskcsi-driver/patches/external-azurediskcsi-driver.yaml delete mode 100644 templates/test/ci/prow-external-cloud-provider-ci-version/kustomization.yaml delete mode 100644 templates/test/ci/prow-external-cloud-provider-ci-version/patches/external-cloud-provider-windows.yaml delete mode 100644 templates/test/ci/prow-external-cloud-provider-vmss-flex/kustomization.yaml delete mode 100644 templates/test/ci/prow-external-cloud-provider/kustomization.yaml create mode 100644 templates/test/ci/prow-intree-cloud-provider-machine-pool/kustomization.yaml create mode 100644 templates/test/ci/prow-intree-cloud-provider-machine-pool/patches/intree-mp.yaml rename templates/test/ci/{prow-external-azurediskcsi-driver => prow-intree-cloud-provider}/kustomization.yaml (54%) create mode 100644 templates/test/ci/prow-intree-cloud-provider/patches/intree-cp.yaml create mode 100644 templates/test/ci/prow-intree-cloud-provider/patches/intree-md-0.yaml create mode 100644 templates/test/ci/prow-intree-cloud-provider/patches/intree-md-win.yaml rename templates/test/ci/{prow-azurediskcsi-migration-off => prow-machine-pool-flex}/kustomization.yaml (56%) rename templates/{flavors/external-cloud-provider-machinepool => test/ci/prow-machine-pool-flex}/patches/vmss-flex.yaml (50%) delete mode 100644 templates/test/ci/prow-user-assigned-managed-identity/kustomization.yaml diff --git a/Tiltfile b/Tiltfile index 3d9f5a1f906..13f33d85cba 100644 --- a/Tiltfile +++ b/Tiltfile @@ -365,7 +365,7 @@ def deploy_worker_templates(template, substitutions): else: calico_values = "./templates/addons/calico/values.yaml" flavor_cmd += "; " + helm_cmd + " repo add projectcalico https://docs.tigera.io/calico/charts; " + helm_cmd + " --kubeconfig ./${CLUSTER_NAME}.kubeconfig install calico projectcalico/tigera-operator -f " + calico_values + " --namespace tigera-operator --create-namespace" - if "external-cloud-provider" in flavor_name: + if "intree-cloud-provider" not in flavor_name and "ipv6" not in flavor_name: # TODO: remove ipv6 once https://github.com/kubernetes-sigs/cloud-provider-azure/issues/3401 is fixed. flavor_cmd += "; " + helm_cmd + " --kubeconfig ./${CLUSTER_NAME}.kubeconfig install --repo https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo cloud-provider-azure --generate-name --set infra.clusterName=${CLUSTER_NAME}" local_resource( name = flavor_name, diff --git a/docs/book/src/developers/kubernetes-developers.md b/docs/book/src/developers/kubernetes-developers.md index b3ecaf70c2c..88934348300 100644 --- a/docs/book/src/developers/kubernetes-developers.md +++ b/docs/book/src/developers/kubernetes-developers.md @@ -44,12 +44,11 @@ $ export CNM_IMAGE_NAME=azure-node-controller-manager $ export IMAGE_TAG=canary ``` -Then, use the `external-cloud-provider` flavor to create a cluster: +Then, create a cluster: ```bash $ export CLUSTER_NAME=my-cluster -$ CLUSTER_TEMPLATE=cluster-template-external-cloud-provider.yaml \ -make create-workload-cluster +$ make create-workload-cluster ``` Once your cluster deploys, you should receive the kubeconfig to the workload cluster. Set your `KUBECONFIG` environment variable to point to the kubeconfig file, then use the official cloud-provider-azure Helm chart to deploy the cloud-provider-azure components using your custom built images: diff --git a/docs/book/src/topics/addons.md b/docs/book/src/topics/addons.md index b9b085e76b0..f143599bcdc 100644 --- a/docs/book/src/topics/addons.md +++ b/docs/book/src/topics/addons.md @@ -149,9 +149,9 @@ kubectl apply -f kube-flannel.yml # External Cloud Provider -To deploy a cluster using [external cloud provider](https://github.com/kubernetes-sigs/cloud-provider-azure), create a cluster configuration with the [external cloud provider template](https://raw.githubusercontent.com/kubernetes-sigs/cluster-api-provider-azure/main/templates/cluster-template-external-cloud-provider.yaml). +The "external" or "out-of-tree" cloud provider for Azure is the recommended cloud provider for CAPZ clusters. The "in-tree" cloud provider has been deprecated since v1.20 and only bug fixes are allowed in its Kubernetes repository directory. -After the cluster has provisioned, install the `cloud-provider-azure` components using the official helm chart: +Below are instructions to install [external cloud provider](https://github.com/kubernetes-sigs/cloud-provider-azure) components on a self-managed cluster using the official helm chart. For more information see the official [`cloud-provider-azure` helm chart documentation](https://github.com/kubernetes-sigs/cloud-provider-azure/tree/master/helm/cloud-provider-azure). Grab the CIDR ranges from your cluster by running this kubectl statement against the management cluster: @@ -180,7 +180,7 @@ kube-system cloud-node-manager-mfsdg kube-system cloud-node-manager-qrz74 1/1 Running 0 24s ``` -For more information see the official [`cloud-provider-azure` helm chart documentation](https://github.com/kubernetes-sigs/cloud-provider-azure/tree/master/helm/cloud-provider-azure). +To know more about configuring cloud-provider-azure, see [Configuring the Kubernetes Cloud Provider for Azure](./cloud-provider-config.md). ## Storage Drivers diff --git a/docs/book/src/topics/machinepools.md b/docs/book/src/topics/machinepools.md index 8dfcd5d83d4..8e0930e546d 100644 --- a/docs/book/src/topics/machinepools.md +++ b/docs/book/src/topics/machinepools.md @@ -42,7 +42,7 @@ which provides the cloud provider-specific resource for orchestrating a group of Azure Virtual Machine Scale Sets support two orchestration modes: `Uniform` and `Flexible`. CAPZ defaults to `Uniform` mode. See [VMSS Orchestration modes in Azure](https://learn.microsoft.com/azure/virtual-machine-scale-sets/virtual-machine-scale-sets-orchestration-modes) for more information. -To use `Flexible` mode requires Kubernetes v1.26.0 or later with a workload cluster template like CAPZ's "external-cloud-provider-machinepool" flavor. Ensure that `orchestrationMode` on the `AzureMachinePool` spec is set: +To use `Flexible` mode requires Kubernetes v1.26.0 or later. Ensure that `orchestrationMode` on the `AzureMachinePool` spec is set: ```yaml apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 @@ -190,8 +190,5 @@ spec: permissions: "0644" joinConfiguration: nodeRegistration: - kubeletExtraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure name: '{{ ds.meta_data["local_hostname"] }}' ``` diff --git a/hack/util.sh b/hack/util.sh index 6039db92cf2..d489ab2a805 100755 --- a/hack/util.sh +++ b/hack/util.sh @@ -41,6 +41,16 @@ capz::util::should_build_kubernetes() { echo "false" } +capz::util::should_build_ccm() { + if [[ -n "${TEST_CCM:-}" ]]; then + echo "true" && return + fi + if [[ "${E2E_ARGS:-}" == "-kubetest.use-ci-artifacts" ]]; then + echo "true" && return + fi + echo "false" +} + # all test regions must support AvailabilityZones capz::util::get_random_region() { local REGIONS=("canadacentral" "eastus" "eastus2" "northeurope" "uksouth" "westeurope" "westus2" "westus3") diff --git a/scripts/ci-conformance.sh b/scripts/ci-conformance.sh index 464c716232f..f6c32c52029 100755 --- a/scripts/ci-conformance.sh +++ b/scripts/ci-conformance.sh @@ -55,6 +55,13 @@ else export KUBE_BUILD_CONFORMANCE="y" source "${REPO_ROOT}/scripts/ci-build-kubernetes.sh" fi + + if [[ "$(capz::util::should_build_ccm)" == "true" ]]; then + # shellcheck source=scripts/ci-build-azure-ccm.sh + source "${REPO_ROOT}/scripts/ci-build-azure-ccm.sh" + echo "Will use the ${IMAGE_REGISTRY}/${CCM_IMAGE_NAME}:${IMAGE_TAG} cloud-controller-manager image for external cloud-provider-cluster" + echo "Will use the ${IMAGE_REGISTRY}/${CNM_IMAGE_NAME}:${IMAGE_TAG} cloud-node-manager image for external cloud-provider-azure cluster" + fi fi defaultTag=$(date -u '+%Y%m%d%H%M%S') diff --git a/scripts/ci-entrypoint.sh b/scripts/ci-entrypoint.sh index 61427d609a3..0d29d0b3553 100755 --- a/scripts/ci-entrypoint.sh +++ b/scripts/ci-entrypoint.sh @@ -55,6 +55,13 @@ setup() { source "${REPO_ROOT}/scripts/ci-build-azure-ccm.sh" echo "Will use the ${IMAGE_REGISTRY}/${CCM_IMAGE_NAME}:${IMAGE_TAG} cloud-controller-manager image for external cloud-provider-cluster" echo "Will use the ${IMAGE_REGISTRY}/${CNM_IMAGE_NAME}:${IMAGE_TAG} cloud-node-manager image for external cloud-provider-azure cluster" + + export CCM_IMG_ARGS=(--set cloudControllerManager.imageRepository="${IMAGE_REGISTRY}" + --set cloudNodeManager.imageRepository="${IMAGE_REGISTRY}" + --set cloudControllerManager.imageName="${CCM_IMAGE_NAME}" + --set cloudNodeManager.imageName="${CNM_IMAGE_NAME}" + --set-string cloudControllerManager.imageTag="${IMAGE_TAG}" + --set-string cloudNodeManager.imageTag="${IMAGE_TAG}") fi if [[ "$(capz::util::should_build_kubernetes)" == "true" ]]; then @@ -115,11 +122,6 @@ select_cluster_template() { export CLUSTER_TEMPLATE="test/ci/cluster-template-prow.yaml" fi - if [[ -n "${TEST_CCM:-}" ]]; then - # replace 'prow' with 'prow-external-cloud-provider' in the template name if testing out-of-tree - export CLUSTER_TEMPLATE="${CLUSTER_TEMPLATE/prow/prow-external-cloud-provider}" - fi - if [[ "${EXP_MACHINE_POOL:-}" == "true" ]]; then if [[ "${CLUSTER_TEMPLATE}" =~ "prow" ]]; then export CLUSTER_TEMPLATE="${CLUSTER_TEMPLATE/prow/prow-machine-pool}" @@ -148,6 +150,18 @@ get_cidrs() { fi } +# get_cloud_provider determines if the Cluster is using an intree or external cloud-provider from the KubeadmConfigSpec. +# any retryable operation in this function must return a non-zero exit code on failure so that we can +# retry it using a `until get_cloud_provider; do sleep 5; done` pattern; +# and any statement must be idempotent so that subsequent retry attempts can make forward progress. +get_cloud_provider() { + CLOUD_PROVIDER=$("${KUBECTL}" get kubeadmcontrolplane -l cluster.x-k8s.io/cluster-name="${CLUSTER_NAME}" -o=jsonpath='{.items[0].spec.kubeadmConfigSpec.clusterConfiguration.controllerManager.extraArgs.cloud-provider}') + if [[ "${CLOUD_PROVIDER:-}" = "azure" ]]; then + IN_TREE="true" + export IN_TREE + fi +} + # install_calico installs Calico CNI componentry onto the Cluster # any retryable operation in this function must return a non-zero exit code on failure so that we can # retry it using a `until install_calico; do sleep 5; done` pattern; @@ -205,18 +219,12 @@ install_cloud_provider_azure() { echo "Installing cloud-provider-azure components via helm" "${HELM}" upgrade cloud-provider-azure --install --repo https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo cloud-provider-azure \ --set infra.clusterName="${CLUSTER_NAME}" \ - --set cloudControllerManager.imageRepository="${IMAGE_REGISTRY}" \ - --set cloudNodeManager.imageRepository="${IMAGE_REGISTRY}" \ - --set cloudControllerManager.imageName="${CCM_IMAGE_NAME}" \ - --set cloudNodeManager.imageName="${CNM_IMAGE_NAME}" \ - --set-string cloudControllerManager.imageTag="${IMAGE_TAG}" \ - --set-string cloudNodeManager.imageTag="${IMAGE_TAG}" \ --set cloudControllerManager.replicas="${CCM_COUNT}" \ --set cloudControllerManager.enableDynamicReloading="${ENABLE_DYNAMIC_RELOADING}" \ --set cloudControllerManager.cloudConfig="${CLOUD_CONFIG}" \ --set cloudControllerManager.cloudConfigSecretName="${CONFIG_SECRET_NAME}" \ --set cloudControllerManager.logVerbosity="${CCM_LOG_VERBOSITY}" \ - --set-string cloudControllerManager.clusterCIDR="${CCM_CLUSTER_CIDR}" + --set-string cloudControllerManager.clusterCIDR="${CCM_CLUSTER_CIDR}" "${CCM_IMG_ARGS[@]}" } # wait_for_nodes returns when all nodes in the workload cluster are Ready. @@ -265,7 +273,10 @@ install_addons() { sleep 5 done # install cloud-provider-azure components, if using out-of-tree - if [[ -n "${TEST_CCM:-}" ]]; then + until get_cloud_provider; do + sleep 5 + done + if [[ -z "${IN_TREE:-}" ]]; then until install_cloud_provider_azure; do sleep 5 done diff --git a/templates/cluster-template-aad.yaml b/templates/cluster-template-aad.yaml index 9b81a0ff1e1..2ee92c5a40d 100644 --- a/templates/cluster-template-aad.yaml +++ b/templates/cluster-template-aad.yaml @@ -51,30 +51,18 @@ spec: clusterConfiguration: apiServer: extraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external oidc-client-id: ${AZURE_SERVER_APP_ID} oidc-groups-claim: groups oidc-issuer-url: https://sts.windows.net/${AZURE_TENANT_ID}/ oidc-username-claim: oid oidc-username-prefix: '-' - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true timeoutForControlPlane: 20m controllerManager: extraArgs: allocate-node-cidrs: "false" - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true etcd: local: dataDir: /var/lib/etcddisk/etcd @@ -109,15 +97,13 @@ spec: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -210,8 +196,7 @@ spec: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 diff --git a/templates/cluster-template-azure-bastion.yaml b/templates/cluster-template-azure-bastion.yaml index 9c7cf319f53..e922f76a8cc 100644 --- a/templates/cluster-template-azure-bastion.yaml +++ b/templates/cluster-template-azure-bastion.yaml @@ -53,25 +53,13 @@ spec: clusterConfiguration: apiServer: extraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true + cloud-provider: external timeoutForControlPlane: 20m controllerManager: extraArgs: allocate-node-cidrs: "false" - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true etcd: local: dataDir: /var/lib/etcddisk/etcd @@ -106,15 +94,13 @@ spec: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -205,8 +191,7 @@ spec: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: [] --- diff --git a/templates/cluster-template-clusterclass.yaml b/templates/cluster-template-clusterclass.yaml index 0d1314681f6..7c9c39da4a2 100644 --- a/templates/cluster-template-clusterclass.yaml +++ b/templates/cluster-template-clusterclass.yaml @@ -170,25 +170,13 @@ spec: clusterConfiguration: apiServer: extraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true + cloud-provider: external timeoutForControlPlane: 20m controllerManager: extraArgs: allocate-node-cidrs: "false" - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true etcd: local: dataDir: /var/lib/etcddisk/etcd @@ -223,15 +211,13 @@ spec: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -259,8 +245,7 @@ spec: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: [] --- diff --git a/templates/cluster-template-dual-stack.yaml b/templates/cluster-template-dual-stack.yaml index d4a50f8d9e2..1a72e1a0eca 100644 --- a/templates/cluster-template-dual-stack.yaml +++ b/templates/cluster-template-dual-stack.yaml @@ -65,27 +65,15 @@ spec: clusterConfiguration: apiServer: extraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true + cloud-provider: external timeoutForControlPlane: 20m controllerManager: extraArgs: allocate-node-cidrs: "true" - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-cidr: 10.244.0.0/16,2001:1234:5678:9a40::/58 cluster-name: ${CLUSTER_NAME} configure-cloud-routes: "true" - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true etcd: local: dataDir: /var/lib/etcddisk/etcd @@ -122,8 +110,7 @@ spec: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: controlPlane: @@ -132,8 +119,7 @@ spec: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -248,8 +234,7 @@ spec: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' postKubeadmCommands: - echo "DNSStubListener=no" >> /etc/systemd/resolved.conf diff --git a/templates/cluster-template-edgezone.yaml b/templates/cluster-template-edgezone.yaml index 230733a1fad..54b93177496 100644 --- a/templates/cluster-template-edgezone.yaml +++ b/templates/cluster-template-edgezone.yaml @@ -52,27 +52,13 @@ spec: clusterConfiguration: apiServer: extraArgs: - cloud-config: /etc/kubernetes/azure.json cloud-provider: external - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true timeoutForControlPlane: 20m controllerManager: extraArgs: allocate-node-cidrs: "false" - cloud-config: /etc/kubernetes/azure.json cloud-provider: external cluster-name: ${CLUSTER_NAME} - external-cloud-volume-plugin: azure - feature-gates: CSIMigrationAzureDisk=true - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true etcd: local: dataDir: /var/lib/etcddisk/etcd @@ -107,17 +93,13 @@ spec: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json cloud-provider: external - feature-gates: CSIMigrationAzureDisk=true name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json cloud-provider: external - feature-gates: CSIMigrationAzureDisk=true name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -208,9 +190,7 @@ spec: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json cloud-provider: external - feature-gates: CSIMigrationAzureDisk=true name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: [] --- diff --git a/templates/cluster-template-ephemeral.yaml b/templates/cluster-template-ephemeral.yaml index 4f4aa314961..b815021e241 100644 --- a/templates/cluster-template-ephemeral.yaml +++ b/templates/cluster-template-ephemeral.yaml @@ -51,25 +51,13 @@ spec: clusterConfiguration: apiServer: extraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true + cloud-provider: external timeoutForControlPlane: 20m controllerManager: extraArgs: allocate-node-cidrs: "false" - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true etcd: local: dataDir: /var/lib/etcddisk/etcd @@ -104,15 +92,13 @@ spec: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -209,8 +195,7 @@ spec: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: [] --- diff --git a/templates/cluster-template-external-cloud-provider-machinepool.yaml b/templates/cluster-template-external-cloud-provider-machinepool.yaml deleted file mode 100644 index e9db35bbe15..00000000000 --- a/templates/cluster-template-external-cloud-provider-machinepool.yaml +++ /dev/null @@ -1,233 +0,0 @@ -apiVersion: cluster.x-k8s.io/v1beta1 -kind: Cluster -metadata: - name: ${CLUSTER_NAME} - namespace: default -spec: - clusterNetwork: - pods: - cidrBlocks: - - 192.168.0.0/16 - controlPlaneRef: - apiVersion: controlplane.cluster.x-k8s.io/v1beta1 - kind: KubeadmControlPlane - name: ${CLUSTER_NAME}-control-plane - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureCluster - name: ${CLUSTER_NAME} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureCluster -metadata: - name: ${CLUSTER_NAME} - namespace: default -spec: - identityRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureClusterIdentity - name: ${CLUSTER_IDENTITY_NAME} - location: ${AZURE_LOCATION} - networkSpec: - subnets: - - name: control-plane-subnet - role: control-plane - - name: node-subnet - natGateway: - name: node-natgateway - role: node - vnet: - name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} - resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} - subscriptionID: ${AZURE_SUBSCRIPTION_ID} ---- -apiVersion: controlplane.cluster.x-k8s.io/v1beta1 -kind: KubeadmControlPlane -metadata: - name: ${CLUSTER_NAME}-control-plane - namespace: default -spec: - kubeadmConfigSpec: - clusterConfiguration: - apiServer: - extraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: external - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true - timeoutForControlPlane: 20m - controllerManager: - extraArgs: - allocate-node-cidrs: "false" - cloud-config: /etc/kubernetes/azure.json - cloud-provider: external - cluster-name: ${CLUSTER_NAME} - external-cloud-volume-plugin: azure - feature-gates: CSIMigrationAzureDisk=true - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true - etcd: - local: - dataDir: /var/lib/etcddisk/etcd - extraArgs: - quota-backend-bytes: "8589934592" - diskSetup: - filesystems: - - device: /dev/disk/azure/scsi1/lun0 - extraOpts: - - -E - - lazy_itable_init=1,lazy_journal_init=1 - filesystem: ext4 - label: etcd_disk - - device: ephemeral0.1 - filesystem: ext4 - label: ephemeral0 - replaceFS: ntfs - partitions: - - device: /dev/disk/azure/scsi1/lun0 - layout: true - overwrite: false - tableType: gpt - files: - - contentFrom: - secret: - key: control-plane-azure.json - name: ${CLUSTER_NAME}-control-plane-azure-json - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" - initConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: external - feature-gates: CSIMigrationAzureDisk=true - name: '{{ ds.meta_data["local_hostname"] }}' - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: external - feature-gates: CSIMigrationAzureDisk=true - name: '{{ ds.meta_data["local_hostname"] }}' - mounts: - - - LABEL=etcd_disk - - /var/lib/etcddisk - postKubeadmCommands: [] - preKubeadmCommands: [] - machineTemplate: - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachineTemplate - name: ${CLUSTER_NAME}-control-plane - replicas: ${CONTROL_PLANE_MACHINE_COUNT} - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachineTemplate -metadata: - name: ${CLUSTER_NAME}-control-plane - namespace: default -spec: - template: - spec: - dataDisks: - - diskSizeGB: 256 - lun: 0 - nameSuffix: etcddisk - osDisk: - diskSizeGB: 128 - osType: Linux - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachinePool -metadata: - name: ${CLUSTER_NAME}-mp-0 - namespace: default -spec: - clusterName: ${CLUSTER_NAME} - replicas: ${WORKER_MACHINE_COUNT} - template: - spec: - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfig - name: ${CLUSTER_NAME}-mp-0 - clusterName: ${CLUSTER_NAME} - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachinePool - name: ${CLUSTER_NAME}-mp-0 - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachinePool -metadata: - name: ${CLUSTER_NAME}-mp-0 - namespace: default -spec: - location: ${AZURE_LOCATION} - orchestrationMode: Flexible - strategy: - rollingUpdate: - maxSurge: 0% - maxUnavailable: 0 - type: RollingUpdate - template: - osDisk: - diskSizeGB: 30 - managedDisk: - storageAccountType: Premium_LRS - osType: Linux - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - vmSize: ${AZURE_NODE_MACHINE_TYPE} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfig -metadata: - name: ${CLUSTER_NAME}-mp-0 - namespace: default -spec: - files: - - contentFrom: - secret: - key: worker-node-azure.json - name: ${CLUSTER_NAME}-mp-0-azure-json - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: external - feature-gates: CSIMigrationAzureDisk=true - name: '{{ ds.meta_data["local_hostname"] }}' ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureClusterIdentity -metadata: - labels: - clusterctl.cluster.x-k8s.io/move-hierarchy: "true" - name: ${CLUSTER_IDENTITY_NAME} - namespace: default -spec: - allowedNamespaces: {} - clientID: ${AZURE_CLIENT_ID} - clientSecret: - name: ${AZURE_CLUSTER_IDENTITY_SECRET_NAME} - namespace: ${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE} - tenantID: ${AZURE_TENANT_ID} - type: ServicePrincipal diff --git a/templates/cluster-template-flatcar.yaml b/templates/cluster-template-flatcar.yaml index 062a5433f26..697817b9c62 100644 --- a/templates/cluster-template-flatcar.yaml +++ b/templates/cluster-template-flatcar.yaml @@ -51,25 +51,13 @@ spec: clusterConfiguration: apiServer: extraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true + cloud-provider: external timeoutForControlPlane: 20m controllerManager: extraArgs: allocate-node-cidrs: "false" - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true etcd: local: dataDir: /var/lib/etcddisk/etcd @@ -117,15 +105,13 @@ spec: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '@@HOSTNAME@@' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '@@HOSTNAME@@' mounts: - - etcd_disk @@ -240,8 +226,7 @@ spec: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '@@HOSTNAME@@' postKubeadmCommands: [] preKubeadmCommands: diff --git a/templates/cluster-template-external-cloud-provider.yaml b/templates/cluster-template-intree-cloud-provider.yaml similarity index 97% rename from templates/cluster-template-external-cloud-provider.yaml rename to templates/cluster-template-intree-cloud-provider.yaml index b92c875bd27..73c19b643e6 100644 --- a/templates/cluster-template-external-cloud-provider.yaml +++ b/templates/cluster-template-intree-cloud-provider.yaml @@ -52,7 +52,7 @@ spec: apiServer: extraArgs: cloud-config: /etc/kubernetes/azure.json - cloud-provider: external + cloud-provider: azure extraVolumes: - hostPath: /etc/kubernetes/azure.json mountPath: /etc/kubernetes/azure.json @@ -63,7 +63,7 @@ spec: extraArgs: allocate-node-cidrs: "false" cloud-config: /etc/kubernetes/azure.json - cloud-provider: external + cloud-provider: azure cluster-name: ${CLUSTER_NAME} external-cloud-volume-plugin: azure feature-gates: CSIMigrationAzureDisk=true @@ -107,7 +107,7 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: external + cloud-provider: azure feature-gates: CSIMigrationAzureDisk=true name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: @@ -115,7 +115,7 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: external + cloud-provider: azure feature-gates: CSIMigrationAzureDisk=true name: '{{ ds.meta_data["local_hostname"] }}' mounts: @@ -208,7 +208,7 @@ spec: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: external + cloud-provider: azure feature-gates: CSIMigrationAzureDisk=true name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: [] diff --git a/templates/cluster-template-ipv6.yaml b/templates/cluster-template-ipv6.yaml index d4196e16eed..11a519105c8 100644 --- a/templates/cluster-template-ipv6.yaml +++ b/templates/cluster-template-ipv6.yaml @@ -1,8 +1,6 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: - labels: - cni: calico-ipv6 name: ${CLUSTER_NAME} namespace: default spec: diff --git a/templates/cluster-template-machinepool-windows.yaml b/templates/cluster-template-machinepool-windows.yaml index 3b028348fa3..d9052ba84be 100644 --- a/templates/cluster-template-machinepool-windows.yaml +++ b/templates/cluster-template-machinepool-windows.yaml @@ -55,26 +55,14 @@ spec: clusterConfiguration: apiServer: extraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external feature-gates: WindowsHostProcessContainers=true - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true timeoutForControlPlane: 20m controllerManager: extraArgs: allocate-node-cidrs: "false" - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true etcd: local: dataDir: /var/lib/etcddisk/etcd @@ -109,15 +97,13 @@ spec: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -212,8 +198,7 @@ spec: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 @@ -294,8 +279,7 @@ spec: criSocket: npipe:////./pipe/containerd-containerd kubeletExtraArgs: azure-container-registry-config: c:/k/azure.json - cloud-config: c:/k/azure.json - cloud-provider: azure + cloud-provider: external feature-gates: WindowsHostProcessContainers=true pod-infra-container-image: mcr.microsoft.com/oss/kubernetes/pause:3.4.1 name: '{{ ds.meta_data["local_hostname"] }}' diff --git a/templates/cluster-template-machinepool.yaml b/templates/cluster-template-machinepool.yaml index dbc4043331e..5f01ddae9ef 100644 --- a/templates/cluster-template-machinepool.yaml +++ b/templates/cluster-template-machinepool.yaml @@ -51,25 +51,13 @@ spec: clusterConfiguration: apiServer: extraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true + cloud-provider: external timeoutForControlPlane: 20m controllerManager: extraArgs: allocate-node-cidrs: "false" - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true etcd: local: dataDir: /var/lib/etcddisk/etcd @@ -104,15 +92,13 @@ spec: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -207,8 +193,7 @@ spec: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 diff --git a/templates/cluster-template-nvidia-gpu.yaml b/templates/cluster-template-nvidia-gpu.yaml index 93885b53e26..6a74c18657f 100644 --- a/templates/cluster-template-nvidia-gpu.yaml +++ b/templates/cluster-template-nvidia-gpu.yaml @@ -51,25 +51,13 @@ spec: clusterConfiguration: apiServer: extraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true + cloud-provider: external timeoutForControlPlane: 20m controllerManager: extraArgs: allocate-node-cidrs: "false" - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true etcd: local: dataDir: /var/lib/etcddisk/etcd @@ -104,15 +92,13 @@ spec: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -221,6 +207,5 @@ spec: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' diff --git a/templates/cluster-template-private.yaml b/templates/cluster-template-private.yaml index d9ef8bc6390..e8a8ba7b51f 100644 --- a/templates/cluster-template-private.yaml +++ b/templates/cluster-template-private.yaml @@ -60,25 +60,13 @@ spec: clusterConfiguration: apiServer: extraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true + cloud-provider: external timeoutForControlPlane: 20m controllerManager: extraArgs: allocate-node-cidrs: "false" - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true etcd: local: dataDir: /var/lib/etcddisk/etcd @@ -113,15 +101,13 @@ spec: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -217,8 +203,7 @@ spec: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: [] --- diff --git a/templates/cluster-template-windows.yaml b/templates/cluster-template-windows.yaml index 6a2b7e8cd02..ac94f38119d 100644 --- a/templates/cluster-template-windows.yaml +++ b/templates/cluster-template-windows.yaml @@ -55,26 +55,14 @@ spec: clusterConfiguration: apiServer: extraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external feature-gates: WindowsHostProcessContainers=true - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true timeoutForControlPlane: 20m controllerManager: extraArgs: allocate-node-cidrs: "false" - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true etcd: local: dataDir: /var/lib/etcddisk/etcd @@ -109,15 +97,13 @@ spec: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -208,8 +194,7 @@ spec: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: [] --- @@ -300,8 +285,7 @@ spec: criSocket: npipe:////./pipe/containerd-containerd kubeletExtraArgs: azure-container-registry-config: c:/k/azure.json - cloud-config: c:/k/azure.json - cloud-provider: azure + cloud-provider: external feature-gates: WindowsHostProcessContainers=true v: "2" windows-priorityclass: ABOVE_NORMAL_PRIORITY_CLASS diff --git a/templates/cluster-template.yaml b/templates/cluster-template.yaml index 15cef1bba03..4fc41f16cb9 100644 --- a/templates/cluster-template.yaml +++ b/templates/cluster-template.yaml @@ -51,25 +51,13 @@ spec: clusterConfiguration: apiServer: extraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true + cloud-provider: external timeoutForControlPlane: 20m controllerManager: extraArgs: allocate-node-cidrs: "false" - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true etcd: local: dataDir: /var/lib/etcddisk/etcd @@ -104,15 +92,13 @@ spec: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -203,8 +189,7 @@ spec: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: [] --- diff --git a/templates/flavors/README.md b/templates/flavors/README.md index 33bc09ada15..c06a0f1033b 100644 --- a/templates/flavors/README.md +++ b/templates/flavors/README.md @@ -2,10 +2,12 @@ In `clusterctl` the infrastructure provider authors can provide different type of cluster templates, or flavors; use the --flavor flag to specify which flavor to use; e.g + ```shell -clusterctl generate cluster my-cluster --kubernetes-version v1.22.1 \ - --flavor external-cloud-provider > my-cluster.yaml +clusterctl generate cluster my-cluster --kubernetes-version v1.26.1 \ + --flavor private > my-cluster.yaml ``` + See [`clusterctl` flavors docs](https://cluster-api.sigs.k8s.io/clusterctl/commands/generate-cluster.html#flavors). This directory contains each of the flavors for CAPZ. Each directory besides `base` will be used to diff --git a/templates/flavors/aad/machine-deployment.yaml b/templates/flavors/aad/machine-deployment.yaml index d0855a7cf48..2e373f69a40 100644 --- a/templates/flavors/aad/machine-deployment.yaml +++ b/templates/flavors/aad/machine-deployment.yaml @@ -48,9 +48,8 @@ spec: nodeRegistration: name: '{{ ds.meta_data["local_hostname"] }}' kubeletExtraArgs: - cloud-provider: azure - cloud-config: /etc/kubernetes/azure.json azure-container-registry-config: /etc/kubernetes/azure.json + cloud-provider: external files: - contentFrom: secret: diff --git a/templates/flavors/aad/patches/kubeadm-controlplane.yaml b/templates/flavors/aad/patches/kubeadm-controlplane.yaml index ed38feb0f55..8dde2f89450 100644 --- a/templates/flavors/aad/patches/kubeadm-controlplane.yaml +++ b/templates/flavors/aad/patches/kubeadm-controlplane.yaml @@ -8,28 +8,20 @@ spec: nodeRegistration: name: '{{ ds.meta_data["local_hostname"] }}' kubeletExtraArgs: - cloud-provider: azure - cloud-config: /etc/kubernetes/azure.json azure-container-registry-config: /etc/kubernetes/azure.json + cloud-provider: external joinConfiguration: nodeRegistration: name: '{{ ds.meta_data["local_hostname"] }}' kubeletExtraArgs: - cloud-provider: azure - cloud-config: /etc/kubernetes/azure.json azure-container-registry-config: /etc/kubernetes/azure.json + cloud-provider: external clusterConfiguration: apiServer: timeoutForControlPlane: 20m extraArgs: - cloud-provider: azure - cloud-config: /etc/kubernetes/azure.json oidc-username-claim: oid oidc-groups-claim: groups oidc-client-id: "${AZURE_SERVER_APP_ID}" oidc-issuer-url: "https://sts.windows.net/${AZURE_TENANT_ID}/" - oidc-username-prefix: "-" - controllerManager: - extraArgs: - cloud-provider: azure - cloud-config: /etc/kubernetes/azure.json + oidc-username-prefix: "-" \ No newline at end of file diff --git a/templates/flavors/base/cluster-template.yaml b/templates/flavors/base/cluster-template.yaml index 9c0c2eef7fb..679eda71982 100644 --- a/templates/flavors/base/cluster-template.yaml +++ b/templates/flavors/base/cluster-template.yaml @@ -51,38 +51,24 @@ spec: nodeRegistration: name: '{{ ds.meta_data["local_hostname"] }}' kubeletExtraArgs: - cloud-provider: azure - cloud-config: /etc/kubernetes/azure.json azure-container-registry-config: /etc/kubernetes/azure.json + cloud-provider: external joinConfiguration: nodeRegistration: name: '{{ ds.meta_data["local_hostname"] }}' kubeletExtraArgs: - cloud-provider: azure - cloud-config: /etc/kubernetes/azure.json azure-container-registry-config: /etc/kubernetes/azure.json + cloud-provider: external clusterConfiguration: apiServer: timeoutForControlPlane: 20m extraArgs: - cloud-provider: azure - cloud-config: /etc/kubernetes/azure.json - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true + cloud-provider: external controllerManager: extraArgs: - cloud-provider: azure - cloud-config: /etc/kubernetes/azure.json allocate-node-cidrs: "false" cluster-name: ${CLUSTER_NAME} - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true + cloud-provider: external etcd: local: dataDir: "/var/lib/etcddisk/etcd" diff --git a/templates/flavors/clusterclass/kubeadm-config-template.yaml b/templates/flavors/clusterclass/kubeadm-config-template.yaml index 7b597447b82..2991ab7dddd 100644 --- a/templates/flavors/clusterclass/kubeadm-config-template.yaml +++ b/templates/flavors/clusterclass/kubeadm-config-template.yaml @@ -18,7 +18,6 @@ spec: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: [] \ No newline at end of file diff --git a/templates/flavors/clusterclass/kubeadm-controlplane-template.yaml b/templates/flavors/clusterclass/kubeadm-controlplane-template.yaml index fc3bc5fd3b1..f749208c36b 100644 --- a/templates/flavors/clusterclass/kubeadm-controlplane-template.yaml +++ b/templates/flavors/clusterclass/kubeadm-controlplane-template.yaml @@ -10,25 +10,13 @@ spec: clusterConfiguration: apiServer: extraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true + cloud-provider: external timeoutForControlPlane: 20m controllerManager: extraArgs: allocate-node-cidrs: "false" - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true etcd: local: dataDir: /var/lib/etcddisk/etcd @@ -63,15 +51,13 @@ spec: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk diff --git a/templates/flavors/default/machine-deployment.yaml b/templates/flavors/default/machine-deployment.yaml index c2ce6a416a4..f63d224948a 100644 --- a/templates/flavors/default/machine-deployment.yaml +++ b/templates/flavors/default/machine-deployment.yaml @@ -47,9 +47,8 @@ spec: nodeRegistration: name: '{{ ds.meta_data["local_hostname"] }}' kubeletExtraArgs: - cloud-provider: azure - cloud-config: /etc/kubernetes/azure.json azure-container-registry-config: /etc/kubernetes/azure.json + cloud-provider: external files: - contentFrom: secret: diff --git a/templates/flavors/dual-stack/machine-deployment.yaml b/templates/flavors/dual-stack/machine-deployment.yaml index 42dd2ec26ee..f3c56ca609e 100644 --- a/templates/flavors/dual-stack/machine-deployment.yaml +++ b/templates/flavors/dual-stack/machine-deployment.yaml @@ -54,9 +54,8 @@ spec: nodeRegistration: name: '{{ ds.meta_data["local_hostname"] }}' kubeletExtraArgs: - cloud-provider: azure - cloud-config: /etc/kubernetes/azure.json azure-container-registry-config: /etc/kubernetes/azure.json + cloud-provider: external files: - contentFrom: secret: diff --git a/templates/flavors/dual-stack/patches/kubeadm-controlplane.yaml b/templates/flavors/dual-stack/patches/kubeadm-controlplane.yaml index d59c7081330..29e0525e140 100644 --- a/templates/flavors/dual-stack/patches/kubeadm-controlplane.yaml +++ b/templates/flavors/dual-stack/patches/kubeadm-controlplane.yaml @@ -13,32 +13,26 @@ spec: nodeRegistration: name: '{{ ds.meta_data["local_hostname"] }}' kubeletExtraArgs: - cloud-provider: azure - cloud-config: /etc/kubernetes/azure.json azure-container-registry-config: /etc/kubernetes/azure.json + cloud-provider: external localAPIEndpoint: bindPort: 6443 joinConfiguration: nodeRegistration: name: '{{ ds.meta_data["local_hostname"] }}' kubeletExtraArgs: - cloud-provider: azure - cloud-config: /etc/kubernetes/azure.json azure-container-registry-config: /etc/kubernetes/azure.json + cloud-provider: external controlPlane: localAPIEndpoint: bindPort: 6443 clusterConfiguration: apiServer: timeoutForControlPlane: 20m - extraArgs: - cloud-provider: azure - cloud-config: /etc/kubernetes/azure.json controllerManager: extraArgs: - cloud-provider: azure - cloud-config: /etc/kubernetes/azure.json #required for ipv6 using calico allocate-node-cidrs: "true" cluster-cidr: "10.244.0.0/16,2001:1234:5678:9a40::/58" configure-cloud-routes: "true" + cloud-provider: external diff --git a/templates/flavors/edgezone/kustomization.yaml b/templates/flavors/edgezone/kustomization.yaml index b8592cf5409..89e733b6ee1 100644 --- a/templates/flavors/edgezone/kustomization.yaml +++ b/templates/flavors/edgezone/kustomization.yaml @@ -3,6 +3,5 @@ resources: - ../default patchesStrategicMerge: - - ../external-cloud-provider/patches/external-cloud-provider.yaml - patches/azure-extendedlocation.yaml - patches/azure-remove-natgateway.yaml diff --git a/templates/flavors/external-cloud-provider-machinepool/kustomization.yaml b/templates/flavors/external-cloud-provider-machinepool/kustomization.yaml deleted file mode 100644 index a83b7479daf..00000000000 --- a/templates/flavors/external-cloud-provider-machinepool/kustomization.yaml +++ /dev/null @@ -1,7 +0,0 @@ -namespace: default -resources: - - ../machinepool - -patchesStrategicMerge: - - patches/external-cloud-provider.yaml - - patches/vmss-flex.yaml diff --git a/templates/flavors/external-cloud-provider-machinepool/machine-pool-deployment.yaml b/templates/flavors/external-cloud-provider-machinepool/machine-pool-deployment.yaml deleted file mode 100644 index a29f6fe9cc8..00000000000 --- a/templates/flavors/external-cloud-provider-machinepool/machine-pool-deployment.yaml +++ /dev/null @@ -1,62 +0,0 @@ ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachinePool -metadata: - name: "${CLUSTER_NAME}-mp-0" -spec: - clusterName: "${CLUSTER_NAME}" - replicas: ${WORKER_MACHINE_COUNT} - template: - spec: - clusterName: "${CLUSTER_NAME}" - version: "${KUBERNETES_VERSION}" - bootstrap: - configRef: - name: "${CLUSTER_NAME}-mp-0" - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfig - infrastructureRef: - name: "${CLUSTER_NAME}-mp-0" - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachinePool ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachinePool -metadata: - name: "${CLUSTER_NAME}-mp-0" -spec: - location: ${AZURE_LOCATION} - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 0% - maxUnavailable: 0 - template: - vmSize: ${AZURE_NODE_MACHINE_TYPE} - osDisk: - osType: "Linux" - diskSizeGB: 30 - managedDisk: - storageAccountType: "Premium_LRS" - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfig -metadata: - name: "${CLUSTER_NAME}-mp-0" -spec: - joinConfiguration: - nodeRegistration: - name: '{{ ds.meta_data["local_hostname"] }}' - kubeletExtraArgs: - cloud-provider: azure - cloud-config: /etc/kubernetes/azure.json - azure-container-registry-config: /etc/kubernetes/azure.json - files: - - contentFrom: - secret: - name: ${CLUSTER_NAME}-mp-0-azure-json - key: worker-node-azure.json - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" diff --git a/templates/flavors/external-cloud-provider-machinepool/patches/external-cloud-provider.yaml b/templates/flavors/external-cloud-provider-machinepool/patches/external-cloud-provider.yaml deleted file mode 100644 index 03ea70e29b4..00000000000 --- a/templates/flavors/external-cloud-provider-machinepool/patches/external-cloud-provider.yaml +++ /dev/null @@ -1,41 +0,0 @@ ---- -kind: KubeadmControlPlane -apiVersion: controlplane.cluster.x-k8s.io/v1beta1 -metadata: - name: "${CLUSTER_NAME}-control-plane" -spec: - kubeadmConfigSpec: - initConfiguration: - nodeRegistration: - kubeletExtraArgs: - cloud-provider: external - feature-gates: CSIMigrationAzureDisk=true - azure-container-registry-config: /etc/kubernetes/azure.json - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - cloud-provider: external - feature-gates: CSIMigrationAzureDisk=true - azure-container-registry-config: /etc/kubernetes/azure.json - clusterConfiguration: - apiServer: - timeoutForControlPlane: 20m - extraArgs: - cloud-provider: external - controllerManager: - extraArgs: - cloud-provider: external - external-cloud-volume-plugin: azure - feature-gates: CSIMigrationAzureDisk=true - version: "${KUBERNETES_VERSION}" ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfig -metadata: - name: ${CLUSTER_NAME}-mp-0 -spec: - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - feature-gates: CSIMigrationAzureDisk=true - cloud-provider: external diff --git a/templates/flavors/external-cloud-provider/kustomization.yaml b/templates/flavors/external-cloud-provider/kustomization.yaml deleted file mode 100644 index 00f4e4e4d9e..00000000000 --- a/templates/flavors/external-cloud-provider/kustomization.yaml +++ /dev/null @@ -1,6 +0,0 @@ -namespace: default -resources: - - ../default - -patchesStrategicMerge: - - patches/external-cloud-provider.yaml diff --git a/templates/flavors/external-cloud-provider/patches/external-cloud-provider.yaml b/templates/flavors/external-cloud-provider/patches/external-cloud-provider.yaml deleted file mode 100644 index 89af6cd5130..00000000000 --- a/templates/flavors/external-cloud-provider/patches/external-cloud-provider.yaml +++ /dev/null @@ -1,44 +0,0 @@ ---- -kind: KubeadmControlPlane -apiVersion: controlplane.cluster.x-k8s.io/v1beta1 -metadata: - name: "${CLUSTER_NAME}-control-plane" -spec: - kubeadmConfigSpec: - initConfiguration: - nodeRegistration: - kubeletExtraArgs: - cloud-provider: external - feature-gates: CSIMigrationAzureDisk=true - azure-container-registry-config: /etc/kubernetes/azure.json - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - cloud-provider: external - feature-gates: CSIMigrationAzureDisk=true - azure-container-registry-config: /etc/kubernetes/azure.json - clusterConfiguration: - apiServer: - timeoutForControlPlane: 20m - extraArgs: - cloud-provider: external - controllerManager: - extraArgs: - cloud-provider: external - external-cloud-volume-plugin: azure - feature-gates: CSIMigrationAzureDisk=true - version: "${KUBERNETES_VERSION}" ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfigTemplate -metadata: - name: "${CLUSTER_NAME}-md-0" -spec: - template: - spec: - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - cloud-provider: external - feature-gates: CSIMigrationAzureDisk=true - azure-container-registry-config: /etc/kubernetes/azure.json diff --git a/templates/flavors/flatcar/machine-deployment.yaml b/templates/flavors/flatcar/machine-deployment.yaml index ff408db3ba7..59ccfde5794 100644 --- a/templates/flavors/flatcar/machine-deployment.yaml +++ b/templates/flavors/flatcar/machine-deployment.yaml @@ -71,8 +71,7 @@ spec: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '@@HOSTNAME@@' postKubeadmCommands: [] preKubeadmCommands: diff --git a/templates/flavors/ipv6/kustomization.yaml b/templates/flavors/ipv6/kustomization.yaml index 20b7e499b7b..6a07db80c85 100644 --- a/templates/flavors/ipv6/kustomization.yaml +++ b/templates/flavors/ipv6/kustomization.yaml @@ -5,7 +5,9 @@ resources: - machine-deployment.yaml patchesStrategicMerge: + - ../../azure-cluster-identity/azurecluster-identity-ref.yaml + - ../../test/ci/prow-intree-cloud-provider/patches/intree-cp.yaml # TODO: enable CCM for ipv6, see https://github.com/kubernetes-sigs/cloud-provider-azure/issues/3401. + - ../../test/ci/prow-intree-cloud-provider/patches/intree-md-0.yaml # TODO: enable CCM for ipv6, see https://github.com/kubernetes-sigs/cloud-provider-azure/issues/3401. - patches/ipv6.yaml - patches/kubeadm-controlplane.yaml - patches/controlplane-azuremachinetemplate.yaml - - ../../azure-cluster-identity/azurecluster-identity-ref.yaml diff --git a/templates/flavors/ipv6/machine-deployment.yaml b/templates/flavors/ipv6/machine-deployment.yaml index b843f63e7c8..9d2b1e644ed 100644 --- a/templates/flavors/ipv6/machine-deployment.yaml +++ b/templates/flavors/ipv6/machine-deployment.yaml @@ -54,8 +54,6 @@ spec: nodeRegistration: name: '{{ ds.meta_data["local_hostname"] }}' kubeletExtraArgs: - cloud-provider: azure - cloud-config: /etc/kubernetes/azure.json azure-container-registry-config: /etc/kubernetes/azure.json node-ip: "::" cluster-dns: "[fd00::10]" diff --git a/templates/flavors/ipv6/patches/ipv6.yaml b/templates/flavors/ipv6/patches/ipv6.yaml index 1731bfa559a..ea0eff7286d 100644 --- a/templates/flavors/ipv6/patches/ipv6.yaml +++ b/templates/flavors/ipv6/patches/ipv6.yaml @@ -3,8 +3,6 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: name: ${CLUSTER_NAME} - labels: - cni: "calico-ipv6" spec: clusterNetwork: pods: diff --git a/templates/flavors/ipv6/patches/kubeadm-controlplane.yaml b/templates/flavors/ipv6/patches/kubeadm-controlplane.yaml index 83fdc707283..7905e2e79b5 100644 --- a/templates/flavors/ipv6/patches/kubeadm-controlplane.yaml +++ b/templates/flavors/ipv6/patches/kubeadm-controlplane.yaml @@ -13,9 +13,6 @@ spec: nodeRegistration: name: '{{ ds.meta_data["local_hostname"] }}' kubeletExtraArgs: - cloud-provider: azure - cloud-config: /etc/kubernetes/azure.json - azure-container-registry-config: /etc/kubernetes/azure.json node-ip: "::" cluster-dns: "fd00::10" localAPIEndpoint: @@ -25,9 +22,6 @@ spec: nodeRegistration: name: '{{ ds.meta_data["local_hostname"] }}' kubeletExtraArgs: - cloud-provider: azure - cloud-config: /etc/kubernetes/azure.json - azure-container-registry-config: /etc/kubernetes/azure.json node-ip: "::" cluster-dns: "fd00::10" controlPlane: @@ -38,13 +32,9 @@ spec: apiServer: timeoutForControlPlane: 20m extraArgs: - cloud-provider: azure - cloud-config: /etc/kubernetes/azure.json bind-address: "::" controllerManager: extraArgs: - cloud-provider: azure - cloud-config: /etc/kubernetes/azure.json #required for ipv6 using calico allocate-node-cidrs: "true" cluster-cidr: "2001:1234:5678:9a40::/58" diff --git a/templates/flavors/machinepool-windows/machine-pool-deployment-windows.yaml b/templates/flavors/machinepool-windows/machine-pool-deployment-windows.yaml index 7f90b786d32..a104072e950 100644 --- a/templates/flavors/machinepool-windows/machine-pool-deployment-windows.yaml +++ b/templates/flavors/machinepool-windows/machine-pool-deployment-windows.yaml @@ -57,9 +57,8 @@ spec: name: '{{ ds.meta_data["local_hostname"] }}' criSocket: npipe:////./pipe/containerd-containerd kubeletExtraArgs: - cloud-provider: azure - cloud-config: 'c:/k/azure.json' azure-container-registry-config: 'c:/k/azure.json' + cloud-provider: external pod-infra-container-image: "mcr.microsoft.com/oss/kubernetes/pause:3.4.1" feature-gates: "WindowsHostProcessContainers=true" files: diff --git a/templates/flavors/machinepool/machine-pool-deployment.yaml b/templates/flavors/machinepool/machine-pool-deployment.yaml index cf3e50b6a6b..389be0f30de 100644 --- a/templates/flavors/machinepool/machine-pool-deployment.yaml +++ b/templates/flavors/machinepool/machine-pool-deployment.yaml @@ -50,9 +50,8 @@ spec: nodeRegistration: name: '{{ ds.meta_data["local_hostname"] }}' kubeletExtraArgs: - cloud-provider: azure - cloud-config: /etc/kubernetes/azure.json azure-container-registry-config: /etc/kubernetes/azure.json + cloud-provider: external files: - contentFrom: secret: diff --git a/templates/flavors/nvidia-gpu/machine-deployment.yaml b/templates/flavors/nvidia-gpu/machine-deployment.yaml index 88e97237f23..80917be61ab 100644 --- a/templates/flavors/nvidia-gpu/machine-deployment.yaml +++ b/templates/flavors/nvidia-gpu/machine-deployment.yaml @@ -48,9 +48,8 @@ spec: nodeRegistration: name: '{{ ds.meta_data["local_hostname"] }}' kubeletExtraArgs: - cloud-provider: azure - cloud-config: /etc/kubernetes/azure.json azure-container-registry-config: /etc/kubernetes/azure.json + cloud-provider: external files: - contentFrom: secret: diff --git a/templates/flavors/windows/machine-deployment-windows.yaml b/templates/flavors/windows/machine-deployment-windows.yaml index 98ed536ff79..2deda6c8744 100644 --- a/templates/flavors/windows/machine-deployment-windows.yaml +++ b/templates/flavors/windows/machine-deployment-windows.yaml @@ -63,9 +63,8 @@ spec: name: '{{ ds.meta_data["local_hostname"] }}' criSocket: npipe:////./pipe/containerd-containerd kubeletExtraArgs: - cloud-provider: azure - cloud-config: 'c:/k/azure.json' azure-container-registry-config: 'c:/k/azure.json' + cloud-provider: external feature-gates: "WindowsHostProcessContainers=true" v: "2" windows-priorityclass: "ABOVE_NORMAL_PRIORITY_CLASS" diff --git a/templates/test/ci/cluster-template-prow-azurediskcsi-migration-off.yaml b/templates/test/ci/cluster-template-prow-azurediskcsi-migration-off.yaml deleted file mode 100644 index 083dbc448c7..00000000000 --- a/templates/test/ci/cluster-template-prow-azurediskcsi-migration-off.yaml +++ /dev/null @@ -1,244 +0,0 @@ -apiVersion: cluster.x-k8s.io/v1beta1 -kind: Cluster -metadata: - labels: - ccm: intree - csi: intree - name: ${CLUSTER_NAME} - namespace: default -spec: - clusterNetwork: - pods: - cidrBlocks: - - 192.168.0.0/16 - controlPlaneRef: - apiVersion: controlplane.cluster.x-k8s.io/v1beta1 - kind: KubeadmControlPlane - name: ${CLUSTER_NAME}-control-plane - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureCluster - name: ${CLUSTER_NAME} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureCluster -metadata: - name: ${CLUSTER_NAME} - namespace: default -spec: - additionalTags: - buildProvenance: ${BUILD_PROVENANCE} - creationTimestamp: ${TIMESTAMP} - jobName: ${JOB_NAME} - identityRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureClusterIdentity - name: ${CLUSTER_IDENTITY_NAME} - location: ${AZURE_LOCATION} - networkSpec: - subnets: - - name: control-plane-subnet - role: control-plane - - name: node-subnet - natGateway: - name: node-natgateway - role: node - vnet: - name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} - resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} - subscriptionID: ${AZURE_SUBSCRIPTION_ID} ---- -apiVersion: controlplane.cluster.x-k8s.io/v1beta1 -kind: KubeadmControlPlane -metadata: - name: ${CLUSTER_NAME}-control-plane - namespace: default -spec: - kubeadmConfigSpec: - clusterConfiguration: - apiServer: - extraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - feature-gates: MixedProtocolLBService=true - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true - timeoutForControlPlane: 20m - controllerManager: - extraArgs: - allocate-node-cidrs: "false" - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - cluster-name: ${CLUSTER_NAME} - feature-gates: CSIMigrationAzureDisk=false - v: "4" - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true - etcd: - local: - dataDir: /var/lib/etcddisk/etcd - extraArgs: - quota-backend-bytes: "8589934592" - diskSetup: - filesystems: - - device: /dev/disk/azure/scsi1/lun0 - extraOpts: - - -E - - lazy_itable_init=1,lazy_journal_init=1 - filesystem: ext4 - label: etcd_disk - - device: ephemeral0.1 - filesystem: ext4 - label: ephemeral0 - replaceFS: ntfs - partitions: - - device: /dev/disk/azure/scsi1/lun0 - layout: true - overwrite: false - tableType: gpt - files: - - contentFrom: - secret: - key: control-plane-azure.json - name: ${CLUSTER_NAME}-control-plane-azure-json - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" - initConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - feature-gates: CSIMigrationAzureDisk=false - name: '{{ ds.meta_data["local_hostname"] }}' - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - feature-gates: CSIMigrationAzureDisk=false - name: '{{ ds.meta_data["local_hostname"] }}' - mounts: - - - LABEL=etcd_disk - - /var/lib/etcddisk - postKubeadmCommands: [] - preKubeadmCommands: [] - machineTemplate: - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachineTemplate - name: ${CLUSTER_NAME}-control-plane - replicas: ${CONTROL_PLANE_MACHINE_COUNT} - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachineTemplate -metadata: - name: ${CLUSTER_NAME}-control-plane - namespace: default -spec: - template: - spec: - dataDisks: - - diskSizeGB: 256 - lun: 0 - nameSuffix: etcddisk - identity: UserAssigned - osDisk: - diskSizeGB: 128 - osType: Linux - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - userAssignedIdentities: - - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG:=capz-ci}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY:=cloud-provider-user-identity} - vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachineDeployment -metadata: - name: ${CLUSTER_NAME}-md-0 - namespace: default -spec: - clusterName: ${CLUSTER_NAME} - replicas: ${WORKER_MACHINE_COUNT} - selector: - matchLabels: null - template: - spec: - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfigTemplate - name: ${CLUSTER_NAME}-md-0 - clusterName: ${CLUSTER_NAME} - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachineTemplate - name: ${CLUSTER_NAME}-md-0 - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachineTemplate -metadata: - name: ${CLUSTER_NAME}-md-0 - namespace: default -spec: - template: - spec: - identity: UserAssigned - osDisk: - diskSizeGB: 128 - osType: Linux - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - userAssignedIdentities: - - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG:=capz-ci}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY:=cloud-provider-user-identity} - vmSize: ${AZURE_NODE_MACHINE_TYPE} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfigTemplate -metadata: - name: ${CLUSTER_NAME}-md-0 - namespace: default -spec: - template: - spec: - files: - - contentFrom: - secret: - key: worker-node-azure.json - name: ${CLUSTER_NAME}-md-0-azure-json - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - feature-gates: CSIMigrationAzureDisk=false - name: '{{ ds.meta_data["local_hostname"] }}' - preKubeadmCommands: [] ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureClusterIdentity -metadata: - labels: - clusterctl.cluster.x-k8s.io/move-hierarchy: "true" - name: ${CLUSTER_IDENTITY_NAME} - namespace: default -spec: - allowedNamespaces: {} - clientID: ${AZURE_CLIENT_ID} - clientSecret: - name: ${AZURE_CLUSTER_IDENTITY_SECRET_NAME} - namespace: ${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE} - tenantID: ${AZURE_TENANT_ID} - type: ServicePrincipal diff --git a/templates/test/ci/cluster-template-prow-ci-version-windows-containerd-2022.yaml b/templates/test/ci/cluster-template-prow-ci-version-windows-containerd-2022.yaml index 1b094c2d069..b3a809157dd 100644 --- a/templates/test/ci/cluster-template-prow-ci-version-windows-containerd-2022.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version-windows-containerd-2022.yaml @@ -60,28 +60,16 @@ spec: clusterConfiguration: apiServer: extraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external feature-gates: ${K8S_FEATURE_GATES:-""} - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true timeoutForControlPlane: 20m controllerManager: extraArgs: allocate-node-cidrs: "false" - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} feature-gates: HPAContainerMetrics=true v: "4" - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true etcd: local: dataDir: /var/lib/etcddisk/etcd @@ -180,15 +168,13 @@ spec: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -367,8 +353,7 @@ spec: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: - bash -c /tmp/kubeadm-bootstrap.sh @@ -516,8 +501,7 @@ spec: criSocket: npipe:////./pipe/containerd-containerd kubeletExtraArgs: azure-container-registry-config: c:/k/azure.json - cloud-config: c:/k/azure.json - cloud-provider: azure + cloud-provider: external feature-gates: WindowsHostProcessContainers=true v: "2" windows-priorityclass: ABOVE_NORMAL_PRIORITY_CLASS diff --git a/templates/test/ci/cluster-template-prow-ci-version.yaml b/templates/test/ci/cluster-template-prow-ci-version.yaml index 1aff1651c04..e36abe4ee0e 100644 --- a/templates/test/ci/cluster-template-prow-ci-version.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version.yaml @@ -60,28 +60,16 @@ spec: clusterConfiguration: apiServer: extraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external feature-gates: ${K8S_FEATURE_GATES:-""} - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true timeoutForControlPlane: 20m controllerManager: extraArgs: allocate-node-cidrs: "false" - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} feature-gates: HPAContainerMetrics=true v: "4" - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true etcd: local: dataDir: /var/lib/etcddisk/etcd @@ -180,15 +168,13 @@ spec: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -367,8 +353,7 @@ spec: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: - bash -c /tmp/kubeadm-bootstrap.sh @@ -516,8 +501,7 @@ spec: criSocket: npipe:////./pipe/containerd-containerd kubeletExtraArgs: azure-container-registry-config: c:/k/azure.json - cloud-config: c:/k/azure.json - cloud-provider: azure + cloud-provider: external feature-gates: WindowsHostProcessContainers=true v: "2" windows-priorityclass: ABOVE_NORMAL_PRIORITY_CLASS diff --git a/templates/test/ci/cluster-template-prow-clusterclass-ci-default.yaml b/templates/test/ci/cluster-template-prow-clusterclass-ci-default.yaml index 64950a6d554..b127b0b40f4 100644 --- a/templates/test/ci/cluster-template-prow-clusterclass-ci-default.yaml +++ b/templates/test/ci/cluster-template-prow-clusterclass-ci-default.yaml @@ -380,25 +380,13 @@ spec: clusterConfiguration: apiServer: extraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true + cloud-provider: external timeoutForControlPlane: 20m controllerManager: extraArgs: allocate-node-cidrs: "false" - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true etcd: local: dataDir: /var/lib/etcddisk/etcd @@ -433,15 +421,13 @@ spec: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -512,8 +498,7 @@ spec: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: [] --- @@ -557,8 +542,7 @@ spec: criSocket: npipe:////./pipe/containerd-containerd kubeletExtraArgs: azure-container-registry-config: c:/k/azure.json - cloud-config: c:/k/azure.json - cloud-provider: azure + cloud-provider: external feature-gates: WindowsHostProcessContainers=true v: "2" windows-priorityclass: ABOVE_NORMAL_PRIORITY_CLASS diff --git a/templates/test/ci/cluster-template-prow-custom-vnet.yaml b/templates/test/ci/cluster-template-prow-custom-vnet.yaml index dce49e27b5e..877f8c4521e 100644 --- a/templates/test/ci/cluster-template-prow-custom-vnet.yaml +++ b/templates/test/ci/cluster-template-prow-custom-vnet.yaml @@ -60,26 +60,14 @@ spec: clusterConfiguration: apiServer: extraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true + cloud-provider: external timeoutForControlPlane: 20m controllerManager: extraArgs: allocate-node-cidrs: "false" - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} v: "4" - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true etcd: local: dataDir: /var/lib/etcddisk/etcd @@ -114,15 +102,13 @@ spec: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -221,8 +207,7 @@ spec: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: [] --- diff --git a/templates/test/ci/cluster-template-prow-dual-stack.yaml b/templates/test/ci/cluster-template-prow-dual-stack.yaml index 06e4641ed0a..809444c80a3 100644 --- a/templates/test/ci/cluster-template-prow-dual-stack.yaml +++ b/templates/test/ci/cluster-template-prow-dual-stack.yaml @@ -69,28 +69,16 @@ spec: clusterConfiguration: apiServer: extraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true + cloud-provider: external timeoutForControlPlane: 20m controllerManager: extraArgs: allocate-node-cidrs: "true" - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-cidr: 10.244.0.0/16,2001:1234:5678:9a40::/58 cluster-name: ${CLUSTER_NAME} configure-cloud-routes: "true" v: "4" - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true etcd: local: dataDir: /var/lib/etcddisk/etcd @@ -127,8 +115,7 @@ spec: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: controlPlane: @@ -137,8 +124,7 @@ spec: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -259,8 +245,7 @@ spec: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' postKubeadmCommands: - echo "DNSStubListener=no" >> /etc/systemd/resolved.conf diff --git a/templates/test/ci/cluster-template-prow-edgezone.yaml b/templates/test/ci/cluster-template-prow-edgezone.yaml index 0bede7d68d8..c15c46bab89 100644 --- a/templates/test/ci/cluster-template-prow-edgezone.yaml +++ b/templates/test/ci/cluster-template-prow-edgezone.yaml @@ -56,29 +56,15 @@ spec: clusterConfiguration: apiServer: extraArgs: - cloud-config: /etc/kubernetes/azure.json cloud-provider: external feature-gates: MixedProtocolLBService=true - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true timeoutForControlPlane: 20m controllerManager: extraArgs: allocate-node-cidrs: "false" - cloud-config: /etc/kubernetes/azure.json cloud-provider: external cluster-name: ${CLUSTER_NAME} - external-cloud-volume-plugin: azure - feature-gates: CSIMigrationAzureDisk=true v: "4" - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true etcd: local: dataDir: /var/lib/etcddisk/etcd @@ -113,17 +99,13 @@ spec: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json cloud-provider: external - feature-gates: CSIMigrationAzureDisk=true name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json cloud-provider: external - feature-gates: CSIMigrationAzureDisk=true name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -225,9 +207,7 @@ spec: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json cloud-provider: external - feature-gates: CSIMigrationAzureDisk=true name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: [] --- diff --git a/templates/test/ci/cluster-template-prow-external-azurediskcsi-driver.yaml b/templates/test/ci/cluster-template-prow-external-azurediskcsi-driver.yaml deleted file mode 100644 index 8db47a1ad93..00000000000 --- a/templates/test/ci/cluster-template-prow-external-azurediskcsi-driver.yaml +++ /dev/null @@ -1,244 +0,0 @@ -apiVersion: cluster.x-k8s.io/v1beta1 -kind: Cluster -metadata: - labels: - ccm: intree - csi: external - name: ${CLUSTER_NAME} - namespace: default -spec: - clusterNetwork: - pods: - cidrBlocks: - - 192.168.0.0/16 - controlPlaneRef: - apiVersion: controlplane.cluster.x-k8s.io/v1beta1 - kind: KubeadmControlPlane - name: ${CLUSTER_NAME}-control-plane - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureCluster - name: ${CLUSTER_NAME} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureCluster -metadata: - name: ${CLUSTER_NAME} - namespace: default -spec: - additionalTags: - buildProvenance: ${BUILD_PROVENANCE} - creationTimestamp: ${TIMESTAMP} - jobName: ${JOB_NAME} - identityRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureClusterIdentity - name: ${CLUSTER_IDENTITY_NAME} - location: ${AZURE_LOCATION} - networkSpec: - subnets: - - name: control-plane-subnet - role: control-plane - - name: node-subnet - natGateway: - name: node-natgateway - role: node - vnet: - name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} - resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} - subscriptionID: ${AZURE_SUBSCRIPTION_ID} ---- -apiVersion: controlplane.cluster.x-k8s.io/v1beta1 -kind: KubeadmControlPlane -metadata: - name: ${CLUSTER_NAME}-control-plane - namespace: default -spec: - kubeadmConfigSpec: - clusterConfiguration: - apiServer: - extraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - feature-gates: MixedProtocolLBService=true - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true - timeoutForControlPlane: 20m - controllerManager: - extraArgs: - allocate-node-cidrs: "false" - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - cluster-name: ${CLUSTER_NAME} - feature-gates: CSIMigrationAzureDisk=true - v: "4" - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true - etcd: - local: - dataDir: /var/lib/etcddisk/etcd - extraArgs: - quota-backend-bytes: "8589934592" - diskSetup: - filesystems: - - device: /dev/disk/azure/scsi1/lun0 - extraOpts: - - -E - - lazy_itable_init=1,lazy_journal_init=1 - filesystem: ext4 - label: etcd_disk - - device: ephemeral0.1 - filesystem: ext4 - label: ephemeral0 - replaceFS: ntfs - partitions: - - device: /dev/disk/azure/scsi1/lun0 - layout: true - overwrite: false - tableType: gpt - files: - - contentFrom: - secret: - key: control-plane-azure.json - name: ${CLUSTER_NAME}-control-plane-azure-json - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" - initConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - feature-gates: CSIMigrationAzureDisk=true - name: '{{ ds.meta_data["local_hostname"] }}' - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - feature-gates: CSIMigrationAzureDisk=true - name: '{{ ds.meta_data["local_hostname"] }}' - mounts: - - - LABEL=etcd_disk - - /var/lib/etcddisk - postKubeadmCommands: [] - preKubeadmCommands: [] - machineTemplate: - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachineTemplate - name: ${CLUSTER_NAME}-control-plane - replicas: ${CONTROL_PLANE_MACHINE_COUNT} - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachineTemplate -metadata: - name: ${CLUSTER_NAME}-control-plane - namespace: default -spec: - template: - spec: - dataDisks: - - diskSizeGB: 256 - lun: 0 - nameSuffix: etcddisk - identity: UserAssigned - osDisk: - diskSizeGB: 128 - osType: Linux - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - userAssignedIdentities: - - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG:=capz-ci}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY:=cloud-provider-user-identity} - vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachineDeployment -metadata: - name: ${CLUSTER_NAME}-md-0 - namespace: default -spec: - clusterName: ${CLUSTER_NAME} - replicas: ${WORKER_MACHINE_COUNT} - selector: - matchLabels: null - template: - spec: - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfigTemplate - name: ${CLUSTER_NAME}-md-0 - clusterName: ${CLUSTER_NAME} - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachineTemplate - name: ${CLUSTER_NAME}-md-0 - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachineTemplate -metadata: - name: ${CLUSTER_NAME}-md-0 - namespace: default -spec: - template: - spec: - identity: UserAssigned - osDisk: - diskSizeGB: 128 - osType: Linux - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - userAssignedIdentities: - - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG:=capz-ci}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY:=cloud-provider-user-identity} - vmSize: ${AZURE_NODE_MACHINE_TYPE} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfigTemplate -metadata: - name: ${CLUSTER_NAME}-md-0 - namespace: default -spec: - template: - spec: - files: - - contentFrom: - secret: - key: worker-node-azure.json - name: ${CLUSTER_NAME}-md-0-azure-json - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - feature-gates: CSIMigrationAzureDisk=true - name: '{{ ds.meta_data["local_hostname"] }}' - preKubeadmCommands: [] ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureClusterIdentity -metadata: - labels: - clusterctl.cluster.x-k8s.io/move-hierarchy: "true" - name: ${CLUSTER_IDENTITY_NAME} - namespace: default -spec: - allowedNamespaces: {} - clientID: ${AZURE_CLIENT_ID} - clientSecret: - name: ${AZURE_CLUSTER_IDENTITY_SECRET_NAME} - namespace: ${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE} - tenantID: ${AZURE_TENANT_ID} - type: ServicePrincipal diff --git a/templates/test/ci/cluster-template-prow-external-cloud-provider-vmss-flex.yaml b/templates/test/ci/cluster-template-prow-external-cloud-provider-vmss-flex.yaml deleted file mode 100644 index fc8bd817d09..00000000000 --- a/templates/test/ci/cluster-template-prow-external-cloud-provider-vmss-flex.yaml +++ /dev/null @@ -1,245 +0,0 @@ -apiVersion: cluster.x-k8s.io/v1beta1 -kind: Cluster -metadata: - name: ${CLUSTER_NAME} - namespace: default -spec: - clusterNetwork: - pods: - cidrBlocks: - - 192.168.0.0/16 - controlPlaneRef: - apiVersion: controlplane.cluster.x-k8s.io/v1beta1 - kind: KubeadmControlPlane - name: ${CLUSTER_NAME}-control-plane - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureCluster - name: ${CLUSTER_NAME} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureCluster -metadata: - name: ${CLUSTER_NAME} - namespace: default -spec: - additionalTags: - buildProvenance: ${BUILD_PROVENANCE} - creationTimestamp: ${TIMESTAMP} - jobName: ${JOB_NAME} - identityRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureClusterIdentity - name: ${CLUSTER_IDENTITY_NAME} - location: ${AZURE_LOCATION} - networkSpec: - subnets: - - name: control-plane-subnet - role: control-plane - - name: node-subnet - natGateway: - name: node-natgateway - role: node - vnet: - name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} - resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} - subscriptionID: ${AZURE_SUBSCRIPTION_ID} ---- -apiVersion: controlplane.cluster.x-k8s.io/v1beta1 -kind: KubeadmControlPlane -metadata: - name: ${CLUSTER_NAME}-control-plane - namespace: default -spec: - kubeadmConfigSpec: - clusterConfiguration: - apiServer: - extraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: external - feature-gates: MixedProtocolLBService=true - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true - timeoutForControlPlane: 20m - controllerManager: - extraArgs: - allocate-node-cidrs: "false" - cloud-config: /etc/kubernetes/azure.json - cloud-provider: external - cluster-name: ${CLUSTER_NAME} - external-cloud-volume-plugin: azure - feature-gates: CSIMigrationAzureDisk=true - v: "4" - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true - etcd: - local: - dataDir: /var/lib/etcddisk/etcd - extraArgs: - quota-backend-bytes: "8589934592" - diskSetup: - filesystems: - - device: /dev/disk/azure/scsi1/lun0 - extraOpts: - - -E - - lazy_itable_init=1,lazy_journal_init=1 - filesystem: ext4 - label: etcd_disk - - device: ephemeral0.1 - filesystem: ext4 - label: ephemeral0 - replaceFS: ntfs - partitions: - - device: /dev/disk/azure/scsi1/lun0 - layout: true - overwrite: false - tableType: gpt - files: - - contentFrom: - secret: - key: control-plane-azure.json - name: ${CLUSTER_NAME}-control-plane-azure-json - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" - initConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: external - feature-gates: CSIMigrationAzureDisk=true - name: '{{ ds.meta_data["local_hostname"] }}' - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: external - feature-gates: CSIMigrationAzureDisk=true - name: '{{ ds.meta_data["local_hostname"] }}' - mounts: - - - LABEL=etcd_disk - - /var/lib/etcddisk - postKubeadmCommands: [] - preKubeadmCommands: [] - machineTemplate: - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachineTemplate - name: ${CLUSTER_NAME}-control-plane - replicas: ${CONTROL_PLANE_MACHINE_COUNT} - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachineTemplate -metadata: - name: ${CLUSTER_NAME}-control-plane - namespace: default -spec: - template: - spec: - dataDisks: - - diskSizeGB: 256 - lun: 0 - nameSuffix: etcddisk - identity: UserAssigned - osDisk: - diskSizeGB: 128 - osType: Linux - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - userAssignedIdentities: - - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG:=capz-ci}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY:=cloud-provider-user-identity} - vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachinePool -metadata: - name: ${CLUSTER_NAME}-mp-0 - namespace: default -spec: - clusterName: ${CLUSTER_NAME} - replicas: ${WORKER_MACHINE_COUNT} - template: - spec: - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfig - name: ${CLUSTER_NAME}-mp-0 - clusterName: ${CLUSTER_NAME} - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachinePool - name: ${CLUSTER_NAME}-mp-0 - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachinePool -metadata: - name: ${CLUSTER_NAME}-mp-0 - namespace: default -spec: - identity: UserAssigned - location: ${AZURE_LOCATION} - orchestrationMode: Flexible - strategy: - rollingUpdate: - maxSurge: 0% - maxUnavailable: 0 - type: RollingUpdate - template: - osDisk: - diskSizeGB: 30 - managedDisk: - storageAccountType: Premium_LRS - osType: Linux - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - vmSize: ${AZURE_NODE_MACHINE_TYPE} - userAssignedIdentities: - - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG:=capz-ci}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY:=cloud-provider-user-identity} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfig -metadata: - name: ${CLUSTER_NAME}-mp-0 - namespace: default -spec: - files: - - contentFrom: - secret: - key: worker-node-azure.json - name: ${CLUSTER_NAME}-mp-0-azure-json - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: external - feature-gates: CSIMigrationAzureDisk=true - name: '{{ ds.meta_data["local_hostname"] }}' ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureClusterIdentity -metadata: - labels: - clusterctl.cluster.x-k8s.io/move-hierarchy: "true" - name: ${CLUSTER_IDENTITY_NAME} - namespace: default -spec: - allowedNamespaces: {} - clientID: ${AZURE_CLIENT_ID} - clientSecret: - name: ${AZURE_CLUSTER_IDENTITY_SECRET_NAME} - namespace: ${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE} - tenantID: ${AZURE_TENANT_ID} - type: ServicePrincipal diff --git a/templates/test/ci/cluster-template-prow-external-cloud-provider.yaml b/templates/test/ci/cluster-template-prow-external-cloud-provider.yaml deleted file mode 100644 index 82725efef71..00000000000 --- a/templates/test/ci/cluster-template-prow-external-cloud-provider.yaml +++ /dev/null @@ -1,242 +0,0 @@ -apiVersion: cluster.x-k8s.io/v1beta1 -kind: Cluster -metadata: - name: ${CLUSTER_NAME} - namespace: default -spec: - clusterNetwork: - pods: - cidrBlocks: - - 192.168.0.0/16 - controlPlaneRef: - apiVersion: controlplane.cluster.x-k8s.io/v1beta1 - kind: KubeadmControlPlane - name: ${CLUSTER_NAME}-control-plane - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureCluster - name: ${CLUSTER_NAME} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureCluster -metadata: - name: ${CLUSTER_NAME} - namespace: default -spec: - additionalTags: - buildProvenance: ${BUILD_PROVENANCE} - creationTimestamp: ${TIMESTAMP} - jobName: ${JOB_NAME} - identityRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureClusterIdentity - name: ${CLUSTER_IDENTITY_NAME} - location: ${AZURE_LOCATION} - networkSpec: - subnets: - - name: control-plane-subnet - role: control-plane - - name: node-subnet - natGateway: - name: node-natgateway - role: node - vnet: - name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} - resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} - subscriptionID: ${AZURE_SUBSCRIPTION_ID} ---- -apiVersion: controlplane.cluster.x-k8s.io/v1beta1 -kind: KubeadmControlPlane -metadata: - name: ${CLUSTER_NAME}-control-plane - namespace: default -spec: - kubeadmConfigSpec: - clusterConfiguration: - apiServer: - extraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: external - feature-gates: MixedProtocolLBService=true - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true - timeoutForControlPlane: 20m - controllerManager: - extraArgs: - allocate-node-cidrs: "false" - cloud-config: /etc/kubernetes/azure.json - cloud-provider: external - cluster-name: ${CLUSTER_NAME} - external-cloud-volume-plugin: azure - feature-gates: CSIMigrationAzureDisk=true - v: "4" - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true - etcd: - local: - dataDir: /var/lib/etcddisk/etcd - extraArgs: - quota-backend-bytes: "8589934592" - diskSetup: - filesystems: - - device: /dev/disk/azure/scsi1/lun0 - extraOpts: - - -E - - lazy_itable_init=1,lazy_journal_init=1 - filesystem: ext4 - label: etcd_disk - - device: ephemeral0.1 - filesystem: ext4 - label: ephemeral0 - replaceFS: ntfs - partitions: - - device: /dev/disk/azure/scsi1/lun0 - layout: true - overwrite: false - tableType: gpt - files: - - contentFrom: - secret: - key: control-plane-azure.json - name: ${CLUSTER_NAME}-control-plane-azure-json - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" - initConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: external - feature-gates: CSIMigrationAzureDisk=true - name: '{{ ds.meta_data["local_hostname"] }}' - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: external - feature-gates: CSIMigrationAzureDisk=true - name: '{{ ds.meta_data["local_hostname"] }}' - mounts: - - - LABEL=etcd_disk - - /var/lib/etcddisk - postKubeadmCommands: [] - preKubeadmCommands: [] - machineTemplate: - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachineTemplate - name: ${CLUSTER_NAME}-control-plane - replicas: ${CONTROL_PLANE_MACHINE_COUNT} - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachineTemplate -metadata: - name: ${CLUSTER_NAME}-control-plane - namespace: default -spec: - template: - spec: - dataDisks: - - diskSizeGB: 256 - lun: 0 - nameSuffix: etcddisk - identity: UserAssigned - osDisk: - diskSizeGB: 128 - osType: Linux - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - userAssignedIdentities: - - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG:=capz-ci}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY:=cloud-provider-user-identity} - vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachineDeployment -metadata: - name: ${CLUSTER_NAME}-md-0 - namespace: default -spec: - clusterName: ${CLUSTER_NAME} - replicas: ${WORKER_MACHINE_COUNT} - selector: - matchLabels: null - template: - spec: - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfigTemplate - name: ${CLUSTER_NAME}-md-0 - clusterName: ${CLUSTER_NAME} - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachineTemplate - name: ${CLUSTER_NAME}-md-0 - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachineTemplate -metadata: - name: ${CLUSTER_NAME}-md-0 - namespace: default -spec: - template: - spec: - identity: UserAssigned - osDisk: - diskSizeGB: 128 - osType: Linux - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - userAssignedIdentities: - - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG:=capz-ci}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY:=cloud-provider-user-identity} - vmSize: ${AZURE_NODE_MACHINE_TYPE} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfigTemplate -metadata: - name: ${CLUSTER_NAME}-md-0 - namespace: default -spec: - template: - spec: - files: - - contentFrom: - secret: - key: worker-node-azure.json - name: ${CLUSTER_NAME}-md-0-azure-json - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: external - feature-gates: CSIMigrationAzureDisk=true - name: '{{ ds.meta_data["local_hostname"] }}' - preKubeadmCommands: [] ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureClusterIdentity -metadata: - labels: - clusterctl.cluster.x-k8s.io/move-hierarchy: "true" - name: ${CLUSTER_IDENTITY_NAME} - namespace: default -spec: - allowedNamespaces: {} - clientID: ${AZURE_CLIENT_ID} - clientSecret: - name: ${AZURE_CLUSTER_IDENTITY_SECRET_NAME} - namespace: ${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE} - tenantID: ${AZURE_TENANT_ID} - type: ServicePrincipal diff --git a/templates/test/ci/cluster-template-prow-flatcar.yaml b/templates/test/ci/cluster-template-prow-flatcar.yaml index f682e9100b3..2e8d9c7f732 100644 --- a/templates/test/ci/cluster-template-prow-flatcar.yaml +++ b/templates/test/ci/cluster-template-prow-flatcar.yaml @@ -55,26 +55,14 @@ spec: clusterConfiguration: apiServer: extraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true + cloud-provider: external timeoutForControlPlane: 20m controllerManager: extraArgs: allocate-node-cidrs: "false" - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} v: "4" - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true etcd: local: dataDir: /var/lib/etcddisk/etcd @@ -122,15 +110,13 @@ spec: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '@@HOSTNAME@@' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '@@HOSTNAME@@' mounts: - - etcd_disk @@ -245,8 +231,7 @@ spec: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '@@HOSTNAME@@' postKubeadmCommands: [] preKubeadmCommands: diff --git a/templates/test/ci/cluster-template-prow-identity-from-env.yaml b/templates/test/ci/cluster-template-prow-identity-from-env.yaml deleted file mode 100644 index 2d7d6f6620f..00000000000 --- a/templates/test/ci/cluster-template-prow-identity-from-env.yaml +++ /dev/null @@ -1,4393 +0,0 @@ -apiVersion: cluster.x-k8s.io/v1alpha4 -kind: Cluster -metadata: - labels: - cni: ${CLUSTER_NAME}-calico - name: ${CLUSTER_NAME} - namespace: default -spec: - clusterNetwork: - pods: - cidrBlocks: - - 192.168.0.0/16 - controlPlaneRef: - apiVersion: controlplane.cluster.x-k8s.io/v1alpha4 - kind: KubeadmControlPlane - name: ${CLUSTER_NAME}-control-plane - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4 - kind: AzureCluster - name: ${CLUSTER_NAME} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4 -kind: AzureCluster -metadata: - name: ${CLUSTER_NAME} - namespace: default -spec: - additionalTags: - buildProvenance: ${BUILD_PROVENANCE} - creationTimestamp: ${TIMESTAMP} - jobName: ${JOB_NAME} - location: ${AZURE_LOCATION} - networkSpec: - vnet: - name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} - resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} - subscriptionID: ${AZURE_SUBSCRIPTION_ID} ---- -apiVersion: controlplane.cluster.x-k8s.io/v1alpha4 -kind: KubeadmControlPlane -metadata: - name: ${CLUSTER_NAME}-control-plane - namespace: default -spec: - kubeadmConfigSpec: - clusterConfiguration: - apiServer: - extraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true - timeoutForControlPlane: 20m - controllerManager: - extraArgs: - allocate-node-cidrs: "false" - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - cluster-name: ${CLUSTER_NAME} - v: "4" - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true - etcd: - local: - dataDir: /var/lib/etcddisk/etcd - diskSetup: - filesystems: - - device: /dev/disk/azure/scsi1/lun0 - extraOpts: - - -E - - lazy_itable_init=1,lazy_journal_init=1 - filesystem: ext4 - label: etcd_disk - - device: ephemeral0.1 - filesystem: ext4 - label: ephemeral0 - replaceFS: ntfs - partitions: - - device: /dev/disk/azure/scsi1/lun0 - layout: true - overwrite: false - tableType: gpt - files: - - contentFrom: - secret: - key: control-plane-azure.json - name: ${CLUSTER_NAME}-control-plane-azure-json - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" - initConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - name: '{{ ds.meta_data["local_hostname"] }}' - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - name: '{{ ds.meta_data["local_hostname"] }}' - mounts: - - - LABEL=etcd_disk - - /var/lib/etcddisk - postKubeadmCommands: [] - preKubeadmCommands: [] - machineTemplate: - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4 - kind: AzureMachineTemplate - name: ${CLUSTER_NAME}-control-plane - replicas: ${CONTROL_PLANE_MACHINE_COUNT} - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4 -kind: AzureMachineTemplate -metadata: - name: ${CLUSTER_NAME}-control-plane - namespace: default -spec: - template: - spec: - dataDisks: - - diskSizeGB: 256 - lun: 0 - nameSuffix: etcddisk - osDisk: - diskSizeGB: 128 - osType: Linux - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} ---- -apiVersion: cluster.x-k8s.io/v1alpha4 -kind: MachineDeployment -metadata: - name: ${CLUSTER_NAME}-md-0 - namespace: default -spec: - clusterName: ${CLUSTER_NAME} - replicas: ${WORKER_MACHINE_COUNT} - selector: {} - template: - metadata: - labels: - nodepool: pool1 - spec: - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1alpha4 - kind: KubeadmConfigTemplate - name: ${CLUSTER_NAME}-md-0 - clusterName: ${CLUSTER_NAME} - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4 - kind: AzureMachineTemplate - name: ${CLUSTER_NAME}-md-0 - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4 -kind: AzureMachineTemplate -metadata: - name: ${CLUSTER_NAME}-md-0 - namespace: default -spec: - template: - spec: - osDisk: - diskSizeGB: 128 - osType: Linux - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - vmSize: ${AZURE_NODE_MACHINE_TYPE} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1alpha4 -kind: KubeadmConfigTemplate -metadata: - name: ${CLUSTER_NAME}-md-0 - namespace: default -spec: - template: - spec: - files: - - contentFrom: - secret: - key: worker-node-azure.json - name: ${CLUSTER_NAME}-md-0-azure-json - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - name: '{{ ds.meta_data["local_hostname"] }}' - preKubeadmCommands: [] ---- -apiVersion: cluster.x-k8s.io/v1alpha4 -kind: MachineHealthCheck -metadata: - name: ${CLUSTER_NAME}-mhc-0 - namespace: default -spec: - clusterName: ${CLUSTER_NAME} - maxUnhealthy: 100% - selector: - matchLabels: - nodepool: pool1 - unhealthyConditions: - - status: "True" - timeout: 30s - type: E2ENodeUnhealthy ---- -apiVersion: addons.cluster.x-k8s.io/v1alpha4 -kind: ClusterResourceSet -metadata: - name: ${CLUSTER_NAME}-calico - namespace: default -spec: - clusterSelector: - matchLabels: - cni: ${CLUSTER_NAME}-calico - resources: - - kind: ConfigMap - name: cni-${CLUSTER_NAME}-calico - strategy: ApplyOnce ---- -apiVersion: v1 -data: - resources: | - --- - # Source: calico/templates/calico-config.yaml - # This ConfigMap is used to configure a self-hosted Calico installation. - kind: ConfigMap - apiVersion: v1 - metadata: - name: calico-config - namespace: kube-system - data: - # Typha is disabled. - typha_service_name: "none" - # Configure the backend to use. - calico_backend: "vxlan" - # Configure the MTU to use for workload interfaces and tunnels. - # - If Wireguard is enabled, set to your network MTU - 60 - # - Otherwise, if VXLAN or BPF mode is enabled, set to your network MTU - 50 - # - Otherwise, if IPIP is enabled, set to your network MTU - 20 - # - Otherwise, if not using any encapsulation, set to your network MTU. - veth_mtu: "1440" - - # The CNI network configuration to install on each node. The special - # values in this config will be automatically populated. - cni_network_config: |- - { - "name": "k8s-pod-network", - "cniVersion": "0.3.1", - "plugins": [ - { - "type": "calico", - "log_level": "info", - "log_file_path": "/var/log/calico/cni/cni.log", - "datastore_type": "kubernetes", - "nodename": "__KUBERNETES_NODE_NAME__", - "mtu": __CNI_MTU__, - "ipam": { - "type": "calico-ipam" - }, - "policy": { - "type": "k8s" - }, - "kubernetes": { - "kubeconfig": "__KUBECONFIG_FILEPATH__" - } - }, - { - "type": "portmap", - "snat": true, - "capabilities": {"portMappings": true} - }, - { - "type": "bandwidth", - "capabilities": {"bandwidth": true} - } - ] - } - - --- - # Source: calico/templates/kdd-crds.yaml - - - --- - apiVersion: apiextensions.k8s.io/v1 - kind: CustomResourceDefinition - metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null - name: bgpconfigurations.crd.projectcalico.org - spec: - group: crd.projectcalico.org - names: - kind: BGPConfiguration - listKind: BGPConfigurationList - plural: bgpconfigurations - singular: bgpconfiguration - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - description: BGPConfiguration contains the configuration for any BGP routing. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: BGPConfigurationSpec contains the values of the BGP configuration. - properties: - asNumber: - description: 'ASNumber is the default AS number used by a node. [Default: - 64512]' - format: int32 - type: integer - communities: - description: Communities is a list of BGP community values and their - arbitrary names for tagging routes. - items: - description: Community contains standard or large community value - and its name. - properties: - name: - description: Name given to community value. - type: string - value: - description: Value must be of format `aa:nn` or `aa:nn:mm`. - For standard community use `aa:nn` format, where `aa` and - `nn` are 16 bit number. For large community use `aa:nn:mm` - format, where `aa`, `nn` and `mm` are 32 bit number. Where, - `aa` is an AS Number, `nn` and `mm` are per-AS identifier. - pattern: ^(\d+):(\d+)$|^(\d+):(\d+):(\d+)$ - type: string - type: object - type: array - listenPort: - description: ListenPort is the port where BGP protocol should listen. - Defaults to 179 - maximum: 65535 - minimum: 1 - type: integer - logSeverityScreen: - description: 'LogSeverityScreen is the log severity above which logs - are sent to the stdout. [Default: INFO]' - type: string - nodeToNodeMeshEnabled: - description: 'NodeToNodeMeshEnabled sets whether full node to node - BGP mesh is enabled. [Default: true]' - type: boolean - prefixAdvertisements: - description: PrefixAdvertisements contains per-prefix advertisement - configuration. - items: - description: PrefixAdvertisement configures advertisement properties - for the specified CIDR. - properties: - cidr: - description: CIDR for which properties should be advertised. - type: string - communities: - description: Communities can be list of either community names - already defined in `Specs.Communities` or community value - of format `aa:nn` or `aa:nn:mm`. For standard community use - `aa:nn` format, where `aa` and `nn` are 16 bit number. For - large community use `aa:nn:mm` format, where `aa`, `nn` and - `mm` are 32 bit number. Where,`aa` is an AS Number, `nn` and - `mm` are per-AS identifier. - items: - type: string - type: array - type: object - type: array - serviceClusterIPs: - description: ServiceClusterIPs are the CIDR blocks from which service - cluster IPs are allocated. If specified, Calico will advertise these - blocks, as well as any cluster IPs within them. - items: - description: ServiceClusterIPBlock represents a single allowed ClusterIP - CIDR block. - properties: - cidr: - type: string - type: object - type: array - serviceExternalIPs: - description: ServiceExternalIPs are the CIDR blocks for Kubernetes - Service External IPs. Kubernetes Service ExternalIPs will only be - advertised if they are within one of these blocks. - items: - description: ServiceExternalIPBlock represents a single allowed - External IP CIDR block. - properties: - cidr: - type: string - type: object - type: array - serviceLoadBalancerIPs: - description: ServiceLoadBalancerIPs are the CIDR blocks for Kubernetes - Service LoadBalancer IPs. Kubernetes Service status.LoadBalancer.Ingress - IPs will only be advertised if they are within one of these blocks. - items: - description: ServiceLoadBalancerIPBlock represents a single allowed - LoadBalancer IP CIDR block. - properties: - cidr: - type: string - type: object - type: array - type: object - type: object - served: true - storage: true - status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - - --- - - --- - apiVersion: apiextensions.k8s.io/v1 - kind: CustomResourceDefinition - metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null - name: bgppeers.crd.projectcalico.org - spec: - group: crd.projectcalico.org - names: - kind: BGPPeer - listKind: BGPPeerList - plural: bgppeers - singular: bgppeer - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: BGPPeerSpec contains the specification for a BGPPeer resource. - properties: - asNumber: - description: The AS Number of the peer. - format: int32 - type: integer - keepOriginalNextHop: - description: Option to keep the original nexthop field when routes - are sent to a BGP Peer. Setting "true" configures the selected BGP - Peers node to use the "next hop keep;" instead of "next hop self;"(default) - in the specific branch of the Node on "bird.cfg". - type: boolean - maxRestartTime: - description: Time to allow for software restart. When specified, this - is configured as the graceful restart timeout. When not specified, - the BIRD default of 120s is used. - type: string - node: - description: The node name identifying the Calico node instance that - is targeted by this peer. If this is not set, and no nodeSelector - is specified, then this BGP peer selects all nodes in the cluster. - type: string - nodeSelector: - description: Selector for the nodes that should have this peering. When - this is set, the Node field must be empty. - type: string - password: - description: Optional BGP password for the peerings generated by this - BGPPeer resource. - properties: - secretKeyRef: - description: Selects a key of a secret in the node pod's namespace. - properties: - key: - description: The key of the secret to select from. Must be - a valid secret key. - type: string - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' - type: string - optional: - description: Specify whether the Secret or its key must be - defined - type: boolean - required: - - key - type: object - type: object - peerIP: - description: The IP address of the peer followed by an optional port - number to peer with. If port number is given, format should be `[]:port` - or `:` for IPv4. If optional port number is not set, - and this peer IP and ASNumber belongs to a calico/node with ListenPort - set in BGPConfiguration, then we use that port to peer. - type: string - peerSelector: - description: Selector for the remote nodes to peer with. When this - is set, the PeerIP and ASNumber fields must be empty. For each - peering between the local node and selected remote nodes, we configure - an IPv4 peering if both ends have NodeBGPSpec.IPv4Address specified, - and an IPv6 peering if both ends have NodeBGPSpec.IPv6Address specified. The - remote AS number comes from the remote node’s NodeBGPSpec.ASNumber, - or the global default if that is not set. - type: string - sourceAddress: - description: Specifies whether and how to configure a source address - for the peerings generated by this BGPPeer resource. Default value - "UseNodeIP" means to configure the node IP as the source address. "None" - means not to configure a source address. - type: string - type: object - type: object - served: true - storage: true - status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - - --- - - --- - apiVersion: apiextensions.k8s.io/v1 - kind: CustomResourceDefinition - metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null - name: blockaffinities.crd.projectcalico.org - spec: - group: crd.projectcalico.org - names: - kind: BlockAffinity - listKind: BlockAffinityList - plural: blockaffinities - singular: blockaffinity - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: BlockAffinitySpec contains the specification for a BlockAffinity - resource. - properties: - cidr: - type: string - deleted: - description: Deleted indicates that this block affinity is being deleted. - This field is a string for compatibility with older releases that - mistakenly treat this field as a string. - type: string - node: - type: string - state: - type: string - required: - - cidr - - deleted - - node - - state - type: object - type: object - served: true - storage: true - status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - - --- - - --- - apiVersion: apiextensions.k8s.io/v1 - kind: CustomResourceDefinition - metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null - name: clusterinformations.crd.projectcalico.org - spec: - group: crd.projectcalico.org - names: - kind: ClusterInformation - listKind: ClusterInformationList - plural: clusterinformations - singular: clusterinformation - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - description: ClusterInformation contains the cluster specific information. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: ClusterInformationSpec contains the values of describing - the cluster. - properties: - calicoVersion: - description: CalicoVersion is the version of Calico that the cluster - is running - type: string - clusterGUID: - description: ClusterGUID is the GUID of the cluster - type: string - clusterType: - description: ClusterType describes the type of the cluster - type: string - datastoreReady: - description: DatastoreReady is used during significant datastore migrations - to signal to components such as Felix that it should wait before - accessing the datastore. - type: boolean - variant: - description: Variant declares which variant of Calico should be active. - type: string - type: object - type: object - served: true - storage: true - status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - - --- - - --- - apiVersion: apiextensions.k8s.io/v1 - kind: CustomResourceDefinition - metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null - name: felixconfigurations.crd.projectcalico.org - spec: - group: crd.projectcalico.org - names: - kind: FelixConfiguration - listKind: FelixConfigurationList - plural: felixconfigurations - singular: felixconfiguration - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - description: Felix Configuration contains the configuration for Felix. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: FelixConfigurationSpec contains the values of the Felix configuration. - properties: - allowIPIPPacketsFromWorkloads: - description: 'AllowIPIPPacketsFromWorkloads controls whether Felix - will add a rule to drop IPIP encapsulated traffic from workloads - [Default: false]' - type: boolean - allowVXLANPacketsFromWorkloads: - description: 'AllowVXLANPacketsFromWorkloads controls whether Felix - will add a rule to drop VXLAN encapsulated traffic from workloads - [Default: false]' - type: boolean - awsSrcDstCheck: - description: 'Set source-destination-check on AWS EC2 instances. Accepted - value must be one of "DoNothing", "Enabled" or "Disabled". [Default: - DoNothing]' - enum: - - DoNothing - - Enable - - Disable - type: string - bpfConnectTimeLoadBalancingEnabled: - description: 'BPFConnectTimeLoadBalancingEnabled when in BPF mode, - controls whether Felix installs the connection-time load balancer. The - connect-time load balancer is required for the host to be able to - reach Kubernetes services and it improves the performance of pod-to-service - connections. The only reason to disable it is for debugging purposes. [Default: - true]' - type: boolean - bpfDataIfacePattern: - description: 'BPFDataIfacePattern is a regular expression that controls - which interfaces Felix should attach BPF programs to in order to - catch traffic to/from the network. This needs to match the interfaces - that Calico workload traffic flows over as well as any interfaces - that handle incoming traffic to nodeports and services from outside - the cluster. It should not match the workload interfaces (usually - named cali...). [Default: ^(en.*|eth.*|tunl0$)]' - type: string - bpfDisableUnprivileged: - description: 'BPFDisableUnprivileged, if enabled, Felix sets the kernel.unprivileged_bpf_disabled - sysctl to disable unprivileged use of BPF. This ensures that unprivileged - users cannot access Calico''s BPF maps and cannot insert their own - BPF programs to interfere with Calico''s. [Default: true]' - type: boolean - bpfEnabled: - description: 'BPFEnabled, if enabled Felix will use the BPF dataplane. - [Default: false]' - type: boolean - bpfExtToServiceConnmark: - description: 'BPFExtToServiceConnmark in BPF mode, control a 32bit - mark that is set on connections from an external client to a local - service. This mark allows us to control how packets of that connection - are routed within the host and how is routing interpreted by RPF - check. [Default: 0]' - type: integer - bpfExternalServiceMode: - description: 'BPFExternalServiceMode in BPF mode, controls how connections - from outside the cluster to services (node ports and cluster IPs) - are forwarded to remote workloads. If set to "Tunnel" then both - request and response traffic is tunneled to the remote node. If - set to "DSR", the request traffic is tunneled but the response traffic - is sent directly from the remote node. In "DSR" mode, the remote - node appears to use the IP of the ingress node; this requires a - permissive L2 network. [Default: Tunnel]' - type: string - bpfKubeProxyEndpointSlicesEnabled: - description: BPFKubeProxyEndpointSlicesEnabled in BPF mode, controls - whether Felix's embedded kube-proxy accepts EndpointSlices or not. - type: boolean - bpfKubeProxyIptablesCleanupEnabled: - description: 'BPFKubeProxyIptablesCleanupEnabled, if enabled in BPF - mode, Felix will proactively clean up the upstream Kubernetes kube-proxy''s - iptables chains. Should only be enabled if kube-proxy is not running. [Default: - true]' - type: boolean - bpfKubeProxyMinSyncPeriod: - description: 'BPFKubeProxyMinSyncPeriod, in BPF mode, controls the - minimum time between updates to the dataplane for Felix''s embedded - kube-proxy. Lower values give reduced set-up latency. Higher values - reduce Felix CPU usage by batching up more work. [Default: 1s]' - type: string - bpfLogLevel: - description: 'BPFLogLevel controls the log level of the BPF programs - when in BPF dataplane mode. One of "Off", "Info", or "Debug". The - logs are emitted to the BPF trace pipe, accessible with the command - `tc exec bpf debug`. [Default: Off].' - type: string - chainInsertMode: - description: 'ChainInsertMode controls whether Felix hooks the kernel’s - top-level iptables chains by inserting a rule at the top of the - chain or by appending a rule at the bottom. insert is the safe default - since it prevents Calico’s rules from being bypassed. If you switch - to append mode, be sure that the other rules in the chains signal - acceptance by falling through to the Calico rules, otherwise the - Calico policy will be bypassed. [Default: insert]' - type: string - dataplaneDriver: - type: string - debugDisableLogDropping: - type: boolean - debugMemoryProfilePath: - type: string - debugSimulateCalcGraphHangAfter: - type: string - debugSimulateDataplaneHangAfter: - type: string - defaultEndpointToHostAction: - description: 'DefaultEndpointToHostAction controls what happens to - traffic that goes from a workload endpoint to the host itself (after - the traffic hits the endpoint egress policy). By default Calico - blocks traffic from workload endpoints to the host itself with an - iptables “DROP” action. If you want to allow some or all traffic - from endpoint to host, set this parameter to RETURN or ACCEPT. Use - RETURN if you have your own rules in the iptables “INPUT” chain; - Calico will insert its rules at the top of that chain, then “RETURN” - packets to the “INPUT” chain once it has completed processing workload - endpoint egress policy. Use ACCEPT to unconditionally accept packets - from workloads after processing workload endpoint egress policy. - [Default: Drop]' - type: string - deviceRouteProtocol: - description: This defines the route protocol added to programmed device - routes, by default this will be RTPROT_BOOT when left blank. - type: integer - deviceRouteSourceAddress: - description: This is the source address to use on programmed device - routes. By default the source address is left blank, leaving the - kernel to choose the source address used. - type: string - disableConntrackInvalidCheck: - type: boolean - endpointReportingDelay: - type: string - endpointReportingEnabled: - type: boolean - externalNodesList: - description: ExternalNodesCIDRList is a list of CIDR's of external-non-calico-nodes - which may source tunnel traffic and have the tunneled traffic be - accepted at calico nodes. - items: - type: string - type: array - failsafeInboundHostPorts: - description: 'FailsafeInboundHostPorts is a list of UDP/TCP ports - and CIDRs that Felix will allow incoming traffic to host endpoints - on irrespective of the security policy. This is useful to avoid - accidentally cutting off a host with incorrect configuration. For - back-compatibility, if the protocol is not specified, it defaults - to "tcp". If a CIDR is not specified, it will allow traffic from - all addresses. To disable all inbound host ports, use the value - none. The default value allows ssh access and DHCP. [Default: tcp:22, - udp:68, tcp:179, tcp:2379, tcp:2380, tcp:6443, tcp:6666, tcp:6667]' - items: - description: ProtoPort is combination of protocol, port, and CIDR. - Protocol and port must be specified. - properties: - net: - type: string - port: - type: integer - protocol: - type: string - required: - - port - - protocol - type: object - type: array - failsafeOutboundHostPorts: - description: 'FailsafeOutboundHostPorts is a list of UDP/TCP ports - and CIDRs that Felix will allow outgoing traffic from host endpoints - to irrespective of the security policy. This is useful to avoid - accidentally cutting off a host with incorrect configuration. For - back-compatibility, if the protocol is not specified, it defaults - to "tcp". If a CIDR is not specified, it will allow traffic from - all addresses. To disable all outbound host ports, use the value - none. The default value opens etcd''s standard ports to ensure that - Felix does not get cut off from etcd as well as allowing DHCP and - DNS. [Default: tcp:179, tcp:2379, tcp:2380, tcp:6443, tcp:6666, - tcp:6667, udp:53, udp:67]' - items: - description: ProtoPort is combination of protocol, port, and CIDR. - Protocol and port must be specified. - properties: - net: - type: string - port: - type: integer - protocol: - type: string - required: - - port - - protocol - type: object - type: array - featureDetectOverride: - description: FeatureDetectOverride is used to override the feature - detection. Values are specified in a comma separated list with no - spaces, example; "SNATFullyRandom=true,MASQFullyRandom=false,RestoreSupportsLock=". - "true" or "false" will force the feature, empty or omitted values - are auto-detected. - type: string - genericXDPEnabled: - description: 'GenericXDPEnabled enables Generic XDP so network cards - that don''t support XDP offload or driver modes can use XDP. This - is not recommended since it doesn''t provide better performance - than iptables. [Default: false]' - type: boolean - healthEnabled: - type: boolean - healthHost: - type: string - healthPort: - type: integer - interfaceExclude: - description: 'InterfaceExclude is a comma-separated list of interfaces - that Felix should exclude when monitoring for host endpoints. The - default value ensures that Felix ignores Kubernetes'' IPVS dummy - interface, which is used internally by kube-proxy. If you want to - exclude multiple interface names using a single value, the list - supports regular expressions. For regular expressions you must wrap - the value with ''/''. For example having values ''/^kube/,veth1'' - will exclude all interfaces that begin with ''kube'' and also the - interface ''veth1''. [Default: kube-ipvs0]' - type: string - interfacePrefix: - description: 'InterfacePrefix is the interface name prefix that identifies - workload endpoints and so distinguishes them from host endpoint - interfaces. Note: in environments other than bare metal, the orchestrators - configure this appropriately. For example our Kubernetes and Docker - integrations set the ‘cali’ value, and our OpenStack integration - sets the ‘tap’ value. [Default: cali]' - type: string - interfaceRefreshInterval: - description: InterfaceRefreshInterval is the period at which Felix - rescans local interfaces to verify their state. The rescan can be - disabled by setting the interval to 0. - type: string - ipipEnabled: - type: boolean - ipipMTU: - description: 'IPIPMTU is the MTU to set on the tunnel device. See - Configuring MTU [Default: 1440]' - type: integer - ipsetsRefreshInterval: - description: 'IpsetsRefreshInterval is the period at which Felix re-checks - all iptables state to ensure that no other process has accidentally - broken Calico’s rules. Set to 0 to disable iptables refresh. [Default: - 90s]' - type: string - iptablesBackend: - description: IptablesBackend specifies which backend of iptables will - be used. The default is legacy. - type: string - iptablesFilterAllowAction: - type: string - iptablesLockFilePath: - description: 'IptablesLockFilePath is the location of the iptables - lock file. You may need to change this if the lock file is not in - its standard location (for example if you have mapped it into Felix’s - container at a different path). [Default: /run/xtables.lock]' - type: string - iptablesLockProbeInterval: - description: 'IptablesLockProbeInterval is the time that Felix will - wait between attempts to acquire the iptables lock if it is not - available. Lower values make Felix more responsive when the lock - is contended, but use more CPU. [Default: 50ms]' - type: string - iptablesLockTimeout: - description: 'IptablesLockTimeout is the time that Felix will wait - for the iptables lock, or 0, to disable. To use this feature, Felix - must share the iptables lock file with all other processes that - also take the lock. When running Felix inside a container, this - requires the /run directory of the host to be mounted into the calico/node - or calico/felix container. [Default: 0s disabled]' - type: string - iptablesMangleAllowAction: - type: string - iptablesMarkMask: - description: 'IptablesMarkMask is the mask that Felix selects its - IPTables Mark bits from. Should be a 32 bit hexadecimal number with - at least 8 bits set, none of which clash with any other mark bits - in use on the system. [Default: 0xff000000]' - format: int32 - type: integer - iptablesNATOutgoingInterfaceFilter: - type: string - iptablesPostWriteCheckInterval: - description: 'IptablesPostWriteCheckInterval is the period after Felix - has done a write to the dataplane that it schedules an extra read - back in order to check the write was not clobbered by another process. - This should only occur if another application on the system doesn’t - respect the iptables lock. [Default: 1s]' - type: string - iptablesRefreshInterval: - description: 'IptablesRefreshInterval is the period at which Felix - re-checks the IP sets in the dataplane to ensure that no other process - has accidentally broken Calico''s rules. Set to 0 to disable IP - sets refresh. Note: the default for this value is lower than the - other refresh intervals as a workaround for a Linux kernel bug that - was fixed in kernel version 4.11. If you are using v4.11 or greater - you may want to set this to, a higher value to reduce Felix CPU - usage. [Default: 10s]' - type: string - ipv6Support: - type: boolean - kubeNodePortRanges: - description: 'KubeNodePortRanges holds list of port ranges used for - service node ports. Only used if felix detects kube-proxy running - in ipvs mode. Felix uses these ranges to separate host and workload - traffic. [Default: 30000:32767].' - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - logFilePath: - description: 'LogFilePath is the full path to the Felix log. Set to - none to disable file logging. [Default: /var/log/calico/felix.log]' - type: string - logPrefix: - description: 'LogPrefix is the log prefix that Felix uses when rendering - LOG rules. [Default: calico-packet]' - type: string - logSeverityFile: - description: 'LogSeverityFile is the log severity above which logs - are sent to the log file. [Default: Info]' - type: string - logSeverityScreen: - description: 'LogSeverityScreen is the log severity above which logs - are sent to the stdout. [Default: Info]' - type: string - logSeveritySys: - description: 'LogSeveritySys is the log severity above which logs - are sent to the syslog. Set to None for no logging to syslog. [Default: - Info]' - type: string - maxIpsetSize: - type: integer - metadataAddr: - description: 'MetadataAddr is the IP address or domain name of the - server that can answer VM queries for cloud-init metadata. In OpenStack, - this corresponds to the machine running nova-api (or in Ubuntu, - nova-api-metadata). A value of none (case insensitive) means that - Felix should not set up any NAT rule for the metadata path. [Default: - 127.0.0.1]' - type: string - metadataPort: - description: 'MetadataPort is the port of the metadata server. This, - combined with global.MetadataAddr (if not ‘None’), is used to set - up a NAT rule, from 169.254.169.254:80 to MetadataAddr:MetadataPort. - In most cases this should not need to be changed [Default: 8775].' - type: integer - mtuIfacePattern: - description: MTUIfacePattern is a regular expression that controls - which interfaces Felix should scan in order to calculate the host's - MTU. This should not match workload interfaces (usually named cali...). - type: string - natOutgoingAddress: - description: NATOutgoingAddress specifies an address to use when performing - source NAT for traffic in a natOutgoing pool that is leaving the - network. By default the address used is an address on the interface - the traffic is leaving on (ie it uses the iptables MASQUERADE target) - type: string - natPortRange: - anyOf: - - type: integer - - type: string - description: NATPortRange specifies the range of ports that is used - for port mapping when doing outgoing NAT. When unset the default - behavior of the network stack is used. - pattern: ^.* - x-kubernetes-int-or-string: true - netlinkTimeout: - type: string - openstackRegion: - description: 'OpenstackRegion is the name of the region that a particular - Felix belongs to. In a multi-region Calico/OpenStack deployment, - this must be configured somehow for each Felix (here in the datamodel, - or in felix.cfg or the environment on each compute node), and must - match the [calico] openstack_region value configured in neutron.conf - on each node. [Default: Empty]' - type: string - policySyncPathPrefix: - description: 'PolicySyncPathPrefix is used to by Felix to communicate - policy changes to external services, like Application layer policy. - [Default: Empty]' - type: string - prometheusGoMetricsEnabled: - description: 'PrometheusGoMetricsEnabled disables Go runtime metrics - collection, which the Prometheus client does by default, when set - to false. This reduces the number of metrics reported, reducing - Prometheus load. [Default: true]' - type: boolean - prometheusMetricsEnabled: - description: 'PrometheusMetricsEnabled enables the Prometheus metrics - server in Felix if set to true. [Default: false]' - type: boolean - prometheusMetricsHost: - description: 'PrometheusMetricsHost is the host that the Prometheus - metrics server should bind to. [Default: empty]' - type: string - prometheusMetricsPort: - description: 'PrometheusMetricsPort is the TCP port that the Prometheus - metrics server should bind to. [Default: 9091]' - type: integer - prometheusProcessMetricsEnabled: - description: 'PrometheusProcessMetricsEnabled disables process metrics - collection, which the Prometheus client does by default, when set - to false. This reduces the number of metrics reported, reducing - Prometheus load. [Default: true]' - type: boolean - removeExternalRoutes: - description: Whether or not to remove device routes that have not - been programmed by Felix. Disabling this will allow external applications - to also add device routes. This is enabled by default which means - we will remove externally added routes. - type: boolean - reportingInterval: - description: 'ReportingInterval is the interval at which Felix reports - its status into the datastore or 0 to disable. Must be non-zero - in OpenStack deployments. [Default: 30s]' - type: string - reportingTTL: - description: 'ReportingTTL is the time-to-live setting for process-wide - status reports. [Default: 90s]' - type: string - routeRefreshInterval: - description: 'RouterefreshInterval is the period at which Felix re-checks - the routes in the dataplane to ensure that no other process has - accidentally broken Calico’s rules. Set to 0 to disable route refresh. - [Default: 90s]' - type: string - routeSource: - description: 'RouteSource configures where Felix gets its routing - information. - WorkloadIPs: use workload endpoints to construct - routes. - CalicoIPAM: the default - use IPAM data to construct routes.' - type: string - routeTableRange: - description: Calico programs additional Linux route tables for various - purposes. RouteTableRange specifies the indices of the route tables - that Calico should use. - properties: - max: - type: integer - min: - type: integer - required: - - max - - min - type: object - serviceLoopPrevention: - description: 'When service IP advertisement is enabled, prevent routing - loops to service IPs that are not in use, by dropping or rejecting - packets that do not get DNAT''d by kube-proxy. Unless set to "Disabled", - in which case such routing loops continue to be allowed. [Default: - Drop]' - type: string - sidecarAccelerationEnabled: - description: 'SidecarAccelerationEnabled enables experimental sidecar - acceleration [Default: false]' - type: boolean - usageReportingEnabled: - description: 'UsageReportingEnabled reports anonymous Calico version - number and cluster size to projectcalico.org. Logs warnings returned - by the usage server. For example, if a significant security vulnerability - has been discovered in the version of Calico being used. [Default: - true]' - type: boolean - usageReportingInitialDelay: - description: 'UsageReportingInitialDelay controls the minimum delay - before Felix makes a report. [Default: 300s]' - type: string - usageReportingInterval: - description: 'UsageReportingInterval controls the interval at which - Felix makes reports. [Default: 86400s]' - type: string - useInternalDataplaneDriver: - type: boolean - vxlanEnabled: - type: boolean - vxlanMTU: - description: 'VXLANMTU is the MTU to set on the tunnel device. See - Configuring MTU [Default: 1440]' - type: integer - vxlanPort: - type: integer - vxlanVNI: - type: integer - wireguardEnabled: - description: 'WireguardEnabled controls whether Wireguard is enabled. - [Default: false]' - type: boolean - wireguardInterfaceName: - description: 'WireguardInterfaceName specifies the name to use for - the Wireguard interface. [Default: wg.calico]' - type: string - wireguardListeningPort: - description: 'WireguardListeningPort controls the listening port used - by Wireguard. [Default: 51820]' - type: integer - wireguardMTU: - description: 'WireguardMTU controls the MTU on the Wireguard interface. - See Configuring MTU [Default: 1420]' - type: integer - wireguardRoutingRulePriority: - description: 'WireguardRoutingRulePriority controls the priority value - to use for the Wireguard routing rule. [Default: 99]' - type: integer - xdpEnabled: - description: 'XDPEnabled enables XDP acceleration for suitable untracked - incoming deny rules. [Default: true]' - type: boolean - xdpRefreshInterval: - description: 'XDPRefreshInterval is the period at which Felix re-checks - all XDP state to ensure that no other process has accidentally broken - Calico''s BPF maps or attached programs. Set to 0 to disable XDP - refresh. [Default: 90s]' - type: string - type: object - type: object - served: true - storage: true - status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - - --- - - --- - apiVersion: apiextensions.k8s.io/v1 - kind: CustomResourceDefinition - metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null - name: globalnetworkpolicies.crd.projectcalico.org - spec: - group: crd.projectcalico.org - names: - kind: GlobalNetworkPolicy - listKind: GlobalNetworkPolicyList - plural: globalnetworkpolicies - singular: globalnetworkpolicy - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - properties: - applyOnForward: - description: ApplyOnForward indicates to apply the rules in this policy - on forward traffic. - type: boolean - doNotTrack: - description: DoNotTrack indicates whether packets matched by the rules - in this policy should go through the data plane's connection tracking, - such as Linux conntrack. If True, the rules in this policy are - applied before any data plane connection tracking, and packets allowed - by this policy are marked as not to be tracked. - type: boolean - egress: - description: The ordered set of egress rules. Each rule contains - a set of packet match criteria and a corresponding action to apply. - items: - description: "A Rule encapsulates a set of match criteria and an - action. Both selector-based security Policy and security Profiles - reference rules - separated out as a list of rules for both ingress - and egress packet matching. \n Each positive match criteria has - a negated version, prefixed with ”Not”. All the match criteria - within a rule must be satisfied for a packet to match. A single - rule can contain the positive and negative version of a match - and both must be satisfied for the rule to match." - properties: - action: - type: string - destination: - description: Destination contains the match criteria that apply - to destination entity. - properties: - namespaceSelector: - description: "NamespaceSelector is an optional field that - contains a selector expression. Only traffic that originates - from (or terminates at) endpoints within the selected - namespaces will be matched. When both NamespaceSelector - and Selector are defined on the same rule, then only workload - endpoints that are matched by both selectors will be selected - by the rule. \n For NetworkPolicy, an empty NamespaceSelector - implies that the Selector is limited to selecting only - workload endpoints in the same namespace as the NetworkPolicy. - \n For NetworkPolicy, `global()` NamespaceSelector implies - that the Selector is limited to selecting only GlobalNetworkSet - or HostEndpoint. \n For GlobalNetworkPolicy, an empty - NamespaceSelector implies the Selector applies to workload - endpoints across all namespaces." - type: string - nets: - description: Nets is an optional field that restricts the - rule to only apply to traffic that originates from (or - terminates at) IP addresses in any of the given subnets. - items: - type: string - type: array - notNets: - description: NotNets is the negated version of the Nets - field. - items: - type: string - type: array - notPorts: - description: NotPorts is the negated version of the Ports - field. Since only some protocols have ports, if any ports - are specified it requires the Protocol match in the Rule - to be set to "TCP" or "UDP". - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - notSelector: - description: NotSelector is the negated version of the Selector - field. See Selector field for subtleties with negated - selectors. - type: string - ports: - description: "Ports is an optional field that restricts - the rule to only apply to traffic that has a source (destination) - port that matches one of these ranges/values. This value - is a list of integers or strings that represent ranges - of ports. \n Since only some protocols have ports, if - any ports are specified it requires the Protocol match - in the Rule to be set to \"TCP\" or \"UDP\"." - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - selector: - description: "Selector is an optional field that contains - a selector expression (see Policy for sample syntax). - \ Only traffic that originates from (terminates at) endpoints - matching the selector will be matched. \n Note that: in - addition to the negated version of the Selector (see NotSelector - below), the selector expression syntax itself supports - negation. The two types of negation are subtly different. - One negates the set of matched endpoints, the other negates - the whole match: \n \tSelector = \"!has(my_label)\" matches - packets that are from other Calico-controlled \tendpoints - that do not have the label “my_label”. \n \tNotSelector - = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label “my_label”. - \n The effect is that the latter will accept packets from - non-Calico sources whereas the former is limited to packets - from Calico-controlled endpoints." - type: string - serviceAccounts: - description: ServiceAccounts is an optional field that restricts - the rule to only apply to traffic that originates from - (or terminates at) a pod running as a matching service - account. - properties: - names: - description: Names is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account whose name is in the list. - items: - type: string - type: array - selector: - description: Selector is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account that matches the given label selector. If - both Names and Selector are specified then they are - AND'ed. - type: string - type: object - services: - description: "Services is an optional field that contains - options for matching Kubernetes Services. If specified, - only traffic that originates from or terminates at endpoints - within the selected service(s) will be matched, and only - to/from each endpoint's port. \n Services cannot be specified - on the same rule as Selector, NotSelector, NamespaceSelector, - Ports, NotPorts, Nets, NotNets or ServiceAccounts. \n - Only valid on egress rules." - properties: - name: - description: Name specifies the name of a Kubernetes - Service to match. - type: string - namespace: - description: Namespace specifies the namespace of the - given Service. If left empty, the rule will match - within this policy's namespace. - type: string - type: object - type: object - http: - description: HTTP contains match criteria that apply to HTTP - requests. - properties: - methods: - description: Methods is an optional field that restricts - the rule to apply only to HTTP requests that use one of - the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple - methods are OR'd together. - items: - type: string - type: array - paths: - description: 'Paths is an optional field that restricts - the rule to apply to HTTP requests that use one of the - listed HTTP Paths. Multiple paths are OR''d together. - e.g: - exact: /foo - prefix: /bar NOTE: Each entry may - ONLY specify either a `exact` or a `prefix` match. The - validator will check for it.' - items: - description: 'HTTPPath specifies an HTTP path to match. - It may be either of the form: exact: : which matches - the path exactly or prefix: : which matches - the path prefix' - properties: - exact: - type: string - prefix: - type: string - type: object - type: array - type: object - icmp: - description: ICMP is an optional field that restricts the rule - to apply to a specific type and code of ICMP traffic. This - should only be specified if the Protocol field is set to "ICMP" - or "ICMPv6". - properties: - code: - description: Match on a specific ICMP code. If specified, - the Type value must also be specified. This is a technical - limitation imposed by the kernel’s iptables firewall, - which Calico uses to enforce the rule. - type: integer - type: - description: Match on a specific ICMP type. For example - a value of 8 refers to ICMP Echo Request (i.e. pings). - type: integer - type: object - ipVersion: - description: IPVersion is an optional field that restricts the - rule to only match a specific IP version. - type: integer - metadata: - description: Metadata contains additional information for this - rule - properties: - annotations: - additionalProperties: - type: string - description: Annotations is a set of key value pairs that - give extra information about the rule - type: object - type: object - notICMP: - description: NotICMP is the negated version of the ICMP field. - properties: - code: - description: Match on a specific ICMP code. If specified, - the Type value must also be specified. This is a technical - limitation imposed by the kernel’s iptables firewall, - which Calico uses to enforce the rule. - type: integer - type: - description: Match on a specific ICMP type. For example - a value of 8 refers to ICMP Echo Request (i.e. pings). - type: integer - type: object - notProtocol: - anyOf: - - type: integer - - type: string - description: NotProtocol is the negated version of the Protocol - field. - pattern: ^.* - x-kubernetes-int-or-string: true - protocol: - anyOf: - - type: integer - - type: string - description: "Protocol is an optional field that restricts the - rule to only apply to traffic of a specific IP protocol. Required - if any of the EntityRules contain Ports (because ports only - apply to certain protocols). \n Must be one of these string - values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", - \"UDPLite\" or an integer in the range 1-255." - pattern: ^.* - x-kubernetes-int-or-string: true - source: - description: Source contains the match criteria that apply to - source entity. - properties: - namespaceSelector: - description: "NamespaceSelector is an optional field that - contains a selector expression. Only traffic that originates - from (or terminates at) endpoints within the selected - namespaces will be matched. When both NamespaceSelector - and Selector are defined on the same rule, then only workload - endpoints that are matched by both selectors will be selected - by the rule. \n For NetworkPolicy, an empty NamespaceSelector - implies that the Selector is limited to selecting only - workload endpoints in the same namespace as the NetworkPolicy. - \n For NetworkPolicy, `global()` NamespaceSelector implies - that the Selector is limited to selecting only GlobalNetworkSet - or HostEndpoint. \n For GlobalNetworkPolicy, an empty - NamespaceSelector implies the Selector applies to workload - endpoints across all namespaces." - type: string - nets: - description: Nets is an optional field that restricts the - rule to only apply to traffic that originates from (or - terminates at) IP addresses in any of the given subnets. - items: - type: string - type: array - notNets: - description: NotNets is the negated version of the Nets - field. - items: - type: string - type: array - notPorts: - description: NotPorts is the negated version of the Ports - field. Since only some protocols have ports, if any ports - are specified it requires the Protocol match in the Rule - to be set to "TCP" or "UDP". - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - notSelector: - description: NotSelector is the negated version of the Selector - field. See Selector field for subtleties with negated - selectors. - type: string - ports: - description: "Ports is an optional field that restricts - the rule to only apply to traffic that has a source (destination) - port that matches one of these ranges/values. This value - is a list of integers or strings that represent ranges - of ports. \n Since only some protocols have ports, if - any ports are specified it requires the Protocol match - in the Rule to be set to \"TCP\" or \"UDP\"." - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - selector: - description: "Selector is an optional field that contains - a selector expression (see Policy for sample syntax). - \ Only traffic that originates from (terminates at) endpoints - matching the selector will be matched. \n Note that: in - addition to the negated version of the Selector (see NotSelector - below), the selector expression syntax itself supports - negation. The two types of negation are subtly different. - One negates the set of matched endpoints, the other negates - the whole match: \n \tSelector = \"!has(my_label)\" matches - packets that are from other Calico-controlled \tendpoints - that do not have the label “my_label”. \n \tNotSelector - = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label “my_label”. - \n The effect is that the latter will accept packets from - non-Calico sources whereas the former is limited to packets - from Calico-controlled endpoints." - type: string - serviceAccounts: - description: ServiceAccounts is an optional field that restricts - the rule to only apply to traffic that originates from - (or terminates at) a pod running as a matching service - account. - properties: - names: - description: Names is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account whose name is in the list. - items: - type: string - type: array - selector: - description: Selector is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account that matches the given label selector. If - both Names and Selector are specified then they are - AND'ed. - type: string - type: object - services: - description: "Services is an optional field that contains - options for matching Kubernetes Services. If specified, - only traffic that originates from or terminates at endpoints - within the selected service(s) will be matched, and only - to/from each endpoint's port. \n Services cannot be specified - on the same rule as Selector, NotSelector, NamespaceSelector, - Ports, NotPorts, Nets, NotNets or ServiceAccounts. \n - Only valid on egress rules." - properties: - name: - description: Name specifies the name of a Kubernetes - Service to match. - type: string - namespace: - description: Namespace specifies the namespace of the - given Service. If left empty, the rule will match - within this policy's namespace. - type: string - type: object - type: object - required: - - action - type: object - type: array - ingress: - description: The ordered set of ingress rules. Each rule contains - a set of packet match criteria and a corresponding action to apply. - items: - description: "A Rule encapsulates a set of match criteria and an - action. Both selector-based security Policy and security Profiles - reference rules - separated out as a list of rules for both ingress - and egress packet matching. \n Each positive match criteria has - a negated version, prefixed with ”Not”. All the match criteria - within a rule must be satisfied for a packet to match. A single - rule can contain the positive and negative version of a match - and both must be satisfied for the rule to match." - properties: - action: - type: string - destination: - description: Destination contains the match criteria that apply - to destination entity. - properties: - namespaceSelector: - description: "NamespaceSelector is an optional field that - contains a selector expression. Only traffic that originates - from (or terminates at) endpoints within the selected - namespaces will be matched. When both NamespaceSelector - and Selector are defined on the same rule, then only workload - endpoints that are matched by both selectors will be selected - by the rule. \n For NetworkPolicy, an empty NamespaceSelector - implies that the Selector is limited to selecting only - workload endpoints in the same namespace as the NetworkPolicy. - \n For NetworkPolicy, `global()` NamespaceSelector implies - that the Selector is limited to selecting only GlobalNetworkSet - or HostEndpoint. \n For GlobalNetworkPolicy, an empty - NamespaceSelector implies the Selector applies to workload - endpoints across all namespaces." - type: string - nets: - description: Nets is an optional field that restricts the - rule to only apply to traffic that originates from (or - terminates at) IP addresses in any of the given subnets. - items: - type: string - type: array - notNets: - description: NotNets is the negated version of the Nets - field. - items: - type: string - type: array - notPorts: - description: NotPorts is the negated version of the Ports - field. Since only some protocols have ports, if any ports - are specified it requires the Protocol match in the Rule - to be set to "TCP" or "UDP". - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - notSelector: - description: NotSelector is the negated version of the Selector - field. See Selector field for subtleties with negated - selectors. - type: string - ports: - description: "Ports is an optional field that restricts - the rule to only apply to traffic that has a source (destination) - port that matches one of these ranges/values. This value - is a list of integers or strings that represent ranges - of ports. \n Since only some protocols have ports, if - any ports are specified it requires the Protocol match - in the Rule to be set to \"TCP\" or \"UDP\"." - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - selector: - description: "Selector is an optional field that contains - a selector expression (see Policy for sample syntax). - \ Only traffic that originates from (terminates at) endpoints - matching the selector will be matched. \n Note that: in - addition to the negated version of the Selector (see NotSelector - below), the selector expression syntax itself supports - negation. The two types of negation are subtly different. - One negates the set of matched endpoints, the other negates - the whole match: \n \tSelector = \"!has(my_label)\" matches - packets that are from other Calico-controlled \tendpoints - that do not have the label “my_label”. \n \tNotSelector - = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label “my_label”. - \n The effect is that the latter will accept packets from - non-Calico sources whereas the former is limited to packets - from Calico-controlled endpoints." - type: string - serviceAccounts: - description: ServiceAccounts is an optional field that restricts - the rule to only apply to traffic that originates from - (or terminates at) a pod running as a matching service - account. - properties: - names: - description: Names is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account whose name is in the list. - items: - type: string - type: array - selector: - description: Selector is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account that matches the given label selector. If - both Names and Selector are specified then they are - AND'ed. - type: string - type: object - services: - description: "Services is an optional field that contains - options for matching Kubernetes Services. If specified, - only traffic that originates from or terminates at endpoints - within the selected service(s) will be matched, and only - to/from each endpoint's port. \n Services cannot be specified - on the same rule as Selector, NotSelector, NamespaceSelector, - Ports, NotPorts, Nets, NotNets or ServiceAccounts. \n - Only valid on egress rules." - properties: - name: - description: Name specifies the name of a Kubernetes - Service to match. - type: string - namespace: - description: Namespace specifies the namespace of the - given Service. If left empty, the rule will match - within this policy's namespace. - type: string - type: object - type: object - http: - description: HTTP contains match criteria that apply to HTTP - requests. - properties: - methods: - description: Methods is an optional field that restricts - the rule to apply only to HTTP requests that use one of - the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple - methods are OR'd together. - items: - type: string - type: array - paths: - description: 'Paths is an optional field that restricts - the rule to apply to HTTP requests that use one of the - listed HTTP Paths. Multiple paths are OR''d together. - e.g: - exact: /foo - prefix: /bar NOTE: Each entry may - ONLY specify either a `exact` or a `prefix` match. The - validator will check for it.' - items: - description: 'HTTPPath specifies an HTTP path to match. - It may be either of the form: exact: : which matches - the path exactly or prefix: : which matches - the path prefix' - properties: - exact: - type: string - prefix: - type: string - type: object - type: array - type: object - icmp: - description: ICMP is an optional field that restricts the rule - to apply to a specific type and code of ICMP traffic. This - should only be specified if the Protocol field is set to "ICMP" - or "ICMPv6". - properties: - code: - description: Match on a specific ICMP code. If specified, - the Type value must also be specified. This is a technical - limitation imposed by the kernel’s iptables firewall, - which Calico uses to enforce the rule. - type: integer - type: - description: Match on a specific ICMP type. For example - a value of 8 refers to ICMP Echo Request (i.e. pings). - type: integer - type: object - ipVersion: - description: IPVersion is an optional field that restricts the - rule to only match a specific IP version. - type: integer - metadata: - description: Metadata contains additional information for this - rule - properties: - annotations: - additionalProperties: - type: string - description: Annotations is a set of key value pairs that - give extra information about the rule - type: object - type: object - notICMP: - description: NotICMP is the negated version of the ICMP field. - properties: - code: - description: Match on a specific ICMP code. If specified, - the Type value must also be specified. This is a technical - limitation imposed by the kernel’s iptables firewall, - which Calico uses to enforce the rule. - type: integer - type: - description: Match on a specific ICMP type. For example - a value of 8 refers to ICMP Echo Request (i.e. pings). - type: integer - type: object - notProtocol: - anyOf: - - type: integer - - type: string - description: NotProtocol is the negated version of the Protocol - field. - pattern: ^.* - x-kubernetes-int-or-string: true - protocol: - anyOf: - - type: integer - - type: string - description: "Protocol is an optional field that restricts the - rule to only apply to traffic of a specific IP protocol. Required - if any of the EntityRules contain Ports (because ports only - apply to certain protocols). \n Must be one of these string - values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", - \"UDPLite\" or an integer in the range 1-255." - pattern: ^.* - x-kubernetes-int-or-string: true - source: - description: Source contains the match criteria that apply to - source entity. - properties: - namespaceSelector: - description: "NamespaceSelector is an optional field that - contains a selector expression. Only traffic that originates - from (or terminates at) endpoints within the selected - namespaces will be matched. When both NamespaceSelector - and Selector are defined on the same rule, then only workload - endpoints that are matched by both selectors will be selected - by the rule. \n For NetworkPolicy, an empty NamespaceSelector - implies that the Selector is limited to selecting only - workload endpoints in the same namespace as the NetworkPolicy. - \n For NetworkPolicy, `global()` NamespaceSelector implies - that the Selector is limited to selecting only GlobalNetworkSet - or HostEndpoint. \n For GlobalNetworkPolicy, an empty - NamespaceSelector implies the Selector applies to workload - endpoints across all namespaces." - type: string - nets: - description: Nets is an optional field that restricts the - rule to only apply to traffic that originates from (or - terminates at) IP addresses in any of the given subnets. - items: - type: string - type: array - notNets: - description: NotNets is the negated version of the Nets - field. - items: - type: string - type: array - notPorts: - description: NotPorts is the negated version of the Ports - field. Since only some protocols have ports, if any ports - are specified it requires the Protocol match in the Rule - to be set to "TCP" or "UDP". - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - notSelector: - description: NotSelector is the negated version of the Selector - field. See Selector field for subtleties with negated - selectors. - type: string - ports: - description: "Ports is an optional field that restricts - the rule to only apply to traffic that has a source (destination) - port that matches one of these ranges/values. This value - is a list of integers or strings that represent ranges - of ports. \n Since only some protocols have ports, if - any ports are specified it requires the Protocol match - in the Rule to be set to \"TCP\" or \"UDP\"." - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - selector: - description: "Selector is an optional field that contains - a selector expression (see Policy for sample syntax). - \ Only traffic that originates from (terminates at) endpoints - matching the selector will be matched. \n Note that: in - addition to the negated version of the Selector (see NotSelector - below), the selector expression syntax itself supports - negation. The two types of negation are subtly different. - One negates the set of matched endpoints, the other negates - the whole match: \n \tSelector = \"!has(my_label)\" matches - packets that are from other Calico-controlled \tendpoints - that do not have the label “my_label”. \n \tNotSelector - = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label “my_label”. - \n The effect is that the latter will accept packets from - non-Calico sources whereas the former is limited to packets - from Calico-controlled endpoints." - type: string - serviceAccounts: - description: ServiceAccounts is an optional field that restricts - the rule to only apply to traffic that originates from - (or terminates at) a pod running as a matching service - account. - properties: - names: - description: Names is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account whose name is in the list. - items: - type: string - type: array - selector: - description: Selector is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account that matches the given label selector. If - both Names and Selector are specified then they are - AND'ed. - type: string - type: object - services: - description: "Services is an optional field that contains - options for matching Kubernetes Services. If specified, - only traffic that originates from or terminates at endpoints - within the selected service(s) will be matched, and only - to/from each endpoint's port. \n Services cannot be specified - on the same rule as Selector, NotSelector, NamespaceSelector, - Ports, NotPorts, Nets, NotNets or ServiceAccounts. \n - Only valid on egress rules." - properties: - name: - description: Name specifies the name of a Kubernetes - Service to match. - type: string - namespace: - description: Namespace specifies the namespace of the - given Service. If left empty, the rule will match - within this policy's namespace. - type: string - type: object - type: object - required: - - action - type: object - type: array - namespaceSelector: - description: NamespaceSelector is an optional field for an expression - used to select a pod based on namespaces. - type: string - order: - description: Order is an optional field that specifies the order in - which the policy is applied. Policies with higher "order" are applied - after those with lower order. If the order is omitted, it may be - considered to be "infinite" - i.e. the policy will be applied last. Policies - with identical order will be applied in alphanumerical order based - on the Policy "Name". - type: number - preDNAT: - description: PreDNAT indicates to apply the rules in this policy before - any DNAT. - type: boolean - selector: - description: "The selector is an expression used to pick pick out - the endpoints that the policy should be applied to. \n Selector - expressions follow this syntax: \n \tlabel == \"string_literal\" - \ -> comparison, e.g. my_label == \"foo bar\" \tlabel != \"string_literal\" - \ -> not equal; also matches if label is not present \tlabel in - { \"a\", \"b\", \"c\", ... } -> true if the value of label X is - one of \"a\", \"b\", \"c\" \tlabel not in { \"a\", \"b\", \"c\", - ... } -> true if the value of label X is not one of \"a\", \"b\", - \"c\" \thas(label_name) -> True if that label is present \t! expr - -> negation of expr \texpr && expr -> Short-circuit and \texpr - || expr -> Short-circuit or \t( expr ) -> parens for grouping \tall() - or the empty selector -> matches all endpoints. \n Label names are - allowed to contain alphanumerics, -, _ and /. String literals are - more permissive but they do not support escape characters. \n Examples - (with made-up labels): \n \ttype == \"webserver\" && deployment - == \"prod\" \ttype in {\"frontend\", \"backend\"} \tdeployment != - \"dev\" \t! has(label_name)" - type: string - serviceAccountSelector: - description: ServiceAccountSelector is an optional field for an expression - used to select a pod based on service accounts. - type: string - types: - description: "Types indicates whether this policy applies to ingress, - or to egress, or to both. When not explicitly specified (and so - the value on creation is empty or nil), Calico defaults Types according - to what Ingress and Egress rules are present in the policy. The - default is: \n - [ PolicyTypeIngress ], if there are no Egress rules - (including the case where there are also no Ingress rules) \n - - [ PolicyTypeEgress ], if there are Egress rules but no Ingress - rules \n - [ PolicyTypeIngress, PolicyTypeEgress ], if there are - both Ingress and Egress rules. \n When the policy is read back again, - Types will always be one of these values, never empty or nil." - items: - description: PolicyType enumerates the possible values of the PolicySpec - Types field. - type: string - type: array - type: object - type: object - served: true - storage: true - status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - - --- - - --- - apiVersion: apiextensions.k8s.io/v1 - kind: CustomResourceDefinition - metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null - name: globalnetworksets.crd.projectcalico.org - spec: - group: crd.projectcalico.org - names: - kind: GlobalNetworkSet - listKind: GlobalNetworkSetList - plural: globalnetworksets - singular: globalnetworkset - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - description: GlobalNetworkSet contains a set of arbitrary IP sub-networks/CIDRs - that share labels to allow rules to refer to them via selectors. The labels - of GlobalNetworkSet are not namespaced. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: GlobalNetworkSetSpec contains the specification for a NetworkSet - resource. - properties: - nets: - description: The list of IP networks that belong to this set. - items: - type: string - type: array - type: object - type: object - served: true - storage: true - status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - - --- - - --- - apiVersion: apiextensions.k8s.io/v1 - kind: CustomResourceDefinition - metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null - name: hostendpoints.crd.projectcalico.org - spec: - group: crd.projectcalico.org - names: - kind: HostEndpoint - listKind: HostEndpointList - plural: hostendpoints - singular: hostendpoint - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: HostEndpointSpec contains the specification for a HostEndpoint - resource. - properties: - expectedIPs: - description: "The expected IP addresses (IPv4 and IPv6) of the endpoint. - If \"InterfaceName\" is not present, Calico will look for an interface - matching any of the IPs in the list and apply policy to that. Note: - \tWhen using the selector match criteria in an ingress or egress - security Policy \tor Profile, Calico converts the selector into - a set of IP addresses. For host \tendpoints, the ExpectedIPs field - is used for that purpose. (If only the interface \tname is specified, - Calico does not learn the IPs of the interface for use in match - \tcriteria.)" - items: - type: string - type: array - interfaceName: - description: "Either \"*\", or the name of a specific Linux interface - to apply policy to; or empty. \"*\" indicates that this HostEndpoint - governs all traffic to, from or through the default network namespace - of the host named by the \"Node\" field; entering and leaving that - namespace via any interface, including those from/to non-host-networked - local workloads. \n If InterfaceName is not \"*\", this HostEndpoint - only governs traffic that enters or leaves the host through the - specific interface named by InterfaceName, or - when InterfaceName - is empty - through the specific interface that has one of the IPs - in ExpectedIPs. Therefore, when InterfaceName is empty, at least - one expected IP must be specified. Only external interfaces (such - as “eth0”) are supported here; it isn't possible for a HostEndpoint - to protect traffic through a specific local workload interface. - \n Note: Only some kinds of policy are implemented for \"*\" HostEndpoints; - initially just pre-DNAT policy. Please check Calico documentation - for the latest position." - type: string - node: - description: The node name identifying the Calico node instance. - type: string - ports: - description: Ports contains the endpoint's named ports, which may - be referenced in security policy rules. - items: - properties: - name: - type: string - port: - type: integer - protocol: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - required: - - name - - port - - protocol - type: object - type: array - profiles: - description: A list of identifiers of security Profile objects that - apply to this endpoint. Each profile is applied in the order that - they appear in this list. Profile rules are applied after the selector-based - security policy. - items: - type: string - type: array - type: object - type: object - served: true - storage: true - status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - - --- - - --- - apiVersion: apiextensions.k8s.io/v1 - kind: CustomResourceDefinition - metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null - name: ipamblocks.crd.projectcalico.org - spec: - group: crd.projectcalico.org - names: - kind: IPAMBlock - listKind: IPAMBlockList - plural: ipamblocks - singular: ipamblock - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: IPAMBlockSpec contains the specification for an IPAMBlock - resource. - properties: - affinity: - type: string - allocations: - items: - type: integer - # TODO: This nullable is manually added in. We should update controller-gen - # to handle []*int properly itself. - nullable: true - type: array - attributes: - items: - properties: - handle_id: - type: string - secondary: - additionalProperties: - type: string - type: object - type: object - type: array - cidr: - type: string - deleted: - type: boolean - strictAffinity: - type: boolean - unallocated: - items: - type: integer - type: array - required: - - allocations - - attributes - - cidr - - strictAffinity - - unallocated - type: object - type: object - served: true - storage: true - status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - - --- - - --- - apiVersion: apiextensions.k8s.io/v1 - kind: CustomResourceDefinition - metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null - name: ipamconfigs.crd.projectcalico.org - spec: - group: crd.projectcalico.org - names: - kind: IPAMConfig - listKind: IPAMConfigList - plural: ipamconfigs - singular: ipamconfig - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: IPAMConfigSpec contains the specification for an IPAMConfig - resource. - properties: - autoAllocateBlocks: - type: boolean - maxBlocksPerHost: - description: MaxBlocksPerHost, if non-zero, is the max number of blocks - that can be affine to each host. - type: integer - strictAffinity: - type: boolean - required: - - autoAllocateBlocks - - strictAffinity - type: object - type: object - served: true - storage: true - status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - - --- - - --- - apiVersion: apiextensions.k8s.io/v1 - kind: CustomResourceDefinition - metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null - name: ipamhandles.crd.projectcalico.org - spec: - group: crd.projectcalico.org - names: - kind: IPAMHandle - listKind: IPAMHandleList - plural: ipamhandles - singular: ipamhandle - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: IPAMHandleSpec contains the specification for an IPAMHandle - resource. - properties: - block: - additionalProperties: - type: integer - type: object - deleted: - type: boolean - handleID: - type: string - required: - - block - - handleID - type: object - type: object - served: true - storage: true - status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - - --- - - --- - apiVersion: apiextensions.k8s.io/v1 - kind: CustomResourceDefinition - metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null - name: ippools.crd.projectcalico.org - spec: - group: crd.projectcalico.org - names: - kind: IPPool - listKind: IPPoolList - plural: ippools - singular: ippool - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: IPPoolSpec contains the specification for an IPPool resource. - properties: - blockSize: - description: The block size to use for IP address assignments from - this pool. Defaults to 26 for IPv4 and 112 for IPv6. - type: integer - cidr: - description: The pool CIDR. - type: string - disabled: - description: When disabled is true, Calico IPAM will not assign addresses - from this pool. - type: boolean - ipip: - description: 'Deprecated: this field is only used for APIv1 backwards - compatibility. Setting this field is not allowed, this field is - for internal use only.' - properties: - enabled: - description: When enabled is true, ipip tunneling will be used - to deliver packets to destinations within this pool. - type: boolean - mode: - description: The IPIP mode. This can be one of "always" or "cross-subnet". A - mode of "always" will also use IPIP tunneling for routing to - destination IP addresses within this pool. A mode of "cross-subnet" - will only use IPIP tunneling when the destination node is on - a different subnet to the originating node. The default value - (if not specified) is "always". - type: string - type: object - ipipMode: - description: Contains configuration for IPIP tunneling for this pool. - If not specified, then this is defaulted to "Never" (i.e. IPIP tunneling - is disabled). - type: string - nat-outgoing: - description: 'Deprecated: this field is only used for APIv1 backwards - compatibility. Setting this field is not allowed, this field is - for internal use only.' - type: boolean - natOutgoing: - description: When nat-outgoing is true, packets sent from Calico networked - containers in this pool to destinations outside of this pool will - be masqueraded. - type: boolean - nodeSelector: - description: Allows IPPool to allocate for a specific node by label - selector. - type: string - vxlanMode: - description: Contains configuration for VXLAN tunneling for this pool. - If not specified, then this is defaulted to "Never" (i.e. VXLAN - tunneling is disabled). - type: string - required: - - cidr - type: object - type: object - served: true - storage: true - status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - - --- - - --- - apiVersion: apiextensions.k8s.io/v1 - kind: CustomResourceDefinition - metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null - name: kubecontrollersconfigurations.crd.projectcalico.org - spec: - group: crd.projectcalico.org - names: - kind: KubeControllersConfiguration - listKind: KubeControllersConfigurationList - plural: kubecontrollersconfigurations - singular: kubecontrollersconfiguration - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: KubeControllersConfigurationSpec contains the values of the - Kubernetes controllers configuration. - properties: - controllers: - description: Controllers enables and configures individual Kubernetes - controllers - properties: - namespace: - description: Namespace enables and configures the namespace controller. - Enabled by default, set to nil to disable. - properties: - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform reconciliation - with the Calico datastore. [Default: 5m]' - type: string - type: object - node: - description: Node enables and configures the node controller. - Enabled by default, set to nil to disable. - properties: - hostEndpoint: - description: HostEndpoint controls syncing nodes to host endpoints. - Disabled by default, set to nil to disable. - properties: - autoCreate: - description: 'AutoCreate enables automatic creation of - host endpoints for every node. [Default: Disabled]' - type: string - type: object - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform reconciliation - with the Calico datastore. [Default: 5m]' - type: string - syncLabels: - description: 'SyncLabels controls whether to copy Kubernetes - node labels to Calico nodes. [Default: Enabled]' - type: string - type: object - policy: - description: Policy enables and configures the policy controller. - Enabled by default, set to nil to disable. - properties: - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform reconciliation - with the Calico datastore. [Default: 5m]' - type: string - type: object - serviceAccount: - description: ServiceAccount enables and configures the service - account controller. Enabled by default, set to nil to disable. - properties: - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform reconciliation - with the Calico datastore. [Default: 5m]' - type: string - type: object - workloadEndpoint: - description: WorkloadEndpoint enables and configures the workload - endpoint controller. Enabled by default, set to nil to disable. - properties: - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform reconciliation - with the Calico datastore. [Default: 5m]' - type: string - type: object - type: object - etcdV3CompactionPeriod: - description: 'EtcdV3CompactionPeriod is the period between etcdv3 - compaction requests. Set to 0 to disable. [Default: 10m]' - type: string - healthChecks: - description: 'HealthChecks enables or disables support for health - checks [Default: Enabled]' - type: string - logSeverityScreen: - description: 'LogSeverityScreen is the log severity above which logs - are sent to the stdout. [Default: Info]' - type: string - prometheusMetricsPort: - description: 'PrometheusMetricsPort is the TCP port that the Prometheus - metrics server should bind to. Set to 0 to disable. [Default: 9094]' - type: integer - required: - - controllers - type: object - status: - description: KubeControllersConfigurationStatus represents the status - of the configuration. It's useful for admins to be able to see the actual - config that was applied, which can be modified by environment variables - on the kube-controllers process. - properties: - environmentVars: - additionalProperties: - type: string - description: EnvironmentVars contains the environment variables on - the kube-controllers that influenced the RunningConfig. - type: object - runningConfig: - description: RunningConfig contains the effective config that is running - in the kube-controllers pod, after merging the API resource with - any environment variables. - properties: - controllers: - description: Controllers enables and configures individual Kubernetes - controllers - properties: - namespace: - description: Namespace enables and configures the namespace - controller. Enabled by default, set to nil to disable. - properties: - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform - reconciliation with the Calico datastore. [Default: - 5m]' - type: string - type: object - node: - description: Node enables and configures the node controller. - Enabled by default, set to nil to disable. - properties: - hostEndpoint: - description: HostEndpoint controls syncing nodes to host - endpoints. Disabled by default, set to nil to disable. - properties: - autoCreate: - description: 'AutoCreate enables automatic creation - of host endpoints for every node. [Default: Disabled]' - type: string - type: object - leakGracePeriod: - description: 'LeakGracePeriod is the period used by the - controller to determine if an IP address has been leaked. - Set to 0 to disable IP garbage collection. [Default: - 15m]' - type: string - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform - reconciliation with the Calico datastore. [Default: - 5m]' - type: string - syncLabels: - description: 'SyncLabels controls whether to copy Kubernetes - node labels to Calico nodes. [Default: Enabled]' - type: string - type: object - policy: - description: Policy enables and configures the policy controller. - Enabled by default, set to nil to disable. - properties: - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform - reconciliation with the Calico datastore. [Default: - 5m]' - type: string - type: object - serviceAccount: - description: ServiceAccount enables and configures the service - account controller. Enabled by default, set to nil to disable. - properties: - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform - reconciliation with the Calico datastore. [Default: - 5m]' - type: string - type: object - workloadEndpoint: - description: WorkloadEndpoint enables and configures the workload - endpoint controller. Enabled by default, set to nil to disable. - properties: - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform - reconciliation with the Calico datastore. [Default: - 5m]' - type: string - type: object - type: object - etcdV3CompactionPeriod: - description: 'EtcdV3CompactionPeriod is the period between etcdv3 - compaction requests. Set to 0 to disable. [Default: 10m]' - type: string - healthChecks: - description: 'HealthChecks enables or disables support for health - checks [Default: Enabled]' - type: string - logSeverityScreen: - description: 'LogSeverityScreen is the log severity above which - logs are sent to the stdout. [Default: Info]' - type: string - prometheusMetricsPort: - description: 'PrometheusMetricsPort is the TCP port that the Prometheus - metrics server should bind to. Set to 0 to disable. [Default: - 9094]' - type: integer - required: - - controllers - type: object - type: object - type: object - served: true - storage: true - status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - - --- - - --- - apiVersion: apiextensions.k8s.io/v1 - kind: CustomResourceDefinition - metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null - name: networkpolicies.crd.projectcalico.org - spec: - group: crd.projectcalico.org - names: - kind: NetworkPolicy - listKind: NetworkPolicyList - plural: networkpolicies - singular: networkpolicy - scope: Namespaced - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - properties: - egress: - description: The ordered set of egress rules. Each rule contains - a set of packet match criteria and a corresponding action to apply. - items: - description: "A Rule encapsulates a set of match criteria and an - action. Both selector-based security Policy and security Profiles - reference rules - separated out as a list of rules for both ingress - and egress packet matching. \n Each positive match criteria has - a negated version, prefixed with ”Not”. All the match criteria - within a rule must be satisfied for a packet to match. A single - rule can contain the positive and negative version of a match - and both must be satisfied for the rule to match." - properties: - action: - type: string - destination: - description: Destination contains the match criteria that apply - to destination entity. - properties: - namespaceSelector: - description: "NamespaceSelector is an optional field that - contains a selector expression. Only traffic that originates - from (or terminates at) endpoints within the selected - namespaces will be matched. When both NamespaceSelector - and Selector are defined on the same rule, then only workload - endpoints that are matched by both selectors will be selected - by the rule. \n For NetworkPolicy, an empty NamespaceSelector - implies that the Selector is limited to selecting only - workload endpoints in the same namespace as the NetworkPolicy. - \n For NetworkPolicy, `global()` NamespaceSelector implies - that the Selector is limited to selecting only GlobalNetworkSet - or HostEndpoint. \n For GlobalNetworkPolicy, an empty - NamespaceSelector implies the Selector applies to workload - endpoints across all namespaces." - type: string - nets: - description: Nets is an optional field that restricts the - rule to only apply to traffic that originates from (or - terminates at) IP addresses in any of the given subnets. - items: - type: string - type: array - notNets: - description: NotNets is the negated version of the Nets - field. - items: - type: string - type: array - notPorts: - description: NotPorts is the negated version of the Ports - field. Since only some protocols have ports, if any ports - are specified it requires the Protocol match in the Rule - to be set to "TCP" or "UDP". - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - notSelector: - description: NotSelector is the negated version of the Selector - field. See Selector field for subtleties with negated - selectors. - type: string - ports: - description: "Ports is an optional field that restricts - the rule to only apply to traffic that has a source (destination) - port that matches one of these ranges/values. This value - is a list of integers or strings that represent ranges - of ports. \n Since only some protocols have ports, if - any ports are specified it requires the Protocol match - in the Rule to be set to \"TCP\" or \"UDP\"." - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - selector: - description: "Selector is an optional field that contains - a selector expression (see Policy for sample syntax). - \ Only traffic that originates from (terminates at) endpoints - matching the selector will be matched. \n Note that: in - addition to the negated version of the Selector (see NotSelector - below), the selector expression syntax itself supports - negation. The two types of negation are subtly different. - One negates the set of matched endpoints, the other negates - the whole match: \n \tSelector = \"!has(my_label)\" matches - packets that are from other Calico-controlled \tendpoints - that do not have the label “my_label”. \n \tNotSelector - = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label “my_label”. - \n The effect is that the latter will accept packets from - non-Calico sources whereas the former is limited to packets - from Calico-controlled endpoints." - type: string - serviceAccounts: - description: ServiceAccounts is an optional field that restricts - the rule to only apply to traffic that originates from - (or terminates at) a pod running as a matching service - account. - properties: - names: - description: Names is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account whose name is in the list. - items: - type: string - type: array - selector: - description: Selector is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account that matches the given label selector. If - both Names and Selector are specified then they are - AND'ed. - type: string - type: object - services: - description: "Services is an optional field that contains - options for matching Kubernetes Services. If specified, - only traffic that originates from or terminates at endpoints - within the selected service(s) will be matched, and only - to/from each endpoint's port. \n Services cannot be specified - on the same rule as Selector, NotSelector, NamespaceSelector, - Ports, NotPorts, Nets, NotNets or ServiceAccounts. \n - Only valid on egress rules." - properties: - name: - description: Name specifies the name of a Kubernetes - Service to match. - type: string - namespace: - description: Namespace specifies the namespace of the - given Service. If left empty, the rule will match - within this policy's namespace. - type: string - type: object - type: object - http: - description: HTTP contains match criteria that apply to HTTP - requests. - properties: - methods: - description: Methods is an optional field that restricts - the rule to apply only to HTTP requests that use one of - the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple - methods are OR'd together. - items: - type: string - type: array - paths: - description: 'Paths is an optional field that restricts - the rule to apply to HTTP requests that use one of the - listed HTTP Paths. Multiple paths are OR''d together. - e.g: - exact: /foo - prefix: /bar NOTE: Each entry may - ONLY specify either a `exact` or a `prefix` match. The - validator will check for it.' - items: - description: 'HTTPPath specifies an HTTP path to match. - It may be either of the form: exact: : which matches - the path exactly or prefix: : which matches - the path prefix' - properties: - exact: - type: string - prefix: - type: string - type: object - type: array - type: object - icmp: - description: ICMP is an optional field that restricts the rule - to apply to a specific type and code of ICMP traffic. This - should only be specified if the Protocol field is set to "ICMP" - or "ICMPv6". - properties: - code: - description: Match on a specific ICMP code. If specified, - the Type value must also be specified. This is a technical - limitation imposed by the kernel’s iptables firewall, - which Calico uses to enforce the rule. - type: integer - type: - description: Match on a specific ICMP type. For example - a value of 8 refers to ICMP Echo Request (i.e. pings). - type: integer - type: object - ipVersion: - description: IPVersion is an optional field that restricts the - rule to only match a specific IP version. - type: integer - metadata: - description: Metadata contains additional information for this - rule - properties: - annotations: - additionalProperties: - type: string - description: Annotations is a set of key value pairs that - give extra information about the rule - type: object - type: object - notICMP: - description: NotICMP is the negated version of the ICMP field. - properties: - code: - description: Match on a specific ICMP code. If specified, - the Type value must also be specified. This is a technical - limitation imposed by the kernel’s iptables firewall, - which Calico uses to enforce the rule. - type: integer - type: - description: Match on a specific ICMP type. For example - a value of 8 refers to ICMP Echo Request (i.e. pings). - type: integer - type: object - notProtocol: - anyOf: - - type: integer - - type: string - description: NotProtocol is the negated version of the Protocol - field. - pattern: ^.* - x-kubernetes-int-or-string: true - protocol: - anyOf: - - type: integer - - type: string - description: "Protocol is an optional field that restricts the - rule to only apply to traffic of a specific IP protocol. Required - if any of the EntityRules contain Ports (because ports only - apply to certain protocols). \n Must be one of these string - values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", - \"UDPLite\" or an integer in the range 1-255." - pattern: ^.* - x-kubernetes-int-or-string: true - source: - description: Source contains the match criteria that apply to - source entity. - properties: - namespaceSelector: - description: "NamespaceSelector is an optional field that - contains a selector expression. Only traffic that originates - from (or terminates at) endpoints within the selected - namespaces will be matched. When both NamespaceSelector - and Selector are defined on the same rule, then only workload - endpoints that are matched by both selectors will be selected - by the rule. \n For NetworkPolicy, an empty NamespaceSelector - implies that the Selector is limited to selecting only - workload endpoints in the same namespace as the NetworkPolicy. - \n For NetworkPolicy, `global()` NamespaceSelector implies - that the Selector is limited to selecting only GlobalNetworkSet - or HostEndpoint. \n For GlobalNetworkPolicy, an empty - NamespaceSelector implies the Selector applies to workload - endpoints across all namespaces." - type: string - nets: - description: Nets is an optional field that restricts the - rule to only apply to traffic that originates from (or - terminates at) IP addresses in any of the given subnets. - items: - type: string - type: array - notNets: - description: NotNets is the negated version of the Nets - field. - items: - type: string - type: array - notPorts: - description: NotPorts is the negated version of the Ports - field. Since only some protocols have ports, if any ports - are specified it requires the Protocol match in the Rule - to be set to "TCP" or "UDP". - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - notSelector: - description: NotSelector is the negated version of the Selector - field. See Selector field for subtleties with negated - selectors. - type: string - ports: - description: "Ports is an optional field that restricts - the rule to only apply to traffic that has a source (destination) - port that matches one of these ranges/values. This value - is a list of integers or strings that represent ranges - of ports. \n Since only some protocols have ports, if - any ports are specified it requires the Protocol match - in the Rule to be set to \"TCP\" or \"UDP\"." - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - selector: - description: "Selector is an optional field that contains - a selector expression (see Policy for sample syntax). - \ Only traffic that originates from (terminates at) endpoints - matching the selector will be matched. \n Note that: in - addition to the negated version of the Selector (see NotSelector - below), the selector expression syntax itself supports - negation. The two types of negation are subtly different. - One negates the set of matched endpoints, the other negates - the whole match: \n \tSelector = \"!has(my_label)\" matches - packets that are from other Calico-controlled \tendpoints - that do not have the label “my_label”. \n \tNotSelector - = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label “my_label”. - \n The effect is that the latter will accept packets from - non-Calico sources whereas the former is limited to packets - from Calico-controlled endpoints." - type: string - serviceAccounts: - description: ServiceAccounts is an optional field that restricts - the rule to only apply to traffic that originates from - (or terminates at) a pod running as a matching service - account. - properties: - names: - description: Names is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account whose name is in the list. - items: - type: string - type: array - selector: - description: Selector is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account that matches the given label selector. If - both Names and Selector are specified then they are - AND'ed. - type: string - type: object - services: - description: "Services is an optional field that contains - options for matching Kubernetes Services. If specified, - only traffic that originates from or terminates at endpoints - within the selected service(s) will be matched, and only - to/from each endpoint's port. \n Services cannot be specified - on the same rule as Selector, NotSelector, NamespaceSelector, - Ports, NotPorts, Nets, NotNets or ServiceAccounts. \n - Only valid on egress rules." - properties: - name: - description: Name specifies the name of a Kubernetes - Service to match. - type: string - namespace: - description: Namespace specifies the namespace of the - given Service. If left empty, the rule will match - within this policy's namespace. - type: string - type: object - type: object - required: - - action - type: object - type: array - ingress: - description: The ordered set of ingress rules. Each rule contains - a set of packet match criteria and a corresponding action to apply. - items: - description: "A Rule encapsulates a set of match criteria and an - action. Both selector-based security Policy and security Profiles - reference rules - separated out as a list of rules for both ingress - and egress packet matching. \n Each positive match criteria has - a negated version, prefixed with ”Not”. All the match criteria - within a rule must be satisfied for a packet to match. A single - rule can contain the positive and negative version of a match - and both must be satisfied for the rule to match." - properties: - action: - type: string - destination: - description: Destination contains the match criteria that apply - to destination entity. - properties: - namespaceSelector: - description: "NamespaceSelector is an optional field that - contains a selector expression. Only traffic that originates - from (or terminates at) endpoints within the selected - namespaces will be matched. When both NamespaceSelector - and Selector are defined on the same rule, then only workload - endpoints that are matched by both selectors will be selected - by the rule. \n For NetworkPolicy, an empty NamespaceSelector - implies that the Selector is limited to selecting only - workload endpoints in the same namespace as the NetworkPolicy. - \n For NetworkPolicy, `global()` NamespaceSelector implies - that the Selector is limited to selecting only GlobalNetworkSet - or HostEndpoint. \n For GlobalNetworkPolicy, an empty - NamespaceSelector implies the Selector applies to workload - endpoints across all namespaces." - type: string - nets: - description: Nets is an optional field that restricts the - rule to only apply to traffic that originates from (or - terminates at) IP addresses in any of the given subnets. - items: - type: string - type: array - notNets: - description: NotNets is the negated version of the Nets - field. - items: - type: string - type: array - notPorts: - description: NotPorts is the negated version of the Ports - field. Since only some protocols have ports, if any ports - are specified it requires the Protocol match in the Rule - to be set to "TCP" or "UDP". - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - notSelector: - description: NotSelector is the negated version of the Selector - field. See Selector field for subtleties with negated - selectors. - type: string - ports: - description: "Ports is an optional field that restricts - the rule to only apply to traffic that has a source (destination) - port that matches one of these ranges/values. This value - is a list of integers or strings that represent ranges - of ports. \n Since only some protocols have ports, if - any ports are specified it requires the Protocol match - in the Rule to be set to \"TCP\" or \"UDP\"." - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - selector: - description: "Selector is an optional field that contains - a selector expression (see Policy for sample syntax). - \ Only traffic that originates from (terminates at) endpoints - matching the selector will be matched. \n Note that: in - addition to the negated version of the Selector (see NotSelector - below), the selector expression syntax itself supports - negation. The two types of negation are subtly different. - One negates the set of matched endpoints, the other negates - the whole match: \n \tSelector = \"!has(my_label)\" matches - packets that are from other Calico-controlled \tendpoints - that do not have the label “my_label”. \n \tNotSelector - = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label “my_label”. - \n The effect is that the latter will accept packets from - non-Calico sources whereas the former is limited to packets - from Calico-controlled endpoints." - type: string - serviceAccounts: - description: ServiceAccounts is an optional field that restricts - the rule to only apply to traffic that originates from - (or terminates at) a pod running as a matching service - account. - properties: - names: - description: Names is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account whose name is in the list. - items: - type: string - type: array - selector: - description: Selector is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account that matches the given label selector. If - both Names and Selector are specified then they are - AND'ed. - type: string - type: object - services: - description: "Services is an optional field that contains - options for matching Kubernetes Services. If specified, - only traffic that originates from or terminates at endpoints - within the selected service(s) will be matched, and only - to/from each endpoint's port. \n Services cannot be specified - on the same rule as Selector, NotSelector, NamespaceSelector, - Ports, NotPorts, Nets, NotNets or ServiceAccounts. \n - Only valid on egress rules." - properties: - name: - description: Name specifies the name of a Kubernetes - Service to match. - type: string - namespace: - description: Namespace specifies the namespace of the - given Service. If left empty, the rule will match - within this policy's namespace. - type: string - type: object - type: object - http: - description: HTTP contains match criteria that apply to HTTP - requests. - properties: - methods: - description: Methods is an optional field that restricts - the rule to apply only to HTTP requests that use one of - the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple - methods are OR'd together. - items: - type: string - type: array - paths: - description: 'Paths is an optional field that restricts - the rule to apply to HTTP requests that use one of the - listed HTTP Paths. Multiple paths are OR''d together. - e.g: - exact: /foo - prefix: /bar NOTE: Each entry may - ONLY specify either a `exact` or a `prefix` match. The - validator will check for it.' - items: - description: 'HTTPPath specifies an HTTP path to match. - It may be either of the form: exact: : which matches - the path exactly or prefix: : which matches - the path prefix' - properties: - exact: - type: string - prefix: - type: string - type: object - type: array - type: object - icmp: - description: ICMP is an optional field that restricts the rule - to apply to a specific type and code of ICMP traffic. This - should only be specified if the Protocol field is set to "ICMP" - or "ICMPv6". - properties: - code: - description: Match on a specific ICMP code. If specified, - the Type value must also be specified. This is a technical - limitation imposed by the kernel’s iptables firewall, - which Calico uses to enforce the rule. - type: integer - type: - description: Match on a specific ICMP type. For example - a value of 8 refers to ICMP Echo Request (i.e. pings). - type: integer - type: object - ipVersion: - description: IPVersion is an optional field that restricts the - rule to only match a specific IP version. - type: integer - metadata: - description: Metadata contains additional information for this - rule - properties: - annotations: - additionalProperties: - type: string - description: Annotations is a set of key value pairs that - give extra information about the rule - type: object - type: object - notICMP: - description: NotICMP is the negated version of the ICMP field. - properties: - code: - description: Match on a specific ICMP code. If specified, - the Type value must also be specified. This is a technical - limitation imposed by the kernel’s iptables firewall, - which Calico uses to enforce the rule. - type: integer - type: - description: Match on a specific ICMP type. For example - a value of 8 refers to ICMP Echo Request (i.e. pings). - type: integer - type: object - notProtocol: - anyOf: - - type: integer - - type: string - description: NotProtocol is the negated version of the Protocol - field. - pattern: ^.* - x-kubernetes-int-or-string: true - protocol: - anyOf: - - type: integer - - type: string - description: "Protocol is an optional field that restricts the - rule to only apply to traffic of a specific IP protocol. Required - if any of the EntityRules contain Ports (because ports only - apply to certain protocols). \n Must be one of these string - values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", - \"UDPLite\" or an integer in the range 1-255." - pattern: ^.* - x-kubernetes-int-or-string: true - source: - description: Source contains the match criteria that apply to - source entity. - properties: - namespaceSelector: - description: "NamespaceSelector is an optional field that - contains a selector expression. Only traffic that originates - from (or terminates at) endpoints within the selected - namespaces will be matched. When both NamespaceSelector - and Selector are defined on the same rule, then only workload - endpoints that are matched by both selectors will be selected - by the rule. \n For NetworkPolicy, an empty NamespaceSelector - implies that the Selector is limited to selecting only - workload endpoints in the same namespace as the NetworkPolicy. - \n For NetworkPolicy, `global()` NamespaceSelector implies - that the Selector is limited to selecting only GlobalNetworkSet - or HostEndpoint. \n For GlobalNetworkPolicy, an empty - NamespaceSelector implies the Selector applies to workload - endpoints across all namespaces." - type: string - nets: - description: Nets is an optional field that restricts the - rule to only apply to traffic that originates from (or - terminates at) IP addresses in any of the given subnets. - items: - type: string - type: array - notNets: - description: NotNets is the negated version of the Nets - field. - items: - type: string - type: array - notPorts: - description: NotPorts is the negated version of the Ports - field. Since only some protocols have ports, if any ports - are specified it requires the Protocol match in the Rule - to be set to "TCP" or "UDP". - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - notSelector: - description: NotSelector is the negated version of the Selector - field. See Selector field for subtleties with negated - selectors. - type: string - ports: - description: "Ports is an optional field that restricts - the rule to only apply to traffic that has a source (destination) - port that matches one of these ranges/values. This value - is a list of integers or strings that represent ranges - of ports. \n Since only some protocols have ports, if - any ports are specified it requires the Protocol match - in the Rule to be set to \"TCP\" or \"UDP\"." - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - selector: - description: "Selector is an optional field that contains - a selector expression (see Policy for sample syntax). - \ Only traffic that originates from (terminates at) endpoints - matching the selector will be matched. \n Note that: in - addition to the negated version of the Selector (see NotSelector - below), the selector expression syntax itself supports - negation. The two types of negation are subtly different. - One negates the set of matched endpoints, the other negates - the whole match: \n \tSelector = \"!has(my_label)\" matches - packets that are from other Calico-controlled \tendpoints - that do not have the label “my_label”. \n \tNotSelector - = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label “my_label”. - \n The effect is that the latter will accept packets from - non-Calico sources whereas the former is limited to packets - from Calico-controlled endpoints." - type: string - serviceAccounts: - description: ServiceAccounts is an optional field that restricts - the rule to only apply to traffic that originates from - (or terminates at) a pod running as a matching service - account. - properties: - names: - description: Names is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account whose name is in the list. - items: - type: string - type: array - selector: - description: Selector is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account that matches the given label selector. If - both Names and Selector are specified then they are - AND'ed. - type: string - type: object - services: - description: "Services is an optional field that contains - options for matching Kubernetes Services. If specified, - only traffic that originates from or terminates at endpoints - within the selected service(s) will be matched, and only - to/from each endpoint's port. \n Services cannot be specified - on the same rule as Selector, NotSelector, NamespaceSelector, - Ports, NotPorts, Nets, NotNets or ServiceAccounts. \n - Only valid on egress rules." - properties: - name: - description: Name specifies the name of a Kubernetes - Service to match. - type: string - namespace: - description: Namespace specifies the namespace of the - given Service. If left empty, the rule will match - within this policy's namespace. - type: string - type: object - type: object - required: - - action - type: object - type: array - order: - description: Order is an optional field that specifies the order in - which the policy is applied. Policies with higher "order" are applied - after those with lower order. If the order is omitted, it may be - considered to be "infinite" - i.e. the policy will be applied last. Policies - with identical order will be applied in alphanumerical order based - on the Policy "Name". - type: number - selector: - description: "The selector is an expression used to pick pick out - the endpoints that the policy should be applied to. \n Selector - expressions follow this syntax: \n \tlabel == \"string_literal\" - \ -> comparison, e.g. my_label == \"foo bar\" \tlabel != \"string_literal\" - \ -> not equal; also matches if label is not present \tlabel in - { \"a\", \"b\", \"c\", ... } -> true if the value of label X is - one of \"a\", \"b\", \"c\" \tlabel not in { \"a\", \"b\", \"c\", - ... } -> true if the value of label X is not one of \"a\", \"b\", - \"c\" \thas(label_name) -> True if that label is present \t! expr - -> negation of expr \texpr && expr -> Short-circuit and \texpr - || expr -> Short-circuit or \t( expr ) -> parens for grouping \tall() - or the empty selector -> matches all endpoints. \n Label names are - allowed to contain alphanumerics, -, _ and /. String literals are - more permissive but they do not support escape characters. \n Examples - (with made-up labels): \n \ttype == \"webserver\" && deployment - == \"prod\" \ttype in {\"frontend\", \"backend\"} \tdeployment != - \"dev\" \t! has(label_name)" - type: string - serviceAccountSelector: - description: ServiceAccountSelector is an optional field for an expression - used to select a pod based on service accounts. - type: string - types: - description: "Types indicates whether this policy applies to ingress, - or to egress, or to both. When not explicitly specified (and so - the value on creation is empty or nil), Calico defaults Types according - to what Ingress and Egress are present in the policy. The default - is: \n - [ PolicyTypeIngress ], if there are no Egress rules (including - the case where there are also no Ingress rules) \n - [ PolicyTypeEgress - ], if there are Egress rules but no Ingress rules \n - [ PolicyTypeIngress, - PolicyTypeEgress ], if there are both Ingress and Egress rules. - \n When the policy is read back again, Types will always be one - of these values, never empty or nil." - items: - description: PolicyType enumerates the possible values of the PolicySpec - Types field. - type: string - type: array - type: object - type: object - served: true - storage: true - status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - - --- - - --- - apiVersion: apiextensions.k8s.io/v1 - kind: CustomResourceDefinition - metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - creationTimestamp: null - name: networksets.crd.projectcalico.org - spec: - group: crd.projectcalico.org - names: - kind: NetworkSet - listKind: NetworkSetList - plural: networksets - singular: networkset - scope: Namespaced - versions: - - name: v1 - schema: - openAPIV3Schema: - description: NetworkSet is the Namespaced-equivalent of the GlobalNetworkSet. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: NetworkSetSpec contains the specification for a NetworkSet - resource. - properties: - nets: - description: The list of IP networks that belong to this set. - items: - type: string - type: array - type: object - type: object - served: true - storage: true - status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - - --- - --- - # Source: calico/templates/calico-kube-controllers-rbac.yaml - - # Include a clusterrole for the kube-controllers component, - # and bind it to the calico-kube-controllers serviceaccount. - kind: ClusterRole - apiVersion: rbac.authorization.k8s.io/v1 - metadata: - name: calico-kube-controllers - rules: - # Nodes are watched to monitor for deletions. - - apiGroups: [""] - resources: - - nodes - verbs: - - watch - - list - - get - # Pods are watched to check for existence as part of IPAM controller. - - apiGroups: [""] - resources: - - pods - verbs: - - get - - list - - watch - # IPAM resources are manipulated when nodes are deleted. - - apiGroups: ["crd.projectcalico.org"] - resources: - - ippools - verbs: - - list - - apiGroups: ["crd.projectcalico.org"] - resources: - - blockaffinities - - ipamblocks - - ipamhandles - verbs: - - get - - list - - create - - update - - delete - - watch - # kube-controllers manages hostendpoints. - - apiGroups: ["crd.projectcalico.org"] - resources: - - hostendpoints - verbs: - - get - - list - - create - - update - - delete - # Needs access to update clusterinformations. - - apiGroups: ["crd.projectcalico.org"] - resources: - - clusterinformations - verbs: - - get - - create - - update - # KubeControllersConfiguration is where it gets its config - - apiGroups: ["crd.projectcalico.org"] - resources: - - kubecontrollersconfigurations - verbs: - # read its own config - - get - # create a default if none exists - - create - # update status - - update - # watch for changes - - watch - --- - kind: ClusterRoleBinding - apiVersion: rbac.authorization.k8s.io/v1 - metadata: - name: calico-kube-controllers - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: calico-kube-controllers - subjects: - - kind: ServiceAccount - name: calico-kube-controllers - namespace: kube-system - --- - - --- - # Source: calico/templates/calico-node-rbac.yaml - # Include a clusterrole for the calico-node DaemonSet, - # and bind it to the calico-node serviceaccount. - kind: ClusterRole - apiVersion: rbac.authorization.k8s.io/v1 - metadata: - name: calico-node - rules: - # The CNI plugin needs to get pods, nodes, and namespaces. - - apiGroups: [""] - resources: - - pods - - nodes - - namespaces - verbs: - - get - # EndpointSlices are used for Service-based network policy rule - # enforcement. - - apiGroups: ["discovery.k8s.io"] - resources: - - endpointslices - verbs: - - watch - - list - - apiGroups: [""] - resources: - - endpoints - - services - verbs: - # Used to discover service IPs for advertisement. - - watch - - list - # Used to discover Typhas. - - get - # Pod CIDR auto-detection on kubeadm needs access to config maps. - - apiGroups: [""] - resources: - - configmaps - verbs: - - get - - apiGroups: [""] - resources: - - nodes/status - verbs: - # Needed for clearing NodeNetworkUnavailable flag. - - patch - # Calico stores some configuration information in node annotations. - - update - # Watch for changes to Kubernetes NetworkPolicies. - - apiGroups: ["networking.k8s.io"] - resources: - - networkpolicies - verbs: - - watch - - list - # Used by Calico for policy information. - - apiGroups: [""] - resources: - - pods - - namespaces - - serviceaccounts - verbs: - - list - - watch - # The CNI plugin patches pods/status. - - apiGroups: [""] - resources: - - pods/status - verbs: - - patch - # Calico monitors various CRDs for config. - - apiGroups: ["crd.projectcalico.org"] - resources: - - globalfelixconfigs - - felixconfigurations - - bgppeers - - globalbgpconfigs - - bgpconfigurations - - ippools - - ipamblocks - - globalnetworkpolicies - - globalnetworksets - - networkpolicies - - networksets - - clusterinformations - - hostendpoints - - blockaffinities - verbs: - - get - - list - - watch - # Calico must create and update some CRDs on startup. - - apiGroups: ["crd.projectcalico.org"] - resources: - - ippools - - felixconfigurations - - clusterinformations - verbs: - - create - - update - # Calico stores some configuration information on the node. - - apiGroups: [""] - resources: - - nodes - verbs: - - get - - list - - watch - # These permissions are only required for upgrade from v2.6, and can - # be removed after upgrade or on fresh installations. - - apiGroups: ["crd.projectcalico.org"] - resources: - - bgpconfigurations - - bgppeers - verbs: - - create - - update - # These permissions are required for Calico CNI to perform IPAM allocations. - - apiGroups: ["crd.projectcalico.org"] - resources: - - blockaffinities - - ipamblocks - - ipamhandles - verbs: - - get - - list - - create - - update - - delete - - apiGroups: ["crd.projectcalico.org"] - resources: - - ipamconfigs - verbs: - - get - # Block affinities must also be watchable by confd for route aggregation. - - apiGroups: ["crd.projectcalico.org"] - resources: - - blockaffinities - verbs: - - watch - # The Calico IPAM migration needs to get daemonsets. These permissions can be - # removed if not upgrading from an installation using host-local IPAM. - - apiGroups: ["apps"] - resources: - - daemonsets - verbs: - - get - - --- - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRoleBinding - metadata: - name: calico-node - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: calico-node - subjects: - - kind: ServiceAccount - name: calico-node - namespace: kube-system - - --- - # Source: calico/templates/calico-node.yaml - # This manifest installs the calico-node container, as well - # as the CNI plugins and network config on - # each master and worker node in a Kubernetes cluster. - kind: DaemonSet - apiVersion: apps/v1 - metadata: - name: calico-node - namespace: kube-system - labels: - k8s-app: calico-node - spec: - selector: - matchLabels: - k8s-app: calico-node - updateStrategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 1 - template: - metadata: - labels: - k8s-app: calico-node - spec: - nodeSelector: - kubernetes.io/os: linux - hostNetwork: true - tolerations: - # Make sure calico-node gets scheduled on all nodes. - - effect: NoSchedule - operator: Exists - # Mark the pod as a critical add-on for rescheduling. - - key: CriticalAddonsOnly - operator: Exists - - effect: NoExecute - operator: Exists - serviceAccountName: calico-node - # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force - # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods. - terminationGracePeriodSeconds: 0 - priorityClassName: system-node-critical - initContainers: - # This container performs upgrade from host-local IPAM to calico-ipam. - # It can be deleted if this is a fresh installation, or if you have already - # upgraded to use calico-ipam. - - name: upgrade-ipam - image: calico/cni:v3.20.0 - command: ["/opt/cni/bin/calico-ipam", "-upgrade"] - envFrom: - - configMapRef: - # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. - name: kubernetes-services-endpoint - optional: true - env: - - name: KUBERNETES_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: CALICO_NETWORKING_BACKEND - valueFrom: - configMapKeyRef: - name: calico-config - key: calico_backend - volumeMounts: - - mountPath: /var/lib/cni/networks - name: host-local-net-dir - - mountPath: /host/opt/cni/bin - name: cni-bin-dir - securityContext: - privileged: true - # This container installs the CNI binaries - # and CNI network config file on each node. - - name: install-cni - image: calico/cni:v3.20.0 - command: ["/opt/cni/bin/install"] - envFrom: - - configMapRef: - # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. - name: kubernetes-services-endpoint - optional: true - env: - # Name of the CNI config file to create. - - name: CNI_CONF_NAME - value: "10-calico.conflist" - # The CNI network config to install on each node. - - name: CNI_NETWORK_CONFIG - valueFrom: - configMapKeyRef: - name: calico-config - key: cni_network_config - # Set the hostname based on the k8s node name. - - name: KUBERNETES_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - # CNI MTU Config variable - - name: CNI_MTU - valueFrom: - configMapKeyRef: - name: calico-config - key: veth_mtu - # Prevents the container from sleeping forever. - - name: SLEEP - value: "false" - volumeMounts: - - mountPath: /host/opt/cni/bin - name: cni-bin-dir - - mountPath: /host/etc/cni/net.d - name: cni-net-dir - securityContext: - privileged: true - # Adds a Flex Volume Driver that creates a per-pod Unix Domain Socket to allow Dikastes - # to communicate with Felix over the Policy Sync API. - - name: flexvol-driver - image: calico/pod2daemon-flexvol:v3.20.0 - volumeMounts: - - name: flexvol-driver-host - mountPath: /host/driver - securityContext: - privileged: true - containers: - # Runs calico-node container on each Kubernetes node. This - # container programs network policy and routes on each - # host. - - name: calico-node - image: calico/node:v3.20.0 - envFrom: - - configMapRef: - # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. - name: kubernetes-services-endpoint - optional: true - env: - # Use Kubernetes API as the backing datastore. - - name: DATASTORE_TYPE - value: "kubernetes" - # Wait for the datastore. - - name: WAIT_FOR_DATASTORE - value: "true" - # Set based on the k8s node name. - - name: NODENAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - # Choose the backend to use. - - name: CALICO_NETWORKING_BACKEND - valueFrom: - configMapKeyRef: - name: calico-config - key: calico_backend - # Cluster type to identify the deployment type - - name: CLUSTER_TYPE - value: "k8s,bgp" - # Auto-detect the BGP IP address. - - name: IP - value: "autodetect" - # Enable VXLAN - - name: CALICO_IPV4POOL_VXLAN - value: "Always" - # Set MTU for tunnel device used if ipip is enabled - - name: FELIX_IPINIPMTU - valueFrom: - configMapKeyRef: - name: calico-config - key: veth_mtu - # Set MTU for the VXLAN tunnel device. - - name: FELIX_VXLANMTU - valueFrom: - configMapKeyRef: - name: calico-config - key: veth_mtu - # Set MTU for the Wireguard tunnel device. - - name: FELIX_WIREGUARDMTU - valueFrom: - configMapKeyRef: - name: calico-config - key: veth_mtu - # The default IPv4 pool to create on startup if none exists. Pod IPs will be - # chosen from this range. Changing this value after installation will have - # no effect. This should fall within `--cluster-cidr`. - # - name: CALICO_IPV4POOL_CIDR - # value: "192.168.0.0/16" - # Disable file logging so `kubectl logs` works. - - name: CALICO_DISABLE_FILE_LOGGING - value: "true" - # Set Felix endpoint to host default action to ACCEPT. - - name: FELIX_DEFAULTENDPOINTTOHOSTACTION - value: "ACCEPT" - # Disable IPv6 on Kubernetes. - - name: FELIX_IPV6SUPPORT - value: "false" - - name: FELIX_FEATUREDETECTOVERRIDE - value: "ChecksumOffloadBroken=true" - - name: FELIX_HEALTHENABLED - value: "true" - securityContext: - privileged: true - resources: - requests: - cpu: 250m - livenessProbe: - exec: - command: - - /bin/calico-node - - -felix-live - periodSeconds: 10 - initialDelaySeconds: 10 - failureThreshold: 6 - readinessProbe: - exec: - command: - - /bin/calico-node - - -felix-ready - periodSeconds: 10 - volumeMounts: - - mountPath: /host/etc/cni/net.d - name: cni-net-dir - readOnly: false - - mountPath: /lib/modules - name: lib-modules - readOnly: true - - mountPath: /run/xtables.lock - name: xtables-lock - readOnly: false - - mountPath: /var/run/calico - name: var-run-calico - readOnly: false - - mountPath: /var/lib/calico - name: var-lib-calico - readOnly: false - - name: policysync - mountPath: /var/run/nodeagent - # For eBPF mode, we need to be able to mount the BPF filesystem at /sys/fs/bpf so we mount in the - # parent directory. - - name: sysfs - mountPath: /sys/fs/ - # Bidirectional means that, if we mount the BPF filesystem at /sys/fs/bpf it will propagate to the host. - # If the host is known to mount that filesystem already then Bidirectional can be omitted. - mountPropagation: Bidirectional - - name: cni-log-dir - mountPath: /var/log/calico/cni - readOnly: true - volumes: - # Used by calico-node. - - name: lib-modules - hostPath: - path: /lib/modules - - name: var-run-calico - hostPath: - path: /var/run/calico - - name: var-lib-calico - hostPath: - path: /var/lib/calico - - name: xtables-lock - hostPath: - path: /run/xtables.lock - type: FileOrCreate - - name: sysfs - hostPath: - path: /sys/fs/ - type: DirectoryOrCreate - # Used to install CNI. - - name: cni-bin-dir - hostPath: - path: /opt/cni/bin - - name: cni-net-dir - hostPath: - path: /etc/cni/net.d - # Used to access CNI logs. - - name: cni-log-dir - hostPath: - path: /var/log/calico/cni - # Mount in the directory for host-local IPAM allocations. This is - # used when upgrading from host-local to calico-ipam, and can be removed - # if not using the upgrade-ipam init container. - - name: host-local-net-dir - hostPath: - path: /var/lib/cni/networks - # Used to create per-pod Unix Domain Sockets - - name: policysync - hostPath: - type: DirectoryOrCreate - path: /var/run/nodeagent - # Used to install Flex Volume Driver - - name: flexvol-driver-host - hostPath: - type: DirectoryOrCreate - path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent~uds - --- - - apiVersion: v1 - kind: ServiceAccount - metadata: - name: calico-node - namespace: kube-system - - --- - # Source: calico/templates/calico-kube-controllers.yaml - # See https://github.com/projectcalico/kube-controllers - apiVersion: apps/v1 - kind: Deployment - metadata: - name: calico-kube-controllers - namespace: kube-system - labels: - k8s-app: calico-kube-controllers - spec: - # The controllers can only have a single active instance. - replicas: 1 - selector: - matchLabels: - k8s-app: calico-kube-controllers - strategy: - type: Recreate - template: - metadata: - name: calico-kube-controllers - namespace: kube-system - labels: - k8s-app: calico-kube-controllers - spec: - nodeSelector: - kubernetes.io/os: linux - tolerations: - # Mark the pod as a critical add-on for rescheduling. - - key: CriticalAddonsOnly - operator: Exists - - key: node-role.kubernetes.io/master - effect: NoSchedule - - key: node-role.kubernetes.io/control-plane - effect: NoSchedule - serviceAccountName: calico-kube-controllers - priorityClassName: system-cluster-critical - containers: - - name: calico-kube-controllers - image: calico/kube-controllers:v3.20.0 - env: - # Choose which controllers to run. - - name: ENABLED_CONTROLLERS - value: node - - name: DATASTORE_TYPE - value: kubernetes - livenessProbe: - exec: - command: - - /usr/bin/check-status - - -l - periodSeconds: 10 - initialDelaySeconds: 10 - failureThreshold: 6 - timeoutSeconds: 10 - readinessProbe: - exec: - command: - - /usr/bin/check-status - - -r - periodSeconds: 10 - - --- - - apiVersion: v1 - kind: ServiceAccount - metadata: - name: calico-kube-controllers - namespace: kube-system - - --- - - # This manifest creates a Pod Disruption Budget for Controller to allow K8s Cluster Autoscaler to evict - - apiVersion: policy/v1beta1 - kind: PodDisruptionBudget - metadata: - name: calico-kube-controllers - namespace: kube-system - labels: - k8s-app: calico-kube-controllers - spec: - maxUnavailable: 1 - selector: - matchLabels: - k8s-app: calico-kube-controllers - --- - # Source: calico/templates/calico-etcd-secrets.yaml - - --- - # Source: calico/templates/calico-typha.yaml - - --- - # Source: calico/templates/configure-canal.yaml -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: cni-${CLUSTER_NAME}-calico - namespace: default diff --git a/templates/test/ci/cluster-template-prow-intree-cloud-provider-machine-pool.yaml b/templates/test/ci/cluster-template-prow-intree-cloud-provider-machine-pool.yaml new file mode 100644 index 00000000000..368e1e98e99 --- /dev/null +++ b/templates/test/ci/cluster-template-prow-intree-cloud-provider-machine-pool.yaml @@ -0,0 +1,689 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + labels: + cni-windows: ${CLUSTER_NAME}-calico + containerd-logger: enabled + csi-proxy: enabled + windows: enabled + name: ${CLUSTER_NAME} + namespace: default +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlane + name: ${CLUSTER_NAME}-control-plane + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureCluster + name: ${CLUSTER_NAME} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureCluster +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + additionalTags: + buildProvenance: ${BUILD_PROVENANCE} + creationTimestamp: ${TIMESTAMP} + jobName: ${JOB_NAME} + identityRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureClusterIdentity + name: ${CLUSTER_IDENTITY_NAME} + location: ${AZURE_LOCATION} + networkSpec: + subnets: + - name: control-plane-subnet + role: control-plane + - name: node-subnet + natGateway: + name: node-natgateway + role: node + vnet: + name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} + resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} + subscriptionID: ${AZURE_SUBSCRIPTION_ID} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlane +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + kubeadmConfigSpec: + clusterConfiguration: + apiServer: + extraArgs: + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + feature-gates: ${K8S_FEATURE_GATES:-""} + extraVolumes: + - hostPath: /etc/kubernetes/azure.json + mountPath: /etc/kubernetes/azure.json + name: cloud-config + readOnly: true + timeoutForControlPlane: 20m + controllerManager: + extraArgs: + allocate-node-cidrs: "false" + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + cluster-name: ${CLUSTER_NAME} + v: "4" + extraVolumes: + - hostPath: /etc/kubernetes/azure.json + mountPath: /etc/kubernetes/azure.json + name: cloud-config + readOnly: true + etcd: + local: + dataDir: /var/lib/etcddisk/etcd + extraArgs: + quota-backend-bytes: "8589934592" + diskSetup: + filesystems: + - device: /dev/disk/azure/scsi1/lun0 + extraOpts: + - -E + - lazy_itable_init=1,lazy_journal_init=1 + filesystem: ext4 + label: etcd_disk + - device: ephemeral0.1 + filesystem: ext4 + label: ephemeral0 + replaceFS: ntfs + partitions: + - device: /dev/disk/azure/scsi1/lun0 + layout: true + overwrite: false + tableType: gpt + files: + - contentFrom: + secret: + key: control-plane-azure.json + name: ${CLUSTER_NAME}-control-plane-azure-json + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + azure-container-registry-config: /etc/kubernetes/azure.json + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + name: '{{ ds.meta_data["local_hostname"] }}' + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + azure-container-registry-config: /etc/kubernetes/azure.json + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + name: '{{ ds.meta_data["local_hostname"] }}' + mounts: + - - LABEL=etcd_disk + - /var/lib/etcddisk + postKubeadmCommands: [] + preKubeadmCommands: [] + machineTemplate: + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-control-plane + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + template: + spec: + dataDisks: + - diskSizeGB: 256 + lun: 0 + nameSuffix: etcddisk + osDisk: + diskSizeGB: 128 + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachinePool +metadata: + name: ${CLUSTER_NAME}-mp-0 + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT} + template: + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfig + name: ${CLUSTER_NAME}-mp-0 + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachinePool + name: ${CLUSTER_NAME}-mp-0 + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachinePool +metadata: + name: ${CLUSTER_NAME}-mp-0 + namespace: default +spec: + location: ${AZURE_LOCATION} + strategy: + rollingUpdate: + deletePolicy: Oldest + maxSurge: 25% + maxUnavailable: 1 + type: RollingUpdate + template: + osDisk: + diskSizeGB: 30 + managedDisk: + storageAccountType: Premium_LRS + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + vmExtensions: + - name: CustomScript + protectedSettings: + commandToExecute: | + #!/bin/sh + echo "This script is a no-op used for extension testing purposes ..." + touch test_file + publisher: Microsoft.Azure.Extensions + version: "2.1" + vmSize: ${AZURE_NODE_MACHINE_TYPE} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfig +metadata: + name: ${CLUSTER_NAME}-mp-0 + namespace: default +spec: + files: + - contentFrom: + secret: + key: worker-node-azure.json + name: ${CLUSTER_NAME}-mp-0-azure-json + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + azure-container-registry-config: /etc/kubernetes/azure.json + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + name: '{{ ds.meta_data["local_hostname"] }}' +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureClusterIdentity +metadata: + labels: + clusterctl.cluster.x-k8s.io/move-hierarchy: "true" + name: ${CLUSTER_IDENTITY_NAME} + namespace: default +spec: + allowedNamespaces: {} + clientID: ${AZURE_CLIENT_ID} + clientSecret: + name: ${AZURE_CLUSTER_IDENTITY_SECRET_NAME} + namespace: ${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE} + tenantID: ${AZURE_TENANT_ID} + type: ServicePrincipal +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachinePool +metadata: + name: ${CLUSTER_NAME}-mp-win + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WINDOWS_WORKER_MACHINE_COUNT:-0} + template: + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfig + name: ${CLUSTER_NAME}-mp-win + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachinePool + name: ${CLUSTER_NAME}-mp-win + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachinePool +metadata: + annotations: + runtime: containerd + name: ${CLUSTER_NAME}-mp-win + namespace: default +spec: + location: ${AZURE_LOCATION} + template: + osDisk: + diskSizeGB: 30 + managedDisk: + storageAccountType: Premium_LRS + osType: Windows + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + vmSize: ${AZURE_NODE_MACHINE_TYPE} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfig +metadata: + name: ${CLUSTER_NAME}-mp-win + namespace: default +spec: + files: + - contentFrom: + secret: + key: worker-node-azure.json + name: ${CLUSTER_NAME}-mp-win-azure-json + owner: root:root + path: c:/k/azure.json + permissions: "0644" + - content: Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico.exe + path: C:/defender-exclude-calico.ps1 + permissions: "0744" + joinConfiguration: + nodeRegistration: + criSocket: npipe:////./pipe/containerd-containerd + kubeletExtraArgs: + azure-container-registry-config: c:/k/azure.json + cloud-config: c:/k/azure.json + cloud-provider: azure + feature-gates: WindowsHostProcessContainers=true + pod-infra-container-image: mcr.microsoft.com/oss/kubernetes/pause:3.4.1 + name: '{{ ds.meta_data["local_hostname"] }}' + postKubeadmCommands: + - nssm set kubelet start SERVICE_AUTO_START + - powershell C:/defender-exclude-calico.ps1 + preKubeadmCommands: + - powershell c:/create-external-network.ps1 + users: + - groups: Administrators + name: capi + sshAuthorizedKeys: + - ${AZURE_SSH_PUBLIC_KEY:=""} +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: ${CLUSTER_NAME}-calico-windows + namespace: default +spec: + clusterSelector: + matchLabels: + cni-windows: ${CLUSTER_NAME}-calico + resources: + - kind: ConfigMap + name: cni-${CLUSTER_NAME}-calico-windows + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: csi-proxy + namespace: default +spec: + clusterSelector: + matchLabels: + csi-proxy: enabled + resources: + - kind: ConfigMap + name: csi-proxy-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: containerd-logger-${CLUSTER_NAME} + namespace: default +spec: + clusterSelector: + matchLabels: + containerd-logger: enabled + resources: + - kind: ConfigMap + name: containerd-logger-${CLUSTER_NAME} + strategy: ApplyOnce +--- +apiVersion: v1 +data: + proxy: | + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + k8s-app: kube-proxy + name: kube-proxy-windows + namespace: kube-system + spec: + selector: + matchLabels: + k8s-app: kube-proxy-windows + template: + metadata: + labels: + k8s-app: kube-proxy-windows + spec: + serviceAccountName: kube-proxy + securityContext: + windowsOptions: + hostProcess: true + runAsUserName: "NT AUTHORITY\\system" + hostNetwork: true + containers: + - image: sigwindowstools/kube-proxy:${KUBERNETES_VERSION/+/_}-calico-hostprocess + args: ["$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/start.ps1"] + workingDir: "$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/" + name: kube-proxy + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: KUBEPROXY_PATH + valueFrom: + configMapKeyRef: + name: windows-kubeproxy-ci + key: KUBEPROXY_PATH + optional: true + volumeMounts: + - mountPath: /var/lib/kube-proxy + name: kube-proxy + nodeSelector: + kubernetes.io/os: windows + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - operator: Exists + volumes: + - configMap: + name: kube-proxy + name: kube-proxy + updateStrategy: + type: RollingUpdate + windows-cni: "# strictAffinity required for windows\napiVersion: crd.projectcalico.org/v1\nkind: + IPAMConfig\nmetadata:\n name: default\nspec:\n autoAllocateBlocks: true\n strictAffinity: + true\n---\nkind: ConfigMap\napiVersion: v1\nmetadata:\n name: calico-static-rules\n + \ namespace: calico-system\n labels:\n tier: node\n app: calico\ndata:\n + \ static-rules.json: |\n {\n \"Provider\": \"azure\",\n \"Version\": + \"0.1\",\n \"Rules\": [\n {\n \"Name\": \"EndpointPolicy\",\n + \ \"Rule\": {\n \"Id\": \"wireserver\",\n \"Type\": + \"ACL\",\n \"Protocol\": 6,\n \"Action\": \"Block\",\n + \ \"Direction\": \"Out\",\n \"RemoteAddresses\": \"168.63.129.16/32\",\n + \ \"RemotePorts\": \"80\",\n \"Priority\": 200,\n \"RuleType\": + \"Switch\"\n }\n }\n ]\n } \n---\nkind: ConfigMap\napiVersion: + v1\nmetadata:\n name: calico-config-windows\n namespace: calico-system\n labels:\n + \ tier: node\n app: calico\ndata:\n veth_mtu: \"1350\"\n \n cni_network_config: + |\n {\n \"name\": \"Calico\",\n \"cniVersion\": \"0.3.1\",\n \"plugins\": + [\n {\n \"windows_use_single_network\": true,\n \"type\": + \"calico\",\n \"mode\": \"vxlan\",\n \"nodename\": \"__KUBERNETES_NODE_NAME__\",\n + \ \"nodename_file_optional\": true,\n \"log_file_path\": \"c:/cni.log\",\n + \ \"log_level\": \"debug\",\n\n \"vxlan_mac_prefix\": \"0E-2A\",\n + \ \"vxlan_vni\": 4096,\n \"mtu\": __CNI_MTU__,\n \"policy\": + {\n \"type\": \"k8s\"\n },\n\n \"log_level\": \"info\",\n\n + \ \"capabilities\": {\"dns\": true},\n \"DNS\": {\n \"Search\": + \ [\n \"svc.cluster.local\"\n ]\n },\n\n \"datastore_type\": + \"kubernetes\",\n\n \"kubernetes\": {\n \"kubeconfig\": \"__KUBECONFIG_FILEPATH__\"\n + \ },\n\n \"ipam\": {\n \"type\": \"calico-ipam\",\n + \ \"subnet\": \"usePodCidr\"\n },\n\n \"policies\": + \ [\n {\n \"Name\": \"EndpointPolicy\",\n \"Value\": + \ {\n \"Type\": \"OutBoundNAT\",\n \"ExceptionList\": + \ [\n \"__K8S_SERVICE_CIDR__\"\n ]\n }\n + \ },\n {\n \"Name\": \"EndpointPolicy\",\n + \ \"Value\": {\n \"Type\": \"SDNROUTE\",\n \"DestinationPrefix\": + \ \"__K8S_SERVICE_CIDR__\",\n \"NeedEncap\": true\n }\n + \ }\n ]\n }\n ]\n\n }\n---\napiVersion: apps/v1\nkind: + DaemonSet\nmetadata:\n name: calico-node-windows\n labels:\n tier: node\n + \ app: calico\n namespace: calico-system\nspec:\n selector:\n matchLabels:\n + \ app: calico\n template:\n metadata:\n labels:\n tier: node\n + \ app: calico\n spec:\n affinity:\n nodeAffinity:\n requiredDuringSchedulingIgnoredDuringExecution:\n + \ nodeSelectorTerms:\n - matchExpressions:\n - + key: kubernetes.io/os\n operator: In\n values:\n + \ - windows\n - key: kubernetes.io/arch\n + \ operator: In\n values:\n - + amd64\n securityContext:\n windowsOptions:\n hostProcess: + true\n runAsUserName: \"NT AUTHORITY\\\\system\"\n hostNetwork: + true\n serviceAccountName: calico-node\n tolerations:\n - operator: + Exists\n effect: NoSchedule\n # Mark the pod as a critical add-on + for rescheduling.\n - key: CriticalAddonsOnly\n operator: Exists\n + \ - effect: NoExecute\n operator: Exists\n initContainers:\n # + This container installs the CNI binaries\n # and CNI network config file + on each node.\n - name: install-cni\n image: sigwindowstools/calico-install:v3.25.0-hostprocess\n + \ args: [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/install.ps1\"]\n + \ imagePullPolicy: Always\n env:\n # Name of the CNI + config file to create.\n - name: CNI_CONF_NAME\n value: + \"10-calico.conflist\"\n # The CNI network config to install on each + node.\n - name: CNI_NETWORK_CONFIG\n valueFrom:\n configMapKeyRef:\n + \ name: calico-config-windows\n key: cni_network_config\n + \ # Set the hostname based on the k8s node name.\n - name: + KUBERNETES_NODE_NAME\n valueFrom:\n fieldRef:\n fieldPath: + spec.nodeName\n # CNI MTU Config variable\n - name: CNI_MTU\n + \ valueFrom:\n configMapKeyRef:\n name: + calico-config-windows\n key: veth_mtu\n # Prevents + the container from sleeping forever.\n - name: SLEEP\n value: + \"false\"\n - name: K8S_SERVICE_CIDR\n value: \"10.96.0.0/12\"\n + \ volumeMounts:\n - mountPath: /host/opt/cni/bin\n name: + cni-bin-dir\n - mountPath: /host/etc/cni/net.d\n name: + cni-net-dir\n - name: kubeadm-config\n mountPath: /etc/kubeadm-config/\n + \ securityContext:\n windowsOptions:\n hostProcess: + true\n runAsUserName: \"NT AUTHORITY\\\\system\"\n containers:\n + \ - name: calico-node-startup\n image: sigwindowstools/calico-node:v3.25.0-hostprocess\n + \ args: [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/node-service.ps1\"]\n + \ workingDir: \"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/\"\n imagePullPolicy: + Always\n volumeMounts:\n - name: calico-config-windows\n mountPath: + /etc/kube-calico-windows/\n env:\n - name: POD_NAME\n valueFrom:\n + \ fieldRef:\n apiVersion: v1\n fieldPath: + metadata.name\n - name: POD_NAMESPACE\n valueFrom:\n fieldRef:\n + \ apiVersion: v1\n fieldPath: metadata.namespace\n - + name: CNI_IPAM_TYPE\n value: \"calico-ipam\"\n - name: CALICO_NETWORKING_BACKEND\n + \ value: \"vxlan\"\n - name: KUBECONFIG\n value: \"C:/etc/cni/net.d/calico-kubeconfig\"\n + \ - name: VXLAN_VNI\n value: \"4096\"\n - name: calico-node-felix\n + \ image: sigwindowstools/calico-node:v3.25.0-hostprocess\n args: + [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/felix-service.ps1\"]\n imagePullPolicy: + Always\n workingDir: \"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/\"\n volumeMounts:\n + \ - name: calico-config-windows\n mountPath: /etc/kube-calico-windows/\n + \ - name: calico-static-rules\n mountPath: /calico/static-rules.json\n + \ subPath: static-rules.json\n env:\n - name: POD_NAME\n + \ valueFrom:\n fieldRef:\n apiVersion: v1\n fieldPath: + metadata.name\n - name: POD_NAMESPACE\n valueFrom:\n fieldRef:\n + \ apiVersion: v1\n fieldPath: metadata.namespace\n - + name: VXLAN_VNI\n value: \"4096\"\n - name: KUBECONFIG\n value: + \"C:/etc/cni/net.d/calico-kubeconfig\"\n volumes:\n - name: calico-config-windows\n + \ configMap:\n name: calico-config-windows\n - name: calico-static-rules\n + \ configMap:\n name: calico-static-rules\n # Used to install + CNI.\n - name: cni-bin-dir\n hostPath:\n path: /opt/cni/bin\n + \ - name: cni-net-dir\n hostPath:\n path: /etc/cni/net.d\n + \ - name: kubeadm-config\n configMap:\n name: kubeadm-config\n" +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cni-${CLUSTER_NAME}-calico-windows + namespace: default +--- +apiVersion: v1 +data: + csi-proxy: | + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + k8s-app: csi-proxy + name: csi-proxy + namespace: kube-system + spec: + selector: + matchLabels: + k8s-app: csi-proxy + template: + metadata: + labels: + k8s-app: csi-proxy + spec: + nodeSelector: + "kubernetes.io/os": windows + securityContext: + windowsOptions: + hostProcess: true + runAsUserName: "NT AUTHORITY\\SYSTEM" + hostNetwork: true + containers: + - name: csi-proxy + image: ghcr.io/kubernetes-sigs/sig-windows/csi-proxy:v1.0.2 +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: csi-proxy-addon + namespace: default +--- +apiVersion: v1 +data: + containerd-windows-logger: | + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + k8s-app: containerd-logger + name: containerd-logger + namespace: kube-system + spec: + selector: + matchLabels: + k8s-app: containerd-logger + template: + metadata: + labels: + k8s-app: containerd-logger + spec: + securityContext: + windowsOptions: + hostProcess: true + runAsUserName: "NT AUTHORITY\\system" + hostNetwork: true + containers: + - image: ghcr.io/kubernetes-sigs/sig-windows/eventflow-logger:v0.1.0 + args: [ "config.json" ] + name: containerd-logger + imagePullPolicy: Always + volumeMounts: + - name: containerd-logger-config + mountPath: /config.json + subPath: config.json + nodeSelector: + kubernetes.io/os: windows + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - operator: Exists + volumes: + - configMap: + name: containerd-logger-config + name: containerd-logger-config + updateStrategy: + type: RollingUpdate + --- + kind: ConfigMap + apiVersion: v1 + metadata: + name: containerd-logger-config + namespace: kube-system + data: + config.json: | + { + "inputs": [ + { + "type": "ETW", + "sessionNamePrefix": "containerd", + "cleanupOldSessions": true, + "reuseExistingSession": true, + "providers": [ + { + "providerName": "Microsoft.Virtualization.RunHCS", + "providerGuid": "0B52781F-B24D-5685-DDF6-69830ED40EC3", + "level": "Verbose" + }, + { + "providerName": "ContainerD", + "providerGuid": "2acb92c0-eb9b-571a-69cf-8f3410f383ad", + "level": "Verbose" + } + ] + } + ], + "filters": [ + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == Stats && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::LayerID && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::NameToGuid && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.Stats && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.State && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetProcessProperties && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetComputeSystemProperties && hasnoproperty error" + } + ], + "outputs": [ + { + "type": "StdOutput" + } + ], + "schemaVersion": "2016-08-11" + } +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: containerd-logger-${CLUSTER_NAME} + namespace: default diff --git a/templates/test/ci/cluster-template-prow-external-cloud-provider-ci-version.yaml b/templates/test/ci/cluster-template-prow-intree-cloud-provider.yaml similarity index 63% rename from templates/test/ci/cluster-template-prow-external-cloud-provider-ci-version.yaml rename to templates/test/ci/cluster-template-prow-intree-cloud-provider.yaml index 452f1733597..93d07bc5c46 100644 --- a/templates/test/ci/cluster-template-prow-external-cloud-provider-ci-version.yaml +++ b/templates/test/ci/cluster-template-prow-intree-cloud-provider.yaml @@ -5,7 +5,6 @@ metadata: cni-windows: ${CLUSTER_NAME}-calico containerd-logger: enabled csi-proxy: enabled - metrics-server: enabled name: ${CLUSTER_NAME} namespace: default spec: @@ -61,7 +60,7 @@ spec: apiServer: extraArgs: cloud-config: /etc/kubernetes/azure.json - cloud-provider: external + cloud-provider: azure feature-gates: ${K8S_FEATURE_GATES:-""} extraVolumes: - hostPath: /etc/kubernetes/azure.json @@ -73,10 +72,8 @@ spec: extraArgs: allocate-node-cidrs: "false" cloud-config: /etc/kubernetes/azure.json - cloud-provider: external + cloud-provider: azure cluster-name: ${CLUSTER_NAME} - external-cloud-volume-plugin: azure - feature-gates: CSIMigrationAzureDisk=true v: "4" extraVolumes: - hostPath: /etc/kubernetes/azure.json @@ -88,7 +85,6 @@ spec: dataDir: /var/lib/etcddisk/etcd extraArgs: quota-backend-bytes: "8589934592" - kubernetesVersion: ci/${CI_VERSION} diskSetup: filesystems: - device: /dev/disk/azure/scsi1/lun0 @@ -114,92 +110,25 @@ spec: owner: root:root path: /etc/kubernetes/azure.json permissions: "0644" - - content: | - #!/bin/bash - - set -o nounset - set -o pipefail - set -o errexit - [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" - - # This test installs release packages or binaries that are a result of the CI and release builds. - # It runs '... --version' commands to verify that the binaries are correctly installed - # and finally uninstalls the packages. - # For the release packages it tests all versions in the support skew. - LINE_SEPARATOR="*************************************************" - echo "$$LINE_SEPARATOR" - CI_VERSION=${CI_VERSION} - if [[ "$${CI_VERSION}" != "" ]]; then - CI_DIR=/tmp/k8s-ci - mkdir -p $$CI_DIR - declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm") - declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler") - CONTAINER_EXT="tar" - echo "* testing CI version $$CI_VERSION" - # Check for semver - if [[ "$${CI_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - VERSION_WITHOUT_PREFIX="${CI_VERSION#v}" - DEBIAN_FRONTEND=noninteractive apt-get install -y apt-transport-https curl - curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - - echo 'deb https://apt.kubernetes.io/ kubernetes-xenial main' > /etc/apt/sources.list.d/kubernetes.list - apt-get update - # replace . with \. - VERSION_REGEX="${VERSION_WITHOUT_PREFIX//./\\.}" - PACKAGE_VERSION="$(apt-cache madison kubelet|grep $${VERSION_REGEX}- | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')" - for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do - echo "* installing package: $$CI_PACKAGE $${PACKAGE_VERSION}" - DEBIAN_FRONTEND=noninteractive apt-get install -y $$CI_PACKAGE=$$PACKAGE_VERSION - done - else - CI_URL="https://storage.googleapis.com/k8s-release-dev/ci/$${CI_VERSION}/bin/linux/amd64" - for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do - echo "* downloading binary: $$CI_URL/$$CI_PACKAGE" - wget "$$CI_URL/$$CI_PACKAGE" -nv -O "$$CI_DIR/$$CI_PACKAGE" - chmod +x "$$CI_DIR/$$CI_PACKAGE" - mv "$$CI_DIR/$$CI_PACKAGE" "/usr/bin/$$CI_PACKAGE" - done - IMAGE_REGISTRY_PREFIX=registry.k8s.io - for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do - echo "* downloading package: $$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" - wget "$$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" -nv -O "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" - $${SUDO} ctr -n k8s.io images import "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" || echo "* ignoring expected 'ctr images import' result" - $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER:"$${CI_VERSION//+/_}" - $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" gcr.io/k8s-staging-ci-images/$$CI_CONTAINER:"$${CI_VERSION//+/_}" - done - fi - fi - echo "* checking binary versions" - echo "ctr version: " $(ctr version) - echo "kubeadm version: " $(kubeadm version -o=short) - echo "kubectl version: " $(kubectl version --client=true --short=true) - echo "kubelet version: " $(kubelet --version) - echo "$$LINE_SEPARATOR" - owner: root:root - path: /tmp/kubeadm-bootstrap.sh - permissions: "0744" initConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: external - feature-gates: CSIMigrationAzureDisk=true + cloud-provider: azure name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: external - feature-gates: CSIMigrationAzureDisk=true + cloud-provider: azure name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk - /var/lib/etcddisk postKubeadmCommands: [] - preKubeadmCommands: - - bash -c /tmp/kubeadm-bootstrap.sh - verbosity: 5 + preKubeadmCommands: [] machineTemplate: infrastructureRef: apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 @@ -220,12 +149,6 @@ spec: - diskSizeGB: 256 lun: 0 nameSuffix: etcddisk - image: - marketplace: - offer: capi - publisher: cncf-upstream - sku: ubuntu-1804-gen1 - version: latest osDisk: diskSizeGB: 128 osType: Linux @@ -266,12 +189,6 @@ metadata: spec: template: spec: - image: - marketplace: - offer: capi - publisher: cncf-upstream - sku: ubuntu-1804-gen1 - version: latest osDisk: diskSizeGB: 128 osType: Linux @@ -303,80 +220,14 @@ spec: owner: root:root path: /etc/kubernetes/azure.json permissions: "0644" - - content: | - #!/bin/bash - - set -o nounset - set -o pipefail - set -o errexit - [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" - - # This test installs release packages or binaries that are a result of the CI and release builds. - # It runs '... --version' commands to verify that the binaries are correctly installed - # and finally uninstalls the packages. - # For the release packages it tests all versions in the support skew. - LINE_SEPARATOR="*************************************************" - echo "$$LINE_SEPARATOR" - CI_VERSION=${CI_VERSION} - if [[ "$${CI_VERSION}" != "" ]]; then - CI_DIR=/tmp/k8s-ci - mkdir -p $$CI_DIR - declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm") - declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler") - CONTAINER_EXT="tar" - echo "* testing CI version $$CI_VERSION" - # Check for semver - if [[ "$${CI_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - VERSION_WITHOUT_PREFIX="${CI_VERSION#v}" - DEBIAN_FRONTEND=noninteractive apt-get install -y apt-transport-https curl - curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - - echo 'deb https://apt.kubernetes.io/ kubernetes-xenial main' > /etc/apt/sources.list.d/kubernetes.list - apt-get update - # replace . with \. - VERSION_REGEX="${VERSION_WITHOUT_PREFIX//./\\.}" - PACKAGE_VERSION="$(apt-cache madison kubelet|grep $${VERSION_REGEX}- | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')" - for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do - echo "* installing package: $$CI_PACKAGE $${PACKAGE_VERSION}" - DEBIAN_FRONTEND=noninteractive apt-get install -y $$CI_PACKAGE=$$PACKAGE_VERSION - done - else - CI_URL="https://storage.googleapis.com/k8s-release-dev/ci/$${CI_VERSION}/bin/linux/amd64" - for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do - echo "* downloading binary: $$CI_URL/$$CI_PACKAGE" - wget "$$CI_URL/$$CI_PACKAGE" -nv -O "$$CI_DIR/$$CI_PACKAGE" - chmod +x "$$CI_DIR/$$CI_PACKAGE" - mv "$$CI_DIR/$$CI_PACKAGE" "/usr/bin/$$CI_PACKAGE" - done - IMAGE_REGISTRY_PREFIX=registry.k8s.io - for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do - echo "* downloading package: $$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" - wget "$$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" -nv -O "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" - $${SUDO} ctr -n k8s.io images import "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" || echo "* ignoring expected 'ctr images import' result" - $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER:"$${CI_VERSION//+/_}" - $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" gcr.io/k8s-staging-ci-images/$$CI_CONTAINER:"$${CI_VERSION//+/_}" - done - fi - fi - echo "* checking binary versions" - echo "ctr version: " $(ctr version) - echo "kubeadm version: " $(kubeadm version -o=short) - echo "kubectl version: " $(kubectl version --client=true --short=true) - echo "kubelet version: " $(kubelet --version) - echo "$$LINE_SEPARATOR" - owner: root:root - path: /tmp/kubeadm-bootstrap.sh - permissions: "0744" joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json cloud-config: /etc/kubernetes/azure.json - cloud-provider: external - feature-gates: CSIMigrationAzureDisk=true + cloud-provider: azure name: '{{ ds.meta_data["local_hostname"] }}' - preKubeadmCommands: - - bash -c /tmp/kubeadm-bootstrap.sh - verbosity: 5 + preKubeadmCommands: [] --- apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineDeployment @@ -415,12 +266,6 @@ spec: runtime: containerd windowsServerVersion: ${WINDOWS_SERVER_VERSION:=""} spec: - image: - marketplace: - offer: capi-windows - publisher: cncf-upstream - sku: ${WINDOWS_SERVER_VERSION:=windows-2019}-containerd-gen1 - version: latest osDisk: diskSizeGB: 128 managedDisk: @@ -487,42 +332,14 @@ spec: sc.exe start sshd path: C:/collect-hns-crashes.ps1 permissions: "0744" - - content: | - $ErrorActionPreference = 'Stop' - - Stop-Service kubelet -Force - - $$CI_VERSION="${CI_VERSION}" - if($$CI_VERSION -ne "") - { - $$binaries=@("kubeadm", "kubectl", "kubelet", "kube-proxy") - $$ci_url="https://storage.googleapis.com/k8s-release-dev/ci/$$CI_VERSION/bin/windows/amd64" - foreach ( $$binary in $$binaries ) - { - echo "downloading binary: $$ci_url/$$binary.exe" - curl.exe --retry 10 --retry-delay 5 "$$ci_url/$$binary.exe" --output "c:/k/$$binary.exe" - } - } - - # Tag it to the ci version. The image knows how to use the copy locally with the configmap - # that is applied at at this stage (windows-kubeproxy-ci.yaml) - ctr.exe -n k8s.io images pull docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess - ctr.exe -n k8s.io images tag docker.io/sigwindowstools/kube-proxy:v1.23.1-calico-hostprocess "docker.io/sigwindowstools/kube-proxy:${CI_VERSION/+/_}-calico-hostprocess" - - kubeadm.exe version -o=short - kubectl.exe version --client=true --short=true - kubelet.exe --version - kube-proxy.exe --version - path: C:/replace-ci-binaries.ps1 - permissions: "0744" joinConfiguration: nodeRegistration: criSocket: npipe:////./pipe/containerd-containerd kubeletExtraArgs: azure-container-registry-config: c:/k/azure.json cloud-config: c:/k/azure.json - cloud-provider: external - feature-gates: WindowsHostProcessContainers=true,CSIMigrationAzureDisk=true + cloud-provider: azure + feature-gates: WindowsHostProcessContainers=true v: "2" windows-priorityclass: ABOVE_NORMAL_PRIORITY_CLASS name: '{{ ds.meta_data["local_hostname"] }}' @@ -533,7 +350,6 @@ spec: - powershell C:/create-temp-folder.ps1 - powershell C:/replace-containerd.ps1 - powershell C:/collect-hns-crashes.ps1 - - powershell C:/replace-ci-binaries.ps1 users: - groups: Administrators name: capi @@ -616,14 +432,6 @@ spec: --- apiVersion: v1 data: - kube-proxy-patch: |- - apiVersion: v1 - kind: ConfigMap - metadata: - name: windows-kubeproxy-ci - namespace: kube-system - data: - KUBEPROXY_PATH: "c:/k/kube-proxy.exe" proxy: | apiVersion: apps/v1 kind: DaemonSet @@ -942,233 +750,3 @@ metadata: type: generated name: containerd-logger-${CLUSTER_NAME} namespace: default ---- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: metrics-server-${CLUSTER_NAME} - namespace: default -spec: - clusterSelector: - matchLabels: - metrics-server: enabled - resources: - - kind: ConfigMap - name: metrics-server-${CLUSTER_NAME} - strategy: ApplyOnce ---- -apiVersion: v1 -data: - metrics-server: | - apiVersion: v1 - kind: ServiceAccount - metadata: - labels: - k8s-app: metrics-server - name: metrics-server - namespace: kube-system - --- - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRole - metadata: - labels: - k8s-app: metrics-server - rbac.authorization.k8s.io/aggregate-to-admin: "true" - rbac.authorization.k8s.io/aggregate-to-edit: "true" - rbac.authorization.k8s.io/aggregate-to-view: "true" - name: system:aggregated-metrics-reader - rules: - - apiGroups: - - metrics.k8s.io - resources: - - pods - - nodes - verbs: - - get - - list - - watch - --- - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRole - metadata: - labels: - k8s-app: metrics-server - name: system:metrics-server - rules: - - apiGroups: - - "" - resources: - - nodes/metrics - verbs: - - get - - apiGroups: - - "" - resources: - - pods - - nodes - verbs: - - get - - list - - watch - --- - apiVersion: rbac.authorization.k8s.io/v1 - kind: RoleBinding - metadata: - labels: - k8s-app: metrics-server - name: metrics-server-auth-reader - namespace: kube-system - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: extension-apiserver-authentication-reader - subjects: - - kind: ServiceAccount - name: metrics-server - namespace: kube-system - --- - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRoleBinding - metadata: - labels: - k8s-app: metrics-server - name: metrics-server:system:auth-delegator - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:auth-delegator - subjects: - - kind: ServiceAccount - name: metrics-server - namespace: kube-system - --- - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRoleBinding - metadata: - labels: - k8s-app: metrics-server - name: system:metrics-server - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:metrics-server - subjects: - - kind: ServiceAccount - name: metrics-server - namespace: kube-system - --- - apiVersion: v1 - kind: Service - metadata: - labels: - k8s-app: metrics-server - name: metrics-server - namespace: kube-system - spec: - ports: - - name: https - port: 443 - protocol: TCP - targetPort: https - selector: - k8s-app: metrics-server - --- - apiVersion: apps/v1 - kind: Deployment - metadata: - labels: - k8s-app: metrics-server - name: metrics-server - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: metrics-server - strategy: - rollingUpdate: - maxUnavailable: 0 - template: - metadata: - labels: - k8s-app: metrics-server - spec: - containers: - - args: - - --cert-dir=/tmp - - --secure-port=4443 - - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname - - --kubelet-use-node-status-port - - --metric-resolution=15s - - --kubelet-insecure-tls - image: k8s.gcr.io/metrics-server/metrics-server:v0.6.2 - imagePullPolicy: IfNotPresent - livenessProbe: - failureThreshold: 3 - httpGet: - path: /livez - port: https - scheme: HTTPS - periodSeconds: 10 - name: metrics-server - ports: - - containerPort: 4443 - name: https - protocol: TCP - readinessProbe: - failureThreshold: 3 - httpGet: - path: /readyz - port: https - scheme: HTTPS - initialDelaySeconds: 20 - periodSeconds: 10 - resources: - requests: - cpu: 100m - memory: 200Mi - securityContext: - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true - runAsNonRoot: true - runAsUser: 1000 - volumeMounts: - - mountPath: /tmp - name: tmp-dir - nodeSelector: - kubernetes.io/os: linux - priorityClassName: system-cluster-critical - serviceAccountName: metrics-server - tolerations: - - effect: NoSchedule - key: node-role.kubernetes.io/master - operator: Exists - - effect: NoSchedule - key: node-role.kubernetes.io/control-plane - operator: Exists - volumes: - - emptyDir: {} - name: tmp-dir - --- - apiVersion: apiregistration.k8s.io/v1 - kind: APIService - metadata: - labels: - k8s-app: metrics-server - name: v1beta1.metrics.k8s.io - spec: - group: metrics.k8s.io - groupPriorityMinimum: 100 - insecureSkipTLSVerify: true - service: - name: metrics-server - namespace: kube-system - version: v1beta1 - versionPriority: 100 -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: metrics-server-${CLUSTER_NAME} - namespace: default diff --git a/templates/test/ci/cluster-template-prow-ipv6.yaml b/templates/test/ci/cluster-template-prow-ipv6.yaml index 8f78ac50ad7..b76f78a6712 100644 --- a/templates/test/ci/cluster-template-prow-ipv6.yaml +++ b/templates/test/ci/cluster-template-prow-ipv6.yaml @@ -1,8 +1,6 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: - labels: - cni: calico-ipv6 name: ${CLUSTER_NAME} namespace: default spec: diff --git a/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml b/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml index 0aa0b1019e2..12ee2241e0e 100644 --- a/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml +++ b/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml @@ -60,27 +60,15 @@ spec: clusterConfiguration: apiServer: extraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external feature-gates: ${K8S_FEATURE_GATES:-""} - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true timeoutForControlPlane: 20m controllerManager: extraArgs: allocate-node-cidrs: "false" - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} v: "4" - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true etcd: local: dataDir: /var/lib/etcddisk/etcd @@ -179,15 +167,13 @@ spec: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -369,8 +355,7 @@ spec: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: - bash -c /tmp/kubeadm-bootstrap.sh @@ -486,8 +471,7 @@ spec: criSocket: npipe:////./pipe/containerd-containerd kubeletExtraArgs: azure-container-registry-config: c:/k/azure.json - cloud-config: c:/k/azure.json - cloud-provider: azure + cloud-provider: external feature-gates: WindowsHostProcessContainers=true pod-infra-container-image: mcr.microsoft.com/oss/kubernetes/pause:3.4.1 name: '{{ ds.meta_data["local_hostname"] }}' diff --git a/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml b/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml new file mode 100644 index 00000000000..e5f9e574c19 --- /dev/null +++ b/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml @@ -0,0 +1,679 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + labels: + cni-windows: ${CLUSTER_NAME}-calico + containerd-logger: enabled + csi-proxy: enabled + windows: enabled + name: ${CLUSTER_NAME} + namespace: default +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlane + name: ${CLUSTER_NAME}-control-plane + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureCluster + name: ${CLUSTER_NAME} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureCluster +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + additionalTags: + buildProvenance: ${BUILD_PROVENANCE} + creationTimestamp: ${TIMESTAMP} + jobName: ${JOB_NAME} + identityRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureClusterIdentity + name: ${CLUSTER_IDENTITY_NAME} + location: ${AZURE_LOCATION} + networkSpec: + subnets: + - name: control-plane-subnet + role: control-plane + - name: node-subnet + natGateway: + name: node-natgateway + role: node + vnet: + name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} + resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} + subscriptionID: ${AZURE_SUBSCRIPTION_ID} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlane +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + kubeadmConfigSpec: + clusterConfiguration: + apiServer: + extraArgs: + cloud-provider: external + feature-gates: ${K8S_FEATURE_GATES:-""} + timeoutForControlPlane: 20m + controllerManager: + extraArgs: + allocate-node-cidrs: "false" + cloud-provider: external + cluster-name: ${CLUSTER_NAME} + v: "4" + etcd: + local: + dataDir: /var/lib/etcddisk/etcd + extraArgs: + quota-backend-bytes: "8589934592" + diskSetup: + filesystems: + - device: /dev/disk/azure/scsi1/lun0 + extraOpts: + - -E + - lazy_itable_init=1,lazy_journal_init=1 + filesystem: ext4 + label: etcd_disk + - device: ephemeral0.1 + filesystem: ext4 + label: ephemeral0 + replaceFS: ntfs + partitions: + - device: /dev/disk/azure/scsi1/lun0 + layout: true + overwrite: false + tableType: gpt + files: + - contentFrom: + secret: + key: control-plane-azure.json + name: ${CLUSTER_NAME}-control-plane-azure-json + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + azure-container-registry-config: /etc/kubernetes/azure.json + cloud-provider: external + name: '{{ ds.meta_data["local_hostname"] }}' + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + azure-container-registry-config: /etc/kubernetes/azure.json + cloud-provider: external + name: '{{ ds.meta_data["local_hostname"] }}' + mounts: + - - LABEL=etcd_disk + - /var/lib/etcddisk + postKubeadmCommands: [] + preKubeadmCommands: [] + machineTemplate: + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-control-plane + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + template: + spec: + dataDisks: + - diskSizeGB: 256 + lun: 0 + nameSuffix: etcddisk + osDisk: + diskSizeGB: 128 + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachinePool +metadata: + name: ${CLUSTER_NAME}-mp-0 + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT} + template: + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfig + name: ${CLUSTER_NAME}-mp-0 + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachinePool + name: ${CLUSTER_NAME}-mp-0 + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachinePool +metadata: + name: ${CLUSTER_NAME}-mp-0 + namespace: default +spec: + location: ${AZURE_LOCATION} + orchestrationMode: Flexible + strategy: + rollingUpdate: + maxSurge: 0% + maxUnavailable: 0 + type: RollingUpdate + template: + osDisk: + diskSizeGB: 30 + managedDisk: + storageAccountType: Premium_LRS + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + vmExtensions: + - name: CustomScript + protectedSettings: + commandToExecute: | + #!/bin/sh + echo "This script is a no-op used for extension testing purposes ..." + touch test_file + publisher: Microsoft.Azure.Extensions + version: "2.1" + vmSize: ${AZURE_NODE_MACHINE_TYPE} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfig +metadata: + name: ${CLUSTER_NAME}-mp-0 + namespace: default +spec: + files: + - contentFrom: + secret: + key: worker-node-azure.json + name: ${CLUSTER_NAME}-mp-0-azure-json + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + azure-container-registry-config: /etc/kubernetes/azure.json + cloud-provider: external + name: '{{ ds.meta_data["local_hostname"] }}' +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureClusterIdentity +metadata: + labels: + clusterctl.cluster.x-k8s.io/move-hierarchy: "true" + name: ${CLUSTER_IDENTITY_NAME} + namespace: default +spec: + allowedNamespaces: {} + clientID: ${AZURE_CLIENT_ID} + clientSecret: + name: ${AZURE_CLUSTER_IDENTITY_SECRET_NAME} + namespace: ${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE} + tenantID: ${AZURE_TENANT_ID} + type: ServicePrincipal +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachinePool +metadata: + name: ${CLUSTER_NAME}-mp-win + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WINDOWS_WORKER_MACHINE_COUNT:-0} + template: + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfig + name: ${CLUSTER_NAME}-mp-win + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachinePool + name: ${CLUSTER_NAME}-mp-win + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachinePool +metadata: + annotations: + runtime: containerd + name: ${CLUSTER_NAME}-mp-win + namespace: default +spec: + location: ${AZURE_LOCATION} + orchestrationMode: Flexible + strategy: + rollingUpdate: + maxSurge: 0% + maxUnavailable: 0 + type: RollingUpdate + template: + osDisk: + diskSizeGB: 30 + managedDisk: + storageAccountType: Premium_LRS + osType: Windows + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + vmSize: ${AZURE_NODE_MACHINE_TYPE} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfig +metadata: + name: ${CLUSTER_NAME}-mp-win + namespace: default +spec: + files: + - contentFrom: + secret: + key: worker-node-azure.json + name: ${CLUSTER_NAME}-mp-win-azure-json + owner: root:root + path: c:/k/azure.json + permissions: "0644" + - content: Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico.exe + path: C:/defender-exclude-calico.ps1 + permissions: "0744" + joinConfiguration: + nodeRegistration: + criSocket: npipe:////./pipe/containerd-containerd + kubeletExtraArgs: + azure-container-registry-config: c:/k/azure.json + cloud-provider: external + feature-gates: WindowsHostProcessContainers=true + pod-infra-container-image: mcr.microsoft.com/oss/kubernetes/pause:3.4.1 + name: '{{ ds.meta_data["local_hostname"] }}' + postKubeadmCommands: + - nssm set kubelet start SERVICE_AUTO_START + - powershell C:/defender-exclude-calico.ps1 + preKubeadmCommands: + - powershell c:/create-external-network.ps1 + users: + - groups: Administrators + name: capi + sshAuthorizedKeys: + - ${AZURE_SSH_PUBLIC_KEY:=""} +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: ${CLUSTER_NAME}-calico-windows + namespace: default +spec: + clusterSelector: + matchLabels: + cni-windows: ${CLUSTER_NAME}-calico + resources: + - kind: ConfigMap + name: cni-${CLUSTER_NAME}-calico-windows + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: csi-proxy + namespace: default +spec: + clusterSelector: + matchLabels: + csi-proxy: enabled + resources: + - kind: ConfigMap + name: csi-proxy-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: containerd-logger-${CLUSTER_NAME} + namespace: default +spec: + clusterSelector: + matchLabels: + containerd-logger: enabled + resources: + - kind: ConfigMap + name: containerd-logger-${CLUSTER_NAME} + strategy: ApplyOnce +--- +apiVersion: v1 +data: + proxy: | + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + k8s-app: kube-proxy + name: kube-proxy-windows + namespace: kube-system + spec: + selector: + matchLabels: + k8s-app: kube-proxy-windows + template: + metadata: + labels: + k8s-app: kube-proxy-windows + spec: + serviceAccountName: kube-proxy + securityContext: + windowsOptions: + hostProcess: true + runAsUserName: "NT AUTHORITY\\system" + hostNetwork: true + containers: + - image: sigwindowstools/kube-proxy:${KUBERNETES_VERSION/+/_}-calico-hostprocess + args: ["$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/start.ps1"] + workingDir: "$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/" + name: kube-proxy + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: KUBEPROXY_PATH + valueFrom: + configMapKeyRef: + name: windows-kubeproxy-ci + key: KUBEPROXY_PATH + optional: true + volumeMounts: + - mountPath: /var/lib/kube-proxy + name: kube-proxy + nodeSelector: + kubernetes.io/os: windows + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - operator: Exists + volumes: + - configMap: + name: kube-proxy + name: kube-proxy + updateStrategy: + type: RollingUpdate + windows-cni: "# strictAffinity required for windows\napiVersion: crd.projectcalico.org/v1\nkind: + IPAMConfig\nmetadata:\n name: default\nspec:\n autoAllocateBlocks: true\n strictAffinity: + true\n---\nkind: ConfigMap\napiVersion: v1\nmetadata:\n name: calico-static-rules\n + \ namespace: calico-system\n labels:\n tier: node\n app: calico\ndata:\n + \ static-rules.json: |\n {\n \"Provider\": \"azure\",\n \"Version\": + \"0.1\",\n \"Rules\": [\n {\n \"Name\": \"EndpointPolicy\",\n + \ \"Rule\": {\n \"Id\": \"wireserver\",\n \"Type\": + \"ACL\",\n \"Protocol\": 6,\n \"Action\": \"Block\",\n + \ \"Direction\": \"Out\",\n \"RemoteAddresses\": \"168.63.129.16/32\",\n + \ \"RemotePorts\": \"80\",\n \"Priority\": 200,\n \"RuleType\": + \"Switch\"\n }\n }\n ]\n } \n---\nkind: ConfigMap\napiVersion: + v1\nmetadata:\n name: calico-config-windows\n namespace: calico-system\n labels:\n + \ tier: node\n app: calico\ndata:\n veth_mtu: \"1350\"\n \n cni_network_config: + |\n {\n \"name\": \"Calico\",\n \"cniVersion\": \"0.3.1\",\n \"plugins\": + [\n {\n \"windows_use_single_network\": true,\n \"type\": + \"calico\",\n \"mode\": \"vxlan\",\n \"nodename\": \"__KUBERNETES_NODE_NAME__\",\n + \ \"nodename_file_optional\": true,\n \"log_file_path\": \"c:/cni.log\",\n + \ \"log_level\": \"debug\",\n\n \"vxlan_mac_prefix\": \"0E-2A\",\n + \ \"vxlan_vni\": 4096,\n \"mtu\": __CNI_MTU__,\n \"policy\": + {\n \"type\": \"k8s\"\n },\n\n \"log_level\": \"info\",\n\n + \ \"capabilities\": {\"dns\": true},\n \"DNS\": {\n \"Search\": + \ [\n \"svc.cluster.local\"\n ]\n },\n\n \"datastore_type\": + \"kubernetes\",\n\n \"kubernetes\": {\n \"kubeconfig\": \"__KUBECONFIG_FILEPATH__\"\n + \ },\n\n \"ipam\": {\n \"type\": \"calico-ipam\",\n + \ \"subnet\": \"usePodCidr\"\n },\n\n \"policies\": + \ [\n {\n \"Name\": \"EndpointPolicy\",\n \"Value\": + \ {\n \"Type\": \"OutBoundNAT\",\n \"ExceptionList\": + \ [\n \"__K8S_SERVICE_CIDR__\"\n ]\n }\n + \ },\n {\n \"Name\": \"EndpointPolicy\",\n + \ \"Value\": {\n \"Type\": \"SDNROUTE\",\n \"DestinationPrefix\": + \ \"__K8S_SERVICE_CIDR__\",\n \"NeedEncap\": true\n }\n + \ }\n ]\n }\n ]\n\n }\n---\napiVersion: apps/v1\nkind: + DaemonSet\nmetadata:\n name: calico-node-windows\n labels:\n tier: node\n + \ app: calico\n namespace: calico-system\nspec:\n selector:\n matchLabels:\n + \ app: calico\n template:\n metadata:\n labels:\n tier: node\n + \ app: calico\n spec:\n affinity:\n nodeAffinity:\n requiredDuringSchedulingIgnoredDuringExecution:\n + \ nodeSelectorTerms:\n - matchExpressions:\n - + key: kubernetes.io/os\n operator: In\n values:\n + \ - windows\n - key: kubernetes.io/arch\n + \ operator: In\n values:\n - + amd64\n securityContext:\n windowsOptions:\n hostProcess: + true\n runAsUserName: \"NT AUTHORITY\\\\system\"\n hostNetwork: + true\n serviceAccountName: calico-node\n tolerations:\n - operator: + Exists\n effect: NoSchedule\n # Mark the pod as a critical add-on + for rescheduling.\n - key: CriticalAddonsOnly\n operator: Exists\n + \ - effect: NoExecute\n operator: Exists\n initContainers:\n # + This container installs the CNI binaries\n # and CNI network config file + on each node.\n - name: install-cni\n image: sigwindowstools/calico-install:v3.25.0-hostprocess\n + \ args: [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/install.ps1\"]\n + \ imagePullPolicy: Always\n env:\n # Name of the CNI + config file to create.\n - name: CNI_CONF_NAME\n value: + \"10-calico.conflist\"\n # The CNI network config to install on each + node.\n - name: CNI_NETWORK_CONFIG\n valueFrom:\n configMapKeyRef:\n + \ name: calico-config-windows\n key: cni_network_config\n + \ # Set the hostname based on the k8s node name.\n - name: + KUBERNETES_NODE_NAME\n valueFrom:\n fieldRef:\n fieldPath: + spec.nodeName\n # CNI MTU Config variable\n - name: CNI_MTU\n + \ valueFrom:\n configMapKeyRef:\n name: + calico-config-windows\n key: veth_mtu\n # Prevents + the container from sleeping forever.\n - name: SLEEP\n value: + \"false\"\n - name: K8S_SERVICE_CIDR\n value: \"10.96.0.0/12\"\n + \ volumeMounts:\n - mountPath: /host/opt/cni/bin\n name: + cni-bin-dir\n - mountPath: /host/etc/cni/net.d\n name: + cni-net-dir\n - name: kubeadm-config\n mountPath: /etc/kubeadm-config/\n + \ securityContext:\n windowsOptions:\n hostProcess: + true\n runAsUserName: \"NT AUTHORITY\\\\system\"\n containers:\n + \ - name: calico-node-startup\n image: sigwindowstools/calico-node:v3.25.0-hostprocess\n + \ args: [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/node-service.ps1\"]\n + \ workingDir: \"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/\"\n imagePullPolicy: + Always\n volumeMounts:\n - name: calico-config-windows\n mountPath: + /etc/kube-calico-windows/\n env:\n - name: POD_NAME\n valueFrom:\n + \ fieldRef:\n apiVersion: v1\n fieldPath: + metadata.name\n - name: POD_NAMESPACE\n valueFrom:\n fieldRef:\n + \ apiVersion: v1\n fieldPath: metadata.namespace\n - + name: CNI_IPAM_TYPE\n value: \"calico-ipam\"\n - name: CALICO_NETWORKING_BACKEND\n + \ value: \"vxlan\"\n - name: KUBECONFIG\n value: \"C:/etc/cni/net.d/calico-kubeconfig\"\n + \ - name: VXLAN_VNI\n value: \"4096\"\n - name: calico-node-felix\n + \ image: sigwindowstools/calico-node:v3.25.0-hostprocess\n args: + [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/felix-service.ps1\"]\n imagePullPolicy: + Always\n workingDir: \"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/\"\n volumeMounts:\n + \ - name: calico-config-windows\n mountPath: /etc/kube-calico-windows/\n + \ - name: calico-static-rules\n mountPath: /calico/static-rules.json\n + \ subPath: static-rules.json\n env:\n - name: POD_NAME\n + \ valueFrom:\n fieldRef:\n apiVersion: v1\n fieldPath: + metadata.name\n - name: POD_NAMESPACE\n valueFrom:\n fieldRef:\n + \ apiVersion: v1\n fieldPath: metadata.namespace\n - + name: VXLAN_VNI\n value: \"4096\"\n - name: KUBECONFIG\n value: + \"C:/etc/cni/net.d/calico-kubeconfig\"\n volumes:\n - name: calico-config-windows\n + \ configMap:\n name: calico-config-windows\n - name: calico-static-rules\n + \ configMap:\n name: calico-static-rules\n # Used to install + CNI.\n - name: cni-bin-dir\n hostPath:\n path: /opt/cni/bin\n + \ - name: cni-net-dir\n hostPath:\n path: /etc/cni/net.d\n + \ - name: kubeadm-config\n configMap:\n name: kubeadm-config\n" +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cni-${CLUSTER_NAME}-calico-windows + namespace: default +--- +apiVersion: v1 +data: + csi-proxy: | + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + k8s-app: csi-proxy + name: csi-proxy + namespace: kube-system + spec: + selector: + matchLabels: + k8s-app: csi-proxy + template: + metadata: + labels: + k8s-app: csi-proxy + spec: + nodeSelector: + "kubernetes.io/os": windows + securityContext: + windowsOptions: + hostProcess: true + runAsUserName: "NT AUTHORITY\\SYSTEM" + hostNetwork: true + containers: + - name: csi-proxy + image: ghcr.io/kubernetes-sigs/sig-windows/csi-proxy:v1.0.2 +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: csi-proxy-addon + namespace: default +--- +apiVersion: v1 +data: + containerd-windows-logger: | + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + k8s-app: containerd-logger + name: containerd-logger + namespace: kube-system + spec: + selector: + matchLabels: + k8s-app: containerd-logger + template: + metadata: + labels: + k8s-app: containerd-logger + spec: + securityContext: + windowsOptions: + hostProcess: true + runAsUserName: "NT AUTHORITY\\system" + hostNetwork: true + containers: + - image: ghcr.io/kubernetes-sigs/sig-windows/eventflow-logger:v0.1.0 + args: [ "config.json" ] + name: containerd-logger + imagePullPolicy: Always + volumeMounts: + - name: containerd-logger-config + mountPath: /config.json + subPath: config.json + nodeSelector: + kubernetes.io/os: windows + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - operator: Exists + volumes: + - configMap: + name: containerd-logger-config + name: containerd-logger-config + updateStrategy: + type: RollingUpdate + --- + kind: ConfigMap + apiVersion: v1 + metadata: + name: containerd-logger-config + namespace: kube-system + data: + config.json: | + { + "inputs": [ + { + "type": "ETW", + "sessionNamePrefix": "containerd", + "cleanupOldSessions": true, + "reuseExistingSession": true, + "providers": [ + { + "providerName": "Microsoft.Virtualization.RunHCS", + "providerGuid": "0B52781F-B24D-5685-DDF6-69830ED40EC3", + "level": "Verbose" + }, + { + "providerName": "ContainerD", + "providerGuid": "2acb92c0-eb9b-571a-69cf-8f3410f383ad", + "level": "Verbose" + } + ] + } + ], + "filters": [ + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == Stats && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::LayerID && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::NameToGuid && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.Stats && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.State && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetProcessProperties && hasnoproperty error" + }, + { + "type": "drop", + "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetComputeSystemProperties && hasnoproperty error" + } + ], + "outputs": [ + { + "type": "StdOutput" + } + ], + "schemaVersion": "2016-08-11" + } +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: containerd-logger-${CLUSTER_NAME} + namespace: default diff --git a/templates/test/ci/cluster-template-prow-machine-pool.yaml b/templates/test/ci/cluster-template-prow-machine-pool.yaml index 368e1e98e99..eb5a6d54722 100644 --- a/templates/test/ci/cluster-template-prow-machine-pool.yaml +++ b/templates/test/ci/cluster-template-prow-machine-pool.yaml @@ -60,27 +60,15 @@ spec: clusterConfiguration: apiServer: extraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external feature-gates: ${K8S_FEATURE_GATES:-""} - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true timeoutForControlPlane: 20m controllerManager: extraArgs: allocate-node-cidrs: "false" - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} v: "4" - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true etcd: local: dataDir: /var/lib/etcddisk/etcd @@ -115,15 +103,13 @@ spec: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -227,8 +213,7 @@ spec: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 @@ -309,8 +294,7 @@ spec: criSocket: npipe:////./pipe/containerd-containerd kubeletExtraArgs: azure-container-registry-config: c:/k/azure.json - cloud-config: c:/k/azure.json - cloud-provider: azure + cloud-provider: external feature-gates: WindowsHostProcessContainers=true pod-infra-container-image: mcr.microsoft.com/oss/kubernetes/pause:3.4.1 name: '{{ ds.meta_data["local_hostname"] }}' diff --git a/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml b/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml index 53686d7b8bf..818f39cd8ec 100644 --- a/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml +++ b/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml @@ -55,26 +55,14 @@ spec: clusterConfiguration: apiServer: extraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true + cloud-provider: external timeoutForControlPlane: 20m controllerManager: extraArgs: allocate-node-cidrs: "false" - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} v: "4" - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true etcd: local: dataDir: /var/lib/etcddisk/etcd @@ -109,15 +97,13 @@ spec: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -226,6 +212,5 @@ spec: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' diff --git a/templates/test/ci/cluster-template-prow-user-assigned-managed-identity.yaml b/templates/test/ci/cluster-template-prow-user-assigned-managed-identity.yaml deleted file mode 100644 index ccfa830c96d..00000000000 --- a/templates/test/ci/cluster-template-prow-user-assigned-managed-identity.yaml +++ /dev/null @@ -1,237 +0,0 @@ -apiVersion: cluster.x-k8s.io/v1beta1 -kind: Cluster -metadata: - name: ${CLUSTER_NAME} - namespace: default -spec: - clusterNetwork: - pods: - cidrBlocks: - - 192.168.0.0/16 - controlPlaneRef: - apiVersion: controlplane.cluster.x-k8s.io/v1beta1 - kind: KubeadmControlPlane - name: ${CLUSTER_NAME}-control-plane - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureCluster - name: ${CLUSTER_NAME} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureCluster -metadata: - name: ${CLUSTER_NAME} - namespace: default -spec: - additionalTags: - buildProvenance: ${BUILD_PROVENANCE} - creationTimestamp: ${TIMESTAMP} - jobName: ${JOB_NAME} - identityRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureClusterIdentity - name: ${CLUSTER_IDENTITY_NAME} - location: ${AZURE_LOCATION} - networkSpec: - subnets: - - name: control-plane-subnet - role: control-plane - - name: node-subnet - natGateway: - name: node-natgateway - role: node - vnet: - name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} - resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} - subscriptionID: ${AZURE_SUBSCRIPTION_ID} ---- -apiVersion: controlplane.cluster.x-k8s.io/v1beta1 -kind: KubeadmControlPlane -metadata: - name: ${CLUSTER_NAME}-control-plane - namespace: default -spec: - kubeadmConfigSpec: - clusterConfiguration: - apiServer: - extraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - feature-gates: MixedProtocolLBService=true - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true - timeoutForControlPlane: 20m - controllerManager: - extraArgs: - allocate-node-cidrs: "false" - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - cluster-name: ${CLUSTER_NAME} - v: "4" - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true - etcd: - local: - dataDir: /var/lib/etcddisk/etcd - extraArgs: - quota-backend-bytes: "8589934592" - diskSetup: - filesystems: - - device: /dev/disk/azure/scsi1/lun0 - extraOpts: - - -E - - lazy_itable_init=1,lazy_journal_init=1 - filesystem: ext4 - label: etcd_disk - - device: ephemeral0.1 - filesystem: ext4 - label: ephemeral0 - replaceFS: ntfs - partitions: - - device: /dev/disk/azure/scsi1/lun0 - layout: true - overwrite: false - tableType: gpt - files: - - contentFrom: - secret: - key: control-plane-azure.json - name: ${CLUSTER_NAME}-control-plane-azure-json - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" - initConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - name: '{{ ds.meta_data["local_hostname"] }}' - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - name: '{{ ds.meta_data["local_hostname"] }}' - mounts: - - - LABEL=etcd_disk - - /var/lib/etcddisk - postKubeadmCommands: [] - preKubeadmCommands: [] - machineTemplate: - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachineTemplate - name: ${CLUSTER_NAME}-control-plane - replicas: ${CONTROL_PLANE_MACHINE_COUNT} - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachineTemplate -metadata: - name: ${CLUSTER_NAME}-control-plane - namespace: default -spec: - template: - spec: - dataDisks: - - diskSizeGB: 256 - lun: 0 - nameSuffix: etcddisk - identity: UserAssigned - osDisk: - diskSizeGB: 128 - osType: Linux - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - userAssignedIdentities: - - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG:=capz-ci}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY:=cloud-provider-user-identity} - vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachineDeployment -metadata: - name: ${CLUSTER_NAME}-md-0 - namespace: default -spec: - clusterName: ${CLUSTER_NAME} - replicas: ${WORKER_MACHINE_COUNT} - selector: - matchLabels: null - template: - spec: - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfigTemplate - name: ${CLUSTER_NAME}-md-0 - clusterName: ${CLUSTER_NAME} - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachineTemplate - name: ${CLUSTER_NAME}-md-0 - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachineTemplate -metadata: - name: ${CLUSTER_NAME}-md-0 - namespace: default -spec: - template: - spec: - identity: UserAssigned - osDisk: - diskSizeGB: 128 - osType: Linux - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - userAssignedIdentities: - - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG:=capz-ci}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY:=cloud-provider-user-identity} - vmSize: ${AZURE_NODE_MACHINE_TYPE} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfigTemplate -metadata: - name: ${CLUSTER_NAME}-md-0 - namespace: default -spec: - template: - spec: - files: - - contentFrom: - secret: - key: worker-node-azure.json - name: ${CLUSTER_NAME}-md-0-azure-json - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - name: '{{ ds.meta_data["local_hostname"] }}' - preKubeadmCommands: [] ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureClusterIdentity -metadata: - labels: - clusterctl.cluster.x-k8s.io/move-hierarchy: "true" - name: ${CLUSTER_IDENTITY_NAME} - namespace: default -spec: - allowedNamespaces: {} - clientID: ${AZURE_CLIENT_ID} - clientSecret: - name: ${AZURE_CLUSTER_IDENTITY_SECRET_NAME} - namespace: ${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE} - tenantID: ${AZURE_TENANT_ID} - type: ServicePrincipal diff --git a/templates/test/ci/cluster-template-prow.yaml b/templates/test/ci/cluster-template-prow.yaml index 93d07bc5c46..a2615e1831d 100644 --- a/templates/test/ci/cluster-template-prow.yaml +++ b/templates/test/ci/cluster-template-prow.yaml @@ -59,27 +59,15 @@ spec: clusterConfiguration: apiServer: extraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external feature-gates: ${K8S_FEATURE_GATES:-""} - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true timeoutForControlPlane: 20m controllerManager: extraArgs: allocate-node-cidrs: "false" - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} v: "4" - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true etcd: local: dataDir: /var/lib/etcddisk/etcd @@ -114,15 +102,13 @@ spec: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -224,8 +210,7 @@ spec: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: [] --- @@ -337,8 +322,7 @@ spec: criSocket: npipe:////./pipe/containerd-containerd kubeletExtraArgs: azure-container-registry-config: c:/k/azure.json - cloud-config: c:/k/azure.json - cloud-provider: azure + cloud-provider: external feature-gates: WindowsHostProcessContainers=true v: "2" windows-priorityclass: ABOVE_NORMAL_PRIORITY_CLASS diff --git a/templates/test/ci/prow-clusterclass-ci-default/kubeadm-config-template.yaml b/templates/test/ci/prow-clusterclass-ci-default/kubeadm-config-template.yaml index 8bb3464adba..ada8ec7852b 100644 --- a/templates/test/ci/prow-clusterclass-ci-default/kubeadm-config-template.yaml +++ b/templates/test/ci/prow-clusterclass-ci-default/kubeadm-config-template.yaml @@ -18,7 +18,6 @@ spec: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: [] \ No newline at end of file diff --git a/templates/test/ci/prow-clusterclass-ci-default/windows.yaml b/templates/test/ci/prow-clusterclass-ci-default/windows.yaml index 071d5bd000f..a57dc956ecc 100644 --- a/templates/test/ci/prow-clusterclass-ci-default/windows.yaml +++ b/templates/test/ci/prow-clusterclass-ci-default/windows.yaml @@ -27,8 +27,7 @@ spec: name: '{{ ds.meta_data["local_hostname"] }}' criSocket: npipe:////./pipe/containerd-containerd kubeletExtraArgs: - cloud-provider: azure - cloud-config: 'c:/k/azure.json' + cloud-provider: external azure-container-registry-config: 'c:/k/azure.json' feature-gates: "WindowsHostProcessContainers=true" v: "2" diff --git a/templates/test/ci/prow-external-azurediskcsi-driver/patches/external-azurediskcsi-driver.yaml b/templates/test/ci/prow-external-azurediskcsi-driver/patches/external-azurediskcsi-driver.yaml deleted file mode 100644 index 7c1d02e4961..00000000000 --- a/templates/test/ci/prow-external-azurediskcsi-driver/patches/external-azurediskcsi-driver.yaml +++ /dev/null @@ -1,45 +0,0 @@ ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: Cluster -metadata: - name: ${CLUSTER_NAME} - labels: - ccm: "intree" - csi: "external" ---- -kind: KubeadmControlPlane -apiVersion: controlplane.cluster.x-k8s.io/v1beta1 -metadata: - name: "${CLUSTER_NAME}-control-plane" -spec: - kubeadmConfigSpec: - initConfiguration: - nodeRegistration: - kubeletExtraArgs: - feature-gates: CSIMigrationAzureDisk=true - azure-container-registry-config: /etc/kubernetes/azure.json - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - feature-gates: CSIMigrationAzureDisk=true - azure-container-registry-config: /etc/kubernetes/azure.json - clusterConfiguration: - apiServer: - timeoutForControlPlane: 20m - controllerManager: - extraArgs: - feature-gates: CSIMigrationAzureDisk=true - version: "${KUBERNETES_VERSION}" ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfigTemplate -metadata: - name: "${CLUSTER_NAME}-md-0" -spec: - template: - spec: - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - feature-gates: CSIMigrationAzureDisk=true - azure-container-registry-config: /etc/kubernetes/azure.json diff --git a/templates/test/ci/prow-external-cloud-provider-ci-version/kustomization.yaml b/templates/test/ci/prow-external-cloud-provider-ci-version/kustomization.yaml deleted file mode 100644 index f042b876221..00000000000 --- a/templates/test/ci/prow-external-cloud-provider-ci-version/kustomization.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: default -resources: - - ../prow-ci-version -patchesStrategicMerge: - - ../../../flavors/external-cloud-provider/patches/external-cloud-provider.yaml - - patches/external-cloud-provider-windows.yaml diff --git a/templates/test/ci/prow-external-cloud-provider-ci-version/patches/external-cloud-provider-windows.yaml b/templates/test/ci/prow-external-cloud-provider-ci-version/patches/external-cloud-provider-windows.yaml deleted file mode 100644 index fc0d2b681c6..00000000000 --- a/templates/test/ci/prow-external-cloud-provider-ci-version/patches/external-cloud-provider-windows.yaml +++ /dev/null @@ -1,13 +0,0 @@ ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfigTemplate -metadata: - name: ${CLUSTER_NAME}-md-win -spec: - template: - spec: - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - cloud-provider: external - feature-gates: WindowsHostProcessContainers=true,CSIMigrationAzureDisk=true diff --git a/templates/test/ci/prow-external-cloud-provider-vmss-flex/kustomization.yaml b/templates/test/ci/prow-external-cloud-provider-vmss-flex/kustomization.yaml deleted file mode 100644 index 959ef488b6a..00000000000 --- a/templates/test/ci/prow-external-cloud-provider-vmss-flex/kustomization.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: default -resources: - - ../../../flavors/external-cloud-provider-machinepool -patchesStrategicMerge: - - ../patches/tags.yaml - - ../patches/controller-manager.yaml - - ../patches/apiserver.yaml - - ../patches/uami-mp-0.yaml - - ../patches/uami-control-plane.yaml diff --git a/templates/test/ci/prow-external-cloud-provider/kustomization.yaml b/templates/test/ci/prow-external-cloud-provider/kustomization.yaml deleted file mode 100644 index 728f8b2945c..00000000000 --- a/templates/test/ci/prow-external-cloud-provider/kustomization.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: default -resources: - - ../../../flavors/external-cloud-provider -patchesStrategicMerge: - - ../patches/tags.yaml - - ../patches/controller-manager.yaml - - ../patches/apiserver.yaml - - ../patches/uami-md-0.yaml - - ../patches/uami-control-plane.yaml diff --git a/templates/test/ci/prow-intree-cloud-provider-machine-pool/kustomization.yaml b/templates/test/ci/prow-intree-cloud-provider-machine-pool/kustomization.yaml new file mode 100644 index 00000000000..f7bc246d986 --- /dev/null +++ b/templates/test/ci/prow-intree-cloud-provider-machine-pool/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: default +resources: + - ../prow-machine-pool +patchesStrategicMerge: + - ../prow-intree-cloud-provider/patches/intree-cp.yaml + - patches/intree-mp.yaml diff --git a/templates/test/ci/prow-intree-cloud-provider-machine-pool/patches/intree-mp.yaml b/templates/test/ci/prow-intree-cloud-provider-machine-pool/patches/intree-mp.yaml new file mode 100644 index 00000000000..2bfc52700b8 --- /dev/null +++ b/templates/test/ci/prow-intree-cloud-provider-machine-pool/patches/intree-mp.yaml @@ -0,0 +1,28 @@ +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfig +metadata: + name: ${CLUSTER_NAME}-mp-0 + namespace: default +spec: + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + azure-container-registry-config: /etc/kubernetes/azure.json + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfig +metadata: + name: ${CLUSTER_NAME}-mp-win + namespace: default +spec: + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + feature-gates: WindowsHostProcessContainers=true + pod-infra-container-image: mcr.microsoft.com/oss/kubernetes/pause:3.4.1 + azure-container-registry-config: c:/k/azure.json + cloud-config: c:/k/azure.json + cloud-provider: azure + diff --git a/templates/test/ci/prow-external-azurediskcsi-driver/kustomization.yaml b/templates/test/ci/prow-intree-cloud-provider/kustomization.yaml similarity index 54% rename from templates/test/ci/prow-external-azurediskcsi-driver/kustomization.yaml rename to templates/test/ci/prow-intree-cloud-provider/kustomization.yaml index 249b6bbc25d..fc5e636076e 100644 --- a/templates/test/ci/prow-external-azurediskcsi-driver/kustomization.yaml +++ b/templates/test/ci/prow-intree-cloud-provider/kustomization.yaml @@ -2,6 +2,8 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization namespace: default resources: - - ../prow-user-assigned-managed-identity + - ../prow patchesStrategicMerge: - - ./patches/external-azurediskcsi-driver.yaml + - patches/intree-cp.yaml + - patches/intree-md-0.yaml + - patches/intree-md-win.yaml \ No newline at end of file diff --git a/templates/test/ci/prow-intree-cloud-provider/patches/intree-cp.yaml b/templates/test/ci/prow-intree-cloud-provider/patches/intree-cp.yaml new file mode 100644 index 00000000000..78bcc083644 --- /dev/null +++ b/templates/test/ci/prow-intree-cloud-provider/patches/intree-cp.yaml @@ -0,0 +1,40 @@ +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlane +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + kubeadmConfigSpec: + clusterConfiguration: + apiServer: + extraArgs: + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + extraVolumes: + - hostPath: /etc/kubernetes/azure.json + mountPath: /etc/kubernetes/azure.json + name: cloud-config + readOnly: true + controllerManager: + extraArgs: + allocate-node-cidrs: "false" + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + cluster-name: ${CLUSTER_NAME} + extraVolumes: + - hostPath: /etc/kubernetes/azure.json + mountPath: /etc/kubernetes/azure.json + name: cloud-config + readOnly: true + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + azure-container-registry-config: /etc/kubernetes/azure.json + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + azure-container-registry-config: /etc/kubernetes/azure.json + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure diff --git a/templates/test/ci/prow-intree-cloud-provider/patches/intree-md-0.yaml b/templates/test/ci/prow-intree-cloud-provider/patches/intree-md-0.yaml new file mode 100644 index 00000000000..bdf25e4b437 --- /dev/null +++ b/templates/test/ci/prow-intree-cloud-provider/patches/intree-md-0.yaml @@ -0,0 +1,14 @@ +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + template: + spec: + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + azure-container-registry-config: /etc/kubernetes/azure.json + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure diff --git a/templates/test/ci/prow-intree-cloud-provider/patches/intree-md-win.yaml b/templates/test/ci/prow-intree-cloud-provider/patches/intree-md-win.yaml new file mode 100644 index 00000000000..9daa69f7214 --- /dev/null +++ b/templates/test/ci/prow-intree-cloud-provider/patches/intree-md-win.yaml @@ -0,0 +1,18 @@ +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-win + namespace: default +spec: + template: + spec: + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + feature-gates: WindowsHostProcessContainers=true + v: "2" + windows-priorityclass: ABOVE_NORMAL_PRIORITY_CLASS + azure-container-registry-config: c:/k/azure.json + cloud-config: c:/k/azure.json + cloud-provider: azure + diff --git a/templates/test/ci/prow-azurediskcsi-migration-off/kustomization.yaml b/templates/test/ci/prow-machine-pool-flex/kustomization.yaml similarity index 56% rename from templates/test/ci/prow-azurediskcsi-migration-off/kustomization.yaml rename to templates/test/ci/prow-machine-pool-flex/kustomization.yaml index ee4376e1821..7a153b20c45 100644 --- a/templates/test/ci/prow-azurediskcsi-migration-off/kustomization.yaml +++ b/templates/test/ci/prow-machine-pool-flex/kustomization.yaml @@ -2,7 +2,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization namespace: default resources: - - ../prow-user-assigned-managed-identity + - ../prow-machine-pool patchesStrategicMerge: - - ../patches/azurediskcsi-migration-off.yaml - + - patches/vmss-flex.yaml \ No newline at end of file diff --git a/templates/flavors/external-cloud-provider-machinepool/patches/vmss-flex.yaml b/templates/test/ci/prow-machine-pool-flex/patches/vmss-flex.yaml similarity index 50% rename from templates/flavors/external-cloud-provider-machinepool/patches/vmss-flex.yaml rename to templates/test/ci/prow-machine-pool-flex/patches/vmss-flex.yaml index c424f6427a2..b772d9dd2b5 100644 --- a/templates/flavors/external-cloud-provider-machinepool/patches/vmss-flex.yaml +++ b/templates/test/ci/prow-machine-pool-flex/patches/vmss-flex.yaml @@ -1,4 +1,3 @@ ---- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: AzureMachinePool metadata: @@ -11,3 +10,16 @@ spec: maxSurge: 0% maxUnavailable: 0 deletePolicy: +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachinePool +metadata: + name: "${CLUSTER_NAME}-mp-win" +spec: + orchestrationMode: Flexible + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 0% + maxUnavailable: 0 + deletePolicy: diff --git a/templates/test/ci/prow-private/kustomization.yaml b/templates/test/ci/prow-private/kustomization.yaml index 6cb2c34d0ca..a3d71fa83f6 100644 --- a/templates/test/ci/prow-private/kustomization.yaml +++ b/templates/test/ci/prow-private/kustomization.yaml @@ -12,6 +12,8 @@ patchesStrategicMerge: - patches/vnet-peerings.yaml - ../patches/uami-md-0.yaml - ../patches/uami-control-plane.yaml + - ../prow-intree-cloud-provider/patches/intree-cp.yaml # TODO: remove once CAPI supports Helm addons + - ../prow-intree-cloud-provider/patches/intree-md-0.yaml # TODO: remove once CAPI supports Helm addons patches: - path: patches/user-assigned.yaml target: diff --git a/templates/test/ci/prow-user-assigned-managed-identity/kustomization.yaml b/templates/test/ci/prow-user-assigned-managed-identity/kustomization.yaml deleted file mode 100644 index 595fd35b099..00000000000 --- a/templates/test/ci/prow-user-assigned-managed-identity/kustomization.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: default -resources: - - ../../../flavors/default -patchesStrategicMerge: - - ../patches/tags.yaml - - ../patches/controller-manager.yaml - - ../patches/apiserver.yaml - - ../patches/uami-md-0.yaml - - ../patches/uami-control-plane.yaml - diff --git a/templates/test/dev/custom-builds-machine-pool/kustomization.yaml b/templates/test/dev/custom-builds-machine-pool/kustomization.yaml index 9095bd0bcab..24b8a07b388 100644 --- a/templates/test/dev/custom-builds-machine-pool/kustomization.yaml +++ b/templates/test/dev/custom-builds-machine-pool/kustomization.yaml @@ -1,6 +1,6 @@ namespace: default resources: - - ../../../test/ci/prow-machine-pool + - ../../../test/ci/prow-intree-cloud-provider-machine-pool patchesStrategicMerge: - ../patches/control-plane-custom-builds.yaml - patches/custom-builds.yaml diff --git a/templates/test/dev/custom-builds/kustomization.yaml b/templates/test/dev/custom-builds/kustomization.yaml index 62dd8a93ad5..f11ae217124 100644 --- a/templates/test/dev/custom-builds/kustomization.yaml +++ b/templates/test/dev/custom-builds/kustomization.yaml @@ -1,6 +1,6 @@ namespace: default resources: - - ../../../test/ci/prow + - ../../../test/ci/prow-intree-cloud-provider - ../../../addons/metrics-server/metrics-server-resource-set.yaml patchesStrategicMerge: - patches/machine-deployment-pr-version.yaml diff --git a/test/e2e/azure_privatecluster.go b/test/e2e/azure_privatecluster.go index bb27d32cae2..953d39c28d0 100644 --- a/test/e2e/azure_privatecluster.go +++ b/test/e2e/azure_privatecluster.go @@ -134,7 +134,7 @@ func AzurePrivateClusterSpec(ctx context.Context, inputGetter func() AzurePrivat result := &clusterctl.ApplyClusterTemplateAndWaitResult{} // NOTE: We don't add control plane waiters here because Helm install will fail since the apiserver is private and not reachable from the prow cluster. - // As a workaround, we use still ClusterResourceSet to install CNI on the private cluster until a Helm integration is available. + // As a workaround, we use in-tree cloud-provider-azure and ClusterResourceSet to install CNI on the private cluster until a Helm integration is available. clusterctl.ApplyClusterTemplateAndWait(ctx, createApplyClusterTemplateInput( specName, withClusterProxy(publicClusterProxy), diff --git a/test/e2e/azure_test.go b/test/e2e/azure_test.go index dc267aa8f77..970fa189a7f 100644 --- a/test/e2e/azure_test.go +++ b/test/e2e/azure_test.go @@ -513,15 +513,12 @@ var _ = Describe("Workload cluster creation", func() { }) // ci-e2e.sh and Prow CI skip this test by default. To include this test, set `GINKGO_SKIP=""`. - // This spec expects a user-assigned identity named "cloud-provider-user-identity" in a "capz-ci" - // resource group. Override these defaults by setting the USER_IDENTITY and CI_RG environment variables. - Context("Creating a cluster that uses the external cloud provider and machinepools [OPTIONAL]", func() { + Context("Creating a cluster with VMSS flex machinepools [OPTIONAL]", func() { It("with 1 control plane node and 1 machinepool", func() { - By("using user-assigned identity") clusterName = getClusterName(clusterNamePrefix, "flex") clusterctl.ApplyClusterTemplateAndWait(ctx, createApplyClusterTemplateInput( specName, - withFlavor("external-cloud-provider-vmss-flex"), + withFlavor("machine-pool-flex"), withNamespace(namespace.Name), withClusterName(clusterName), withControlPlaneMachineCount(1), @@ -573,15 +570,13 @@ var _ = Describe("Workload cluster creation", func() { }) // ci-e2e.sh and Prow CI skip this test by default. To include this test, set `GINKGO_SKIP=""`. - // This spec expects a user-assigned identity named "cloud-provider-user-identity" in a "capz-ci" - // resource group. Override these defaults by setting the USER_IDENTITY and CI_RG environment variables. - Context("Creating a cluster that uses the external cloud provider and external azurediskcsi driver [OPTIONAL]", func() { + Context("Creating a cluster that uses the intree cloud provider [OPTIONAL]", func() { It("with a 1 control plane nodes and 2 worker nodes", func() { By("using user-assigned identity") - clusterName = getClusterName(clusterNamePrefix, "oot") + clusterName = getClusterName(clusterNamePrefix, "intree") clusterctl.ApplyClusterTemplateAndWait(ctx, createApplyClusterTemplateInput( specName, - withFlavor("external-cloud-provider"), + withFlavor("intree-cloud-provider"), withNamespace(namespace.Name), withClusterName(clusterName), withControlPlaneMachineCount(1), diff --git a/test/e2e/cloud-provider-azure.go b/test/e2e/cloud-provider-azure.go index 107aa0f255d..92a1ab554d6 100644 --- a/test/e2e/cloud-provider-azure.go +++ b/test/e2e/cloud-provider-azure.go @@ -22,6 +22,7 @@ package e2e import ( "context" "fmt" + "os" "strings" . "github.com/onsi/ginkgo/v2" @@ -44,8 +45,19 @@ func InstallCalicoAndCloudProviderAzureHelmChart(ctx context.Context, input clus specName := "cloud-provider-azure-install" By("Installing cloud-provider-azure components via helm") options := &helmVals.Options{ - Values: []string{fmt.Sprintf("infra.clusterName=%s", input.ConfigCluster.ClusterName), fmt.Sprintf("cloudControllerManager.clusterCIDR=%s", strings.Join(cidrBlocks, `,`))}, + Values: []string{fmt.Sprintf("infra.clusterName=%s", input.ConfigCluster.ClusterName)}, + StringValues: []string{fmt.Sprintf("cloudControllerManager.clusterCIDR=%s", strings.Join(cidrBlocks, `\,`))}, } + // If testing a CI version of Kubernetes, use CCM and CNM images built from source. + if useCIArtifacts { + options.Values = append(options.Values, fmt.Sprintf("cloudControllerManager.imageName=%s", os.Getenv("CCM_IMAGE_NAME"))) + options.Values = append(options.Values, fmt.Sprintf("cloudNodeManager.imageName=%s", os.Getenv("CNM_IMAGE_NAME"))) + options.Values = append(options.Values, fmt.Sprintf("cloudControllerManager.imageRepository=%s", os.Getenv("IMAGE_REGISTRY"))) + options.Values = append(options.Values, fmt.Sprintf("cloudNodeManager.imageRepository=%s", os.Getenv("IMAGE_REGISTRY"))) + options.StringValues = append(options.StringValues, fmt.Sprintf("cloudControllerManager.imageTag=%s", os.Getenv("IMAGE_TAG"))) + options.StringValues = append(options.StringValues, fmt.Sprintf("cloudNodeManager.imageTag=%s", os.Getenv("IMAGE_TAG"))) + } + clusterProxy := input.ClusterProxy.GetWorkloadCluster(ctx, input.ConfigCluster.Namespace, input.ConfigCluster.ClusterName) InstallHelmChart(ctx, clusterProxy, defaultNamespace, cloudProviderAzureHelmRepoURL, cloudProviderAzureChartName, cloudProviderAzureHelmReleaseName, options) diff --git a/test/e2e/common.go b/test/e2e/common.go index 57b56d26027..996e9e0281f 100644 --- a/test/e2e/common.go +++ b/test/e2e/common.go @@ -274,7 +274,7 @@ func EnsureControlPlaneInitialized(ctx context.Context, input clusterctl.ApplyCl }, input.WaitForControlPlaneIntervals...).Should(Succeed(), "API Server was not reachable in time") _, hasWindows := cluster.Labels["cni-windows"] - if kubeadmControlPlane.Spec.KubeadmConfigSpec.ClusterConfiguration.ControllerManager.ExtraArgs["cloud-provider"] == "external" { + if kubeadmControlPlane.Spec.KubeadmConfigSpec.ClusterConfiguration.ControllerManager.ExtraArgs["cloud-provider"] != "azure" { // There is a co-dependency between cloud-provider and CNI so we install both together if cloud-provider is external. InstallCalicoAndCloudProviderAzureHelmChart(ctx, input, cluster.Spec.ClusterNetwork.Pods.CIDRBlocks, hasWindows) } else { diff --git a/test/e2e/config/azure-dev.yaml b/test/e2e/config/azure-dev.yaml index 19d6d9f2739..0a5cdaa6bfe 100644 --- a/test/e2e/config/azure-dev.yaml +++ b/test/e2e/config/azure-dev.yaml @@ -124,16 +124,10 @@ providers: targetName: "cluster-template-conformance-presubmit-artifacts.yaml" - sourcePath: "${PWD}/templates/test/dev/cluster-template-custom-builds.yaml" targetName: "cluster-template-conformance-presubmit-artifacts-windows-containerd.yaml" - - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-external-cloud-provider.yaml" - targetName: "cluster-template-external-cloud-provider.yaml" - - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-external-cloud-provider-vmss-flex.yaml" - targetName: "cluster-template-external-cloud-provider-vmss-flex.yaml" - - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-azurediskcsi-migration-off.yaml" - targetName: "cluster-template-azurediskcsi-migration-off.yaml" - - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-external-azurediskcsi-driver.yaml" - targetName: "cluster-template-external-azurediskcsi-driver.yaml" - - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-user-assigned-managed-identity.yaml" - targetName: "cluster-template-user-assigned-managed-identity.yaml" + - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-intree-cloud-provider.yaml" + targetName: "cluster-template-intree-cloud-provider.yaml" + - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml" + targetName: "cluster-template-machine-pool-flex.yaml" - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-aks.yaml" targetName: "cluster-template-aks.yaml" - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-custom-vnet.yaml" diff --git a/test/e2e/data/infrastructure-azure/v1beta1/bases/cluster-with-kcp.yaml b/test/e2e/data/infrastructure-azure/v1beta1/bases/cluster-with-kcp.yaml index 18e1f3df402..a016095731a 100644 --- a/test/e2e/data/infrastructure-azure/v1beta1/bases/cluster-with-kcp.yaml +++ b/test/e2e/data/infrastructure-azure/v1beta1/bases/cluster-with-kcp.yaml @@ -73,8 +73,7 @@ spec: clusterConfiguration: apiServer: extraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external extraVolumes: - hostPath: /etc/kubernetes/azure.json mountPath: /etc/kubernetes/azure.json @@ -84,8 +83,7 @@ spec: controllerManager: extraArgs: allocate-node-cidrs: "false" - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} extraVolumes: - hostPath: /etc/kubernetes/azure.json @@ -123,17 +121,15 @@ spec: initConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk - /var/lib/etcddisk replicas: ${CONTROL_PLANE_MACHINE_COUNT} - version: ${KUBERNETES_VERSION} + version: ${KUBERNETES_VERSION} \ No newline at end of file diff --git a/test/e2e/data/infrastructure-azure/v1beta1/bases/md.yaml b/test/e2e/data/infrastructure-azure/v1beta1/bases/md.yaml index 1129d20f919..93bdfe0af0c 100644 --- a/test/e2e/data/infrastructure-azure/v1beta1/bases/md.yaml +++ b/test/e2e/data/infrastructure-azure/v1beta1/bases/md.yaml @@ -36,8 +36,7 @@ spec: joinConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' --- # MachineDeployment object diff --git a/test/e2e/data/infrastructure-azure/v1beta1/bases/mp.yaml b/test/e2e/data/infrastructure-azure/v1beta1/bases/mp.yaml index f8316bc48a8..e4dc44d3907 100644 --- a/test/e2e/data/infrastructure-azure/v1beta1/bases/mp.yaml +++ b/test/e2e/data/infrastructure-azure/v1beta1/bases/mp.yaml @@ -59,6 +59,5 @@ spec: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - name: '{{ ds.meta_data["local_hostname"] }}' + cloud-provider: external + name: '{{ ds.meta_data["local_hostname"] }}' \ No newline at end of file diff --git a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-kcp-remediation.yaml b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-kcp-remediation.yaml index 0aaef661063..1137655eb4d 100644 --- a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-kcp-remediation.yaml +++ b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-kcp-remediation.yaml @@ -17,8 +17,7 @@ spec: joinConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' --- apiVersion: cluster.x-k8s.io/v1beta1 @@ -90,8 +89,7 @@ spec: clusterConfiguration: apiServer: extraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external extraVolumes: - hostPath: /etc/kubernetes/azure.json mountPath: /etc/kubernetes/azure.json @@ -101,8 +99,7 @@ spec: controllerManager: extraArgs: allocate-node-cidrs: "false" - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} extraVolumes: - hostPath: /etc/kubernetes/azure.json @@ -140,14 +137,12 @@ spec: initConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk diff --git a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-kcp-scale-in.yaml b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-kcp-scale-in.yaml index 5411b203700..48432e9c3d1 100644 --- a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-kcp-scale-in.yaml +++ b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-kcp-scale-in.yaml @@ -17,8 +17,7 @@ spec: joinConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' --- apiVersion: cluster.x-k8s.io/v1beta1 @@ -75,8 +74,7 @@ spec: clusterConfiguration: apiServer: extraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external extraVolumes: - hostPath: /etc/kubernetes/azure.json mountPath: /etc/kubernetes/azure.json @@ -86,8 +84,7 @@ spec: controllerManager: extraArgs: allocate-node-cidrs: "false" - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} extraVolumes: - hostPath: /etc/kubernetes/azure.json @@ -125,14 +122,12 @@ spec: initConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk diff --git a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-machine-pool.yaml b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-machine-pool.yaml index 097a08239b9..45cb31ca3ec 100644 --- a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-machine-pool.yaml +++ b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-machine-pool.yaml @@ -16,8 +16,7 @@ spec: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' --- apiVersion: cluster.x-k8s.io/v1beta1 @@ -72,8 +71,7 @@ spec: clusterConfiguration: apiServer: extraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external extraVolumes: - hostPath: /etc/kubernetes/azure.json mountPath: /etc/kubernetes/azure.json @@ -83,8 +81,7 @@ spec: controllerManager: extraArgs: allocate-node-cidrs: "false" - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} extraVolumes: - hostPath: /etc/kubernetes/azure.json @@ -122,14 +119,12 @@ spec: initConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk diff --git a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-md-remediation.yaml b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-md-remediation.yaml index 5540bbb04c7..8a7cd741faa 100644 --- a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-md-remediation.yaml +++ b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-md-remediation.yaml @@ -17,8 +17,7 @@ spec: joinConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' --- apiVersion: cluster.x-k8s.io/v1beta1 @@ -92,8 +91,7 @@ spec: clusterConfiguration: apiServer: extraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external extraVolumes: - hostPath: /etc/kubernetes/azure.json mountPath: /etc/kubernetes/azure.json @@ -103,8 +101,7 @@ spec: controllerManager: extraArgs: allocate-node-cidrs: "false" - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} extraVolumes: - hostPath: /etc/kubernetes/azure.json @@ -142,14 +139,12 @@ spec: initConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk diff --git a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-node-drain.yaml b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-node-drain.yaml index 7059a776323..135e1b761ea 100644 --- a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-node-drain.yaml +++ b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-node-drain.yaml @@ -17,8 +17,7 @@ spec: joinConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' --- apiVersion: cluster.x-k8s.io/v1beta1 @@ -75,8 +74,7 @@ spec: clusterConfiguration: apiServer: extraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external extraVolumes: - hostPath: /etc/kubernetes/azure.json mountPath: /etc/kubernetes/azure.json @@ -86,8 +84,7 @@ spec: controllerManager: extraArgs: allocate-node-cidrs: "false" - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} extraVolumes: - hostPath: /etc/kubernetes/azure.json @@ -125,14 +122,12 @@ spec: initConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk diff --git a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-upgrades.yaml b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-upgrades.yaml index 5f441e414dd..d8174eb9b76 100644 --- a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-upgrades.yaml +++ b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-upgrades.yaml @@ -16,8 +16,7 @@ spec: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' --- apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 @@ -39,8 +38,7 @@ spec: joinConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' --- apiVersion: cluster.x-k8s.io/v1beta1 @@ -119,8 +117,7 @@ spec: clusterConfiguration: apiServer: extraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external extraVolumes: - hostPath: /etc/kubernetes/azure.json mountPath: /etc/kubernetes/azure.json @@ -130,8 +127,7 @@ spec: controllerManager: extraArgs: allocate-node-cidrs: "false" - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} extraVolumes: - hostPath: /etc/kubernetes/azure.json @@ -169,14 +165,12 @@ spec: initConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk diff --git a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template.yaml b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template.yaml index bdf162601e4..4836890b4a6 100644 --- a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template.yaml +++ b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template.yaml @@ -17,8 +17,7 @@ spec: joinConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' --- apiVersion: cluster.x-k8s.io/v1beta1 @@ -75,8 +74,7 @@ spec: clusterConfiguration: apiServer: extraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external extraVolumes: - hostPath: /etc/kubernetes/azure.json mountPath: /etc/kubernetes/azure.json @@ -86,8 +84,7 @@ spec: controllerManager: extraArgs: allocate-node-cidrs: "false" - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} extraVolumes: - hostPath: /etc/kubernetes/azure.json @@ -125,14 +122,12 @@ spec: initConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk