diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/_helpers.tpl b/charts/t8s-cluster/templates/management-cluster/clusterClass/_helpers.tpl index 25e2fb355a..4078793499 100644 --- a/charts/t8s-cluster/templates/management-cluster/clusterClass/_helpers.tpl +++ b/charts/t8s-cluster/templates/management-cluster/clusterClass/_helpers.tpl @@ -35,6 +35,14 @@ openstack {{- toYaml $args -}} {{- end -}} +{{- define "t8s-cluster.clusterClass.argsMapToArray" }} + {{- $argsArray := list -}} + {{- range $key, $value := .args -}} + {{- $argsArray = append $argsArray (dict "name" $key "value" $value) -}} + {{- end -}} + {{- toYaml $argsArray -}} +{{- end -}} + {{/* this can be split up with containerd >=2.0.0 */}} {{- define "t8s-cluster.clusterClass.containerdConfig.plugins" -}} {{- $_ := mustMerge . (pick .context "Values") -}} diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/bootstrapConfigTemplate/_bootstrapConfigTemplate.yaml b/charts/t8s-cluster/templates/management-cluster/clusterClass/bootstrapConfigTemplate/_bootstrapConfigTemplate.yaml index 636fb5b70a..e06744910b 100644 --- a/charts/t8s-cluster/templates/management-cluster/clusterClass/bootstrapConfigTemplate/_bootstrapConfigTemplate.yaml +++ b/charts/t8s-cluster/templates/management-cluster/clusterClass/bootstrapConfigTemplate/_bootstrapConfigTemplate.yaml @@ -1,6 +1,6 @@ {{- define "t8s-cluster.clusterClass.bootstrapConfigTemplate" -}} {{- $_ := mustMerge . (pick .context "Values" "Release" "Chart") -}} -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 kind: KubeadmConfigTemplate metadata: name: {{ printf "%s-%s-compute-plane" .Release.Name (.gpu | ternary "gpu" "standard") }} diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/bootstrapConfigTemplate/_kubeadmConfigTemplateSpec.yaml b/charts/t8s-cluster/templates/management-cluster/clusterClass/bootstrapConfigTemplate/_kubeadmConfigTemplateSpec.yaml index 9faa06e7ab..0bd0a6346e 100644 --- a/charts/t8s-cluster/templates/management-cluster/clusterClass/bootstrapConfigTemplate/_kubeadmConfigTemplateSpec.yaml +++ b/charts/t8s-cluster/templates/management-cluster/clusterClass/bootstrapConfigTemplate/_kubeadmConfigTemplateSpec.yaml @@ -2,11 +2,15 @@ {{- $_ := mustMerge . (pick .context "Values") -}} joinConfiguration: nodeRegistration: - kubeletExtraArgs: {{- include "t8s-cluster.clusterClass.kubeletExtraArgs" (dict "context" .context) | nindent 6 }} + kubeletExtraArgs: {{- toYaml (include "t8s-cluster.clusterClass.argsMapToArray" (dict "args" (include "t8s-cluster.clusterClass.kubeletExtraArgs" (dict "context" .context) | fromYaml)) | fromYamlArray) | nindent 6 }} imagePullSerial: false patches: directory: /etc/kubernetes/patches files: {{- include "t8s-cluster.clusterClass.configTemplate.files" (dict "context" .context "gpu" .gpu) | nindent 2 }} -preKubeadmCommands: {{- include "t8s-cluster.clusterClass.preKubeadmCommands" (dict "context" .context) | nindent 2 }} -postKubeadmCommands: {{- include "t8s-cluster.clusterClass.postKubeadmCommands" (dict) | nindent 2 }} + {{- with include "t8s-cluster.clusterClass.preKubeadmCommands" (dict "context" .context) | fromYamlArray }} +preKubeadmCommands: {{- toYaml . | nindent 2 }} + {{- end }} + {{- with include "t8s-cluster.clusterClass.postKubeadmCommands" (dict) | fromYamlArray }} +postKubeadmCommands: {{- toYaml . | nindent 2 }} + {{- end }} {{- end -}} diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/clusterClass.yaml b/charts/t8s-cluster/templates/management-cluster/clusterClass/clusterClass.yaml index ed5d7af805..3c91b55065 100644 --- a/charts/t8s-cluster/templates/management-cluster/clusterClass/clusterClass.yaml +++ b/charts/t8s-cluster/templates/management-cluster/clusterClass/clusterClass.yaml @@ -2,7 +2,7 @@ {{- range $name, $spec := .Values.nodePools -}} {{- $machineDeploymentClasses = set $machineDeploymentClasses $name ($spec.flavor | contains "gpu") -}} {{- end -}} -apiVersion: cluster.x-k8s.io/v1beta1 +apiVersion: cluster.x-k8s.io/v1beta2 kind: ClusterClass metadata: name: {{ $.Release.Name }} @@ -11,25 +11,29 @@ metadata: spec: controlPlane: {{- if not .Values.controlPlane.hosted }} - machineHealthCheck: - maxUnhealthy: 1 - nodeStartupTimeout: 10m - unhealthyConditions: - - status: Unknown - timeout: 600s - type: Ready - - status: 'False' - timeout: 600s - type: Ready - nodeDrainTimeout: "8m" + healthCheck: + remediation: + triggerIf: + unhealthyLessThanOrEqualTo: 1 + checks: + nodeStartupTimeoutSeconds: 600 + unhealthyNodeConditions: + - status: Unknown + timeoutSeconds: 600 + type: Ready + - status: 'False' + timeoutSeconds: 600 + type: Ready + deletion: + nodeDrainTimeoutSeconds: 480 machineInfrastructure: - ref: + templateRef: apiVersion: {{ include "t8s-cluster.clusterClass.infrastructureApiVersion" (dict) }} kind: OpenStackMachineTemplate name: {{ printf "%s-control-plane-%s" $.Release.Name (include "t8s-cluster.clusterClass.openStackMachineTemplate.specHashOfControlPlane" (dict "context" $)) }} {{- end }} - ref: - apiVersion: controlplane.cluster.x-k8s.io/{{ .Values.controlPlane.hosted | ternary "v1alpha1" "v1beta1" }} + templateRef: + apiVersion: controlplane.cluster.x-k8s.io/{{ .Values.controlPlane.hosted | ternary "v1alpha1" "v1beta2" }} {{- if .Values.controlPlane.hosted }} kind: HostedControlPlaneTemplate name: {{/* the full context is needed for .Files.Get */}}{{ printf "%s-%s" $.Release.Name (include "t8s-cluster.clusterClass.hostedControlPlaneTemplate.specHash" .) }} @@ -38,7 +42,7 @@ spec: name: {{/* the full context is needed for .Files.Get */}}{{ printf "%s-%s" $.Release.Name (include "t8s-cluster.clusterClass.kubeadmControlPlaneTemplate.specHash" .) }} {{- end }} infrastructure: - ref: + templateRef: apiVersion: {{ include "t8s-cluster.clusterClass.infrastructureApiVersion" (dict) }} kind: OpenStackClusterTemplate name: {{ printf "%s-%s" $.Release.Name (include "t8s-cluster.clusterClass.openStackClusterTemplate.specHash" (dict "context" $)) }} @@ -138,38 +142,40 @@ spec: machineDeployments: {{- range $name, $isGpuDeploymentClass := $machineDeploymentClasses }} - class: {{ $name }} - machineHealthCheck: - nodeStartupTimeout: 8m - unhealthyConditions: - - status: Unknown - timeout: 300s - type: Ready - - status: 'False' - timeout: 300s - type: Ready - nodeDrainTimeout: 8m - nodeDeletionTimeout: 15m - strategy: - type: RollingUpdate - rollingUpdate: - deletePolicy: Oldest - {{/* TODO: this is only for our beta phase */}} - {{- if $isGpuDeploymentClass }} - maxSurge: 0 - maxUnavailable: 1 - {{- else }} - maxSurge: 75% - maxUnavailable: 0 - {{- end }} - template: - bootstrap: - ref: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfigTemplate - name: {{ printf "%s-%s-compute-plane" $.Release.Name ($isGpuDeploymentClass | ternary "gpu" "standard") }} - infrastructure: - ref: - apiVersion: {{ include "t8s-cluster.clusterClass.infrastructureApiVersion" (dict) }} - kind: OpenStackMachineTemplate - name: {{ printf "%s-compute-plane-%s" $.Release.Name (include "t8s-cluster.clusterClass.openStackMachineTemplate.specHashOfNodePools" (dict "context" $)) }} + healthCheck: + checks: + nodeStartupTimeoutSeconds: 480 + unhealthyNodeConditions: + - status: Unknown + timeoutSeconds: 300 + type: Ready + - status: 'False' + timeoutSeconds: 300 + type: Ready + deletion: + order: Oldest + nodeDrainTimeoutSeconds: 480 + nodeDeletionTimeoutSeconds: 900 + rollout: + strategy: + type: RollingUpdate + rollingUpdate: + {{/* TODO: this is only for our beta phase */}} + {{- if $isGpuDeploymentClass }} + maxSurge: 0 + maxUnavailable: 1 + {{- else }} + maxSurge: 75% + maxUnavailable: 0 + {{- end }} + bootstrap: + templateRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 + kind: KubeadmConfigTemplate + name: {{ printf "%s-%s-compute-plane" $.Release.Name ($isGpuDeploymentClass | ternary "gpu" "standard") }} + infrastructure: + templateRef: + apiVersion: {{ include "t8s-cluster.clusterClass.infrastructureApiVersion" (dict) }} + kind: OpenStackMachineTemplate + name: {{ printf "%s-compute-plane-%s" $.Release.Name (include "t8s-cluster.clusterClass.openStackMachineTemplate.specHashOfNodePools" (dict "context" $)) }} {{- end }} diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmControlPlaneTemplate/_kubeadmControlPlaneTemplateSpec.yaml b/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmControlPlaneTemplate/_kubeadmControlPlaneTemplateSpec.yaml index ee92ff7c72..4aa0c492a4 100644 --- a/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmControlPlaneTemplate/_kubeadmControlPlaneTemplateSpec.yaml +++ b/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmControlPlaneTemplate/_kubeadmControlPlaneTemplateSpec.yaml @@ -9,28 +9,33 @@ rolloutBefore: kubeadmConfigSpec: clusterConfiguration: apiServer: - extraArgs: {{- include "t8s-cluster.clusterClass.args.apiServer" (dict "context" .) | nindent 8 }} + extraArgs: {{- toYaml (include "t8s-cluster.clusterClass.argsMapToArray" (dict "args" (include "t8s-cluster.clusterClass.args.apiServer" (dict "context" .) | fromYaml)) | fromYamlArray) | nindent 8 }} {{- $extraVolumes := list -}} {{- range $name, $file := mustMerge (include "t8s-cluster.clusterClass.apiServer.staticFiles" (dict) | fromYaml) (include "t8s-cluster.clusterClass.apiServer.dynamicFiles" (dict "context" .) | fromYaml) -}} {{- $extraVolumes = append $extraVolumes (dict "name" ($name | trimSuffix ".yaml" | replace "." "-") "hostPath" (get $file "path" ) "mountPath" (get $file "path") "readOnly" true) -}} {{- end }} extraVolumes: {{- toYaml $extraVolumes | nindent 8 }} controllerManager: - extraArgs: {{- include "t8s-cluster.clusterClass.args.controllerManager" (dict "context" .) | nindent 8 }} + extraArgs: {{- toYaml (include "t8s-cluster.clusterClass.argsMapToArray" (dict "args" (include "t8s-cluster.clusterClass.args.controllerManager" (dict "context" .) | fromYaml)) | fromYamlArray) | nindent 8 }} etcd: local: extraArgs: - listen-metrics-urls: http://0.0.0.0:2381 + - name: listen-metrics-urls + value: http://0.0.0.0:2381 scheduler: - extraArgs: {{- include "t8s-cluster.clusterClass.args.scheduler" (dict) | nindent 8 }} + extraArgs: {{- toYaml (include "t8s-cluster.clusterClass.argsMapToArray" (dict "args" (include "t8s-cluster.clusterClass.args.scheduler" (dict "context" .) | fromYaml)) | fromYamlArray) | nindent 8 }} files: {{- include "t8s-cluster.clusterClass.kubeadmControlPlaneTemplate.files" . | nindent 4 }} initConfiguration: &configuration nodeRegistration: - kubeletExtraArgs: {{- include "t8s-cluster.clusterClass.kubeletExtraArgs" (dict "context" .) | nindent 8 }} + kubeletExtraArgs: {{- toYaml (include "t8s-cluster.clusterClass.argsMapToArray" (dict "args" (include "t8s-cluster.clusterClass.kubeletExtraArgs" (dict "context" .) | fromYaml)) | fromYamlArray) | nindent 8 }} imagePullSerial: false patches: directory: {{ include "t8s-cluster.patches.directory" (dict) }} joinConfiguration: *configuration - preKubeadmCommands: {{- include "t8s-cluster.clusterClass.kubeadmControlPlaneTemplate.preKubeadmCommands" (dict "context" .) | nindent 4 }} - postKubeadmCommands: {{- include "t8s-cluster.clusterClass.postKubeadmCommands" (dict) | nindent 4 }} + {{- with include "t8s-cluster.clusterClass.kubeadmControlPlaneTemplate.preKubeadmCommands" (dict "context" .) | fromYamlArray }} + preKubeadmCommands: {{- toYaml . | nindent 4 }} + {{- end }} + {{- with include "t8s-cluster.clusterClass.postKubeadmCommands" (dict) | fromYamlArray }} + postKubeadmCommands: {{- toYaml . | nindent 4 }} + {{- end }} {{- end -}} diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmControlPlaneTemplate/kubeadmControlPlaneTemplate.yaml b/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmControlPlaneTemplate/kubeadmControlPlaneTemplate.yaml index 997a2b92b7..c5dcf6b8c8 100644 --- a/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmControlPlaneTemplate/kubeadmControlPlaneTemplate.yaml +++ b/charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmControlPlaneTemplate/kubeadmControlPlaneTemplate.yaml @@ -1,5 +1,5 @@ {{- if not .Values.controlPlane.hosted -}} -apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +apiVersion: controlplane.cluster.x-k8s.io/v1beta2 kind: KubeadmControlPlaneTemplate metadata: name: {{/* the full context is needed for .Files.Get */}}{{ printf "%s-%s" $.Release.Name (include "t8s-cluster.clusterClass.kubeadmControlPlaneTemplate.specHash" .) }}