diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/_helpers.tpl b/charts/t8s-cluster/templates/management-cluster/clusterClass/_helpers.tpl index 25e2fb355a..641247d976 100644 --- a/charts/t8s-cluster/templates/management-cluster/clusterClass/_helpers.tpl +++ b/charts/t8s-cluster/templates/management-cluster/clusterClass/_helpers.tpl @@ -145,9 +145,13 @@ server = {{ printf "https://%s" .registry | quote }} {{- end }} {{- define "t8s-cluster.clusterClass.args.controllerManager" -}} + {{- $_ := mustMerge . (pick .context "Values") -}} {{- $args := include "t8s-cluster.clusterClass.args.shared" (dict) | fromYaml -}} {{- $args = mustMerge (include "t8s-cluster.clusterClass.args.sharedController" (dict "context" .context) | fromYaml) $args -}} {{- $args = set $args "terminated-pod-gc-threshold" "100" -}} + {{- if .Values.controlPlane.hosted -}} + {{- $args = set $args "allocate-node-cidrs" "true" -}} + {{- end }} {{- toYaml $args -}} {{- end }} diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/hostedControlPlaneTemplate/_hostedControlPlaneTemplateSpec.yaml b/charts/t8s-cluster/templates/management-cluster/clusterClass/hostedControlPlaneTemplate/_hostedControlPlaneTemplateSpec.yaml index 35152c847e..17d57c9f91 100644 --- a/charts/t8s-cluster/templates/management-cluster/clusterClass/hostedControlPlaneTemplate/_hostedControlPlaneTemplateSpec.yaml +++ b/charts/t8s-cluster/templates/management-cluster/clusterClass/hostedControlPlaneTemplate/_hostedControlPlaneTemplateSpec.yaml @@ -65,6 +65,8 @@ deployment: args: {{- include "t8s-cluster.clusterClass.args.scheduler" (dict) | nindent 6 }} resources: {{- include "common.resources" .Values.controlPlane | nindent 6 }} replicas: 1 +kubeProxy: + disabled: {{ .Values.controlPlane.hosted }} gateway: namespace: capi-hosted-control-plane-system name: controlplane diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/openStackClusterTemplate/_openStackClusterTemplateSpec.yaml b/charts/t8s-cluster/templates/management-cluster/clusterClass/openStackClusterTemplate/_openStackClusterTemplateSpec.yaml index 8c7fe2a5e4..523c569e44 100644 --- a/charts/t8s-cluster/templates/management-cluster/clusterClass/openStackClusterTemplate/_openStackClusterTemplateSpec.yaml +++ b/charts/t8s-cluster/templates/management-cluster/clusterClass/openStackClusterTemplate/_openStackClusterTemplateSpec.yaml @@ -34,9 +34,11 @@ identityRef: name: {{ include "t8s-cluster.clusterClass.getIdentityRefSecretName" (dict "context" .context) }} cloudName: {{ include "t8s-cluster.clusterClass.cloudName" (dict) }} - {{- $securityGroupRules := dict -}} - {{- $cni := include "t8s-cluster.cni" .context -}} + + {{- $allowNativeRouting := and .Values.controlPlane.hosted (eq $cni "cilium") -}} + + {{- $securityGroupRules := dict -}} {{- $controlPlaneSecurityGroup := "controlplane" -}} {{- $computePlaneSecurityGroup := "worker" -}} {{- $allRemoteManagedGroups := list $computePlaneSecurityGroup -}} @@ -44,29 +46,37 @@ identityRef: {{ $allRemoteManagedGroups = append $allRemoteManagedGroups $controlPlaneSecurityGroup -}} {{- end -}} - {{- $cniSecurityGroupRules := dict -}} - {{- if eq $cni "calico" }} - {{- $cniSecurityGroupRules = set $cniSecurityGroupRules "BGP" (dict "port" 179) -}} - {{- $cniSecurityGroupRules = set $cniSecurityGroupRules "IP-in-IP" (dict "protocol" "4") -}} - {{- else if eq $cni "cilium" -}} - {{- $cniSecurityGroupRules = set $cniSecurityGroupRules "VXLAN" (dict "port" 8472 "protocol" "udp") -}} - {{- $cniSecurityGroupRules = set $cniSecurityGroupRules "health (http)" (dict "port" 4240) -}} - {{- $cniSecurityGroupRules = set $cniSecurityGroupRules "health (ping)" (dict "protocol" "icmp") -}} + {{- if not $allowNativeRouting -}} + {{- $cniSecurityGroupRules := dict -}} + {{- if eq $cni "calico" }} + {{- $cniSecurityGroupRules = set $cniSecurityGroupRules "BGP" (dict "port" 179) -}} + {{- $cniSecurityGroupRules = set $cniSecurityGroupRules "IP-in-IP" (dict "protocol" "4") -}} + {{- else if eq $cni "cilium" -}} + {{- $cniSecurityGroupRules = set $cniSecurityGroupRules "VXLAN" (dict "port" 8472 "protocol" "udp") -}} + {{- $cniSecurityGroupRules = set $cniSecurityGroupRules "health (http)" (dict "port" 4240) -}} + {{- $cniSecurityGroupRules = set $cniSecurityGroupRules "health (ping)" (dict "protocol" "icmp") -}} + {{- if .Values.controlPlane.hosted -}} + {{- $cniSecurityGroupRules = set $cniSecurityGroupRules "allow pod-pod native routing (ingress)" (dict "remoteIPPrefix" "10.0.0.0/16" "protocol" nil) -}} + {{- $cniSecurityGroupRules = set $cniSecurityGroupRules "allow node-pod native routing (ingress)" (dict "remoteIPPrefix" "10.6.0.0/16" "protocol" nil) -}} + {{- end -}} {{- end }} - {{- range $name, $securityGroupRule := $cniSecurityGroupRules -}} - {{- $securityGroupRules = set $securityGroupRules (printf "%s %s" $cni $name) $securityGroupRule -}} - {{- end -}} + {{- range $name, $securityGroupRule := $cniSecurityGroupRules -}} + {{- $securityGroupRules = set $securityGroupRules (printf "%s %s" $cni $name) $securityGroupRule -}} + {{- end -}} - {{- $metricsSecurityGroupRules := dict -}} - {{- $metricsSecurityGroupRules = set $metricsSecurityGroupRules "prometheus node-exporter" (dict "port" 9100 "remoteManagedGroups" (list $computePlaneSecurityGroup)) -}} - {{- if not .Values.controlPlane.hosted -}} - {{- $metricsSecurityGroupRules = set $metricsSecurityGroupRules "etcd" (dict "port" 2381 "securityGroup" $controlPlaneSecurityGroup "remoteManagedGroups" (list $computePlaneSecurityGroup)) -}} - {{- $metricsSecurityGroupRules = set $metricsSecurityGroupRules "kube-proxy" (dict "port" 10249 "securityGroup" $controlPlaneSecurityGroup "remoteManagedGroups" (list $computePlaneSecurityGroup)) -}} - {{- $metricsSecurityGroupRules = set $metricsSecurityGroupRules "kube-controller-manager" (dict "port" 10257 "securityGroup" $controlPlaneSecurityGroup "remoteManagedGroups" (list $computePlaneSecurityGroup)) -}} - {{- $metricsSecurityGroupRules = set $metricsSecurityGroupRules "kube-scheduler" (dict "port" 10259 "securityGroup" $controlPlaneSecurityGroup "remoteManagedGroups" (list $computePlaneSecurityGroup)) -}} - {{- end }} - {{- range $name, $securityGroupRule := $metricsSecurityGroupRules -}} - {{- $securityGroupRules = set $securityGroupRules (printf "metrics (%s)" $name) $securityGroupRule -}} + {{- $metricsSecurityGroupRules := dict -}} + {{- $metricsSecurityGroupRules = set $metricsSecurityGroupRules "prometheus node-exporter" (dict "port" 9100 "remoteManagedGroups" (list $computePlaneSecurityGroup)) -}} + {{- if not .Values.controlPlane.hosted -}} + {{- $metricsSecurityGroupRules = set $metricsSecurityGroupRules "etcd" (dict "port" 2381 "securityGroup" $controlPlaneSecurityGroup "remoteManagedGroups" (list $computePlaneSecurityGroup)) -}} + {{- $metricsSecurityGroupRules = set $metricsSecurityGroupRules "kube-proxy" (dict "port" 10249 "securityGroup" $controlPlaneSecurityGroup "remoteManagedGroups" (list $computePlaneSecurityGroup)) -}} + {{- $metricsSecurityGroupRules = set $metricsSecurityGroupRules "kube-controller-manager" (dict "port" 10257 "securityGroup" $controlPlaneSecurityGroup "remoteManagedGroups" (list $computePlaneSecurityGroup)) -}} + {{- $metricsSecurityGroupRules = set $metricsSecurityGroupRules "kube-scheduler" (dict "port" 10259 "securityGroup" $controlPlaneSecurityGroup "remoteManagedGroups" (list $computePlaneSecurityGroup)) -}} + {{- end }} + {{- range $name, $securityGroupRule := $metricsSecurityGroupRules -}} + {{- $securityGroupRules = set $securityGroupRules (printf "metrics (%s)" $name) $securityGroupRule -}} + {{- end -}} + {{- else -}} + {{- $securityGroupRules = set $securityGroupRules "cilium allow pod-pod native routing" (dict "remoteIPPrefix" "10.0.0.0/16" "protocol" nil) -}} {{- end -}} {{- range $name, $securityGroupRule := .Values.controlPlane.additionalSecurityGroupRules -}} @@ -97,17 +107,18 @@ identityRef: "description" ($securityGroupRule.description | default $name) "direction" "ingress" "etherType" "IPv4" - "protocol" ($securityGroupRule.protocol | default "tcp") + "protocol" (hasKey $securityGroupRule "protocol" | ternary $securityGroupRule.protocol "tcp") -}} - {{- if hasKey $securityGroupRule "remoteGroupID" -}} - {{- $_securityGroupRule = set $_securityGroupRule "remoteGroupID" $securityGroupRule.remoteGroupID -}} - {{- end -}} {{- if or (hasKey $securityGroupRule "port") (and (hasKey $securityGroupRule "portMin") (hasKey $securityGroupRule "portMax")) -}} {{- $_securityGroupRule = set $_securityGroupRule "portRangeMin" ($securityGroupRule.portMin | default $securityGroupRule.port) -}} {{- $_securityGroupRule = set $_securityGroupRule "portRangeMax" ($securityGroupRule.portMax | default $securityGroupRule.port) -}} {{- end -}} - {{- if hasKey $securityGroupRule "remoteManagedGroups" -}} + {{- if hasKey $securityGroupRule "remoteGroupID" -}} + {{- $_securityGroupRule = set $_securityGroupRule "remoteGroupID" $securityGroupRule.remoteGroupID -}} + {{- else if hasKey $securityGroupRule "remoteManagedGroups" -}} {{- $_securityGroupRule = set $_securityGroupRule "remoteManagedGroups" $securityGroupRule.remoteManagedGroups -}} + {{- else if hasKey $securityGroupRule "remoteIPPrefix" -}} + {{- $_securityGroupRule = set $_securityGroupRule "remoteIPPrefix" $securityGroupRule.remoteIPPrefix -}} {{- else -}} {{- $_securityGroupRule = set $_securityGroupRule "remoteManagedGroups" $allRemoteManagedGroups -}} {{- end -}} @@ -124,6 +135,8 @@ managedSecurityGroups: allNodesSecurityGroupRules: {{- toYaml $allNodesSecurityGroupRules | nindent 4 }} controlPlaneNodesSecurityGroupRules: {{- toYaml $controlPlaneNodesSecurityGroupRules | nindent 4 }} workerNodesSecurityGroupRules: {{- toYaml $workerNodesSecurityGroupRules | nindent 4 }} + {{/* for native routing we'd have to allow ingress from the pods and the nodes; which is just everything... */}} + allowAllInClusterTraffic: {{ $allowNativeRouting }} managedSubnets: - cidr: 10.6.0.0/24 {{- end -}} diff --git a/charts/t8s-cluster/templates/management-cluster/clusterClass/openStackMachineTemplates/_openstackMachineTemplateSpec.yaml b/charts/t8s-cluster/templates/management-cluster/clusterClass/openStackMachineTemplates/_openstackMachineTemplateSpec.yaml index 54d3d0acf2..cce05ed0d9 100644 --- a/charts/t8s-cluster/templates/management-cluster/clusterClass/openStackMachineTemplates/_openstackMachineTemplateSpec.yaml +++ b/charts/t8s-cluster/templates/management-cluster/clusterClass/openStackMachineTemplates/_openstackMachineTemplateSpec.yaml @@ -5,6 +5,11 @@ Here we are generating a hash suffix. {{- define "t8s-cluster.clusterClass.openStackMachineTemplate.spec" -}} {{- $_ := mustMerge . (pick .context "Values") -}} flavor: {{ eq .name "control-plane" | ternary .Values.controlPlane.flavor "compute-plane-placeholder" | required "flavor is required" }} + {{- if .Values.controlPlane.hosted }} +ports: + - allowedAddressPairs: + - ipAddress: 10.0.0.0/16 + {{- end }} image: filter: name: placeholder diff --git a/charts/t8s-cluster/templates/workload-cluster/cni-cilium.yaml b/charts/t8s-cluster/templates/workload-cluster/cni-cilium.yaml index d36ac9eec9..13593f7608 100644 --- a/charts/t8s-cluster/templates/workload-cluster/cni-cilium.yaml +++ b/charts/t8s-cluster/templates/workload-cluster/cni-cilium.yaml @@ -27,19 +27,45 @@ spec: values: nodePort: enabled: true - # enable eBPF based routing instead of iptables + {{- if .Values.controlPlane.hosted }} bpf: - masquerade: false # disable for now as this creates routing problems + masquerade: true + tproxy: true + enableTCX: true + endpointRoutes: + enabled: true + bandwidthManager: + enabled: true egressGateway: - enabled: false # disable for now as this depends on bpf.masquerade - # enable eBPF bases host routing - # currently not really possible with CAPI, as they don't support disabling the built-in kube-proxy - # kubeProxyReplacement: strict + enabled: true + # currently not really possible with kubeadmcontrolplane, as they don't support disabling the built-in kube-proxy + kubeProxyReplacement: true + ipam: + mode: kubernetes + routingMode: native + autoDirectNodeRoutes: true + directRoutingSkipUnreachable: true + localRedirectPolicies: + enabled: true + k8s: + requireIPv4PodCIDR: true + {{- $gateway := lookup "gateway.networking.k8s.io/v1" "Gateway" "capi-hosted-control-plane-system" "controlplane" -}} + {{- if not $gateway -}} + {{- fail "Hosted control plane Gateway 'controlplane' in namespace 'capi-hosted-control-plane-system' not found" -}} + {{- else }} + k8sServiceHost: {{ printf "%s.%s.%s" .Release.Name .Release.Namespace (replace "*." "" (index $gateway.spec.listeners 0).hostname) }} + {{- end }} + k8sServicePort: 443 + ipv4NativeRoutingCIDR: 10.0.0.0/16 # default net, see hosted control plane controller + {{- end }} rollOutCiliumPods: true encryption: enabled: false nodeEncryption: false + envoy: + prometheus: + enabled: true hubble: metrics: enabled: @@ -57,6 +83,12 @@ spec: ui: rollOutPods: true enabled: true + export: + static: + enabled: true + filePath: stdout + allowList: + - '{"verdict":["DROPPED","ERROR"]}' operator: rollOutPods: true prometheus: