diff --git a/bindata/network/ovn-kubernetes/managed/008-route.yaml b/bindata/network/ovn-kubernetes/managed/008-route.yaml deleted file mode 100644 index 245c7126d1..0000000000 --- a/bindata/network/ovn-kubernetes/managed/008-route.yaml +++ /dev/null @@ -1,28 +0,0 @@ -{{ if eq .OVNDbServiceType "ClusterIP" }} -# There are two service types supported for ovn sb db in hypershift: ClusterIP and NodePort -# When ClusterIP is used we expose the service through the following route -apiVersion: route.openshift.io/v1 -kind: Route -metadata: - name: ovnkube-sbdb - namespace: {{.HostedClusterNamespace}} - annotations: - network.operator.openshift.io/cluster-name: {{.ManagementClusterName}} - {{ if .OVNSbDbRouteLabels }} - labels: - {{ range $key, $value := .OVNSbDbRouteLabels }} - "{{$key}}": "{{$value}}" - {{ end }}{{ end }} -spec: - port: - targetPort: {{.OVN_SB_PORT}} - tls: - insecureEdgeTerminationPolicy: None - termination: passthrough - to: - kind: Service - name: ovnkube-master-external - {{ if .OVNSbDbRouteHost }} - host: {{.OVNSbDbRouteHost}} - {{ end }} -{{ end }} diff --git a/bindata/network/ovn-kubernetes/managed/ovnkube-cluster-manager.yaml b/bindata/network/ovn-kubernetes/managed/ovnkube-cluster-manager.yaml new file mode 100644 index 0000000000..3deeb2136d --- /dev/null +++ b/bindata/network/ovn-kubernetes/managed/ovnkube-cluster-manager.yaml @@ -0,0 +1,281 @@ +# The ovnkube control-plane components + +{{ if not .IsSNO }} +# The pod disruption budget ensures that we keep a raft quorum +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: ovn-raft-quorum-guard + namespace: {{.HostedClusterNamespace}} + annotations: + network.operator.openshift.io/cluster-name: {{.ManagementClusterName}} +spec: + minAvailable: {{.OVN_MIN_AVAILABLE}} + selector: + matchLabels: + app: ovnkube-cluster-manager +--- +{{ end }} +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: ovnkube-cluster-manager + namespace: {{.HostedClusterNamespace}} + annotations: + network.operator.openshift.io/cluster-name: {{.ManagementClusterName}} + kubernetes.io/description: | + This daemonset launches the ovn-kubernetes controller (master) networking components. + release.openshift.io/version: "{{.ReleaseVersion}}" + labels: + # used by PodAffinity to prefer co-locating pods that belong to the same hosted cluster. + hypershift.openshift.io/hosted-control-plane: {{.HostedClusterNamespace}} +spec: + podManagementPolicy: Parallel + selector: + matchLabels: + app: ovnkube-cluster-manager + serviceName: ovnkube-cluster-manager-internal + volumeClaimTemplates: + replicas: {{.OvnkubeMasterReplicas}} + template: + metadata: + annotations: + target.workload.openshift.io/management: '{"effect": "PreferredDuringScheduling"}' + labels: + app: ovnkube-cluster-manager + ovn-db-pod: "true" + component: network + type: infra + openshift.io/component: network + hypershift.openshift.io/control-plane-component: ovnkube-cluster-manager + kubernetes.io/os: "linux" + spec: + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 50 + preference: + matchExpressions: + - key: hypershift.openshift.io/control-plane + operator: In + values: + - "true" + - weight: 100 + preference: + matchExpressions: + - key: hypershift.openshift.io/cluster + operator: In + values: + - {{.HostedClusterNamespace}} + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app: ovnkube-cluster-manager + topologyKey: topology.kubernetes.io/zone + podAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchLabels: + hypershift.openshift.io/hosted-control-plane: {{.HostedClusterNamespace}} + topologyKey: kubernetes.io/hostname + priorityClassName: hypershift-api-critical + initContainers: + # Remove once https://github.com/kubernetes/kubernetes/issues/85966 is addressed + - name: init-ip + command: + - /bin/bash + - -c + - | + cat <<-EOF + Kubelet only sets a pod's Status.PodIP when all containers of the pod have started at least once (successfully or unsuccessfully) + or at least one of the initContainers finished. + Container start is blocked by postStart hooks. See https://github.com/kubernetes/kubernetes/issues/85966 for more details. + The NB and SB DB postStart hooks block until the DBs join the RAFT cluster or until a timeout is reached. + In a standalone cluster every pod is host networked and the DBs use host IP to communicate between the RAFT members. + In HyperShift OVN-Kubernetes master is run as a statefulset and the pods are not host networked, meaning we cannot rely on the podIP not changing. + To provide a stable network identity for each pod in the statefulset we use a headless service, + the downside of this approach is the DNS entry for the pod will only start to work after the pod has its Status.PodIP set. + + Until https://github.com/kubernetes/kubernetes/issues/85966 is fixed use a no-op init container as a workaround. + This allows for pod-pod connectivity in postStart hooks the first time they run. + EOF + image: "{{.OvnImage}}" + containers: + # token-minter creates a token with the default service account path + # The token is read by ovn-k containers to authenticate against the hosted cluster api server + - name: token-minter + image: "{{.TokenMinterImage}}" + command: ["/usr/bin/control-plane-operator", "token-minter"] + args: + - --service-account-namespace=openshift-ovn-kubernetes + - --service-account-name=ovn-kubernetes-controller + - --token-audience={{.TokenAudience}} + - --token-file=/var/run/secrets/hosted_cluster/token + - --kubeconfig=/etc/kubernetes/kubeconfig + resources: + requests: + cpu: 10m + memory: 30Mi + volumeMounts: + - mountPath: /etc/kubernetes + name: admin-kubeconfig + - mountPath: /var/run/secrets/hosted_cluster + name: hosted-cluster-api-access + # ovnkube master: convert kubernetes objects in to nbdb logical network components + - name: ovnkube-cluster-manager + image: "{{.OvnImage}}" + command: + - /bin/bash + - -c + - | + set -xe + if [[ -f "/env/_master" ]]; then + set -o allexport + source "/env/_master" + set +o allexport + fi + + # TLS for ovnkube-cluster-manager metrics + TLS_PK=/etc/pki/tls/metrics-cert/tls.key + TLS_CERT=/etc/pki/tls/metrics-cert/tls.crt + + if [ "{{.OVN_GATEWAY_MODE}}" == "shared" ]; then + gateway_mode_flags="--gateway-mode shared --gateway-interface br-ex" + elif [ "{{.OVN_GATEWAY_MODE}}" == "local" ]; then + gateway_mode_flags="--gateway-mode local --gateway-interface br-ex" + else + echo "Invalid OVN_GATEWAY_MODE: \"{{.OVN_GATEWAY_MODE}}\". Must be \"local\" or \"shared\"." + exit 1 + fi + + retries=0 + while [ ! -f /var/run/secrets/hosted_cluster/token ]; do + (( retries += 1 )) + sleep 1 + if [[ "${retries}" -gt 30 ]]; then + echo "$(date -Iseconds) - Hosted cluster token not found" + exit 1 + fi + done + + multi_network_enabled_flag= + if [[ "{{.OVN_MULTI_NETWORK_ENABLE}}" == "true" ]]; then + multi_network_enabled_flag="--enable-multi-network" + fi + + echo "I$(date "+%m%d %H:%M:%S.%N") - ovnkube-cluster-manager - start ovnkube --init-master ${K8S_NODE}" + exec /usr/bin/ovnkube \ + --init-cluster-manager "${K8S_NODE}" \ + --config-file=/run/ovnkube-config/ovnkube.conf \ + --k8s-token-file=/var/run/secrets/hosted_cluster/token \ + --ovn-empty-lb-events \ + --loglevel "${OVN_KUBE_LOG_LEVEL}" \ + #--metrics-bind-address "0.0.0.0:9102" \ + --metrics-bind-address "127.0.0.1:29104" \ + --metrics-enable-pprof \ + --metrics-enable-config-duration \ + ${gateway_mode_flags} \ + --sb-address "{{.OVN_SB_DB_LIST}}" \ + --sb-client-privkey /ovn-cert/tls.key \ + --sb-client-cert /ovn-cert/tls.crt \ + --sb-client-cacert /ovn-ca/ca-bundle.crt \ + --sb-cert-common-name "{{.OVN_CERT_CN}}" \ + --nb-address "{{.OVN_NB_DB_LIST}}" \ + --nb-client-privkey /ovn-cert/tls.key \ + --nb-client-cert /ovn-cert/tls.crt \ + --nb-client-cacert /ovn-ca/ca-bundle.crt \ + --nb-cert-common-name "{{.OVN_CERT_CN}}" \ + --enable-multicast \ + --disable-snat-multiple-gws \ + --node-server-privkey ${TLS_PK} \ + --node-server-cert ${TLS_CERT} \ + ${multi_network_enabled_flag} \ + --acl-logging-rate-limit "{{.OVNPolicyAuditRateLimit}}" + volumeMounts: + - mountPath: /run/ovnkube-config/ + name: ovnkube-config + - mountPath: /env + name: env-overrides + - mountPath: /ovn-cert + name: ovn-cert + - mountPath: /ovn-ca + name: ovn-ca + - mountPath: /hosted-ca + name: hosted-ca-cert + - mountPath: /var/run/secrets/hosted_cluster + name: hosted-cluster-api-access + - name: ovn-master-metrics-cert + mountPath: /etc/pki/tls/metrics-cert + readOnly: True + resources: + requests: + cpu: 10m + memory: 200Mi + env: + - name: OVN_KUBE_LOG_LEVEL + value: "4" + - name: K8S_NODE + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: ALL_PROXY + value: socks5://127.0.0.1:8090 + - name: NO_PROXY + value: kube-apiserver + ports: + - name: metrics-port + containerPort: 29102 + terminationMessagePolicy: FallbackToLogsOnError + {{ if .HCPNodeSelector }} + nodeSelector: + {{ range $key, $value := .HCPNodeSelector }} + "{{$key}}": "{{$value}}" + {{ end }} + {{ end }} + volumes: + - name: ovnkube-config + configMap: + name: ovnkube-config + - name: konnectivity-proxy-ca + configMap: + name: konnectivity-ca-bundle + - name: konnectivity-proxy-cert + secret: + defaultMode: 0640 + secretName: konnectivity-client + - name: env-overrides + configMap: + name: env-overrides + optional: true + - name: ovn-ca + configMap: + name: ovn-ca + - name: ovn-cert + secret: + secretName: ovn-cert + - name: ovn-master-metrics-cert + secret: + secretName: ovn-master-metrics-cert + - name: admin-kubeconfig + secret: + secretName: service-network-admin-kubeconfig + - name: hosted-cluster-api-access + emptyDir: {} + - name: hosted-ca-cert + secret: + secretName: root-ca + items: + - key: ca.crt + path: ca.crt + tolerations: + - key: "hypershift.openshift.io/control-plane" + operator: "Equal" + value: "true" + effect: "NoSchedule" + - key: "hypershift.openshift.io/cluster" + operator: "Equal" + value: {{.HostedClusterNamespace}} + effect: "NoSchedule" diff --git a/bindata/network/ovn-kubernetes/managed/ovnkube-node.yaml b/bindata/network/ovn-kubernetes/managed/ovnkube-local.yaml similarity index 62% rename from bindata/network/ovn-kubernetes/managed/ovnkube-node.yaml rename to bindata/network/ovn-kubernetes/managed/ovnkube-local.yaml index 70c6347e76..34167f7895 100644 --- a/bindata/network/ovn-kubernetes/managed/ovnkube-node.yaml +++ b/bindata/network/ovn-kubernetes/managed/ovnkube-local.yaml @@ -2,13 +2,7 @@ kind: DaemonSet apiVersion: apps/v1 metadata: - {{ if eq .OVN_NODE_MODE "dpu-host" }} - name: ovnkube-node-dpu-host - {{ else if eq .OVN_NODE_MODE "smart-nic" }} - name: ovnkube-node-smart-nic - {{ else }} - name: ovnkube-node - {{ end }} + name: ovnkube-local namespace: openshift-ovn-kubernetes annotations: kubernetes.io/description: | @@ -17,13 +11,7 @@ metadata: spec: selector: matchLabels: - {{ if eq .OVN_NODE_MODE "dpu-host" }} - app: ovnkube-node-dpu-host - {{ else if eq .OVN_NODE_MODE "smart-nic" }} - app: ovnkube-node-smart-nic - {{ else }} app: ovnkube-node - {{ end }} updateStrategy: type: RollingUpdate rollingUpdate: @@ -33,13 +21,7 @@ spec: annotations: target.workload.openshift.io/management: '{"effect": "PreferredDuringScheduling"}' labels: - {{ if eq .OVN_NODE_MODE "dpu-host" }} - app: ovnkube-node-dpu-host - {{ else if eq .OVN_NODE_MODE "smart-nic" }} - app: ovnkube-node-smart-nic - {{ else }} app: ovnkube-node - {{ end }} component: network type: infra openshift.io/component: network @@ -74,7 +56,7 @@ spec: - key: {{ .DpuModeLabel }} operator: DoesNotExist {{ end }} - serviceAccountName: ovn-kubernetes-node + serviceAccountName: ovn-kubernetes-controller hostNetwork: true dnsPolicy: Default hostPID: true @@ -83,59 +65,6 @@ spec: # to 127.0.0.1 with hostAlias settings. Ovnkube-node-proxy container # listens on 127.0.0.1 and connects to destination ovn sbdb route or # node port service through http proxy server. - {{if .ENABLE_OVN_NODE_PROXY}} - hostAliases: - - hostnames: - - {{.OVN_SB_DB_ROUTE_HOST}} - ip: 127.0.0.1 - {{end}} - initContainers: - # ovnkube-node-init: wait for sbdb ready - {{ if or (eq .OVN_NODE_MODE "full") (eq .OVN_NODE_MODE "smart-nic") }} - - name: ovnkube-node-init - image: "{{.OvnImage}}" - command: - - /bin/bash - - -c - - | - set -xe - if [[ -f "/env/${K8S_NODE}" ]]; then - set -o allexport - source "/env/${K8S_NODE}" - set +o allexport - fi - echo "$(date -Iseconds) - checking sbdb" - - {{if .ENABLE_OVN_NODE_PROXY}} - # Don't use /etc/hosts to resolve OVN_SB_DB_ROUTE_HOST which points to 127.0.0.1 (see spec.hostAliases) - echo "hosts: dns files" >> /etc/nsswitch.conf - exec socat TCP-LISTEN:{{.OVN_SB_DB_ROUTE_LOCAL_PORT}},reuseaddr,fork PROXY:{{.HTTP_PROXY_IP}}:{{.OVN_SB_DB_ROUTE_HOST}}:{{.OVN_SB_DB_ROUTE_PORT}},proxyport={{.HTTP_PROXY_PORT}} & - proxypid=$! - {{end}} - - ovndb_ctl_ssl_opts="-p /ovn-cert/tls.key -c /ovn-cert/tls.crt -C /ovn-ca/ca-bundle.crt" - sbdb_ip="{{.OVN_SB_DB_ENDPOINT}}" - retries=0 - while ! ovn-sbctl --no-leader-only --timeout=5 --db=${sbdb_ip} ${ovndb_ctl_ssl_opts} get-connection; do - (( retries += 1 )) - if [[ "${retries}" -gt 40 ]]; then - echo "$(date -Iseconds) - ERROR RESTARTING - sbdb - too many failed ovn-sbctl attempts, giving up" - exit 1 - fi - sleep 2 - done - - {{if .ENABLE_OVN_NODE_PROXY}} - kill $proxypid - {{end}} - volumeMounts: - - mountPath: /env - name: env-overrides - - mountPath: /ovn-cert - name: ovn-cert - - mountPath: /ovn-ca - name: ovn-ca - {{ end }} # volumes in all containers: # (container) -> (host) @@ -145,38 +74,6 @@ spec: # /env -> configmap env-overrides - debug overrides containers: {{ if or (eq .OVN_NODE_MODE "full") (eq .OVN_NODE_MODE "smart-nic") }} - {{if .ENABLE_OVN_NODE_PROXY}} - # ovnkube-node-proxy redirects ovn sbdb traffic to http proxy - - name: ovnkube-node-proxy - image: "{{.OvnImage}}" - command: - - /bin/bash - - -c - - | - set -xe - if [[ -f "/env/${K8S_NODE}" ]]; then - set -o allexport - source "/env/${K8S_NODE}" - set +o allexport - fi - echo "$(date -Iseconds) - starting ovnkube-node-proxy" - # Don't use /etc/hosts to resolve OVN_SB_DB_ROUTE_HOST which points to 127.0.0.1 (see spec.hostAliases) - echo "hosts: dns files" >> /etc/nsswitch.conf - exec socat TCP-LISTEN:{{.OVN_SB_DB_ROUTE_LOCAL_PORT}},reuseaddr,fork PROXY:{{.HTTP_PROXY_IP}}:{{.OVN_SB_DB_ROUTE_HOST}}:{{.OVN_SB_DB_ROUTE_PORT}},proxyport={{.HTTP_PROXY_PORT}} - env: - - name: K8S_NODE - valueFrom: - fieldRef: - fieldPath: spec.nodeName - volumeMounts: - - mountPath: /env - name: env-overrides - terminationMessagePolicy: FallbackToLogsOnError - resources: - requests: - cpu: 10m - memory: 20Mi - {{end}} # ovn-controller: programs the vswitch with flows from the sbdb - name: ovn-controller image: "{{.OvnImage}}" @@ -389,6 +286,437 @@ spec: - name: ovn-node-metrics-cert mountPath: /etc/pki/tls/metrics-cert readOnly: True + - name: northd + image: "{{.OvnImage}}" + command: + - /bin/bash + - -c + - | + set -xem + if [[ -f /env/_master ]]; then + set -o allexport + source /env/_master + set +o allexport + fi + + quit() { + echo "$(date -Iseconds) - stopping ovn-northd" + OVN_MANAGE_OVSDB=no /usr/share/ovn/scripts/ovn-ctl stop_northd + echo "$(date -Iseconds) - ovn-northd stopped" + rm -f /var/run/ovn/ovn-northd.pid + exit 0 + } + # end of quit + trap quit TERM INT + + echo "$(date -Iseconds) - starting ovn-northd" + exec ovn-northd \ + --no-chdir "-vconsole:${OVN_LOG_LEVEL}" -vfile:off "-vPATTERN:console:{{.OVN_LOG_PATTERN_CONSOLE}}" \ + --pidfile /var/run/ovn/ovn-northd.pid & + + wait $! + lifecycle: + preStop: + exec: + command: + - OVN_MANAGE_OVSDB=no + - /usr/share/ovn/scripts/ovn-ctl + - stop_northd + env: + - name: OVN_LOG_LEVEL + value: info + volumeMounts: + - mountPath: /etc/openvswitch/ + name: etc-openvswitch + - mountPath: /var/lib/openvswitch/ + name: var-lib-openvswitch + - mountPath: /run/openvswitch/ + name: run-openvswitch + - mountPath: /run/ovn/ + name: run-ovn + - mountPath: /env + name: env-overrides + - mountPath: /ovn-cert # not needed, but useful when exec'ing in to pod. + name: ovn-cert + - mountPath: /ovn-ca + name: ovn-ca + resources: + requests: + cpu: 10m + memory: 300Mi + terminationMessagePolicy: FallbackToLogsOnError + + # nbdb: the northbound, or logical network object DB. In standalone mode listening on unix socket. + - name: nbdb + image: "{{.OvnImage}}" + command: + - /bin/bash + - -c + - | + set -xem + if [[ -f /env/_master ]]; then + set -o allexport + source /env/_master + set +o allexport + fi + + quit() { + echo "$(date -Iseconds) - stopping nbdb" + /usr/share/ovn/scripts/ovn-ctl stop_nb_ovsdb + echo "$(date -Iseconds) - nbdb stopped" + rm -f /var/run/ovn/ovnnb_db.pid + exit 0 + } + # end of quit + trap quit TERM INT + + bracketify() { case "$1" in *:*) echo "[$1]" ;; *) echo "$1" ;; esac } + + # checks if a db pod is part of a current cluster + + OVN_ARGS="--no-monitor" + exec /usr/share/ovn/scripts/ovn-ctl ${OVN_ARGS} \ + --ovn-nb-log="-vconsole:${OVN_LOG_LEVEL} -vfile:off -vPATTERN:console:{{.OVN_LOG_PATTERN_CONSOLE}}" \ + run_nb_ovsdb & + + wait $! + + lifecycle: + postStart: + exec: + command: + - /bin/bash + - -c + - | + set -x + rm -f /var/run/ovn/ovnnb_db.pid + #configure northd_probe_interval + OVN_NB_CTL="ovn-nbctl" + # Set the zone + #${OVN_NB_CTL} set NB_Global . name=${K8S_NODE} + #${OVN_NB_CTL} set NB_Global . options:name=${K8S_NODE} + echo "Setting the IC zone to ${K8S_NODE}" + retries=0 + current_probe_interval=0 + while [[ "${retries}" -lt 10 ]]; do + current_probe_interval=$(${OVN_NB_CTL} set NB_Global . name="${K8S_NODE}" options:name="${K8S_NODE}") + if [[ $? == 0 ]]; then + current_probe_interval=$(echo ${current_probe_interval} | tr -d '\"') + break + else + sleep 2 + (( retries += 1 )) + fi + done + + northd_probe_interval=${OVN_NORTHD_PROBE_INTERVAL:-5000} + echo "Setting northd probe interval to ${northd_probe_interval} ms" + retries=0 + current_probe_interval=0 + while [[ "${retries}" -lt 10 ]]; do + current_probe_interval=$(${OVN_NB_CTL} --if-exists get NB_GLOBAL . options:northd_probe_interval) + if [[ $? == 0 ]]; then + current_probe_interval=$(echo ${current_probe_interval} | tr -d '\"') + break + else + sleep 2 + (( retries += 1 )) + fi + done + + if [[ "${current_probe_interval}" != "${northd_probe_interval}" ]]; then + retries=0 + while [[ "${retries}" -lt 10 ]]; do + ${OVN_NB_CTL} set NB_GLOBAL . options:northd_probe_interval=${northd_probe_interval} + if [[ $? != 0 ]]; then + echo "Failed to set northd probe interval to ${northd_probe_interval}. retrying....." + sleep 2 + (( retries += 1 )) + else + echo "Successfully set northd probe interval to ${northd_probe_interval} ms" + break + fi + done + fi + preStop: + exec: + command: + - /usr/bin/ovn-appctl + - -t + - /var/run/ovn/ovnnb_db.ctl + - exit + readinessProbe: +{{ if not .IsSNO }} + initialDelaySeconds: 90 +{{ end }} + timeoutSeconds: 5 + exec: + command: + - /bin/bash + - -c + - | + set -xeo pipefail + status=$(/usr/bin/ovn-appctl -t /var/run/ovn/ovnnb_db.ctl --timeout=3 ovsdb-server/sync-status 2>/dev/null | { grep "state: active" || false; }) + if [[ -z "${status}" ]]; then + echo "NB DB is not running or active." + exit 1 + fi + + env: + - name: OVN_LOG_LEVEL + value: info + - name: OVN_NORTHD_PROBE_INTERVAL + value: "{{.OVN_NORTHD_PROBE_INTERVAL}}" + - name: K8S_NODE + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - mountPath: /etc/openvswitch/ + name: etc-openvswitch + - mountPath: /etc/ovn/ + name: etc-openvswitch + - mountPath: /var/lib/openvswitch/ + name: var-lib-openvswitch + - mountPath: /run/openvswitch/ + name: run-openvswitch + - mountPath: /run/ovn/ + name: run-ovn + - mountPath: /env + name: env-overrides + - mountPath: /ovn-cert + name: ovn-cert + - mountPath: /ovn-ca + name: ovn-ca + resources: + requests: + cpu: 10m + memory: 300Mi + terminationMessagePolicy: FallbackToLogsOnError + # sbdb: the southbound, or logical network object DB. In standalone mode listening on unix socket + - name: sbdb + image: "{{.OvnImage}}" + command: + - /bin/bash + - -c + - | + set -xem + if [[ -f /env/_master ]]; then + set -o allexport + source /env/_master + set +o allexport + fi + + quit() { + echo "$(date -Iseconds) - stopping sbdb" + /usr/share/ovn/scripts/ovn-ctl stop_sb_ovsdb + echo "$(date -Iseconds) - sbdb stopped" + rm -f /var/run/ovn/ovnsb_db.pid + exit 0 + } + # end of quit + trap quit TERM INT + + bracketify() { case "$1" in *:*) echo "[$1]" ;; *) echo "$1" ;; esac } + + OVN_ARGS="--no-monitor" + exec /usr/share/ovn/scripts/ovn-ctl ${OVN_ARGS} \ + --ovn-sb-log="-vconsole:${OVN_LOG_LEVEL} -vfile:off -vPATTERN:console:{{.OVN_LOG_PATTERN_CONSOLE}}" \ + run_sb_ovsdb & + + wait $! + + lifecycle: + postStart: + exec: + command: + - /bin/bash + - -c + - | + set -x + rm -f /var/run/ovn/ovnsb_db.pid + + preStop: + exec: + command: + - /usr/bin/ovn-appctl + - -t + - /var/run/ovn/ovnsb_db.ctl + - exit + readinessProbe: +{{ if not .IsSNO }} + initialDelaySeconds: 90 +{{ end }} + timeoutSeconds: 5 + exec: + command: + - /bin/bash + - -c + - | + set -xeo pipefail + status=$(/usr/bin/ovn-appctl -t /var/run/ovn/ovnsb_db.ctl --timeout=3 ovsdb-server/sync-status 2>/dev/null | { grep "state: active" || false; }) + if [[ -z "${status}" ]]; then + echo "SB DB is not running or active." + exit 1 + fi + env: + - name: OVN_LOG_LEVEL + value: info + volumeMounts: + - mountPath: /etc/openvswitch/ + name: etc-openvswitch + - mountPath: /etc/ovn/ + name: etc-openvswitch + - mountPath: /var/lib/openvswitch/ + name: var-lib-openvswitch + - mountPath: /run/openvswitch/ + name: run-openvswitch + - mountPath: /run/ovn/ + name: run-ovn + - mountPath: /env + name: env-overrides + - mountPath: /ovn-cert + name: ovn-cert + - mountPath: /ovn-ca + name: ovn-ca + resources: + requests: + cpu: 10m + memory: 300Mi + terminationMessagePolicy: FallbackToLogsOnError + - name: kube-rbac-proxy-ncm + image: {{.KubeRBACProxyImage}} + command: + - /bin/bash + - -c + - | + #!/bin/bash + set -euo pipefail + TLS_PK=/etc/pki/tls/metrics-cert/tls.key + TLS_CERT=/etc/pki/tls/metrics-cert/tls.crt + # As the secret mount is optional we must wait for the files to be present. + # The service is created in monitor.yaml and this is created in sdn.yaml. + TS=$(date +%s) + WARN_TS=$(( ${TS} + $(( 20 * 60)) )) + HAS_LOGGED_INFO=0 + + log_missing_certs(){ + CUR_TS=$(date +%s) + if [[ "${CUR_TS}" -gt "WARN_TS" ]]; then + echo $(date -Iseconds) WARN: ovn-master-metrics-cert not mounted after 20 minutes. + elif [[ "${HAS_LOGGED_INFO}" -eq 0 ]] ; then + echo $(date -Iseconds) INFO: ovn-master-metrics-cert not mounted. Waiting 20 minutes. + HAS_LOGGED_INFO=1 + fi + } + while [[ ! -f "${TLS_PK}" || ! -f "${TLS_CERT}" ]] ; do + log_missing_certs + sleep 5 + done + + echo $(date -Iseconds) INFO: ovn-master-metrics-certs mounted, starting kube-rbac-proxy + exec /usr/bin/kube-rbac-proxy \ + --logtostderr \ + --secure-listen-address=:9102 \ + --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 \ + --upstream=http://127.0.0.1:29102/ \ + --tls-private-key-file=${TLS_PK} \ + --tls-cert-file=${TLS_CERT} + ports: + - containerPort: 9102 + name: https + resources: + requests: + cpu: 10m + memory: 20Mi + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - name: ovn-master-metrics-cert + mountPath: /etc/pki/tls/metrics-cert + readOnly: True + # ovnkube local: convert kubernetes objects in to nbdb logical network components + - name: ovnkube-controller + image: "{{.OvnImage}}" + command: + - /bin/bash + - -c + - | + set -xe + if [[ -f "/env/_local" ]]; then + set -o allexport + source "/env/_local" + set +o allexport + fi + + if [ "{{.OVN_GATEWAY_MODE}}" == "shared" ]; then + gateway_mode_flags="--gateway-mode shared --gateway-interface br-ex" + elif [ "{{.OVN_GATEWAY_MODE}}" == "local" ]; then + gateway_mode_flags="--gateway-mode local --gateway-interface br-ex" + else + echo "Invalid OVN_GATEWAY_MODE: \"{{.OVN_GATEWAY_MODE}}\". Must be \"local\" or \"shared\"." + exit 1 + fi + + multi_network_enabled_flag= + if [[ "{{.OVN_MULTI_NETWORK_ENABLE}}" == "true" ]]; then + multi_network_enabled_flag="--enable-multi-network" + fi + + echo "I$(date "+%m%d %H:%M:%S.%N") - ovnkube-controller - start ovnkube --init-ovnkube-controller ${K8S_NODE}" + exec /usr/bin/ovnkube \ + --init-ovnkube-controller "${K8S_NODE}" \ + --config-file=/run/ovnkube-config/ovnkube.conf \ + --ovn-empty-lb-events \ + --loglevel "${OVN_KUBE_LOG_LEVEL}" \ + --metrics-bind-address "127.0.0.1:29102" \ + --metrics-enable-pprof \ + --metrics-enable-config-duration \ + ${gateway_mode_flags} \ + --disable-snat-multiple-gws \ + ${multi_network_enabled_flag} \ + --enable-multicast \ + --zone ${K8S_NODE} \ + --enable-interconnect \ + --acl-logging-rate-limit "{{.OVNPolicyAuditRateLimit}}" + volumeMounts: + # for checking ovs-configuration service + - mountPath: /etc/systemd/system + name: systemd-units + readOnly: true + - mountPath: /etc/openvswitch/ + name: etc-openvswitch + - mountPath: /etc/ovn/ + name: etc-openvswitch + - mountPath: /var/lib/openvswitch/ + name: var-lib-openvswitch + - mountPath: /run/openvswitch/ + name: run-openvswitch + - mountPath: /run/ovn/ + name: run-ovn + - mountPath: /run/ovnkube-config/ + name: ovnkube-config + - mountPath: /env + name: env-overrides + - mountPath: /ovn-cert + name: ovn-cert + - mountPath: /ovn-ca + name: ovn-ca + resources: + requests: + cpu: 10m + memory: 300Mi + env: + - name: OVN_KUBE_LOG_LEVEL + value: "4" + - name: K8S_NODE + valueFrom: + fieldRef: + fieldPath: spec.nodeName + ports: + - name: metrics-port + containerPort: 29102 + terminationMessagePolicy: FallbackToLogsOnError + # ovnkube-node: does node-level bookkeeping and configuration - name: ovnkube-node image: "{{.OvnImage}}" @@ -402,7 +730,6 @@ spec: source "/env/${K8S_NODE}" set +o allexport fi - echo "I$(date "+%m%d %H:%M:%S.%N") - waiting for db_ip addresses" cp -f /usr/libexec/cni/ovn-k8s-cni-overlay /cni-bin-dir/ ovn_config_namespace=openshift-ovn-kubernetes echo "I$(date "+%m%d %H:%M:%S.%N") - disable conntrack on geneve port" @@ -410,9 +737,7 @@ spec: iptables -t raw -A OUTPUT -p udp --dport {{.GenevePort}} -j NOTRACK ip6tables -t raw -A PREROUTING -p udp --dport {{.GenevePort}} -j NOTRACK ip6tables -t raw -A OUTPUT -p udp --dport {{.GenevePort}} -j NOTRACK - retries=0 - - echo "I$(date "+%m%d %H:%M:%S.%N") - starting ovnkube-node db_ip ${db_ip}" + echo "I$(date "+%m%d %H:%M:%S.%N") - starting ovnkube-node" if [ "{{.OVN_GATEWAY_MODE}}" == "shared" ]; then gateway_mode_flags="--gateway-mode shared --gateway-interface br-ex" @@ -452,9 +777,6 @@ spec: if [[ -n "${OVNKUBE_NODE_MGMT_PORT_NETDEV}" ]] ; then node_mgmt_port_netdev_flags="--ovnkube-node-mgmt-port-netdev ${OVNKUBE_NODE_MGMT_PORT_NETDEV}" fi - if [[ -n "${OVNKUBE_NODE_MGMT_PORT_DP_RESOURCE_NAME}" ]] ; then - node_mgmt_port_netdev_flags="$node_mgmt_port_netdev_flags --ovnkube-node-mgmt-port-dp-resource-name ${OVNKUBE_NODE_MGMT_PORT_DP_RESOURCE_NAME}" - fi multi_network_enabled_flag= if [[ "{{.OVN_MULTI_NETWORK_ENABLE}}" == "true" ]]; then @@ -467,16 +789,6 @@ spec: fi exec /usr/bin/ovnkube --init-node "${K8S_NODE}" \ - --nb-address "{{.OVN_NB_DB_ENDPOINT}}" \ - --sb-address "{{.OVN_SB_DB_ENDPOINT}}" \ - --nb-client-privkey /ovn-cert/tls.key \ - --nb-client-cert /ovn-cert/tls.crt \ - --nb-client-cacert /ovn-ca/ca-bundle.crt \ - --nb-cert-common-name "{{.OVN_CERT_CN}}" \ - --sb-client-privkey /ovn-cert/tls.key \ - --sb-client-cert /ovn-cert/tls.crt \ - --sb-client-cacert /ovn-ca/ca-bundle.crt \ - --sb-cert-common-name "{{.OVN_CERT_CN}}" \ --config-file=/run/ovnkube-config/ovnkube.conf \ --loglevel "${OVN_KUBE_LOG_LEVEL}" \ --inactivity-probe="${OVN_CONTROLLER_INACTIVITY_PROBE}" \ @@ -492,7 +804,8 @@ spec: --disable-snat-multiple-gws \ ${export_network_flows_flags} \ ${multi_network_enabled_flag} \ - ${multi_network_policy_enabled_flag} \ + --zone ${K8S_NODE} \ + --enable-interconnect \ ${gw_interface_flag} env: # for kubectl @@ -504,18 +817,6 @@ spec: value: "{{.OVN_CONTROLLER_INACTIVITY_PROBE}}" - name: OVN_KUBE_LOG_LEVEL value: "4" -{{ if .HTTP_PROXY }} - - name: "HTTP_PROXY" - value: "{{ .HTTP_PROXY}}" -{{ end }} -{{ if .HTTPS_PROXY }} - - name: "HTTPS_PROXY" - value: "{{ .HTTPS_PROXY}}" -{{ end }} -{{ if .NO_PROXY }} - - name: "NO_PROXY" - value: "{{ .NO_PROXY}}" -{{ end }} {{ if .NetFlowCollectors }} - name: NETFLOW_COLLECTORS value: "{{.NetFlowCollectors}}" @@ -540,10 +841,6 @@ spec: - name: IPFIX_SAMPLING value: "{{.IPFIXSampling}}" {{ end }} - {{ if and (.MgmtPortResourceName) (or (eq .OVN_NODE_MODE "smart-nic") (eq .OVN_NODE_MODE "dpu-host")) }} - - name: OVNKUBE_NODE_MGMT_PORT_DP_RESOURCE_NAME - value: {{ .MgmtPortResourceName }} - {{ end }} - name: K8S_NODE valueFrom: fieldRef: @@ -569,6 +866,9 @@ spec: name: host-slash readOnly: true mountPropagation: HostToContainer + # for installing the CNI plugin binary + - mountPath: /cni-bin-dir + name: host-cni-bin # for the CNI server socket - mountPath: /run/ovn-kubernetes/ name: host-run-ovn-kubernetes @@ -578,9 +878,6 @@ spec: readOnly: true mountPropagation: HostToContainer # for installing the CNI plugin binary - - mountPath: /cni-bin-dir - name: host-cni-bin - # for installing the CNI configuration file - mountPath: /etc/cni/net.d name: host-cni-netd # Where we store IP allocations @@ -592,6 +889,9 @@ spec: name: run-ovn - mountPath: /etc/openvswitch name: etc-openvswitch + # for installing the CNI plugin binary + # - mountPath: /cni-bin-dir + # name: host-cni-bin - mountPath: /etc/ovn/ name: etc-openvswitch - mountPath: /var/lib/openvswitch @@ -608,13 +908,6 @@ spec: requests: cpu: 10m memory: 300Mi - {{ if and (.MgmtPortResourceName) (or (eq .OVN_NODE_MODE "smart-nic") (eq .OVN_NODE_MODE "dpu-host")) }} - {{ .MgmtPortResourceName }}: '1' - {{ end }} - {{ if and (.MgmtPortResourceName) (or (eq .OVN_NODE_MODE "smart-nic") (eq .OVN_NODE_MODE "dpu-host")) }} - limits: - {{ .MgmtPortResourceName }}: '1' - {{ end }} lifecycle: preStop: exec: @@ -624,6 +917,7 @@ spec: command: ["test", "-f", "/etc/cni/net.d/10-ovn-kubernetes.conf"] initialDelaySeconds: 5 periodSeconds: 30 + {{- if .OVNPlatformAzure}} - name: drop-icmp image: "{{.OvnImage}}" @@ -757,5 +1051,9 @@ spec: secret: secretName: ovn-node-metrics-cert optional: true + - name: ovn-master-metrics-cert + secret: + secretName: ovn-master-metrics-cert + optional: true tolerations: - operator: "Exists" diff --git a/bindata/network/ovn-kubernetes/managed/ovnkube-master.yaml b/bindata/network/ovn-kubernetes/managed/ovnkube-master.yaml deleted file mode 100644 index eae98c396e..0000000000 --- a/bindata/network/ovn-kubernetes/managed/ovnkube-master.yaml +++ /dev/null @@ -1,1102 +0,0 @@ -# The ovnkube control-plane components - -{{ if not .IsSNO }} -# The pod disruption budget ensures that we keep a raft quorum -apiVersion: policy/v1 -kind: PodDisruptionBudget -metadata: - name: ovn-raft-quorum-guard - namespace: {{.HostedClusterNamespace}} - annotations: - network.operator.openshift.io/cluster-name: {{.ManagementClusterName}} -spec: - minAvailable: {{.OVN_MIN_AVAILABLE}} - selector: - matchLabels: - app: ovnkube-master ---- -{{ end }} -kind: StatefulSet -apiVersion: apps/v1 -metadata: - name: ovnkube-master - namespace: {{.HostedClusterNamespace}} - annotations: - network.operator.openshift.io/cluster-name: {{.ManagementClusterName}} - kubernetes.io/description: | - This daemonset launches the ovn-kubernetes controller (master) networking components. - release.openshift.io/version: "{{.ReleaseVersion}}" - labels: - # used by PodAffinity to prefer co-locating pods that belong to the same hosted cluster. - hypershift.openshift.io/hosted-control-plane: {{.HostedClusterNamespace}} -spec: - podManagementPolicy: Parallel - selector: - matchLabels: - app: ovnkube-master - serviceName: ovnkube-master-internal - volumeClaimTemplates: - - apiVersion: v1 - kind: PersistentVolumeClaim - metadata: - name: datadir - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi - volumeMode: Filesystem - replicas: {{.OvnkubeMasterReplicas}} - template: - metadata: - annotations: - hypershift.openshift.io/release-image: {{.ReleaseImage}} - target.workload.openshift.io/management: '{"effect": "PreferredDuringScheduling"}' - cluster-autoscaler.kubernetes.io/safe-to-evict-local-volumes: "hosted-cluster-api-access" - labels: - app: ovnkube-master - ovn-db-pod: "true" - component: network - type: infra - openshift.io/component: network - hypershift.openshift.io/control-plane-component: ovnkube-master - kubernetes.io/os: "linux" - spec: - affinity: - nodeAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 50 - preference: - matchExpressions: - - key: hypershift.openshift.io/control-plane - operator: In - values: - - "true" - - weight: 100 - preference: - matchExpressions: - - key: hypershift.openshift.io/cluster - operator: In - values: - - {{.HostedClusterNamespace}} - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchLabels: - app: ovnkube-master - topologyKey: topology.kubernetes.io/zone - podAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - podAffinityTerm: - labelSelector: - matchLabels: - hypershift.openshift.io/hosted-control-plane: {{.HostedClusterNamespace}} - topologyKey: kubernetes.io/hostname - priorityClassName: hypershift-api-critical - initContainers: - # Remove once https://github.com/kubernetes/kubernetes/issues/85966 is addressed - - name: init-ip - command: - - /bin/bash - - -c - - | - cat <<-EOF - Kubelet only sets a pod's Status.PodIP when all containers of the pod have started at least once (successfully or unsuccessfully) - or at least one of the initContainers finished. - Container start is blocked by postStart hooks. See https://github.com/kubernetes/kubernetes/issues/85966 for more details. - The NB and SB DB postStart hooks block until the DBs join the RAFT cluster or until a timeout is reached. - In a standalone cluster every pod is host networked and the DBs use host IP to communicate between the RAFT members. - In HyperShift OVN-Kubernetes master is run as a statefulset and the pods are not host networked, meaning we cannot rely on the podIP not changing. - To provide a stable network identity for each pod in the statefulset we use a headless service, - the downside of this approach is the DNS entry for the pod will only start to work after the pod has its Status.PodIP set. - - Until https://github.com/kubernetes/kubernetes/issues/85966 is fixed use a no-op init container as a workaround. - This allows for pod-pod connectivity in postStart hooks the first time they run. - EOF - image: "{{.OvnImage}}" - containers: - # token-minter creates a token with the default service account path - # The token is read by ovn-k containers to authenticate against the hosted cluster api server - - name: token-minter - image: "{{.TokenMinterImage}}" - command: ["/usr/bin/control-plane-operator", "token-minter"] - args: - - --service-account-namespace=openshift-ovn-kubernetes - - --service-account-name=ovn-kubernetes-controller - - --token-audience={{.TokenAudience}} - - --token-file=/var/run/secrets/hosted_cluster/token - - --kubeconfig=/etc/kubernetes/kubeconfig - resources: - requests: - cpu: 10m - memory: 30Mi - volumeMounts: - - mountPath: /etc/kubernetes - name: admin-kubeconfig - - mountPath: /var/run/secrets/hosted_cluster - name: hosted-cluster-api-access - # ovn-northd: convert network objects in nbdb to flows in sbdb - - name: northd - image: "{{.OvnImage}}" - command: - - /bin/bash - - -c - - | - set -xem - if [[ -f /env/_master ]]; then - set -o allexport - source /env/_master - set +o allexport - fi - - quit() { - echo "$(date -Iseconds) - stopping ovn-northd" - OVN_MANAGE_OVSDB=no /usr/share/ovn/scripts/ovn-ctl stop_northd - echo "$(date -Iseconds) - ovn-northd stopped" - rm -f /var/run/ovn/ovn-northd.pid - exit 0 - } - # end of quit - trap quit TERM INT - - echo "$(date -Iseconds) - starting ovn-northd" - exec ovn-northd \ - --no-chdir "-vconsole:${OVN_LOG_LEVEL}" -vfile:off "-vPATTERN:console:{{.OVN_LOG_PATTERN_CONSOLE}}" \ - --ovnnb-db "{{.OVN_NB_DB_LIST}}" \ - --ovnsb-db "{{.OVN_SB_DB_LIST}}" \ - --pidfile /var/run/ovn/ovn-northd.pid \ - --n-threads={{.NorthdThreads}} \ - -p /ovn-cert/tls.key \ - -c /ovn-cert/tls.crt \ - -C /ovn-ca/ca-bundle.crt & - - wait $! - lifecycle: - preStop: - exec: - command: - - /bin/bash - - -c - - OVN_MANAGE_OVSDB=no /usr/share/ovn/scripts/ovn-ctl stop_northd - env: - - name: OVN_LOG_LEVEL - value: info - volumeMounts: - - mountPath: /var/run/ovn - name: datadir - - mountPath: /etc/ovn - name: datadir - - mountPath: /var/log/ovn - name: datadir - - mountPath: /env - name: env-overrides - - mountPath: /ovn-cert # not needed, but useful when exec'ing in to pod. - name: ovn-cert - - mountPath: /ovn-ca - name: ovn-ca - resources: - requests: - cpu: 10m - memory: 70Mi - terminationMessagePolicy: FallbackToLogsOnError - - # nbdb: the northbound, or logical network object DB. In raft mode - - name: nbdb - image: "{{.OvnImage}}" - command: - - /bin/bash - - -c - - | - set -xem - if [[ -f /env/_master ]]; then - set -o allexport - source /env/_master - set +o allexport - fi - - quit() { - echo "$(date -Iseconds) - stopping nbdb" - /usr/share/ovn/scripts/ovn-ctl stop_nb_ovsdb - echo "$(date -Iseconds) - nbdb stopped" - rm -f /var/run/ovn/ovnnb_db.pid - exit 0 - } - # end of quit - trap quit TERM INT - - bracketify() { case "$1" in *:*) echo "[$1]" ;; *) echo "$1" ;; esac } - - # initialize variables - pod_dns_name="${K8S_POD_NAME}.ovnkube-master-internal.{{.HostedClusterNamespace}}.svc.cluster.local" - ovndb_ctl_ssl_opts="-p /ovn-cert/tls.key -c /ovn-cert/tls.crt -C /ovn-ca/ca-bundle.crt" - transport="ssl" - ovn_raft_conn_ip_url_suffix="" - if [[ "${K8S_POD_IP}" == *":"* ]]; then - ovn_raft_conn_ip_url_suffix=":[::]" - fi - db="nb" - db_port="{{.OVN_NB_PORT}}" - ovn_db_file="/etc/ovn/ovn${db}_db.db" - # checks if a db pod is part of a current cluster - db_part_of_cluster() { - local pod_index=${1} - local db=${2} - local port=${3} - echo "Checking if ovnkube-master-${pod_index} is part of cluster" - - init_ip="ovnkube-master-${pod_index}.ovnkube-master-internal.{{.HostedClusterNamespace}}.svc.cluster.local" - - - - - echo "ovnkube-master-${pod_index} ip: $init_ip" - init_ip=$(bracketify $init_ip) - target=$(ovn-${db}ctl --timeout=5 --db=${transport}:${init_ip}:${port} ${ovndb_ctl_ssl_opts} \ - --data=bare --no-headings --columns=target list connection || true) - if [[ "${target}" != "p${transport}:${port}${ovn_raft_conn_ip_url_suffix}" ]]; then - echo "Unable to check correct target ${target} " - return 1 - fi - echo "ovnkube-master-${pod_index} is part of cluster" - return 0 - } - # end of db_part_of_cluster - - # Checks if cluster has already been initialized. - # If not it returns false and sets init_ip to CLUSTER_INITIATOR_IP - cluster_exists() { - local db=${1} - local port=${2} - for ((i=0; i<{{.OvnkubeMasterReplicas}}; i++ )); do - if db_part_of_cluster $i $db $port; then - echo "ovnkube-master-${i} is part of current cluster with ip: ${init_ip}!" - return 0 - fi - done - # if we get here there is no cluster, set init_ip and get out - init_ip=$(bracketify $CLUSTER_INITIATOR_IP) - return 1 - } - # end of cluster_exists() - - # RAFT clusters need an odd number of members to achieve consensus. - # The CNO determines which members make up the cluster, so if this container - # is not supposed to be part of the cluster, wait forever doing nothing - # (instad of exiting and causing CrashLoopBackoffs for no reason). - if [[ ! "{{.OVN_NB_DB_LIST}}" =~ .*":${pod_dns_name}:".* ]] && [[ ! "{{.OVN_NB_DB_LIST}}" =~ .*":[${pod_dns_name}]:".* ]]; then - echo "$(date -Iseconds) - not selected as RAFT member; sleeping..." - sleep 1500d - exit 0 - fi - - OVN_ARGS="--db-nb-cluster-local-port={{.OVN_NB_RAFT_PORT}} \ - --db-nb-cluster-local-addr=${pod_dns_name} \ - --no-monitor \ - --db-nb-cluster-local-proto=ssl \ - --ovn-nb-db-ssl-key=/ovn-cert/tls.key \ - --ovn-nb-db-ssl-cert=/ovn-cert/tls.crt \ - --ovn-nb-db-ssl-ca-cert=/ovn-ca/ca-bundle.crt" - - CLUSTER_INITIATOR_IP="{{.OVN_DB_CLUSTER_INITIATOR}}" - echo "$(date -Iseconds) - starting nbdb CLUSTER_INITIATOR_IP=${CLUSTER_INITIATOR_IP}, K8S_NODE_IP=${K8S_NODE_IP}" - initialize="false" - - if [[ ! -e ${ovn_db_file} ]]; then - initialize="true" - fi - - if [[ "${initialize}" == "true" ]]; then - # check to see if a cluster already exists. If it does, just join it. - counter=0 - cluster_found=false - while [ $counter -lt 5 ]; do - if cluster_exists ${db} ${db_port}; then - cluster_found=true - break - fi - sleep 1 - counter=$((counter+1)) - done - - if ${cluster_found}; then - echo "Cluster already exists for DB: ${db}" - # join existing cluster - exec /usr/share/ovn/scripts/ovn-ctl ${OVN_ARGS} \ - --db-nb-cluster-remote-port={{.OVN_NB_RAFT_PORT}} \ - --db-nb-cluster-remote-addr=${init_ip} \ - --db-nb-cluster-remote-proto=ssl \ - --ovn-nb-log="-vconsole:${OVN_LOG_LEVEL} -vfile:off -vPATTERN:console:{{.OVN_LOG_PATTERN_CONSOLE}}" \ - run_nb_ovsdb & - - wait $! - else - # either we need to initialize a new cluster or wait for master to create it - if [[ "${pod_dns_name}" == "${CLUSTER_INITIATOR_IP}" ]]; then - # set DB election timer at DB creation time if OVN supports it - election_timer= - if test -n "$(/usr/share/ovn/scripts/ovn-ctl --help 2>&1 | grep "\--db-nb-election-timer")"; then - election_timer="--db-nb-election-timer=$(({{.OVN_NB_RAFT_ELECTION_TIMER}}*1000))" - fi - - exec /usr/share/ovn/scripts/ovn-ctl ${OVN_ARGS} \ - --ovn-nb-log="-vconsole:${OVN_LOG_LEVEL} -vfile:off -vPATTERN:console:{{.OVN_LOG_PATTERN_CONSOLE}}" \ - ${election_timer} \ - run_nb_ovsdb & - - wait $! - else - echo "Joining the nbdb cluster with init_ip=${init_ip}..." - exec /usr/share/ovn/scripts/ovn-ctl ${OVN_ARGS} \ - --db-nb-cluster-remote-port={{.OVN_NB_RAFT_PORT}} \ - --db-nb-cluster-remote-addr=${init_ip} \ - --db-nb-cluster-remote-proto=ssl \ - --ovn-nb-log="-vconsole:${OVN_LOG_LEVEL} -vfile:off -vPATTERN:console:{{.OVN_LOG_PATTERN_CONSOLE}}" \ - run_nb_ovsdb & - - wait $! - fi - fi - else - exec /usr/share/ovn/scripts/ovn-ctl ${OVN_ARGS} \ - --ovn-nb-log="-vconsole:${OVN_LOG_LEVEL} -vfile:off -vPATTERN:console:{{.OVN_LOG_PATTERN_CONSOLE}}" \ - run_nb_ovsdb & - - wait $! - fi - - lifecycle: - postStart: - exec: - command: - - /bin/bash - - -c - - | - set -x - CLUSTER_INITIATOR_IP="{{.OVN_DB_CLUSTER_INITIATOR}}" - rm -f /var/run/ovn/ovnnb_db.pid - - pod_dns_name="${K8S_POD_NAME}.ovnkube-master-internal.{{.HostedClusterNamespace}}.svc.cluster.local" - - # exit early if this DB is not supposed to be part of the cluster - if [[ ! "{{.OVN_NB_DB_LIST}}" =~ .*":${pod_dns_name}:".* ]] && [[ ! "{{.OVN_NB_DB_LIST}}" =~ .*":[${pod_dns_name}]:".* ]]; then - exit 0 - fi - - # retry an operation a number of times, sleeping 2 seconds between each try - retry() { - local tries=${1} - local desc=${2} - local cmd=${3} - - local retries=0 - while ! ${cmd}; do - (( retries += 1 )) - if [[ "${retries}" -gt ${tries} ]]; then - echo "$(date -Iseconds) - ERROR - nbdb ${desc} - too many failed attempts, giving up" - return 1 - fi - echo "$(date -Iseconds) - WARN - nbdb ${desc} - failed try ${retries}, retrying..." - sleep 2 - done - echo "$(date -Iseconds) - INFO - nbdb ${desc} - success" - return 0 - } - - if [[ "${pod_dns_name}" == "${CLUSTER_INITIATOR_IP}" ]]; then - echo "$(date -Iseconds) - nbdb - postStart - waiting for master to be selected" - - # set the connection and inactivity probe - if ! retry 60 "inactivity-probe" "ovn-nbctl --no-leader-only -t 5 set-connection pssl:{{.OVN_NB_PORT}}{{.LISTEN_DUAL_STACK}} -- set connection . inactivity_probe={{.OVN_NB_INACTIVITY_PROBE}}"; then - exit 1 - fi - - # Upgrade the db if required. - DB_SCHEMA="/usr/share/ovn/ovn-nb.ovsschema" - DB_SERVER="unix:/var/run/ovn/ovnnb_db.sock" - schema_name=$(ovsdb-tool schema-name $DB_SCHEMA) - db_version=$(ovsdb-client -t 10 get-schema-version "$DB_SERVER" "$schema_name") - target_version=$(ovsdb-tool schema-version "$DB_SCHEMA") - - if ovsdb-tool compare-versions "$db_version" == "$target_version"; then - : - elif ovsdb-tool compare-versions "$db_version" ">" "$target_version"; then - echo "Database $schema_name has newer schema version ($db_version) than our local schema ($target_version), possibly an upgrade is partially complete?" - else - echo "Upgrading database $schema_name from schema version $db_version to $target_version" - ovsdb-client -t 30 convert "$DB_SERVER" "$DB_SCHEMA" - fi - fi - - # read the current northd_probe_interval from the DB - OVN_NB_CTL="ovn-nbctl -p /ovn-cert/tls.key -c /ovn-cert/tls.crt -C /ovn-ca/ca-bundle.crt --db "{{.OVN_NB_DB_LIST}}"" - northd_probe_interval=${OVN_NORTHD_PROBE_INTERVAL:-10000} - echo "Setting northd probe interval to ${northd_probe_interval} ms" - retries=0 - current_probe_interval=0 - while [[ "${retries}" -lt 20 ]]; do - current_probe_interval=$(${OVN_NB_CTL} --if-exists get NB_GLOBAL . options:northd_probe_interval) - if [[ $? == 0 ]]; then - current_probe_interval=$(echo ${current_probe_interval} | tr -d '\"') - break - else - sleep 2 - (( retries += 1 )) - fi - done - - # ensure the northd_probe_interval is set to the configured value - if [[ "${current_probe_interval}" != "${northd_probe_interval}" ]]; then - if ! retry 20 "northd-probe" "${OVN_NB_CTL} set NB_GLOBAL . options:northd_probe_interval=${northd_probe_interval}"; then - exit 1 - fi - fi - - # Enable/disable IPsec - {{ if .OVNIPsecEnable }} - ipsec=true - {{ else }} - ipsec=false - {{ end }} - if ! retry 20 "ipsec" "${OVN_NB_CTL} set nb_global . ipsec=${ipsec}"; then - exit 1 - fi - preStop: - exec: - command: - - /bin/bash - - -c - - | - echo "$(date -Iseconds) - stopping nbdb" - /usr/share/ovn/scripts/ovn-ctl stop_nb_ovsdb - echo "$(date -Iseconds) - nbdb stopped" - rm -f /var/run/ovn/ovnnb_db.pid - readinessProbe: - initialDelaySeconds: 90 - timeoutSeconds: 5 - exec: - command: - - /bin/bash - - -c - - | - set -xeo pipefail - - # exit early if this DB is not supposed to be part of the cluster - pod_dns_name="${K8S_POD_NAME}.ovnkube-master-internal.{{.HostedClusterNamespace}}.svc.cluster.local" - if [[ ! "{{.OVN_NB_DB_LIST}}" =~ .*":${pod_dns_name}:".* ]] && [[ ! "{{.OVN_NB_DB_LIST}}" =~ .*":[${pod_dns_name}]:".* ]]; then - exit 0 - fi - - leader_status=$(/usr/bin/ovn-appctl -t /var/run/ovn/ovnnb_db.ctl --timeout=3 cluster/status OVN_Northbound 2>/dev/null | { grep "Leader: unknown" || true; }) - if [[ ! -z "${leader_status}" ]]; then - echo "NB DB Raft leader is unknown to the cluster node." - exit 1 - fi - # set trim-on-compaction - /usr/bin/ovn-appctl -t /var/run/ovn/ovnnb_db.ctl --timeout=5 ovsdb-server/memory-trim-on-compaction on 2>/dev/null - env: - - name: OVN_LOG_LEVEL - value: info - - name: OVN_NORTHD_PROBE_INTERVAL - value: "{{.OVN_NORTHD_PROBE_INTERVAL}}" - - name: K8S_NODE_IP - valueFrom: - fieldRef: - fieldPath: status.hostIP - - name: K8S_POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: K8S_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - volumeMounts: - - mountPath: /etc/ovn - name: datadir - - mountPath: /var/run/ovn - name: datadir - - mountPath: /var/log/ovn - name: datadir - - mountPath: /env - name: env-overrides - - mountPath: /ovn-cert - name: ovn-cert - - mountPath: /ovn-ca - name: ovn-ca - resources: - requests: - cpu: 10m - memory: 30Mi - ports: - - name: nb-db-port - containerPort: {{.OVN_NB_PORT}} - - name: nb-db-raft-port - containerPort: {{.OVN_NB_RAFT_PORT}} - terminationMessagePolicy: FallbackToLogsOnError - - # sbdb: The southbound, or flow DB. In raft mode - - name: sbdb - image: "{{.OvnImage}}" - command: - - /bin/bash - - -c - - | - set -xm - if [[ -f /env/_master ]]; then - set -o allexport - source /env/_master - set +o allexport - fi - - quit() { - echo "$(date -Iseconds) - stopping sbdb" - /usr/share/ovn/scripts/ovn-ctl stop_sb_ovsdb - echo "$(date -Iseconds) - sbdb stopped" - rm -f /var/run/ovn/ovnsb_db.pid - exit 0 - } - # end of quit - trap quit TERM INT - - # initialize variables - pod_dns_name="${K8S_POD_NAME}.ovnkube-master-internal.{{.HostedClusterNamespace}}.svc.cluster.local" - ovndb_ctl_ssl_opts="-p /ovn-cert/tls.key -c /ovn-cert/tls.crt -C /ovn-ca/ca-bundle.crt" - transport="ssl" - ovn_raft_conn_ip_url_suffix="" - if [[ "${K8S_POD_IP}" == *":"* ]]; then - ovn_raft_conn_ip_url_suffix=":[::]" - fi - db="sb" - db_port="{{.OVN_SB_PORT}}" - ovn_db_file="/etc/ovn/ovn${db}_db.db" - # checks if a db pod is part of a current cluster - db_part_of_cluster() { - local pod_index=${1} - local db=${2} - local port=${3} - echo "Checking if ovnkube-master-${pod_index} is part of cluster" - init_ip="ovnkube-master-${pod_index}.ovnkube-master-internal.{{.HostedClusterNamespace}}.svc.cluster.local" - echo "ovnkube-master-${pod_index} ip: $init_ip" - target=$(ovn-${db}ctl --timeout=5 --db=${transport}:${init_ip}:${port} ${ovndb_ctl_ssl_opts} \ - --data=bare --no-headings --columns=target list connection || true) - if [[ "${target}" != "p${transport}:${port}${ovn_raft_conn_ip_url_suffix}" ]]; then - echo "Unable to check correct target ${target} " - return 1 - fi - echo "ovnkube-master-${pod_index} is part of cluster" - return 0 - } - # end of db_part_of_cluster - - # Checks if cluster has already been initialized. - # If not it returns false and sets init_ip to CLUSTER_INITIATOR_IP - cluster_exists() { - local db=${1} - local port=${2} - for ((i=0; i<{{.OvnkubeMasterReplicas}}; i++ )); do - if db_part_of_cluster $i $db $port; then - echo "ovnkube-master-${i} is part of current cluster with ip: ${init_ip}!" - return 0 - fi - done - # if we get here there is no cluster, set init_ip and get out - init_ip=$CLUSTER_INITIATOR_IP - return 1 - } - # end of cluster_exists() - - # RAFT clusters need an odd number of members to achieve consensus. - # The CNO determines which members make up the cluster, so if this container - # is not supposed to be part of the cluster, wait forever doing nothing - # (instad of exiting and causing CrashLoopBackoffs for no reason). - if [[ ! "{{.OVN_SB_DB_LIST}}" =~ .*":${pod_dns_name}:".* ]] && [[ ! "{{.OVN_SB_DB_LIST}}" =~ .*":[${pod_dns_name}]:".* ]]; then - echo "$(date -Iseconds) - not selected as RAFT member; sleeping..." - sleep 1500d - exit 0 - fi - - OVN_ARGS="--db-sb-cluster-local-port={{.OVN_SB_RAFT_PORT}} \ - --db-sb-cluster-local-addr="${pod_dns_name}" \ - --no-monitor \ - --db-sb-cluster-local-proto=ssl \ - --ovn-sb-db-ssl-key=/ovn-cert/tls.key \ - --ovn-sb-db-ssl-cert=/ovn-cert/tls.crt \ - --ovn-sb-db-ssl-ca-cert=/ovn-ca/ca-bundle.crt" - - CLUSTER_INITIATOR_IP="{{.OVN_DB_CLUSTER_INITIATOR}}" - echo "$(date -Iseconds) - starting sbdb CLUSTER_INITIATOR_IP=${CLUSTER_INITIATOR_IP}" - initialize="false" - - if [[ ! -e ${ovn_db_file} ]]; then - initialize="true" - fi - - if [[ "${initialize}" == "true" ]]; then - # check to see if a cluster already exists. If it does, just join it. - counter=0 - cluster_found=false - while [ $counter -lt 5 ]; do - if cluster_exists ${db} ${db_port}; then - cluster_found=true - break - fi - sleep 1 - counter=$((counter+1)) - done - - if ${cluster_found}; then - echo "Cluster already exists for DB: ${db}" - # join existing cluster - exec /usr/share/ovn/scripts/ovn-ctl ${OVN_ARGS} \ - --db-sb-cluster-remote-port={{.OVN_SB_RAFT_PORT}} \ - --db-sb-cluster-remote-addr=${init_ip} \ - --db-sb-cluster-remote-proto=ssl \ - --ovn-sb-log="-vconsole:${OVN_LOG_LEVEL} -vfile:off -vPATTERN:console:{{.OVN_LOG_PATTERN_CONSOLE}}" \ - run_sb_ovsdb & - - wait $! - else - # either we need to initialize a new cluster or wait for master to create it - if [[ "${pod_dns_name}" == "${CLUSTER_INITIATOR_IP}" ]]; then - # set DB election timer at DB creation time if OVN supports it - election_timer= - if test -n "$(/usr/share/ovn/scripts/ovn-ctl --help 2>&1 | grep "\--db-sb-election-timer")"; then - election_timer="--db-sb-election-timer=$(({{.OVN_SB_RAFT_ELECTION_TIMER}}*1000))" - fi - - exec /usr/share/ovn/scripts/ovn-ctl ${OVN_ARGS} \ - --ovn-sb-log="-vconsole:${OVN_LOG_LEVEL} -vfile:off -vPATTERN:console:{{.OVN_LOG_PATTERN_CONSOLE}}" \ - ${election_timer} \ - run_sb_ovsdb & - - wait $! - else - exec /usr/share/ovn/scripts/ovn-ctl ${OVN_ARGS} \ - --db-sb-cluster-remote-port={{.OVN_SB_RAFT_PORT}} \ - --db-sb-cluster-remote-addr=${init_ip} \ - --db-sb-cluster-remote-proto=ssl \ - --ovn-sb-log="-vconsole:${OVN_LOG_LEVEL} -vfile:off -vPATTERN:console:{{.OVN_LOG_PATTERN_CONSOLE}}" \ - run_sb_ovsdb & - - wait $! - fi - fi - else - exec /usr/share/ovn/scripts/ovn-ctl ${OVN_ARGS} \ - --ovn-sb-log="-vconsole:${OVN_LOG_LEVEL} -vfile:off -vPATTERN:console:{{.OVN_LOG_PATTERN_CONSOLE}}" \ - run_sb_ovsdb & - - wait $! - fi - lifecycle: - postStart: - exec: - command: - - /bin/bash - - -c - - | - set -x - CLUSTER_INITIATOR_IP="{{.OVN_DB_CLUSTER_INITIATOR}}" - rm -f /var/run/ovn/ovnsb_db.pid - - pod_dns_name="${K8S_POD_NAME}.ovnkube-master-internal.{{.HostedClusterNamespace}}.svc.cluster.local" - - # exit early if this DB is not supposed to be part of the cluster - if [[ ! "{{.OVN_SB_DB_LIST}}" =~ .*":${pod_dns_name}:".* ]] && [[ ! "{{.OVN_SB_DB_LIST}}" =~ .*":[${pod_dns_name}]:".* ]]; then - exit 0 - fi - - # retry an operation a number of times, sleeping 2 seconds between each try - retry() { - local tries=${1} - local desc=${2} - local cmd=${3} - - local retries=0 - while ! ${cmd}; do - (( retries += 1 )) - if [[ "${retries}" -gt ${tries} ]]; then - echo "$(date -Iseconds) - ERROR - nbdb ${desc} - too many failed attempts, giving up" - return 1 - fi - echo "$(date -Iseconds) - WARN - nbdb ${desc} - failed try ${retries}, retrying..." - sleep 2 - done - echo "$(date -Iseconds) - INFO - nbdb ${desc} - success" - return 0 - } - - if [[ "${pod_dns_name}" == "${CLUSTER_INITIATOR_IP}" ]]; then - echo "$(date -Iseconds) - sdb - postStart - waiting for master to be selected" - - # set the connection and inactivity probe - if ! retry 60 "inactivity-probe" "ovn-sbctl --no-leader-only -t 5 set-connection pssl:{{.OVN_SB_PORT}}{{.LISTEN_DUAL_STACK}} -- set connection . inactivity_probe={{.OVN_CONTROLLER_INACTIVITY_PROBE}}"; then - exit 1 - fi - - # Upgrade the db if required. - DB_SCHEMA="/usr/share/ovn/ovn-sb.ovsschema" - DB_SERVER="unix:/var/run/ovn/ovnsb_db.sock" - schema_name=$(ovsdb-tool schema-name $DB_SCHEMA) - db_version=$(ovsdb-client -t 10 get-schema-version "$DB_SERVER" "$schema_name") - target_version=$(ovsdb-tool schema-version "$DB_SCHEMA") - - if ovsdb-tool compare-versions "$db_version" == "$target_version"; then - : - elif ovsdb-tool compare-versions "$db_version" ">" "$target_version"; then - echo "Database $schema_name has newer schema version ($db_version) than our local schema ($target_version), possibly an upgrade is partially complete?" - else - echo "Upgrading database $schema_name from schema version $db_version to $target_version" - ovsdb-client -t 30 convert "$DB_SERVER" "$DB_SCHEMA" - fi - fi - - # Kill some time while the cluster converges by checking IPsec status - OVN_SB_CTL="ovn-sbctl -p /ovn-cert/tls.key -c /ovn-cert/tls.crt -C /ovn-ca/ca-bundle.crt --db "{{.OVN_SB_DB_LIST}}"" - if ! retry 20 "ipsec" "${OVN_SB_CTL} get sb_global . ipsec"; then - exit 1 - fi - preStop: - exec: - command: - - /bin/bash - - -c - - | - echo "$(date -Iseconds) - stopping sbdb" - /usr/share/ovn/scripts/ovn-ctl stop_sb_ovsdb - echo "$(date -Iseconds) - sbdb stopped" - rm -f /var/run/ovn/ovnsb_db.pid - readinessProbe: - initialDelaySeconds: 90 - timeoutSeconds: 5 - exec: - command: - - /bin/bash - - -c - - | - set -xeo pipefail - - # exit early if this DB is not supposed to be part of the cluster - pod_dns_name="${K8S_POD_NAME}.ovnkube-master-internal.{{.HostedClusterNamespace}}.svc.cluster.local" - if [[ ! "{{.OVN_SB_DB_LIST}}" =~ .*":${pod_dns_name}:".* ]] && [[ ! "{{.OVN_SB_DB_LIST}}" =~ .*":[${pod_dns_name}]:".* ]]; then - exit 0 - fi - - leader_status=$(/usr/bin/ovn-appctl -t /var/run/ovn/ovnsb_db.ctl --timeout=3 cluster/status OVN_Southbound 2>/dev/null | { grep "Leader: unknown" || true; }) - if [[ ! -z "${leader_status}" ]]; then - echo "SB DB Raft leader is unknown to the cluster node." - exit 1 - fi - # set trim-on-compaction - /usr/bin/ovn-appctl -t /var/run/ovn/ovnsb_db.ctl --timeout=5 ovsdb-server/memory-trim-on-compaction on 2>/dev/null - env: - - name: OVN_LOG_LEVEL - value: info - - name: K8S_NODE_IP - valueFrom: - fieldRef: - fieldPath: status.hostIP - - name: K8S_POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: K8S_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - volumeMounts: - - mountPath: /etc/ovn - name: datadir - - mountPath: /var/run/ovn - name: datadir - - mountPath: /var/log/ovn - name: datadir - - mountPath: /env - name: env-overrides - - mountPath: /ovn-cert - name: ovn-cert - - mountPath: /ovn-ca - name: ovn-ca - ports: - - name: sb-db-port - containerPort: {{.OVN_SB_PORT}} - - name: sb-db-raft-port - containerPort: {{.OVN_SB_RAFT_PORT}} - resources: - requests: - cpu: 10m - memory: 130Mi - terminationMessagePolicy: FallbackToLogsOnError - - # ovnkube master: convert kubernetes objects in to nbdb logical network components - - name: ovnkube-master - image: "{{.OvnImage}}" - command: - - /bin/bash - - -c - - | - set -xe - if [[ -f "/env/_master" ]]; then - set -o allexport - source "/env/_master" - set +o allexport - fi - - # TLS for ovnkube-master metrics - TLS_PK=/etc/pki/tls/metrics-cert/tls.key - TLS_CERT=/etc/pki/tls/metrics-cert/tls.crt - - if [ "{{.OVN_GATEWAY_MODE}}" == "shared" ]; then - gateway_mode_flags="--gateway-mode shared --gateway-interface br-ex" - elif [ "{{.OVN_GATEWAY_MODE}}" == "local" ]; then - gateway_mode_flags="--gateway-mode local --gateway-interface br-ex" - else - echo "Invalid OVN_GATEWAY_MODE: \"{{.OVN_GATEWAY_MODE}}\". Must be \"local\" or \"shared\"." - exit 1 - fi - - retries=0 - while [ ! -f /var/run/secrets/hosted_cluster/token ]; do - (( retries += 1 )) - sleep 1 - if [[ "${retries}" -gt 30 ]]; then - echo "$(date -Iseconds) - Hosted cluster token not found" - exit 1 - fi - done - - multi_network_enabled_flag= - if [[ "{{.OVN_MULTI_NETWORK_ENABLE}}" == "true" ]]; then - multi_network_enabled_flag="--enable-multi-network" - fi - - multi_network_policy_enabled_flag= - if [[ "{{.OVN_MULTI_NETWORK_POLICY_ENABLE}}" == "true" ]]; then - multi_network_policy_enabled_flag="--enable-multi-networkpolicy" - fi - - echo "I$(date "+%m%d %H:%M:%S.%N") - ovnkube-master - start ovnkube --init-master ${K8S_NODE}" - exec /usr/bin/ovnkube \ - --init-master "${K8S_NODE}" \ - --config-file=/run/ovnkube-config/ovnkube.conf \ - --k8s-token-file=/var/run/secrets/hosted_cluster/token \ - --ovn-empty-lb-events \ - --loglevel "${OVN_KUBE_LOG_LEVEL}" \ - --metrics-bind-address "0.0.0.0:9102" \ - --metrics-enable-pprof \ - --metrics-enable-config-duration \ - ${gateway_mode_flags} \ - --sb-address "{{.OVN_SB_DB_LIST}}" \ - --sb-client-privkey /ovn-cert/tls.key \ - --sb-client-cert /ovn-cert/tls.crt \ - --sb-client-cacert /ovn-ca/ca-bundle.crt \ - --sb-cert-common-name "{{.OVN_CERT_CN}}" \ - --nb-address "{{.OVN_NB_DB_LIST}}" \ - --nb-client-privkey /ovn-cert/tls.key \ - --nb-client-cert /ovn-cert/tls.crt \ - --nb-client-cacert /ovn-ca/ca-bundle.crt \ - --nb-cert-common-name "{{.OVN_CERT_CN}}" \ - --enable-multicast \ - --disable-snat-multiple-gws \ - --node-server-privkey ${TLS_PK} \ - --node-server-cert ${TLS_CERT} \ - ${multi_network_enabled_flag} \ - ${multi_network_policy_enabled_flag} \ - --acl-logging-rate-limit "{{.OVNPolicyAuditRateLimit}}" - volumeMounts: - - mountPath: /etc/ovn - name: datadir - - mountPath: /var/run/ovn - name: datadir - - mountPath: /run/ovnkube-config/ - name: ovnkube-config - - mountPath: /env - name: env-overrides - - mountPath: /ovn-cert - name: ovn-cert - - mountPath: /ovn-ca - name: ovn-ca - - mountPath: /hosted-ca - name: hosted-ca-cert - - mountPath: /var/run/secrets/hosted_cluster - name: hosted-cluster-api-access - - name: ovn-master-metrics-cert - mountPath: /etc/pki/tls/metrics-cert - readOnly: True - resources: - requests: - cpu: 10m - memory: 200Mi - env: - - name: OVN_KUBE_LOG_LEVEL - value: "4" - - name: K8S_NODE - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: ALL_PROXY - value: socks5://127.0.0.1:8090 - - name: NO_PROXY - value: kube-apiserver - ports: - - name: metrics-port - containerPort: 29102 - terminationMessagePolicy: FallbackToLogsOnError - # ovn-dbchecker: monitor clustered ovn databases for db health and stale raft members - - name: ovn-dbchecker - image: "{{.OvnImage}}" - command: - - /bin/bash - - -c - - | - set -xe - if [[ -f "/env/_master" ]]; then - set -o allexport - source "/env/_master" - set +o allexport - fi - - retries=0 - while [ ! -f /var/run/secrets/hosted_cluster/token ]; do - (( retries += 1 )) - sleep 1 - if [[ "${retries}" -gt 30 ]]; then - echo "$(date -Iseconds) - Hosted cluster token not found" - exit 1 - fi - done - - echo "I$(date "+%m%d %H:%M:%S.%N") - ovn-dbchecker - start ovn-dbchecker" - - # RAFT clusters need an odd number of members to achieve consensus. - # The CNO determines which members make up the cluster, so if this container - # is not supposed to be part of the cluster, wait forever doing nothing - # (instad of exiting and causing CrashLoopBackoffs for no reason). - pod_dns_name="${K8S_POD_NAME}.ovnkube-master-internal.{{.HostedClusterNamespace}}.svc.cluster.local" - if [[ ! "{{.OVN_NB_DB_LIST}}" =~ .*":${pod_dns_name}:".* ]] && [[ ! "{{.OVN_NB_DB_LIST}}" =~ .*":[${pod_dns_name}]:".* ]]; then - echo "$(date -Iseconds) - not selected as RAFT member; sleeping..." - sleep 1500d - exit 0 - fi - - exec /usr/bin/ovndbchecker \ - --config-file=/run/ovnkube-config/ovnkube.conf \ - --k8s-token-file=/var/run/secrets/hosted_cluster/token \ - --loglevel "${OVN_KUBE_LOG_LEVEL}" \ - --sb-address "{{.OVN_SB_DB_LIST}}" \ - --sb-client-privkey /ovn-cert/tls.key \ - --sb-client-cert /ovn-cert/tls.crt \ - --sb-client-cacert /ovn-ca/ca-bundle.crt \ - --sb-cert-common-name "{{.OVN_CERT_CN}}" \ - --sb-raft-election-timer "{{.OVN_SB_RAFT_ELECTION_TIMER}}" \ - --nb-address "{{.OVN_NB_DB_LIST}}" \ - --nb-client-privkey /ovn-cert/tls.key \ - --nb-client-cert /ovn-cert/tls.crt \ - --nb-client-cacert /ovn-ca/ca-bundle.crt \ - --nb-cert-common-name "{{.OVN_CERT_CN}}" \ - --nb-raft-election-timer "{{.OVN_NB_RAFT_ELECTION_TIMER}}" - volumeMounts: - - mountPath: /etc/ovn - name: datadir - - mountPath: /var/run/ovn - name: datadir - - mountPath: /run/ovnkube-config/ - name: ovnkube-config - - mountPath: /env - name: env-overrides - - mountPath: /ovn-cert - name: ovn-cert - - mountPath: /ovn-ca - name: ovn-ca - - mountPath: /hosted-ca - name: hosted-ca-cert - - mountPath: /var/run/secrets/hosted_cluster - name: hosted-cluster-api-access - resources: - requests: - cpu: 10m - memory: 50Mi - env: - - name: OVN_KUBE_LOG_LEVEL - value: "4" - - name: K8S_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - terminationMessagePolicy: FallbackToLogsOnError - - name: socks-proxy - image: "{{.Socks5ProxyImage}}" - command: - - /usr/bin/control-plane-operator - - konnectivity-socks5-proxy - args: - - "run" - volumeMounts: - - mountPath: /etc/kubernetes/ - name: admin-kubeconfig - - mountPath: /etc/konnectivity/proxy-client - name: konnectivity-proxy-cert - readOnly: true - - mountPath: /etc/konnectivity/proxy-ca - name: konnectivity-proxy-ca - readOnly: true - resources: - requests: - cpu: 10m - memory: 10Mi - env: - - name: KUBECONFIG - value: "/etc/kubernetes/kubeconfig" - terminationMessagePolicy: FallbackToLogsOnError - {{ if .HCPNodeSelector }} - nodeSelector: - {{ range $key, $value := .HCPNodeSelector }} - "{{$key}}": "{{$value}}" - {{ end }} - {{ end }} - volumes: - - name: ovnkube-config - configMap: - name: ovnkube-config - - name: konnectivity-proxy-ca - configMap: - name: konnectivity-ca-bundle - - name: konnectivity-proxy-cert - secret: - defaultMode: 0640 - secretName: konnectivity-client - - name: env-overrides - configMap: - name: env-overrides - optional: true - - name: ovn-ca - configMap: - name: ovn-ca - - name: ovn-cert - secret: - secretName: ovn-cert - - name: ovn-master-metrics-cert - secret: - secretName: ovn-master-metrics-cert - - name: admin-kubeconfig - secret: - secretName: service-network-admin-kubeconfig - - name: hosted-cluster-api-access - emptyDir: {} - - name: hosted-ca-cert - secret: - secretName: root-ca - items: - - key: ca.crt - path: ca.crt - tolerations: - - key: "hypershift.openshift.io/control-plane" - operator: "Equal" - value: "true" - effect: "NoSchedule" - - key: "hypershift.openshift.io/cluster" - operator: "Equal" - value: {{.HostedClusterNamespace}} - effect: "NoSchedule"