diff --git a/README.md b/README.md index fa1b51e0d4..71e4282942 100644 --- a/README.md +++ b/README.md @@ -132,8 +132,9 @@ spec: ``` ### Configuring OVNKubernetes -OVNKubernetes supports the following configuration options, all of which are optional: +OVNKubernetes supports the following configuration options, all of which are optional and once set at cluster creation, they can't be changed: * `MTU`: The MTU to use for the geneve overlay. The default is the MTU of the node that the cluster-network-operator is first run on, minus 100 bytes for geneve overhead. If the nodes in your cluster don't all have the same MTU then you may need to set this explicitly. +* `genevePort`: The UDP port to use for the Geneve overlay. The default is 6081. These configuration flags are only in the Operator configuration object. @@ -144,6 +145,7 @@ spec: type: OVNKubernetes ovnKubernetesConfig: mtu: 1400 + genevePort: 6081 ``` Additionally, you can configure per-node verbosity for ovn-kubernetes. This is useful @@ -366,7 +368,7 @@ spec: The operator is expected to run as a pod (via a Deployment) inside a kubernetes cluster. It will retrieve the configuration above and reconcile the desired configuration. A suitable manifest for running the operator is located in `manifests/`. ## Unsafe changes -Most network changes are unsafe to roll out to a production cluster. Therefore, the network operator will stop reconciling if it detects that an unsafe change has been requested. +Most network changes are unsafe to roll out to a production cluster. Therefore, the network operator will stop reconciling if it detects that an unsafe change has been requested. ### Safe changes to apply: It is safe to edit the following fields in the Operator configuration: diff --git a/bindata/network/ovn-kubernetes/004-config.yaml b/bindata/network/ovn-kubernetes/004-config.yaml index f59a1ef6e0..30d7a12faa 100644 --- a/bindata/network/ovn-kubernetes/004-config.yaml +++ b/bindata/network/ovn-kubernetes/004-config.yaml @@ -1,11 +1,25 @@ --- -# The network cidr and service cidr are set in the ovn-config configmap +# The ovnconfig config file. Used by both node and master processes. kind: ConfigMap apiVersion: v1 metadata: - name: ovn-config + name: ovnkube-config namespace: openshift-ovn-kubernetes data: - net_cidr: {{.OVN_cidr}} - svc_cidr: {{.OVN_service_cidr}} - k8s_apiserver: "{{.K8S_APISERVER}}" + ovnkube.conf: |- + [default] + mtu="{{.MTU}}" + cluster-subnets="{{.OVN_cidr}}" + encap-port="{{.GenevePort}}" + + [kubernetes] + service-cidr="{{.OVN_service_cidr}}" + ovn-config-namespace="openshift-ovn-kubernetes" + apiserver="{{.K8S_APISERVER}}" + + [logging] + logfile="/dev/stdout" + + [gateway] + mode=local + nodeport=true diff --git a/bindata/network/ovn-kubernetes/005-service.yaml b/bindata/network/ovn-kubernetes/005-service.yaml index fcf3426ad1..0899952f69 100644 --- a/bindata/network/ovn-kubernetes/005-service.yaml +++ b/bindata/network/ovn-kubernetes/005-service.yaml @@ -10,7 +10,7 @@ metadata: namespace: openshift-ovn-kubernetes spec: selector: - name: ovnkube-master + app: ovnkube-master ports: - name: north port: {{.OVN_NB_PORT}} diff --git a/bindata/network/ovn-kubernetes/alert-rules-control-plane.yaml b/bindata/network/ovn-kubernetes/alert-rules-control-plane.yaml index 28180f7981..e32d6f6f37 100644 --- a/bindata/network/ovn-kubernetes/alert-rules-control-plane.yaml +++ b/bindata/network/ovn-kubernetes/alert-rules-control-plane.yaml @@ -17,7 +17,7 @@ spec: message: | there is no running ovn-kubernetes master expr: | - absent(up{job="ovn-kubernetes-master",namespace="openshift-ovn-kubernetes"} ==1) + absent(up{job="ovnkube-master",namespace="openshift-ovn-kubernetes"} ==1) for: 10m labels: severity: warning diff --git a/bindata/network/ovn-kubernetes/monitor.yaml b/bindata/network/ovn-kubernetes/monitor.yaml index e8b1461cf2..711ee88880 100644 --- a/bindata/network/ovn-kubernetes/monitor.yaml +++ b/bindata/network/ovn-kubernetes/monitor.yaml @@ -3,7 +3,7 @@ apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor metadata: labels: - name: ovnkube-master + app: ovnkube-master annotations: networkoperator.openshift.io/ignore-errors: "" name: monitor-ovn-master @@ -18,19 +18,20 @@ spec: - openshift-ovn-kubernetes selector: matchLabels: - name: ovnkube-master + app: ovnkube-master --- apiVersion: v1 kind: Service metadata: labels: - name: ovnkube-master + app: ovnkube-master name: ovn-kubernetes-master namespace: openshift-ovn-kubernetes spec: selector: - name: ovnkube-master + app: ovnkube-master clusterIP: None + publishNotReadyAddresses: true ports: - name: metrics port: 9102 @@ -72,6 +73,7 @@ spec: selector: app: ovnkube-node clusterIP: None + publishNotReadyAddresses: true ports: - name: metrics port: 9101 diff --git a/bindata/network/ovn-kubernetes/ovnkube-master.yaml b/bindata/network/ovn-kubernetes/ovnkube-master.yaml index d9196a7758..58118a3481 100644 --- a/bindata/network/ovn-kubernetes/ovnkube-master.yaml +++ b/bindata/network/ovn-kubernetes/ovnkube-master.yaml @@ -1,3 +1,19 @@ +# The ovnkube control-plane components + +# The pod disruption budget ensures that we keep a raft quorum +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: ovn-raft-quorum-guard + namespace: openshift-ovn-kubernetes +spec: + minAvailable: {{.OVN_MIN_AVAILABLE}} + selector: + matchLabels: + app: ovnkube-master + +--- + kind: DaemonSet apiVersion: apps/v1 metadata: @@ -11,7 +27,7 @@ spec: replicas: 1 selector: matchLabels: - name: ovnkube-master + app: ovnkube-master strategy: type: RollingUpdate rollingUpdate: @@ -22,7 +38,7 @@ spec: template: metadata: labels: - name: ovnkube-master + app: ovnkube-master component: network type: infra openshift.io/component: network @@ -35,12 +51,12 @@ spec: # (container) -> (host) # /etc/openvswitch -> /var/lib/ovn/etc - ovsdb data # /var/lib/openvswitch -> /var/lib/ovn/data - ovsdb pki state - # /run/openvswitch -> tmpfs - sockets & pids + # /run/openvswitch -> tmpfs - sockets # /env -> configmap env-overrides - debug overrides containers: # ovn-northd: convert network objects in nbdb to flows in sbdb - name: northd - image: {{.OvnImage}} + image: "{{.OvnImage}}" command: - /bin/bash - -c @@ -54,7 +70,6 @@ spec: exec ovn-northd \ --no-chdir "-vconsole:${OVN_LOG_LEVEL}" -vfile:off \ - --pidfile=/var/run/openvswitch/ovn-northd.pid \ --ovnnb-db "{{.OVN_NB_DB_LIST}}" \ --ovnsb-db "{{.OVN_SB_DB_LIST}}" \ -p /ovn-cert/tls.key \ @@ -70,6 +85,8 @@ spec: name: var-lib-openvswitch - mountPath: /run/openvswitch/ name: run-openvswitch + - mountPath: /run/ovn/ + name: run-ovn - mountPath: /env name: env-overrides - mountPath: /ovn-cert # not needed, but useful when exec'ing in to pod. @@ -80,10 +97,11 @@ spec: requests: cpu: 100m memory: 300Mi + terminationMessagePolicy: FallbackToLogsOnError # nbdb: the northbound, or logical network object DB. In raft mode - name: nbdb - image: {{.OvnImage}} + image: "{{.OvnImage}}" command: - /bin/bash - -c @@ -95,11 +113,20 @@ spec: set +o allexport fi + # Determine the ovn rundir. + if [[ -f /usr/bin/ovn-appctl ]] ; then + # ovn-appctl is present. Use new ovn run dir path. + OVNCTL_PATH=/usr/share/ovn/scripts/ovn-ctl + else + # ovn-appctl is not present. Use openvswitch run dir path. + OVNCTL_PATH=/usr/share/openvswitch/scripts/ovn-ctl + fi + bracketify() { case "$1" in *:*) echo "[$1]" ;; *) echo "$1" ;; esac } MASTER_IP="{{.OVN_MASTER_IP}}" if [[ "${K8S_NODE_IP}" == "${MASTER_IP}" ]]; then - exec /usr/share/openvswitch/scripts/ovn-ctl \ + exec ${OVNCTL_PATH} \ --db-nb-cluster-local-port={{.OVN_NB_RAFT_PORT}} \ --db-nb-cluster-local-addr=$(bracketify ${K8S_NODE_IP}) \ --no-monitor \ @@ -110,7 +137,7 @@ spec: --ovn-nb-log="-vconsole:${OVN_LOG_LEVEL} -vfile:off" \ run_nb_ovsdb else - exec /usr/share/openvswitch/scripts/ovn-ctl \ + exec ${OVNCTL_PATH} \ --db-nb-cluster-local-port={{.OVN_NB_RAFT_PORT}} \ --db-nb-cluster-remote-port={{.OVN_NB_RAFT_PORT}} \ --db-nb-cluster-local-addr=$(bracketify ${K8S_NODE_IP}) \ @@ -143,6 +170,25 @@ spec: sleep 2 done fi + readinessProbe: + initialDelaySeconds: 30 + exec: + command: + - /bin/bash + - -c + - | + set -xe + # Determine the ovn rundir. + if [[ -f /usr/bin/ovn-appctl ]] ; then + # ovn-appctl is present. Use new ovn run dir path. + DB_SOCK_PATH=/var/run/ovn/ovnnb_db.ctl + APPCTL_PATH=/usr/bin/ovn-appctl + else + # ovn-appctl is not present. Use openvswitch run dir path. + DB_SOCK_PATH=/var/run/openvswitch/ovnnb_db.ctl + APPCTL_PATH=/usr/bin/ovs-appctl + fi + exec ${APPCTL_PATH} -t ${DB_SOCK_PATH} cluster/status OVN_Northbound 2>/dev/null | grep ${K8S_NODE_IP} | grep -v Address -q env: - name: OVN_LOG_LEVEL value: info @@ -153,10 +199,14 @@ spec: volumeMounts: - mountPath: /etc/openvswitch/ name: etc-openvswitch + - mountPath: /etc/ovn/ + name: etc-openvswitch - mountPath: /var/lib/openvswitch/ name: var-lib-openvswitch - mountPath: /run/openvswitch/ name: run-openvswitch + - mountPath: /run/ovn/ + name: run-ovn - mountPath: /env name: env-overrides - mountPath: /ovn-cert @@ -172,10 +222,11 @@ spec: containerPort: {{.OVN_NB_PORT}} - name: nb-db-raft-port containerPort: {{.OVN_NB_RAFT_PORT}} + terminationMessagePolicy: FallbackToLogsOnError # sbdb: The southbound, or flow DB. In raft mode - name: sbdb - image: {{.OvnImage}} + image: "{{.OvnImage}}" command: - /bin/bash - -c @@ -187,11 +238,20 @@ spec: set +o allexport fi + # Determine the ovn rundir. + if [[ -f /usr/bin/ovn-appctl ]] ; then + # ovn-appctl is present. Use new ovn run dir path. + OVNCTL_PATH=/usr/share/ovn/scripts/ovn-ctl + else + # ovn-appctl is not present. Use openvswitch run dir path. + OVNCTL_PATH=/usr/share/openvswitch/scripts/ovn-ctl + fi + bracketify() { case "$1" in *:*) echo "[$1]" ;; *) echo "$1" ;; esac } MASTER_IP="{{.OVN_MASTER_IP}}" if [[ "${K8S_NODE_IP}" == "${MASTER_IP}" ]]; then - exec /usr/share/openvswitch/scripts/ovn-ctl \ + exec ${OVNCTL_PATH} \ --db-sb-cluster-local-port={{.OVN_SB_RAFT_PORT}} \ --db-sb-cluster-local-addr=$(bracketify ${K8S_NODE_IP}) \ --no-monitor \ @@ -203,7 +263,7 @@ spec: run_sb_ovsdb else echo "joining cluster at ${MASTER_IP}" - exec /usr/share/openvswitch/scripts/ovn-ctl \ + exec ${OVNCTL_PATH} \ --db-sb-cluster-local-port={{.OVN_SB_RAFT_PORT}} \ --db-sb-cluster-remote-port={{.OVN_SB_RAFT_PORT}} \ --db-sb-cluster-local-addr=$(bracketify ${K8S_NODE_IP}) \ @@ -236,6 +296,25 @@ spec: sleep 2 done fi + readinessProbe: + initialDelaySeconds: 30 + exec: + command: + - /bin/bash + - -c + - | + set -xe + # Determine the ovn rundir. + if [[ -f /usr/bin/ovn-appctl ]] ; then + # ovn-appctl is present. Use new ovn run dir path. + DB_SOCK_PATH=/var/run/ovn/ovnsb_db.ctl + APPCTL_PATH=/usr/bin/ovn-appctl + else + # ovn-appctl is not present. Use openvswitch run dir path. + DB_SOCK_PATH=/var/run/openvswitch/ovnsb_db.ctl + APPCTL_PATH=/usr/bin/ovs-appctl + fi + exec ${APPCTL_PATH} -t ${DB_SOCK_PATH} cluster/status OVN_Southbound 2>/dev/null | grep ${K8S_NODE_IP} | grep -v Address -q env: - name: OVN_LOG_LEVEL value: info @@ -246,10 +325,14 @@ spec: volumeMounts: - mountPath: /etc/openvswitch/ name: etc-openvswitch + - mountPath: /etc/ovn/ + name: etc-openvswitch - mountPath: /var/lib/openvswitch/ name: var-lib-openvswitch - mountPath: /run/openvswitch/ name: run-openvswitch + - mountPath: /run/ovn/ + name: run-ovn - mountPath: /env name: env-overrides - mountPath: /ovn-cert @@ -261,10 +344,11 @@ spec: containerPort: {{.OVN_SB_PORT}} - name: sb-db-raft-port containerPort: {{.OVN_SB_RAFT_PORT}} + terminationMessagePolicy: FallbackToLogsOnError # ovnkube master: convert kubernetes objects in to nbdb logical network components - name: ovnkube-master - image: {{.OvnImage}} + image: "{{.OvnImage}}" command: - /bin/bash - -c @@ -285,22 +369,17 @@ spec: fi # start nbctl daemon for caching - export OVN_NB_DAEMON=$(ovn-nbctl --pidfile=/run/openvswitch/ovnk-nbctl.pid \ + export OVN_NB_DAEMON=$(ovn-nbctl --pidfile=/tmp/ovnk-nbctl.pid \ --detach \ -p /ovn-cert/tls.key -c /ovn-cert/tls.crt -C /ovn-ca/ca-bundle.crt \ --db "{{.OVN_NB_DB_LIST}}") exec /usr/bin/ovnkube \ --init-master "${K8S_NODE}" \ - --cluster-subnets "${OVN_NET_CIDR}" \ - --k8s-service-cidr "${OVN_SVC_CIDR}" \ - --k8s-apiserver "{{.K8S_APISERVER}}" \ - --ovn-config-namespace openshift-ovn-kubernetes \ - --nodeport \ + --config-file=/run/ovnkube-config/ovnkube.conf \ --ovn-empty-lb-events \ --loglevel "${OVN_KUBE_LOG_LEVEL}" \ ${hybrid_overlay_flags} \ - --logfile /dev/stdout \ --metrics-bind-address "0.0.0.0:9102" \ --sb-address "{{.OVN_SB_ADDR_LIST}}" \ --sb-client-privkey /ovn-cert/tls.key \ @@ -309,14 +388,20 @@ spec: lifecycle: preStop: exec: - command: ["/bin/bash", "-c", "kill $(cat /run/openvswitch/ovnk-nbctl.pid)"] + command: ["/bin/bash", "-c", "kill $(cat /tmp/ovnk-nbctl.pid) && unset OVN_NB_DAEMON"] volumeMounts: - mountPath: /etc/openvswitch/ name: etc-openvswitch + - mountPath: /etc/ovn/ + name: etc-openvswitch - mountPath: /var/lib/openvswitch/ name: var-lib-openvswitch - mountPath: /run/openvswitch/ name: run-openvswitch + - mountPath: /run/ovn/ + name: run-ovn + - mountPath: /run/ovnkube-config/ + name: ovnkube-config - mountPath: /env name: env-overrides - mountPath: /ovn-cert @@ -330,16 +415,6 @@ spec: env: - name: OVN_KUBE_LOG_LEVEL value: "4" - - name: OVN_NET_CIDR - valueFrom: - configMapKeyRef: - name: ovn-config - key: net_cidr - - name: OVN_SVC_CIDR - valueFrom: - configMapKeyRef: - name: ovn-config - key: svc_cidr - name: K8S_NODE valueFrom: fieldRef: @@ -347,6 +422,7 @@ spec: ports: - name: metrics-port containerPort: 9102 + terminationMessagePolicy: FallbackToLogsOnError nodeSelector: node-role.kubernetes.io/master: "" @@ -360,6 +436,12 @@ spec: path: /var/lib/ovn/data - name: run-openvswitch emptyDir: {} + - name: run-ovn + hostPath: + path: /var/run/ovn + - name: ovnkube-config + configMap: + name: ovnkube-config - name: env-overrides configMap: name: env-overrides diff --git a/bindata/network/ovn-kubernetes/ovnkube-node.yaml b/bindata/network/ovn-kubernetes/ovnkube-node.yaml index 2d8b03dcb3..4ccd24a348 100644 --- a/bindata/network/ovn-kubernetes/ovnkube-node.yaml +++ b/bindata/network/ovn-kubernetes/ovnkube-node.yaml @@ -35,7 +35,7 @@ spec: containers: # ovsdb and ovs-vswitchd - name: ovs-daemons - image: {{.OvnImage}} + image: "{{.OvnImage}}" command: - /bin/bash - -c @@ -53,7 +53,9 @@ spec: chown -R openvswitch:openvswitch /run/openvswitch chown -R openvswitch:openvswitch /etc/openvswitch function quit { - /usr/share/openvswitch/scripts/ovs-ctl stop + # Don't allow ovs-vswitchd to clear datapath flows on exit + kill -9 $(cat /var/run/openvswitch/ovs-vswitchd.pid 2>/dev/null) 2>/dev/null || true + kill $(cat /var/run/openvswitch/ovsdb-server.pid 2>/dev/null) 2>/dev/null || true exit 0 } trap quit SIGTERM @@ -109,15 +111,11 @@ spec: - status initialDelaySeconds: 15 periodSeconds: 5 - lifecycle: - preStop: - exec: - command: ["/usr/share/openvswitch/scripts/ovs-ctl", "stop"] terminationGracePeriodSeconds: 10 # ovn-controller: programs the vswitch with flows from the sbdb - name: ovn-controller - image: {{.OvnImage}} + image: "{{.OvnImage}}" command: - /bin/bash - -c @@ -128,8 +126,16 @@ spec: source "/env/${K8S_NODE}" set +o allexport fi + # Determine the ovn rundir. + if [[ -f /usr/bin/ovn-appctl ]] ; then + # ovn-appctl is present. Use new ovn run dir path. + OVNCTL_DIR=ovn + else + # ovn-appctl is not present. Use openvswitch run dir path. + OVNCTL_DIR=openvswitch + fi exec ovn-controller unix:/var/run/openvswitch/db.sock -vfile:off \ - --no-chdir --pidfile=/var/run/openvswitch/ovn-controller.pid \ + --no-chdir --pidfile=/var/run/${OVNCTL_DIR}/ovn-controller.pid \ -p /ovn-cert/tls.key -c /ovn-cert/tls.crt -C /ovn-ca/ca-bundle.crt \ -vconsole:"${OVN_LOG_LEVEL}" securityContext: @@ -144,8 +150,12 @@ spec: volumeMounts: - mountPath: /run/openvswitch name: run-openvswitch + - mountPath: /run/ovn/ + name: run-ovn - mountPath: /etc/openvswitch name: etc-openvswitch + - mountPath: /etc/ovn/ + name: etc-openvswitch - mountPath: /var/lib/openvswitch name: var-lib-openvswitch - mountPath: /env @@ -162,7 +172,7 @@ spec: # ovnkube-node: does node-level bookkeeping and configuration - name: ovnkube-node - image: {{.OvnImage}} + image: "{{.OvnImage}}" command: - /bin/bash - -c @@ -199,10 +209,6 @@ spec: fi exec /usr/bin/ovnkube --init-node "${K8S_NODE}" \ - --cluster-subnets "${OVN_NET_CIDR}" \ - --k8s-service-cidr "${OVN_SVC_CIDR}" \ - --k8s-apiserver "{{.K8S_APISERVER}}" \ - --ovn-config-namespace ${ovn_config_namespace} \ --nb-address "{{.OVN_NB_ADDR_LIST}}" \ --sb-address "{{.OVN_SB_ADDR_LIST}}" \ --nb-client-privkey /ovn-cert/tls.key \ @@ -211,10 +217,9 @@ spec: --sb-client-privkey /ovn-cert/tls.key \ --sb-client-cert /ovn-cert/tls.crt \ --sb-client-cacert /ovn-ca/ca-bundle.crt \ - --nodeport --gateway-mode local \ + --config-file=/run/ovnkube-config/ovnkube.conf \ + --loglevel "${OVN_KUBE_LOG_LEVEL}" \ ${hybrid_overlay_flags} \ - --pidfile /var/run/openvswitch/ovnkube-node.pid \ - --loglevel "${OVN_KUBE_LOG_LEVEL}" --logfile /dev/stdout \ --metrics-bind-address "0.0.0.0:9101" env: - name: OVN_KUBE_LOG_LEVEL @@ -224,16 +229,8 @@ spec: value: "{{.KUBERNETES_SERVICE_PORT}}" - name: KUBERNETES_SERVICE_HOST value: "{{.KUBERNETES_SERVICE_HOST}}" - - name: OVN_NET_CIDR - valueFrom: - configMapKeyRef: - name: ovn-config - key: net_cidr - - name: OVN_SVC_CIDR - valueFrom: - configMapKeyRef: - name: ovn-config - key: svc_cidr + - name: OVN_KUBE_LOG_LEVEL + value: "4" - name: K8S_NODE valueFrom: fieldRef: @@ -243,6 +240,7 @@ spec: containerPort: 9101 securityContext: privileged: true + terminationMessagePolicy: FallbackToLogsOnError volumeMounts: # for the iptables wrapper - mountPath: /host @@ -266,10 +264,16 @@ spec: name: host-var-lib-cni-networks-ovn-kubernetes - mountPath: /run/openvswitch name: run-openvswitch + - mountPath: /run/ovn/ + name: run-ovn - mountPath: /etc/openvswitch name: etc-openvswitch + - mountPath: /etc/ovn/ + name: etc-openvswitch - mountPath: /var/lib/openvswitch name: var-lib-openvswitch + - mountPath: /run/ovnkube-config/ + name: ovnkube-config - mountPath: /env name: env-overrides - mountPath: /ovn-cert @@ -309,7 +313,11 @@ spec: hostPath: path: /var/lib/openvswitch/etc - name: run-openvswitch - emptyDir: {} + hostPath: + path: /var/run/openvswitch + - name: run-ovn + hostPath: + path: /var/run/ovn # commit 0ac2cd changed the location of the ovs database, mount and check if a database already exists - name: old-openvswitch-database path: /etc/origin/openvswitch @@ -322,13 +330,16 @@ spec: path: /sys - name: host-cni-bin hostPath: - path: {{.CNIBinDir}} + path: "{{.CNIBinDir}}" - name: host-cni-netd hostPath: - path: {{.CNIConfDir}} + path: "{{.CNIConfDir}}" - name: host-var-lib-cni-networks-ovn-kubernetes hostPath: path: /var/lib/cni/networks/ovn-k8s-cni-overlay + - name: ovnkube-config + configMap: + name: ovnkube-config - name: env-overrides configMap: name: env-overrides diff --git a/pkg/network/ovn_kubernetes.go b/pkg/network/ovn_kubernetes.go index dcc9df1070..98e74fc482 100644 --- a/pkg/network/ovn_kubernetes.go +++ b/pkg/network/ovn_kubernetes.go @@ -45,6 +45,7 @@ func renderOVNKubernetes(conf *operv1.NetworkSpec, bootstrapResult *bootstrap.Bo data.Data["KUBERNETES_SERVICE_PORT"] = os.Getenv("KUBERNETES_SERVICE_PORT") data.Data["K8S_APISERVER"] = fmt.Sprintf("https://%s:%s", os.Getenv("KUBERNETES_SERVICE_HOST"), os.Getenv("KUBERNETES_SERVICE_PORT")) data.Data["MTU"] = c.MTU + data.Data["GenevePort"] = c.GenevePort data.Data["CNIConfDir"] = pluginCNIConfDir(conf) data.Data["CNIBinDir"] = CNIBinDir data.Data["OVN_NB_PORT"] = OVN_NB_PORT @@ -57,6 +58,7 @@ func renderOVNKubernetes(conf *operv1.NetworkSpec, bootstrapResult *bootstrap.Bo data.Data["OVN_SB_ADDR_LIST"] = addrList(bootstrapResult.OVN.MasterIPs, OVN_SB_PORT) data.Data["OVN_MASTER_IP"] = bootstrapResult.OVN.MasterIPs[0] data.Data["LISTEN_DUAL_STACK"] = listenDualStack(bootstrapResult.OVN.MasterIPs[0]) + data.Data["OVN_MIN_AVAILABLE"] = len(bootstrapResult.OVN.MasterIPs)/2 + 1 var ippools string for _, net := range conf.ClusterNetwork { @@ -110,6 +112,9 @@ func validateOVNKubernetes(conf *operv1.NetworkSpec) []error { if oc.MTU != nil && (*oc.MTU < 576 || *oc.MTU > 65536) { out = append(out, errors.Errorf("invalid MTU %d", *oc.MTU)) } + if oc.GenevePort != nil && (*oc.GenevePort < 1 || *oc.GenevePort > 65535) { + out = append(out, errors.Errorf("invalid GenevePort %d", *oc.GenevePort)) + } } return out @@ -125,11 +130,15 @@ func isOVNKubernetesChangeSafe(prev, next *operv1.NetworkSpec) []error { if !reflect.DeepEqual(pn.MTU, nn.MTU) { errs = append(errs, errors.Errorf("cannot change ovn-kubernetes MTU")) } + if !reflect.DeepEqual(pn.GenevePort, nn.GenevePort) { + errs = append(errs, errors.Errorf("cannot change ovn-kubernetes genevePort")) + } if pn.HybridOverlayConfig != nil { if !reflect.DeepEqual(pn.HybridOverlayConfig, nn.HybridOverlayConfig) { errs = append(errs, errors.Errorf("once set cannot change ovn-kubernetes Hybrid Overlay Config")) } } + return errs } @@ -139,16 +148,22 @@ func fillOVNKubernetesDefaults(conf, previous *operv1.NetworkSpec, hostMTU int) } sc := conf.DefaultNetwork.OVNKubernetesConfig - // MTU is currently the only field we pull from previous. - // If it's not supplied, we infer it from the node on which we're running. + // MTU is currently the only field we pull from previous. + // If MTU is not supplied, we infer it from the host on which CNO is running + // (which may not be a node in the cluster). // However, this can never change, so we always prefer previous. if sc.MTU == nil { var mtu uint32 = uint32(hostMTU) - 100 // 100 byte geneve header - if previous != nil && previous.DefaultNetwork.OVNKubernetesConfig != nil { + if previous != nil && previous.DefaultNetwork.OVNKubernetesConfig != nil && + previous.DefaultNetwork.OVNKubernetesConfig.MTU != nil { mtu = *previous.DefaultNetwork.OVNKubernetesConfig.MTU } sc.MTU = &mtu } + if sc.GenevePort == nil { + var geneve uint32 = uint32(6081) + sc.GenevePort = &geneve + } } func networkPluginName() string { diff --git a/pkg/network/ovn_kubernetes_test.go b/pkg/network/ovn_kubernetes_test.go index d975718f03..fccc3c07b2 100644 --- a/pkg/network/ovn_kubernetes_test.go +++ b/pkg/network/ovn_kubernetes_test.go @@ -13,6 +13,8 @@ import ( uns "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" ) +// vars +var g = uint32(8061) var OVNKubernetesConfig = operv1.Network{ Spec: operv1.NetworkSpec{ ServiceNetwork: []string{"172.30.0.0/16"}, @@ -27,8 +29,10 @@ var OVNKubernetesConfig = operv1.Network{ }, }, DefaultNetwork: operv1.DefaultNetworkDefinition{ - Type: operv1.NetworkTypeOVNKubernetes, - OVNKubernetesConfig: &operv1.OVNKubernetesConfig{}, + Type: operv1.NetworkTypeOVNKubernetes, + OVNKubernetesConfig: &operv1.OVNKubernetesConfig{ + GenevePort: &g, + }, }, }, } @@ -186,6 +190,7 @@ func TestFillOVNKubernetesDefaults(t *testing.T) { // vars m := uint32(8900) + p := uint32(6081) expected := operv1.NetworkSpec{ ServiceNetwork: []string{"172.30.0.0/16"}, @@ -202,7 +207,8 @@ func TestFillOVNKubernetesDefaults(t *testing.T) { DefaultNetwork: operv1.DefaultNetworkDefinition{ Type: operv1.NetworkTypeOVNKubernetes, OVNKubernetesConfig: &operv1.OVNKubernetesConfig{ - MTU: &m, + MTU: &m, + GenevePort: &p, }, }, } @@ -236,6 +242,11 @@ func TestValidateOVNKubernetes(t *testing.T) { ovnConfig.MTU = &mtu errExpect("invalid MTU 70000") + // set geneve port to insanity + geneve := uint32(70001) + ovnConfig.GenevePort = &geneve + errExpect("invalid GenevePort 70001") + config.ClusterNetwork = nil errExpect("ClusterNetworks cannot be empty") } @@ -254,8 +265,12 @@ func TestOVNKubernetesIsSafe(t *testing.T) { // change the mtu mtu := uint32(70000) next.DefaultNetwork.OVNKubernetesConfig.MTU = &mtu - errs = isOVNKubernetesChangeSafe(prev, next) - g.Expect(errs).To(HaveLen(1)) + // change the geneve port + geneve := uint32(34001) + next.DefaultNetwork.OVNKubernetesConfig.GenevePort = &geneve + errs = isOVNKubernetesChangeSafe(prev, next) + g.Expect(errs).To(HaveLen(2)) g.Expect(errs[0]).To(MatchError("cannot change ovn-kubernetes MTU")) + g.Expect(errs[1]).To(MatchError("cannot change ovn-kubernetes genevePort")) }