OVN: use config file via ConfigMap rather than environment variables

bindata/network/ovn-kubernetes/004-config.yaml

@@ -1,11 +1,24 @@
 ---
-# The network cidr and service cidr are set in the ovn-config configmap
+# The ovnconfig config file. Used by both node and master processes.
 kind: ConfigMap
 apiVersion: v1
 metadata:
-  name: ovn-config
+  name: ovnkube-config
   namespace: openshift-ovn-kubernetes
 data:
-  net_cidr:   {{.OVN_cidr}}
-  svc_cidr:   {{.OVN_service_cidr}}
-  k8s_apiserver: "{{.K8S_APISERVER}}"
+  ovnkube.conf:   |-
+    [default]
+    mtu="{{.MTU}}"
+    cluster-subnets="{{.OVN_cidr}}"
+
+    [kubernetes]
+    service-cidr="{{.OVN_service_cidr}}"
+    ovn-config-namespace="openshift-ovn-kubernetes"
+    apiserver="{{.K8S_APISERVER}}"
+
+    [logging]
+    logfile="/dev/stdout"
+
+    [gateway]
+    mode=local
+    nodeport=true

bindata/network/ovn-kubernetes/ovnkube-master.yaml

@@ -40,7 +40,7 @@ spec:
       containers:
       # ovn-northd: convert network objects in nbdb to flows in sbdb
       - name: northd
-        image: {{.OvnImage}}
+        image: "{{.OvnImage}}"
         command:
         - /bin/bash
         - -c
@@ -66,6 +66,12 @@ spec:
             nb_addr_list="${nb_addr_list}ssl:${host}:{{.OVN_NB_PORT}}"
             sb_addr_list="${sb_addr_list}ssl:${host}:{{.OVN_SB_PORT}}"
           done
+          echo /ovn-cert/tls.key
+          cat /ovn-cert/tls.key
+          echo /ovn-cert/tls.crt
+          cat /ovn-cert/tls.crt
+          echo /ovn-ca/ca-bundle.crt
+          cat /ovn-ca/ca-bundle.crt
 
           exec ovn-northd \
             --no-chdir "-vconsole:${OVN_LOG_LEVEL}" -vfile:off \
@@ -98,7 +104,7 @@ spec:
 
       # nbdb: the northbound, or logical network object DB. In raft mode 
       - name: nbdb
-        image: {{.OvnImage}}
+        image: "{{.OvnImage}}"
         command:
         - /bin/bash
         - -c
@@ -113,6 +119,12 @@ spec:
           OVN_NODES_ARRAY=({{.OVN_NODES}})
           MASTER_NODE=$(getent ahostsv4 "${OVN_NODES_ARRAY[0]}" | grep RAW | awk '{print $1}')
           LOCALHOST=$(getent ahostsv4 "${K8S_NODE}" | grep RAW | awk '{print $1}')
+          echo /ovn-cert/tls.key
+          cat /ovn-cert/tls.key
+          echo /ovn-cert/tls.crt
+          cat /ovn-cert/tls.crt
+          echo /ovn-ca/ca-bundle.crt
+          cat /ovn-ca/ca-bundle.crt
 
           if [[ "$LOCALHOST" == "$MASTER_NODE" ]]; then
             exec /usr/share/openvswitch/scripts/ovn-ctl \
@@ -191,7 +203,7 @@ spec:
 
       # sbdb: The southbound, or flow DB. In raft mode 
       - name: sbdb
-        image: {{.OvnImage}}
+        image: "{{.OvnImage}}"
         command:
         - /bin/bash
         - -c
@@ -206,6 +218,12 @@ spec:
           OVN_NODES_ARRAY=({{.OVN_NODES}})
           MASTER_NODE=$(getent ahostsv4 "${OVN_NODES_ARRAY[0]}" | grep RAW | awk '{print $1}')
           LOCALHOST=$(getent ahostsv4 "${K8S_NODE}" | grep RAW | awk '{print $1}')
+          echo /ovn-cert/tls.key
+          cat /ovn-cert/tls.key
+          echo /ovn-cert/tls.crt
+          cat /ovn-cert/tls.crt
+          echo /ovn-ca/ca-bundle.crt
+          cat /ovn-ca/ca-bundle.crt
 
           if [[ "$LOCALHOST" == "$MASTER_NODE" ]]; then
             exec /usr/share/openvswitch/scripts/ovn-ctl \
@@ -281,7 +299,7 @@ spec:
 
       # ovnkube master: convert kubernetes objects in to nbdb logical network components
       - name: ovnkube-master
-        image: {{.OvnImage}}
+        image: "{{.OvnImage}}"
         command:
         - /bin/bash
         - -c
@@ -313,6 +331,12 @@ spec:
             nb_addr_list="${nb_addr_list}ssl:${host}:{{.OVN_NB_PORT}}"
             sb_addr_list="${sb_addr_list}ssl://${host}:{{.OVN_SB_PORT}}"
           done
+          echo /ovn-cert/tls.key
+          cat /ovn-cert/tls.key
+          echo /ovn-cert/tls.crt
+          cat /ovn-cert/tls.crt
+          echo /ovn-ca/ca-bundle.crt
+          cat /ovn-ca/ca-bundle.crt
 
           # start nbctl daemon for caching
           export OVN_NB_DAEMON=$(ovn-nbctl --pidfile=/run/openvswitch/ovnk-nbctl.pid \
@@ -322,15 +346,10 @@ spec:
 
           exec /usr/bin/ovnkube \
             --init-master "${K8S_NODE}" \
-            --cluster-subnets "${OVN_NET_CIDR}" \
-            --k8s-service-cidr "${OVN_SVC_CIDR}" \
-            --k8s-apiserver "{{.K8S_APISERVER}}" \
-            --ovn-config-namespace openshift-ovn-kubernetes \
-            --nodeport \
+            --config-file=/run/ovnkube-config/ovnkube.conf \
             --ovn-empty-lb-events \
             --loglevel "${OVN_KUBE_LOG_LEVEL}" \
             ${hybrid_overlay_flags} \
-            --logfile /dev/stdout \
             --metrics-bind-address "0.0.0.0:9102" \
             --sb-address "${sb_addr_list}" \
             --sb-client-privkey /ovn-cert/tls.key \
@@ -347,6 +366,8 @@ spec:
           name: var-lib-openvswitch
         - mountPath: /run/openvswitch/
           name: run-openvswitch
+        - mountPath: /run/ovnkube-config/
+          name: ovnkube-config
         - mountPath: /env
           name: env-overrides
         - mountPath: /ovn-cert
@@ -359,17 +380,7 @@ spec:
             memory: 300Mi
         env:
         - name: OVN_KUBE_LOG_LEVEL
-          value: "4"
-        - name: OVN_NET_CIDR
-          valueFrom:
-            configMapKeyRef:
-              name: ovn-config
-              key: net_cidr
-        - name: OVN_SVC_CIDR
-          valueFrom:
-            configMapKeyRef:
-              name: ovn-config
-              key: svc_cidr
+          value: "{{.OvnkubeLogLevel}}"
         - name: K8S_NODE
           valueFrom:
             fieldRef:
@@ -394,6 +405,9 @@ spec:
           path: /var/lib/ovn/data
       - name: run-openvswitch
         emptyDir: {}
+      - name: ovnkube-config
+        configMap:
+          name: ovnkube-config
       - name: env-overrides
         configMap:
           name: env-overrides

bindata/network/ovn-kubernetes/ovnkube-node.yaml

@@ -35,7 +35,7 @@ spec:
       containers:
       # ovsdb and ovs-vswitchd
       - name: ovs-daemons
-        image: {{.OvnImage}}
+        image: "{{.OvnImage}}"
         command:
         - /bin/bash
         - -c
@@ -117,7 +117,7 @@ spec:
 
       # ovn-controller: programs the vswitch with flows from the sbdb
       - name: ovn-controller
-        image: {{.OvnImage}}
+        image: "{{.OvnImage}}"
         command:
         - /bin/bash
         - -c
@@ -128,6 +128,12 @@ spec:
             source "/env/${K8S_NODE}"
             set +o allexport
           fi
+          echo /ovn-cert/tls.key
+          cat /ovn-cert/tls.key
+          echo /ovn-cert/tls.crt
+          cat /ovn-cert/tls.crt
+          echo /ovn-ca/ca-bundle.crt
+          cat /ovn-ca/ca-bundle.crt
           exec ovn-controller unix:/var/run/openvswitch/db.sock -vfile:off \
             --no-chdir --pidfile=/var/run/openvswitch/ovn-controller.pid \
             -p /ovn-cert/tls.key -c /ovn-cert/tls.crt -C /ovn-ca/ca-bundle.crt \
@@ -162,7 +168,7 @@ spec:
 
       # ovnkube-node: does node-level bookkeeping and configuration
       - name: ovnkube-node
-        image: {{.OvnImage}}
+        image: "{{.OvnImage}}"
         command:
         - /bin/bash
         - -c
@@ -210,6 +216,12 @@ spec:
             nb_addr_list="${nb_addr_list}ssl://${host}:{{.OVN_NB_PORT}}"
             sb_addr_list="${sb_addr_list}ssl://${host}:{{.OVN_SB_PORT}}"
           done
+          echo /ovn-cert/tls.key
+          cat /ovn-cert/tls.key
+          echo /ovn-cert/tls.crt
+          cat /ovn-cert/tls.crt
+          echo /ovn-ca/ca-bundle.crt
+          cat /ovn-ca/ca-bundle.crt
 
           exec /usr/bin/ovnkube --init-node "${K8S_NODE}" \
             --cluster-subnets "${OVN_NET_CIDR}" \
@@ -224,14 +236,12 @@ spec:
             --sb-client-privkey /ovn-cert/tls.key \
             --sb-client-cert /ovn-cert/tls.crt \
             --sb-client-cacert /ovn-ca/ca-bundle.crt \
-            --nodeport --gateway-mode local \
+            --config-file=/run/ovnkube-config/ovnkube.conf \
+            --loglevel "${OVN_KUBE_LOG_LEVEL}" \
             ${hybrid_overlay_flags} \
             --pidfile /var/run/openvswitch/ovnkube-node.pid \
-            --loglevel "${OVN_KUBE_LOG_LEVEL}" --logfile /dev/stdout \
             --metrics-bind-address "0.0.0.0:9101"
         env:
-        - name: OVN_KUBE_LOG_LEVEL
-          value: "4"
         - name: OVN_HYBRID_OVERLAY_ENABLE
           value: "{{ .OVNHybridOverlayEnable }}"
         - name: OVN_HYBRID_OVERLAY_NET_CIDR
@@ -241,16 +251,8 @@ spec:
           value: "{{.KUBERNETES_SERVICE_PORT}}"
         - name: KUBERNETES_SERVICE_HOST
           value: "{{.KUBERNETES_SERVICE_HOST}}"
-        - name: OVN_NET_CIDR
-          valueFrom:
-            configMapKeyRef:
-              name: ovn-config
-              key: net_cidr
-        - name: OVN_SVC_CIDR
-          valueFrom:
-            configMapKeyRef:
-              name: ovn-config
-              key: svc_cidr
+        - name: OVN_KUBE_LOG_LEVEL
+          value: "{{.OvnkubeLogLevel}}"
         - name: K8S_NODE
           valueFrom:
             fieldRef:
@@ -287,6 +289,8 @@ spec:
           name: etc-openvswitch
         - mountPath: /var/lib/openvswitch
           name: var-lib-openvswitch
+        - mountPath: /run/ovnkube-config/
+          name: ovnkube-config
         - mountPath: /env
           name: env-overrides
         - mountPath: /ovn-cert
@@ -339,13 +343,16 @@ spec:
           path: /sys
       - name: host-cni-bin
         hostPath:
-          path: {{.CNIBinDir}}
+          path: "{{.CNIBinDir}}"
       - name: host-cni-netd
         hostPath:
-          path: {{.CNIConfDir}}
+          path: "{{.CNIConfDir}}"
       - name: host-var-lib-cni-networks-ovn-kubernetes
         hostPath:
           path: /var/lib/cni/networks/ovn-k8s-cni-overlay
+      - name: ovnkube-config
+        configMap:
+          name: ovnkube-config
       - name: env-overrides
         configMap:
           name: env-overrides

pkg/network/ovn_kubernetes.go

@@ -71,6 +71,9 @@ func renderOVNKubernetes(conf *operv1.NetworkSpec, bootstrapResult *bootstrap.Bo
 	}
 	data.Data["OVN_service_cidr"] = svcpools
 
+	// ovnkube config file
+	data.Data["OvnkubeLogLevel"] = 4
+
 	if c.HybridOverlayConfig != nil {
 		data.Data["OVNHybridOverlayNetCIDR"] = c.HybridOverlayConfig.HybridClusterNetwork[0].CIDR
 		data.Data["OVNHybridOverlayEnable"] = "true"
@@ -161,6 +164,10 @@ func boostrapOVN(kubeClient client.Client) (*bootstrap.BootstrapResult, error) {
 		return nil, fmt.Errorf("unable to bootstrap OVN, no master nodes found")
 	}
 
+	if len(masterNodeList.Items) < 3 {
+		return nil, fmt.Errorf("PHIL Need at least 3 nodes, have %d", len(masterNodeList.Items))
+	}
+
 	ovnMasterNodes := []string{}
 	for _, masterNode := range masterNodeList.Items {
 		ovnMasterNodes = append(ovnMasterNodes, masterNode.Name)