[wip][poc]Interconnect on hypershift

bindata/network/ovn-kubernetes/managed/ovnkube-cluster-manager.yaml

@@ -0,0 +1,281 @@
+# The ovnkube control-plane components
+
+{{ if not .IsSNO }}
+# The pod disruption budget ensures that we keep a raft quorum
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: ovn-raft-quorum-guard
+  namespace: {{.HostedClusterNamespace}}
+  annotations:
+    network.operator.openshift.io/cluster-name:  {{.ManagementClusterName}}
+spec:
+  minAvailable: {{.OVN_MIN_AVAILABLE}}
+  selector:
+    matchLabels:
+      app: ovnkube-cluster-manager
+---
+{{ end }}
+kind: StatefulSet
+apiVersion: apps/v1
+metadata:
+  name: ovnkube-cluster-manager
+  namespace: {{.HostedClusterNamespace}}
+  annotations:
+    network.operator.openshift.io/cluster-name:  {{.ManagementClusterName}}
+    kubernetes.io/description: |
+      This daemonset launches the ovn-kubernetes controller (master) networking components.
+    release.openshift.io/version: "{{.ReleaseVersion}}"
+  labels:
+    # used by PodAffinity to prefer co-locating pods that belong to the same hosted cluster.
+    hypershift.openshift.io/hosted-control-plane: {{.HostedClusterNamespace}}
+spec:
+  podManagementPolicy: Parallel
+  selector:
+    matchLabels:
+      app: ovnkube-cluster-manager
+  serviceName: ovnkube-cluster-manager-internal
+  volumeClaimTemplates:
+  replicas: {{.OvnkubeMasterReplicas}}
+  template:
+    metadata:
+      annotations:
+        target.workload.openshift.io/management: '{"effect": "PreferredDuringScheduling"}'
+      labels:
+        app: ovnkube-cluster-manager
+        ovn-db-pod: "true"
+        component: network
+        type: infra
+        openshift.io/component: network
+        hypershift.openshift.io/control-plane-component: ovnkube-cluster-manager
+        kubernetes.io/os: "linux"
+    spec:
+      affinity:
+        nodeAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 50
+            preference:
+              matchExpressions:
+                - key: hypershift.openshift.io/control-plane
+                  operator: In
+                  values:
+                    - "true"
+          - weight: 100
+            preference:
+              matchExpressions:
+                - key: hypershift.openshift.io/cluster
+                  operator: In
+                  values:
+                    - {{.HostedClusterNamespace}}
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+          - labelSelector:
+              matchLabels:
+                app: ovnkube-cluster-manager
+            topologyKey: topology.kubernetes.io/zone
+        podAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+            - weight: 100
+              podAffinityTerm:
+                labelSelector:
+                  matchLabels:
+                    hypershift.openshift.io/hosted-control-plane: {{.HostedClusterNamespace}}
+                topologyKey: kubernetes.io/hostname
+      priorityClassName: hypershift-api-critical
+      initContainers:
+      # Remove once https://github.com/kubernetes/kubernetes/issues/85966 is addressed
+      - name: init-ip
+        command:
+          - /bin/bash
+          - -c
+          - |
+            cat <<-EOF
+            Kubelet only sets a pod's Status.PodIP when all containers of the pod have started at least once (successfully or unsuccessfully)
+             or at least one of the initContainers finished.
+            Container start is blocked by postStart hooks. See https://github.com/kubernetes/kubernetes/issues/85966 for more details.
+            The NB and SB DB postStart hooks block until the DBs join the RAFT cluster or until a timeout is reached.
+            In a standalone cluster every pod is host networked and the DBs use host IP to communicate between the RAFT members.
+            In HyperShift OVN-Kubernetes master is run as a statefulset and the pods are not host networked, meaning we cannot rely on the podIP not changing.
+            To provide a stable network identity for each pod in the statefulset we use a headless service,
+             the downside of this approach is the DNS entry for the pod will only start to work after the pod has its Status.PodIP set.
+
+            Until https://github.com/kubernetes/kubernetes/issues/85966 is fixed use a no-op init container as a workaround.
+            This allows for pod-pod connectivity in postStart hooks the first time they run.
+            EOF
+        image: "{{.OvnImage}}"
+      containers:
+      # token-minter creates a token with the default service account path
+      # The token is read by ovn-k containers to authenticate against the hosted cluster api server
+      - name: token-minter
+        image: "{{.TokenMinterImage}}"
+        command: ["/usr/bin/control-plane-operator", "token-minter"]
+        args:
+        - --service-account-namespace=openshift-ovn-kubernetes
+        - --service-account-name=ovn-kubernetes-controller
+        - --token-audience={{.TokenAudience}}
+        - --token-file=/var/run/secrets/hosted_cluster/token
+        - --kubeconfig=/etc/kubernetes/kubeconfig
+        resources:
+          requests:
+            cpu: 10m
+            memory: 30Mi
+        volumeMounts:
+        - mountPath: /etc/kubernetes
+          name: admin-kubeconfig
+        - mountPath: /var/run/secrets/hosted_cluster
+          name: hosted-cluster-api-access
+      # ovnkube master: convert kubernetes objects in to nbdb logical network components
+      - name: ovnkube-cluster-manager
+        image: "{{.OvnImage}}"
+        command:
+        - /bin/bash
+        - -c
+        - |
+          set -xe
+          if [[ -f "/env/_master" ]]; then
+            set -o allexport
+            source "/env/_master"
+            set +o allexport
+          fi
+
+          # TLS for ovnkube-cluster-manager metrics
+          TLS_PK=/etc/pki/tls/metrics-cert/tls.key
+          TLS_CERT=/etc/pki/tls/metrics-cert/tls.crt
+
+          if [ "{{.OVN_GATEWAY_MODE}}" == "shared" ]; then
+            gateway_mode_flags="--gateway-mode shared --gateway-interface br-ex"
+          elif [ "{{.OVN_GATEWAY_MODE}}" == "local" ]; then
+            gateway_mode_flags="--gateway-mode local --gateway-interface br-ex"
+          else
+            echo "Invalid OVN_GATEWAY_MODE: \"{{.OVN_GATEWAY_MODE}}\". Must be \"local\" or \"shared\"."
+            exit 1
+          fi
+
+          retries=0
+          while [ ! -f /var/run/secrets/hosted_cluster/token ]; do
+            (( retries += 1 ))
+            sleep 1
+            if [[ "${retries}" -gt 30 ]]; then
+              echo "$(date -Iseconds) - Hosted cluster token not found"
+                exit 1
+            fi
+          done
+
+          multi_network_enabled_flag=
+          if [[ "{{.OVN_MULTI_NETWORK_ENABLE}}" == "true" ]]; then
+            multi_network_enabled_flag="--enable-multi-network"
+          fi
+
+          echo "I$(date "+%m%d %H:%M:%S.%N") - ovnkube-cluster-manager - start ovnkube --init-master ${K8S_NODE}"
+          exec /usr/bin/ovnkube \
+            --init-cluster-manager "${K8S_NODE}" \
+            --config-file=/run/ovnkube-config/ovnkube.conf \
+            --k8s-token-file=/var/run/secrets/hosted_cluster/token \
+            --ovn-empty-lb-events \
+            --loglevel "${OVN_KUBE_LOG_LEVEL}" \
+            #--metrics-bind-address "0.0.0.0:9102" \
+            --metrics-bind-address "127.0.0.1:29104" \
+            --metrics-enable-pprof \
+            --metrics-enable-config-duration \
+            ${gateway_mode_flags} \
+            --sb-address "{{.OVN_SB_DB_LIST}}" \
+            --sb-client-privkey /ovn-cert/tls.key \
+            --sb-client-cert /ovn-cert/tls.crt \
+            --sb-client-cacert /ovn-ca/ca-bundle.crt \
+            --sb-cert-common-name "{{.OVN_CERT_CN}}" \
+            --nb-address "{{.OVN_NB_DB_LIST}}" \
+            --nb-client-privkey /ovn-cert/tls.key \
+            --nb-client-cert /ovn-cert/tls.crt \
+            --nb-client-cacert /ovn-ca/ca-bundle.crt \
+            --nb-cert-common-name "{{.OVN_CERT_CN}}" \
+            --enable-multicast \
+            --disable-snat-multiple-gws \
+            --node-server-privkey ${TLS_PK} \
+            --node-server-cert ${TLS_CERT} \
+            ${multi_network_enabled_flag} \
+            --acl-logging-rate-limit "{{.OVNPolicyAuditRateLimit}}"
+        volumeMounts:
+        - mountPath: /run/ovnkube-config/
+          name: ovnkube-config
+        - mountPath: /env
+          name: env-overrides
+        - mountPath: /ovn-cert
+          name: ovn-cert
+        - mountPath: /ovn-ca
+          name: ovn-ca
+        - mountPath: /hosted-ca
+          name: hosted-ca-cert
+        - mountPath: /var/run/secrets/hosted_cluster
+          name: hosted-cluster-api-access
+        - name: ovn-master-metrics-cert
+          mountPath: /etc/pki/tls/metrics-cert
+          readOnly: True
+        resources:
+          requests:
+            cpu: 10m
+            memory: 200Mi
+        env:
+        - name: OVN_KUBE_LOG_LEVEL
+          value: "4"
+        - name: K8S_NODE
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        - name: ALL_PROXY
+          value: socks5://127.0.0.1:8090
+        - name: NO_PROXY
+          value: kube-apiserver
+        ports:
+        - name: metrics-port
+          containerPort: 29102
+        terminationMessagePolicy: FallbackToLogsOnError
+      {{ if .HCPNodeSelector }}
+      nodeSelector:
+        {{ range $key, $value := .HCPNodeSelector }}
+        "{{$key}}": "{{$value}}"
+        {{ end }}
+      {{ end }}
+      volumes:
+      - name: ovnkube-config
+        configMap:
+          name: ovnkube-config
+      - name: konnectivity-proxy-ca
+        configMap:
+          name: konnectivity-ca-bundle
+      - name: konnectivity-proxy-cert
+        secret:
+          defaultMode: 0640
+          secretName: konnectivity-client
+      - name: env-overrides
+        configMap:
+          name: env-overrides
+          optional: true
+      - name: ovn-ca
+        configMap:
+          name: ovn-ca
+      - name: ovn-cert
+        secret:
+          secretName: ovn-cert
+      - name: ovn-master-metrics-cert
+        secret:
+          secretName: ovn-master-metrics-cert
+      - name: admin-kubeconfig
+        secret:
+          secretName: service-network-admin-kubeconfig
+      - name: hosted-cluster-api-access
+        emptyDir: {}
+      - name: hosted-ca-cert
+        secret:
+          secretName: root-ca
+          items:
+            - key: ca.crt
+              path: ca.crt
+      tolerations:
+        - key: "hypershift.openshift.io/control-plane"
+          operator: "Equal"
+          value: "true"
+          effect: "NoSchedule"
+        - key: "hypershift.openshift.io/cluster"
+          operator: "Equal"
+          value: {{.HostedClusterNamespace}}
+          effect: "NoSchedule"