Update vSphere templates and manifests

manifests/vsphere/coredns-corefile.tmpl

@@ -5,8 +5,14 @@
     forward . {{`{{- range $upstream := .DNSUpstreams}} {{$upstream}}{{- end}}`}}
     cache 30
     reload
-    hosts /etc/coredns/api-int.hosts {{ .ControllerConfig.EtcdDiscoveryDomain }} {
-        {{ .ControllerConfig.Infra.Status.PlatformStatus.VSphere.APIServerInternalIP }} api-int.{{ .ControllerConfig.EtcdDiscoveryDomain }} api.{{ .ControllerConfig.EtcdDiscoveryDomain }}
+    hosts {
+        {{ .ControllerConfig.Infra.Status.PlatformStatus.VSphere.APIServerInternalIP }} api-int.{{ .ControllerConfig.EtcdDiscoveryDomain }}
+        {{ .ControllerConfig.Infra.Status.PlatformStatus.VSphere.APIServerInternalIP }} api.{{ .ControllerConfig.EtcdDiscoveryDomain }}
+        fallthrough
+    }
+    template IN A {{ .ControllerConfig.EtcdDiscoveryDomain }} {
+        match .*.apps.{{ .ControllerConfig.EtcdDiscoveryDomain }}
+        answer "{{`{{"{{ .Name }}"}}`}} 60 in a {{ .ControllerConfig.Infra.Status.PlatformStatus.VSphere.IngressIP }}"
         fallthrough
     }
 }

pkg/operator/assets/bindata.go

@@ -3449,8 +3449,14 @@ var _manifestsVsphereCorednsCorefileTmpl = []byte(`. {
     forward . {{`+"`"+`{{- range $upstream := .DNSUpstreams}} {{$upstream}}{{- end}}`+"`"+`}}
     cache 30
     reload
-    hosts /etc/coredns/api-int.hosts {{ .ControllerConfig.EtcdDiscoveryDomain }} {
-        {{ .ControllerConfig.Infra.Status.PlatformStatus.VSphere.APIServerInternalIP }} api-int.{{ .ControllerConfig.EtcdDiscoveryDomain }} api.{{ .ControllerConfig.EtcdDiscoveryDomain }}
+    hosts {
+        {{ .ControllerConfig.Infra.Status.PlatformStatus.VSphere.APIServerInternalIP }} api-int.{{ .ControllerConfig.EtcdDiscoveryDomain }}
+        {{ .ControllerConfig.Infra.Status.PlatformStatus.VSphere.APIServerInternalIP }} api.{{ .ControllerConfig.EtcdDiscoveryDomain }}
+        fallthrough
+    }
+    template IN A {{ .ControllerConfig.EtcdDiscoveryDomain }} {
+        match .*.apps.{{ .ControllerConfig.EtcdDiscoveryDomain }}
+        answer "{{`+"`"+`{{"{{ .Name }}"}}`+"`"+`}} 60 in a {{ .ControllerConfig.Infra.Status.PlatformStatus.VSphere.IngressIP }}"
         fallthrough
     }
 }

templates/common/vsphere/files/NetworkManager-mdns-hostname.yaml

@@ -0,0 +1,26 @@
+filesystem: "root"
+mode: 0755
+path: "/etc/NetworkManager/dispatcher.d/40-mdns-hostname"
+contents:
+  inline: |
+    {{ if .Infra.Status.PlatformStatus.VSphere -}}
+    {{ if .Infra.Status.PlatformStatus.VSphere.APIServerInternalIP -}}
+    #!/bin/bash
+    STATUS=$2
+    case "$STATUS" in
+        up|down|dhcp4-change|dhcp6-change|hostname)
+        logger -s "NM mdns-hostname triggered by ${2}."
+        set +e
+        t_hostname=$(hostname)
+        if [ -z "${t_hostname}" ]; then
+           t_hostname="localhost"
+        fi
+        mkdir -p /etc/mdns
+        echo "${t_hostname}">/etc/mdns/hostname
+        logger -s "Hostname changed: ${t_hostname}"
+        ;;
+        *)
+        ;;
+    esac
+    {{ end -}}
+    {{ end -}}

templates/common/vsphere/files/etc-systemd-system-crio-stream-address.conf.yaml

@@ -0,0 +1,16 @@
+filesystem: "root"
+mode: 0644
+path: "/etc/systemd/system/crio.service.d/20-stream-address.conf"
+contents:
+  inline: |
+    {{ if .Infra.Status.PlatformStatus.VSphere -}}
+    {{ if .Infra.Status.PlatformStatus.VSphere.APIServerInternalIP -}}
+    [Service]
+    ExecStart=
+    ExecStart=/usr/bin/crio \
+          --stream-address="${CONTAINER_STREAM_ADDRESS}" \
+          $CRIO_STORAGE_OPTIONS \
+          $CRIO_NETWORK_OPTIONS \
+          $CRIO_METRICS_OPTIONS
+    {{ end -}}
+    {{ end -}}

templates/common/vsphere/files/nodeip-finder.yaml

@@ -0,0 +1,110 @@
+filesystem: "root"
+mode: 0755
+path: "/usr/local/bin/nodeip-finder"
+contents:
+  inline: |
+    #!/usr/libexec/platform-python
+    # /* vim: set filetype=python : */
+    """Writes Kubelet and CRI-O configuration to choose the right IP address
+
+    For kubelet, a systemd environment file with a KUBELET_NODE_IP setting
+    For CRI-O it drops a config file in /etc/crio/crio.conf.d"""
+    import argparse
+    import ipaddress
+    from importlib import util as iutil
+    from importlib import machinery as imachinery
+    from types import ModuleType
+    import os
+    import pathlib
+    import socket
+    import sys
+    import time
+
+    loader = imachinery.SourceFileLoader(
+        'non_virtual_ip',
+        os.path.join(os.path.dirname(os.path.realpath(__file__)), 'non_virtual_ip'))
+    spec = iutil.spec_from_loader('non_virtual_ip', loader)
+    non_virtual_ip = iutil.module_from_spec(spec)
+    loader.exec_module(non_virtual_ip)
+
+
+    KUBELET_WORKAROUND_PATH = '/etc/systemd/system/kubelet.service.d/20-nodenet.conf'
+    CRIO_WORKAROUND_PATH = '/etc/systemd/system/crio.service.d/20-nodenet.conf'
+
+
+    def first_candidate_addr(api_vip: str) -> non_virtual_ip.Address:
+        filters = (non_virtual_ip.non_host_scope,
+                   non_virtual_ip.non_deprecated,
+                   non_virtual_ip.non_secondary)
+        iface_addrs = list(non_virtual_ip.interface_addrs(filters))
+        subnet, candidates = non_virtual_ip.vip_subnet_and_addrs_in_it(api_vip, iface_addrs)
+        sys.stderr.write('VIP Subnet %s\n' % subnet.cidr)
+
+        for addr in candidates:
+            return addr
+        raise non_virtual_ip.AddressNotFoundException()
+
+
+    class IPAction(argparse.Action):
+        def __call__(self, parser: argparse.ArgumentParser, namespace: argparse.Namespace, values, option_string: str=None):
+            print('Processing CustomAction for {}'.format(self.dest))
+            print('  parser = {}'.format(id(parser)))
+            print('  values = {!r}'.format(values))
+            print('  option_string = {!r}'.format(option_string))
+
+            # Do some arbitrary processing of the input values
+            if values is None:
+                raise argparse.ArgumentError(self, 'Not provided nor found from Environment')
+            if isinstance(values, list):
+                target = values[0]
+            else:
+                target = values
+
+            try:
+                ipaddress.ip_address(target)
+                setattr(namespace, self.dest, values)
+            except ValueError:  # Possibly got the name, try to resolve
+                try:
+                    sstream_tuple = socket.getaddrinfo(target, None)[0]
+                    _, _, _, _, sockaddr = sstream_tuple
+                    resolved = sockaddr[0]
+                    setattr(namespace, self.dest, resolved)
+                    sys.stderr.write(f'Found {target} resolves to {resolved}\n')
+                except socket.gaierror:
+                    raise argparse.ArgumentError(
+                        self, f'IP not provided and failed to resolve {target}')
+
+    def main() -> None:
+        parser = argparse.ArgumentParser()
+        parser.add_argument('target', nargs='?', action=IPAction, default=os.getenv('API_INT'), help='Target IP address to find a local address that directly routes to it. If not provided checks API_INT Environment variable')
+        parser.add_argument('-r', '--retry-on-failure', action='store_true', dest='retry')
+        args = parser.parse_args()
+        while True:
+            try:
+                first: non_virtual_ip.Address = first_candidate_addr(args.target)
+                prefixless = first.cidr.split('/')[0]
+                break
+            except (non_virtual_ip.AddressNotFoundException, non_virtual_ip.SubnetNotFoundException):
+                sys.stderr.write('Failed to find suitable node ip. ')
+                if args.retry:
+                    sys.stderr.write('Retrying...\n')
+                    time.sleep(1)
+                    continue
+                else:
+                    sys.stderr.write('Exiting\n')
+                    sys.exit(1)
+
+        # Kubelet
+        with open(KUBELET_WORKAROUND_PATH, 'w') as kwf:
+            print(f'[Service]\nEnvironment="KUBELET_NODE_IP={prefixless}"', file=kwf)
+
+        # CRI-O
+        crio_confd = pathlib.Path(CRIO_WORKAROUND_PATH).parent
+        crio_confd.mkdir(parents=True, exist_ok=True)
+        with open(CRIO_WORKAROUND_PATH, 'w') as cwf:
+            print(f'[Service]\nEnvironment="CONTAINER_STREAM_ADDRESS={prefixless}"', file=cwf)
+
+
+
+    if __name__ == '__main__':
+        main()

templates/common/vsphere/files/vsphere-NetworkManager-kni-conf.yaml

@@ -7,5 +7,8 @@ contents:
     {{ if .Infra.Status.PlatformStatus.VSphere.APIServerInternalIP -}}
     [main]
     dhcp=dhclient
+    rc-manager=unmanaged
+    [connection]
+    ipv6.dhcp-duid=ll
     {{ end -}}
     {{ end -}}

templates/common/vsphere/files/vsphere-coredns-corefile.yaml

@@ -12,7 +12,16 @@ contents:
         forward . {{`{{- range $upstream := .DNSUpstreams}} {{$upstream}}{{- end}}`}}
         cache 30
         reload
-        file /etc/coredns/node-dns-db {{ .EtcdDiscoveryDomain }}
+        hosts {
+            {{ .Infra.Status.PlatformStatus.VSphere.APIServerInternalIP }} api-int.{{ .EtcdDiscoveryDomain }}
+            {{ .Infra.Status.PlatformStatus.VSphere.APIServerInternalIP }} api.{{ .EtcdDiscoveryDomain }}
+            fallthrough
+        }
+        template IN A {{ .EtcdDiscoveryDomain }} {
+            match .*.apps.{{ .EtcdDiscoveryDomain }}
+            answer "{{`{{"{{ .Name }}"}}`}} 60 in a {{ .Infra.Status.PlatformStatus.VSphere.IngressIP }}"
+            fallthrough
+        }
     }
     {{ end -}}
     {{ end -}}

templates/common/vsphere/files/vsphere-coredns.yaml

@@ -25,6 +25,9 @@ contents:
       - name: conf-dir
         hostPath:
           path: "/etc/coredns"
+      - name: nm-resolv
+        hostPath:
+          path: "/var/run/NetworkManager"
       initContainers:
       - name: render-config
         image: {{ .Images.baremetalRuntimeCfgImage }}
@@ -86,6 +89,35 @@ contents:
           failureThreshold: 5
         terminationMessagePolicy: FallbackToLogsOnError
         imagePullPolicy: IfNotPresent
+      - name: coredns-monitor
+        securityContext:
+          privileged: true
+        image: {{ .Images.baremetalRuntimeCfgImage }}
+        command:
+        - corednsmonitor
+        - "/etc/kubernetes/kubeconfig"
+        - "/config/Corefile.tmpl"
+        - "/etc/coredns/Corefile"
+        - "--api-vip"
+        - "{{ .Infra.Status.PlatformStatus.VSphere.APIServerInternalIP }}"
+        - "--dns-vip"
+        - "{{ .Infra.Status.PlatformStatus.VSphere.NodeDNSIP }}"
+        - "--ingress-vip"
+        - "{{ .Infra.Status.PlatformStatus.VSphere.IngressIP }}"
+        resources:
+          requests:
+            cpu: 100m
+            memory: 200Mi
+        volumeMounts:
+        - name: kubeconfig
+          mountPath: "/etc/kubernetes/kubeconfig"
+        - name: resource-dir
+          mountPath: "/config"
+        - name: conf-dir
+          mountPath: "/etc/coredns"
+        - name: nm-resolv
+          mountPath: "/var/run/NetworkManager"
+        imagePullPolicy: IfNotPresent
       hostNetwork: true
       tolerations:
       - operator: Exists

templates/common/vsphere/files/vsphere-hostname.yaml

@@ -3,6 +3,8 @@ mode: 0755
 path: "/usr/local/bin/vsphere-hostname.sh"
 contents:
   inline: |
+    {{ if .Infra.Status.PlatformStatus.VSphere -}}
+    {{ if .Infra.Status.PlatformStatus.VSphere.APIServerInternalIP -}}
     #!/usr/bin/env bash
     set -e
 
@@ -11,4 +13,6 @@ contents:
           /usr/bin/hostnamectl --transient --static set-hostname ${hostname}
       fi
     fi
+    {{ end -}}
+    {{ end -}}
 

templates/common/vsphere/files/vsphere-keepalived.yaml

@@ -103,6 +103,10 @@ contents:
         - "{{ .Infra.Status.PlatformStatus.VSphere.NodeDNSIP }}"
         - "--ingress-vip"
         - "{{ .Infra.Status.PlatformStatus.VSphere.IngressIP }}"
+        resources:
+          requests:
+            cpu: 100m
+            memory: 200Mi
         volumeMounts:
         - name: resource-dir
           mountPath: "/config"

templates/common/vsphere/files/vsphere-mdns-publisher.yaml

@@ -26,8 +26,38 @@ contents:
         hostPath:
           path: "/etc/mdns"
       initContainers:
+      - name: verify-hostname
+        image: {{ .Images.baremetalRuntimeCfgImage }}
+        env:
+          - name: RUNTIMECFG_HOSTNAME_PATH
+            value: "/etc/mdns/hostname"
+        command:
+        - "/bin/bash"
+        - "-c"
+        - |
+          #!/bin/bash
+          set -xv
+          function get_hostname()
+          {
+            if [[ -s $RUNTIMECFG_HOSTNAME_PATH ]]; then
+              cat $RUNTIMECFG_HOSTNAME_PATH
+            else
+              # if hostname wasn't updated by NM script, read hostname
+              hostname
+            fi
+          }
+          while [[ "$(get_hostname)" =~ ^localhost(.localdomain)?$ ]]; do
+            echo "hostname is still set to a default value"
+            sleep 1
+          done
+        volumeMounts:
+        - name: conf-dir
+          mountPath: "/etc/mdns"
       - name: render-config
         image: {{ .Images.baremetalRuntimeCfgImage }}
+        env:
+          - name: RUNTIMECFG_HOSTNAME_PATH
+            value: "/etc/mdns/hostname"
         command:
         - runtimecfg
         - render
@@ -41,6 +71,7 @@ contents:
         - "/config"
         - "--out-dir"
         - "/etc/mdns"
+        - "--verbose"
         resources: {}
         volumeMounts:
         - name: kubeconfig