diff --git a/templates/common/openstack/files/NetworkManager-mdns-hostname.yaml b/templates/common/openstack/files/NetworkManager-mdns-hostname.yaml new file mode 100644 index 0000000000..cd3219a97c --- /dev/null +++ b/templates/common/openstack/files/NetworkManager-mdns-hostname.yaml @@ -0,0 +1,22 @@ +filesystem: "root" +mode: 0755 +path: "/etc/NetworkManager/dispatcher.d/40-mdns-hostname" +contents: + inline: | + #!/bin/bash + STATUS=$2 + case "$STATUS" in + up|down|dhcp4-change|dhcp6-change|hostname) + logger -s "NM mdns-hostname triggered by ${2}." + set +e + t_hostname=$(hostname) + if [ -z "${t_hostname}" ]; then + t_hostname="localhost" + fi + mkdir -p /etc/mdns + echo "${t_hostname}">/etc/mdns/hostname + logger -s "Hostname changed: ${t_hostname}" + ;; + *) + ;; + esac diff --git a/templates/common/openstack/files/nodeip-finder.yaml b/templates/common/openstack/files/nodeip-finder.yaml new file mode 100644 index 0000000000..19621af6fb --- /dev/null +++ b/templates/common/openstack/files/nodeip-finder.yaml @@ -0,0 +1,78 @@ +filesystem: "root" +mode: 0755 +path: "/usr/local/bin/nodeip-finder" +contents: + inline: | + #!/usr/libexec/platform-python + # /* vim: set filetype=python : */ + """Writes Kubelet and CRI-O configuration to choose the right IP address + + For kubelet, a systemd environment file with a KUBELET_NODE_IP setting + For CRI-O it drops a config file in /etc/crio/crio.conf.d""" + from importlib import util as iutil + from importlib import machinery as imachinery + from types import ModuleType + import os + import pathlib + import socket + import sys + + loader = imachinery.SourceFileLoader( + 'non_virtual_ip', + os.path.join(os.path.dirname(os.path.realpath(__file__)), 'non_virtual_ip')) + spec = iutil.spec_from_loader('non_virtual_ip', loader) + non_virtual_ip = iutil.module_from_spec(spec) + loader.exec_module(non_virtual_ip) + + + KUBELET_WORKAROUND_PATH = '/etc/systemd/system/kubelet.service.d/20-nodenet.conf' + CRIO_WORKAROUND_PATH = '/etc/systemd/system/crio.service.d/20-nodenet.conf' + + + def first_candidate_addr(api_vip: str) -> non_virtual_ip.Address: + filters = (non_virtual_ip.non_host_scope, + non_virtual_ip.non_deprecated, + non_virtual_ip.non_secondary) + iface_addrs = list(non_virtual_ip.interface_addrs(filters)) + subnet, candidates = non_virtual_ip.vip_subnet_and_addrs_in_it(api_vip, iface_addrs) + sys.stderr.write('VIP Subnet %s\n' % subnet.cidr) + + for addr in candidates: + return addr + raise non_virtual_ip.AddressNotFoundException() + + + def main() -> None: + if len(sys.argv) > 1: + api_vip = sys.argv[1] + else: + api_int_name = os.getenv('API_INT') + try: + sstream_tuple = socket.getaddrinfo(api_int_name, None)[0] + _, _, _, _, sockaddr = sstream_tuple + api_vip = sockaddr[0] + sys.stderr.write(f'Found {api_int_name} to resolve to {api_vip}\n') + except socket.gaierror: + sys.stderr.write(f'api-int VIP not provided and failed to resolve {api_int_name}\n') + sys.exit(1) + try: + first: non_virtual_ip.Address = first_candidate_addr(api_vip) + prefixless = first.cidr.split('/')[0] + + # Kubelet + with open(KUBELET_WORKAROUND_PATH, 'w') as kwf: + print(f'[Service]\nEnvironment="KUBELET_NODE_IP={prefixless}"', file=kwf) + + # CRI-O + crio_confd = pathlib.Path(CRIO_WORKAROUND_PATH).parent + crio_confd.mkdir(parents=True, exist_ok=True) + with open(CRIO_WORKAROUND_PATH, 'w') as cwf: + print(f'[Service]\nEnvironment="CONTAINER_STREAM_ADDRESS={prefixless}"', file=cwf) + + except (non_virtual_ip.AddressNotFoundException, non_virtual_ip.SubnetNotFoundException): + sys.stderr.write('Failed to find suitable node ip') + sys.exit(1) + + + if __name__ == '__main__': + main() diff --git a/templates/common/openstack/files/openstack-NetworkManager-conf.yaml b/templates/common/openstack/files/openstack-NetworkManager-conf.yaml index 77098627ea..ba45f40d8b 100644 --- a/templates/common/openstack/files/openstack-NetworkManager-conf.yaml +++ b/templates/common/openstack/files/openstack-NetworkManager-conf.yaml @@ -5,3 +5,6 @@ contents: inline: | [main] dhcp=dhclient + rc-manager=unmanaged + [connection] + ipv6.dhcp-duid=ll diff --git a/templates/common/openstack/files/openstack-mdns-publisher.yaml b/templates/common/openstack/files/openstack-mdns-publisher.yaml index d40fc2deab..b3981d763a 100644 --- a/templates/common/openstack/files/openstack-mdns-publisher.yaml +++ b/templates/common/openstack/files/openstack-mdns-publisher.yaml @@ -24,8 +24,40 @@ contents: hostPath: path: "/etc/mdns" initContainers: + - name: verify-hostname + image: {{ .Images.baremetalRuntimeCfgImage }} + env: + - name: DEFAULT_LOCAL_HOSTNAME + value: "localhost" + - name: RUNTIMECFG_HOSTNAME_PATH + value: "/etc/mdns/hostname" + command: + - "/bin/bash" + - "-c" + - | + #/bin/bash + function get_hostname() + { + if [[ -s $RUNTIMECFG_HOSTNAME_PATH ]]; then + cat $RUNTIMECFG_HOSTNAME_PATH + else + # if hostname wasn't updated by NM script, read hostname + hostname + fi + } + while [ "$(get_hostname)" == "$DEFAULT_LOCAL_HOSTNAME" ] + do + echo "hostname is still ${DEFAULT_LOCAL_HOSTNAME}" + sleep 1 + done + volumeMounts: + - name: conf-dir + mountPath: "/etc/mdns" - name: render-config image: {{ .Images.baremetalRuntimeCfgImage }} + env: + - name: RUNTIMECFG_HOSTNAME_PATH + value: "/etc/mdns/hostname" command: - runtimecfg - render diff --git a/templates/common/openstack/files/openstack-non-virtual-ip.yaml b/templates/common/openstack/files/openstack-non-virtual-ip.yaml new file mode 100644 index 0000000000..57c9e7e785 --- /dev/null +++ b/templates/common/openstack/files/openstack-non-virtual-ip.yaml @@ -0,0 +1,240 @@ +filesystem: "root" +mode: 0755 +path: "/usr/local/bin/non_virtual_ip" +contents: + inline: | + #!/usr/libexec/platform-python + # /* vim: set filetype=python : */ + import collections + import itertools + import socket + import struct + import subprocess + import sys + from typing import Callable, Iterable, Iterator, List, Optional, Tuple, Type, TypeVar + + + class SubnetNotFoundException(Exception): + """ + Exception raised when no subnet in the systems ifaces is on the VIP subnet + """ + + + class AddressNotFoundException(Exception): + """ + Exception raised when no Address in the systems ifaces is on the VIP subnet + """ + + + TA = TypeVar('TA', bound='Address') + + + class Address: + def __init__(self, cidr: str, name: str, family: str, index: int = -1, scope: str = '', flags: Iterable[str] = tuple(), label: Optional[str] = None) -> None: + self.index = index + self.name = name + self.family = family + self.cidr = cidr + self.scope = scope + self.flags = flags + self.label = label + + @classmethod + def from_line(cls: Type[TA], line: str) -> TA: + tokens = collections.deque(line.split()) + index = int(tokens.popleft()[:-1]) + name = tokens.popleft() + family = tokens.popleft() + cidr = tokens.popleft() + _ = tokens.popleft() # dump scope label + scope = tokens.popleft() + flags = [] + label = None + while True: + token = tokens.popleft() + if token[-1] == '\\': + if len(token) > 1: + label = token[:-1] + break + flags.append(token) + return cls(cidr, name, family, index, scope, flags, label) + + def __str__(self) -> str: + return f'{self.__class__.__name__}({self.cidr}, dev={self.name})' + + + TR = TypeVar('TR', bound='V6Route') + + + class V6Route: + def __init__(self, destination: str, dev: Optional[str] = None, proto: Optional[str] = None, metric: Optional[int] = None, pref: Optional[str] = None, via: Optional[str] = None) -> None: + self.destination: str = destination + self.via: Optional[str] = via + self.dev: Optional[str] = dev + self.proto: Optional[str] = proto + self.metric: Optional[int] = metric + self.pref: Optional[str] = pref + + @classmethod + def from_line(cls: Type[TR], line: str) -> TR: + items = line.split() + dest = items[0] + if dest == 'default': + dest = '::/0' + attrs = dict(itertools.zip_longest(*[iter(items[1:])]*2, fillvalue=None)) + attrs['destination'] = dest + return cls(**attrs) + + def __str__(self) -> str: + return f'{self.__class__.__name__}({self.destination}, dev={self.dev})' + + + SUBNET_MASK_LEN = { + 'inet': 32, + 'inet6': 128 + } + + + def ntoa(family: str, num: int) -> str: + if family == 'inet': + result = socket.inet_ntoa(struct.pack("!I", num)) + else: + lo_half = num & 0xFFFFFFFFFFFFFFFF + hi_half = num >> 64 + result = socket.inet_ntop(socket.AF_INET6, + struct.pack(">QQ", hi_half, lo_half)) + return result + + + def aton(family: str, rep: str) -> int: + if family == 'inet': + result = struct.unpack("!I", socket.inet_aton(rep))[0] + else: + hi_half, lo_half = struct.unpack(">QQ", socket.inet_pton(socket.AF_INET6, rep)) + result = (hi_half << 64) | lo_half + return result + + + def addr_subnet_int_min_max(addr: Address) -> Tuple[int, int]: + ip_addr, prefix = addr.cidr.split('/') + ip_int = aton(addr.family, ip_addr) + + prefix_int = int(prefix) + mask = int('1' * prefix_int + + '0' * (SUBNET_MASK_LEN[addr.family] - prefix_int), 2) + + subnet_ip_int_min = ip_int & mask + + remainder = '1' * (SUBNET_MASK_LEN[addr.family] - prefix_int) + subnet_ip_int_max = subnet_ip_int_min | ( + 0 if remainder == '' else int(remainder, 2)) + return subnet_ip_int_min, subnet_ip_int_max + + + def vip_subnet_and_addrs_in_it(vip: str, addrs: List[Address]) -> Tuple[Address, List[Address]]: + try: + vip_int = aton('inet', vip) + except Exception: + vip_int = aton('inet6', vip) + subnet = None + candidates = [] + for addr in addrs: + subnet_ip_int_min, subnet_ip_int_max = addr_subnet_int_min_max(addr) + subnet_ip = ntoa(addr.family, subnet_ip_int_min) + subnet_ip_max = ntoa(addr.family, subnet_ip_int_max) + + sys.stderr.write('Is %s between %s and %s\n' % + (vip, subnet_ip, subnet_ip_max)) + if subnet_ip_int_min < vip_int < subnet_ip_int_max: + subnet_ip = ntoa(addr.family, subnet_ip_int_min) + subnet = Address(name="subnet", + cidr='%s/%s' % (subnet_ip, addr.cidr.split('/')[1]), + family=addr.family, + scope='') + candidates.append(addr) + if subnet is None: + raise SubnetNotFoundException() + return subnet, candidates + + + def interface_addrs(filters: Optional[Iterable[Callable[[Address], bool]]] = None) -> Iterator[Address]: + out = subprocess.check_output(["ip", "-o", "addr", "show"], encoding=sys.stdout.encoding) + for addr in (Address.from_line(line) for line in out.splitlines()): + if not filters or all(f(addr) for f in filters): + if (addr.family == 'inet6' and + int(addr.cidr.split('/')[1]) == SUBNET_MASK_LEN[addr.family]): + route_out = subprocess.check_output(["ip", "-o", "-6", "route", "show"], + encoding=sys.stdout.encoding) + for route in (V6Route.from_line(rline) for rline in route_out.splitlines()): + if (route.dev == addr.name and route.proto == 'ra' and + route.destination != '::/0'): + sys.stderr.write('Checking %s for %s\n' % (route, addr)) + route_net = Address(name=route.dev, cidr=route.destination, family='inet6') + route_filter = in_subnet(route_net) + if route_filter(addr): + ip_addr = addr.cidr.split('/')[0] + route_prefix = route_net.cidr.split('/')[1] + cidr = '%s/%s' % (ip_addr, route_prefix) + yield Address(cidr=cidr, + family=addr.family, + name=addr.name) + yield addr + + + def non_host_scope(addr: Address) -> bool: + if addr.scope == 'host': + sys.stderr.write(f'Filtering out {addr} due to it having host scope\n') + res = False + else: + res = True + return res + + + def non_deprecated(addr: Address) -> bool: + if 'deprecated' in addr.flags: + sys.stderr.write(f'Filtering out {addr} due to it being deprecated\n') + res = False + else: + res = True + return res + + + def non_secondary(addr: Address) -> bool: + if 'secondary' in addr.flags: + sys.stderr.write(f'Filtering out {addr} due to it being secondary\n') + res = False + else: + res = True + return res + + + def in_subnet(subnet: Address) -> Callable[[Address], bool]: + subnet_ip_int_min, subnet_ip_int_max = addr_subnet_int_min_max(subnet) + + def filt(addr: Address) -> bool: + ip_addr, _ = addr.cidr.split('/') + ip_int = aton(addr.family, ip_addr) + return subnet_ip_int_min < ip_int < subnet_ip_int_max + return filt + + + def main() -> None: + api_vip = sys.argv[1] + vips = set(sys.argv[1:4]) + filters = (non_host_scope, non_deprecated, non_secondary) + iface_addrs = list(interface_addrs(filters)) + try: + subnet, candidates = vip_subnet_and_addrs_in_it(api_vip, iface_addrs) + sys.stderr.write('VIP Subnet %s\n' % subnet.cidr) + + for addr in candidates: + ip_addr, _ = addr.cidr.split('/') + if ip_addr not in vips: + print(ip_addr) + sys.exit(0) + except SubnetNotFoundException: + sys.exit(1) + + + if __name__ == '__main__': + main() diff --git a/templates/common/openstack/units/nodeip-configuration.service b/templates/common/openstack/units/nodeip-configuration.service new file mode 100644 index 0000000000..21623b2e67 --- /dev/null +++ b/templates/common/openstack/units/nodeip-configuration.service @@ -0,0 +1,20 @@ +name: "nodeip-configuration.service" +enabled: true +contents: | + [Unit] + Description=Writes IP address configuration so that kubelet and crio services select a valid node IP + # This only applies to VIP managing environments where the kubelet and crio IP + # address picking logic is flawed and may end up selecting an address from a + # different subnet or a deprecated address + Wants=network-online.target + After=network-online.target ignition-firstboot-complete.service + Before=kubelet.service crio.service + + [Service] + # Need oneshot to delay kubelet + Type=oneshot + ExecStart=/usr/local/bin/nodeip-finder {{.Infra.Status.PlatformStatus.OpenStack.APIServerInternalIP }} + + [Install] + WantedBy=multi-user.target + diff --git a/templates/master/00-master/openstack/files/NetworkManager-resolv-prepender.yaml b/templates/master/00-master/openstack/files/NetworkManager-resolv-prepender.yaml new file mode 100644 index 0000000000..294f09b02e --- /dev/null +++ b/templates/master/00-master/openstack/files/NetworkManager-resolv-prepender.yaml @@ -0,0 +1,27 @@ +filesystem: "root" +mode: 0755 +path: "/etc/NetworkManager/dispatcher.d/30-resolv-prepender" +contents: + inline: | + #!/bin/bash + IFACE=$1 + STATUS=$2 + case "$STATUS" in + up|down|dhcp4-change|dhcp6-change) + logger -s "NM resolv-prepender triggered by ${1} ${2}." + NAMESERVER_IP="{{.Infra.Status.PlatformStatus.OpenStack.NodeDNSIP}}" + DOMAIN="{{.EtcdDiscoveryDomain}}" + set +e + if [[ -n "$NAMESERVER_IP" ]]; then + logger -s "NM resolv-prepender: Prepending 'nameserver $NAMESERVER_IP' to /etc/resolv.conf (other nameservers from /var/run/NetworkManager/resolv.conf)" + sed -e "/^search/d" \ + -e "/Generated by/c# Generated by OpenStack resolv prepender NM dispatcher script\nsearch $DOMAIN\nnameserver $NAMESERVER_IP" \ + /var/run/NetworkManager/resolv.conf > /etc/resolv.conf + else + logger -s "NM resolv-prepender: Couldn't find a Virtual IP, just updating resolv.conf" + cp /var/run/NetworkManager/resolv.conf /etc/resolv.conf + fi + ;; + *) + ;; + esac diff --git a/templates/master/00-master/openstack/files/dhcp-dhclient-conf.yaml b/templates/master/00-master/openstack/files/dhcp-dhclient-conf.yaml index aa84b764c0..0a3d18bc86 100644 --- a/templates/master/00-master/openstack/files/dhcp-dhclient-conf.yaml +++ b/templates/master/00-master/openstack/files/dhcp-dhclient-conf.yaml @@ -4,4 +4,3 @@ path: "/etc/dhcp/dhclient.conf" contents: inline: | supersede domain-search "{{ .EtcdDiscoveryDomain }}"; - prepend domain-name-servers {{ .Infra.Status.PlatformStatus.OpenStack.NodeDNSIP }}; diff --git a/templates/master/00-master/openstack/files/openstack-haproxy-haproxy.yaml b/templates/master/00-master/openstack/files/openstack-haproxy-haproxy.yaml index bf8384ac6e..4a425f5fb8 100644 --- a/templates/master/00-master/openstack/files/openstack-haproxy-haproxy.yaml +++ b/templates/master/00-master/openstack/files/openstack-haproxy-haproxy.yaml @@ -25,7 +25,7 @@ contents: monitor-uri /healthz option dontlognull listen stats - bind 127.0.0.1:{{`{{ .LBConfig.StatPort }}`}} + bind localhost:{{`{{ .LBConfig.StatPort }}`}} mode http stats enable stats hide-version diff --git a/templates/master/00-master/openstack/files/openstack-keepalived-keepalived.yaml b/templates/master/00-master/openstack/files/openstack-keepalived-keepalived.yaml index 55644d6493..761d874eac 100644 --- a/templates/master/00-master/openstack/files/openstack-keepalived-keepalived.yaml +++ b/templates/master/00-master/openstack/files/openstack-keepalived-keepalived.yaml @@ -4,7 +4,7 @@ path: "/etc/kubernetes/static-pod-resources/keepalived/keepalived.conf.tmpl" contents: inline: | vrrp_script chk_ocp { - script "/usr/bin/curl -o /dev/null -kLs https://0:6443/readyz" + script "/usr/bin/curl -o /dev/null -kLs https://localhost:6443/readyz" interval 1 weight 50 } @@ -16,7 +16,7 @@ contents: # TODO: Improve this check. The port is assumed to be alive. # Need to assess what is the ramification if the port is not there. vrrp_script chk_ingress { - script "/usr/bin/curl -o /dev/null -kLs http://0:1936/healthz" + script "/usr/bin/curl -o /dev/null -kLs http://localhost:1936/healthz" interval 1 weight 50 } diff --git a/templates/master/01-master-kubelet/openstack/units/kubelet.yaml b/templates/master/01-master-kubelet/openstack/units/kubelet.yaml new file mode 100644 index 0000000000..8889162bf0 --- /dev/null +++ b/templates/master/01-master-kubelet/openstack/units/kubelet.yaml @@ -0,0 +1,39 @@ +name: "kubelet.service" +enabled: true +contents: | + [Unit] + Description=Kubernetes Kubelet + Wants=rpc-statd.service network-online.target crio.service + After=network-online.target crio.service + + [Service] + Type=notify + ExecStartPre=/bin/mkdir --parents /etc/kubernetes/manifests + ExecStartPre=/bin/rm -f /var/lib/kubelet/cpu_manager_state + Environment="KUBELET_LOG_LEVEL=3" + EnvironmentFile=/etc/os-release + EnvironmentFile=-/etc/kubernetes/kubelet-workaround + EnvironmentFile=-/etc/kubernetes/kubelet-env + + ExecStart=/usr/bin/hyperkube \ + kubelet \ + --config=/etc/kubernetes/kubelet.conf \ + --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --container-runtime=remote \ + --container-runtime-endpoint=/var/run/crio/crio.sock \ + --node-labels=node-role.kubernetes.io/master,node.openshift.io/os_id=${ID} \ + --node-ip="${KUBELET_NODE_IP}" \ + --address="${KUBELET_NODE_IP}" \ + --minimum-container-ttl-duration=6m0s \ + --cloud-provider={{cloudProvider .}} \ + --volume-plugin-dir=/etc/kubernetes/kubelet-plugins/volume/exec \ + {{cloudConfigFlag . }} \ + --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \ + --v=${KUBELET_LOG_LEVEL} + + Restart=always + RestartSec=10 + + [Install] + WantedBy=multi-user.target diff --git a/templates/worker/00-worker/openstack/files/NetworkManager-non-virtual-ip-prepender.yaml b/templates/worker/00-worker/openstack/files/NetworkManager-non-virtual-ip-prepender.yaml deleted file mode 100644 index 9607da4138..0000000000 --- a/templates/worker/00-worker/openstack/files/NetworkManager-non-virtual-ip-prepender.yaml +++ /dev/null @@ -1,41 +0,0 @@ -filesystem: "root" -mode: 0755 -path: "/etc/NetworkManager/dispatcher.d/pre-up.d/non-virtual-ip-prepender" -contents: - inline: | - #!/bin/bash - IFACE=$1 - STATUS=$2 - case "$STATUS" in - pre-up) - logger -s "NM non-virtual-ip-prepender triggered by pre-upping ${1}." - NON_VIRTUAL_IP=$(/usr/local/bin/non_virtual_ip \ - "{{.Infra.Status.PlatformStatus.OpenStack.APIServerInternalIP}}" \ - "{{.Infra.Status.PlatformStatus.OpenStack.NodeDNSIP}}" \ - "{{.Infra.Status.PlatformStatus.OpenStack.IngressIP}}") - set +e - if [[ -n $NON_VIRTUAL_IP ]]; then - logger -s "NM non-virtual-ip-prepender: Checking if worker non virtual IP is the first entry in resolv.conf" - if grep nameserver /etc/resolv.conf | head -n 1 | grep -q "$NON_VIRTUAL_IP" ; then - logger -s "NM non-virtual-ip-prepender: worker node non virtual IP already is the first entry in resolv.conf" - exit 0 - else - logger -s "NM non-virtual-ip-prepender: Setting dhclient to prepend non virtual IP in resolv.conf" - sed "s/{{`{{ .NonVirtualIP }}`}}/$NON_VIRTUAL_IP/" /etc/dhcp/dhclient.conf.template > /etc/dhcp/dhclient.conf - logger -s "NM non-virtual-ip-prepender: Looking for 'search' in /etc/resolv.conf to place 'nameserver $NON_VIRTUAL_IP'" - sed -i "/^search .*$/a nameserver $NON_VIRTUAL_IP" /etc/resolv.conf - fi - fi - ;; - down) - logger -s "NM non-virtual-ip-prepender triggered by downing $IFACE" - ;; - up) - logger -s "NM non-virtual-ip-prepender triggered by upping $IFACE" - ;; - post-down) - logger -s "NM non-virtual-ip-prepender triggered by post-downing $IFACE" - ;; - *) - ;; - esac diff --git a/templates/worker/00-worker/openstack/files/NetworkManager-resolv-prepender.yaml b/templates/worker/00-worker/openstack/files/NetworkManager-resolv-prepender.yaml new file mode 100644 index 0000000000..77c70a6c53 --- /dev/null +++ b/templates/worker/00-worker/openstack/files/NetworkManager-resolv-prepender.yaml @@ -0,0 +1,30 @@ +filesystem: "root" +mode: 0755 +path: "/etc/NetworkManager/dispatcher.d/30-resolv-prepender" +contents: + inline: | + #!/bin/bash + IFACE=$1 + STATUS=$2 + case "$STATUS" in + up|down|dhcp4-change|dhcp6-change) + logger -s "NM resolv-prepender triggered by ${1} ${2}." + NAMESERVER_IP=$(/usr/local/bin/non_virtual_ip \ + "{{.Infra.Status.PlatformStatus.OpenStack.APIServerInternalIP}}" \ + "{{.Infra.Status.PlatformStatus.OpenStack.NodeDNSIP}}" \ + "{{.Infra.Status.PlatformStatus.OpenStack.IngressIP}}") + DOMAIN="{{.EtcdDiscoveryDomain}}" + set +e + if [[ -n "$NAMESERVER_IP" ]]; then + logger -s "NM resolv-prepender: Prepending 'nameserver $NAMESERVER_IP' to /etc/resolv.conf (other nameservers from /var/run/NetworkManager/resolv.conf)" + sed -e "/^search/d" \ + -e "/Generated by/c# Generated by OpenStack resolv prepender NM dispatcher script\nsearch $DOMAIN\nnameserver $NAMESERVER_IP" \ + /var/run/NetworkManager/resolv.conf > /etc/resolv.conf + else + logger -s "Couldn't find a non-virtual IP, just updating resolv.conf" + cp /var/run/NetworkManager/resolv.conf /etc/resolv.conf + fi + ;; + *) + ;; + esac diff --git a/templates/worker/00-worker/openstack/files/dhcp-dhclient-conf.yaml b/templates/worker/00-worker/openstack/files/dhcp-dhclient-conf.yaml index ee141be526..0a3d18bc86 100644 --- a/templates/worker/00-worker/openstack/files/dhcp-dhclient-conf.yaml +++ b/templates/worker/00-worker/openstack/files/dhcp-dhclient-conf.yaml @@ -1,7 +1,6 @@ filesystem: "root" mode: 0644 -path: "/etc/dhcp/dhclient.conf.template" +path: "/etc/dhcp/dhclient.conf" contents: inline: | supersede domain-search "{{ .EtcdDiscoveryDomain }}"; - prepend domain-name-servers {{`{{ .NonVirtualIP }}`}}; diff --git a/templates/worker/00-worker/openstack/files/openstack-non-virtual-ip.yaml b/templates/worker/00-worker/openstack/files/openstack-non-virtual-ip.yaml deleted file mode 100644 index c66cd2c5ba..0000000000 --- a/templates/worker/00-worker/openstack/files/openstack-non-virtual-ip.yaml +++ /dev/null @@ -1,138 +0,0 @@ -filesystem: "root" -mode: 0755 -path: "/usr/local/bin/non_virtual_ip" -contents: - inline: | - #!/usr/libexec/platform-python - import collections - import socket - import struct - import subprocess - import sys - - - class SubnetNotFoundException(Exception): - """ - Exception raised when no subnet in the systems ifaces is on the VIP subnet - """ - pass - - - Address = collections.namedtuple('Address', 'index name family cidr scope') - - SUBNET_MASK_LEN = { - 'inet': 32, - 'inet6': 128 - } - - - def ntoa(family, num): - if family == 'inet': - result = socket.inet_ntoa(struct.pack("!I", num)) - else: - lo_half = num & 0xFFFFFFFFFFFFFFFF - hi_half = num >> 64 - result = socket.inet_ntop(socket.AF_INET6, - struct.pack(">QQ", hi_half, lo_half)) - return result - - - def aton(family, rep): - if family == 'inet': - result = struct.unpack("!I", socket.inet_aton(rep))[0] - else: - hi_half, lo_half = struct.unpack(">QQ", socket.inet_pton(socket.AF_INET6, rep)) - result = (hi_half << 64) | lo_half - return result - - - def addr_subnet_int_min_max(addr): - ip, prefix = addr.cidr.split('/') - ip_int = aton(addr.family, ip) - - prefix_int = int(prefix) - mask = int('1' * prefix_int + - '0' * (SUBNET_MASK_LEN[addr.family] - prefix_int), 2) - - subnet_ip_int_min = ip_int & mask - - remainder = '1' * (SUBNET_MASK_LEN[addr.family] - prefix_int) - subnet_ip_int_max = subnet_ip_int_min | ( - 0 if remainder == '' else int(remainder, 2)) - return subnet_ip_int_min, subnet_ip_int_max - - - def vip_subnet_cidr(vip, addrs): - try: - vip_int = aton('inet', vip) - except Exception: - vip_int = aton('inet6', vip) - for addr in addrs: - subnet_ip_int_min, subnet_ip_int_max = addr_subnet_int_min_max(addr) - subnet_ip = ntoa(addr.family, subnet_ip_int_min) - subnet_ip_max = ntoa(addr.family, subnet_ip_int_max) - - sys.stderr.write('Is %s between %s and %s\n' % - (vip, subnet_ip, subnet_ip_max)) - if subnet_ip_int_min < vip_int < subnet_ip_int_max: - subnet_ip = socket.inet_ntoa(struct.pack("!I", subnet_ip_int_min)) - return Address(index="-1", - name="subnet", - cidr='%s/%s' % (subnet_ip, addr.cidr.split('/')[1]), - family=addr.family, - scope='') - raise SubnetNotFoundException() - - - def line_to_address(line): - spl = line.split() - return Address(index=spl[0][:-1], - name=spl[1], - family=spl[2], - cidr=spl[3], - scope=spl[5]) - - - def interface_cidrs(filter_func=None): - try: - out = subprocess.check_output(["ip", "-o", "addr", "show"], encoding=sys.stdout.encoding) - except TypeError: # python2 - out = subprocess.check_output(["ip", "-o", "addr", "show"]) - for addr in (line_to_address(line) for line in out.splitlines()): - if filter_func(addr): - yield addr - - - def non_host_scope(addr): - return addr.scope != 'host' - - - def in_subnet(subnet): - subnet_ip_int_min, subnet_ip_int_max = addr_subnet_int_min_max(subnet) - - def filt(addr): - ip, _ = addr.cidr.split('/') - ip_int = aton(addr.family, ip) - return subnet_ip_int_min < ip_int < subnet_ip_int_max - return filt - - - def main(): - api_vip, dns_vip, ingress_vip = sys.argv[1:4] - vips = set(sys.argv[1:4]) - iface_cidrs = list(interface_cidrs(non_host_scope)) - try: - subnet = vip_subnet_cidr(api_vip, iface_cidrs) - - sys.stderr.write('VIP Subnet %s\n' % subnet.cidr) - - for addr in interface_cidrs(in_subnet(subnet)): - ip, prefix = addr.cidr.split('/') - if ip not in vips: - print(addr.cidr.split('/')[0]) - sys.exit(0) - except SubnetNotFoundException: - sys.exit(1) - - if __name__ == '__main__': - main() diff --git a/templates/worker/01-worker-kubelet/openstack/units/kubelet.yaml b/templates/worker/01-worker-kubelet/openstack/units/kubelet.yaml new file mode 100644 index 0000000000..07fc68f3c9 --- /dev/null +++ b/templates/worker/01-worker-kubelet/openstack/units/kubelet.yaml @@ -0,0 +1,38 @@ +name: "kubelet.service" +enabled: true +contents: | + [Unit] + Description=Kubernetes Kubelet + Wants=rpc-statd.service network-online.target crio.service + After=network-online.target crio.service + + [Service] + Type=notify + ExecStartPre=/bin/mkdir --parents /etc/kubernetes/manifests + ExecStartPre=/bin/rm -f /var/lib/kubelet/cpu_manager_state + Environment="KUBELET_LOG_LEVEL=3" + EnvironmentFile=/etc/os-release + EnvironmentFile=-/etc/kubernetes/kubelet-workaround + EnvironmentFile=-/etc/kubernetes/kubelet-env + + ExecStart=/usr/bin/hyperkube \ + kubelet \ + --config=/etc/kubernetes/kubelet.conf \ + --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --container-runtime=remote \ + --container-runtime-endpoint=/var/run/crio/crio.sock \ + --node-labels=node-role.kubernetes.io/worker,node.openshift.io/os_id=${ID} \ + --node-ip="${KUBELET_NODE_IP}" \ + --address="${KUBELET_NODE_IP}" \ + --minimum-container-ttl-duration=6m0s \ + --volume-plugin-dir=/etc/kubernetes/kubelet-plugins/volume/exec \ + --cloud-provider={{cloudProvider .}} \ + {{cloudConfigFlag . }} \ + --v=${KUBELET_LOG_LEVEL} + + Restart=always + RestartSec=10 + + [Install] + WantedBy=multi-user.target