diff --git a/templates/common/vsphere/files/nodeip-finder.yaml b/templates/common/vsphere/files/nodeip-finder.yaml deleted file mode 100644 index bfc73c7c62..0000000000 --- a/templates/common/vsphere/files/nodeip-finder.yaml +++ /dev/null @@ -1,109 +0,0 @@ -mode: 0755 -path: "/usr/local/bin/nodeip-finder" -contents: - inline: | - #!/usr/libexec/platform-python - # /* vim: set filetype=python : */ - """Writes Kubelet and CRI-O configuration to choose the right IP address - - For kubelet, a systemd environment file with a KUBELET_NODE_IP setting - For CRI-O it drops a config file in /etc/crio/crio.conf.d""" - import argparse - import ipaddress - from importlib import util as iutil - from importlib import machinery as imachinery - from types import ModuleType - import os - import pathlib - import socket - import sys - import time - - loader = imachinery.SourceFileLoader( - 'non_virtual_ip', - os.path.join(os.path.dirname(os.path.realpath(__file__)), 'non_virtual_ip')) - spec = iutil.spec_from_loader('non_virtual_ip', loader) - non_virtual_ip = iutil.module_from_spec(spec) - loader.exec_module(non_virtual_ip) - - - KUBELET_WORKAROUND_PATH = '/etc/systemd/system/kubelet.service.d/20-nodenet.conf' - CRIO_WORKAROUND_PATH = '/etc/systemd/system/crio.service.d/20-nodenet.conf' - - - def first_candidate_addr(api_vip: str) -> non_virtual_ip.Address: - filters = (non_virtual_ip.non_host_scope, - non_virtual_ip.non_deprecated, - non_virtual_ip.non_secondary) - iface_addrs = list(non_virtual_ip.interface_addrs(filters)) - subnet, candidates = non_virtual_ip.vip_subnet_and_addrs_in_it(api_vip, iface_addrs) - sys.stderr.write('VIP Subnet %s\n' % subnet.cidr) - - for addr in candidates: - return addr - raise non_virtual_ip.AddressNotFoundException() - - - class IPAction(argparse.Action): - def __call__(self, parser: argparse.ArgumentParser, namespace: argparse.Namespace, values, option_string: str=None): - print('Processing CustomAction for {}'.format(self.dest)) - print(' parser = {}'.format(id(parser))) - print(' values = {!r}'.format(values)) - print(' option_string = {!r}'.format(option_string)) - - # Do some arbitrary processing of the input values - if values is None: - raise argparse.ArgumentError(self, 'Not provided nor found from Environment') - if isinstance(values, list): - target = values[0] - else: - target = values - - try: - ipaddress.ip_address(target) - setattr(namespace, self.dest, values) - except ValueError: # Possibly got the name, try to resolve - try: - sstream_tuple = socket.getaddrinfo(target, None)[0] - _, _, _, _, sockaddr = sstream_tuple - resolved = sockaddr[0] - setattr(namespace, self.dest, resolved) - sys.stderr.write(f'Found {target} resolves to {resolved}\n') - except socket.gaierror: - raise argparse.ArgumentError( - self, f'IP not provided and failed to resolve {target}') - - def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument('target', nargs='?', action=IPAction, default=os.getenv('API_INT'), help='Target IP address to find a local address that directly routes to it. If not provided checks API_INT Environment variable') - parser.add_argument('-r', '--retry-on-failure', action='store_true', dest='retry') - args = parser.parse_args() - while True: - try: - first: non_virtual_ip.Address = first_candidate_addr(args.target) - prefixless = first.cidr.split('/')[0] - break - except (non_virtual_ip.AddressNotFoundException, non_virtual_ip.SubnetNotFoundException): - sys.stderr.write('Failed to find suitable node ip. ') - if args.retry: - sys.stderr.write('Retrying...\n') - time.sleep(1) - continue - else: - sys.stderr.write('Exiting\n') - sys.exit(1) - - # Kubelet - with open(KUBELET_WORKAROUND_PATH, 'w') as kwf: - print(f'[Service]\nEnvironment="KUBELET_NODE_IP={prefixless}"', file=kwf) - - # CRI-O - crio_confd = pathlib.Path(CRIO_WORKAROUND_PATH).parent - crio_confd.mkdir(parents=True, exist_ok=True) - with open(CRIO_WORKAROUND_PATH, 'w') as cwf: - print(f'[Service]\nEnvironment="CONTAINER_STREAM_ADDRESS={prefixless}"', file=cwf) - - - - if __name__ == '__main__': - main() diff --git a/templates/common/vsphere/files/vsphere-non-virtual-ip.yaml b/templates/common/vsphere/files/vsphere-non-virtual-ip.yaml deleted file mode 100644 index da5f62eb67..0000000000 --- a/templates/common/vsphere/files/vsphere-non-virtual-ip.yaml +++ /dev/null @@ -1,239 +0,0 @@ -mode: 0755 -path: "/usr/local/bin/non_virtual_ip" -contents: - inline: | - #!/usr/libexec/platform-python - # /* vim: set filetype=python : */ - import collections - import itertools - import socket - import struct - import subprocess - import sys - from typing import Callable, Iterable, Iterator, List, Optional, Tuple, Type, TypeVar - - - class SubnetNotFoundException(Exception): - """ - Exception raised when no subnet in the systems ifaces is on the VIP subnet - """ - - - class AddressNotFoundException(Exception): - """ - Exception raised when no Address in the systems ifaces is on the VIP subnet - """ - - - TA = TypeVar('TA', bound='Address') - - - class Address: - def __init__(self, cidr: str, name: str, family: str, index: int = -1, scope: str = '', flags: Iterable[str] = tuple(), label: Optional[str] = None) -> None: - self.index = index - self.name = name - self.family = family - self.cidr = cidr - self.scope = scope - self.flags = flags - self.label = label - - @classmethod - def from_line(cls: Type[TA], line: str) -> TA: - tokens = collections.deque(line.split()) - index = int(tokens.popleft()[:-1]) - name = tokens.popleft() - family = tokens.popleft() - cidr = tokens.popleft() - _ = tokens.popleft() # dump scope label - scope = tokens.popleft() - flags = [] - label = None - while True: - token = tokens.popleft() - if token[-1] == '\\': - if len(token) > 1: - label = token[:-1] - break - flags.append(token) - return cls(cidr, name, family, index, scope, flags, label) - - def __str__(self) -> str: - return f'{self.__class__.__name__}({self.cidr}, dev={self.name})' - - - TR = TypeVar('TR', bound='V6Route') - - - class V6Route: - def __init__(self, destination: str, dev: Optional[str] = None, proto: Optional[str] = None, metric: Optional[int] = None, pref: Optional[str] = None, via: Optional[str] = None, **kwargs) -> None: - self.destination: str = destination - self.via: Optional[str] = via - self.dev: Optional[str] = dev - self.proto: Optional[str] = proto - self.metric: Optional[int] = metric - self.pref: Optional[str] = pref - - @classmethod - def from_line(cls: Type[TR], line: str) -> TR: - items = [i for i in line.split() if i != '\\'] - dest = items[0] - if dest == 'default': - dest = '::/0' - attrs = dict(itertools.zip_longest(*[iter(items[1:])]*2, fillvalue=None)) - attrs['destination'] = dest - return cls(**attrs) - - def __str__(self) -> str: - return f'{self.__class__.__name__}({self.destination}, dev={self.dev})' - - - SUBNET_MASK_LEN = { - 'inet': 32, - 'inet6': 128 - } - - - def ntoa(family: str, num: int) -> str: - if family == 'inet': - result = socket.inet_ntoa(struct.pack("!I", num)) - else: - lo_half = num & 0xFFFFFFFFFFFFFFFF - hi_half = num >> 64 - result = socket.inet_ntop(socket.AF_INET6, - struct.pack(">QQ", hi_half, lo_half)) - return result - - - def aton(family: str, rep: str) -> int: - if family == 'inet': - result = struct.unpack("!I", socket.inet_aton(rep))[0] - else: - hi_half, lo_half = struct.unpack(">QQ", socket.inet_pton(socket.AF_INET6, rep)) - result = (hi_half << 64) | lo_half - return result - - - def addr_subnet_int_min_max(addr: Address) -> Tuple[int, int]: - ip_addr, prefix = addr.cidr.split('/') - ip_int = aton(addr.family, ip_addr) - - prefix_int = int(prefix) - mask = int('1' * prefix_int + - '0' * (SUBNET_MASK_LEN[addr.family] - prefix_int), 2) - - subnet_ip_int_min = ip_int & mask - - remainder = '1' * (SUBNET_MASK_LEN[addr.family] - prefix_int) - subnet_ip_int_max = subnet_ip_int_min | ( - 0 if remainder == '' else int(remainder, 2)) - return subnet_ip_int_min, subnet_ip_int_max - - - def vip_subnet_and_addrs_in_it(vip: str, addrs: List[Address]) -> Tuple[Address, List[Address]]: - try: - vip_int = aton('inet', vip) - except Exception: - vip_int = aton('inet6', vip) - subnet = None - candidates = [] - for addr in addrs: - subnet_ip_int_min, subnet_ip_int_max = addr_subnet_int_min_max(addr) - subnet_ip = ntoa(addr.family, subnet_ip_int_min) - subnet_ip_max = ntoa(addr.family, subnet_ip_int_max) - - sys.stderr.write('Is %s between %s and %s\n' % - (vip, subnet_ip, subnet_ip_max)) - if subnet_ip_int_min < vip_int < subnet_ip_int_max: - subnet_ip = ntoa(addr.family, subnet_ip_int_min) - subnet = Address(name="subnet", - cidr='%s/%s' % (subnet_ip, addr.cidr.split('/')[1]), - family=addr.family, - scope='') - candidates.append(addr) - if subnet is None: - raise SubnetNotFoundException() - return subnet, candidates - - - def interface_addrs(filters: Optional[Iterable[Callable[[Address], bool]]] = None) -> Iterator[Address]: - out = subprocess.check_output(["ip", "-o", "addr", "show"], encoding=sys.stdout.encoding) - for addr in (Address.from_line(line) for line in out.splitlines()): - if not filters or all(f(addr) for f in filters): - if (addr.family == 'inet6' and - int(addr.cidr.split('/')[1]) == SUBNET_MASK_LEN[addr.family]): - route_out = subprocess.check_output(["ip", "-o", "-6", "route", "show"], - encoding=sys.stdout.encoding) - for route in (V6Route.from_line(rline) for rline in route_out.splitlines()): - if (route.dev == addr.name and route.proto == 'ra' and - route.destination != '::/0'): - sys.stderr.write('Checking %s for %s\n' % (route, addr)) - route_net = Address(name=route.dev, cidr=route.destination, family='inet6') - route_filter = in_subnet(route_net) - if route_filter(addr): - ip_addr = addr.cidr.split('/')[0] - route_prefix = route_net.cidr.split('/')[1] - cidr = '%s/%s' % (ip_addr, route_prefix) - yield Address(cidr=cidr, - family=addr.family, - name=addr.name) - yield addr - - - def non_host_scope(addr: Address) -> bool: - if addr.scope == 'host': - sys.stderr.write(f'Filtering out {addr} due to it having host scope\n') - res = False - else: - res = True - return res - - - def non_deprecated(addr: Address) -> bool: - if 'deprecated' in addr.flags: - sys.stderr.write(f'Filtering out {addr} due to it being deprecated\n') - res = False - else: - res = True - return res - - - def non_secondary(addr: Address) -> bool: - if 'secondary' in addr.flags: - sys.stderr.write(f'Filtering out {addr} due to it being secondary\n') - res = False - else: - res = True - return res - - - def in_subnet(subnet: Address) -> Callable[[Address], bool]: - subnet_ip_int_min, subnet_ip_int_max = addr_subnet_int_min_max(subnet) - - def filt(addr: Address) -> bool: - ip_addr, _ = addr.cidr.split('/') - ip_int = aton(addr.family, ip_addr) - return subnet_ip_int_min < ip_int < subnet_ip_int_max - return filt - - - def main() -> None: - api_vip = sys.argv[1] - vips = set(sys.argv[1:4]) - filters = (non_host_scope, non_deprecated, non_secondary) - iface_addrs = list(interface_addrs(filters)) - try: - subnet, candidates = vip_subnet_and_addrs_in_it(api_vip, iface_addrs) - sys.stderr.write('VIP Subnet %s\n' % subnet.cidr) - - for addr in candidates: - ip_addr, _ = addr.cidr.split('/') - if ip_addr not in vips: - print(ip_addr) - sys.exit(0) - except SubnetNotFoundException: - sys.exit(1) - - - if __name__ == '__main__': - main() diff --git a/templates/common/vsphere/units/nodeip-configuration.service.yaml b/templates/common/vsphere/units/nodeip-configuration.service.yaml index e2e7b0d67f..94cd0287d9 100644 --- a/templates/common/vsphere/units/nodeip-configuration.service.yaml +++ b/templates/common/vsphere/units/nodeip-configuration.service.yaml @@ -18,7 +18,15 @@ contents: | [Service] # Need oneshot to delay kubelet Type=oneshot - ExecStart=/usr/local/bin/nodeip-finder --retry-on-failure {{.Infra.Status.PlatformStatus.VSphere.APIServerInternalIP }} + ExecStart=/usr/bin/podman run --rm \ + --authfile /var/lib/kubelet/config.json \ + --net=host \ + --volume /etc/systemd/system:/etc/systemd/system:z \ + {{ .Images.baremetalRuntimeCfgImage }} \ + node-ip \ + set --retry-on-failure \ + {{.Infra.Status.PlatformStatus.VSphere.APIServerInternalIP }} + ExecStart=/bin/systemctl daemon-reload [Install] WantedBy=multi-user.target @@ -27,4 +35,3 @@ contents: | {{ end -}} {{ end -}} {{ end -}} - diff --git a/templates/master/00-master/vsphere/files/NetworkManager-resolv-prepender.yaml b/templates/master/00-master/vsphere/files/NetworkManager-resolv-prepender.yaml index 60818fc1ac..3b46c2e8f6 100644 --- a/templates/master/00-master/vsphere/files/NetworkManager-resolv-prepender.yaml +++ b/templates/master/00-master/vsphere/files/NetworkManager-resolv-prepender.yaml @@ -20,7 +20,12 @@ contents: # Ensure resolv.conf exists before we try to run podman cp /var/run/NetworkManager/resolv.conf /etc/resolv.conf - NAMESERVER_IP=$(/usr/local/bin/non_virtual_ip \ + NAMESERVER_IP=$(/usr/bin/podman run --rm \ + --authfile /var/lib/kubelet/config.json \ + --net=host \ + {{ .Images.baremetalRuntimeCfgImage }} \ + node-ip \ + show \ "{{.Infra.Status.PlatformStatus.VSphere.APIServerInternalIP}}" \ "{{.Infra.Status.PlatformStatus.VSphere.IngressIP}}") DOMAIN="{{.EtcdDiscoveryDomain}}" diff --git a/templates/worker/00-worker/ovirt/files/ovirt-non-virtual-ip.yaml b/templates/worker/00-worker/ovirt/files/ovirt-non-virtual-ip.yaml deleted file mode 100644 index 92e6414933..0000000000 --- a/templates/worker/00-worker/ovirt/files/ovirt-non-virtual-ip.yaml +++ /dev/null @@ -1,137 +0,0 @@ -mode: 0755 -path: "/usr/local/bin/non_virtual_ip" -contents: - inline: | - #!/usr/libexec/platform-python - import collections - import socket - import struct - import subprocess - import sys - - - class SubnetNotFoundException(Exception): - """ - Exception raised when no subnet in the systems ifaces is on the VIP subnet - """ - pass - - - Address = collections.namedtuple('Address', 'index name family cidr scope') - - SUBNET_MASK_LEN = { - 'inet': 32, - 'inet6': 128 - } - - - def ntoa(family, num): - if family == 'inet': - result = socket.inet_ntoa(struct.pack("!I", num)) - else: - lo_half = num & 0xFFFFFFFFFFFFFFFF - hi_half = num >> 64 - result = socket.inet_ntop(socket.AF_INET6, - struct.pack(">QQ", hi_half, lo_half)) - return result - - - def aton(family, rep): - if family == 'inet': - result = struct.unpack("!I", socket.inet_aton(rep))[0] - else: - hi_half, lo_half = struct.unpack(">QQ", socket.inet_pton(socket.AF_INET6, rep)) - result = (hi_half << 64) | lo_half - return result - - - def addr_subnet_int_min_max(addr): - ip, prefix = addr.cidr.split('/') - ip_int = aton(addr.family, ip) - - prefix_int = int(prefix) - mask = int('1' * prefix_int + - '0' * (SUBNET_MASK_LEN[addr.family] - prefix_int), 2) - - subnet_ip_int_min = ip_int & mask - - remainder = '1' * (SUBNET_MASK_LEN[addr.family] - prefix_int) - subnet_ip_int_max = subnet_ip_int_min | ( - 0 if remainder == '' else int(remainder, 2)) - return subnet_ip_int_min, subnet_ip_int_max - - - def vip_subnet_cidr(vip, addrs): - try: - vip_int = aton('inet', vip) - except Exception: - vip_int = aton('inet6', vip) - for addr in addrs: - subnet_ip_int_min, subnet_ip_int_max = addr_subnet_int_min_max(addr) - subnet_ip = ntoa(addr.family, subnet_ip_int_min) - subnet_ip_max = ntoa(addr.family, subnet_ip_int_max) - - sys.stderr.write('Is %s between %s and %s\n' % - (vip, subnet_ip, subnet_ip_max)) - if subnet_ip_int_min < vip_int < subnet_ip_int_max: - subnet_ip = socket.inet_ntoa(struct.pack("!I", subnet_ip_int_min)) - return Address(index="-1", - name="subnet", - cidr='%s/%s' % (subnet_ip, addr.cidr.split('/')[1]), - family=addr.family, - scope='') - raise SubnetNotFoundException() - - - def line_to_address(line): - spl = line.split() - return Address(index=spl[0][:-1], - name=spl[1], - family=spl[2], - cidr=spl[3], - scope=spl[5]) - - - def interface_cidrs(filter_func=None): - try: - out = subprocess.check_output(["ip", "-o", "addr", "show"], encoding=sys.stdout.encoding) - except TypeError: # python2 - out = subprocess.check_output(["ip", "-o", "addr", "show"]) - for addr in (line_to_address(line) for line in out.splitlines()): - if filter_func(addr): - yield addr - - - def non_host_scope(addr): - return addr.scope != 'host' - - - def in_subnet(subnet): - subnet_ip_int_min, subnet_ip_int_max = addr_subnet_int_min_max(subnet) - - def filt(addr): - ip, _ = addr.cidr.split('/') - ip_int = aton(addr.family, ip) - return subnet_ip_int_min < ip_int < subnet_ip_int_max - return filt - - - def main(): - api_vip, dns_vip, ingress_vip = sys.argv[1:4] - vips = set(sys.argv[1:4]) - iface_cidrs = list(interface_cidrs(non_host_scope)) - try: - subnet = vip_subnet_cidr(api_vip, iface_cidrs) - - sys.stderr.write('VIP Subnet %s\n' % subnet.cidr) - - for addr in interface_cidrs(in_subnet(subnet)): - ip, prefix = addr.cidr.split('/') - if ip not in vips: - print(addr.cidr.split('/')[0]) - sys.exit(0) - except SubnetNotFoundException: - sys.exit(1) - - if __name__ == '__main__': - main() diff --git a/templates/worker/00-worker/vsphere/files/NetworkManager-resolv-prepender.yaml b/templates/worker/00-worker/vsphere/files/NetworkManager-resolv-prepender.yaml index 5c20ead069..3d677f499d 100644 --- a/templates/worker/00-worker/vsphere/files/NetworkManager-resolv-prepender.yaml +++ b/templates/worker/00-worker/vsphere/files/NetworkManager-resolv-prepender.yaml @@ -15,10 +15,15 @@ contents: case "$STATUS" in up|down|dhcp4-change|dhcp6-change) logger -s "NM resolv-prepender triggered by ${1} ${2}." - NAMESERVER_IP=$(/usr/local/bin/non_virtual_ip \ + NAMESERVER_IP=$(/usr/bin/podman run --rm \ + --authfile /var/lib/kubelet/config.json \ + --net=host \ + {{ .Images.baremetalRuntimeCfgImage }} \ + node-ip \ + show \ "{{.Infra.Status.PlatformStatus.VSphere.APIServerInternalIP}}" \ "{{.Infra.Status.PlatformStatus.VSphere.IngressIP}}") - DOMAIN="{{.EtcdDiscoveryDomain}}" + DOMAIN="{{.Infra.Status.EtcdDiscoveryDomain}}" if [[ -n "$NAMESERVER_IP" ]]; then logger -s "NM resolv-prepender: Prepending 'nameserver $NAMESERVER_IP' to /etc/resolv.conf (other nameservers from /var/run/NetworkManager/resolv.conf)" sed -e "/^search/d" \