From 20aeed5298704eeac72c9eb2f56c45907a65eefc Mon Sep 17 00:00:00 2001 From: "Yc.S" Date: Fri, 21 Feb 2020 18:03:36 +0900 Subject: [PATCH 01/10] Support libvirtd network resources --- examples/virtd-network.nix | 28 +++ nix/default.nix | 4 +- nix/libvirtd-network.nix | 62 ++++++ nix/libvirtd.nix | 43 +++- nixopsvirtd/backends/libvirtd.py | 48 ++++- nixopsvirtd/plugin.py | 1 + nixopsvirtd/resources/__init__.py | 2 + nixopsvirtd/resources/libvirtd_network.py | 235 ++++++++++++++++++++++ release.nix | 2 +- setup.py | 2 +- 10 files changed, 415 insertions(+), 12 deletions(-) create mode 100644 examples/virtd-network.nix create mode 100644 nix/libvirtd-network.nix create mode 100644 nixopsvirtd/resources/__init__.py create mode 100644 nixopsvirtd/resources/libvirtd_network.py diff --git a/examples/virtd-network.nix b/examples/virtd-network.nix new file mode 100644 index 0000000..fa92e90 --- /dev/null +++ b/examples/virtd-network.nix @@ -0,0 +1,28 @@ +{ + resources.libvirtdNetworks.net2 = { + type = "nat"; + cidrBlock = "172.16.100.0/16"; + staticIPs = [ + { + machine = "node1"; + address = "172.16.100.12"; + } + { + machine = "node2"; + address = "172.16.100.5"; + } + ]; + }; + + node1 = { + deployment.targetEnv = "libvirtd"; + deployment.libvirtd.imageDir = "/var/lib/libvirt/images"; + deployment.libvirtd.networks = [ "net2" ]; + }; + + node2 = { + deployment.targetEnv = "libvirtd"; + deployment.libvirtd.imageDir = "/var/lib/libvirt/images"; + deployment.libvirtd.networks = [ "net2" ]; + }; +} diff --git a/nix/default.nix b/nix/default.nix index 5e4eeec..ee2ff11 100644 --- a/nix/default.nix +++ b/nix/default.nix @@ -5,5 +5,7 @@ options = [ ./libvirtd.nix ]; - resources = { ... }: {}; + resources = { evalResources, zipAttrs, resourcesByType, ...}: { + libvirtdNetworks = evalResources ./libvirtd-network.nix (zipAttrs resourcesByType.libvirtdNetworks or []); + }; } diff --git a/nix/libvirtd-network.nix b/nix/libvirtd-network.nix new file mode 100644 index 0000000..79c8019 --- /dev/null +++ b/nix/libvirtd-network.nix @@ -0,0 +1,62 @@ +{ config, lib, pkgs, uuid, name, ... }: + +with lib; +with import lib; + +rec { + options = { + type = mkOption { + default = "nat"; + description = '' + The type of the libvirt network. + Either NAT network or isolated network can be specified. Defaults to NAT Network. + ''; + type = types.enum [ "nat" "isolate" ]; + }; + + cidrBlock = mkOption { + example = "192.168.56.0/24"; + description = '' + The IPv4 CIDR block for the libvirt network. The following IP addresses are reserved for the network: + Network - The first address in the IP range, e.g. 192.168.56.0 in 192.168.56.0/24 + Gateway - The second address in the IP range, e.g. 192.168.56.1 in 192.168.56.0/24 + Broadcast - The last address in the IP range, e.g. 192.168.56.255 in 192.168.56.0/24 + ''; + type = types.str; + }; + + staticIPs = mkOption { + default = []; + description = "The list of machine to IPv4 address bindings for fixing IP address of the machine in the network"; + type = with types; listOf (submodule { + options = { + machine = mkOption { + type = either str (resource "machine"); + apply = x: if builtins.isString x then x else x._name; + description = "The name of the machine in the network"; + }; + address = mkOption { + example = "192.168.56.3"; + type = str; + description = '' + The IPv4 address assigned to the machine as static IP. + The static IP must be a non-reserved IP address. + ''; + }; + }; + }); + }; + + URI = mkOption { + type = types.str; + default = "qemu:///system"; + description = '' + Connection URI. + ''; + }; + }; + + config = { + _type = "libvirtd-network"; + }; +} diff --git a/nix/libvirtd.nix b/nix/libvirtd.nix index e956bd4..593d8c7 100644 --- a/nix/libvirtd.nix +++ b/nix/libvirtd.nix @@ -1,6 +1,7 @@ { config, pkgs, lib, ... }: with lib; +with import lib; let the_key = builtins.getEnv "NIXOPS_LIBVIRTD_PUBKEY"; @@ -80,10 +81,48 @@ in disk image as a base. ''; }; - deployment.libvirtd.networks = mkOption { default = [ "default" ]; - type = types.listOf types.str; + type = with types; addCheck (nonEmptyListOf + (either + str # for backward compatibility + (either + (resource "libvirtd-network") + (submodule { + options = { + name = mkOption { + default = ""; + description = "The name of the network not managed by NixOps"; + type = str; + }; + type = mkOption { + description = "The type of the network"; + type = enum [ "nat" "isolate" "bridge" "direct" ]; + }; + mode = mkOption { + default = "bridge"; + description = "The mode of the direct (macvtap) network"; + type = enum [ "bridge" "vepa" "private" "passthrough" ]; + }; + virtualport = mkOption { + default = null; + description = "The virtualport for specific bridge devices such as Open vSwitch"; + type = nullOr (either str (submodule { + optiones = { + type = mkOption { + description = "The type of the virtualport"; + type = str; + }; + parameters = mkOption { + description = "The parameters of the virtualport"; + type = attrset; + }; + }; + })); + }; + }; + })) + )) (l: any (n: (builtins.isString n || (n ? type && builtins.elem n.type [ "nat" "isolated" ]) )) l); description = "Names of libvirt networks to attach the VM to."; }; diff --git a/nixopsvirtd/backends/libvirtd.py b/nixopsvirtd/backends/libvirtd.py index e383b40..bd47cdf 100644 --- a/nixopsvirtd/backends/libvirtd.py +++ b/nixopsvirtd/backends/libvirtd.py @@ -42,9 +42,7 @@ def __init__(self, xml, config): self.storage_pool_name = x.find("attr[@name='storagePool']/string").get("value") self.uri = x.find("attr[@name='URI']/string").get("value") - self.networks = [ - k.get("value") - for k in x.findall("attr[@name='networks']/list/string")] + self.networks = self.config["libvirtd"]["networks"] assert len(self.networks) > 0 @@ -144,7 +142,7 @@ def _generate_primary_mac(self): def create(self, defn, check, allow_reboot, allow_recreate): assert isinstance(defn, LibvirtdDefinition) self.set_common_state(defn) - self.primary_net = defn.networks[0] + self.primary_net = next(n for n in defn.networks if isinstance(n, basestring) or (isinstance(n, dict) and n.get("type") in ["nat" "isolate"])) self.storage_pool_name = defn.storage_pool_name self.uri = defn.uri @@ -249,13 +247,49 @@ def maybe_mac(n): else: return "" + def vport(n): + v = n.get("virtualport") + + if isinstance(v, basestring) or (isinstance(v, dict) and not v.get("parameters")): + return ''.format(v.get("type") if isinstance(v, dict) else v) + + if isinstance(v, dict) and v.get("parameters"): + return ''' + + + + '''.format( + type=v.get("type"), + params=" ".join('{key}="{value}"' for key, value in v.get("parameters")) + ) + return "" + def iface(n): - return "\n".join([ + # virtual network + if isinstance(n, basestring) or (isinstance(n, dict) and n.get("type") in ["nat" "isolate"]): return "\n".join([ ' ', - maybe_mac(n), ' ', + maybe_mac(n), + ' ', + ]).format(n.get("_name", n.get("name")) if isinstance(n, dict) else n) + + # macvtap device + if isinstance(n, dict) and n.get("type") == "direct": return "\n".join([ + ' ', + ' ', + maybe_mac(n), + vport(n), + ' ', + ]).format(n.get("name"), n.get("mode")) + + # bridge + if isinstance(n, dict) and n.get("type") == "bridge": return "\n".join([ + ' ', + ' ', + maybe_mac(n), + vport(n), ' ', - ]).format(n) + ]).format(n.get("name")) def _make_os(defn): return [ diff --git a/nixopsvirtd/plugin.py b/nixopsvirtd/plugin.py index c048ec0..78f153b 100644 --- a/nixopsvirtd/plugin.py +++ b/nixopsvirtd/plugin.py @@ -16,5 +16,6 @@ def nixexprs(): @nixops.plugins.hookimpl def load(): return [ + "nixopsvirtd.resources", "nixopsvirtd.backends.libvirtd", ] diff --git a/nixopsvirtd/resources/__init__.py b/nixopsvirtd/resources/__init__.py new file mode 100644 index 0000000..de47710 --- /dev/null +++ b/nixopsvirtd/resources/__init__.py @@ -0,0 +1,2 @@ +import libvirtd_network +import __init__ diff --git a/nixopsvirtd/resources/libvirtd_network.py b/nixopsvirtd/resources/libvirtd_network.py new file mode 100644 index 0000000..99effd1 --- /dev/null +++ b/nixopsvirtd/resources/libvirtd_network.py @@ -0,0 +1,235 @@ +# -*- coding: utf-8 -*- + +# Automatic provisioning of Virtualbox Networks. + +import os +import ipaddress +import libvirt +from nixops.util import attr_property, logged_exec +from nixops.resources import ResourceDefinition, ResourceState +from nixopsvirtd.backends.libvirtd import LibvirtdDefinition, LibvirtdState + +class LibvirtdNetworkDefinition(ResourceDefinition): + """Definition of the Libvirtd Network""" + + @classmethod + def get_type(cls): + return "libvirtd-network" + + @classmethod + def get_resource_type(cls): + return "libvirtdNetworks" + + def __init__(self, xml): + ResourceDefinition.__init__(self, xml) + self.network_type = xml.find("attrs/attr[@name='type']/string").get("value") + self.network_cidr = xml.find("attrs/attr[@name='cidrBlock']/string").get("value") + + self.static_ips = { x.find("attr[@name='machine']/string").get("value"): + x.find("attr[@name='address']/string").get("value") for x in xml.findall("attrs/attr[@name='staticIPs']/list/attrs") } + + self.uri = xml.find("attrs/attr[@name='URI']/string").get("value") + + def show_type(self): + return "{0} [{1:8} {2}]".format(self.get_type(), self.network_type, self.network_cidr) + +class LibvirtdNetworkState(ResourceState): + """State of the Libvirtd Network""" + + network_name = attr_property("libvirtd.network_name", None) + network_type = attr_property("libvirtd.network_type", None) + network_cidr = attr_property("libvirtd.network_cidr", None) + static_ips = attr_property("libvirtd.static_ips", [], "json") + + uri = attr_property("libvirtd.URI", "qemu:///system") + + @classmethod + def get_type(cls): + return "libvirtd-network" + + def __init__(self, depl, name, id): + ResourceState.__init__(self, depl, name, id) + self._conn = None + self._net = None + + def show_type(self): + s = super(LibvirtdNetworkState, self).show_type() + if self.state == self.UP: s = "{0} [{1}]".format(s, self.network_type) + return s + + @property + def resource_id(self): + return self.network_name + + @property + def public_ipv4(self): + return self.network_cidr if self.state == self.UP else None; + + nix_name = "libvirtdNetworks" + + @property + def full_name(self): + return "Libvirtd network '{}'".format(self.name) + + @property + def conn(self): + if self._conn is None: + self.logger.log('Connecting to {}...'.format(self.uri)) + try: + self._conn = libvirt.open(self.uri) + except libvirt.libvirtError as error: + self.logger.error(error.get_error_message()) + if error.get_error_code() == libvirt.VIR_ERR_NO_CONNECT: + # this error code usually means "no connection driver available for qemu:///..." + self.logger.error('make sure qemu-system-x86_64 is installed on the target host') + raise Exception('Failed to connect to the hypervisor at {}'.format(self.uri)) + return self._conn + + @property + def net(self): + if self._net is None: + try: + self._net = self.conn.networkLookupByName(self.name) + except Exception as e: + self.log("Warning: %s" % e) + return self._net + + def create(self, defn, check, allow_reboot, allow_recreate): + assert isinstance(defn, LibvirtdNetworkDefinition) + + if check: self.check() + + subnet = ipaddress.ip_network(unicode(defn.network_cidr), strict=False) + + if self.state != self.UP: + self.log("Creating {}...".format(self.full_name)) + self.network_type = defn.network_type + self.network_cidr = defn.network_cidr + self.static_ips = defn.static_ips + self.uri = defn.uri + + self._net = self.conn.networkDefineXML(''' + + {name} + {maybeForwardTag} + + + + {dhcpHosts} + + + + '''.format( + name=self.name, + maybeForwardTag='' if defn.network_type == "nat" else "", + gateway=subnet[1], + netmask=subnet.netmask, + lowerip=subnet[2], + upperip=subnet[-2], + dhcpHosts="".join("".format( + name=machine, + ip=address, + ) for machine, address in defn.static_ips.iteritems()) + )) + + self.net.create() + self.net.setAutostart(1) + self.network_name = self.net.bridgeName() + self.state = self.UP + return + + if self._can_update(defn, allow_reboot, allow_recreate): + self.log("Updating {}...".format(self.full_name)) + + if self.network_type != defn.network_type: + self.conn.networkUpdate( + libvirt.VIR_NETWORK_UPDATE_COMMAND_ADD if defn.network_type == "nat" else libvirt.VIR_NETWORK_UPDATE_COMMAND_DELETE, + libvirt.VIR_NETWORK_SECTION_FORWARD, + -1, + '' + ) + self.network_type = defn.network_type + + if self.static_ips != defn.static_ips: + # Remove obsolete + for machine, address in self.static_ips.iteritems(): + if not defn.static_ips.get(machine): + self.net.update( + libvirt.VIR_NETWORK_UPDATE_COMMAND_DELETE, + libvirt.VIR_NETWORK_SECTION_IP_DHCP_HOST, + -1, + "".format( + name=machine, + ip=address, + ) + ) + + # Add or update existings + for machine, address in defn.static_ips.iteritems(): + mstate = self.depl.resources.get(machine) + mdefn = self.depl.definitions.get(machine) + if isinstance(mstate, LibvirtdState) and isinstance(mdefn, LibvirtdDefinition): + for net in mdefn.config["libvirtd"]["networks"]: + if net == defn.name or (net.get("_name", net.get("name")) == defn.name and net.get("type") == defn.network_type): + try: + if ipaddress.ip_address(unicode(address)) not in subnet: + raise Exception("cannot assign a static IP out of the network CIDR") + self.net.update( + libvirt.VIR_NETWORK_UPDATE_COMMAND_MODIFY if self.static_ips.get(machine) else libvirt.VIR_NETWORK_UPDATE_COMMAND_ADD_LAST, + libvirt.VIR_NETWORK_SECTION_IP_DHCP_HOST, + -1, + "".format( + name=machine, + ip=address, + ) + ) + except Exception as e: + print(e) + self.warn("Cannot assign static IP '{0}' to machine '{1}' in subnet '{2}'".format(address, machine, defn.network_cidr)) + break; + else: + self.warn("Cannot assign static IP '{0}' to non-attached machine '{1}'".format(address, machine)) + else: + self.warn("Cannot assign static IP '{0}' to non-existent machine '{1}'".format(address, machine)) + + self.static_ips = defn.static_ips + + def _can_update(self, defn, allow_reboot, allow_recreate): + if self.uri != defn.uri: + self.warn("Change of the connection URI from {0} to {1} is not supported; skipping".format(self.uri, defn.uri)) + return False + + if self.network_cidr != defn.network_cidr: + self.warn("Change of the network CIDR from {0} to {1} is not supported; skipping".format(self.network_cidr, defn.network_cidr)) + return False + + # checkme: the state of the attached machine should also be considered + if any(defn.static_ips.get(machine) != address for machine, address in self.static_ips.iteritems()) and not allow_reboot: + self.warn("change of existing bindings for static IPs requires reboot; skipping") + return False + + return True + + def destroy(self, wipe=False): + if self.state != self.UP or not self.net: return True + if not self.depl.logger.confirm("are you sure you want to destroy {}?".format(self.full_name)): + return False + + self.log("destroying {}...".format(self.full_name)) + + self.net.destroy() + self.net.undefine() + + return True + + def _check(self): + if self.net: + if self.network_name != self.net.bridgeName(): + self.network_name = self.net.bridgeName() + return super(LibvirtdNetworkState, self)._check() + + with self.depl._db: + self.network_name = None + self.state = self.MISSING + + return False diff --git a/release.nix b/release.nix index 6946898..eb62eb9 100644 --- a/release.nix +++ b/release.nix @@ -18,7 +18,7 @@ in done ''; buildInputs = [ python2Packages.nose python2Packages.coverage ]; - propagatedBuildInputs = [ python2Packages.libvirt ]; + propagatedBuildInputs = with python2Packages; [ libvirt ipaddress ]; doCheck = true; postInstall = '' mkdir -p $out/share/nix/nixops-libvirtd diff --git a/setup.py b/setup.py index 7fd02b2..42dcb54 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ url='https://github.com/AmineChikhaoui/nixops-libvirtd', maintainer='Amine Chikhaoui', maintainer_email='amine.chikhaoui91@gmail.com', - packages=['nixopsvirtd', 'nixopsvirtd.backends'], + packages=['nixopsvirtd', 'nixopsvirtd.backends', 'nixopsvirtd.resources'], entry_points={'nixops': ['virtd = nixopsvirtd.plugin']}, py_modules=['plugin'] ) From 3547a30dc765e20c291a29f68b2f8586e80022dc Mon Sep 17 00:00:00 2001 From: "Yc.S" Date: Mon, 2 Mar 2020 13:55:02 +0900 Subject: [PATCH 02/10] Allow define staticIPs in a compact form using attrset --- nix/libvirtd-network.nix | 24 ++++++++++++++-- nixopsvirtd/resources/libvirtd_network.py | 35 +++++++++++++++-------- 2 files changed, 44 insertions(+), 15 deletions(-) diff --git a/nix/libvirtd-network.nix b/nix/libvirtd-network.nix index 79c8019..e7afa7c 100644 --- a/nix/libvirtd-network.nix +++ b/nix/libvirtd-network.nix @@ -3,6 +3,9 @@ with lib; with import lib; +let + toMachineName = m: if builtins.isString m then m else m._name; +in rec { options = { type = mkOption { @@ -26,13 +29,28 @@ rec { }; staticIPs = mkOption { + example = '' + # As an attrset + { + "192.168.56.10" = "node1"; + "192.168.56.11" = "node2"; + ... + } + # Or as a list + [ + { address = "192.168.56.10"; machine = "node1"; } + { address = "192.168.56.11"; machine = "node2"; } + ... + ] + ''; default = []; description = "The list of machine to IPv4 address bindings for fixing IP address of the machine in the network"; - type = with types; listOf (submodule { + apply = a: if builtins.isAttrs a then mapAttrs (k: toMachineName) a else a; + type = with types; either attrs (listOf (submodule { options = { machine = mkOption { type = either str (resource "machine"); - apply = x: if builtins.isString x then x else x._name; + apply = toMachineName; description = "The name of the machine in the network"; }; address = mkOption { @@ -44,7 +62,7 @@ rec { ''; }; }; - }); + })); }; URI = mkOption { diff --git a/nixopsvirtd/resources/libvirtd_network.py b/nixopsvirtd/resources/libvirtd_network.py index 99effd1..e81e5a7 100644 --- a/nixopsvirtd/resources/libvirtd_network.py +++ b/nixopsvirtd/resources/libvirtd_network.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Automatic provisioning of Virtualbox Networks. +# Automatic provisioning of Libvirt Virtual Networks. import os import ipaddress @@ -27,11 +27,15 @@ def __init__(self, xml): self.static_ips = { x.find("attr[@name='machine']/string").get("value"): x.find("attr[@name='address']/string").get("value") for x in xml.findall("attrs/attr[@name='staticIPs']/list/attrs") } + self.static_ips.update({ + x.find("string").get("value"): x.get("name") + for x in xml.findall("attrs/attr[@name='staticIPs']/attrs/attr") + }) self.uri = xml.find("attrs/attr[@name='URI']/string").get("value") def show_type(self): - return "{0} [{1:8} {2}]".format(self.get_type(), self.network_type, self.network_cidr) + return "{0} [{1} {2}]".format(self.get_type(), self.network_type, self.network_cidr) class LibvirtdNetworkState(ResourceState): """State of the Libvirtd Network""" @@ -39,7 +43,7 @@ class LibvirtdNetworkState(ResourceState): network_name = attr_property("libvirtd.network_name", None) network_type = attr_property("libvirtd.network_type", None) network_cidr = attr_property("libvirtd.network_cidr", None) - static_ips = attr_property("libvirtd.static_ips", [], "json") + static_ips = attr_property("libvirtd.static_ips", {}, "json") uri = attr_property("libvirtd.URI", "qemu:///system") @@ -102,7 +106,7 @@ def create(self, defn, check, allow_reboot, allow_recreate): subnet = ipaddress.ip_network(unicode(defn.network_cidr), strict=False) if self.state != self.UP: - self.log("Creating {}...".format(self.full_name)) + self.log("creating {}...".format(self.full_name)) self.network_type = defn.network_type self.network_cidr = defn.network_cidr self.static_ips = defn.static_ips @@ -111,7 +115,7 @@ def create(self, defn, check, allow_reboot, allow_recreate): self._net = self.conn.networkDefineXML(''' {name} - {maybeForwardTag} + {maybeForward} @@ -121,12 +125,12 @@ def create(self, defn, check, allow_reboot, allow_recreate): '''.format( name=self.name, - maybeForwardTag='' if defn.network_type == "nat" else "", + maybeForward='' if defn.network_type == "nat" else "", gateway=subnet[1], netmask=subnet.netmask, lowerip=subnet[2], upperip=subnet[-2], - dhcpHosts="".join("".format( + dhcpHosts="".join("".format( name=machine, ip=address, ) for machine, address in defn.static_ips.iteritems()) @@ -138,8 +142,8 @@ def create(self, defn, check, allow_reboot, allow_recreate): self.state = self.UP return - if self._can_update(defn, allow_reboot, allow_recreate): - self.log("Updating {}...".format(self.full_name)) + if self._need_update(defn, allow_reboot, allow_recreate): + self.log("updating {}...".format(self.full_name)) if self.network_type != defn.network_type: self.conn.networkUpdate( @@ -158,7 +162,7 @@ def create(self, defn, check, allow_reboot, allow_recreate): libvirt.VIR_NETWORK_UPDATE_COMMAND_DELETE, libvirt.VIR_NETWORK_SECTION_IP_DHCP_HOST, -1, - "".format( + "".format( name=machine, ip=address, ) @@ -178,7 +182,7 @@ def create(self, defn, check, allow_reboot, allow_recreate): libvirt.VIR_NETWORK_UPDATE_COMMAND_MODIFY if self.static_ips.get(machine) else libvirt.VIR_NETWORK_UPDATE_COMMAND_ADD_LAST, libvirt.VIR_NETWORK_SECTION_IP_DHCP_HOST, -1, - "".format( + "".format( name=machine, ip=address, ) @@ -194,7 +198,7 @@ def create(self, defn, check, allow_reboot, allow_recreate): self.static_ips = defn.static_ips - def _can_update(self, defn, allow_reboot, allow_recreate): + def _need_update(self, defn, allow_reboot, allow_recreate): if self.uri != defn.uri: self.warn("Change of the connection URI from {0} to {1} is not supported; skipping".format(self.uri, defn.uri)) return False @@ -203,6 +207,13 @@ def _can_update(self, defn, allow_reboot, allow_recreate): self.warn("Change of the network CIDR from {0} to {1} is not supported; skipping".format(self.network_cidr, defn.network_cidr)) return False + if self.network_type == defn.network_type and self.static_ips == defn.static_ips: # no changes + return False + + if self.network_type != defn.network_type and not allow_reboot: + self.warn("change of the network type requires reboot; skipping") + return False + # checkme: the state of the attached machine should also be considered if any(defn.static_ips.get(machine) != address for machine, address in self.static_ips.iteritems()) and not allow_reboot: self.warn("change of existing bindings for static IPs requires reboot; skipping") From 84ed5cfee1bf19a947045716d22c36fd49998b39 Mon Sep 17 00:00:00 2001 From: "Yc.S" Date: Mon, 2 Mar 2020 13:56:51 +0900 Subject: [PATCH 03/10] Update changed network interfaces if reboot is allowed --- nixopsvirtd/backends/libvirtd.py | 156 ++++++++++++++++++++----------- 1 file changed, 102 insertions(+), 54 deletions(-) diff --git a/nixopsvirtd/backends/libvirtd.py b/nixopsvirtd/backends/libvirtd.py index bd47cdf..dcd2a56 100644 --- a/nixopsvirtd/backends/libvirtd.py +++ b/nixopsvirtd/backends/libvirtd.py @@ -57,6 +57,7 @@ class LibvirtdState(MachineState): storage_volume_name = nixops.util.attr_property("libvirtd.storageVolume", None) storage_pool_name = nixops.util.attr_property("libvirtd.storagePool", None) vcpu = nixops.util.attr_property("libvirtd.vcpu", None) + networks = nixops.util.attr_property("libvirtd.networks", [], "json") # older deployments may not have a libvirtd.URI attribute in the state file # using qemu:///system in such case @@ -132,6 +133,16 @@ def address_to(self, m): def _vm_id(self): return "nixops-{0}-{1}".format(self.depl.uuid, self.name) + def _get_network_name(self, n): + if isinstance(n, basestring): return n + if isinstance(n, dict): return n.get("_name", n.get("name")) + + def _get_primary_net(self): + assert len(self.networks) > 0 + return next(self._get_network_name(n) for n in self.networks + if isinstance(n, basestring) or (isinstance(n, dict) and n.get("type") in ["nat", "isolate"]) + ) + def _generate_primary_mac(self): mac = [0x52, 0x54, 0x00, random.randint(0x00, 0x7f), @@ -142,7 +153,6 @@ def _generate_primary_mac(self): def create(self, defn, check, allow_reboot, allow_recreate): assert isinstance(defn, LibvirtdDefinition) self.set_common_state(defn) - self.primary_net = next(n for n in defn.networks if isinstance(n, basestring) or (isinstance(n, dict) and n.get("type") in ["nat" "isolate"])) self.storage_pool_name = defn.storage_pool_name self.uri = defn.uri @@ -161,9 +171,11 @@ def create(self, defn, check, allow_reboot, allow_recreate): self._prepare_storage_volume() self.storage_volume_name = self.vol.name() - self.domain_xml = self._make_domain_xml(defn) - if self.vm_id is None: + self.networks = defn.networks + self.primary_net = self._get_primary_net() + self.domain_xml = self._make_domain_xml(defn) + # By using "define" we ensure that the domain is # "persistent", as opposed to "transient" (i.e. removed on reboot). self._dom = self.conn.defineXML(self.domain_xml) @@ -173,6 +185,27 @@ def create(self, defn, check, allow_reboot, allow_recreate): self.vm_id = self._vm_id() + # Update networks for redeployment + if self.networks != defn.networks: + if not allow_reboot: + self.warn("change of the networks requires reboot; skipping") + else: + self.log('update networks...') + self.stop() + + for n in self.networks: + try: + self.dom.detachDeviceFlags(self._make_iface(n), libvirt.VIR_DOMAIN_AFFECT_CONFIG) + except: + pass + + self.networks = defn.networks + self.primary_net = self._get_primary_net() + self.domain_xml = self._make_domain_xml(defn) + + for n in defn.networks: + self.dom.attachDeviceFlags(self._make_iface(n), libvirt.VIR_DOMAIN_AFFECT_CONFIG) + self.start() return True @@ -241,56 +274,6 @@ def _get_qemu_executable(self): def _make_domain_xml(self, defn): qemu = self._get_qemu_executable() - def maybe_mac(n): - if n == self.primary_net: - return '' - else: - return "" - - def vport(n): - v = n.get("virtualport") - - if isinstance(v, basestring) or (isinstance(v, dict) and not v.get("parameters")): - return ''.format(v.get("type") if isinstance(v, dict) else v) - - if isinstance(v, dict) and v.get("parameters"): - return ''' - - - - '''.format( - type=v.get("type"), - params=" ".join('{key}="{value}"' for key, value in v.get("parameters")) - ) - return "" - - def iface(n): - # virtual network - if isinstance(n, basestring) or (isinstance(n, dict) and n.get("type") in ["nat" "isolate"]): return "\n".join([ - ' ', - ' ', - maybe_mac(n), - ' ', - ]).format(n.get("_name", n.get("name")) if isinstance(n, dict) else n) - - # macvtap device - if isinstance(n, dict) and n.get("type") == "direct": return "\n".join([ - ' ', - ' ', - maybe_mac(n), - vport(n), - ' ', - ]).format(n.get("name"), n.get("mode")) - - # bridge - if isinstance(n, dict) and n.get("type") == "bridge": return "\n".join([ - ' ', - ' ', - maybe_mac(n), - vport(n), - ' ', - ]).format(n.get("name")) - def _make_os(defn): return [ '', @@ -313,7 +296,7 @@ def _make_os(defn): ' ', ' ', ' ', - '\n'.join([iface(n) for n in defn.networks]), + '\n'.join([self._make_iface(n) for n in defn.networks]), ' ' if not defn.headless else "", ' ', ' ', @@ -332,6 +315,71 @@ def _make_os(defn): defn.domain_type ) + def _make_iface(self, n): + def maybe_mac(n): + if self._get_network_name(n) == self.primary_net: + return '' + else: + return "" + + def virt_port(n): + v = n.get("virtualport") + + if isinstance(v, basestring) or (isinstance(v, dict) and not v.get("parameters")): + return ''.format(v.get("type") if isinstance(v, dict) else v) + + if isinstance(v, dict) and v.get("parameters"): + return ''' + + + + '''.format( + type=v.get("type"), + params=" ".join('{key}="{value}"' for key, value in v.get("parameters")) + ) + return "" + + # virtual network + if isinstance(n, basestring) or (isinstance(n, dict) and n.get("type") in ["nat", "isolate"]): + return ''' + + + {mac} + + '''.format( + name=self._get_network_name(n), + mac=maybe_mac(n) + ) + + # macvtap device + if isinstance(n, dict) and n.get("type") == "direct": + return ''' + + + {mac} + {vport} + + '''.format( + name=n.get("name"), + mode=n.get("mode"), + mac=maybe_mac(n), + vport=virt_port(n), + ) + + # bridge + if isinstance(n, dict) and n.get("type") == "bridge": + return ''' + + + {mac} + {vport} + + '''.format( + name=n.get("name"), + mac=maybe_mac(n), + vport=virt_port(n), + ) + def _parse_ip(self): """ return an ip v4 From d6e268a90e38778ac2451ec9cea6fbd1500b870c Mon Sep 17 00:00:00 2001 From: "Yc.S" Date: Mon, 2 Mar 2020 16:09:35 +0900 Subject: [PATCH 04/10] Fix wrong indentations --- nixopsvirtd/resources/libvirtd_network.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/nixopsvirtd/resources/libvirtd_network.py b/nixopsvirtd/resources/libvirtd_network.py index e81e5a7..0e11c82 100644 --- a/nixopsvirtd/resources/libvirtd_network.py +++ b/nixopsvirtd/resources/libvirtd_network.py @@ -186,15 +186,16 @@ def create(self, defn, check, allow_reboot, allow_recreate): name=machine, ip=address, ) + , + libvirt.VIR_NETWORK_UPDATE_AFFECT_CONFIG | libvirt.VIR_NETWORK_UPDATE_AFFECT_LIVE ) except Exception as e: - print(e) self.warn("Cannot assign static IP '{0}' to machine '{1}' in subnet '{2}'".format(address, machine, defn.network_cidr)) - break; + break; + else: + self.warn("Cannot assign static IP '{0}' to non-attached machine '{1}'".format(address, machine)) else: - self.warn("Cannot assign static IP '{0}' to non-attached machine '{1}'".format(address, machine)) - else: - self.warn("Cannot assign static IP '{0}' to non-existent machine '{1}'".format(address, machine)) + self.warn("Cannot assign static IP '{0}' to non-existent machine '{1}'".format(address, machine)) self.static_ips = defn.static_ips From 1b7599ef673b66f7f439ee2590371cc0fdea122f Mon Sep 17 00:00:00 2001 From: "Yc.S" Date: Mon, 2 Mar 2020 16:15:20 +0900 Subject: [PATCH 05/10] Add more variations in examples --- examples/virtd-network.nix | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/examples/virtd-network.nix b/examples/virtd-network.nix index fa92e90..b099a09 100644 --- a/examples/virtd-network.nix +++ b/examples/virtd-network.nix @@ -17,12 +17,19 @@ node1 = { deployment.targetEnv = "libvirtd"; deployment.libvirtd.imageDir = "/var/lib/libvirt/images"; - deployment.libvirtd.networks = [ "net2" ]; + deployment.libvirtd.networks = [ + "net2" + # { + # name = "ovsbr0"; + # type = "bridge"; + # virtualport = "openvswitch"; + # } + ]; }; - node2 = { + node2 = {resources, ...}: { deployment.targetEnv = "libvirtd"; deployment.libvirtd.imageDir = "/var/lib/libvirt/images"; - deployment.libvirtd.networks = [ "net2" ]; + deployment.libvirtd.networks = [ resources.libvirtdNetworks.net2 ]; }; } From 1178e085081afa1349e555db200d94bd51ca8988 Mon Sep 17 00:00:00 2001 From: "Yc.S" Date: Tue, 10 Mar 2020 11:13:48 +0900 Subject: [PATCH 06/10] Revert breaking changes for determining primary network interface It fails deploying to a remote libvirtd if the primary network is limited to be a NAT or isolated network. Also in order to keep backward compatibility, it is better to keep the first interface as primary. --- nix/libvirtd.nix | 4 ++-- nixopsvirtd/backends/libvirtd.py | 4 +--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/nix/libvirtd.nix b/nix/libvirtd.nix index 593d8c7..bb67803 100644 --- a/nix/libvirtd.nix +++ b/nix/libvirtd.nix @@ -83,7 +83,7 @@ in }; deployment.libvirtd.networks = mkOption { default = [ "default" ]; - type = with types; addCheck (nonEmptyListOf + type = with types; nonEmptyListOf (either str # for backward compatibility (either @@ -122,7 +122,7 @@ in }; }; })) - )) (l: any (n: (builtins.isString n || (n ? type && builtins.elem n.type [ "nat" "isolated" ]) )) l); + ); description = "Names of libvirt networks to attach the VM to."; }; diff --git a/nixopsvirtd/backends/libvirtd.py b/nixopsvirtd/backends/libvirtd.py index dcd2a56..826a709 100644 --- a/nixopsvirtd/backends/libvirtd.py +++ b/nixopsvirtd/backends/libvirtd.py @@ -139,9 +139,7 @@ def _get_network_name(self, n): def _get_primary_net(self): assert len(self.networks) > 0 - return next(self._get_network_name(n) for n in self.networks - if isinstance(n, basestring) or (isinstance(n, dict) and n.get("type") in ["nat", "isolate"]) - ) + return self._get_network_name(self.networks[0]) def _generate_primary_mac(self): mac = [0x52, 0x54, 0x00, From 992b06a699faa2ae51f82ecab826db3ce5ec9887 Mon Sep 17 00:00:00 2001 From: "Yc.S" Date: Wed, 11 Mar 2020 09:32:28 +0900 Subject: [PATCH 07/10] Respect user specified privateIPv4 --- nixopsvirtd/backends/libvirtd.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/nixopsvirtd/backends/libvirtd.py b/nixopsvirtd/backends/libvirtd.py index 826a709..a5b2628 100644 --- a/nixopsvirtd/backends/libvirtd.py +++ b/nixopsvirtd/backends/libvirtd.py @@ -43,6 +43,8 @@ def __init__(self, xml, config): self.uri = x.find("attr[@name='URI']/string").get("value") self.networks = self.config["libvirtd"]["networks"] + self.private_ipv4 = self.config["privateIPv4"] + assert len(self.networks) > 0 @@ -52,6 +54,7 @@ class LibvirtdState(MachineState): client_private_key = nixops.util.attr_property("libvirtd.clientPrivateKey", None) primary_net = nixops.util.attr_property("libvirtd.primaryNet", None) primary_mac = nixops.util.attr_property("libvirtd.primaryMAC", None) + primary_ip = nixops.util.attr_property("libvirtd.primaryIp", None) domain_xml = nixops.util.attr_property("libvirtd.domainXML", None) disk_path = nixops.util.attr_property("libvirtd.diskPath", None) storage_volume_name = nixops.util.attr_property("libvirtd.storageVolume", None) @@ -169,6 +172,9 @@ def create(self, defn, check, allow_reboot, allow_recreate): self._prepare_storage_volume() self.storage_volume_name = self.vol.name() + if defn.private_ipv4: + self.primary_ip = defn.private_ipv4; + if self.vm_id is None: self.networks = defn.networks self.primary_net = self._get_primary_net() @@ -382,6 +388,9 @@ def _parse_ip(self): """ return an ip v4 """ + if self.primary_ip: + return self.primary_ip + # alternative is VIR_DOMAIN_INTERFACE_ADDRESSES_SRC_LEASE if qemu agent is available ifaces = self.dom.interfaceAddresses(libvirt.VIR_DOMAIN_INTERFACE_ADDRESSES_SRC_LEASE, 0) if ifaces is None: From 92401ede533a9269f1d33916c7e60ac2174c260c Mon Sep 17 00:00:00 2001 From: "Yc.S" Date: Thu, 2 Jul 2020 11:51:52 +0900 Subject: [PATCH 08/10] Use virtio network driver --- nixopsvirtd/backends/libvirtd.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nixopsvirtd/backends/libvirtd.py b/nixopsvirtd/backends/libvirtd.py index a5b2628..4dcc5a5 100644 --- a/nixopsvirtd/backends/libvirtd.py +++ b/nixopsvirtd/backends/libvirtd.py @@ -348,6 +348,7 @@ def virt_port(n): return ''' + {mac} '''.format( @@ -360,6 +361,7 @@ def virt_port(n): return ''' + {mac} {vport} @@ -375,6 +377,7 @@ def virt_port(n): return ''' + {mac} {vport} From cfee9233d3f8e1dbb7ae22ebf8d2006f6b9107b8 Mon Sep 17 00:00:00 2001 From: "Yc.S" Date: Thu, 2 Jul 2020 11:54:36 +0900 Subject: [PATCH 09/10] Use virtio disk driver from https://github.com/NixOS/nixops/issues/9 --- nix/libvirtd.nix | 8 +++++++- nixopsvirtd/backends/libvirtd.py | 4 ++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/nix/libvirtd.nix b/nix/libvirtd.nix index bb67803..95c2b7c 100644 --- a/nix/libvirtd.nix +++ b/nix/libvirtd.nix @@ -173,9 +173,15 @@ in fileSystems."/".device = "/dev/disk/by-label/nixos"; boot.loader.grub.version = 2; - boot.loader.grub.device = "/dev/sda"; + boot.loader.grub.device = "/dev/vda"; boot.loader.timeout = 0; + imports = + [ + ]; + boot.initrd.availableKernelModules = [ "virtio_pci" "virtio_blk" "virtio_net" ]; + boot.kernelModules = [ "kvm-intel" ]; + services.openssh.enable = true; services.openssh.startWhenNeeded = false; services.openssh.extraConfig = "UseDNS no"; diff --git a/nixopsvirtd/backends/libvirtd.py b/nixopsvirtd/backends/libvirtd.py index 4dcc5a5..1c094a6 100644 --- a/nixopsvirtd/backends/libvirtd.py +++ b/nixopsvirtd/backends/libvirtd.py @@ -296,9 +296,9 @@ def _make_os(defn): ' ', ' {2}', ' ', - ' ', + ' ', ' ', - ' ', + ' ', ' ', '\n'.join([self._make_iface(n) for n in defn.networks]), ' ' if not defn.headless else "", From 7268db3e1a17940abe5cb15868d4dfb440b8b846 Mon Sep 17 00:00:00 2001 From: "Yc.S" Date: Thu, 2 Jul 2020 15:56:16 +0900 Subject: [PATCH 10/10] Remove unecessary configuration for virtio enabling --- nix/libvirtd.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nix/libvirtd.nix b/nix/libvirtd.nix index 95c2b7c..52f6847 100644 --- a/nix/libvirtd.nix +++ b/nix/libvirtd.nix @@ -176,9 +176,9 @@ in boot.loader.grub.device = "/dev/vda"; boot.loader.timeout = 0; - imports = - [ - ]; + # imports = + # [ + # ]; boot.initrd.availableKernelModules = [ "virtio_pci" "virtio_blk" "virtio_net" ]; boot.kernelModules = [ "kvm-intel" ];