diff --git a/examples/virtd-network.nix b/examples/virtd-network.nix new file mode 100644 index 0000000..b099a09 --- /dev/null +++ b/examples/virtd-network.nix @@ -0,0 +1,35 @@ +{ + resources.libvirtdNetworks.net2 = { + type = "nat"; + cidrBlock = "172.16.100.0/16"; + staticIPs = [ + { + machine = "node1"; + address = "172.16.100.12"; + } + { + machine = "node2"; + address = "172.16.100.5"; + } + ]; + }; + + node1 = { + deployment.targetEnv = "libvirtd"; + deployment.libvirtd.imageDir = "/var/lib/libvirt/images"; + deployment.libvirtd.networks = [ + "net2" + # { + # name = "ovsbr0"; + # type = "bridge"; + # virtualport = "openvswitch"; + # } + ]; + }; + + node2 = {resources, ...}: { + deployment.targetEnv = "libvirtd"; + deployment.libvirtd.imageDir = "/var/lib/libvirt/images"; + deployment.libvirtd.networks = [ resources.libvirtdNetworks.net2 ]; + }; +} diff --git a/nix/default.nix b/nix/default.nix index 5e4eeec..ee2ff11 100644 --- a/nix/default.nix +++ b/nix/default.nix @@ -5,5 +5,7 @@ options = [ ./libvirtd.nix ]; - resources = { ... }: {}; + resources = { evalResources, zipAttrs, resourcesByType, ...}: { + libvirtdNetworks = evalResources ./libvirtd-network.nix (zipAttrs resourcesByType.libvirtdNetworks or []); + }; } diff --git a/nix/libvirtd-network.nix b/nix/libvirtd-network.nix new file mode 100644 index 0000000..e7afa7c --- /dev/null +++ b/nix/libvirtd-network.nix @@ -0,0 +1,80 @@ +{ config, lib, pkgs, uuid, name, ... }: + +with lib; +with import lib; + +let + toMachineName = m: if builtins.isString m then m else m._name; +in +rec { + options = { + type = mkOption { + default = "nat"; + description = '' + The type of the libvirt network. + Either NAT network or isolated network can be specified. Defaults to NAT Network. + ''; + type = types.enum [ "nat" "isolate" ]; + }; + + cidrBlock = mkOption { + example = "192.168.56.0/24"; + description = '' + The IPv4 CIDR block for the libvirt network. The following IP addresses are reserved for the network: + Network - The first address in the IP range, e.g. 192.168.56.0 in 192.168.56.0/24 + Gateway - The second address in the IP range, e.g. 192.168.56.1 in 192.168.56.0/24 + Broadcast - The last address in the IP range, e.g. 192.168.56.255 in 192.168.56.0/24 + ''; + type = types.str; + }; + + staticIPs = mkOption { + example = '' + # As an attrset + { + "192.168.56.10" = "node1"; + "192.168.56.11" = "node2"; + ... + } + # Or as a list + [ + { address = "192.168.56.10"; machine = "node1"; } + { address = "192.168.56.11"; machine = "node2"; } + ... + ] + ''; + default = []; + description = "The list of machine to IPv4 address bindings for fixing IP address of the machine in the network"; + apply = a: if builtins.isAttrs a then mapAttrs (k: toMachineName) a else a; + type = with types; either attrs (listOf (submodule { + options = { + machine = mkOption { + type = either str (resource "machine"); + apply = toMachineName; + description = "The name of the machine in the network"; + }; + address = mkOption { + example = "192.168.56.3"; + type = str; + description = '' + The IPv4 address assigned to the machine as static IP. + The static IP must be a non-reserved IP address. + ''; + }; + }; + })); + }; + + URI = mkOption { + type = types.str; + default = "qemu:///system"; + description = '' + Connection URI. + ''; + }; + }; + + config = { + _type = "libvirtd-network"; + }; +} diff --git a/nix/libvirtd.nix b/nix/libvirtd.nix index e956bd4..52f6847 100644 --- a/nix/libvirtd.nix +++ b/nix/libvirtd.nix @@ -1,6 +1,7 @@ { config, pkgs, lib, ... }: with lib; +with import lib; let the_key = builtins.getEnv "NIXOPS_LIBVIRTD_PUBKEY"; @@ -80,10 +81,48 @@ in disk image as a base. ''; }; - deployment.libvirtd.networks = mkOption { default = [ "default" ]; - type = types.listOf types.str; + type = with types; nonEmptyListOf + (either + str # for backward compatibility + (either + (resource "libvirtd-network") + (submodule { + options = { + name = mkOption { + default = ""; + description = "The name of the network not managed by NixOps"; + type = str; + }; + type = mkOption { + description = "The type of the network"; + type = enum [ "nat" "isolate" "bridge" "direct" ]; + }; + mode = mkOption { + default = "bridge"; + description = "The mode of the direct (macvtap) network"; + type = enum [ "bridge" "vepa" "private" "passthrough" ]; + }; + virtualport = mkOption { + default = null; + description = "The virtualport for specific bridge devices such as Open vSwitch"; + type = nullOr (either str (submodule { + optiones = { + type = mkOption { + description = "The type of the virtualport"; + type = str; + }; + parameters = mkOption { + description = "The parameters of the virtualport"; + type = attrset; + }; + }; + })); + }; + }; + })) + ); description = "Names of libvirt networks to attach the VM to."; }; @@ -134,9 +173,15 @@ in fileSystems."/".device = "/dev/disk/by-label/nixos"; boot.loader.grub.version = 2; - boot.loader.grub.device = "/dev/sda"; + boot.loader.grub.device = "/dev/vda"; boot.loader.timeout = 0; + # imports = + # [ + # ]; + boot.initrd.availableKernelModules = [ "virtio_pci" "virtio_blk" "virtio_net" ]; + boot.kernelModules = [ "kvm-intel" ]; + services.openssh.enable = true; services.openssh.startWhenNeeded = false; services.openssh.extraConfig = "UseDNS no"; diff --git a/nixopsvirtd/backends/libvirtd.py b/nixopsvirtd/backends/libvirtd.py index e383b40..1c094a6 100644 --- a/nixopsvirtd/backends/libvirtd.py +++ b/nixopsvirtd/backends/libvirtd.py @@ -42,9 +42,9 @@ def __init__(self, xml, config): self.storage_pool_name = x.find("attr[@name='storagePool']/string").get("value") self.uri = x.find("attr[@name='URI']/string").get("value") - self.networks = [ - k.get("value") - for k in x.findall("attr[@name='networks']/list/string")] + self.networks = self.config["libvirtd"]["networks"] + self.private_ipv4 = self.config["privateIPv4"] + assert len(self.networks) > 0 @@ -54,11 +54,13 @@ class LibvirtdState(MachineState): client_private_key = nixops.util.attr_property("libvirtd.clientPrivateKey", None) primary_net = nixops.util.attr_property("libvirtd.primaryNet", None) primary_mac = nixops.util.attr_property("libvirtd.primaryMAC", None) + primary_ip = nixops.util.attr_property("libvirtd.primaryIp", None) domain_xml = nixops.util.attr_property("libvirtd.domainXML", None) disk_path = nixops.util.attr_property("libvirtd.diskPath", None) storage_volume_name = nixops.util.attr_property("libvirtd.storageVolume", None) storage_pool_name = nixops.util.attr_property("libvirtd.storagePool", None) vcpu = nixops.util.attr_property("libvirtd.vcpu", None) + networks = nixops.util.attr_property("libvirtd.networks", [], "json") # older deployments may not have a libvirtd.URI attribute in the state file # using qemu:///system in such case @@ -134,6 +136,14 @@ def address_to(self, m): def _vm_id(self): return "nixops-{0}-{1}".format(self.depl.uuid, self.name) + def _get_network_name(self, n): + if isinstance(n, basestring): return n + if isinstance(n, dict): return n.get("_name", n.get("name")) + + def _get_primary_net(self): + assert len(self.networks) > 0 + return self._get_network_name(self.networks[0]) + def _generate_primary_mac(self): mac = [0x52, 0x54, 0x00, random.randint(0x00, 0x7f), @@ -144,7 +154,6 @@ def _generate_primary_mac(self): def create(self, defn, check, allow_reboot, allow_recreate): assert isinstance(defn, LibvirtdDefinition) self.set_common_state(defn) - self.primary_net = defn.networks[0] self.storage_pool_name = defn.storage_pool_name self.uri = defn.uri @@ -163,9 +172,14 @@ def create(self, defn, check, allow_reboot, allow_recreate): self._prepare_storage_volume() self.storage_volume_name = self.vol.name() - self.domain_xml = self._make_domain_xml(defn) + if defn.private_ipv4: + self.primary_ip = defn.private_ipv4; if self.vm_id is None: + self.networks = defn.networks + self.primary_net = self._get_primary_net() + self.domain_xml = self._make_domain_xml(defn) + # By using "define" we ensure that the domain is # "persistent", as opposed to "transient" (i.e. removed on reboot). self._dom = self.conn.defineXML(self.domain_xml) @@ -175,6 +189,27 @@ def create(self, defn, check, allow_reboot, allow_recreate): self.vm_id = self._vm_id() + # Update networks for redeployment + if self.networks != defn.networks: + if not allow_reboot: + self.warn("change of the networks requires reboot; skipping") + else: + self.log('update networks...') + self.stop() + + for n in self.networks: + try: + self.dom.detachDeviceFlags(self._make_iface(n), libvirt.VIR_DOMAIN_AFFECT_CONFIG) + except: + pass + + self.networks = defn.networks + self.primary_net = self._get_primary_net() + self.domain_xml = self._make_domain_xml(defn) + + for n in defn.networks: + self.dom.attachDeviceFlags(self._make_iface(n), libvirt.VIR_DOMAIN_AFFECT_CONFIG) + self.start() return True @@ -243,20 +278,6 @@ def _get_qemu_executable(self): def _make_domain_xml(self, defn): qemu = self._get_qemu_executable() - def maybe_mac(n): - if n == self.primary_net: - return '' - else: - return "" - - def iface(n): - return "\n".join([ - ' ', - maybe_mac(n), - ' ', - ' ', - ]).format(n) - def _make_os(defn): return [ '', @@ -275,11 +296,11 @@ def _make_os(defn): ' ', ' {2}', ' ', - ' ', + ' ', ' ', - ' ', + ' ', ' ', - '\n'.join([iface(n) for n in defn.networks]), + '\n'.join([self._make_iface(n) for n in defn.networks]), ' ' if not defn.headless else "", ' ', ' ', @@ -298,10 +319,81 @@ def _make_os(defn): defn.domain_type ) + def _make_iface(self, n): + def maybe_mac(n): + if self._get_network_name(n) == self.primary_net: + return '' + else: + return "" + + def virt_port(n): + v = n.get("virtualport") + + if isinstance(v, basestring) or (isinstance(v, dict) and not v.get("parameters")): + return ''.format(v.get("type") if isinstance(v, dict) else v) + + if isinstance(v, dict) and v.get("parameters"): + return ''' + + + + '''.format( + type=v.get("type"), + params=" ".join('{key}="{value}"' for key, value in v.get("parameters")) + ) + return "" + + # virtual network + if isinstance(n, basestring) or (isinstance(n, dict) and n.get("type") in ["nat", "isolate"]): + return ''' + + + + {mac} + + '''.format( + name=self._get_network_name(n), + mac=maybe_mac(n) + ) + + # macvtap device + if isinstance(n, dict) and n.get("type") == "direct": + return ''' + + + + {mac} + {vport} + + '''.format( + name=n.get("name"), + mode=n.get("mode"), + mac=maybe_mac(n), + vport=virt_port(n), + ) + + # bridge + if isinstance(n, dict) and n.get("type") == "bridge": + return ''' + + + + {mac} + {vport} + + '''.format( + name=n.get("name"), + mac=maybe_mac(n), + vport=virt_port(n), + ) + def _parse_ip(self): """ return an ip v4 """ + if self.primary_ip: + return self.primary_ip + # alternative is VIR_DOMAIN_INTERFACE_ADDRESSES_SRC_LEASE if qemu agent is available ifaces = self.dom.interfaceAddresses(libvirt.VIR_DOMAIN_INTERFACE_ADDRESSES_SRC_LEASE, 0) if ifaces is None: diff --git a/nixopsvirtd/plugin.py b/nixopsvirtd/plugin.py index c048ec0..78f153b 100644 --- a/nixopsvirtd/plugin.py +++ b/nixopsvirtd/plugin.py @@ -16,5 +16,6 @@ def nixexprs(): @nixops.plugins.hookimpl def load(): return [ + "nixopsvirtd.resources", "nixopsvirtd.backends.libvirtd", ] diff --git a/nixopsvirtd/resources/__init__.py b/nixopsvirtd/resources/__init__.py new file mode 100644 index 0000000..de47710 --- /dev/null +++ b/nixopsvirtd/resources/__init__.py @@ -0,0 +1,2 @@ +import libvirtd_network +import __init__ diff --git a/nixopsvirtd/resources/libvirtd_network.py b/nixopsvirtd/resources/libvirtd_network.py new file mode 100644 index 0000000..0e11c82 --- /dev/null +++ b/nixopsvirtd/resources/libvirtd_network.py @@ -0,0 +1,247 @@ +# -*- coding: utf-8 -*- + +# Automatic provisioning of Libvirt Virtual Networks. + +import os +import ipaddress +import libvirt +from nixops.util import attr_property, logged_exec +from nixops.resources import ResourceDefinition, ResourceState +from nixopsvirtd.backends.libvirtd import LibvirtdDefinition, LibvirtdState + +class LibvirtdNetworkDefinition(ResourceDefinition): + """Definition of the Libvirtd Network""" + + @classmethod + def get_type(cls): + return "libvirtd-network" + + @classmethod + def get_resource_type(cls): + return "libvirtdNetworks" + + def __init__(self, xml): + ResourceDefinition.__init__(self, xml) + self.network_type = xml.find("attrs/attr[@name='type']/string").get("value") + self.network_cidr = xml.find("attrs/attr[@name='cidrBlock']/string").get("value") + + self.static_ips = { x.find("attr[@name='machine']/string").get("value"): + x.find("attr[@name='address']/string").get("value") for x in xml.findall("attrs/attr[@name='staticIPs']/list/attrs") } + self.static_ips.update({ + x.find("string").get("value"): x.get("name") + for x in xml.findall("attrs/attr[@name='staticIPs']/attrs/attr") + }) + + self.uri = xml.find("attrs/attr[@name='URI']/string").get("value") + + def show_type(self): + return "{0} [{1} {2}]".format(self.get_type(), self.network_type, self.network_cidr) + +class LibvirtdNetworkState(ResourceState): + """State of the Libvirtd Network""" + + network_name = attr_property("libvirtd.network_name", None) + network_type = attr_property("libvirtd.network_type", None) + network_cidr = attr_property("libvirtd.network_cidr", None) + static_ips = attr_property("libvirtd.static_ips", {}, "json") + + uri = attr_property("libvirtd.URI", "qemu:///system") + + @classmethod + def get_type(cls): + return "libvirtd-network" + + def __init__(self, depl, name, id): + ResourceState.__init__(self, depl, name, id) + self._conn = None + self._net = None + + def show_type(self): + s = super(LibvirtdNetworkState, self).show_type() + if self.state == self.UP: s = "{0} [{1}]".format(s, self.network_type) + return s + + @property + def resource_id(self): + return self.network_name + + @property + def public_ipv4(self): + return self.network_cidr if self.state == self.UP else None; + + nix_name = "libvirtdNetworks" + + @property + def full_name(self): + return "Libvirtd network '{}'".format(self.name) + + @property + def conn(self): + if self._conn is None: + self.logger.log('Connecting to {}...'.format(self.uri)) + try: + self._conn = libvirt.open(self.uri) + except libvirt.libvirtError as error: + self.logger.error(error.get_error_message()) + if error.get_error_code() == libvirt.VIR_ERR_NO_CONNECT: + # this error code usually means "no connection driver available for qemu:///..." + self.logger.error('make sure qemu-system-x86_64 is installed on the target host') + raise Exception('Failed to connect to the hypervisor at {}'.format(self.uri)) + return self._conn + + @property + def net(self): + if self._net is None: + try: + self._net = self.conn.networkLookupByName(self.name) + except Exception as e: + self.log("Warning: %s" % e) + return self._net + + def create(self, defn, check, allow_reboot, allow_recreate): + assert isinstance(defn, LibvirtdNetworkDefinition) + + if check: self.check() + + subnet = ipaddress.ip_network(unicode(defn.network_cidr), strict=False) + + if self.state != self.UP: + self.log("creating {}...".format(self.full_name)) + self.network_type = defn.network_type + self.network_cidr = defn.network_cidr + self.static_ips = defn.static_ips + self.uri = defn.uri + + self._net = self.conn.networkDefineXML(''' + + {name} + {maybeForward} + + + + {dhcpHosts} + + + + '''.format( + name=self.name, + maybeForward='' if defn.network_type == "nat" else "", + gateway=subnet[1], + netmask=subnet.netmask, + lowerip=subnet[2], + upperip=subnet[-2], + dhcpHosts="".join("".format( + name=machine, + ip=address, + ) for machine, address in defn.static_ips.iteritems()) + )) + + self.net.create() + self.net.setAutostart(1) + self.network_name = self.net.bridgeName() + self.state = self.UP + return + + if self._need_update(defn, allow_reboot, allow_recreate): + self.log("updating {}...".format(self.full_name)) + + if self.network_type != defn.network_type: + self.conn.networkUpdate( + libvirt.VIR_NETWORK_UPDATE_COMMAND_ADD if defn.network_type == "nat" else libvirt.VIR_NETWORK_UPDATE_COMMAND_DELETE, + libvirt.VIR_NETWORK_SECTION_FORWARD, + -1, + '' + ) + self.network_type = defn.network_type + + if self.static_ips != defn.static_ips: + # Remove obsolete + for machine, address in self.static_ips.iteritems(): + if not defn.static_ips.get(machine): + self.net.update( + libvirt.VIR_NETWORK_UPDATE_COMMAND_DELETE, + libvirt.VIR_NETWORK_SECTION_IP_DHCP_HOST, + -1, + "".format( + name=machine, + ip=address, + ) + ) + + # Add or update existings + for machine, address in defn.static_ips.iteritems(): + mstate = self.depl.resources.get(machine) + mdefn = self.depl.definitions.get(machine) + if isinstance(mstate, LibvirtdState) and isinstance(mdefn, LibvirtdDefinition): + for net in mdefn.config["libvirtd"]["networks"]: + if net == defn.name or (net.get("_name", net.get("name")) == defn.name and net.get("type") == defn.network_type): + try: + if ipaddress.ip_address(unicode(address)) not in subnet: + raise Exception("cannot assign a static IP out of the network CIDR") + self.net.update( + libvirt.VIR_NETWORK_UPDATE_COMMAND_MODIFY if self.static_ips.get(machine) else libvirt.VIR_NETWORK_UPDATE_COMMAND_ADD_LAST, + libvirt.VIR_NETWORK_SECTION_IP_DHCP_HOST, + -1, + "".format( + name=machine, + ip=address, + ) + , + libvirt.VIR_NETWORK_UPDATE_AFFECT_CONFIG | libvirt.VIR_NETWORK_UPDATE_AFFECT_LIVE + ) + except Exception as e: + self.warn("Cannot assign static IP '{0}' to machine '{1}' in subnet '{2}'".format(address, machine, defn.network_cidr)) + break; + else: + self.warn("Cannot assign static IP '{0}' to non-attached machine '{1}'".format(address, machine)) + else: + self.warn("Cannot assign static IP '{0}' to non-existent machine '{1}'".format(address, machine)) + + self.static_ips = defn.static_ips + + def _need_update(self, defn, allow_reboot, allow_recreate): + if self.uri != defn.uri: + self.warn("Change of the connection URI from {0} to {1} is not supported; skipping".format(self.uri, defn.uri)) + return False + + if self.network_cidr != defn.network_cidr: + self.warn("Change of the network CIDR from {0} to {1} is not supported; skipping".format(self.network_cidr, defn.network_cidr)) + return False + + if self.network_type == defn.network_type and self.static_ips == defn.static_ips: # no changes + return False + + if self.network_type != defn.network_type and not allow_reboot: + self.warn("change of the network type requires reboot; skipping") + return False + + # checkme: the state of the attached machine should also be considered + if any(defn.static_ips.get(machine) != address for machine, address in self.static_ips.iteritems()) and not allow_reboot: + self.warn("change of existing bindings for static IPs requires reboot; skipping") + return False + + return True + + def destroy(self, wipe=False): + if self.state != self.UP or not self.net: return True + if not self.depl.logger.confirm("are you sure you want to destroy {}?".format(self.full_name)): + return False + + self.log("destroying {}...".format(self.full_name)) + + self.net.destroy() + self.net.undefine() + + return True + + def _check(self): + if self.net: + if self.network_name != self.net.bridgeName(): + self.network_name = self.net.bridgeName() + return super(LibvirtdNetworkState, self)._check() + + with self.depl._db: + self.network_name = None + self.state = self.MISSING + + return False diff --git a/release.nix b/release.nix index 6946898..eb62eb9 100644 --- a/release.nix +++ b/release.nix @@ -18,7 +18,7 @@ in done ''; buildInputs = [ python2Packages.nose python2Packages.coverage ]; - propagatedBuildInputs = [ python2Packages.libvirt ]; + propagatedBuildInputs = with python2Packages; [ libvirt ipaddress ]; doCheck = true; postInstall = '' mkdir -p $out/share/nix/nixops-libvirtd diff --git a/setup.py b/setup.py index 7fd02b2..42dcb54 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ url='https://github.com/AmineChikhaoui/nixops-libvirtd', maintainer='Amine Chikhaoui', maintainer_email='amine.chikhaoui91@gmail.com', - packages=['nixopsvirtd', 'nixopsvirtd.backends'], + packages=['nixopsvirtd', 'nixopsvirtd.backends', 'nixopsvirtd.resources'], entry_points={'nixops': ['virtd = nixopsvirtd.plugin']}, py_modules=['plugin'] )