Merge branch 'devel-3'

* devel-3:
  Implement VM clone as create + copy data+metadata
  storage: make Volumes sortable
  base: add PropertyHolder.clone_properties
  doc: minor fixes to man pages
  storage: add volume clone method
  doc: fix skel-manpage tool
  tools: add qvm-tags tool
  tags support

doc/manpages/qvm-clone.rst

@@ -14,7 +14,7 @@
 
 Synopsis
 --------
-:command:`qvm-clone` [-h] [--verbose] [--quiet] [-p *POOL:VOLUME* | -P POOL] *VMNAME* *NEWVM*
+:command:`qvm-clone` [*options*] *VMNAME* *NEWVM*
 
 Options
 -------
@@ -23,6 +23,12 @@ Options
 
     Show this help message and exit
 
+.. option:: --class=CLASS, -C CLASS
+
+    Create VM of different class than source VM. The tool will try to copy as
+    much as possible data/metadata from source VM, but some information may be
+    impossible to preserve (for example target VM have no matching properties).
+
 .. option:: -P POOL
 
     Pool to use for the new domain. All volumes besides snapshots volumes are

doc/manpages/qvm-device.rst

@@ -49,7 +49,7 @@ aliases: ls, l
 attach
 ^^^^^^
 
-| :command:`qvm-volume attach` [-h] [--verbose] [--quiet] [--ro] *VMNAME* *BACKEND_DOMAIN:DEVICE_ID*
+| :command:`qvm-device` *DEVICE_CLASS* attach [-h] [--verbose] [--quiet] [--ro] *VMNAME* *BACKEND_DOMAIN:DEVICE_ID*
 
 Attach the device with *DEVICE_ID* from *BACKEND_DOMAIN* to the domain *VMNAME*
 
@@ -67,7 +67,7 @@ aliases: a, at
 detach
 ^^^^^^
 
-| :command:`qvm-volume detach` [-h] [--verbose] [--quiet] *VMNAME* *BACKEND_DOMAIN:DEVICE_ID*
+| :command:`qvm-device` *DEVICE_CLASS* detach [-h] [--verbose] [--quiet] *VMNAME* *BACKEND_DOMAIN:DEVICE_ID*
 
 Detach the device with *BACKEND_DOMAIN:DEVICE_ID* from domain *VMNAME*
 

doc/manpages/qvm-tags.rst

@@ -15,7 +15,10 @@
 Synopsis
 --------
 
-:command:`qvm-tags` [-h] [--verbose] [--quiet] [--query | --set | --unset] *VMNAME* [*TAG*]
+| :command:`qvm-tags` [-h] [--verbose] [--quiet] *VMNAME* {list,ls,l} [*TAG*]
+| :command:`qvm-tags` [-h] [--verbose] [--quiet] *VMNAME* {add,a,set} *TAG* ...
+| :command:`qvm-tags` [-h] [--verbose] [--quiet] *VMNAME* {del,d,unset,u} *TAG* ...
+
 
 Options
 -------
@@ -32,22 +35,37 @@ Options
 
    Decrease verbosity.
 
-.. option:: --query
+Commands
+--------
+
+list
+^^^^
+
+| :command:`qvm-tags` [-h] [--verbose] [--quiet] *VMNAME* list [*TAG*]
+
+List tags. If tag name is given, check if this tag is set for the VM and signal
+this with exit code (0 - tag is set, 1 - it is not).
+
+aliases: ls, l
+
+add
+^^^
+
+| :command:`qvm-tags` [-h] [--verbose] [--quiet] *VMNAME* add *TAG* [*TAG* ...]
+
+Add tag(s) to a VM. If tag is already set for given VM, do nothing.
 
-   Query for the tag. Exit with zero (true) if the qube in question has the tag
-   and with non-zero (false) if it does not. If no tag specified, list all the
-   tags.
+aliases: a, set
 
-   This is the default mode.
+del
+^^^
 
-.. option:: --set, -s
+| :command:`qvm-tags` [-h] [--verbose] [--quiet] *VMNAME* del *TAG* [*TAG* ...]
 
-   Set the tag. The tag argument is mandatory. If tag is already set, do
-   nothing.
+Delete tag(s) from a VM. If tag is not set for given VM, do nothing.
 
-.. option:: --delete, --unset, -D
+aliases: d, unset, u
 
-   Unset the tag. The tag argument is mandatory. If tag is not set, do nothing.
 
 Authors
 -------

doc/manpages/qvm-volume.rst

@@ -1,7 +1,7 @@
 .. program:: qvm-volume
 
 :program:`qvm-volume` -- Qubes volume and block device managment
-===============================================================
+================================================================
 
 Synopsis
 --------

doc/skel-manpage.py

@@ -5,7 +5,7 @@
 sys.path.insert(0, os.path.abspath('../'))
 
 import argparse
-import qubesadmin.dochelpers
+import qubesadmin.tools.dochelpers
 
 parser = argparse.ArgumentParser(description='prepare new manpage for command')
 parser.add_argument('command', metavar='COMMAND',
@@ -14,7 +14,7 @@
 
 def main():
     args = parser.parse_args()
-    sys.stdout.write(qubesadmin.dochelpers.prepare_manpage(args.command))
+    sys.stdout.write(qubesadmin.tools.dochelpers.prepare_manpage(args.command))
 
 if __name__ == '__main__':
     main()

qubesadmin/app.py

@@ -270,7 +270,8 @@ def add_new_vm(self, cls, name, label, template=None, pool=None,
         self.domains.clear_cache()
         return self.domains[name]
 
-    def clone_vm(self, src_vm, new_name, pool=None, pools=None):
+    def clone_vm(self, src_vm, new_name, new_cls=None,
+            pool=None, pools=None, ignore_errors=False):
         '''Clone Virtual Machine
 
         Example usage with custom storage pools:
@@ -281,35 +282,110 @@ def clone_vm(self, src_vm, new_name, pool=None, pools=None):
         >>> vm = app.clone_vm(src_vm, 'my-new-vm', pools=pools)
         >>> vm.label = app.labels['green']
 
-        :param str cls: name of VM class (`AppVM`, `TemplateVM` etc)
-        :param str name: name of VM
-        :param str label: label color for new VM
-        :param str template: template to use (if apply for given VM class),
-        can be also VM object; use None for default value
+        :param QubesVM or str src_vm: source VM
+        :param str new_name: name of new VM
+        :param str new_cls: name of VM class (`AppVM`, `TemplateVM` etc) - use
+        None to copy it from *src_vm*
         :param str pool: storage pool to use instead of default one
         :param dict pools: storage pool for specific volumes
+        :param bool ignore_errors: should errors on meta-data setting be only
+        logged, or abort the whole operation?
         :return new VM object
         '''
 
         if pool and pools:
             raise ValueError('only one of pool= and pools= can be used')
 
-        if not isinstance(src_vm, str):
-            src_vm = str(src_vm)
+        if isinstance(src_vm, str):
+            src_vm = self.domains[src_vm]
+
+        if new_cls is None:
+            new_cls = src_vm.__class__.__name__
 
-        method = 'admin.vm.Clone'
-        payload = 'name={}'.format(new_name)
+        template = getattr(src_vm, 'template', None)
+        if template is not None:
+            template = str(template)
+
+        label = src_vm.label
+
+        method_prefix = 'admin.vm.Create.'
+        payload = 'name={} label={}'.format(new_name, label)
         if pool:
             payload += ' pool={}'.format(str(pool))
-            method = 'admin.vm.CloneInPool'
+            method_prefix = 'admin.vm.CreateInPool.'
         if pools:
             payload += ''.join(' pool:{}={}'.format(vol, str(pool))
                 for vol, pool in sorted(pools.items()))
-            method = 'admin.vm.CloneInPool'
+            method_prefix = 'admin.vm.CreateInPool.'
+
+        self.qubesd_call('dom0', method_prefix + new_cls, template,
+            payload.encode('utf-8'))
+
+        self.domains.clear_cache()
+        dst_vm = self.domains[new_name]
+        try:
+            assert isinstance(dst_vm, qubesadmin.vm.QubesVM)
+            for prop in src_vm.property_list():
+                # handled by admin.vm.Create call
+                if prop in ('name', 'qid', 'template', 'label'):
+                    continue
+                if src_vm.property_is_default(prop):
+                    continue
+                try:
+                    setattr(dst_vm, prop, getattr(src_vm, prop))
+                except AttributeError:
+                    pass
+                except qubesadmin.exc.QubesException as e:
+                    dst_vm.log.error(
+                        'Failed to set {!s} property: {!s}'.format(prop, e))
+                    if not ignore_errors:
+                        raise
+
+            for tag in src_vm.tags:
+                try:
+                    dst_vm.tags.add(tag)
+                except qubesadmin.exc.QubesException as e:
+                    dst_vm.log.error(
+                        'Failed to add {!s} tag: {!s}'.format(tag, e))
+                    if not ignore_errors:
+                        raise
+
+            for feature, value in src_vm.features.items():
+                try:
+                    dst_vm.features[feature] = value
+                except qubesadmin.exc.QubesException as e:
+                    dst_vm.log.error(
+                        'Failed to set {!s} feature: {!s}'.format(feature, e))
+                    if not ignore_errors:
+                        raise
+
+            try:
+                dst_vm.firewall.policy = src_vm.firewall.policy
+                dst_vm.firewall.save_rules(src_vm.firewall.rules)
+            except qubesadmin.exc.QubesException as e:
+                self.log.error('Failed to set firewall: %s', e)
+                if not ignore_errors:
+                    raise
+
+        except qubesadmin.exc.QubesException:
+            if not ignore_errors:
+                del self.domains[dst_vm.name]
+                raise
+
+        try:
+            for dst_volume in sorted(dst_vm.volumes.values()):
+                if not dst_volume.save_on_stop:
+                    # clone only persistent volumes
+                    continue
+                src_volume = src_vm.volumes[dst_volume.name]
+                dst_vm.log.info('Cloning {} volume'.format(dst_volume.name))
+                dst_volume.clone(src_volume)
 
-        self.qubesd_call(src_vm, method, None, payload.encode('utf-8'))
+        except qubesadmin.exc.QubesException:
+            del self.domains[dst_vm.name]
+            raise
 
-        return self.domains[new_name]
+        return dst_vm
 
     def run_service(self, dest, service, filter_esc=False, user=None,
             localcmd=None, wait=True, **kwargs):

qubesadmin/base.py

@@ -154,6 +154,23 @@ def property_is_default(self, item):
         assert isinstance(is_default, bool)
         return is_default
 
+    def clone_properties(self, src, proplist=None):
+        '''Clone properties from other object.
+
+        :param PropertyHolder src: source object
+        :param list proplist: list of properties \
+            (:py:obj:`None` or omit for all properties)
+        '''
+
+        if proplist is None:
+            proplist = self.property_list()
+
+        for prop in proplist:
+            try:
+                setattr(self, prop, getattr(src, prop))
+            except AttributeError:
+                continue
+
     def __getattr__(self, item):
         # pylint: disable=too-many-return-statements
         if item.startswith('_'):

qubesadmin/exc.py

@@ -116,6 +116,10 @@ class QubesFeatureNotFoundError(QubesException, KeyError):
     '''Feature not set for a given domain'''
 
 
+class QubesTagNotFoundError(QubesException, KeyError):
+    '''Tag not set for a given domain'''
+
+
 class StoragePoolException(QubesException):
     ''' A general storage exception '''
 

qubesadmin/storage.py

@@ -93,6 +93,21 @@ def __eq__(self, other):
             return self.pool == other.pool and self.vid == other.vid
         return NotImplemented
 
+    def __lt__(self, other):
+        # pylint: disable=protected-access
+        if isinstance(other, Volume):
+            if self._vm and other._vm:
+                return (self._vm, self._vm_name) < (other._vm, other._vm_name)
+            elif self._vid and other._vid:
+                return (self._pool, self._vid) < (other._pool, other._vid)
+        return NotImplemented
+
+
+    @property
+    def name(self):
+        '''per-VM volume name, if available'''
+        return self._vm_name
+
     @property
     def pool(self):
         '''Storage volume pool name.'''
@@ -195,6 +210,26 @@ def import_data(self, stream):
         '''
         self._qubesd_call('Import', payload_stream=stream)
 
+    def clone(self, source):
+        ''' Clone data from sane volume of another VM.
+
+        This function override existing volume content.
+        This operation is implemented for VM volumes - those in vm.volumes
+        collection (not pool.volumes).
+
+        :param source: source volume object
+        '''
+
+        # pylint: disable=protected-access
+        if source._vm_name is None or self._vm_name is None:
+            raise NotImplementedError(
+                'Operation implemented only for VM volumes')
+        if source._vm_name != self._vm_name:
+            raise ValueError('Source and target volume must have the same type')
+
+        # this call is to *source* volume, because we extract data from there
+        source._qubesd_call('Clone', payload=str(self._vm).encode())
+
 
 class Pool(object):
     ''' A Pool is used to manage different kind of volumes (File