[MXNET-793] Virtual testing with Qemu, refinement and extract test re…

…sults to root MXNet folder (apache#13065)

* Improve Qemu infrastructure
Add documentation about running it interactively

* Separate provision

* Improve provisioning

* Refine provisioning and interactive

* Cant provision when the volumes arent mounted

* Fix running tests

* raise log output to INFO

* adjust logging

* flush stdout and stderr

* Refine by copying test results back to the host

* Fix license

* remove config file and different way to run QEMU

* remove config file and different way to run QEMU, remove ansible

ci/README.md

@@ -98,5 +98,21 @@ To run the unit tests under qemu:
 ./build.py -p armv7 && ./build.py -p test.arm_qemu ./runtime_functions.py run_ut_py3_qemu
 ```
 
-To get a shell on the container and debug issues with the emulator itself:
-Run the output of `./build.py -p test.arm_qemu --print-docker-run`
+To get a shell on the container and debug issues with the emulator itself, we build the container
+and then execute it interactively. We can afterwards use port 2222 on the host to connect with SSH.
+
+
+```
+ci/build.py -p test.arm_qemu -b && docker run -p2222:2222 -ti mxnetci/build.test.arm_qemu
+```
+
+Then from another terminal:
+
+```
+ssh -o StrictHostKeyChecking=no -p 2222 qemu@localhost
+```
+
+There are two pre-configured users: `root` and `qemu` both without passwords.
+
+
+

ci/docker/Dockerfile.build.test.arm_qemu

@@ -39,6 +39,8 @@ RUN /work/ubuntu_adduser.sh
 
 COPY runtime_functions.sh /work/
 COPY qemu/* /work/
-COPY qemu/ansible.cfg /etc/ansible/ansible.cfg 
 
-CMD ["./runtime_functions.py","run_ut_py3_qemu"]
+# SSH to the Qemu VM
+EXPOSE 2222/tcp
+
+CMD ["./runtime_functions.py","run_qemu_interactive"]

ci/docker/install/ubuntu_arm_qemu.sh

@@ -31,6 +31,7 @@ apt-get install -y \
     qemu-system-arm \
     unzip \
     bzip2 \
-    vim-nox
+    vim-nox \
+    toilet
 
-pip3 install ansible ipython
+pip3 install ipython

ci/docker/qemu/runtime_functions.py

@@ -33,6 +33,8 @@
 import sys
 import types
 import glob
+import vmcontrol
+from vmcontrol import qemu_ssh, qemu_provision, qemu_rsync_to_host, VM
 
 def activate_this(base):
     import site
@@ -53,30 +55,45 @@ def activate_this(base):
             sys.path.remove(item)
     sys.path[:0] = new_sys_path
 
+
+
+
 def run_ut_py3_qemu():
+    """Run unit tests in the emulator and copy the results back to the host through the mounted
+    volume in /mxnet"""
     from vmcontrol import VM
     with VM() as vm:
-        logging.info("VM provisioning with ansible")
-        check_call(["ansible-playbook", "-v", "-u", "qemu", "-i", "localhost:{},".format(vm.ssh_port), "playbook.yml"])
-        logging.info("VM provisioned successfully.")
-        logging.info("sync tests")
-        check_call(['rsync', '-e', 'ssh -p{}'.format(vm.ssh_port), '-a', 'mxnet/tests', 'qemu@localhost:mxnet'])
+        qemu_provision(vm.ssh_port)
         logging.info("execute tests")
-        check_call(["ssh", "-o", "ServerAliveInterval=5", "-p{}".format(vm.ssh_port), "qemu@localhost", "./runtime_functions.py", "run_ut_python3_qemu_internal"])
+        qemu_ssh(vm.ssh_port, "./runtime_functions.py", "run_ut_python3_qemu_internal")
+        qemu_rsync_to_host(vm.ssh_port, "*.xml", "mxnet")
+        logging.info("copied to host")
         logging.info("tests finished, vm shutdown.")
         vm.shutdown()
 
 def run_ut_python3_qemu_internal():
-    """this runs inside the vm, it's run by the playbook above by ansible"""
+    """this runs inside the vm"""
     pkg = glob.glob('mxnet_dist/*.whl')[0]
     logging.info("=== NOW Running inside QEMU ===")
     logging.info("PIP Installing %s", pkg)
     check_call(['sudo', 'pip3', 'install', pkg])
     logging.info("PIP Installing mxnet/tests/requirements.txt")
     check_call(['sudo', 'pip3', 'install', '-r', 'mxnet/tests/requirements.txt'])
     logging.info("Running tests in mxnet/tests/python/unittest/")
-    check_call(['nosetests', '--with-timer', '--with-xunit', '--xunit-file', 'nosetests_unittest.xml', '--verbose', 'mxnet/tests/python/unittest/test_ndarray.py:test_ndarray_fluent'])
+    check_call(['nosetests', '--with-timer', '--with-xunit', '--xunit-file', 'nosetests_unittest.xml', '--verbose', 'mxnet/tests/python/unittest/'])
+    # Example to run a single unit test:
+    # check_call(['nosetests', '--with-timer', '--with-xunit', '--xunit-file', 'nosetests_unittest.xml', '--verbose', 'mxnet/tests/python/unittest/test_ndarray.py:test_ndarray_fluent'])
+
+
+
+def run_qemu_interactive():
+    vm = VM(interactive=True)
+    vm.detach()
+    vm.start()
+    vm.wait()
+    logging.info("QEMU finished")
 
+################################
 
 def parsed_args():
     parser = argparse.ArgumentParser(description="""python runtime functions""", epilog="")
@@ -95,7 +112,7 @@ def chdir_to_script_directory():
     os.chdir(base)
 
 def main():
-    logging.getLogger().setLevel(logging.DEBUG)
+    logging.getLogger().setLevel(logging.INFO)
     logging.basicConfig(format='{}: %(asctime)-15s %(message)s'.format(script_name()))
     chdir_to_script_directory()
 

ci/docker/qemu/vmcontrol.py

@@ -42,6 +42,8 @@
 #
 # The VMs are provisioned after boot, tests are run and then they are stopped
 #
+QEMU_SSH_PORT=2222
+QEMU_RAM=4096
 
 QEMU_RUN="""
 qemu-system-arm -M virt -m {ram} \
@@ -55,17 +57,32 @@
   -display none -nographic
 """
 
+QEMU_RUN_INTERACTIVE="""
+qemu-system-arm -M virt -m {ram} \
+  -kernel vmlinuz \
+  -initrd initrd.img \
+  -append 'root=/dev/vda1' \
+  -drive if=none,file=vda.qcow2,format=qcow2,id=hd \
+  -device virtio-blk-device,drive=hd \
+  -netdev user,id=mynet,hostfwd=tcp::{ssh_port}-:22 \
+  -device virtio-net-device,netdev=mynet \
+  -nographic
+"""
+
+
 class VMError(RuntimeError):
     pass
 
 class VM:
     """Control of the virtual machine"""
-    def __init__(self, ssh_port=2222):
+    def __init__(self, ssh_port=QEMU_SSH_PORT, ram=QEMU_RAM, interactive=False):
         self.log = logging.getLogger(VM.__name__)
         self.ssh_port = ssh_port
         self.timeout_s = 300
         self.qemu_process = None
         self._detach = False
+        self._interactive = interactive
+        self.ram = ram
 
     def __enter__(self):
         self.start()
@@ -77,13 +94,22 @@ def __exit__(self, exc_type, exc_value, traceback):
             self.terminate()
 
     def start(self):
-        self.log.info("Starting VM, ssh port redirected to localhost:%s", self.ssh_port)
+        sys.stderr.flush()
+        call(['toilet', '-f', 'smbraille', 'Starting QEMU'])
+        sys.stdout.flush()
+        self.log.info("Starting VM, ssh port redirected to localhost:%s (inside docker, not exposed by default)", self.ssh_port)
         if self.is_running():
             raise VMError("VM is running, shutdown first")
-        self.qemu_process = run_qemu(self.ssh_port)
+        if self._interactive:
+            self.qemu_process = Popen(shlex.split(QEMU_RUN_INTERACTIVE.format(ssh_port=self.ssh_port, ram=self.ram)))
+            return
+        else:
+            self.log.info("Starting in non-interactive mode. Terminal output is disabled.")
+            self.qemu_process = Popen(shlex.split(QEMU_RUN.format(ssh_port=self.ssh_port, ram=self.ram)), stdout=DEVNULL, stdin=DEVNULL, stderr=PIPE)
         def keep_waiting():
             return self.is_running()
 
+        logging.info("waiting for ssh to be open in the VM (timeout {}s)".format(self.timeout_s))
         ssh_working = wait_ssh_open('127.0.0.1', self.ssh_port, keep_waiting, self.timeout_s)
 
         if not self.is_running():
@@ -140,11 +166,28 @@ def __del__(self):
             logging.info("VM destructor hit")
             self.terminate()
 
-def run_qemu(ssh_port=2222):
-    cmd = QEMU_RUN.format(ssh_port=ssh_port, ram=4096)
-    logging.info("QEMU command: %s", cmd)
-    qemu_process = Popen(shlex.split(cmd), stdout=DEVNULL, stdin=DEVNULL, stderr=PIPE)
-    return qemu_process
+
+def qemu_ssh(ssh_port=QEMU_SSH_PORT, *args):
+    check_call(["ssh", "-o", "ServerAliveInterval=5", "-o", "StrictHostKeyChecking=no", "-p{}".format(ssh_port), "qemu@localhost", *args])
+
+
+def qemu_rsync(ssh_port, local_path, remote_path):
+    check_call(['rsync', '-e', 'ssh -o StrictHostKeyChecking=no -p{}'.format(ssh_port), '-a', local_path, 'qemu@localhost:{}'.format(remote_path)])
+
+def qemu_rsync_to_host(ssh_port, remote_path, local_path):
+    check_call(['rsync', '-e', 'ssh -o StrictHostKeyChecking=no -p{}'.format(ssh_port), '-va', 'qemu@localhost:{}'.format(remote_path), local_path])
+
+def qemu_provision(ssh_port=QEMU_SSH_PORT):
+    import glob
+    logging.info("Provisioning the VM with artifacts and sources")
+
+    artifact = glob.glob('/work/mxnet/build/*.whl')
+    for x in artifact:
+        qemu_rsync(ssh_port, x, 'mxnet_dist/')
+    qemu_rsync(ssh_port, '/work/runtime_functions.py','')
+    qemu_rsync(ssh_port, '/work/vmcontrol.py','')
+    qemu_rsync(ssh_port, 'mxnet/tests', 'mxnet')
+    logging.info("Provisioning completed successfully.")
 
 
 def wait_ssh_open(server, port, keep_waiting=None, timeout=None):
@@ -159,7 +202,7 @@ def wait_ssh_open(server, port, keep_waiting=None, timeout=None):
     import errno
     import time
     log = logging.getLogger('wait_ssh_open')
-    sleep_s = 0
+    sleep_s = 1
     if timeout:
         from time import time as now
         # time module is needed to calc timeout shared between two exceptions
@@ -183,7 +226,7 @@ def wait_ssh_open(server, port, keep_waiting=None, timeout=None):
                     log.debug("connect timeout %d s", next_timeout)
                     s.settimeout(next_timeout)
 
-            log.info("connect %s:%d", server, port)
+            log.debug("connect %s:%d", server, port)
             s.connect((server, port))
             ret = s.recv(1024).decode()
             if ret and ret.startswith('SSH'):