From 80e9d6f1e4e960d8a0cf80a871fcb1cbbfe83cab Mon Sep 17 00:00:00 2001 From: Pedro Larroy <928489+larroy@users.noreply.github.com> Date: Mon, 22 Oct 2018 15:06:32 +0200 Subject: [PATCH] =?UTF-8?q?[MXNET-793]=20=E2=98=85=20Virtualized=20testing?= =?UTF-8?q?=20in=20CI=20with=20QEMU=20=E2=98=85=20(#12094)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * virtual testing with qemu * Add install procedure * update installation * Refine test run * use direct ssh * update readme * Fix uneccesary cp * Minor refinements * Refine error conditions in startup * requirements installed inside QEMU * Update base image * Fix license * Dockerfile rename fallout * license fixes * refine documentation * license fix * update readme * Update qemu base image and refine documentation * Address CR comments wrt shebangs. * Address CR comments wrt comments. * adjust vda2 -> vda1 * Disable SMP, bug with newer kernel * Remove commented out code * Fix licenses * CR comments addressed * increase ram to 4096mb * Revert dockerfile renaming * Fix undo rename of dockerfiles * Address CR comments * CR --- ci/README.md | 10 + ci/build.py | 2 +- ci/docker/Dockerfile.build.android_armv7 | 0 ci/docker/Dockerfile.build.android_armv8 | 0 ci/docker/Dockerfile.build.armv6 | 0 ci/docker/Dockerfile.build.armv7 | 0 ci/docker/Dockerfile.build.armv8 | 0 ci/docker/Dockerfile.build.centos7_cpu | 0 ci/docker/Dockerfile.build.centos7_gpu | 0 ci/docker/Dockerfile.build.jetson | 0 ci/docker/Dockerfile.build.test.arm_qemu | 44 +++ ci/docker/Dockerfile.build.ubuntu_base_cpu | 0 ci/docker/Dockerfile.build.ubuntu_base_gpu | 0 ci/docker/Dockerfile.build.ubuntu_blc | 0 ci/docker/Dockerfile.build.ubuntu_build_cuda | 0 ci/docker/Dockerfile.build.ubuntu_cpu | 0 ci/docker/Dockerfile.build.ubuntu_gpu | 0 .../Dockerfile.build.ubuntu_gpu_tensorrt | 0 ci/docker/Dockerfile.build.ubuntu_nightly_cpu | 0 ci/docker/Dockerfile.build.ubuntu_nightly_gpu | 0 ci/docker/Dockerfile.build.ubuntu_rat | 0 ci/docker/install/ubuntu_arm_qemu.sh | 36 +++ ci/docker/install/ubuntu_arm_qemu_bin.sh | 40 +++ ci/docker/install/ubuntu_python.sh | 2 +- ci/docker/qemu/README.md | 1 + ci/docker/qemu/ansible.cfg | 20 ++ ci/docker/qemu/playbook.yml | 48 +++ ci/docker/qemu/qemu_run.sh | 31 ++ ci/docker/qemu/runtime_functions.py | 117 ++++++++ ci/docker/qemu/vmcontrol.py | 274 ++++++++++++++++++ ci/qemu/README.md | 71 +++++ ci/qemu/copy.sh | 23 ++ ci/qemu/init.sh | 23 ++ ci/qemu/initrd_modif/inittab | 38 +++ ci/qemu/install.sh | 32 ++ ci/qemu/mxnet_requirements.txt | 7 + ci/qemu/preseed.cfg | 68 +++++ ci/qemu/preseed.sh | 29 ++ ci/qemu/run.sh | 33 +++ tests/requirements.txt | 1 + tools/license_header.py | 3 +- 41 files changed, 950 insertions(+), 3 deletions(-) mode change 100755 => 100644 ci/docker/Dockerfile.build.android_armv7 mode change 100755 => 100644 ci/docker/Dockerfile.build.android_armv8 mode change 100755 => 100644 ci/docker/Dockerfile.build.armv6 mode change 100755 => 100644 ci/docker/Dockerfile.build.armv7 mode change 100755 => 100644 ci/docker/Dockerfile.build.armv8 mode change 100755 => 100644 ci/docker/Dockerfile.build.centos7_cpu mode change 100755 => 100644 ci/docker/Dockerfile.build.centos7_gpu mode change 100755 => 100644 ci/docker/Dockerfile.build.jetson create mode 100644 ci/docker/Dockerfile.build.test.arm_qemu mode change 100755 => 100644 ci/docker/Dockerfile.build.ubuntu_base_cpu mode change 100755 => 100644 ci/docker/Dockerfile.build.ubuntu_base_gpu mode change 100755 => 100644 ci/docker/Dockerfile.build.ubuntu_blc mode change 100755 => 100644 ci/docker/Dockerfile.build.ubuntu_build_cuda mode change 100755 => 100644 ci/docker/Dockerfile.build.ubuntu_cpu mode change 100755 => 100644 ci/docker/Dockerfile.build.ubuntu_gpu mode change 100755 => 100644 ci/docker/Dockerfile.build.ubuntu_gpu_tensorrt mode change 100755 => 100644 ci/docker/Dockerfile.build.ubuntu_nightly_cpu mode change 100755 => 100644 ci/docker/Dockerfile.build.ubuntu_nightly_gpu mode change 100755 => 100644 ci/docker/Dockerfile.build.ubuntu_rat create mode 100755 ci/docker/install/ubuntu_arm_qemu.sh create mode 100755 ci/docker/install/ubuntu_arm_qemu_bin.sh create mode 100644 ci/docker/qemu/README.md create mode 100644 ci/docker/qemu/ansible.cfg create mode 100644 ci/docker/qemu/playbook.yml create mode 100755 ci/docker/qemu/qemu_run.sh create mode 100755 ci/docker/qemu/runtime_functions.py create mode 100644 ci/docker/qemu/vmcontrol.py create mode 100644 ci/qemu/README.md create mode 100755 ci/qemu/copy.sh create mode 100755 ci/qemu/init.sh create mode 100644 ci/qemu/initrd_modif/inittab create mode 100755 ci/qemu/install.sh create mode 100644 ci/qemu/mxnet_requirements.txt create mode 100644 ci/qemu/preseed.cfg create mode 100755 ci/qemu/preseed.sh create mode 100755 ci/qemu/run.sh diff --git a/ci/README.md b/ci/README.md index 693087569434..3737fc740653 100644 --- a/ci/README.md +++ b/ci/README.md @@ -90,3 +90,13 @@ For all builds a directory from the host system is mapped where ccache will stor compiled object files (defaults to /tmp/ci_ccache). This will speed up rebuilds significantly. You can set this directory explicitly by setting CCACHE_DIR environment variable. All ccache instances are currently set to be 10 Gigabytes max in size. + + +## Testing with QEMU +To run the unit tests under qemu: +``` +./build.py -p armv7 && ./build.py -p test.arm_qemu ./runtime_functions.py run_ut_py3_qemu +``` + +To get a shell on the container and debug issues with the emulator itself: +Run the output of `./build.py -p test.arm_qemu --print-docker-run` diff --git a/ci/build.py b/ci/build.py index b7c86adda715..e2554d9b8cea 100755 --- a/ci/build.py +++ b/ci/build.py @@ -394,7 +394,7 @@ def main() -> int: help="platform", type=str) - parser.add_argument("--build-only", + parser.add_argument("-b", "--build-only", help="Only build the container, don't build the project", action='store_true') diff --git a/ci/docker/Dockerfile.build.android_armv7 b/ci/docker/Dockerfile.build.android_armv7 old mode 100755 new mode 100644 diff --git a/ci/docker/Dockerfile.build.android_armv8 b/ci/docker/Dockerfile.build.android_armv8 old mode 100755 new mode 100644 diff --git a/ci/docker/Dockerfile.build.armv6 b/ci/docker/Dockerfile.build.armv6 old mode 100755 new mode 100644 diff --git a/ci/docker/Dockerfile.build.armv7 b/ci/docker/Dockerfile.build.armv7 old mode 100755 new mode 100644 diff --git a/ci/docker/Dockerfile.build.armv8 b/ci/docker/Dockerfile.build.armv8 old mode 100755 new mode 100644 diff --git a/ci/docker/Dockerfile.build.centos7_cpu b/ci/docker/Dockerfile.build.centos7_cpu old mode 100755 new mode 100644 diff --git a/ci/docker/Dockerfile.build.centos7_gpu b/ci/docker/Dockerfile.build.centos7_gpu old mode 100755 new mode 100644 diff --git a/ci/docker/Dockerfile.build.jetson b/ci/docker/Dockerfile.build.jetson old mode 100755 new mode 100644 diff --git a/ci/docker/Dockerfile.build.test.arm_qemu b/ci/docker/Dockerfile.build.test.arm_qemu new file mode 100644 index 000000000000..fde105c3f983 --- /dev/null +++ b/ci/docker/Dockerfile.build.test.arm_qemu @@ -0,0 +1,44 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to build and run MXNet on Ubuntu 16.04 for CPU + +FROM ubuntu:16.04 + +WORKDIR /work + +RUN apt-get update +COPY install/ubuntu_python.sh /work/ +RUN /work/ubuntu_python.sh + +COPY install/ubuntu_arm_qemu.sh /work +RUN /work/ubuntu_arm_qemu.sh + +COPY install/ubuntu_arm_qemu_bin.sh /work +RUN /work/ubuntu_arm_qemu_bin.sh + +ARG USER_ID=0 +ARG GROUP_ID=0 +COPY install/ubuntu_adduser.sh /work/ +RUN /work/ubuntu_adduser.sh + +COPY runtime_functions.sh /work/ +COPY qemu/* /work/ +COPY qemu/ansible.cfg /etc/ansible/ansible.cfg + +CMD ["./runtime_functions.py","run_ut_py3_qemu"] diff --git a/ci/docker/Dockerfile.build.ubuntu_base_cpu b/ci/docker/Dockerfile.build.ubuntu_base_cpu old mode 100755 new mode 100644 diff --git a/ci/docker/Dockerfile.build.ubuntu_base_gpu b/ci/docker/Dockerfile.build.ubuntu_base_gpu old mode 100755 new mode 100644 diff --git a/ci/docker/Dockerfile.build.ubuntu_blc b/ci/docker/Dockerfile.build.ubuntu_blc old mode 100755 new mode 100644 diff --git a/ci/docker/Dockerfile.build.ubuntu_build_cuda b/ci/docker/Dockerfile.build.ubuntu_build_cuda old mode 100755 new mode 100644 diff --git a/ci/docker/Dockerfile.build.ubuntu_cpu b/ci/docker/Dockerfile.build.ubuntu_cpu old mode 100755 new mode 100644 diff --git a/ci/docker/Dockerfile.build.ubuntu_gpu b/ci/docker/Dockerfile.build.ubuntu_gpu old mode 100755 new mode 100644 diff --git a/ci/docker/Dockerfile.build.ubuntu_gpu_tensorrt b/ci/docker/Dockerfile.build.ubuntu_gpu_tensorrt old mode 100755 new mode 100644 diff --git a/ci/docker/Dockerfile.build.ubuntu_nightly_cpu b/ci/docker/Dockerfile.build.ubuntu_nightly_cpu old mode 100755 new mode 100644 diff --git a/ci/docker/Dockerfile.build.ubuntu_nightly_gpu b/ci/docker/Dockerfile.build.ubuntu_nightly_gpu old mode 100755 new mode 100644 diff --git a/ci/docker/Dockerfile.build.ubuntu_rat b/ci/docker/Dockerfile.build.ubuntu_rat old mode 100755 new mode 100644 diff --git a/ci/docker/install/ubuntu_arm_qemu.sh b/ci/docker/install/ubuntu_arm_qemu.sh new file mode 100755 index 000000000000..c30dc4f13d1e --- /dev/null +++ b/ci/docker/install/ubuntu_arm_qemu.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# build and install are separated so changes to build don't invalidate +# the whole docker cache for the image + +set -exuo pipefail + +apt-get install -y \ + cmake \ + curl \ + wget \ + git \ + qemu \ + qemu-system-arm \ + unzip \ + bzip2 \ + vim-nox + +pip3 install ansible ipython diff --git a/ci/docker/install/ubuntu_arm_qemu_bin.sh b/ci/docker/install/ubuntu_arm_qemu_bin.sh new file mode 100755 index 000000000000..d4f81185c169 --- /dev/null +++ b/ci/docker/install/ubuntu_arm_qemu_bin.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# build and install are separated so changes to build don't invalidate +# the whole docker cache for the image + +set -exuo pipefail + +# +# This disk image and kernels for virtual testing with QEMU is generated with some manual OS +# installation steps with the scripts and documentation found in the ci/qemu/ folder. +# +# The image has a base Debian OS and MXNet runtime dependencies installed. +# The root password is empty and there's a "qemu" user without password. SSH access is enabled as +# well. +# +# See also: ci/qemu/README.md +# + +REMOTE="https://s3-us-west-2.amazonaws.com/mxnet-ci-prod-slave-data" +curl -f ${REMOTE}/vda_debian_stretch.qcow2.bz2 | bunzip2 > vda.qcow2 +curl -f ${REMOTE}/vmlinuz -o vmlinuz +curl -f ${REMOTE}/initrd.img -o initrd.img + diff --git a/ci/docker/install/ubuntu_python.sh b/ci/docker/install/ubuntu_python.sh index a60516386652..d6e66aa45c3d 100755 --- a/ci/docker/install/ubuntu_python.sh +++ b/ci/docker/install/ubuntu_python.sh @@ -22,7 +22,7 @@ set -ex # install libraries for mxnet's python package on ubuntu -apt-get install -y python-dev python3-dev virtualenv +apt-get install -y python-dev python3-dev virtualenv wget # the version of the pip shipped with ubuntu may be too lower, install a recent version here wget -nv https://bootstrap.pypa.io/get-pip.py diff --git a/ci/docker/qemu/README.md b/ci/docker/qemu/README.md new file mode 100644 index 000000000000..1dcaa5aeb607 --- /dev/null +++ b/ci/docker/qemu/README.md @@ -0,0 +1 @@ +These are files used in the docker container that runs QEMU diff --git a/ci/docker/qemu/ansible.cfg b/ci/docker/qemu/ansible.cfg new file mode 100644 index 000000000000..24e2ec87b94f --- /dev/null +++ b/ci/docker/qemu/ansible.cfg @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[defaults] +host_key_checking = False +ansible_python_interpreter = /usr/bin/python3 diff --git a/ci/docker/qemu/playbook.yml b/ci/docker/qemu/playbook.yml new file mode 100644 index 000000000000..3b9e7c52bded --- /dev/null +++ b/ci/docker/qemu/playbook.yml @@ -0,0 +1,48 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +--- +- name: provision QEMU VM + hosts: all + gather_facts: no + become: true + become_user: root + tasks: + - name: Wait until ssh is available + wait_for_connection: + delay: 0 + sleep: 3 + timeout: 400 + - command: hostname + register: h + - debug: msg="{{ h.stdout }}" + + - name: copy mxnet artifacts + copy: + src: "{{ item }}" + dest: mxnet_dist/ + with_fileglob: "/work/mxnet/build/*.whl" + + - name: copy runtime_functions.py + copy: + src: "/work/runtime_functions.py" + dest: . + - file: + path: runtime_functions.py + mode: 0755 + + diff --git a/ci/docker/qemu/qemu_run.sh b/ci/docker/qemu/qemu_run.sh new file mode 100755 index 000000000000..53a6487aa0db --- /dev/null +++ b/ci/docker/qemu/qemu_run.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -exuo pipefail + +qemu-system-arm -M virt -m 1024 \ + -kernel vmlinuz \ + -initrd initrd.img \ + -append 'root=/dev/vda1' \ + -drive if=none,file=vda.qcow2,format=qcow2,id=hd \ + -device virtio-blk-device,drive=hd \ + -netdev user,id=mynet,hostfwd=tcp::2222-:22 \ + -device virtio-net-device,netdev=mynet \ + -nographic \ + -display none diff --git a/ci/docker/qemu/runtime_functions.py b/ci/docker/qemu/runtime_functions.py new file mode 100755 index 000000000000..6cf01b6f9127 --- /dev/null +++ b/ci/docker/qemu/runtime_functions.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# -*- coding: utf-8 -*- +"""Runtime functions to use in docker / testing""" + +__author__ = 'Pedro Larroy' +__version__ = '0.1' + +import os +import sys +import subprocess +import argparse +import logging +from subprocess import call, check_call, Popen, DEVNULL, PIPE +import time +import sys +import types +import glob + +def activate_this(base): + import site + import os + import sys + if sys.platform == 'win32': + site_packages = os.path.join(base, 'Lib', 'site-packages') + else: + site_packages = os.path.join(base, 'lib', 'python%s' % sys.version[:3], 'site-packages') + prev_sys_path = list(sys.path) + sys.real_prefix = sys.prefix + sys.prefix = base + # Move the added items to the front of the path: + new_sys_path = [] + for item in list(sys.path): + if item not in prev_sys_path: + new_sys_path.append(item) + sys.path.remove(item) + sys.path[:0] = new_sys_path + +def run_ut_py3_qemu(): + from vmcontrol import VM + with VM() as vm: + logging.info("VM provisioning with ansible") + check_call(["ansible-playbook", "-v", "-u", "qemu", "-i", "localhost:{},".format(vm.ssh_port), "playbook.yml"]) + logging.info("VM provisioned successfully.") + logging.info("sync tests") + check_call(['rsync', '-e', 'ssh -p{}'.format(vm.ssh_port), '-a', 'mxnet/tests', 'qemu@localhost:mxnet']) + logging.info("execute tests") + check_call(["ssh", "-o", "ServerAliveInterval=5", "-p{}".format(vm.ssh_port), "qemu@localhost", "./runtime_functions.py", "run_ut_python3_qemu_internal"]) + logging.info("tests finished, vm shutdown.") + vm.shutdown() + +def run_ut_python3_qemu_internal(): + """this runs inside the vm, it's run by the playbook above by ansible""" + pkg = glob.glob('mxnet_dist/*.whl')[0] + logging.info("=== NOW Running inside QEMU ===") + logging.info("PIP Installing %s", pkg) + check_call(['sudo', 'pip3', 'install', pkg]) + logging.info("PIP Installing mxnet/tests/requirements.txt") + check_call(['sudo', 'pip3', 'install', '-r', 'mxnet/tests/requirements.txt']) + logging.info("Running tests in mxnet/tests/python/unittest/") + check_call(['nosetests', '--with-timer', '--with-xunit', '--xunit-file', 'nosetests_unittest.xml', '--verbose', 'mxnet/tests/python/unittest/test_ndarray.py:test_ndarray_fluent']) + + +def parsed_args(): + parser = argparse.ArgumentParser(description="""python runtime functions""", epilog="") + parser.add_argument('command',nargs='*', + help="Name of the function to run with arguments") + args = parser.parse_args() + return (args, parser) + +def script_name() -> str: + return os.path.split(sys.argv[0])[1] + +def chdir_to_script_directory(): + # We need to be in the same directory than the script so the commands in the dockerfiles work as + # expected. But the script can be invoked from a different path + base = os.path.split(os.path.realpath(__file__))[0] + os.chdir(base) + +def main(): + logging.getLogger().setLevel(logging.DEBUG) + logging.basicConfig(format='{}: %(asctime)-15s %(message)s'.format(script_name())) + chdir_to_script_directory() + + # Run function with name passed as argument + (args, parser) = parsed_args() + logging.info("%s", args.command) + if args.command: + fargs = args.command[1:] + globals()[args.command[0]](*fargs) + return 0 + else: + parser.print_help() + fnames = [x for x in globals() if type(globals()[x]) is types.FunctionType] + print('\nAvailable functions: {}'.format(' '.join(fnames))) + return 1 + +if __name__ == '__main__': + sys.exit(main()) + diff --git a/ci/docker/qemu/vmcontrol.py b/ci/docker/qemu/vmcontrol.py new file mode 100644 index 000000000000..2262bc77bf9b --- /dev/null +++ b/ci/docker/qemu/vmcontrol.py @@ -0,0 +1,274 @@ +#!/usr/bin/env python3 + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# -*- coding: utf-8 -*- +"""Utilities to control a guest VM, used for virtual testing with QEMU""" + +__author__ = 'Pedro Larroy' +__version__ = '0.1' + +import os +import sys +import subprocess +import argparse +import logging +from subprocess import call, check_call, Popen, DEVNULL, PIPE +import time +import sys +import multiprocessing +import shlex + +################################################### +# +# Virtual testing with QEMU +# +# We start QEMU instances that have a local port in the host redirected to the ssh port. +# +# The VMs are provisioned after boot, tests are run and then they are stopped +# + +QEMU_RUN=""" +qemu-system-arm -M virt -m {ram} \ + -kernel vmlinuz \ + -initrd initrd.img \ + -append 'root=/dev/vda1' \ + -drive if=none,file=vda.qcow2,format=qcow2,id=hd \ + -device virtio-blk-device,drive=hd \ + -netdev user,id=mynet,hostfwd=tcp::{ssh_port}-:22 \ + -device virtio-net-device,netdev=mynet \ + -display none -nographic +""" + +class VMError(RuntimeError): + pass + +class VM: + """Control of the virtual machine""" + def __init__(self, ssh_port=2222): + self.log = logging.getLogger(VM.__name__) + self.ssh_port = ssh_port + self.timeout_s = 300 + self.qemu_process = None + self._detach = False + + def __enter__(self): + self.start() + return self + + def __exit__(self, exc_type, exc_value, traceback): + if not self._detach: + self.shutdown() + self.terminate() + + def start(self): + self.log.info("Starting VM, ssh port redirected to localhost:%s", self.ssh_port) + if self.is_running(): + raise VMError("VM is running, shutdown first") + self.qemu_process = run_qemu(self.ssh_port) + def keep_waiting(): + return self.is_running() + + ssh_working = wait_ssh_open('127.0.0.1', self.ssh_port, keep_waiting, self.timeout_s) + + if not self.is_running(): + (_, stderr) = self.qemu_process.communicate() + raise VMError("VM failed to start, retcode: {}, stderr: {}".format( self.retcode(), stderr.decode())) + + if not ssh_working: + if self.is_running(): + self.log.error("VM running but SSH is not working") + self.terminate() + raise VMError("SSH is not working after {} seconds".format(self.timeout_s)) + self.log.info("VM is online and SSH is up") + + def is_running(self): + return self.qemu_process and self.qemu_process.poll() is None + + def retcode(self): + if self.qemu_process: + return self.qemu_process.poll() + else: + raise RuntimeError('qemu process was not started') + + def terminate(self): + if self.qemu_process: + logging.info("send term signal") + self.qemu_process.terminate() + time.sleep(3) + logging.info("send kill signal") + self.qemu_process.kill() + self.qemu_process.wait() + self.qemu_process = None + else: + logging.warn("VM.terminate: QEMU process not running") + + def detach(self): + self._detach = True + + def shutdown(self): + if self.qemu_process: + logging.info("Shutdown via ssh") + # ssh connection will be closed with an error + call(["ssh", "-o", "StrictHostKeyChecking=no", "-p", str(self.ssh_port), "qemu@localhost", + "sudo", "poweroff"]) + ret = self.qemu_process.wait(timeout=90) + self.log.info("VM on port %s has shutdown (exit code %d)", self.ssh_port, ret) + self.qemu_process = None + + def wait(self): + if self.qemu_process: + self.qemu_process.wait() + + def __del__(self): + if self.is_running and not self._detach: + logging.info("VM destructor hit") + self.terminate() + +def run_qemu(ssh_port=2222): + cmd = QEMU_RUN.format(ssh_port=ssh_port, ram=4096) + logging.info("QEMU command: %s", cmd) + qemu_process = Popen(shlex.split(cmd), stdout=DEVNULL, stdin=DEVNULL, stderr=PIPE) + return qemu_process + + +def wait_ssh_open(server, port, keep_waiting=None, timeout=None): + """ Wait for network service to appear + @param server: host to connect to (str) + @param port: port (int) + @param timeout: in seconds, if None or 0 wait forever + @return: True of False, if timeout is None may return only True or + throw unhandled network exception + """ + import socket + import errno + import time + log = logging.getLogger('wait_ssh_open') + sleep_s = 0 + if timeout: + from time import time as now + # time module is needed to calc timeout shared between two exceptions + end = now() + timeout + + while True: + log.debug("Sleeping for %s second(s)", sleep_s) + time.sleep(sleep_s) + s = socket.socket() + try: + if keep_waiting and not keep_waiting(): + log.debug("keep_waiting() is set and evaluates to False") + return False + + if timeout: + next_timeout = end - now() + if next_timeout < 0: + log.debug("connect time out") + return False + else: + log.debug("connect timeout %d s", next_timeout) + s.settimeout(next_timeout) + + log.info("connect %s:%d", server, port) + s.connect((server, port)) + ret = s.recv(1024).decode() + if ret and ret.startswith('SSH'): + s.close() + log.info("wait_ssh_open: port %s:%s is open and ssh is ready", server, port) + return True + else: + log.debug("Didn't get the SSH banner") + s.close() + + except ConnectionError as err: + log.debug("ConnectionError %s", err) + if sleep_s == 0: + sleep_s = 1 + else: + sleep_s *= 2 + + except socket.gaierror as err: + log.debug("gaierror %s",err) + return False + + except socket.timeout as err: + # this exception occurs only if timeout is set + if timeout: + return False + + except TimeoutError as err: + # catch timeout exception from underlying network library + # this one is different from socket.timeout + raise + + +def wait_port_open(server, port, timeout=None): + """ Wait for network service to appear + @param server: host to connect to (str) + @param port: port (int) + @param timeout: in seconds, if None or 0 wait forever + @return: True of False, if timeout is None may return only True or + throw unhandled network exception + """ + import socket + import errno + import time + sleep_s = 0 + if timeout: + from time import time as now + # time module is needed to calc timeout shared between two exceptions + end = now() + timeout + + while True: + logging.debug("Sleeping for %s second(s)", sleep_s) + time.sleep(sleep_s) + s = socket.socket() + try: + if timeout: + next_timeout = end - now() + if next_timeout < 0: + return False + else: + s.settimeout(next_timeout) + + logging.info("connect %s %d", server, port) + s.connect((server, port)) + + except ConnectionError as err: + logging.debug("ConnectionError %s", err) + if sleep_s == 0: + sleep_s = 1 + + except socket.gaierror as err: + logging.debug("gaierror %s",err) + return False + + except socket.timeout as err: + # this exception occurs only if timeout is set + if timeout: + return False + + except TimeoutError as err: + # catch timeout exception from underlying network library + # this one is different from socket.timeout + raise + + else: + s.close() + logging.info("wait_port_open: port %s:%s is open", server, port) + return True + diff --git a/ci/qemu/README.md b/ci/qemu/README.md new file mode 100644 index 000000000000..6dde8916b28c --- /dev/null +++ b/ci/qemu/README.md @@ -0,0 +1,71 @@ +# QEMU base image creation + +This folder contains scripts and configuration to create a QEMU virtual drive with a debian system. + +The order of execution is: +- `init.sh` to download the installation kernel and ramdisk +- `preseed.sh` to preseed the debian installer so it doesn't ask questions +- `copy.sh` to extract the kernel and ramdisk from the installed system +- `run.sh` to boot the system and fine tune the image + +# Description of the process: + +# Preparing the base image + +First, an installation is made using installer kernel and initrd by using the scripts above. + +# After installation, we extract initrd and kernel from the installation drive + +The commands look like this: + +`virt-copy-out -a hda.qcow2 /boot/initrd.img-4.15.0-30-generic-lpae .` + +In the same way for the kernel. + +Then we install packages and dependencies on the qemu image: + +apt install -y sudo python3-dev virtualenv wget libgfortran3 libopenblas-base rsync build-essential +libopenblas-dev libomp5 + +We enable sudo and passwordless logins: + +Add file `/etc/sudoers.d/01-qemu` +With content: +``` +qemu ALL=(ALL) NOPASSWD: ALL +``` + +Edit: `/etc/ssh/sshd_config` + +And set the following options: +``` +PermitEmptyPasswords yes +PasswordAuthentication yes +PermitRootLogin yes +``` + +Disable root and user passwords with `passwd -d` + +Edit ` /etc/pam.d/common-auth` + +Replace `auth [success=1 default=ignore] pam_unix.so nullok_secure` by +``` +auth [success=1 default=ignore] pam_unix.so nullok +``` + +As root to install system wide: + +``` +wget -nv https://bootstrap.pypa.io/get-pip.py +python3 get-pip.py +apt-get clean +``` + +Afterwards install mxnet python3 deps: + +``` +pip3 install -r mxnet_requirements.txt +``` + + +To access qemu control console from tmux: `ctrl-a a c` diff --git a/ci/qemu/copy.sh b/ci/qemu/copy.sh new file mode 100755 index 000000000000..f39a9d083509 --- /dev/null +++ b/ci/qemu/copy.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash -exuo pipefail + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Extract kernel from image + +set -ex +virt-copy-out -a vda.qcow2 /boot/vmlinuz-3.16.0-6-armmp-lpae /boot/initrd.img-3.16.0-6-armmp-lpae . diff --git a/ci/qemu/init.sh b/ci/qemu/init.sh new file mode 100755 index 000000000000..1698cb10f272 --- /dev/null +++ b/ci/qemu/init.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash -exuo pipefail + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Download the installer and ramdisk for intallation +set -ex +wget -O installer-vmlinuz http://http.us.debian.org/debian/dists/jessie/main/installer-armhf/current/images/netboot/vmlinuz +wget -O installer-initrd.gz http://http.us.debian.org/debian/dists/jessie/main/installer-armhf/current/images/netboot/initrd.gz diff --git a/ci/qemu/initrd_modif/inittab b/ci/qemu/initrd_modif/inittab new file mode 100644 index 000000000000..064512595fbc --- /dev/null +++ b/ci/qemu/initrd_modif/inittab @@ -0,0 +1,38 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# /etc/inittab +# busybox init configuration for debian-installer + +# main rc script +::sysinit:/sbin/reopen-console /sbin/debian-installer-startup + +# main setup program +::respawn:/sbin/reopen-console /sbin/debian-installer + +# convenience shells +tty2::askfirst:-/bin/sh +tty3::askfirst:-/bin/sh + +# logging +#tty4::respawn:/usr/bin/tail -f /var/log/syslog + +# Stuff to do before rebooting +::ctrlaltdel:/sbin/shutdown > /dev/null 2>&1 + +# re-exec init on receipt of SIGHUP/SIGUSR1 +::restart:/sbin/init diff --git a/ci/qemu/install.sh b/ci/qemu/install.sh new file mode 100755 index 000000000000..8531b033d074 --- /dev/null +++ b/ci/qemu/install.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -ex +rm -f vda.qcow2 +sudo ./preseed.sh +qemu-img create -f qcow2 vda.qcow2 10G +qemu-system-arm -M virt -m 1024 \ + -kernel installer-vmlinuz \ + -append BOOT_DEBUG=2,DEBIAN_FRONTEND=noninteractive \ + -initrd installer-initrd_automated.gz \ + -drive if=none,file=vda.qcow2,format=qcow2,id=hd \ + -device virtio-blk-device,drive=hd \ + -netdev user,id=mynet \ + -device virtio-net-device,netdev=mynet \ + -nographic -no-reboot diff --git a/ci/qemu/mxnet_requirements.txt b/ci/qemu/mxnet_requirements.txt new file mode 100644 index 000000000000..a2e485efed19 --- /dev/null +++ b/ci/qemu/mxnet_requirements.txt @@ -0,0 +1,7 @@ +urllib3<1.23,>=1.21.1 +requests<2.19.0,>=2.18.4 +graphviz<0.9.0,>=0.8.1 +numpy<=1.15.0,>=1.8.2 +mock +nose +nose-timer diff --git a/ci/qemu/preseed.cfg b/ci/qemu/preseed.cfg new file mode 100644 index 000000000000..23a8fc3baebf --- /dev/null +++ b/ci/qemu/preseed.cfg @@ -0,0 +1,68 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +d-i debian-installer/locale string en_US +d-i keyboard-configuration/xkb-keymap select us +d-i netcfg/get_hostname string debian-qemu +d-i netcfg/get_domain string lab +d-i passwd/root-login boolean true +d-i passwd/root-password password debian +d-i passwd/root-password-again password debian +d-i clock-setup/utc boolean true +d-i mirror/country string US +d-i mirror/https/proxy string +d-i mirror/http/proxy string +d-i mirror/ftp/proxy string +d-i mirror/http/countries select US +d-i mirror/http/hostname string ftp.us.debian.org +d-i mirror/http/mirror select ftp.us.debian.org +d-i localechooser/preferred-locale select en_US.UTF-8 +apt-mirror-setup apt-setup/use_mirror boolean false +apt-mirror-setup apt-setup/mirror/error select Retry +d-i passwd/username string qemu +d-i passwd/user-password password qemu +d-i passwd/user-password-again password qemu +user-setup-udeb passwd/username string qemu +user-setup-udeb passwd/user-fullname string qemu +d-i time/zone string GMT +d-i partman-auto/choose_recipe select atomic +#partman-auto partman-auto/select_disk select /var/lib/partman/devices/=dev=vda +#partman-auto partman-auto/automatically_partition select +#partman-target partman-target/no_root error +#partman-auto partman-auto/init_automatically_partition select 50some_device__________regular +#partman-auto partman-auto/disk string vda +#partman-auto partman-auto/expert_recipe string \ +# boot-root :: \ +# 100 10000 1000000000 ext4 \ +# $primary{ } \ +# lv_name{ root } \ +# method{ format } \ +# format{ } \ +# use_filesystem{ } \ +# filesystem{ ext4 } \ +# mountpoint{ / } . +# +#d-i partman-partitioning/confirm_write_new_label boolean true +#d-i partman/choose_partition select finish +#d-i partman/confirm boolean true +#d-i partman/confirm_nooverwrite boolean true +#partman-base partman/choose_partition select 90finish__________finish +#partman-basicfilesystems partman-basicfilesystems/swap_check_failed boolean +d-i popularity-contest/participate boolean false +d-i tasksel/first multiselect SSH server, standard system utilities +d-i debian-installer/main-menu select Finish the installation +d-i debian-installer/exit/poweroff boolean true diff --git a/ci/qemu/preseed.sh b/ci/qemu/preseed.sh new file mode 100755 index 000000000000..ad005548fbbe --- /dev/null +++ b/ci/qemu/preseed.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash -exuo pipefail + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -ex +rm -rf initrd +mkdir -p initrd +cd initrd +gunzip -c ../installer-initrd.gz | cpio -i +cp ../preseed.cfg . +cp ../initrd_modif/inittab etc/inittab +cp ../initrd_modif/S10syslog lib/debian-installer-startup.d/S10syslog +find . | cpio --create --format 'newc' | gzip -c > ../installer-initrd_automated.gz +echo "Done!" diff --git a/ci/qemu/run.sh b/ci/qemu/run.sh new file mode 100755 index 000000000000..eeff4e1fdccb --- /dev/null +++ b/ci/qemu/run.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash -exuo pipefail + + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -ex +disk=${1:-vda.qcow2} +qemu-system-arm -M virt -m 1024 \ + -kernel vmlinuz-3.16.0-6-armmp-lpae \ + -initrd initrd.img-3.16.0-6-armmp-lpae \ + -smp 4 \ + -append 'root=/dev/vda1' \ + -drive if=none,file=$disk,format=qcow2,id=hd \ + -device virtio-blk-device,drive=hd \ + -netdev user,id=mynet,hostfwd=tcp::2222-:22 \ + -device virtio-net-device,netdev=mynet \ + -nographic +# -display none diff --git a/tests/requirements.txt b/tests/requirements.txt index 0eca73fbb02a..3ca696b288c9 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,3 +1,4 @@ # Requirements for tests, those are installed before running on the virtualenv mock nose +nose-timer diff --git a/tools/license_header.py b/tools/license_header.py index f6726891f521..2f12a14795c5 100755 --- a/tools/license_header.py +++ b/tools/license_header.py @@ -77,7 +77,8 @@ 'prepare_mkl.sh', 'example/image-classification/predict-cpp/image-classification-predict.cc', 'src/operator/contrib/ctc_include/', - 'julia/REQUIRE'] + 'julia/REQUIRE' + ] # language extensions and the according commment mark _LANGS = {'.cc':'*', '.h':'*', '.cu':'*', '.cuh':'*', '.py':'#',