From c2f3d1f29db412287c694f0a6bb45849c19bd84d Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Tue, 4 Dec 2018 14:55:18 +0100 Subject: [PATCH 1/4] Improve dev_menu, add build command and virtualenv creation with local builds for easy testing --- .gitignore | 2 + ci/requirements_mxnet_virtualenv.txt | 6 +++ dev_menu.py | 76 +++++++++++++++++++++++++--- 3 files changed, 77 insertions(+), 7 deletions(-) create mode 100644 ci/requirements_mxnet_virtualenv.txt diff --git a/.gitignore b/.gitignore index c8a813649bb8..c7530ab69c6a 100644 --- a/.gitignore +++ b/.gitignore @@ -177,3 +177,5 @@ cxx *.gcno coverage.xml +# Local CMake build config +cmake_options.yml diff --git a/ci/requirements_mxnet_virtualenv.txt b/ci/requirements_mxnet_virtualenv.txt new file mode 100644 index 000000000000..f3564c628d55 --- /dev/null +++ b/ci/requirements_mxnet_virtualenv.txt @@ -0,0 +1,6 @@ +opencv-python +ipython +matplotlib +nose +nose-timer +mock diff --git a/dev_menu.py b/dev_menu.py index 0fd78cb222e3..0a1c7a7e2246 100755 --- a/dev_menu.py +++ b/dev_menu.py @@ -20,6 +20,7 @@ # -*- coding: utf-8 -*- """Tool to ease working with the build system and reproducing test results""" +import argparse import os import sys from subprocess import check_call @@ -29,6 +30,11 @@ from collections import OrderedDict import logging import yaml +import shutil + +DEFAULT_PYENV=os.environ.get('DEFAULT_PYENV','py3_venv') +DEFAULT_PYTHON=os.environ.get('DEFAULT_PYTHON','python3') +DEFAULT_CMAKE_OPTIONS=os.environ.get('DEFAULT_CMAKE_OPTIONS','cmake_options.yml') class Confirm(object): def __init__(self, cmds): @@ -46,7 +52,7 @@ def __call__(self): resp = input("Please answer yes or no: ") class CMake(object): - def __init__(self, cmake_options_yaml='cmake_options.yml', cmake_options_yaml_default='cmake/cmake_options.yml'): + def __init__(self, cmake_options_yaml=DEFAULT_CMAKE_OPTIONS, cmake_options_yaml_default='cmake/cmake_options.yml'): if os.path.exists(cmake_options_yaml): self.cmake_options_yaml = cmake_options_yaml else: @@ -87,10 +93,29 @@ def __call__(self, build_dir='build', generator='Ninja', build_cmd='ninja'): logging.info('Now building') check_call(shlex.split(build_cmd)) - +def create_virtualenv(venv_exe, pyexe, venv) -> None: + logging.info("Creating virtualenv in %s with python %s", venv, pyexe) + if not (venv_exe and pyexe and venv): + logging.warn("Skipping creation of virtualenv") + return + check_call([venv_exe, '-p', pyexe, venv]) + activate_this_py = os.path.join(venv, 'bin', 'activate_this.py') + # Activate virtualenv in this interpreter + exec(open(activate_this_py).read(), dict(__file__=activate_this_py)) + check_call(['pip', 'install', '--upgrade','--force-reinstall', '-e', 'python']) + check_call(['pip', 'install', '-r', 'ci/requirements_mxnet_virtualenv.txt']) + +def create_virtualenv_default(): + create_virtualenv('virtualenv', DEFAULT_PYTHON, DEFAULT_PYENV) + logging.info("You can use the virtualenv by executing 'source %s/bin/activate'", DEFAULT_PYENV) COMMANDS = OrderedDict([ - ('[Docker] sanity_check', + ('[Local build] CMake/Ninja build (using cmake_options.yaml (cp cmake/cmake_options.yml .) and edit) (creates {} virtualenv in "{}")'.format(DEFAULT_PYTHON, DEFAULT_PYENV), + [ + CMake(), + create_virtualenv_default, + ]), + ('[Docker] sanity_check. Check for linting and code formatting.', "ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh sanity_check"), ('[Docker] Python3 CPU unittests', [ @@ -117,8 +142,6 @@ def __call__(self, build_dir='build', generator='Ninja', build_cmd='ninja'): "ci/build.py -p armv7", "ci/build.py -p test.arm_qemu ./runtime_functions.py run_ut_py3_qemu" ]), - ('[Local] CMake build (using cmake/cmake_options.yaml)', - CMake()), ('Clean (RESET HARD) repository (Warning! erases local changes / DATA LOSS)', Confirm("ci/docker/runtime_functions.sh clean_repo")) ]) @@ -128,6 +151,9 @@ def clip(x, mini, maxi): @retry((ValueError, RuntimeError), 3, delay_s = 0) def show_menu(items: List[str], header=None) -> int: + print() + print('-- MXNet dev menu --') + print() def hr(): print(''.join(['-']*30)) if header: @@ -156,11 +182,47 @@ def handle_command(cmd): else: raise RuntimeError("handle_commands(cmds): argument should be str or List[str] but is %s", type(cmds)) -def main(): - logging.getLogger().setLevel(logging.INFO) +def use_menu_ui(args) -> None: command_list = list(COMMANDS.keys()) choice = show_menu(command_list, 'Available actions') handle_commands(COMMANDS[command_list[choice]]) + +def build(args) -> None: + """Build using CMake""" + venv_exe = shutil.which('virtualenv') + pyexe = shutil.which(args.pyexe) + if not venv_exe: + logging.warn("virtualenv wasn't found in path, it's recommended to install virutalenv to manage python environments") + if not pyexe: + logging.warn("Python executable %s not found in path", args.pyexe) + if args.cmake_options: + cmake = CMake(args.cmake_options) + else: + cmake = CMake() + cmake() + create_virtualenv(venv_exe, pyexe, args.venv) + +def main(): + logging.getLogger().setLevel(logging.INFO) + parser = argparse.ArgumentParser(description="""Utility for compiling and testing MXNet easily""") + parser.set_defaults(command='use_menu_ui') + + subparsers = parser.add_subparsers(help='sub-command help') + build_parser = subparsers.add_parser('build', help='build with the specified flags from file') + build_parser.add_argument('cmake_options', nargs='?', + help='File containing CMake options in YAML') + build_parser.add_argument('-v', '--venv', + type=str, + default=DEFAULT_PYENV, + help='virtualenv dir') + build_parser.add_argument('-p', '--pyexe', + type=str, + default=DEFAULT_PYTHON, + help='python executable') + + build_parser.set_defaults(command='build') + args = parser.parse_args() + globals()[args.command](args) return 0 if __name__ == '__main__': From 2200601d01b9a64da07c881a2b94841cc6138fa1 Mon Sep 17 00:00:00 2001 From: Jose Luis Contreras Date: Thu, 6 Dec 2018 14:19:28 +0100 Subject: [PATCH 2/4] Update dev_menu.py Co-Authored-By: larroy --- dev_menu.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/dev_menu.py b/dev_menu.py index 0a1c7a7e2246..4b6de4cb42cf 100755 --- a/dev_menu.py +++ b/dev_menu.py @@ -151,9 +151,7 @@ def clip(x, mini, maxi): @retry((ValueError, RuntimeError), 3, delay_s = 0) def show_menu(items: List[str], header=None) -> int: - print() - print('-- MXNet dev menu --') - print() + print('\n-- MXNet dev menu --\n') def hr(): print(''.join(['-']*30)) if header: From 7194bb89f0482a1a83cba90962626540c71d059f Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Mon, 10 Dec 2018 16:41:47 +0100 Subject: [PATCH 3/4] Cuda off by default, use ccache --- cmake/cmake_options.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cmake/cmake_options.yml b/cmake/cmake_options.yml index 01446f7b8f28..a4323feb92d4 100644 --- a/cmake/cmake_options.yml +++ b/cmake/cmake_options.yml @@ -16,7 +16,7 @@ # under the License. --- # CMake configuration -USE_CUDA: "ON" # Build with CUDA support +USE_CUDA: "OFF" # Build with CUDA support USE_OLDCMAKECUDA: "OFF" # Build with old cmake cuda USE_NCCL: "OFF" # Use NVidia NCCL with CUDA USE_OPENCV: "ON" # Build with OpenCV support @@ -48,3 +48,6 @@ USE_TENSORRT: "OFF" # Enable infeference optimization with TensorRT. USE_ASAN: "OFF" # Enable Clang/GCC ASAN sanitizers. ENABLE_TESTCOVERAGE: "OFF" # Enable compilation with test coverage metric output CMAKE_BUILD_TYPE: "Debug" +CMAKE_CUDA_COMPILER_LAUNCHER: "ccache" +CMAKE_C_COMPILER_LAUNCHER: "ccache" +CMAKE_CXX_COMPILER_LAUNCHER: "ccache" From f70db5b292162c9da5af0286b990f0ebc4bdfa1f Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Tue, 11 Dec 2018 17:17:50 +0000 Subject: [PATCH 4/4] address CR --- ci/requirements_mxnet_virtualenv.txt | 6 ------ dev_menu.py | 2 +- tests/requirements.txt | 1 + 3 files changed, 2 insertions(+), 7 deletions(-) delete mode 100644 ci/requirements_mxnet_virtualenv.txt diff --git a/ci/requirements_mxnet_virtualenv.txt b/ci/requirements_mxnet_virtualenv.txt deleted file mode 100644 index f3564c628d55..000000000000 --- a/ci/requirements_mxnet_virtualenv.txt +++ /dev/null @@ -1,6 +0,0 @@ -opencv-python -ipython -matplotlib -nose -nose-timer -mock diff --git a/dev_menu.py b/dev_menu.py index 4b6de4cb42cf..ebffe14f7620 100755 --- a/dev_menu.py +++ b/dev_menu.py @@ -103,7 +103,7 @@ def create_virtualenv(venv_exe, pyexe, venv) -> None: # Activate virtualenv in this interpreter exec(open(activate_this_py).read(), dict(__file__=activate_this_py)) check_call(['pip', 'install', '--upgrade','--force-reinstall', '-e', 'python']) - check_call(['pip', 'install', '-r', 'ci/requirements_mxnet_virtualenv.txt']) + check_call(['pip', 'install', '-r', 'tests/requirements.txt']) def create_virtualenv_default(): create_virtualenv('virtualenv', DEFAULT_PYTHON, DEFAULT_PYENV) diff --git a/tests/requirements.txt b/tests/requirements.txt index 3ca696b288c9..f64f7ffb6705 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -2,3 +2,4 @@ mock nose nose-timer +ipython