Improve dev_menu usability, local build and virtualenv (#13529)

* Improve dev_menu, add build command and virtualenv creation with local builds for easy testing

* Update dev_menu.py

Co-Authored-By: larroy <[email protected]>

* Cuda off by default, use ccache

* address CR

.gitignore

@@ -177,3 +177,5 @@ cxx
 *.gcno
 coverage.xml
 
+# Local CMake build config
+cmake_options.yml

cmake/cmake_options.yml

@@ -16,7 +16,7 @@
 # under the License.
 
 --- # CMake configuration
-USE_CUDA: "ON" # Build with CUDA support
+USE_CUDA: "OFF" # Build with CUDA support
 USE_OLDCMAKECUDA: "OFF" # Build with old cmake cuda
 USE_NCCL: "OFF" # Use NVidia NCCL with CUDA
 USE_OPENCV: "ON" # Build with OpenCV support
@@ -48,3 +48,6 @@ USE_TENSORRT: "OFF" # Enable infeference optimization with TensorRT.
 USE_ASAN: "OFF" # Enable Clang/GCC ASAN sanitizers.
 ENABLE_TESTCOVERAGE: "OFF" # Enable compilation with test coverage metric output
 CMAKE_BUILD_TYPE: "Debug"
+CMAKE_CUDA_COMPILER_LAUNCHER: "ccache"
+CMAKE_C_COMPILER_LAUNCHER: "ccache"
+CMAKE_CXX_COMPILER_LAUNCHER: "ccache"

dev_menu.py

@@ -20,6 +20,7 @@
 # -*- coding: utf-8 -*-
 """Tool to ease working with the build system and reproducing test results"""
 
+import argparse
 import os
 import sys
 from subprocess import check_call
@@ -29,6 +30,11 @@
 from collections import OrderedDict
 import logging
 import yaml
+import shutil
+
+DEFAULT_PYENV=os.environ.get('DEFAULT_PYENV','py3_venv')
+DEFAULT_PYTHON=os.environ.get('DEFAULT_PYTHON','python3')
+DEFAULT_CMAKE_OPTIONS=os.environ.get('DEFAULT_CMAKE_OPTIONS','cmake_options.yml')
 
 class Confirm(object):
     def __init__(self, cmds):
@@ -46,7 +52,7 @@ def __call__(self):
                 resp = input("Please answer yes or no: ")
 
 class CMake(object):
-    def __init__(self, cmake_options_yaml='cmake_options.yml', cmake_options_yaml_default='cmake/cmake_options.yml'):
+    def __init__(self, cmake_options_yaml=DEFAULT_CMAKE_OPTIONS, cmake_options_yaml_default='cmake/cmake_options.yml'):
         if os.path.exists(cmake_options_yaml):
             self.cmake_options_yaml = cmake_options_yaml
         else:
@@ -87,10 +93,29 @@ def __call__(self, build_dir='build', generator='Ninja', build_cmd='ninja'):
             logging.info('Now building')
             check_call(shlex.split(build_cmd))
 
-
+def create_virtualenv(venv_exe, pyexe, venv) -> None:
+    logging.info("Creating virtualenv in %s with python %s", venv, pyexe)
+    if not (venv_exe and pyexe and venv):
+        logging.warn("Skipping creation of virtualenv")
+        return
+    check_call([venv_exe, '-p', pyexe, venv])
+    activate_this_py = os.path.join(venv, 'bin', 'activate_this.py')
+    # Activate virtualenv in this interpreter
+    exec(open(activate_this_py).read(), dict(__file__=activate_this_py))
+    check_call(['pip', 'install', '--upgrade','--force-reinstall', '-e', 'python'])
+    check_call(['pip', 'install', '-r', 'tests/requirements.txt'])
+
+def create_virtualenv_default():
+    create_virtualenv('virtualenv', DEFAULT_PYTHON, DEFAULT_PYENV)
+    logging.info("You can use the virtualenv by executing 'source %s/bin/activate'", DEFAULT_PYENV)
 
 COMMANDS = OrderedDict([
-    ('[Docker] sanity_check',
+    ('[Local build] CMake/Ninja build (using cmake_options.yaml (cp cmake/cmake_options.yml .) and edit) (creates {} virtualenv in "{}")'.format(DEFAULT_PYTHON, DEFAULT_PYENV),
+    [
+        CMake(),
+        create_virtualenv_default,
+    ]),
+    ('[Docker] sanity_check. Check for linting and code formatting.',
         "ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh sanity_check"),
     ('[Docker] Python3 CPU unittests',
     [
@@ -117,8 +142,6 @@ def __call__(self, build_dir='build', generator='Ninja', build_cmd='ninja'):
         "ci/build.py -p armv7",
         "ci/build.py -p test.arm_qemu ./runtime_functions.py run_ut_py3_qemu"
     ]),
-    ('[Local] CMake build (using cmake/cmake_options.yaml)',
-        CMake()),
     ('Clean (RESET HARD) repository (Warning! erases local changes / DATA LOSS)',
        Confirm("ci/docker/runtime_functions.sh clean_repo"))
 ])
@@ -128,6 +151,7 @@ def clip(x, mini, maxi):
 
 @retry((ValueError, RuntimeError), 3, delay_s = 0)
 def show_menu(items: List[str], header=None) -> int:
+    print('\n-- MXNet dev menu --\n')
     def hr():
         print(''.join(['-']*30))
     if header:
@@ -156,11 +180,47 @@ def handle_command(cmd):
     else:
         raise RuntimeError("handle_commands(cmds): argument should be str or List[str] but is %s", type(cmds))
 
-def main():
-    logging.getLogger().setLevel(logging.INFO)
+def use_menu_ui(args) -> None:
     command_list = list(COMMANDS.keys())
     choice = show_menu(command_list, 'Available actions')
     handle_commands(COMMANDS[command_list[choice]])
+
+def build(args) -> None:
+    """Build using CMake"""
+    venv_exe = shutil.which('virtualenv')
+    pyexe = shutil.which(args.pyexe)
+    if not venv_exe:
+        logging.warn("virtualenv wasn't found in path, it's recommended to install virutalenv to manage python environments")
+    if not pyexe:
+        logging.warn("Python executable %s not found in path", args.pyexe)
+    if args.cmake_options:
+        cmake = CMake(args.cmake_options)
+    else:
+        cmake = CMake()
+    cmake()
+    create_virtualenv(venv_exe, pyexe, args.venv)
+
+def main():
+    logging.getLogger().setLevel(logging.INFO)
+    parser = argparse.ArgumentParser(description="""Utility for compiling and testing MXNet easily""")
+    parser.set_defaults(command='use_menu_ui')
+
+    subparsers = parser.add_subparsers(help='sub-command help')
+    build_parser = subparsers.add_parser('build', help='build with the specified flags from file')
+    build_parser.add_argument('cmake_options', nargs='?',
+        help='File containing CMake options in YAML')
+    build_parser.add_argument('-v', '--venv',
+        type=str,
+        default=DEFAULT_PYENV,
+        help='virtualenv dir')
+    build_parser.add_argument('-p', '--pyexe',
+        type=str,
+        default=DEFAULT_PYTHON,
+        help='python executable')
+
+    build_parser.set_defaults(command='build')
+    args = parser.parse_args()
+    globals()[args.command](args)
     return 0
 
 if __name__ == '__main__':

tests/requirements.txt

@@ -2,3 +2,4 @@
 mock
 nose
 nose-timer
+ipython