Execute unit tests in parallel

build.cmd

@@ -388,7 +388,7 @@ if "%InstallPythonPackages%" == "True" (
     echo "Installing python packages ... "
     echo "#################################"
     call "%PythonExe%" -m pip install --upgrade pip
-    call "%PythonExe%" -m pip install --upgrade nose pytest graphviz imageio pytest-cov "jupyter_client>=4.4.0" "nbconvert>=4.2.0"
+    call "%PythonExe%" -m pip install --upgrade nose pytest pytest-xdist graphviz imageio pytest-cov "jupyter_client>=4.4.0" "nbconvert>=4.2.0"
 
     if %PythonVersion% == 2.7 (
         call "%PythonExe%" -m pip install --upgrade pyzmq
@@ -414,11 +414,10 @@ set TestsPath1=%PackagePath%\tests
 set TestsPath2=%__currentScriptDir%src\python\tests
 set TestsPath3=%__currentScriptDir%src\python\tests_extended
 set ReportPath=%__currentScriptDir%build\TestCoverageReport
-call "%PythonExe%" -m pytest --verbose --maxfail=1000 --capture=sys "%TestsPath1%" --cov="%PackagePath%" --cov-report term-missing --cov-report html:"%ReportPath%"
-if errorlevel 1 (
-    goto :Exit_Error
-)
-call "%PythonExe%" -m pytest --verbose --maxfail=1000 --capture=sys "%TestsPath2%" --cov="%PackagePath%" --cov-report term-missing --cov-report html:"%ReportPath%"
+set NumConcurrentTests=%NUMBER_OF_PROCESSORS%
+if %NumConcurrentTests% LSS 4 set NumConcurrentTests=4
+
+call "%PythonExe%" -m pytest -n %NumConcurrentTests% --verbose --maxfail=1000 --capture=sys "%TestsPath2%" "%TestsPath1%" --cov="%PackagePath%" --cov-report term-missing --cov-report html:"%ReportPath%"
 if errorlevel 1 (
     goto :Exit_Error
 )

build.sh

@@ -280,7 +280,7 @@ then
         exit 1
     fi
     # Review: Adding "--upgrade" to pip install will cause problems when using Anaconda as the python distro because of Anaconda's quirks with pytest.
-    "${PythonExe}" -m pip install nose "pytest>=4.4.0" graphviz "pytest-cov>=2.6.1" "jupyter_client>=4.4.0" "nbconvert>=4.2.0"
+    "${PythonExe}" -m pip install nose "pytest>=4.4.0" pytest-xdist graphviz "pytest-cov>=2.6.1" "jupyter_client>=4.4.0" "nbconvert>=4.2.0"
     if [ ${PythonVersion} = 2.7 ]
     then
         "${PythonExe}" -m pip install --upgrade pyzmq
@@ -307,8 +307,7 @@ then
     TestsPath2=${__currentScriptDir}/src/python/tests
     TestsPath3=${__currentScriptDir}/src/python/tests_extended
     ReportPath=${__currentScriptDir}/build/TestCoverageReport
-    "${PythonExe}" -m pytest --verbose --maxfail=1000 --capture=sys "${TestsPath1}"
-    "${PythonExe}" -m pytest --verbose --maxfail=1000 --capture=sys "${TestsPath2}"
+    "${PythonExe}" -m pytest -n 4 --verbose --maxfail=1000 --capture=sys "${TestsPath2}" "${TestsPath1}"
 
     if [ ${__runExtendedTests} = true ]
     then 

src/python/tests/test_estimator_checks.py

@@ -7,6 +7,7 @@
 """
 import json
 import os
+import unittest
 
 from nimbusml.decomposition import FactorizationMachineBinaryClassifier
 from nimbusml.ensemble import EnsembleClassifier
@@ -247,6 +248,14 @@
 
 MULTI_OUTPUT.extend(MULTI_OUTPUT_EX)
 
+skip_epoints = set([
+    'OneVsRestClassifier',
+    'TreeFeaturizer',
+    # skip SymSgdBinaryClassifier for now, because of crashes.
+    'SymSgdBinaryClassifier',
+    'DatasetTransformer'
+])
+
 
 def my_import(name):
     components = name.split('.')
@@ -263,97 +272,105 @@ def load_json(file_path):
         content_without_comments = '\n'.join(lines)
         return json.loads(content_without_comments)
 
+def get_epoints():
+    epoints = []
+    my_path = os.path.realpath(__file__)
+    my_dir = os.path.dirname(my_path)
+    manifest_diff_json = os.path.join(my_dir, '..', 'tools',
+                                      'manifest_diff.json')
+    manifest_diff = load_json(manifest_diff_json)
+    for e in manifest_diff['EntryPoints']:
+        if e['NewName'] not in skip_epoints:
+            epoints.append((e['Module'], e['NewName']))
 
-skip_epoints = set([
-    'OneVsRestClassifier',
-    'TreeFeaturizer',
-    # skip SymSgdBinaryClassifier for now, because of crashes.
-    'SymSgdBinaryClassifier',
-    'DatasetTransformer'
-])
+    return epoints
 
-epoints = []
-my_path = os.path.realpath(__file__)
-my_dir = os.path.dirname(my_path)
-manifest_diff_json = os.path.join(my_dir, '..', 'tools',
-                                  'manifest_diff.json')
-manifest_diff = load_json(manifest_diff_json)
-for e in manifest_diff['EntryPoints']:
-    if e['NewName'] not in skip_epoints:
-        epoints.append((e['Module'], e['NewName']))
-
-all_checks = {}
-all_failed_checks = {}
-all_passed_checks = {}
-total_checks_passed = 0
-
-print("total entrypoints: {}", len(epoints))
-
-for e in epoints:
-    checks = set()
-    failed_checks = set()
-    passed_checks = set()
-    class_name = e[1]
-    print("======== now Estimator is %s =========== " % class_name)
-    # skip LighGbm for now, because of random crashes.
-    if 'LightGbm' in class_name:
-        continue
-
-    mod = __import__('nimbusml.' + e[0], fromlist=[str(class_name)])
-    the_class = getattr(mod, class_name)
-    if class_name in INSTANCES:
-        estimator = INSTANCES[class_name]
-    else:
-        estimator = the_class()
-
-    if estimator._use_single_input_as_string():
-        estimator = estimator << 'F0'
-
-    for check in _yield_all_checks(class_name, estimator):
-        # Skip check_dict_unchanged for estimators which
-        # update the classes_ attribute. For more details
-        # see https://github.com/microsoft/NimbusML/pull/200
-        if (check.__name__ == 'check_dict_unchanged') and \
-            (hasattr(estimator, 'predict_proba') or
-             hasattr(estimator, 'decision_function')):
-            continue
+def check_entry_points(epoints):
+    all_checks = {}
+    all_failed_checks = {}
+    all_passed_checks = {}
+    total_checks_passed = 0
 
-        if check.__name__ in OMITTED_CHECKS_ALWAYS:
-            continue
-        if 'Binary' in class_name and check.__name__ in NOBINARY_CHECKS:
-            continue
-        if class_name in OMITTED_CHECKS and check.__name__ in \
-                OMITTED_CHECKS[class_name]:
-            continue
-        if class_name in OMITTED_CHECKS_TUPLE[0] and check.__name__ in \
-                OMITTED_CHECKS_TUPLE[1]:
+    print("total entrypoints: {}", len(epoints))
+
+    for e in epoints:
+        checks = set()
+        failed_checks = set()
+        passed_checks = set()
+        class_name = e[1]
+        print("======== now Estimator is %s =========== " % class_name)
+        # skip LighGbm for now, because of random crashes.
+        if 'LightGbm' in class_name:
             continue
-        checks.add(check.__name__)
-        try:
-            check(class_name, estimator.clone())
-            passed_checks.add(check.__name__)
-            total_checks_passed = total_checks_passed + 1
-        except Exception as e:
-            failed_checks.add(check.__name__)
-
-    if frozenset(checks) not in all_checks:
-        all_checks[frozenset(checks)] = []
-    all_checks[frozenset(checks)].append(class_name)
-
-    if len(failed_checks) > 0:
-        if frozenset(failed_checks) not in all_failed_checks:
-            all_failed_checks[frozenset(failed_checks)] = []
-        all_failed_checks[frozenset(failed_checks)].append(class_name)
-
-    if frozenset(passed_checks) not in all_passed_checks:
-        all_passed_checks[frozenset(passed_checks)] = []
-    all_passed_checks[frozenset(passed_checks)].append(class_name)
-
-if len(all_failed_checks) > 0:
-    print("Following tests failed for components:")
-    for key, value in all_failed_checks.items():
-        print('========================')
-        print(key)
-        print(value)
-    raise RuntimeError("estimator checks failed")
-print("success, total checks passed %s ", total_checks_passed)
+
+        mod = __import__('nimbusml.' + e[0], fromlist=[str(class_name)])
+        the_class = getattr(mod, class_name)
+        if class_name in INSTANCES:
+            estimator = INSTANCES[class_name]
+        else:
+            estimator = the_class()
+
+        if estimator._use_single_input_as_string():
+            estimator = estimator << 'F0'
+
+        for check in _yield_all_checks(class_name, estimator):
+            # Skip check_dict_unchanged for estimators which
+            # update the classes_ attribute. For more details
+            # see https://github.com/microsoft/NimbusML/pull/200
+            if (check.__name__ == 'check_dict_unchanged') and \
+                (hasattr(estimator, 'predict_proba') or
+                 hasattr(estimator, 'decision_function')):
+                continue
+
+            if check.__name__ in OMITTED_CHECKS_ALWAYS:
+                continue
+            if 'Binary' in class_name and check.__name__ in NOBINARY_CHECKS:
+                continue
+            if class_name in OMITTED_CHECKS and check.__name__ in \
+                    OMITTED_CHECKS[class_name]:
+                continue
+            if class_name in OMITTED_CHECKS_TUPLE[0] and check.__name__ in \
+                    OMITTED_CHECKS_TUPLE[1]:
+                continue
+            checks.add(check.__name__)
+            try:
+                check(class_name, estimator.clone())
+                passed_checks.add(check.__name__)
+                total_checks_passed = total_checks_passed + 1
+            except Exception as e:
+                failed_checks.add(check.__name__)
+
+        if frozenset(checks) not in all_checks:
+            all_checks[frozenset(checks)] = []
+        all_checks[frozenset(checks)].append(class_name)
+
+        if len(failed_checks) > 0:
+            if frozenset(failed_checks) not in all_failed_checks:
+                all_failed_checks[frozenset(failed_checks)] = []
+            all_failed_checks[frozenset(failed_checks)].append(class_name)
+
+        if frozenset(passed_checks) not in all_passed_checks:
+            all_passed_checks[frozenset(passed_checks)] = []
+        all_passed_checks[frozenset(passed_checks)].append(class_name)
+
+    if len(all_failed_checks) > 0:
+        print("Following tests failed for components:")
+        for key, value in all_failed_checks.items():
+            print('========================')
+            print(key)
+            print(value)
+        raise RuntimeError("estimator checks failed")
+    print("success, total checks passed %s ", total_checks_passed)
+
+
+class TestEstimatorChecks(unittest.TestCase):
+
+    def test_estimator_checks_1(self):
+        epoints = get_epoints()
+        check_entry_points(epoints[:len(epoints)//2])
+
+    def test_estimator_checks_2(self):
+        epoints = get_epoints()
+        check_entry_points(epoints[len(epoints)//2:])
+
+