Merge remote-tracking branch 'upstream/master' into es/lpt/lpt_to_ngr…

…aph_integration

.ci/openvino-onnx/Jenkinsfile

@@ -68,7 +68,7 @@ def buildDockerImage() {
 
 def runTests() {
     sh """
-        docker run --rm --name ${DOCKER_CONTAINER_NAME} \
+        docker run --name ${DOCKER_CONTAINER_NAME} \
         --volume ${HOME}/ONNX_CI/models/.onnx:/root/.onnx ${DOCKER_IMAGE_TAG}
     """
 }
@@ -83,6 +83,7 @@ pipeline {
     }
     options {
         skipDefaultCheckout true
+        timeout(activity: true, time: 10, unit: 'MINUTES')
     }
     stages {
         stage("Clone repository") {
@@ -118,6 +119,7 @@ pipeline {
                 deleteDir()
                 sh """
                     docker image prune -f
+                    docker rm -f ${DOCKER_CONTAINER_NAME}
                 """
             }
         }

SECURITY.md

@@ -1,13 +1,12 @@
 # Security Policy
 
-## Reporting a Vulnerability
+## Report a Vulnerability
 
-Please report about security issues or vulnerabilities you find to Intel
-[Security Center].
+Please report security issues or vulnerabilities to the [Intel® Security Center].
 
-For more information on how Intel works to resolve security issues, see:
-[Vulnerability Handling Guidelines]
+For more information on how Intel® works to resolve security issues, see
+[Vulnerability Handling Guidelines].
 
-[Security Center]:https://www.intel.com/security
+[Intel® Security Center]:https://www.intel.com/security
 
-[Vulnerability Handling Guidelines] https://www.intel.com/content/www/us/en/security-center/vulnerability-handling-guidelines.html
+[Vulnerability Handling Guidelines]:https://www.intel.com/content/www/us/en/security-center/vulnerability-handling-guidelines.html

scripts/install_dependencies/install_NEO_OCL_driver.sh

@@ -158,13 +158,13 @@ _uninstall_user_mode_centos()
         echo "rpm -qa | grep $package"
         found_package=$(rpm -qa | grep $package)
         if [[ $? -eq 0 ]]; then
-            echo Found installed user-mode driver, performing uninstall...
+            echo "Found installed user-mode driver, performing uninstall..."
             cmd="rpm -e --nodeps ${found_package}"
             echo $cmd
             eval $cmd
             if [[ $? -ne 0 ]]; then
-                echo ERROR: failed to uninstall existing user-mode driver. >&2
-                echo Please try again manually and run the script again. >&2
+                echo "ERROR: failed to uninstall existing user-mode driver." >&2
+                echo "Please try again manually and run the script again." >&2
                 exit $EXIT_FAILURE
             fi
         fi
@@ -182,11 +182,17 @@ _uninstall_user_mode_ubuntu()
            "intel-igc-opencl")
 
     for package in "${PACKAGES[@]}"; do
-        apt remove -y $package
-        if [[ $? -ne 0 ]]; then
-            echo ERROR: failed to uninstall existing user-mode driver. >&2
-            echo Please try again manually and run the script again. >&2
-            exit $EXIT_FAILURE
+        found_package=$(dpkg-query -W -f='${binary:Package}\n' ${package})
+        if [[ $? -eq 0 ]]; then
+            echo "Found installed user-mode driver, performing uninstall..."
+            cmd="apt-get autoremove -y $package"
+            echo $cmd
+            eval $cmd
+            if [[ $? -ne 0 ]]; then
+                echo "ERROR: failed to uninstall existing user-mode driver." >&2
+                echo "Please try again manually and run the script again." >&2
+                exit $EXIT_FAILURE
+            fi
         fi
     done
 }
@@ -391,7 +397,8 @@ check_current_driver()
 }
 
 install()
-{       
+{   
+
     uninstall_user_mode
     install_prerequisites
     download_packages

tests/time_tests/README.md

@@ -21,6 +21,11 @@ cmake .. -DInferenceEngineDeveloperPackage_DIR=$(realpath ../../../build) && mak
 
 2. Run test:
 ``` bash
-./scripts/run_timetest.py ../../../bin/intel64/Release/timetest_infer -m model.xml -d CPU
+./scripts/run_timetest.py ../../bin/intel64/Release/timetest_infer -m model.xml -d CPU
 ```
 
+2. Run several configurations using `pytest`:
+``` bash
+export PYTHONPATH=./:$PYTHONPATH
+pytest ./test_runner/test_timetest.py --exe ../../bin/intel64/Release/timetest_infer
+```

tests/time_tests/scripts/requirements.txt

@@ -0,0 +1 @@
+PyYAML==5.3.1

tests/time_tests/test_runner/conftest.py

@@ -0,0 +1,98 @@
+# Copyright (C) 2020 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+#
+"""
+Basic high-level plugin file for pytest.
+
+See [Writing plugins](https://docs.pytest.org/en/latest/writing_plugins.html)
+for more information.
+
+This plugin adds the following command-line options:
+
+* `--test_conf` - Path to test configuration file. Used to parametrize tests.
+  Format: YAML file.
+* `--exe` - Path to a timetest binary to execute.
+* `--niter` - Number of times to run executable.
+"""
+
+# pylint:disable=import-error
+import pytest
+from pathlib import Path
+import yaml
+
+from test_runner.utils import expand_env_vars
+
+
+# -------------------- CLI options --------------------
+
+def pytest_addoption(parser):
+    """Specify command-line options for all plugins"""
+    parser.addoption(
+        "--test_conf",
+        type=Path,
+        help="Path to test config",
+        default=Path(__file__).parent / "test_config.yml"
+    )
+    parser.addoption(
+        "--exe",
+        required=True,
+        dest="executable",
+        type=Path,
+        help="Path to a timetest binary to execute",
+    )
+    parser.addoption(
+        "--niter",
+        type=int,
+        help="Number of iterations to run executable and aggregate results",
+        default=3
+    )
+    # TODO: add support of --mo, --omz etc. required for OMZ support
+
+
+@pytest.fixture(scope="session")
+def test_conf(request):
+    """Fixture function for command-line option."""
+    return request.config.getoption('test_conf')
+
+
+@pytest.fixture(scope="session")
+def executable(request):
+    """Fixture function for command-line option."""
+    return request.config.getoption('executable')
+
+
+@pytest.fixture(scope="session")
+def niter(request):
+    """Fixture function for command-line option."""
+    return request.config.getoption('niter')
+
+# -------------------- CLI options --------------------
+
+
+def pytest_generate_tests(metafunc):
+    """Pytest hook for test generation.
+
+    Generate parameterized tests from discovered modules and test config
+    parameters.
+    """
+    with open(metafunc.config.getoption('test_conf'), "r") as file:
+        test_cases = expand_env_vars(yaml.safe_load(file))
+    if test_cases:
+        metafunc.parametrize("instance", test_cases)
+
+
+def pytest_make_parametrize_id(config, val, argname):
+    """Pytest hook for user-friendly test name representation"""
+
+    def get_dict_values(d):
+        """Unwrap dictionary to get all values of nested dictionaries"""
+        if isinstance(d, dict):
+            for v in d.values():
+                yield from get_dict_values(v)
+        else:
+            yield d
+
+    keys = val.keys()
+    values = list(get_dict_values(val))
+
+    return "-".join(["_".join([key, val]) for key, val in zip(keys, values)])

tests/time_tests/test_runner/requirements.txt

@@ -0,0 +1,3 @@
+pytest==4.0.1
+attrs==19.1.0   # required for pytest==4.0.1 to resolve compatibility issues
+PyYAML==5.3.1

tests/time_tests/test_runner/test_config.yml

@@ -0,0 +1,8 @@
+- device:
+    name: CPU
+  model:
+    path: ${SHARE}/stress_tests/master_04d6f112132f92cab563ae7655747e0359687dc9/caffe/FP32/alexnet/alexnet.xml # TODO: add link to `test_data` repo model
+- device:
+    name: GPU
+  model:
+    path: ${SHARE}/stress_tests/master_04d6f112132f92cab563ae7655747e0359687dc9/caffe/FP32/alexnet/alexnet.xml

tests/time_tests/test_runner/test_timetest.py

@@ -1 +1,42 @@
-#TODO: fill
+# Copyright (C) 2020 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+#
+"""Main entry-point to run timetests tests.
+
+Default run:
+$ pytest test_timetest.py
+
+Options[*]:
+--test_conf     Path to test config
+--exe           Path to timetest binary to execute
+--niter         Number of times to run executable
+
+[*] For more information see conftest.py
+"""
+
+from pathlib import Path
+import logging
+
+from scripts.run_timetest import run_timetest
+
+
+def test_timetest(instance, executable, niter):
+    """Parameterized test.
+
+    :param instance: test instance
+    :param executable: timetest executable to run
+    :param niter: number of times to run executable
+    """
+    # Prepare model to get model_path
+    model_path = instance["model"].get("path")
+    assert model_path, "Model path is empty"
+
+    # Run executable
+    exe_args = {
+        "executable": Path(executable),
+        "model": Path(model_path),
+        "device": instance["device"]["name"],
+        "niter": niter
+    }
+    retcode, aggr_stats = run_timetest(exe_args, log=logging)
+    assert retcode == 0, "Run of executable failed"

tests/time_tests/test_runner/utils.py

@@ -0,0 +1,20 @@
+# Copyright (C) 2020 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+#
+"""Utility module."""
+
+import os
+
+
+def expand_env_vars(obj):
+    """Expand environment variables in provided object."""
+
+    if isinstance(obj, list):
+        for i, value in enumerate(obj):
+            obj[i] = expand_env_vars(value)
+    elif isinstance(obj, dict):
+        for name, value in obj.items():
+            obj[name] = expand_env_vars(value)
+    else:
+        obj = os.path.expandvars(obj)
+    return obj