python: Add pytest and coverage (envoyproxy#15888)

- add separate tooling test job in ci
- add pytest and a pytest runner
- add coverage and cov runner
- add pytests for all runners
- upload coverage to https://storage.googleapis.com/envoy-pr/15888/tooling/index.html

Signed-off-by: Ryan Northey <[email protected]>

Author: phlax

.azure-pipelines/pipelines.yml

@@ -33,6 +33,28 @@ stages:
             # not all have fixes so improve condition/handling
             condition: failed()
 
+      - job: tooling
+        dependsOn: []
+        pool:
+          vmImage: "ubuntu-18.04"
+        steps:
+          - script: ci/run_envoy_docker.sh 'ci/do_ci.sh tooling'
+            workingDirectory: $(Build.SourcesDirectory)
+            env:
+              ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory)
+              BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com
+              BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance
+              GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)
+            displayName: "Run tooling checks"
+
+          - script: |
+              ci/run_envoy_docker.sh 'ci/upload_gcs_artifact.sh /source/generated/tooling tooling'
+            displayName: "Upload tooling coverage to GCS"
+            env:
+              ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory)
+              GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)
+              GCS_ARTIFACT_BUCKET: $(GcsArtifactBucket)
+
       - job: format
         dependsOn: ["format_pre"]
         pool:

.coveragerc

@@ -0,0 +1,11 @@
+[run]
+omit =
+     tools/
+     */pytest_*.py
+
+[report]
+exclude_lines =
+        pragma: no cover
+        raise NotImplementedError
+        pass
+        if __name__ == .__main__.:

.github/dependabot.yml

@@ -70,3 +70,8 @@ updates:
     directory: "/configs"
     schedule:
       interval: "daily"
+
+  - package-ecosystem: "pip"
+    directory: "/tools/testing"
+    schedule:
+      interval: "daily"

BUILD

@@ -4,6 +4,8 @@ exports_files([
     "VERSION",
     "API_VERSION",
     ".clang-format",
+    "pytest.ini",
+    ".coveragerc",
 ])
 
 # These two definitions exist to help reduce Envoy upstream core code depending on extensions.

bazel/repositories_extra.bzl

@@ -6,6 +6,13 @@ load("@proxy_wasm_cpp_host//bazel/cargo:crates.bzl", "proxy_wasm_cpp_host_fetch_
 def _python_deps():
     py_repositories()
 
+    # REMOVE!!!
+    pip_install(
+        name = "sometools_pip3",
+        requirements = "@envoy//tools/sometools:requirements.txt",
+        extra_pip_args = ["--require-hashes"],
+    )
+
     pip_install(
         name = "config_validation_pip3",
         requirements = "@envoy//tools/config_validation:requirements.txt",
@@ -88,6 +95,11 @@ def _python_deps():
         requirements = "@envoy//tools/code_format:requirements.txt",
         extra_pip_args = ["--require-hashes"],
     )
+    pip_install(
+        name = "testing_pip3",
+        requirements = "@envoy//tools/testing:requirements.txt",
+        extra_pip_args = ["--require-hashes"],
+    )
     pip_install(
         name = "thrift_pip3",
         requirements = "@envoy//test/extensions/filters/network/thrift_proxy:requirements.txt",

ci/do_ci.sh

@@ -466,6 +466,14 @@ elif [[ "$CI_TARGET" == "cve_scan" ]]; then
   bazel run "${BAZEL_BUILD_OPTIONS[@]}" //tools/dependency:cve_scan_test
   bazel run "${BAZEL_BUILD_OPTIONS[@]}" //tools/dependency:cve_scan
   exit 0
+elif [[ "$CI_TARGET" == "tooling" ]]; then
+  echo "Run pytest tooling tests..."
+  bazel run "${BAZEL_BUILD_OPTIONS[@]}" //tools/testing:pytest_python_pytest -- --cov-collect  /tmp/.coverage-envoy
+  bazel run "${BAZEL_BUILD_OPTIONS[@]}" //tools/testing:pytest_python_coverage -- --cov-collect  /tmp/.coverage-envoy
+  bazel run "${BAZEL_BUILD_OPTIONS[@]}" //tools/base:pytest_runner -- --cov-collect  /tmp/.coverage-envoy
+  bazel run "${BAZEL_BUILD_OPTIONS[@]}" //tools/base:pytest_utils -- --cov-collect  /tmp/.coverage-envoy
+  bazel run "${BAZEL_BUILD_OPTIONS[@]}" //tools/testing:python_coverage -- --fail-under=95 /tmp/.coverage-envoy /source/generated/tooling
+  exit 0
 elif [[ "$CI_TARGET" == "verify_examples" ]]; then
   run_ci_verify "*" wasm-cc
   exit 0

pytest.ini

@@ -0,0 +1,4 @@
+[pytest]
+addopts = -raq --ignore=tools/testing/external/*,__init__.py,testing/conf --color=yes --cov-append -p  tools.testing.plugin --cov-config=.coveragerc -vv tools
+testpaths =
+	  tests