testing: multi project noxfile-template.py (#3700)

* testing: prototpe for multi project noxfile.py

* correct project names

* introduce TEST_CONFIG

* modify noxfile-template, add default config

* changed how we import user config

* fix stale comments

* use different project for python 3.6 and 3.7

* fix a bug

* changed the filename

also simplified the config
stop runnint `gcloud update`
add a warning about editing noxfile.py

* add BUILD_SPECIFIC_GCLOUD_PROJECT

* use session.skip

* print debuggin

* more print debuggin

* adding cwd to sys.path

* removed debug print, display details of ImportError

* use the usual test project

* stop setting gcloud project

* simplified the noxfile-template

.kokoro/python2.7/common.cfg

@@ -43,3 +43,10 @@ env_vars: {
     key: "RUN_TESTS_SESSION"
     value: "py-2.7"
 }
+
+# Declare build specific Cloud project. It still uses the common one,
+# but we'll update the value once we have more Cloud projects.
+env_vars: {
+    key: "BUILD_SPECIFIC_GCLOUD_PROJECT"
+    value: "python-docs-samples-tests"
+}

.kokoro/python3.6/common.cfg

@@ -43,3 +43,10 @@ env_vars: {
     key: "RUN_TESTS_SESSION"
     value: "py-3.6"
 }
+
+# Declare build specific Cloud project. It still uses the common one,
+# but we'll update the value once we have more Cloud projects.
+env_vars: {
+    key: "BUILD_SPECIFIC_GCLOUD_PROJECT"
+    value: "python-docs-samples-tests"
+}

.kokoro/python3.7/common.cfg

@@ -43,3 +43,10 @@ env_vars: {
     key: "RUN_TESTS_SESSION"
     value: "py-3.7"
 }
+
+# Declare build specific Cloud project.
+# Temporary setting my own project for testing the behavior on the PR.
+env_vars: {
+    key: "BUILD_SPECIFIC_GCLOUD_PROJECT"
+    value: "python-docs-samples-tests"
+}

.kokoro/python3.8/common.cfg

@@ -43,3 +43,10 @@ env_vars: {
     key: "RUN_TESTS_SESSION"
     value: "py-3.8"
 }
+
+# Declare build specific Cloud project. It still uses the common one,
+# but we'll update the value once we have more Cloud projects.
+env_vars: {
+    key: "BUILD_SPECIFIC_GCLOUD_PROJECT"
+    value: "python-docs-samples-tests"
+}

.kokoro/tests/run_tests.sh

@@ -43,6 +43,11 @@ SECRETS_PASSWORD=$(cat "${KOKORO_GFILE_DIR}/secrets-password.txt")
 
 source ./testing/test-env.sh
 export GOOGLE_APPLICATION_CREDENTIALS=$(pwd)/testing/service-account.json
+
+# For cloud-run session, we activate the service account for gcloud sdk.
+gcloud auth activate-service-account \
+       --key-file "${GOOGLE_APPLICATION_CREDENTIALS}"
+
 export GOOGLE_CLIENT_SECRETS=$(pwd)/testing/client-secrets.json
 source "${KOKORO_GFILE_DIR}/automl_secrets.txt"
 cp "${KOKORO_GFILE_DIR}/functions-slack-config.json" "functions/slack/config.json"
@@ -130,4 +135,4 @@ cd "$ROOT"
 # Workaround for Kokoro permissions issue: delete secrets
 rm testing/{test-env.sh,client-secrets.json,service-account.json}
 
-exit "$RTN"
+exit "$RTN"

monitoring/api/v3/alerts-client/noxfile_config.py

@@ -0,0 +1,42 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Default TEST_CONFIG_OVERRIDE for python repos.
+
+# You can copy this file into your directory, then it will be inported from
+# the noxfile.py.
+
+# The source of truth:
+# https://github.com/GoogleCloudPlatform/python-docs-samples/blob/master/noxfile_config.py
+
+TEST_CONFIG_OVERRIDE = {
+    # You can opt out from the test for specific Python versions.
+    'ignored_versions': ["2.7"],
+
+    # Declare optional test sessions you want to opt-in. Currently we
+    # have the following optional test sessions:
+    #     'cloud_run' # Test session for Cloud Run application.
+    'opt_in_sessions': [],
+
+    # An envvar key for determining the project id to use. Change it
+    # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a
+    # build specific Cloud project. You can also use your own string
+    # to use your own Cloud project.
+    # 'gcloud_project_env': 'GCLOUD_PROJECT',
+    'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT',
+
+    # A dictionary you want to inject into your test. Don't put any
+    # secrets here. These values will override predefined values.
+    'envs': {},
+}

noxfile-template.py

@@ -16,16 +16,80 @@
 
 import os
 from pathlib import Path
+import sys
 
 import nox
 
 
+# WARNING - WARNING - WARNING - WARNING - WARNING
+# WARNING - WARNING - WARNING - WARNING - WARNING
+#           DO NOT EDIT THIS FILE EVER!
+# WARNING - WARNING - WARNING - WARNING - WARNING
+# WARNING - WARNING - WARNING - WARNING - WARNING
+
+# Copy `noxfile_config.py` to your directory and modify it instead.
+
+
+# `TEST_CONFIG` dict is a configuration hook that allows users to
+# modify the test configurations. The values here should be in sync
+# with `noxfile_config.py`. Users will copy `noxfile_config.py` into
+# their directory and modify it.
+
+TEST_CONFIG = {
+    # You can opt out from the test for specific Python versions.
+    'ignored_versions': ["2.7"],
+
+    # Declare optional test sessions you want to opt-in. Currently we
+    # have the following optional test sessions:
+    #     'cloud_run' # Test session for Cloud Run application.
+    'opt_in_sessions': [],
+
+    # An envvar key for determining the project id to use. Change it
+    # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a
+    # build specific Cloud project. You can also use your own string
+    # to use your own Cloud project.
+    'gcloud_project_env': 'GCLOUD_PROJECT',
+    # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT',
+
+    # A dictionary you want to inject into your test. Don't put any
+    # secrets here. These values will override predefined values.
+    'envs': {},
+}
+
+
+try:
+    # Ensure we can import noxfile_config in the project's directory.
+    sys.path.append('.')
+    from noxfile_config import TEST_CONFIG_OVERRIDE
+except ImportError as e:
+    print("No user noxfile_config found: detail: {}".format(e))
+    TEST_CONFIG_OVERRIDE = {}
+
+# Update the TEST_CONFIG with the user supplied values.
+TEST_CONFIG.update(TEST_CONFIG_OVERRIDE)
+
+
+def get_pytest_env_vars():
+    """Returns a dict for pytest invocation."""
+    ret = {}
+
+    # Override the GCLOUD_PROJECT and the alias.
+    env_key = TEST_CONFIG['gcloud_project_env']
+    # This should error out if not set.
+    ret['GOOGLE_CLOUD_PROJECT'] = os.environ[env_key]
+    ret['GCLOUD_PROJECT'] = os.environ[env_key]
+
+    # Apply user supplied envs.
+    ret.update(TEST_CONFIG['envs'])
+    return ret
+
+
 # DO NOT EDIT - automatically generated.
 # All versions used to tested samples.
 ALL_VERSIONS = ["2.7", "3.6", "3.7", "3.8"]
 
 # Any default versions that should be ignored.
-IGNORED_VERSIONS = ["2.7"]
+IGNORED_VERSIONS = TEST_CONFIG['ignored_versions']
 
 TESTED_VERSIONS = sorted([v for v in ALL_VERSIONS if v not in IGNORED_VERSIONS])
 
@@ -76,11 +140,25 @@ def lint(session):
     session.install("flake8", "flake8-import-order")
 
     local_names = _determine_local_import_names(".")
-    args = FLAKE8_COMMON_ARGS + [
+    options = [
         "--application-import-names",
-        ",".join(local_names),
-        ".",
+        ",".join(local_names)
     ]
+
+    # We currently look at pytest.ini for flake8 config.
+    # You can add your own exclude and ignore by using `extend-`
+    #
+    # Example config:
+    # [flake8]
+    # extend-ignore = I100
+    # extend-exclude = myapp1,myapp2
+    if os.path.isfile("pytest.ini"):
+        options += [
+            "--append-config",
+            "pytest.ini",
+        ]
+    options.append(".")
+    args = FLAKE8_COMMON_ARGS + options
     session.run("flake8", *args)
 
 
@@ -109,7 +187,8 @@ def _session_tests(session, post_install=None):
         # Pytest will return 5 when no tests are collected. This can happen
         # on travis where slow and flaky tests are excluded.
         # See http://doc.pytest.org/en/latest/_modules/_pytest/main.html
-        success_codes=[0, 5]
+        success_codes=[0, 5],
+        env=get_pytest_env_vars()
     )
 
 
@@ -119,8 +198,22 @@ def py(session):
     if session.python in TESTED_VERSIONS:
         _session_tests(session)
     else:
-        print("SKIPPED: {} tests are disabled for this sample.".format(session.python))
+        session.skip("SKIPPED: {} tests are disabled for this sample.".format(
+            session.python
+        ))
+
 
+#
+# cloud_run session
+#
+
+@nox.session
+def cloud_run(session):
+    """Run tests for cloud run."""
+    if 'cloud_run' in TEST_CONFIG['opt_in_sessions']:
+        _session_tests(session)
+    else:
+        session.skip('SKIPPED: cloud_run tests are disabled for this sample.')
 
 #
 # Readmegen

noxfile_config.py

@@ -0,0 +1,42 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Default TEST_CONFIG_OVERRIDE for python repos.
+
+# You can copy this file into your directory, then it will be inported from
+# the noxfile.py.
+
+# The source of truth:
+# https://github.com/GoogleCloudPlatform/python-docs-samples/blob/master/noxfile_config.py
+
+TEST_CONFIG_OVERRIDE = {
+    # You can opt out from the test for specific Python versions.
+    'ignored_versions': ["2.7"],
+
+    # Declare optional test sessions you want to opt-in. Currently we
+    # have the following optional test sessions:
+    #     'cloud_run' # Test session for Cloud Run application.
+    'opt_in_sessions': [],
+
+    # An envvar key for determining the project id to use. Change it
+    # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a
+    # build specific Cloud project. You can also use your own string
+    # to use your own Cloud project.
+    'gcloud_project_env': 'GCLOUD_PROJECT',
+    # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT',
+
+    # A dictionary you want to inject into your test. Don't put any
+    # secrets here. These values will override predefined values.
+    'envs': {},
+}