Skip to content

Commit

Permalink
Simplify noxfile setup.
Browse files Browse the repository at this point in the history
  • Loading branch information
@kurtisvg
kurtisvg committed Feb 8, 2020
1 parent 4624100 commit 882742a
Show file tree
Hide file tree
Showing 3 changed files with 71 additions and 486 deletions.
9 changes: 7 additions & 2 deletions .kokoro/tests/run_tests.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,8 +82,13 @@ for file in **/requirements.txt; do

# If no local noxfile exists, copy the one from root
if [[ ! -f "noxfile.py" ]]; then
cp "$ROOT/noxfile-template.py" "./noxfile.py"
echo -e "\n Using noxfile from project root. \n"
PARENT_DIR=$(cd ../ && pwd)
while [[ "$PARENT_DIR" != "$ROOT" && ! -f "$PARENT_DIR/noxfile-template.py" ]];
do
PARENT_DIR=$(dirname "$PARENT_DIR")
done
cp "$PARENT_DIR/noxfile-template.py" "./noxfile.py"
echo -e "\n Using noxfile-template from parent folder ($PARENT_DIR). \n"
fi

# Use nox to execute the tests for the project.
Expand Down
229 changes: 64 additions & 165 deletions noxfile-template.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,63 +14,29 @@

from __future__ import print_function

import fnmatch
import os
from pathlib import Path
import tempfile

import nox

# Get root of this repository. Assume we don't have directories nested deeper than 10 items.
p = Path(os.getcwd())
for i in range(10):
if p is None:
raise Exception("Unable to detect repository root.")
if Path(p / ".git").exists():
REPO_ROOT = str(p)
break
p = p.parent

#
# Helpers and utility functions
#

# All versions currently being tested on Python. Automatically generated, DO NOT EDIT.
ALL_TESTED_VERSIONS = ["2.7", "3.5", "3.6", "3.7", "3.8"]

def _list_files(folder, pattern):
"""Lists all files below the given folder that match the pattern."""
for root, folders, files in os.walk(folder):
for filename in files:
if fnmatch.fnmatch(filename, pattern):
yield os.path.join(root, filename)

# Any specific versions of Python that should be skipped for this sample
SKIPPED_VERSIONS = []

def _collect_dirs(
start_dir,
blacklist=set(["conftest.py", "noxfile.py", "lib", "third_party"]),
suffix="requirements.txt",
recurse_further=False,
):
"""Recursively collects a list of dirs that contain a file matching the
given suffix.
This works by listing the contents of directories and finding
directories that have `"requirements.text` files.
"""
# Collect all the directories that have tests in them.
for parent, subdirs, files in os.walk(start_dir):
if "./." in parent:
continue # Skip top-level dotfiles
elif any(f for f in files if f.endswith(suffix) and f not in blacklist):
# Don't recurse further for tests, since py.test will do that.
if not recurse_further:
del subdirs[:]
# This dir has desired files in it. yield it.
yield parent
else:
# Filter out dirs we don't want to recurse into
subdirs[:] = [s for s in subdirs if s[0].isalpha() and s not in blacklist]
SAMPLE_TESTED_VERSIONS = [v for v in ALL_TESTED_VERSIONS if v not in SKIPPED_VERSIONS]

#
# Style Checks
#

# Ignore I202 "Additional newline in a section of imports." to accommodate
# region tags in import blocks. Since we specify an explicit ignore, we also
# have to explicitly ignore the list of default ignores:
# `E121,E123,E126,E226,E24,E704,W503,W504` as shown by `flake8 --help`.
def _determine_local_import_names(start_dir):
"""Determines all import names that should be considered "local".
Expand All @@ -82,99 +48,49 @@ def _determine_local_import_names(start_dir):
basename
for basename, extension in file_ext_pairs
if extension == ".py"
or os.path.isdir(os.path.join(start_dir, basename))
and basename not in ("__pycache__")
or os.path.isdir(os.path.join(start_dir, basename))
and basename not in ("__pycache__")
]


#
# App Engine specific helpers
#


_GAE_ROOT = os.environ.get("GAE_ROOT")
if _GAE_ROOT is None:
_GAE_ROOT = tempfile.mkdtemp()


def _setup_appengine_sdk(session):
"""Installs the App Engine SDK, if needed."""
session.env["GAE_SDK_PATH"] = os.path.join(_GAE_ROOT, "google_appengine")
session.run("gcp-devrel-py-tools", "download-appengine-sdk", _GAE_ROOT)


#
# Test sessions
#


PYTEST_COMMON_ARGS = ["--junitxml=sponge_log.xml"]

# Ignore I202 "Additional newline in a section of imports." to accommodate
# region tags in import blocks. Since we specify an explicit ignore, we also
# have to explicitly ignore the list of default ignores:
# `E121,E123,E126,E226,E24,E704,W503,W504` as shown by `flake8 --help`.
FLAKE8_COMMON_ARGS = [
"--show-source",
"--builtin",
"gettext",
"--max-complexity",
"20",
"--import-order-style",
"google",
"--exclude",
".nox,.cache,env,lib,generated_pb2,*_pb2.py,*_pb2_grpc.py",
"--ignore=E121,E123,E126,E203, E226,E24,E266,E501,E704,W503,W504,I100,I201,I202",
"--builtin='gettext'",
"--max-complexity=20",
"--import-order-style=google",
"--exclude='.nox,.cache,env,lib,generated_pb2,*_pb2.py,*_pb2_grpc.py'",
"--ignore=E121,E123,E126,E203,E226,E24,E266,E501,E704,W503,W504,I100,I201,I202",
"--max-line-length=88",
]


# Collect sample directories.
ALL_TESTED_SAMPLES = sorted(list(_collect_dirs(".")))
@nox.session
def lint(session):
session.install("flake8", "flake8-import-order")

GAE_STANDARD_SAMPLES = [
sample
for sample in ALL_TESTED_SAMPLES
if str(Path(sample).absolute().relative_to(REPO_ROOT)).startswith(
"appengine/standard/"
)
]
PY3_ONLY_SAMPLES = [
sample
for sample in ALL_TESTED_SAMPLES
if (
str(Path(sample).absolute().relative_to(REPO_ROOT)).startswith(
"appengine/standard_python37"
)
or str(Path(sample).absolute().relative_to(REPO_ROOT)).startswith(
"appengine/flexible/django_cloudsql"
)
or str(Path(sample).absolute().relative_to(REPO_ROOT)).startswith("functions/")
or str(Path(sample).absolute().relative_to(REPO_ROOT)).startswith(
"bigquery/pandas-gbq-migration"
)
or str(Path(sample).absolute().relative_to(REPO_ROOT)).startswith(
"run/system-package"
)
)
]
NON_GAE_STANDARD_SAMPLES_PY2 = sorted(
list((set(ALL_TESTED_SAMPLES) - set(GAE_STANDARD_SAMPLES)) - set(PY3_ONLY_SAMPLES))
)
NON_GAE_STANDARD_SAMPLES_PY3 = sorted(
list(set(ALL_TESTED_SAMPLES) - set(GAE_STANDARD_SAMPLES))
)
local_names = _determine_local_import_names(".")
args = FLAKE8_COMMON_ARGS + [
"--application-import-names",
",".join(local_names),
".",
]
session.run("flake8", *args)

#
# Sample Tests
#


def _session_tests(session, sample, post_install=None):
"""Runs py.test for a particular sample."""
session.install("-r", REPO_ROOT + "/testing/requirements.txt")
PYTEST_COMMON_ARGS = ["--junitxml=sponge_log.xml"]

session.chdir(sample)

def _session_tests(session, post_install=None):
"""Runs py.test for a particular project."""
if os.path.exists("requirements.txt"):
session.install("-r", "requirements.txt")

if os.path.exists("requirements.txt"):
session.install("-r", "testing/requirements.txt")

if post_install:
post_install(session)

Expand All @@ -188,57 +104,40 @@ def _session_tests(session, sample, post_install=None):
)


@nox.session(python="2.7")
@nox.parametrize("sample", GAE_STANDARD_SAMPLES)
def gae(session, sample):
"""Runs py.test for an App Engine standard sample."""

# Create a lib directory if needed, otherwise the App Engine vendor library
# will complain.
if not os.path.isdir(os.path.join(sample, "lib")):
os.mkdir(os.path.join(sample, "lib"))
@nox.session(python=["2.7", "3.5", "3.6", "3.7"])
def py(session):
"""Runs py.test for a sample using the sepcified version of Python."""
_session_tests(session)

_session_tests(session, sample, _setup_appengine_sdk)

#
# Readmegen
#

@nox.session(python="2.7")
@nox.parametrize("sample", NON_GAE_STANDARD_SAMPLES_PY2)
def py2(session, sample):
"""Runs py.test for a sample using Python 2.7"""
_session_tests(session, sample)


@nox.session(python=["3.5", "3.6", "3.7"])
@nox.parametrize("sample", NON_GAE_STANDARD_SAMPLES_PY3)
def py3(session, sample):
"""Runs py.test for a sample using Python 3.x"""
_session_tests(session, sample)


@nox.session(python="3.6")
def lint(session):
session.install("flake8", "flake8-import-order")

local_names = _determine_local_import_names(".")
args = FLAKE8_COMMON_ARGS + [
"--application-import-names",
",".join(local_names),
".",
]
session.run("flake8", *args)

def _get_repo_root():
""" Returns the root folder of the project. """
# Get root of this repository. Assume we don't have directories nested deeper than 10 items.
p = Path(os.getcwd())
for i in range(10):
if p is None:
break
if Path(p / ".git").exists():
return str(p)
p = p.parent
raise Exception("Unable to detect repository root.")

SAMPLES_WITH_GENERATED_READMES = sorted(list(_collect_dirs(".", suffix=".rst.in")))

GENERATED_READMES = sorted([x for x in Path('.').rglob("*.rst.in")])

@nox.session
@nox.parametrize("sample", SAMPLES_WITH_GENERATED_READMES)
def readmegen(session, sample):
@nox.parametrize("path", GENERATED_READMES)
def readmegen(session, path):
"""(Re-)generates the readme for a sample."""
session.install("jinja2", "pyyaml")

if os.path.exists(os.path.join(sample, "requirements.txt")):
session.install("-r", os.path.join(sample, "requirements.txt"))
if os.path.exists(os.path.join(path, "requirements.txt")):
session.install("-r", os.path.join(path, "requirements.txt"))

in_file = os.path.join(sample, "README.rst.in")
session.run("python", REPO_ROOT + "/scripts/readme-gen/readme_gen.py", in_file)
in_file = os.path.join(path, "README.rst.in")
session.run("python", _get_repo_root() + "/scripts/readme-gen/readme_gen.py", in_file)
Loading

0 comments on commit 882742a

Please sign in to comment.