CI rehash

.circleci/config.yml

@@ -0,0 +1,57 @@
+---
+version: 2
+jobs:
+  build:
+    docker:
+      - image: mrupgrade/deadsnakes:2.7
+    steps:
+      - run:
+          name: Update apt packages.
+          command: apt-get update -qq
+      - run:
+          name: Install apt dependencies.
+          command: apt-get install -y git python3.4 python3.5 python3.6
+      - run:
+          name: Install nox.
+          command: pip install nox-automation
+      - checkout
+      - run:
+          name: Decrypt credentials.
+          command: |
+            if [ -n "$GOOGLE_APPLICATION_CREDENTIALS" ]; then
+              openssl aes-256-cbc -d -a -k "$GOOGLE_CREDENTIALS_PASSPHRASE" \
+                  -in /home/ubuntu/gcp/test_utils/credentials.json.enc \
+                  -out "$GOOGLE_APPLICATION_CREDENTIALS"
+            else
+              echo "No credentials. System tests will not run."
+            fi
+      - run:
+          name: INFO - Declare target packages (changed packages and dependencies).
+          command: python test_utils/scripts/get_target_packages.py
+      - run:
+          name: Run tests for each API.
+          command: |
+            for TARGET in $(python test_utils/scripts/get_target_packages.py); do
+              nox -f $TARGET/nox.py
+            done
+      - run:
+          name: Push to google-cloud-python-private.
+          type: deploy
+          command: |
+            if [[ "$CIRCLE_BRANCH" == "master" ]] && \
+                [[ -z "$CIRCLE_PR_NUMBER" ]] && \
+                [[ -z "$CIRCLE_TAG" ]]; then
+              git remote add private [email protected]:GoogleCloudPlatform/google-cloud-python-private.git
+              git push private master
+            else
+              echo "Not on master; skipping private push."
+            fi
+      - run:
+          name: Update the docs.
+          type: deploy
+          command: |
+            pip install sphinx sphinx_rtd_theme
+            ./test_utils/scripts/update_docs.sh
+
+
+    working_directory: /var/code/gcp/

.gitignore

@@ -25,6 +25,7 @@ pip-log.txt
 
 # Unit test / coverage reports
 .coverage
+.nox
 .tox
 .cache
 

bigquery/.flake8

@@ -0,0 +1,6 @@
+[flake8]
+exclude =
+  __pycache__,
+  .git,
+  *.pyc,
+  conf.py

bigquery/google/cloud/bigquery/__init__.py

@@ -34,3 +34,9 @@
 from google.cloud.bigquery.dataset import Dataset
 from google.cloud.bigquery.schema import SchemaField
 from google.cloud.bigquery.table import Table
+
+__all__ = [
+    '__version__', 'AccessGrant', 'ArrayQueryParameter', 'Client',
+    'Dataset', 'ScalarQueryParameter', 'SchemaField', 'StructQueryParameter',
+    'Table',
+]

bigquery/nox.py

@@ -0,0 +1,90 @@
+# Copyright 2016 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+import nox
+
+
+@nox.session
+@nox.parametrize('python_version', ['2.7', '3.4', '3.5', '3.6'])
+def unit_tests(session, python_version):
+    """Run the unit test suite."""
+
+    # Run unit tests against all supported versions of Python.
+    session.interpreter = 'python%s' % python_version
+
+    # Install all test dependencies, then install this package in-place.
+    session.chdir(os.path.dirname(__file__))
+    session.install('mock', 'pytest', 'pytest-cov', '../core/')
+    session.install('-e', '.')
+
+    # Run py.test against the unit tests.
+    session.run('py.test', '--quiet',
+        '--cov=google.cloud.bigquery', '--cov=tests.unit', '--cov-append',
+        '--cov-config=.coveragerc', '--cov-report=', '--cov-fail-under=97',
+        'tests/unit',
+    )
+
+
+@nox.session
+@nox.parametrize('python_version', ['2.7', '3.6'])
+def system_tests(session, python_version):
+    """Run the system test suite."""
+
+    # Sanity check: Only run system tests if the environment variable is set.
+    if not os.environ.get('GOOGLE_APPLICATION_CREDENTIALS', ''):
+        return
+
+    # Run the system tests against latest Python 2 and Python 3 only.
+    session.interpreter = 'python%s' % python_version
+
+    # Install all test dependencies, then install this package into the
+    # virutalenv's dist-packages.
+    session.chdir(os.path.dirname(__file__))
+    session.install('mock', 'pytest',
+                    '../core/', '../test_utils/',
+                    '../storage/')
+    session.install('.')
+
+    # Run py.test against the system tests.
+    session.run('py.test', '--quiet', 'tests/system.py')
+
+
+@nox.session
+def lint(session):
+    """Run flake8.
+
+    Returns a failure if flake8 finds linting errors or sufficiently
+    serious code quality issues.
+    """
+    session.interpreter = 'python3.6'
+    session.chdir(os.path.dirname(__file__))
+    session.install('flake8')
+    session.install('.')
+    session.run('flake8', 'google/cloud/bigquery')
+
+
+@nox.session
+def cover(session):
+    """Run the final coverage report.
+
+    This outputs the coverage report aggregating coverage from the unit
+    test runs (not system test runs), and then erases coverage data.
+    """
+    session.interpreter = 'python3.6'
+    session.chdir(os.path.dirname(__file__))
+    session.install('coverage', 'pytest-cov')
+    session.run('coverage', 'report', '--show-missing', '--fail-under=100')
+    session.run('coverage', 'erase')

system_tests/bigquery.py → bigquery/tests/system.py

@@ -19,10 +19,10 @@
 from google.cloud import bigquery
 from google.cloud.exceptions import Forbidden
 
-from retry import RetryErrors
-from retry import RetryInstanceState
-from retry import RetryResult
-from system_test_utils import unique_resource_id
+from test_utils.retry import RetryErrors
+from test_utils.retry import RetryInstanceState
+from test_utils.retry import RetryResult
+from test_utils.system import unique_resource_id
 
 
 def _has_rows(result):

bigtable/.flake8

@@ -0,0 +1,11 @@
+[flake8]
+exclude =
+  # BigTable includes generated code in the manual layer;
+  # do not lint this.
+  google/cloud/bigtable/_generated/*.py,
+
+  # Standard linting exemptions.
+  __pycache__,
+  .git,
+  *.pyc,
+  conf.py

bigtable/google/cloud/bigtable/__init__.py

@@ -19,3 +19,6 @@
 __version__ = get_distribution('google-cloud-bigtable').version
 
 from google.cloud.bigtable.client import Client
+
+
+__all__ = ['__version__', 'Client']

bigtable/nox.py

@@ -0,0 +1,89 @@
+# Copyright 2016 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+import nox
+
+
+@nox.session
+@nox.parametrize('python_version', ['2.7', '3.4', '3.5', '3.6'])
+def unit_tests(session, python_version):
+    """Run the unit test suite."""
+
+    # Run unit tests against all supported versions of Python.
+    session.interpreter = 'python%s' % python_version
+
+    # Install all test dependencies, then install this package in-place.
+    session.chdir(os.path.dirname(__file__))
+    session.install('mock', 'pytest', 'pytest-cov', '../core/')
+    session.install('-e', '.')
+
+    # Run py.test against the unit tests.
+    session.run('py.test', '--quiet',
+        '--cov=google.cloud.bigtable', '--cov=tests.unit', '--cov-append',
+        '--cov-config=.coveragerc', '--cov-report=', '--cov-fail-under=97',
+        'tests/unit',
+    )
+
+
+@nox.session
+@nox.parametrize('python_version', ['2.7', '3.6'])
+def system_tests(session, python_version):
+    """Run the system test suite."""
+
+    # Sanity check: Only run system tests if the environment variable is set.
+    if not os.environ.get('GOOGLE_APPLICATION_CREDENTIALS', ''):
+        return
+
+    # Run the system tests against latest Python 2 and Python 3 only.
+    session.interpreter = 'python%s' % python_version
+
+    # Install all test dependencies, then install this package into the
+    # virutalenv's dist-packages.
+    session.chdir(os.path.dirname(__file__))
+    session.install('mock', 'pytest',
+                    '../core/', '../test_utils/')
+    session.install('.')
+
+    # Run py.test against the system tests.
+    session.run('py.test', '--quiet', 'tests/system.py')
+
+
+@nox.session
+def lint(session):
+    """Run flake8.
+
+    Returns a failure if flake8 finds linting errors or sufficiently
+    serious code quality issues.
+    """
+    session.interpreter = 'python3.6'
+    session.chdir(os.path.dirname(__file__))
+    session.install('flake8')
+    session.install('.')
+    session.run('flake8', 'google/cloud/bigtable')
+
+
+@nox.session
+def cover(session):
+    """Run the final coverage report.
+
+    This outputs the coverage report aggregating coverage from the unit
+    test runs (not system test runs), and then erases coverage data.
+    """
+    session.interpreter = 'python3.6'
+    session.chdir(os.path.dirname(__file__))
+    session.install('coverage', 'pytest-cov')
+    session.run('coverage', 'report', '--show-missing', '--fail-under=100')
+    session.run('coverage', 'erase')

system_tests/bigtable.py → bigtable/tests/system.py

@@ -31,10 +31,10 @@
 from google.cloud.bigtable.row_data import PartialRowData
 from google.cloud.environment_vars import BIGTABLE_EMULATOR
 
-from retry import RetryErrors
-from retry import RetryResult
-from system_test_utils import EmulatorCreds
-from system_test_utils import unique_resource_id
+from test_utils.retry import RetryErrors
+from test_utils.retry import RetryResult
+from test_utils.system import EmulatorCreds
+from test_utils.system import unique_resource_id
 
 
 LOCATION_ID = 'us-central1-c'

bigtable/unit_tests/test_client.py → bigtable/tests/unit/test_client.py

@@ -543,7 +543,7 @@ def test_list_instances(self):
             instance_pb2 as data_v2_pb2)
         from google.cloud.bigtable._generated import (
             bigtable_instance_admin_pb2 as messages_v2_pb2)
-        from unit_tests._testing import _FakeStub
+        from tests.unit._testing import _FakeStub
 
         LOCATION = 'projects/' + self.PROJECT + '/locations/locname'
         FAILED_LOCATION = 'FAILED'