diff --git a/system_tests/bigtable.py b/system_tests/bigtable.py index 5a19175266ce..732938c1c722 100644 --- a/system_tests/bigtable.py +++ b/system_tests/bigtable.py @@ -14,6 +14,7 @@ import datetime import operator +import os import unittest @@ -28,9 +29,11 @@ from google.cloud.bigtable.row_filters import RowFilterUnion from google.cloud.bigtable.row_data import Cell from google.cloud.bigtable.row_data import PartialRowData +from google.cloud.environment_vars import BIGTABLE_EMULATOR from retry import RetryErrors from retry import RetryResult +from system_test_utils import EmulatorCreds from system_test_utils import unique_resource_id @@ -59,6 +62,7 @@ class Config(object): """ CLIENT = None INSTANCE = None + IN_EMULATOR = False def _wait_until_complete(operation, max_attempts=5): @@ -91,29 +95,43 @@ def _retry_on_unavailable(exc): def setUpModule(): from google.cloud.exceptions import GrpcRendezvous - Config.CLIENT = Client(admin=True) + Config.IN_EMULATOR = os.getenv(BIGTABLE_EMULATOR) is not None + + if Config.IN_EMULATOR: + credentials = EmulatorCreds() + Config.CLIENT = Client(admin=True, credentials=credentials) + else: + Config.CLIENT = Client(admin=True) + Config.INSTANCE = Config.CLIENT.instance(INSTANCE_ID, LOCATION_ID) - retry = RetryErrors(GrpcRendezvous, error_predicate=_retry_on_unavailable) - instances, failed_locations = retry(Config.CLIENT.list_instances)() - if len(failed_locations) != 0: - raise ValueError('List instances failed in module set up.') + if not Config.IN_EMULATOR: + retry = RetryErrors(GrpcRendezvous, + error_predicate=_retry_on_unavailable) + instances, failed_locations = retry(Config.CLIENT.list_instances)() - EXISTING_INSTANCES[:] = instances + if len(failed_locations) != 0: + raise ValueError('List instances failed in module set up.') - # After listing, create the test instance. - created_op = Config.INSTANCE.create() - if not _wait_until_complete(created_op): - raise RuntimeError('Instance creation exceed 5 seconds.') + EXISTING_INSTANCES[:] = instances + + # After listing, create the test instance. + created_op = Config.INSTANCE.create() + if not _wait_until_complete(created_op): + raise RuntimeError('Instance creation exceed 5 seconds.') def tearDownModule(): - Config.INSTANCE.delete() + if not Config.IN_EMULATOR: + Config.INSTANCE.delete() class TestInstanceAdminAPI(unittest.TestCase): def setUp(self): + if Config.IN_EMULATOR is not None: + self.skipTest( + 'Instance Admin API not supported in Bigtable emulator') self.instances_to_delete = [] def tearDown(self): @@ -292,6 +310,13 @@ def tearDownClass(cls): # Will also delete any data contained in the table. cls._table.delete() + def _maybe_emulator_skip(self, message): + # NOTE: This method is necessary because ``Config.IN_EMULATOR`` + # is set at runtime rather than import time, which means we + # can't use the @unittest.skipIf decorator. + if Config.IN_EMULATOR is not None: + self.skipTest(message) + def setUp(self): self.rows_to_delete = [] @@ -401,6 +426,7 @@ def test_read_rows(self): self.assertEqual(rows_data.rows, expected_rows) def test_read_with_label_applied(self): + self._maybe_emulator_skip('Labels not supported by Bigtable emulator') row = self._table.row(ROW_KEY) self.rows_to_delete.append(row) diff --git a/system_tests/pubsub.py b/system_tests/pubsub.py index 6391945b5d3a..4eaa3062006e 100644 --- a/system_tests/pubsub.py +++ b/system_tests/pubsub.py @@ -45,16 +45,18 @@ class Config(object): global state. """ CLIENT = None + IN_EMULATOR = False def setUpModule(): - if os.getenv(PUBSUB_EMULATOR) is None: - Config.CLIENT = client.Client() - else: + Config.IN_EMULATOR = os.getenv(PUBSUB_EMULATOR) is not None + if Config.IN_EMULATOR: credentials = EmulatorCreds() http = httplib2.Http() # Un-authorized. Config.CLIENT = client.Client(credentials=credentials, http=http) + else: + Config.CLIENT = client.Client() class TestPubsub(unittest.TestCase): @@ -203,11 +205,10 @@ def suction(self): self.assertEqual(message2.attributes['extra'], EXTRA_2) def _maybe_emulator_skip(self): - # NOTE: We check at run-time rather than using the @unittest.skipIf - # decorator. This matches the philosophy behind using - # setUpModule to determine the environment at run-time - # rather than at import time. - if os.getenv(PUBSUB_EMULATOR) is not None: + # NOTE: This method is necessary because ``Config.IN_EMULATOR`` + # is set at runtime rather than import time, which means we + # can't use the @unittest.skipIf decorator. + if Config.IN_EMULATOR: self.skipTest('IAM not supported by Pub/Sub emulator') def test_topic_iam_policy(self): diff --git a/system_tests/run_emulator.py b/system_tests/run_emulator.py index 2a1e5d41a53b..41a79bfb6b67 100644 --- a/system_tests/run_emulator.py +++ b/system_tests/run_emulator.py @@ -25,21 +25,27 @@ import psutil +from google.cloud.environment_vars import BIGTABLE_EMULATOR from google.cloud.environment_vars import GCD_DATASET from google.cloud.environment_vars import GCD_HOST from google.cloud.environment_vars import PUBSUB_EMULATOR from run_system_test import run_module_tests +BIGTABLE = 'bigtable' +DATASTORE = 'datastore' +PUBSUB = 'pubsub' PACKAGE_INFO = { - 'datastore': (GCD_DATASET, GCD_HOST), - 'pubsub': (PUBSUB_EMULATOR,) + BIGTABLE: (BIGTABLE_EMULATOR,), + DATASTORE: (GCD_DATASET, GCD_HOST), + PUBSUB: (PUBSUB_EMULATOR,), } EXTRA = { - 'datastore': ('--no-legacy',), + DATASTORE: ('--no-legacy',), } _DS_READY_LINE = '[datastore] Dev App Server is now running.\n' _PS_READY_LINE_PREFIX = '[pubsub] INFO: Server started, listening on ' +_BT_READY_LINE_PREFIX = '[bigtable] Cloud Bigtable emulator running on ' def get_parser(): @@ -51,8 +57,8 @@ def get_parser(): parser = argparse.ArgumentParser( description='Run google-cloud system tests against local emulator.') parser.add_argument('--package', dest='package', - choices=('datastore', 'pubsub'), - default='datastore', help='Package to be tested.') + choices=sorted(PACKAGE_INFO.keys()), + default=DATASTORE, help='Package to be tested.') return parser @@ -95,16 +101,18 @@ def datastore_wait_ready(popen): emulator_ready = popen.stderr.readline() == _DS_READY_LINE -def pubsub_wait_ready(popen): - """Wait until the pubsub emulator is ready to use. +def wait_ready_prefix(popen, prefix): + """Wait until the a process encounters a line with matching prefix. :type popen: :class:`subprocess.Popen` :param popen: An open subprocess to interact with. + + :type prefix: str + :param prefix: The prefix to match """ emulator_ready = False while not emulator_ready: - emulator_ready = popen.stderr.readline().startswith( - _PS_READY_LINE_PREFIX) + emulator_ready = popen.stderr.readline().startswith(prefix) def wait_ready(package, popen): @@ -117,14 +125,16 @@ def wait_ready(package, popen): :param popen: An open subprocess to interact with. :raises: :class:`KeyError` if the ``package`` is not among - ``datastore``, ``pubsub``. + ``datastore``, ``pubsub`` or ``bigtable``. """ - if package == 'datastore': + if package == DATASTORE: datastore_wait_ready(popen) - elif package == 'pubsub': - pubsub_wait_ready(popen) + elif package == PUBSUB: + wait_ready_prefix(popen, _PS_READY_LINE_PREFIX) + elif package == BIGTABLE: + wait_ready_prefix(popen, _BT_READY_LINE_PREFIX) else: - raise KeyError('') + raise KeyError('Package not supported', package) def cleanup(pid): diff --git a/tox.ini b/tox.ini index 7c7850c16ec9..d0cc47ded48d 100644 --- a/tox.ini +++ b/tox.ini @@ -112,23 +112,34 @@ commands = python {toxinidir}/system_tests/attempt_system_tests.py {posargs} passenv = {[testenv:system-tests]passenv} -[testenv:datastore-emulator] -basepython = - python2.7 -commands = - python {toxinidir}/system_tests/run_emulator.py --package=datastore +[emulator] +deps = + {[testenv]deps} + psutil setenv = GOOGLE_CLOUD_NO_PRINT=true passenv = GOOGLE_CLOUD_DISABLE_GRPC -deps = - {[testenv]deps} - psutil +emulatorcmd = + python {toxinidir}/system_tests/run_emulator.py + +[testenv:datastore-emulator] +commands = + {[emulator]emulatorcmd} --package=datastore +setenv = {[emulator]setenv} +passenv = {[emulator]passenv} +deps = {[emulator]deps} [testenv:pubsub-emulator] -basepython = - python2.7 commands = - python {toxinidir}/system_tests/run_emulator.py --package=pubsub -passenv = GOOGLE_CLOUD_* -deps = {[testenv:datastore-emulator]deps} + {[emulator]emulatorcmd} --package=pubsub +setenv = {[emulator]setenv} +passenv = {[emulator]passenv} +deps = {[emulator]deps} + +[testenv:bigtable-emulator] +commands = + {[emulator]emulatorcmd} --package=bigtable +setenv = {[emulator]setenv} +passenv = {[emulator]passenv} +deps = {[emulator]deps}