From f54fff223246c8288dbed7c79a311a6171ee6052 Mon Sep 17 00:00:00 2001 From: "zachary.burke" Date: Thu, 18 Nov 2021 10:01:16 -0800 Subject: [PATCH] CP-6009 Add to_str and to_byte helpers --- .github/workflows/pre-commit.yml | 86 ++- .gitignore | 7 +- common/.python-version | 1 - common/setup.cfg | 19 +- common/setup.py | 1 + .../dlpx/virtualization/common/__init__.py | 4 +- .../virtualization/common/_common_classes.py | 17 +- .../dlpx/virtualization/common/exceptions.py | 19 +- .../python/dlpx/virtualization/common/util.py | 125 +++++ .../dlpx/virtualization/common/__init__.py | 4 +- .../common/test_common_classes.py | 278 +++++++--- docs/docs/images/.DS_Store | Bin 6148 -> 0 bytes docs/material/.DS_Store | Bin 8196 -> 0 bytes dvp/.python-version | 2 +- dvp/setup.cfg | 24 +- dvp/setup.py | 1 + libs/.python-version | 1 - libs/setup.cfg | 21 +- libs/setup.py | 1 + .../dlpx/virtualization/libs/__init__.py | 6 +- .../dlpx/virtualization/libs/_logging.py | 4 +- .../dlpx/virtualization/libs/exceptions.py | 24 +- .../python/dlpx/virtualization/libs/libs.py | 99 ++-- .../dlpx/virtualization/_engine/libs.py | 4 +- .../python/dlpx/virtualization/conftest.py | 5 +- .../python/dlpx/virtualization/test_libs.py | 489 +++++++++++------- .../dlpx/virtualization/test_logging.py | 4 +- platform/.python-version | 1 - platform/setup.cfg | 21 +- platform/setup.py | 1 + .../dlpx/virtualization/platform/__init__.py | 24 +- .../platform/_plugin_classes.py | 10 +- .../dlpx/virtualization/platform/_virtual.py | 16 +- .../platform/import_validations.py | 9 +- .../platform/migration_helper.py | 3 +- .../dlpx/virtualization/platform/util.py | 5 +- .../fake_generated_definitions.py | 13 +- .../python/dlpx/virtualization/test_plugin.py | 209 +++++--- .../virtualization/test_plugin_classes.py | 109 ++-- .../dlpx/virtualization/test_upgrade.py | 14 +- tools/.python-version | 1 - tools/setup.cfg | 22 +- tools/setup.py | 20 +- .../dlpx/virtualization/_internal/cli.py | 11 +- .../dlpx/virtualization/_internal/codegen.py | 16 +- .../codegen/templates/base_model_.mustache | 47 +- .../_internal/codegen/templates/util.mustache | 22 +- .../_internal/commands/build.py | 34 +- .../_internal/commands/download_logs.py | 14 +- .../_internal/commands/initialize.py | 51 +- .../_internal/commands/upload.py | 15 +- .../_internal/delphix_client.py | 27 +- .../virtualization/_internal/exceptions.py | 13 +- .../virtualization/_internal/package_util.py | 4 +- .../_internal/plugin_dependency_util.py | 27 +- .../_internal/plugin_importer.py | 54 +- .../_internal/plugin_validator.py | 17 +- .../_internal/schema_validator.py | 21 +- .../plugin_config_schema.json | 2 +- .../validation_schemas/plugin_schema.json | 4 - .../_internal/commands/test_build.py | 71 +-- .../_internal/commands/test_codegen.py | 8 +- .../_internal/commands/test_delphix_client.py | 47 +- .../_internal/commands/test_initialize.py | 36 +- .../_internal/commands/test_templates.py | 320 +++++++----- .../dlpx/virtualization/_internal/conftest.py | 16 +- .../dlpx/virtualization/_internal/test_cli.py | 44 +- .../_internal/test_plugin_importer.py | 23 +- .../_internal/test_plugin_validator.py | 6 +- .../_internal/test_schema_validator.py | 6 +- 70 files changed, 1758 insertions(+), 922 deletions(-) delete mode 100644 common/.python-version create mode 100644 common/src/main/python/dlpx/virtualization/common/util.py delete mode 100644 docs/docs/images/.DS_Store delete mode 100644 docs/material/.DS_Store delete mode 100644 libs/.python-version delete mode 100644 platform/.python-version delete mode 100644 tools/.python-version diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index ab5d1efe..eaf4deec 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -3,14 +3,49 @@ name: Pre-commit actions for Delphix Virtualization SDK on: [pull_request] jobs: - pytest: - name: Test ${{ matrix.package }} on ${{ matrix.os }} using pytest + pytest27: + name: Test ${{ matrix.package }} on ${{ matrix.os }} using pytest (Python 2.7) runs-on: ${{ matrix.os }} strategy: max-parallel: 4 matrix: python-version: [2.7] os: [ubuntu-latest, macos-latest, windows-latest] + package: [common, libs, platform] + + steps: + - name: Checkout ${{ matrix.package }} project + uses: actions/checkout@v1 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + + - name: Install ${{ matrix.package }} dependencies + working-directory: ${{ matrix.package }} + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt --find-links https://test.pypi.org/simple/dvp-api/ + + - name: Install ${{ matrix.package }} project + working-directory: ${{ matrix.package }} + run: | + pip install . --find-links https://test.pypi.org/simple/dvp-api/ + + - name: Test ${{ matrix.package }} project with pytest + working-directory: ${{ matrix.package }} + run: | + python -m pytest src/test/python + + pytest38: + name: Test ${{ matrix.package }} on ${{ matrix.os }} using pytest (Python 3.8) + runs-on: ${{ matrix.os }} + strategy: + max-parallel: 4 + matrix: + python-version: [3.8] + os: [ubuntu-latest, macos-latest, windows-latest] package: [common, libs, platform, tools] steps: @@ -38,33 +73,64 @@ jobs: run: | python -m pytest src/test/python - lint: - name: Lint ${{ matrix.package }} + lintpython27: + name: Lint ${{ matrix.package }} - Python27 runs-on: ubuntu-latest strategy: max-parallel: 4 matrix: - package: [tools] - + package: [common, libs, platform] + steps: - name: Checkout ${{ matrix.package }} uses: actions/checkout@v1 - + - name: Set up Python 2.7 uses: actions/setup-python@v1 with: python-version: 2.7 - + - name: Install flake8 run: | python -m pip install --upgrade pip pip install flake8 - + - name: Run flake8 on src directory working-directory: ${{ matrix.package }} run: python -m flake8 src/main/python --max-line-length 88 - + + - name: Run flake8 on test directory + working-directory: ${{ matrix.package }} + run: python -m flake8 src/test/python --max-line-length 88 + + lintpython38: + name: Lint ${{ matrix.package }} - Python38 + + runs-on: ubuntu-latest + strategy: + max-parallel: 4 + matrix: + package: [common, libs, platform, tools] + + steps: + - name: Checkout ${{ matrix.package }} + uses: actions/checkout@v1 + + - name: Set up Python 3.8 + uses: actions/setup-python@v1 + with: + python-version: 3.8 + + - name: Install flake8 + run: | + python -m pip install --upgrade pip + pip install flake8 + + - name: Run flake8 on src directory + working-directory: ${{ matrix.package }} + run: python -m flake8 src/main/python --max-line-length 88 + - name: Run flake8 on test directory working-directory: ${{ matrix.package }} run: python -m flake8 src/test/python --max-line-length 88 diff --git a/.gitignore b/.gitignore index c73fd091..b1fe796c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # # IntelliJ config files @@ -35,3 +35,8 @@ venv/ # Python cache __pycache__ +# .python-version files +*.python-version + +# OSX DS Store files +*.DS_Store diff --git a/common/.python-version b/common/.python-version deleted file mode 100644 index 43c4dbe6..00000000 --- a/common/.python-version +++ /dev/null @@ -1 +0,0 @@ -2.7.17 diff --git a/common/setup.cfg b/common/setup.cfg index 8199f265..a482f5f1 100644 --- a/common/setup.cfg +++ b/common/setup.cfg @@ -1,20 +1,21 @@ # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # [metadata] -Metadata-Version: 1.2 -Author: Delphix -Author-email: virtualization-plugins@delphix.com -Home-page: https://developer.delphix.com -Long-description: file: README.md -Long-description-content-type: text/markdown -Classifiers: +metadata_version: 1.2 +author: Delphix +author_email: virtualization-plugins@delphix.com +home_page: https://developer.delphix.com +long_description: file: README.md +long_description_content_type: text/markdown +classifiers: Development Status :: 5 - Production/Stable Programming Language :: Python Programming Language :: Python :: 2.7 + Programming Language :: Python :: 3.8 License :: OSI Approved :: Apache Software License Operating System :: OS Independent [options] -Requires-Python: 2.7 +requires_python: >=2.7, <=3.8, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*, !=3.7.* diff --git a/common/setup.py b/common/setup.py index 4e6b897e..720fc31e 100644 --- a/common/setup.py +++ b/common/setup.py @@ -15,4 +15,5 @@ install_requires=install_requires, package_dir={'': PYTHON_SRC}, packages=setuptools.find_packages(PYTHON_SRC), + python_requires='>=2.7, <3.9, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*, !=3.7.*', ) diff --git a/common/src/main/python/dlpx/virtualization/common/__init__.py b/common/src/main/python/dlpx/virtualization/common/__init__.py index b4692226..c4c8dcd6 100644 --- a/common/src/main/python/dlpx/virtualization/common/__init__.py +++ b/common/src/main/python/dlpx/virtualization/common/__init__.py @@ -1,7 +1,7 @@ # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # __path__ = __import__('pkgutil').extend_path(__path__, __name__) -from dlpx.virtualization.common._common_classes import * +from dlpx.virtualization.common._common_classes import * # noqa diff --git a/common/src/main/python/dlpx/virtualization/common/_common_classes.py b/common/src/main/python/dlpx/virtualization/common/_common_classes.py index 1cf6b210..57a698f4 100644 --- a/common/src/main/python/dlpx/virtualization/common/_common_classes.py +++ b/common/src/main/python/dlpx/virtualization/common/_common_classes.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # from abc import ABCMeta @@ -134,7 +134,9 @@ def reference(self): return self.__reference def to_proto(self): - """Converts plugin class RemoteEnvironment to protobuf class common_pb2.RemoteEnvironment + """ + Converts plugin class RemoteEnvironment to protobuf + class common_pb2.RemoteEnvironment """ remote_environment = common_pb2.RemoteEnvironment() remote_environment.name = self.name @@ -144,7 +146,9 @@ def to_proto(self): @staticmethod def from_proto(environment): - """Converts protobuf class common_pb2.RemoteEnvironment to plugin class RemoteEnvironment + """ + Converts protobuf class common_pb2.RemoteEnvironment to plugin + class RemoteEnvironment """ if not isinstance(environment, common_pb2.RemoteEnvironment): raise IncorrectTypeError( @@ -353,7 +357,9 @@ def password(self): @staticmethod def from_proto(credentials_result): - """Converts protobuf class libs_pb2.CredentialsResult to plugin class PasswordCredentials + """ + Converts protobuf class libs_pb2.CredentialsResult to plugin + class PasswordCredentials """ if not isinstance(credentials_result, libs_pb2.CredentialsResult): raise IncorrectTypeError( @@ -375,7 +381,8 @@ class KeyPairCredentials(Credentials): Args: username (str): User name. private_key (str): Private key. - public_key (str): Public key corresponding to private key. Empty string if not present. + public_key (str): Public key corresponding to private key. Empty string if + not present. """ def __init__(self, username, private_key, public_key): super(KeyPairCredentials, self).__init__(username) diff --git a/common/src/main/python/dlpx/virtualization/common/exceptions.py b/common/src/main/python/dlpx/virtualization/common/exceptions.py index db84434d..3ecef88e 100644 --- a/common/src/main/python/dlpx/virtualization/common/exceptions.py +++ b/common/src/main/python/dlpx/virtualization/common/exceptions.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # @@ -104,7 +104,8 @@ def _get_type_name(type_object): if len(expected_type) > 1: for index in range(0, len(expected_type)): expected_type[index] = _get_type_name(expected_type[index]) - expected_type[index] = _remove_angle_brackets(str(expected_type[index])) + expected_type[index] = _remove_angle_brackets( + str(expected_type[index])) expected = "any one of the following types: '{}'".format(expected_type) elif len(expected_type) == 0: @@ -120,8 +121,8 @@ def _get_type_name(type_object): raise PlatformError('The thrown TypeError should have had a' ' dict of size 1 as the expected_type') - key_type = expected_type.keys()[0] - value_type = expected_type.values()[0] + key_type = list(expected_type.keys())[0] + value_type = list(expected_type.values())[0] key_type_name = _get_type_name(key_type) value_type_name = _get_type_name(value_type) @@ -147,7 +148,7 @@ def _get_type_name(type_object): raise PlatformError('The thrown TypeError should have had a' ' set of tuples to represent a dict') actual = 'a dict of {{{}}}'.format(', '.join(['{0}:{1}'.format( - _remove_angle_brackets(str(k)), + _remove_angle_brackets(str(k)), _remove_angle_brackets(str(v))) for k, v in actual_type])) else: actual = _remove_angle_brackets(str(actual_type)) @@ -172,12 +173,8 @@ class IncorrectTypeError(PluginRuntimeError): """ def __init__( - self, - object_type, - parameter_name, - actual_type, - expected_type, - required=True): + self, object_type, parameter_name, actual_type, expected_type, + required=True): actual, expected = self.get_actual_and_expected_type( actual_type, expected_type) diff --git a/common/src/main/python/dlpx/virtualization/common/util.py b/common/src/main/python/dlpx/virtualization/common/util.py new file mode 100644 index 00000000..2ff42f6f --- /dev/null +++ b/common/src/main/python/dlpx/virtualization/common/util.py @@ -0,0 +1,125 @@ +# +# Copyright (c) 2021 by Delphix. All rights reserved. +# + + +""" +Utility functions to convert between unicode and bytes. +""" + +import six + + +def to_bytes(string, encoding="utf-8"): + """ + Converts the given object to binary object, bytes (Py3) or str (Py2). + + :param string: The string like object to convert to bytes + :type string: ``object`` + :param encoding: The encoding to encode the string with. + :type encoding: ``str`` + :returns: The encoded string. + :rtype: ``bytes`` + """ + if string is None: + return + + if isinstance(string, dict): + for k, v in string.items(): + string[k] = to_bytes(v, encoding=encoding) + return string + + if isinstance(string, list): + return [to_bytes(i, encoding=encoding) for i in string] + + if isinstance(string, set): + return {to_bytes(i, encoding=encoding) for i in string} + + if isinstance(string, str): + return _to_bytes(string, encoding) + + return string + + +def _to_bytes(string, encoding): + if six.PY3: + if isinstance(string, str): + return string.encode(encoding) + else: + return bytes(string) + else: + if isinstance(string, unicode): # noqa + return string.encode(encoding) + else: + return str(string) + + +def to_str(b, encoding="utf-8"): + """ + Converts the given object to a text object, unicode (Py2) or str (Py3). + + :param b: The object to convert + :type b: ``object`` + :param encoding: The encoding to encode the string with. + :type encoding: ``str`` + :returns: The decoded string. + :rtype: ``str`` + """ + if b is None: + return + + if isinstance(b, dict): + for k, v in b.items(): + b[k] = to_str(v, encoding=encoding) + return b + + if isinstance(b, list): + return [to_str(i, encoding=encoding) for i in b] + + if isinstance(b, set): + return {to_str(i, encoding=encoding) for i in b} + + if isinstance(b, bytes): + return _to_str(b, encoding=encoding) + return b + + +def _to_str(b, encoding): + if six.PY3: + if isinstance(b, bytes): + try: + return str(b, encoding) + except UnicodeDecodeError: + pass + raise UnicodeError( + "Could not decode value with encoding {}".format(encoding) + ) + else: + return b + else: + if isinstance(b, str): + try: + return b.decode(encoding) + except UnicodeDecodeError: + pass + raise UnicodeError( + "Could not decode value with encoding {}".format(encoding) + ) + return b + + +def response_to_str(response): + """ + The response_to_str function ensures all relevant properties of the given + response are unicode (py2) / str (py3). Should be called on a response as + soon as it is received. + + Args: + response (RunPowerShellResponse or RunBashResponse or RunExpectResponse): + Response received by run_bash or run_powershell or run_expect + """ + if response.HasField("return_value"): + if hasattr(response.return_value, "stdout"): + response.return_value.stdout = to_str(response.return_value.stdout) + if hasattr(response.return_value, "stderr"): + response.return_value.stderr = to_str(response.return_value.stderr) diff --git a/common/src/test/python/dlpx/virtualization/common/__init__.py b/common/src/test/python/dlpx/virtualization/common/__init__.py index c7fd3fc1..e9e178ce 100644 --- a/common/src/test/python/dlpx/virtualization/common/__init__.py +++ b/common/src/test/python/dlpx/virtualization/common/__init__.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # -__path__ = __import__('pkgutil').extend_path(__path__, __name__) \ No newline at end of file +__path__ = __import__('pkgutil').extend_path(__path__, __name__) diff --git a/common/src/test/python/dlpx/virtualization/common/test_common_classes.py b/common/src/test/python/dlpx/virtualization/common/test_common_classes.py index 49a7f042..16d6373d 100644 --- a/common/src/test/python/dlpx/virtualization/common/test_common_classes.py +++ b/common/src/test/python/dlpx/virtualization/common/test_common_classes.py @@ -1,12 +1,16 @@ # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # import pytest +import six from dlpx.virtualization.api import common_pb2 from dlpx.virtualization.common._common_classes import ( - KeyPairCredentials, PasswordCredentials, RemoteConnection, RemoteEnvironment, RemoteHost, RemoteUser) -from dlpx.virtualization.common.exceptions import IncorrectTypeError, PluginRuntimeError, PlatformError + KeyPairCredentials, PasswordCredentials, RemoteConnection, RemoteEnvironment, + RemoteHost, RemoteUser) +from dlpx.virtualization.common.exceptions import ( + IncorrectTypeError, PluginRuntimeError, PlatformError) + @pytest.fixture def remote_user(): @@ -32,19 +36,31 @@ def test_init_remote_connection_success(remote_user, remote_environment): def test_init_remote_connection_incorrect_environment(remote_user): with pytest.raises(IncorrectTypeError) as err_info: RemoteConnection('', remote_user) - assert err_info.value.message == ( - "RemoteConnection's parameter 'environment' was" - " type 'str' but should be of class 'dlpx.virtualization" - ".common._common_classes.RemoteEnvironment'.") + if six.PY2: + assert err_info.value.message == ( + "RemoteConnection's parameter 'environment' was" + " type 'str' but should be of class 'dlpx.virtualization" + ".common._common_classes.RemoteEnvironment'.") + else: + assert err_info.value.message == ( + "RemoteConnection's parameter 'environment' was" + " class 'str' but should be of class 'dlpx.virtualization" + ".common._common_classes.RemoteEnvironment'.") @staticmethod def test_init_remote_connection_incorrect_user(remote_environment): with pytest.raises(IncorrectTypeError) as err_info: RemoteConnection(remote_environment, '') - assert err_info.value.message == ( - "RemoteConnection's parameter 'user' was" - " type 'str' but should be of class 'dlpx.virtualization" - ".common._common_classes.RemoteUser'.") + if six.PY2: + assert err_info.value.message == ( + "RemoteConnection's parameter 'user' was" + " type 'str' but should be of class 'dlpx.virtualization" + ".common._common_classes.RemoteUser'.") + else: + assert err_info.value.message == ( + "RemoteConnection's parameter 'user' was" + " class 'str' but should be of class 'dlpx.virtualization" + ".common._common_classes.RemoteUser'.") @staticmethod def test_remote_connection_to_proto(remote_user, remote_environment): @@ -62,10 +78,16 @@ def test_remote_connection_from_proto_success(): def test_remote_connection_from_proto_fail(): with pytest.raises(IncorrectTypeError) as err_info: RemoteConnection.from_proto('') - assert err_info.value.message == ( - "RemoteConnection's parameter 'connection' was" - " type 'str' but should be of class 'dlpx.virtualization.api" - ".common_pb2.RemoteConnection'.") + if six.PY2: + assert err_info.value.message == ( + "RemoteConnection's parameter 'connection' was" + " type 'str' but should be of class 'dlpx.virtualization.api" + ".common_pb2.RemoteConnection'.") + else: + assert err_info.value.message == ( + "RemoteConnection's parameter 'connection' was" + " class 'str' but should be of class 'dlpx.virtualization.api" + ".common_pb2.RemoteConnection'.") class TestRemoteEnvironment: @@ -77,26 +99,42 @@ def test_init_remote_environment_success(remote_host): def test_init_remote_environment_incorrect_name(remote_host): with pytest.raises(IncorrectTypeError) as err_info: RemoteEnvironment(1, '', remote_host) - assert err_info.value.message == ( - "RemoteEnvironment's parameter 'name' was" - " type 'int' but should be of type 'basestring'.") + if six.PY2: + assert err_info.value.message == ( + "RemoteEnvironment's parameter 'name' was" + " type 'int' but should be of type 'basestring'.") + else: + assert err_info.value.message == ( + "RemoteEnvironment's parameter 'name' was" + " class 'int' but should be of class 'str'.") @staticmethod def test_init_remote_environment_incorrect_reference(remote_host): with pytest.raises(IncorrectTypeError) as err_info: RemoteEnvironment('', 1, remote_host) - assert err_info.value.message == ( - "RemoteEnvironment's parameter 'reference' was" - " type 'int' but should be of type 'basestring'.") + if six.PY2: + assert err_info.value.message == ( + "RemoteEnvironment's parameter 'reference' was" + " type 'int' but should be of type 'basestring'.") + else: + assert err_info.value.message == ( + "RemoteEnvironment's parameter 'reference' was" + " class 'int' but should be of class 'str'.") @staticmethod def test_init_remote_environment_incorrect_host(): with pytest.raises(IncorrectTypeError) as err_info: RemoteEnvironment('', '', '') - assert err_info.value.message == ( - "RemoteEnvironment's parameter 'host' was" - " type 'str' but should be of class 'dlpx.virtualization" - ".common._common_classes.RemoteHost'.") + if six.PY2: + assert err_info.value.message == ( + "RemoteEnvironment's parameter 'host' was" + " type 'str' but should be of class 'dlpx.virtualization" + ".common._common_classes.RemoteHost'.") + else: + assert err_info.value.message == ( + "RemoteEnvironment's parameter 'host' was" + " class 'str' but should be of class 'dlpx.virtualization" + ".common._common_classes.RemoteHost'.") @staticmethod def test_remote_environment_to_proto(remote_host): @@ -114,10 +152,16 @@ def test_remote_environment_from_proto_success(): def test_remote_environment_from_proto_fail(): with pytest.raises(IncorrectTypeError) as err_info: RemoteEnvironment.from_proto('') - assert err_info.value.message == ( - "RemoteEnvironment's parameter 'environment' was" - " type 'str' but should be of class 'dlpx.virtualization.api" - ".common_pb2.RemoteEnvironment'.") + if six.PY2: + assert err_info.value.message == ( + "RemoteEnvironment's parameter 'environment' was" + " type 'str' but should be of class 'dlpx.virtualization.api" + ".common_pb2.RemoteEnvironment'.") + else: + assert err_info.value.message == ( + "RemoteEnvironment's parameter 'environment' was" + " class 'str' but should be of class 'dlpx.virtualization.api" + ".common_pb2.RemoteEnvironment'.") class TestRemoteHost: @@ -132,33 +176,53 @@ def test_init_remote_host_success(): def test_init_remote_host_incorrect_name(): with pytest.raises(IncorrectTypeError) as err_info: RemoteHost(1, '', '', '') - assert err_info.value.message == ( - "RemoteHost's parameter 'name' was" - " type 'int' but should be of type 'basestring'.") + if six.PY2: + assert err_info.value.message == ( + "RemoteHost's parameter 'name' was" + " type 'int' but should be of type 'basestring'.") + else: + assert err_info.value.message == ( + "RemoteHost's parameter 'name' was" + " class 'int' but should be of class 'str'.") @staticmethod def test_init_remote_host_incorrect_reference(): with pytest.raises(IncorrectTypeError) as err_info: RemoteHost('', 1, '', '') - assert err_info.value.message == ( - "RemoteHost's parameter 'reference' was" - " type 'int' but should be of type 'basestring'.") + if six.PY2: + assert err_info.value.message == ( + "RemoteHost's parameter 'reference' was" + " type 'int' but should be of type 'basestring'.") + else: + assert err_info.value.message == ( + "RemoteHost's parameter 'reference' was" + " class 'int' but should be of class 'str'.") @staticmethod def test_init_remote_host_incorrect_binary_path(): with pytest.raises(IncorrectTypeError) as err_info: RemoteHost('', '', 1, '') - assert err_info.value.message == ( - "RemoteHost's parameter 'binary_path' was" - " type 'int' but should be of type 'basestring'.") + if six.PY2: + assert err_info.value.message == ( + "RemoteHost's parameter 'binary_path' was" + " type 'int' but should be of type 'basestring'.") + else: + assert err_info.value.message == ( + "RemoteHost's parameter 'binary_path' was" + " class 'int' but should be of class 'str'.") @staticmethod def test_init_remote_host_incorrect_scratch_path(): with pytest.raises(IncorrectTypeError) as err_info: RemoteHost('', '', '', 1) - assert err_info.value.message == ( - "RemoteHost's parameter 'scratch_path' was" - " type 'int' but should be of type 'basestring'.") + if six.PY2: + assert err_info.value.message == ( + "RemoteHost's parameter 'scratch_path' was" + " type 'int' but should be of type 'basestring'.") + else: + assert err_info.value.message == ( + "RemoteHost's parameter 'scratch_path' was" + " class 'int' but should be of class 'str'.") @staticmethod def test_remote_host_to_proto_non_default(): @@ -176,10 +240,16 @@ def test_remote_host_from_proto_success(): def test_remote_host_from_proto_fail(): with pytest.raises(IncorrectTypeError) as err_info: RemoteHost.from_proto('') - assert err_info.value.message == ( - "RemoteHost's parameter 'host' was" - " type 'str' but should be of class 'dlpx.virtualization.api" - ".common_pb2.RemoteHost'.") + if six.PY2: + assert err_info.value.message == ( + "RemoteHost's parameter 'host' was" + " type 'str' but should be of class 'dlpx.virtualization.api" + ".common_pb2.RemoteHost'.") + else: + assert err_info.value.message == ( + "RemoteHost's parameter 'host' was" + " class 'str' but should be of class 'dlpx.virtualization.api" + ".common_pb2.RemoteHost'.") class TestRemoteUser: @@ -191,17 +261,27 @@ def test_init_remote_user_success(): def test_init_remote_user_incorrect_name(): with pytest.raises(IncorrectTypeError) as err_info: RemoteUser(1, '') - assert err_info.value.message == ( - "RemoteUser's parameter 'name' was" - " type 'int' but should be of type 'basestring'.") + if six.PY2: + assert err_info.value.message == ( + "RemoteUser's parameter 'name' was" + " type 'int' but should be of type 'basestring'.") + else: + assert err_info.value.message == ( + "RemoteUser's parameter 'name' was" + " class 'int' but should be of class 'str'.") @staticmethod def test_init_remote_user_incorrect_reference(): with pytest.raises(IncorrectTypeError) as err_info: RemoteUser('', 1) - assert err_info.value.message == ( - "RemoteUser's parameter 'reference' was" - " type 'int' but should be of type 'basestring'.") + if six.PY2: + assert err_info.value.message == ( + "RemoteUser's parameter 'reference' was" + " type 'int' but should be of type 'basestring'.") + else: + assert err_info.value.message == ( + "RemoteUser's parameter 'reference' was" + " class 'int' but should be of class 'str'.") @staticmethod def test_remote_user_to_proto_non_default(): @@ -219,10 +299,16 @@ def test_remote_user_from_proto_success(): def test_remote_user_from_proto_fail(): with pytest.raises(IncorrectTypeError) as err_info: RemoteUser.from_proto('') - assert err_info.value.message == ( - "RemoteUser's parameter 'user' was" - " type 'str' but should be of class 'dlpx.virtualization.api" - ".common_pb2.RemoteUser'.") + if six.PY2: + assert err_info.value.message == ( + "RemoteUser's parameter 'user' was" + " type 'str' but should be of class 'dlpx.virtualization.api" + ".common_pb2.RemoteUser'.") + else: + assert err_info.value.message == ( + "RemoteUser's parameter 'user' was" + " class 'str' but should be of class 'dlpx.virtualization.api" + ".common_pb2.RemoteUser'.") class TestCredentials: @@ -230,10 +316,16 @@ class TestCredentials: def test_init_credentials_incorrect_username_type(): with pytest.raises(IncorrectTypeError) as err_info: KeyPairCredentials(RemoteUser("user1", "reference1"), "1234", "5678") - assert err_info.value.message == ( - "Credentials's parameter 'username' was class " - "'dlpx.virtualization.common._common_classes.RemoteUser' but should be " - "of type 'basestring'.") + if six.PY2: + assert err_info.value.message == ( + "Credentials's parameter 'username' was class " + "'dlpx.virtualization.common._common_classes.RemoteUser' but should be " + "of type 'basestring'.") + else: + assert err_info.value.message == ( + "Credentials's parameter 'username' was class " + "'dlpx.virtualization.common._common_classes.RemoteUser' but should be " + "of class 'str'.") class TestKeyPairCredentials: @@ -241,17 +333,27 @@ class TestKeyPairCredentials: def test_init_key_pair_credentials_incorrect_public_key_type(): with pytest.raises(IncorrectTypeError) as err_info: KeyPairCredentials("user1", "1234", 1) - assert err_info.value.message == ( - "KeyPairCredentials's parameter 'public_key' was type 'int' " - "but should be of type 'basestring'.") + if six.PY2: + assert err_info.value.message == ( + "KeyPairCredentials's parameter 'public_key' was type 'int' " + "but should be of type 'basestring'.") + else: + assert err_info.value.message == ( + "KeyPairCredentials's parameter 'public_key' was class 'int' " + "but should be of class 'str'.") @staticmethod def test_init_key_pair_credentials_incorrect_private_key_type(): with pytest.raises(IncorrectTypeError) as err_info: KeyPairCredentials("user1", 1, "1234") - assert err_info.value.message == ( - "KeyPairCredentials's parameter 'private_key' was type 'int' " - "but should be of type 'basestring'.") + if six.PY2: + assert err_info.value.message == ( + "KeyPairCredentials's parameter 'private_key' was type 'int' " + "but should be of type 'basestring'.") + else: + assert err_info.value.message == ( + "KeyPairCredentials's parameter 'private_key' was class 'int' " + "but should be of class 'str'.") class TestPasswordCredentials: @@ -259,9 +361,14 @@ class TestPasswordCredentials: def test_init_password_credentials_incorrect_password_type(): with pytest.raises(IncorrectTypeError) as err_info: PasswordCredentials("user1", 12345) - assert err_info.value.message == ( - "PasswordCredentials's parameter 'password' was type 'int' but " - "should be of type 'basestring'.") + if six.PY2: + assert err_info.value.message == ( + "PasswordCredentials's parameter 'password' was type 'int' but " + "should be of type 'basestring'.") + else: + assert err_info.value.message == ( + "PasswordCredentials's parameter 'password' was class 'int' but " + "should be of class 'str'.") class TestPluginRuntimeError: @@ -269,43 +376,56 @@ class TestPluginRuntimeError: def test_plugin_runtime_error_get_actual_and_expected_type(): actual, expected = PluginRuntimeError.get_actual_and_expected_type( list([str]), dict({str: dict})) - assert actual == "a list of [type 'str']" + if six.PY2: + assert actual == "a list of [type 'str']" + else: + assert actual == "a list of [class 'str']" assert expected == "type 'dict of str:dict'" @staticmethod def test_plugin_runtime_error_get_actual_and_expected_type_multi_expected_types(): actual, expected = PluginRuntimeError.get_actual_and_expected_type( list([str]), list([str, int, dict, bool])) - assert actual == "a list of [type 'str']" - assert expected == "any one of the following types: '['str', 'int', 'dict', 'bool']'" + if six.PY2: + assert actual == "a list of [type 'str']" + else: + assert actual == "a list of [class 'str']" + assert expected == ( + "any one of the following types: '['str', 'int', 'dict', 'bool']'") @staticmethod - def test_plugin_runtime_error_get_actual_and_expected_type_single_item_list_expected_types(): + def test_plugin_runtime_error_get_actual_and_expected_type_single_item_list_expected_types(): # noqa actual, expected = PluginRuntimeError.get_actual_and_expected_type( list([str]), list([str])) - assert actual == "a list of [type 'str']" + if six.PY2: + assert actual == "a list of [type 'str']" + else: + assert actual == "a list of [class 'str']" assert expected == "type 'list of str'" @staticmethod - def test_plugin_runtime_error_get_actual_and_expected_type_empty_dict_expected_type(): + def test_plugin_runtime_error_get_actual_and_expected_type_empty_dict_expected_type(): # noqa with pytest.raises(PlatformError) as err_info: PluginRuntimeError.get_actual_and_expected_type(list(), dict()) assert err_info.value.message == ( - "The thrown TypeError should have had a dict of size 1 as the expected_type") + "The thrown TypeError should have had a dict of size 1 as the " + "expected_type") @staticmethod - def test_plugin_runtime_error_get_actual_and_expected_type_empty_list_expected_type(): + def test_plugin_runtime_error_get_actual_and_expected_type_empty_list_expected_type(): # noqa with pytest.raises(PlatformError) as err_info: PluginRuntimeError.get_actual_and_expected_type(list(), list()) assert err_info.value.message == ( - "The thrown TypeError should have had a list of size >= 1 as the expected_type") + "The thrown TypeError should have had a list of size >= 1 as the " + "expected_type") @staticmethod - def test_plugin_runtime_error_get_actual_and_expected_type_list_with_duplicate_expected_type(): + def test_plugin_runtime_error_get_actual_and_expected_type_list_with_duplicate_expected_type(): # noqa with pytest.raises(PlatformError) as err_info: PluginRuntimeError.get_actual_and_expected_type(list(), list([str, str])) assert err_info.value.message == ( - "The thrown TypeError should have had a list of size 1 as the expected_type") + "The thrown TypeError should have had a list of size 1 as the " + "expected_type") diff --git a/docs/docs/images/.DS_Store b/docs/docs/images/.DS_Store deleted file mode 100644 index 5008ddfcf53c02e82d7eee2e57c38e5672ef89f6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeH~Jr2S!425mzP>H1@V-^m;4Wg<&0T*E43hX&L&p$$qDprKhvt+--jT7}7np#A3 zem<@ulZcFPQ@L2!n>{z**++&mCkOWA81W14cNZlEfg7;MkzE(HCqgga^y>{tEnwC%0;vJ&^%eQ zLs35+`xjp>T0W7%LH>+6%r*|hl4&cGEwK&; z0t5mC0t5mC0t5mCCIbR=XY(RVaPNECpbiiS5cpprz~2v1dbo`Ha?q3RtAi>x1t3{Q z!)~EA7(47WZaj7p7h)qJa-0vW%wBi+||i1 z>Y6jeJsH#i0s#W02=Lr}8dI1}KUzutUJ*)+)n}}h7B%mr89V>{b*Ij~SgKbqtIzh1 z_&y-V$(N-h-*{MD|n=-5v>CU9GhHg6z4d)S>QZ{v3xgZFVFhkgz?6vxm zj^gC)W+k_q_qHicZ_00XSe7x|R#H23t*=L(m9R`l(M>JqI_s)NCVRScQ?>e!Pky#Z0Eu8ZB;x<7OHNdH{}cOU0x zp*N5JhGG?7f4F{zS|vvv{ki0^i{skZg@gx3=;IomV-RmIn6&vIvbhnBNRisH*Nt0M~+-?VJy znx+jy6>?>jG)tP@RnTocWoVs!y6W`qRP3~7s=9e#7vXO@`W?EK6Dx5{>@by#HdIwJ zXYRaf!{M>k#aX)ZDg9txcMi+7f-qDiS5`#lhowC#oo>{gy7}SoZmvn> zG^$l{a5uFiTrbLs&|%Ygy^*C-Oq6Z0<4z)pI}%!o6hDXA!|ZW(f}LWg*<0)bc7}byF0k*}FYH(L8~dGI z0#J@h%)tUIL>;cjQmnyRtV0~F=)?|mVLue4p+ZLvHu5N-ANS#YJb(vr6i?tuJdNXc z0k7ayyn#3IE^t@| z@%0k>6H_rA)rb;P>(NL|jU#~;tjAWg<92LE2lis0&shT|vN%Y*?Z*J_!65F%LwJN( z`zVg#DPrxjcn&A<5>66#U&HHo8}AT*-^2U(2%n8%YujkH;;Zr4I`SU;cG<}_OKs8&X+EUj6yPJ8kTv0bj66`c2`=acoWp{{D1w#-~X?4nZQ3lAV6ReBY?7& zWJ@z`aH5wB&$XlU?5BqpZa3&j*M%xK9Vh9g<0OawVMy&LnR1`FF9$tIBUJwL4*~z; UpSi*PAKd>V^Vp0vdGG(f0m+@jLjV8( diff --git a/dvp/.python-version b/dvp/.python-version index 43c4dbe6..d20cc2bf 100644 --- a/dvp/.python-version +++ b/dvp/.python-version @@ -1 +1 @@ -2.7.17 +3.8.10 diff --git a/dvp/setup.cfg b/dvp/setup.cfg index 776cfe6a..48fb5749 100644 --- a/dvp/setup.cfg +++ b/dvp/setup.cfg @@ -1,22 +1,22 @@ # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # [metadata] -Metadata-Version: 1.2 -Author: Delphix -Author-email: virtualization-plugins@delphix.com -Home-page: https://developer.delphix.com -Summary: Delphix Virtualization Platform SDK -Long-description: file: README.md -Long-description-content-type: text/markdown -Keywords: virtualization plugin -Classifiers: +metadata_version: 1.2 +author: Delphix +author_email: virtualization-plugins@delphix.com +home_page: https://developer.delphix.com +summary: Delphix Virtualization Platform SDK +long_description: file: README.md +long_description_content_type: text/markdown +keywords: virtualization plugin +classifiers: Development Status :: 5 - Production/Stable Programming Language :: Python - Programming Language :: Python :: 2.7 + Programming Language :: Python :: 3.8 License :: OSI Approved :: Apache Software License Operating System :: OS Independent [options] -Requires-Python: 2.7 +requires_python: 3.8 diff --git a/dvp/setup.py b/dvp/setup.py index b530d493..e7f78c7b 100644 --- a/dvp/setup.py +++ b/dvp/setup.py @@ -18,4 +18,5 @@ install_requires=install_requires, package_dir={'': PYTHON_SRC}, packages=setuptools.find_packages(PYTHON_SRC), + python_requires='>=3.8, <3.9', ) diff --git a/libs/.python-version b/libs/.python-version deleted file mode 100644 index 43c4dbe6..00000000 --- a/libs/.python-version +++ /dev/null @@ -1 +0,0 @@ -2.7.17 diff --git a/libs/setup.cfg b/libs/setup.cfg index 3e46329a..09cd385a 100644 --- a/libs/setup.cfg +++ b/libs/setup.cfg @@ -1,21 +1,22 @@ # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # [metadata] -Metadata-Version: 1.2 -Author: Delphix -Author-email: virtualization-plugins@delphix.com -Home-page: https://developer.delphix.com -Summary: Delphix Virtualization Platform Libraries -Long-description: file: README.md -Long-description-content-type: text/markdown -Classifiers: +metadata_version: 1.2 +author: Delphix +author_email: virtualization-plugins@delphix.com +home_page: https://developer.delphix.com +summary: Delphix Virtualization Platform Libraries +long_description: file: README.md +long_description_content_type: text/markdown +classifiers: Development Status :: 5 - Production/Stable Programming Language :: Python Programming Language :: Python :: 2.7 + Programming Language :: Python :: 3.8 License :: OSI Approved :: Apache Software License Operating System :: OS Independent [options] -Requires-Python: 2.7 +requires_python: >=2.7, <=3.8, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*, !=3.7.* diff --git a/libs/setup.py b/libs/setup.py index 4090d6be..c05ea974 100644 --- a/libs/setup.py +++ b/libs/setup.py @@ -16,4 +16,5 @@ install_requires=install_requires, package_dir={'': PYTHON_SRC}, packages=setuptools.find_packages(PYTHON_SRC), + python_requires='>=2.7, <3.9, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*, !=3.7.*', ) diff --git a/libs/src/main/python/dlpx/virtualization/libs/__init__.py b/libs/src/main/python/dlpx/virtualization/libs/__init__.py index 6af6f0e5..040a606b 100644 --- a/libs/src/main/python/dlpx/virtualization/libs/__init__.py +++ b/libs/src/main/python/dlpx/virtualization/libs/__init__.py @@ -1,8 +1,8 @@ # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # __path__ = __import__('pkgutil').extend_path(__path__, __name__) -from dlpx.virtualization.libs.libs import * -from dlpx.virtualization.libs._logging import * +from dlpx.virtualization.libs.libs import * # noqa +from dlpx.virtualization.libs._logging import * # noqa diff --git a/libs/src/main/python/dlpx/virtualization/libs/_logging.py b/libs/src/main/python/dlpx/virtualization/libs/_logging.py index ec377932..0e97eb7e 100644 --- a/libs/src/main/python/dlpx/virtualization/libs/_logging.py +++ b/libs/src/main/python/dlpx/virtualization/libs/_logging.py @@ -1,10 +1,11 @@ # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # from logging import Handler from dlpx.virtualization.libs import libs +from dlpx.virtualization.common.util import to_str __all__ = [ "PlatformHandler" @@ -17,4 +18,5 @@ class PlatformHandler(Handler): """ def emit(self, record): msg = self.format(record) + msg = to_str(msg) libs._log_request(msg, record.levelno) diff --git a/libs/src/main/python/dlpx/virtualization/libs/exceptions.py b/libs/src/main/python/dlpx/virtualization/libs/exceptions.py index fc4a7c11..f2d4c3d9 100644 --- a/libs/src/main/python/dlpx/virtualization/libs/exceptions.py +++ b/libs/src/main/python/dlpx/virtualization/libs/exceptions.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # import sys @@ -66,22 +66,16 @@ class IncorrectArgumentTypeError(PluginRuntimeError): message (str): A user-readable message describing the exception. """ - def __init__( - self, - parameter_name, - actual_type, - expected_type, - required=True): - actual, expected = self.get_actual_and_expected_type( - actual_type, expected_type) + def __init__(self, parameter_name, actual_type, expected_type, required=True): + actual, expected = self.get_actual_and_expected_type(actual_type, expected_type) - # Get the name of the function that is throwning this error. + # Get the name of the function that is throwing this error. func_name = sys._getframe(1).f_code.co_name message = ("The function {}'s argument '{}' was {} but should" " be of {}{}.".format( - func_name, - parameter_name, - actual, - expected, - (' if defined', '')[required])) + func_name, + parameter_name, + actual, + expected, + (' if defined', '')[required])) super(IncorrectArgumentTypeError, self).__init__(message) diff --git a/libs/src/main/python/dlpx/virtualization/libs/libs.py b/libs/src/main/python/dlpx/virtualization/libs/libs.py index 8284c50a..50b57d5c 100644 --- a/libs/src/main/python/dlpx/virtualization/libs/libs.py +++ b/libs/src/main/python/dlpx/virtualization/libs/libs.py @@ -1,4 +1,4 @@ -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # # -*- coding: utf-8 -*- @@ -33,6 +33,7 @@ from dlpx.virtualization.common._common_classes import (RemoteConnection, PasswordCredentials, KeyPairCredentials) +from dlpx.virtualization.common.util import response_to_str, to_str from google.protobuf import json_format from google.protobuf.struct_pb2 import Struct @@ -75,23 +76,24 @@ def _handle_response(response): def _check_exit_code(response, check): - """ - This functions checks the exitcode received in response and throws PluginScriptError - if check is True. - - Args: - response (RunPowerShellResponse or RunBashResponse or RunExpectResponse): Response received by run_bash or - run_powershell or run_expect - check (bool): if True and non-zero exitcode is received in response, raise PluginScriptError - """ - if (check and response.HasField('return_value') - and response.return_value.exit_code != 0): - raise PluginScriptError('The script failed with exit code {}.' - ' stdout : {} and ' - ' stderr : {}'.format( - response.return_value.exit_code, - response.return_value.stdout, - response.return_value.stderr)) + """ + This functions checks the exitcode received in response and throws + PluginScriptError if check is True. + + Args: + response (RunPowerShellResponse or RunBashResponse or RunExpectResponse): Response + received by run_bash or run_powershell or run_expect + check (bool): if True and non-zero exitcode is received in response, raise + PluginScriptError + """ + if (check and response.HasField('return_value') + and response.return_value.exit_code != 0): + raise PluginScriptError('The script failed with exit code {}.' + ' stdout : {} and ' + ' stderr : {}'.format( + response.return_value.exit_code, + response.return_value.stdout, + response.return_value.stderr)) def run_bash(remote_connection, command, variables=None, use_login_shell=False, @@ -129,6 +131,8 @@ def run_bash(remote_connection, command, variables=None, use_login_shell=False, if variables is None: variables = {} + command = to_str(command) + variables = to_str(variables) # Validate all the arguments passed in are the right types based on docs. if not isinstance(remote_connection, RemoteConnection): @@ -166,6 +170,7 @@ def run_bash(remote_connection, command, variables=None, use_login_shell=False, run_bash_request.variables[variable] = value run_bash_response = internal_libs.run_bash(run_bash_request) + response_to_str(run_bash_response) _check_exit_code(run_bash_response, check) return _handle_response(run_bash_response) @@ -193,6 +198,14 @@ def run_sync(remote_connection, source_directory, rsync_user=None, from dlpx.virtualization._engine import libs as internal_libs + source_directory = to_str(source_directory) + if rsync_user is not None: + rsync_user = to_str(rsync_user) + if exclude_paths is not None: + exclude_paths = to_str(exclude_paths) + if sym_links_to_follow is not None: + sym_links_to_follow = to_str(sym_links_to_follow) + # Validate all the arguments passed in are the right types based on docs. if not isinstance(remote_connection, RemoteConnection): raise IncorrectArgumentTypeError( @@ -246,6 +259,7 @@ def run_sync(remote_connection, source_directory, rsync_user=None, run_sync_request.sym_links_to_follow.extend(sym_links_to_follow) response = internal_libs.run_sync(run_sync_request) + response_to_str(response) _handle_response(response) @@ -283,13 +297,16 @@ def run_powershell(remote_connection, command, variables=None, check=False): if variables is None: variables = {} + command = to_str(command) + variables = to_str(variables) + # Validate all the arguments passed in are the right types based on docs. if not isinstance(remote_connection, RemoteConnection): raise IncorrectArgumentTypeError( 'remote_connection', type(remote_connection), RemoteConnection) - if not isinstance(command, six.string_types[0]): + if not isinstance(command, six.string_types): raise IncorrectArgumentTypeError('command', type(command), six.string_types[0]) if variables and not isinstance(variables, dict): raise IncorrectArgumentTypeError( @@ -315,6 +332,7 @@ def run_powershell(remote_connection, command, variables=None, check=False): run_powershell_request.variables[variable] = value run_powershell_response = internal_libs.run_powershell( run_powershell_request) + response_to_str(run_powershell_response) _check_exit_code(run_powershell_response, check) return _handle_response(run_powershell_response) @@ -344,10 +362,12 @@ def run_expect(remote_connection, command, variables=None, check=False): # scope to allow unit testing of this module. # from dlpx.virtualization._engine import libs as internal_libs - if variables is None: variables = {} + command = to_str(command) + variables = to_str(variables) + # Validate all the arguments passed in are the right types based on docs. if not isinstance(remote_connection, RemoteConnection): raise IncorrectArgumentTypeError( @@ -380,6 +400,7 @@ def run_expect(remote_connection, command, variables=None, check=False): run_expect_request.variables[variable] = value run_expect_response = internal_libs.run_expect(run_expect_request) + response_to_str(run_expect_response) _check_exit_code(run_expect_response, check) return _handle_response(run_expect_response) @@ -401,6 +422,7 @@ def _log_request(message, log_level): """ from dlpx.virtualization._engine import libs as internal_libs + message = to_str(message) log_request = libs_pb2.LogRequest() log_request.message = message @@ -416,23 +438,27 @@ def _log_request(message, log_level): log_request.level = libs_pb2.LogRequest.ERROR response = internal_libs.log(log_request) + response_to_str(response) _handle_response(response) def retrieve_credentials(credentials_supplier): - """This is an internal wrapper around the Virtualization library's credentials retrieval API. - Given a supplier provided by Virtualization, retrieves the credentials from that supplier. + """ + This is an internal wrapper around the Virtualization library's credentials + retrieval API. Given a supplier provided by Virtualization, retrieves the + credentials from that supplier. Args: credentials_supplier (dict): Properties that make up a supplier of credentials. Return: - Subclass of Credentials retrieved from supplier. Either a PasswordCredentials or a KeyPairCredentials - from dlpx.virtualization.common._common_classes. + Subclass of Credentials retrieved from supplier. Either a PasswordCredentials + or a KeyPairCredentials from dlpx.virtualization.common._common_classes. """ from dlpx.virtualization._engine import libs as internal_libs if not isinstance(credentials_supplier, dict): - raise IncorrectArgumentTypeError('credentials_supplier', type(credentials_supplier), dict) + raise IncorrectArgumentTypeError( + 'credentials_supplier', type(credentials_supplier), dict) credentials_request = libs_pb2.CredentialsRequest() credentials_struct = Struct() @@ -440,10 +466,11 @@ def retrieve_credentials(credentials_supplier): credentials_request.credentials_supplier.CopyFrom(credentials_struct) response = internal_libs.retrieve_credentials(credentials_request) - + response_to_str(response) credentials_result = _handle_response(response) if credentials_result.password != "": - return PasswordCredentials(credentials_result.username, credentials_result.password) + return PasswordCredentials( + credentials_result.username, credentials_result.password) return KeyPairCredentials( credentials_result.username, credentials_result.key_pair.private_key, @@ -451,22 +478,26 @@ def retrieve_credentials(credentials_supplier): def upgrade_password(password, username=None): - """This is an internal wrapper around Virtualization's credentials-supplier conversion API. - It is intended for use during plugin upgrade when a plugin needs to transform a password - value into a more generic credentials supplier object. + """ + This is an internal wrapper around Virtualization's credentials-supplier conversion + API. It is intended for use during plugin upgrade when a plugin needs to transform + a password value into a more generic credentials supplier object. Args: password (str): Plain password string. - username (str, defaults to None): User name contained in the password credential supplier to return. + username (str, defaults to None): User name contained in the password + credential supplier to return. Return: Credentials supplier (dict) that supplies the given password and username. """ from dlpx.virtualization._engine import libs as internal_libs if not isinstance(password, six.string_types): - raise IncorrectArgumentTypeError('password', type(password), six.string_types[0]) + raise IncorrectArgumentTypeError( + 'password', type(password), six.string_types[0]) if username and not isinstance(username, six.string_types): - raise IncorrectArgumentTypeError('username', type(username), six.string_types[0], required=False) + raise IncorrectArgumentTypeError( + 'username', type(username), six.string_types[0], required=False) upgrade_password_request = libs_pb2.UpgradePasswordRequest() upgrade_password_request.password = password @@ -474,6 +505,6 @@ def upgrade_password(password, username=None): upgrade_password_request.username = username response = internal_libs.upgrade_password(upgrade_password_request) - + response_to_str(response) upgrade_password_result = _handle_response(response) return json_format.MessageToDict(upgrade_password_result.credentials_supplier) diff --git a/libs/src/test/python/dlpx/virtualization/_engine/libs.py b/libs/src/test/python/dlpx/virtualization/_engine/libs.py index 99356f49..bb0d58dc 100644 --- a/libs/src/test/python/dlpx/virtualization/_engine/libs.py +++ b/libs/src/test/python/dlpx/virtualization/_engine/libs.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # """ @@ -25,4 +25,4 @@ def run_expect(run_expect_request): def log(log_debug_request): - pass \ No newline at end of file + pass diff --git a/libs/src/test/python/dlpx/virtualization/conftest.py b/libs/src/test/python/dlpx/virtualization/conftest.py index 4c805eb0..52e16d2b 100644 --- a/libs/src/test/python/dlpx/virtualization/conftest.py +++ b/libs/src/test/python/dlpx/virtualization/conftest.py @@ -1,9 +1,10 @@ # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # import pytest -from dlpx.virtualization.common._common_classes import RemoteUser, RemoteHost, RemoteEnvironment, RemoteConnection +from dlpx.virtualization.common._common_classes import ( + RemoteUser, RemoteHost, RemoteEnvironment, RemoteConnection) @pytest.fixture diff --git a/libs/src/test/python/dlpx/virtualization/test_libs.py b/libs/src/test/python/dlpx/virtualization/test_libs.py index 4e94d375..abf8eba0 100644 --- a/libs/src/test/python/dlpx/virtualization/test_libs.py +++ b/libs/src/test/python/dlpx/virtualization/test_libs.py @@ -1,16 +1,16 @@ # -# Copyright (c) 2019, 2020 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # import mock import pytest +import six from dlpx.virtualization.api import libs_pb2 from dlpx.virtualization import libs from dlpx.virtualization.libs.exceptions import ( IncorrectArgumentTypeError, LibraryError, PluginScriptError) from google.protobuf import json_format -from dlpx.virtualization.common._common_classes import (PasswordCredentials) class TestLibsRunBash: @@ -53,10 +53,10 @@ def mock_run_bash(actual_run_bash_request): @staticmethod def test_run_bash_check_true_success_exitcode(remote_connection): - expected_run_bash_response = libs_pb2.RunBashResponse() - expected_run_bash_response.return_value.exit_code = 0 - expected_run_bash_response.return_value.stdout = "stdout" - expected_run_bash_response.return_value.stderr = "stderr" + expected_response = libs_pb2.RunBashResponse() + expected_response.return_value.exit_code = 0 + expected_response.return_value.stdout = "stdout" + expected_response.return_value.stderr = "stderr" expected_command = "command" expected_variables = None @@ -73,19 +73,17 @@ def mock_run_bash(actual_run_bash_request): actual_run_bash_request.remote_connection.environment.reference == remote_connection.environment.reference ) - return expected_run_bash_response + return expected_response with mock.patch("dlpx.virtualization._engine.libs.run_bash", side_effect=mock_run_bash, create=True): - actual_run_bash_result = libs.run_bash(remote_connection, - expected_command, - expected_variables, - expected_use_login_shell, - check=True) + actual_result = libs.run_bash( + remote_connection, expected_command, expected_variables, + expected_use_login_shell, check=True) - assert actual_run_bash_result.exit_code == expected_run_bash_response.return_value.exit_code - assert actual_run_bash_result.stdout == expected_run_bash_response.return_value.stdout - assert actual_run_bash_result.stderr == expected_run_bash_response.return_value.stderr + assert actual_result.exit_code == expected_response.return_value.exit_code + assert actual_result.stdout == expected_response.return_value.stdout + assert actual_result.stderr == expected_response.return_value.stderr @staticmethod def test_run_bash_with_check_true_failed_exitcode(remote_connection): @@ -143,11 +141,16 @@ def test_run_bash_bad_remote_connection(): with pytest.raises(IncorrectArgumentTypeError) as err_info: libs.run_bash(connection, command, variables, use_login_shell) - - assert err_info.value.message == ( - "The function run_bash's argument 'remote_connection' was" - " type 'str' but should be of" - " class 'dlpx.virtualization.common._common_classes.RemoteConnection'.") + if six.PY2: + assert err_info.value.message == ( + "The function run_bash's argument 'remote_connection' was" + " type 'str' but should be of" + " class 'dlpx.virtualization.common._common_classes.RemoteConnection'.") + else: + assert err_info.value.message == ( + "The function run_bash's argument 'remote_connection' was" + " class 'str' but should be of" + " class 'dlpx.virtualization.common._common_classes.RemoteConnection'.") @staticmethod def test_run_bash_bad_command(remote_connection): @@ -158,10 +161,14 @@ def test_run_bash_bad_command(remote_connection): with pytest.raises(IncorrectArgumentTypeError) as err_info: libs.run_bash(remote_connection, command, variables, use_login_shell) - - assert err_info.value.message == ( - "The function run_bash's argument 'command' was" - " type 'int' but should be of type 'basestring'.") + if six.PY2: + assert err_info.value.message == ( + "The function run_bash's argument 'command' was" + " type 'int' but should be of type 'basestring'.") + else: + assert err_info.value.message == ( + "The function run_bash's argument 'command' was" + " class 'int' but should be of class 'str'.") @staticmethod def test_run_bash_variables_not_dict(remote_connection): @@ -172,11 +179,16 @@ def test_run_bash_variables_not_dict(remote_connection): with pytest.raises(IncorrectArgumentTypeError) as err_info: libs.run_bash(remote_connection, command, variables, use_login_shell) - - assert err_info.value.message == ( - "The function run_bash's argument 'variables' was" - " type 'str' but should be of" - " type 'dict of basestring:basestring' if defined.") + if six.PY2: + assert err_info.value.message == ( + "The function run_bash's argument 'variables' was" + " type 'unicode' but should be of" + " type 'dict of basestring:basestring' if defined.") + else: + assert err_info.value.message == ( + "The function run_bash's argument 'variables' was" + " class 'str' but should be of" + " type 'dict of str:str' if defined.") @staticmethod def test_run_bash_bad_variables(remote_connection): @@ -190,13 +202,20 @@ def test_run_bash_bad_variables(remote_connection): with pytest.raises(IncorrectArgumentTypeError) as err_info: libs.run_bash(remote_connection, command, variables, use_login_shell) - - message = ("The function run_bash's argument 'variables' was" - " a dict of {{type 'str':type '{}', type 'str':type '{}'}}" - " but should be of" - " type 'dict of basestring:basestring' if defined.") - assert (err_info.value.message == message.format('int', 'str') or - err_info.value.message == message.format('str', 'int')) + if six.PY2: + message = ("The function run_bash's argument 'variables' was" + " a dict of {{type 'str':type '{}', type 'str':type '{}'}}" + " but should be of" + " type 'dict of basestring:basestring' if defined.") + assert (err_info.value.message == message.format('int', 'unicode') or + err_info.value.message == message.format('unicode', 'int')) + else: + message = ("The function run_bash's argument 'variables' was" + " a dict of {{class 'str':class '{}', class 'str':class '{}'}}" + " but should be of" + " type 'dict of str:str' if defined.") + assert (err_info.value.message == message.format('int', 'str') or + err_info.value.message == message.format('str', 'int')) @staticmethod def test_run_bash_bad_use_login_shell(remote_connection): @@ -207,10 +226,14 @@ def test_run_bash_bad_use_login_shell(remote_connection): with pytest.raises(IncorrectArgumentTypeError) as err_info: libs.run_bash(remote_connection, command, variables, use_login_shell) - - assert err_info.value.message == ( - "The function run_bash's argument 'use_login_shell' was" - " type 'str' but should be of type 'bool' if defined.") + if six.PY2: + assert err_info.value.message == ( + "The function run_bash's argument 'use_login_shell' was" + " type 'str' but should be of type 'bool' if defined.") + else: + assert err_info.value.message == ( + "The function run_bash's argument 'use_login_shell' was" + " class 'str' but should be of class 'bool' if defined.") class TestLibsRunSync: @@ -296,11 +319,16 @@ def test_run_sync_bad_remote_connection(): rsync_user, exclude_paths, sym_links_to_follow) - - assert err_info.value.message == ( - "The function run_sync's argument 'remote_connection' was" - " type 'str' but should be of" - " class 'dlpx.virtualization.common._common_classes.RemoteConnection'.") + if six.PY2: + assert err_info.value.message == ( + "The function run_sync's argument 'remote_connection' was" + " type 'str' but should be of" + " class 'dlpx.virtualization.common._common_classes.RemoteConnection'.") + else: + assert err_info.value.message == ( + "The function run_sync's argument 'remote_connection' was" + " class 'str' but should be of" + " class 'dlpx.virtualization.common._common_classes.RemoteConnection'.") @staticmethod def test_run_sync_bad_source_directory(remote_connection): @@ -317,10 +345,14 @@ def test_run_sync_bad_source_directory(remote_connection): rsync_user, exclude_paths, sym_links_to_follow) - - assert err_info.value.message == ( - "The function run_sync's argument 'source_directory' was" - " type 'int' but should be of type 'basestring'.") + if six.PY2: + assert err_info.value.message == ( + "The function run_sync's argument 'source_directory' was" + " type 'int' but should be of type 'basestring'.") + else: + assert err_info.value.message == ( + "The function run_sync's argument 'source_directory' was" + " class 'int' but should be of class 'str'.") @staticmethod def test_run_sync_bad_rsync_user(remote_connection): @@ -337,10 +369,14 @@ def test_run_sync_bad_rsync_user(remote_connection): rsync_user, exclude_paths, sym_links_to_follow) - - assert err_info.value.message == ( - "The function run_sync's argument 'rsync_user' was" - " type 'int' but should be of type 'basestring' if defined.") + if six.PY2: + assert err_info.value.message == ( + "The function run_sync's argument 'rsync_user' was" + " type 'int' but should be of type 'basestring' if defined.") + else: + assert err_info.value.message == ( + "The function run_sync's argument 'rsync_user' was" + " class 'int' but should be of class 'str' if defined.") @staticmethod def test_run_sync_exclude_paths_not_list(remote_connection): @@ -357,11 +393,16 @@ def test_run_sync_exclude_paths_not_list(remote_connection): rsync_user, exclude_paths, sym_links_to_follow) - - assert err_info.value.message == ( - "The function run_sync's argument 'exclude_paths' was" - " type 'str' but should be of" - " type 'list of basestring' if defined.") + if six.PY2: + assert err_info.value.message == ( + "The function run_sync's argument 'exclude_paths' was" + " type 'unicode' but should be of" + " type 'list of basestring' if defined.") + else: + assert err_info.value.message == ( + "The function run_sync's argument 'exclude_paths' was" + " class 'str' but should be of" + " type 'list of str' if defined.") @staticmethod def test_run_sync_bad_exclude_paths(remote_connection): @@ -378,11 +419,16 @@ def test_run_sync_bad_exclude_paths(remote_connection): rsync_user, exclude_paths, sym_links_to_follow) - - assert err_info.value.message == ( - "The function run_sync's argument 'exclude_paths' was a list of" - " [type 'str', type 'int'] but should be of" - " type 'list of basestring' if defined.") + if six.PY2: + assert err_info.value.message == ( + "The function run_sync's argument 'exclude_paths' was a list of" + " [type 'unicode', type 'int'] but should be of" + " type 'list of basestring' if defined.") + else: + assert err_info.value.message == ( + "The function run_sync's argument 'exclude_paths' was a list of" + " [class 'str', class 'int'] but should be of" + " type 'list of str' if defined.") @staticmethod def test_run_sync_sym_links_to_follow_not_list(remote_connection): @@ -399,11 +445,16 @@ def test_run_sync_sym_links_to_follow_not_list(remote_connection): rsync_user, exclude_paths, sym_links_to_follow) - - assert err_info.value.message == ( - "The function run_sync's argument 'sym_links_to_follow' was" - " type 'str' but should be of" - " type 'list of basestring' if defined.") + if six.PY2: + assert err_info.value.message == ( + "The function run_sync's argument 'sym_links_to_follow' was" + " type 'unicode' but should be of" + " type 'list of basestring' if defined.") + else: + assert err_info.value.message == ( + "The function run_sync's argument 'sym_links_to_follow' was" + " class 'str' but should be of" + " type 'list of str' if defined.") @staticmethod def test_run_sync_bad_sym_links_to_follow(remote_connection): @@ -420,11 +471,18 @@ def test_run_sync_bad_sym_links_to_follow(remote_connection): rsync_user, exclude_paths, sym_links_to_follow) - - assert err_info.value.message == ( - "The function run_sync's argument 'sym_links_to_follow' was" - " a list of [type 'str', type 'int'] but should be of" - " type 'list of basestring' if defined.") + if six.PY2: + assert err_info.value.message == ( + "The function run_sync's argument 'sym_links_to_follow' was" + " a list of [type 'unicode', type 'int'] but should be of" + " type 'list of " + "basestring' if defined.") + else: + assert err_info.value.message == ( + "The function run_sync's argument 'sym_links_to_follow' was" + " a list of [class 'str', class 'int'] but should be of" + " type 'list of " + "str' if defined.") class TestLibsRunPowershell: @@ -463,35 +521,35 @@ def mock_run_powershell(actual_run_powershell_request): @staticmethod def test_run_powershell_check_true_exitcode_success(remote_connection): - expected_run_powershell_response = libs_pb2.RunPowerShellResponse() - expected_run_powershell_response.return_value.exit_code = 0 - expected_run_powershell_response.return_value.stdout = "stdout" - expected_run_powershell_response.return_value.stderr = "stderr" + expected_response = libs_pb2.RunPowerShellResponse() + expected_response.return_value.exit_code = 0 + expected_response.return_value.stdout = "stdout" + expected_response.return_value.stderr = "stderr" expected_command = "command" expected_variables = None - def mock_run_powershell(actual_run_powershell_request): - assert actual_run_powershell_request.command == expected_command + def mock_run_powershell(actual_request): + assert actual_request.command == expected_command assert ( - actual_run_powershell_request.remote_connection.environment.name + actual_request.remote_connection.environment.name == remote_connection.environment.name ) assert ( - actual_run_powershell_request.remote_connection.environment.reference + actual_request.remote_connection.environment.reference == remote_connection.environment.reference ) - return expected_run_powershell_response + return expected_response with mock.patch("dlpx.virtualization._engine.libs.run_powershell", side_effect=mock_run_powershell, create=True): - actual_run_powershell_result = libs.run_powershell( + actual_result = libs.run_powershell( remote_connection, expected_command, expected_variables, check=True) - assert actual_run_powershell_result.exit_code == expected_run_powershell_response.return_value.exit_code - assert actual_run_powershell_result.stdout == expected_run_powershell_response.return_value.stdout - assert actual_run_powershell_result.stderr == expected_run_powershell_response.return_value.stderr + assert actual_result.exit_code == expected_response.return_value.exit_code + assert actual_result.stdout == expected_response.return_value.stdout + assert actual_result.stderr == expected_response.return_value.stderr @staticmethod def test_run_powershell_check_true_exitcode_failed(remote_connection): @@ -508,8 +566,8 @@ def test_run_powershell_check_true_exitcode_failed(remote_connection): with mock.patch("dlpx.virtualization._engine.libs.run_powershell", return_value=response, create=True): with pytest.raises(PluginScriptError) as info: - response = libs.run_powershell(remote_connection, "test_command", - check=True) + libs.run_powershell(remote_connection, "test_command", + check=True) assert info.value.message == expected_message @staticmethod @@ -549,11 +607,16 @@ def test_run_powershell_bad_remote_connection(): with pytest.raises(IncorrectArgumentTypeError) as err_info: libs.run_powershell(connection, command, variables) - - assert err_info.value.message == ( - "The function run_powershell's argument 'remote_connection' was" - " type 'str' but should be of" - " class 'dlpx.virtualization.common._common_classes.RemoteConnection'.") + if six.PY2: + assert err_info.value.message == ( + "The function run_powershell's argument 'remote_connection' was" + " type 'str' but should be of" + " class 'dlpx.virtualization.common._common_classes.RemoteConnection'.") + else: + assert err_info.value.message == ( + "The function run_powershell's argument 'remote_connection' was" + " class 'str' but should be of" + " class 'dlpx.virtualization.common._common_classes.RemoteConnection'.") @staticmethod def test_run_powershell_bad_command(remote_connection): @@ -563,10 +626,14 @@ def test_run_powershell_bad_command(remote_connection): with pytest.raises(IncorrectArgumentTypeError) as err_info: libs.run_powershell(remote_connection, command, variables) - - assert err_info.value.message == ( - "The function run_powershell's argument 'command' was" - " type 'int' but should be of type 'basestring'.") + if six.PY2: + assert err_info.value.message == ( + "The function run_powershell's argument 'command' was" + " type 'int' but should be of type 'basestring'.") + else: + assert err_info.value.message == ( + "The function run_powershell's argument 'command' was" + " class 'int' but should be of class 'str'.") @staticmethod def test_run_powershell_variables_not_dict(remote_connection): @@ -576,11 +643,16 @@ def test_run_powershell_variables_not_dict(remote_connection): with pytest.raises(IncorrectArgumentTypeError) as err_info: libs.run_powershell(remote_connection, command, variables) - - assert err_info.value.message == ( - "The function run_powershell's argument 'variables' was" - " type 'str' but should be of" - " type 'dict of basestring:basestring' if defined.") + if six.PY2: + assert err_info.value.message == ( + "The function run_powershell's argument 'variables' was" + " type 'unicode' but should be of" + " type 'dict of basestring:basestring' if defined.") + else: + assert err_info.value.message == ( + "The function run_powershell's argument 'variables' was" + " class 'str' but should be of" + " type 'dict of str:str' if defined.") @staticmethod def test_run_powershell_bad_variables(remote_connection): @@ -593,13 +665,20 @@ def test_run_powershell_bad_variables(remote_connection): with pytest.raises(IncorrectArgumentTypeError) as err_info: libs.run_powershell(remote_connection, command, variables) - - message = ("The function run_powershell's argument 'variables' was" - " a dict of {{type 'str':type '{}', type 'str':type '{}'}}" - " but should be of" - " type 'dict of basestring:basestring' if defined.") - assert (err_info.value.message == message.format('int', 'str') or - err_info.value.message == message.format('str', 'int')) + if six.PY2: + message = ("The function run_powershell's argument 'variables' was" + " a dict of {{type 'str':type '{}', type 'str':type '{}'}}" + " but should be of" + " type 'dict of basestring:basestring' if defined.") + assert (err_info.value.message == message.format('int', 'unicode') or + err_info.value.message == message.format('unicode', 'int')) + else: + message = ("The function run_powershell's argument 'variables' was" + " a dict of {{class 'str':class '{}', class 'str':class '{}'}}" + " but should be of" + " type 'dict of str:str' if defined.") + assert (err_info.value.message == message.format('int', 'str') or + err_info.value.message == message.format('str', 'int')) class TestLibsRunExpect: @@ -638,10 +717,10 @@ def mock_run_expect(actual_run_expect_request): @staticmethod def test_run_expect_check_true_exitcode_success(remote_connection): - expected_run_expect_response = libs_pb2.RunPowerShellResponse() - expected_run_expect_response.return_value.exit_code = 0 - expected_run_expect_response.return_value.stdout = "stdout" - expected_run_expect_response.return_value.stderr = "stderr" + expected_response = libs_pb2.RunPowerShellResponse() + expected_response.return_value.exit_code = 0 + expected_response.return_value.stdout = "stdout" + expected_response.return_value.stderr = "stderr" expected_command = "command" expected_variables = None @@ -656,17 +735,17 @@ def mock_run_expect(actual_run_expect_request): actual_run_expect_request.remote_connection.environment.reference == remote_connection.environment.reference ) - return expected_run_expect_response + return expected_response with mock.patch("dlpx.virtualization._engine.libs.run_expect", side_effect=mock_run_expect, create=True): - actual_run_expect_result = libs.run_expect( + actual_result = libs.run_expect( remote_connection, expected_command, expected_variables, check=True) - assert actual_run_expect_result.exit_code == expected_run_expect_response.return_value.exit_code - assert actual_run_expect_result.stdout == expected_run_expect_response.return_value.stdout - assert actual_run_expect_result.stderr == expected_run_expect_response.return_value.stderr + assert actual_result.exit_code == expected_response.return_value.exit_code + assert actual_result.stdout == expected_response.return_value.stdout + assert actual_result.stderr == expected_response.return_value.stderr @staticmethod def test_run_expect_check_true_exitcode_failed(remote_connection): @@ -683,8 +762,7 @@ def test_run_expect_check_true_exitcode_failed(remote_connection): with mock.patch("dlpx.virtualization._engine.libs.run_expect", return_value=response, create=True): with pytest.raises(PluginScriptError) as info: - response = libs.run_expect(remote_connection, "test_command", - check=True) + libs.run_expect(remote_connection, "test_command", check=True) assert info.value.message == expected_message @staticmethod @@ -724,11 +802,16 @@ def test_run_expect_bad_remote_connection(): with pytest.raises(IncorrectArgumentTypeError) as err_info: libs.run_expect(connection, command, variables) - - assert err_info.value.message == ( - "The function run_expect's argument 'remote_connection' was" - " type 'str' but should be of" - " class 'dlpx.virtualization.common._common_classes.RemoteConnection'.") + if six.PY2: + assert err_info.value.message == ( + "The function run_expect's argument 'remote_connection' was" + " type 'str' but should be of" + " class 'dlpx.virtualization.common._common_classes.RemoteConnection'.") + else: + assert err_info.value.message == ( + "The function run_expect's argument 'remote_connection' was" + " class 'str' but should be of" + " class 'dlpx.virtualization.common._common_classes.RemoteConnection'.") @staticmethod def test_run_expect_bad_command(remote_connection): @@ -739,9 +822,14 @@ def test_run_expect_bad_command(remote_connection): with pytest.raises(IncorrectArgumentTypeError) as err_info: libs.run_expect(remote_connection, command, variables) - assert err_info.value.message == ( - "The function run_expect's argument 'command' was" - " type 'int' but should be of type 'basestring'.") + if six.PY2: + assert err_info.value.message == ( + "The function run_expect's argument 'command' was" + " type 'int' but should be of type 'basestring'.") + else: + assert err_info.value.message == ( + "The function run_expect's argument 'command' was" + " class 'int' but should be of class 'str'.") @staticmethod def test_run_expect_variables_not_dict(remote_connection): @@ -751,11 +839,16 @@ def test_run_expect_variables_not_dict(remote_connection): with pytest.raises(IncorrectArgumentTypeError) as err_info: libs.run_expect(remote_connection, command, variables) - - assert err_info.value.message == ( - "The function run_expect's argument 'variables' was" - " type 'str' but should be of" - " type 'dict of basestring:basestring' if defined.") + if six.PY2: + assert err_info.value.message == ( + "The function run_expect's argument 'variables' was" + " type 'unicode' but should be of" + " type 'dict of basestring:basestring' if defined.") + else: + assert err_info.value.message == ( + "The function run_expect's argument 'variables' was" + " class 'str' but should be of" + " type 'dict of str:str' if defined.") @staticmethod def test_run_expect_bad_variables(remote_connection): @@ -769,60 +862,72 @@ def test_run_expect_bad_variables(remote_connection): with pytest.raises(IncorrectArgumentTypeError) as err_info: libs.run_expect(remote_connection, command, variables) - message = ("The function run_expect's argument 'variables' was" - " a dict of {{type 'str':type '{}', type 'str':type '{}'}}" - " but should be of" - " type 'dict of basestring:basestring' if defined.") - assert (err_info.value.message == message.format('int', 'str') or - err_info.value.message == message.format('str', 'int')) + if six.PY2: + message = ("The function run_expect's argument 'variables' was" + " a dict of {{type 'str':type '{}', type 'str':type '{}'}}" + " but should be of" + " type 'dict of basestring:basestring' if defined.") + assert (err_info.value.message == message.format('int', 'unicode') or + err_info.value.message == message.format('unicode', 'int')) + else: + message = ("The function run_expect's argument 'variables' was" + " a dict of {{class 'str':class '{}', class 'str':class '{}'}}" + " but should be of" + " type 'dict of str:str' if defined.") + assert (err_info.value.message == message.format('int', 'str') or + err_info.value.message == message.format('str', 'int')) class TestLibsRetrieveCredentials: @staticmethod def test_retrieve_password_credentials(): - expected_retrieve_credentials_response = libs_pb2.CredentialsResponse() - expected_retrieve_credentials_response.return_value.username = 'some user' - expected_retrieve_credentials_response.return_value.password = 'some password' + expected_response = libs_pb2.CredentialsResponse() + expected_response.return_value.username = 'some user' + expected_response.return_value.password = 'some password' - expected_credentials_supplier = {'some supplier property': 'some supplier value'} + expected_credentials_supplier = { + 'some supplier property': 'some supplier value'} def mock_retrieve_credentials(actual_retrieve_credentials_request): - assert json_format.MessageToDict(actual_retrieve_credentials_request.credentials_supplier) == \ - expected_credentials_supplier + assert json_format.MessageToDict( + actual_retrieve_credentials_request.credentials_supplier + ) == expected_credentials_supplier - return expected_retrieve_credentials_response + return expected_response with mock.patch('dlpx.virtualization._engine.libs.retrieve_credentials', side_effect=mock_retrieve_credentials, create=True): - actual_retrieve_credentials_result = libs.retrieve_credentials(expected_credentials_supplier) + actual_result = libs.retrieve_credentials(expected_credentials_supplier) - expected = expected_retrieve_credentials_response.return_value - assert actual_retrieve_credentials_result.username == expected.username - assert actual_retrieve_credentials_result.password == expected.password + expected = expected_response.return_value + assert actual_result.username == expected.username + assert actual_result.password == expected.password @staticmethod def test_retrieve_keypair_credentials(): - expected_retrieve_credentials_response = libs_pb2.CredentialsResponse() - expected_retrieve_credentials_response.return_value.username = 'some user' - expected_retrieve_credentials_response.return_value.key_pair.private_key = 'some private key' - expected_retrieve_credentials_response.return_value.key_pair.public_key = 'some public key' + expected_response = libs_pb2.CredentialsResponse() + expected_response.return_value.username = 'some user' + expected_response.return_value.key_pair.private_key = 'some private key' + expected_response.return_value.key_pair.public_key = 'some public key' - expected_credentials_supplier = {'some supplier property': 'some supplier value'} + expected_credentials_supplier = { + 'some supplier property': 'some supplier value'} def mock_retrieve_credentials(actual_retrieve_credentials_request): - assert json_format.MessageToDict(actual_retrieve_credentials_request.credentials_supplier) == \ - expected_credentials_supplier + assert json_format.MessageToDict( + actual_retrieve_credentials_request.credentials_supplier + ) == expected_credentials_supplier - return expected_retrieve_credentials_response + return expected_response with mock.patch('dlpx.virtualization._engine.libs.retrieve_credentials', side_effect=mock_retrieve_credentials, create=True): - actual_retrieve_credentials_result = libs.retrieve_credentials(expected_credentials_supplier) + actual_result = libs.retrieve_credentials(expected_credentials_supplier) - expected = expected_retrieve_credentials_response.return_value - assert actual_retrieve_credentials_result.username == expected.username - assert actual_retrieve_credentials_result.private_key == expected.key_pair.private_key - assert actual_retrieve_credentials_result.public_key == expected.key_pair.public_key + expected = expected_response.return_value + assert actual_result.username == expected.username + assert actual_result.private_key == expected.key_pair.private_key + assert actual_result.public_key == expected.key_pair.public_key @staticmethod def test_retrieve_credentials_with_actionable_error(): @@ -836,7 +941,8 @@ def test_retrieve_credentials_with_actionable_error(): with mock.patch('dlpx.virtualization._engine.libs.retrieve_credentials', return_value=response, create=True): with pytest.raises(LibraryError) as err_info: - libs.retrieve_credentials({'some supplier property': 'some supplier value'}) + libs.retrieve_credentials( + {'some supplier property': 'some supplier value'}) assert err_info.value._id == expected_id assert err_info.value.message == expected_message @@ -850,7 +956,8 @@ def test_retrieve_credentials_with_nonactionable_error(): with mock.patch('dlpx.virtualization._engine.libs.retrieve_credentials', return_value=response, create=True): with pytest.raises(SystemExit): - libs.retrieve_credentials({'some supplier property': 'some supplier value'}) + libs.retrieve_credentials( + {'some supplier property': 'some supplier value'}) @staticmethod def test_retrieve_credentials_bad_supplier(): @@ -859,10 +966,14 @@ def test_retrieve_credentials_bad_supplier(): with pytest.raises(IncorrectArgumentTypeError) as err_info: libs.retrieve_credentials(credentials_supplier) - - assert err_info.value.message == ( - "The function retrieve_credentials's argument 'credentials_supplier' was" - " type 'int' but should be of type 'dict'.") + if six.PY2: + assert err_info.value.message == ( + "The function retrieve_credentials's argument 'credentials_supplier' " + "was type 'int' but should be of type 'dict'.") + else: + assert err_info.value.message == ( + "The function retrieve_credentials's argument 'credentials_supplier' " + "was class 'int' but should be of class 'dict'.") class TestLibsUpgradePassword: @@ -870,9 +981,11 @@ class TestLibsUpgradePassword: def test_upgrade_password(): expected_password = 'some password' - expected_credentials_supplier = {'type': 'NamedPasswordCredential', 'password': expected_password} + expected_credentials_supplier = { + 'type': 'NamedPasswordCredential', 'password': expected_password} expected_upgrade_password_response = libs_pb2.UpgradePasswordResponse() - expected_upgrade_password_response.return_value.credentials_supplier.update(expected_credentials_supplier) + expected_upgrade_password_response.return_value.credentials_supplier.update( + expected_credentials_supplier) def mock_upgrade_password(actual_upgrade_password_request): assert actual_upgrade_password_request.password == expected_password @@ -890,9 +1003,12 @@ def test_upgrade_password_with_username(): expected_password = 'some password' expected_username = 'some user name' - expected_credentials_supplier = {'type': 'NamedPasswordCredential', 'password': expected_password, 'username': expected_username} + expected_credentials_supplier = { + 'type': 'NamedPasswordCredential', 'password': expected_password, + 'username': expected_username} expected_upgrade_password_response = libs_pb2.UpgradePasswordResponse() - expected_upgrade_password_response.return_value.credentials_supplier.update(expected_credentials_supplier) + expected_upgrade_password_response.return_value.credentials_supplier.update( + expected_credentials_supplier) def mock_upgrade_password(actual_upgrade_password_request): assert actual_upgrade_password_request.password == expected_password @@ -902,7 +1018,8 @@ def mock_upgrade_password(actual_upgrade_password_request): with mock.patch('dlpx.virtualization._engine.libs.upgrade_password', side_effect=mock_upgrade_password, create=True): - actual_upgrade_password_result = libs.upgrade_password(expected_password, username=expected_username) + actual_upgrade_password_result = libs.upgrade_password( + expected_password, username=expected_username) assert actual_upgrade_password_result == expected_credentials_supplier @@ -913,10 +1030,14 @@ def test_upgrade_password_invalid_password(): with pytest.raises(IncorrectArgumentTypeError) as err_info: libs.upgrade_password(expected_password, username=expected_username) - - assert err_info.value.message == ( - "The function upgrade_password's argument 'password' was" - " type 'int' but should be of type 'basestring'.") + if six.PY2: + assert err_info.value.message == ( + "The function upgrade_password's argument 'password' was" + " type 'int' but should be of type 'basestring'.") + else: + assert err_info.value.message == ( + "The function upgrade_password's argument 'password' was" + " class 'int' but should be of class 'str'.") @staticmethod def test_upgrade_password_invalid_username(): @@ -925,7 +1046,11 @@ def test_upgrade_password_invalid_username(): with pytest.raises(IncorrectArgumentTypeError) as err_info: libs.upgrade_password(expected_password, username=expected_username) - - assert err_info.value.message == ( - "The function upgrade_password's argument 'username' was" - " type 'int' but should be of type 'basestring' if defined.") + if six.PY2: + assert err_info.value.message == ( + "The function upgrade_password's argument 'username' was" + " type 'int' but should be of type 'basestring' if defined.") + else: + assert err_info.value.message == ( + "The function upgrade_password's argument 'username' was" + " class 'int' but should be of class 'str' if defined.") diff --git a/libs/src/test/python/dlpx/virtualization/test_logging.py b/libs/src/test/python/dlpx/virtualization/test_logging.py index 022cffa8..45a4ddc2 100644 --- a/libs/src/test/python/dlpx/virtualization/test_logging.py +++ b/libs/src/test/python/dlpx/virtualization/test_logging.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # import logging @@ -30,7 +30,6 @@ def successful_response(): (logging.ERROR, LogRequest.ERROR), (logging.CRITICAL, LogRequest.ERROR) ]) - @mock.patch("dlpx.virtualization._engine.libs", create=True) def test_levels(mock_internal_libs, py_level, expected_level, successful_response): mock_internal_libs.log.return_value = successful_response @@ -67,7 +66,6 @@ def test_format(mock_internal_libs, successful_response): mock_internal_libs.log.assert_called_with(log_request) - @staticmethod @mock.patch("dlpx.virtualization._engine.libs", create=True) def test_log_non_string(mock_internal_libs, successful_response): diff --git a/platform/.python-version b/platform/.python-version deleted file mode 100644 index 43c4dbe6..00000000 --- a/platform/.python-version +++ /dev/null @@ -1 +0,0 @@ -2.7.17 diff --git a/platform/setup.cfg b/platform/setup.cfg index 1e4a607a..5e67bf3e 100644 --- a/platform/setup.cfg +++ b/platform/setup.cfg @@ -1,21 +1,22 @@ # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # [metadata] -Metadata-Version: 1.2 -Author: Delphix -Author-email: virtualization-plugins@delphix.com -Home-page: https://developer.delphix.com -Summary: Delphix Virtualization Platform APIs -Long-description: file: README.md -Long-description-content-type: text/markdown -Classifiers: +metadata_version: 1.2 +author: Delphix +author_email: virtualization-plugins@delphix.com +home_page: https://developer.delphix.com +summary: Delphix Virtualization Platform APIs +long_description: file: README.md +long_description_content_type: text/markdown +classifiers: Development Status :: 5 - Production/Stable Programming Language :: Python Programming Language :: Python :: 2.7 + Programming Language :: Python :: 3.8 License :: OSI Approved :: Apache Software License Operating System :: OS Independent [options] -Requires-Python: 2.7 +requires_python: >=2.7, <=3.8, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*, !=3.7.* diff --git a/platform/setup.py b/platform/setup.py index 600fe22f..b3d422c6 100644 --- a/platform/setup.py +++ b/platform/setup.py @@ -17,4 +17,5 @@ install_requires=install_requires, package_dir={'': PYTHON_SRC}, packages=setuptools.find_packages(PYTHON_SRC), + python_requires='>=2.7, <3.9, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*, !=3.7.*', ) diff --git a/platform/src/main/python/dlpx/virtualization/platform/__init__.py b/platform/src/main/python/dlpx/virtualization/platform/__init__.py index 589f76e4..f22e502c 100644 --- a/platform/src/main/python/dlpx/virtualization/platform/__init__.py +++ b/platform/src/main/python/dlpx/virtualization/platform/__init__.py @@ -1,17 +1,17 @@ # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # __path__ = __import__('pkgutil').extend_path(__path__, __name__) -from dlpx.virtualization.platform.validation_util import * -from dlpx.virtualization.platform.migration_helper import * -from dlpx.virtualization.platform._plugin_classes import * -from dlpx.virtualization.platform._discovery import * -from dlpx.virtualization.platform._linked import * -from dlpx.virtualization.platform._upgrade import * -from dlpx.virtualization.platform._virtual import * -from dlpx.virtualization.platform._plugin import * -from dlpx.virtualization.platform.util import * -from dlpx.virtualization.platform.import_util import * -from dlpx.virtualization.platform.import_validations import * +from dlpx.virtualization.platform.validation_util import * # noqa +from dlpx.virtualization.platform.migration_helper import * # noqa +from dlpx.virtualization.platform._plugin_classes import * # noqa +from dlpx.virtualization.platform._discovery import * # noqa +from dlpx.virtualization.platform._linked import * # noqa +from dlpx.virtualization.platform._upgrade import * # noqa +from dlpx.virtualization.platform._virtual import * # noqa +from dlpx.virtualization.platform._plugin import * # noqa +from dlpx.virtualization.platform.util import * # noqa +from dlpx.virtualization.platform.import_util import * # noqa +from dlpx.virtualization.platform.import_validations import * # noqa diff --git a/platform/src/main/python/dlpx/virtualization/platform/_plugin_classes.py b/platform/src/main/python/dlpx/virtualization/platform/_plugin_classes.py index 4d2da24f..97ff49ca 100644 --- a/platform/src/main/python/dlpx/virtualization/platform/_plugin_classes.py +++ b/platform/src/main/python/dlpx/virtualization/platform/_plugin_classes.py @@ -1,10 +1,10 @@ # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # import re +import six from enum import Enum -import six from dlpx.virtualization.common import (RemoteConnection, RemoteEnvironment, RemoteHost) from dlpx.virtualization.common.exceptions import IncorrectTypeError @@ -160,8 +160,8 @@ def __init__(self, remote_environment, mount_path, shared_path=None): plugin writer from attempting to provide parameter values that they won't have access to.""" def __is_correct_reference_format(reference): - unix_format = re.compile("^UNIX_HOST_ENVIRONMENT-\d+$") - win_format = re.compile("^WINDOWS_HOST_ENVIRONMENT-\d+$") + unix_format = re.compile(r"^UNIX_HOST_ENVIRONMENT-\d+$") + win_format = re.compile(r"^WINDOWS_HOST_ENVIRONMENT-\d+$") return (bool(unix_format.match(reference)) or bool(win_format.match(reference))) @@ -199,7 +199,7 @@ def __make_remote_environment_from_reference(reference): six.string_types[0]) self._mount_path = mount_path - if shared_path and not isinstance(shared_path, six.string_types[0]): + if shared_path and not isinstance(shared_path, six.string_types): raise IncorrectTypeError(Mount, 'shared_path', type(shared_path), six.string_types[0], False) self._shared_path = shared_path diff --git a/platform/src/main/python/dlpx/virtualization/platform/_virtual.py b/platform/src/main/python/dlpx/virtualization/platform/_virtual.py index 18d9e1f8..02d8826e 100644 --- a/platform/src/main/python/dlpx/virtualization/platform/_virtual.py +++ b/platform/src/main/python/dlpx/virtualization/platform/_virtual.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # # -*- coding: utf-8 -*- @@ -59,8 +59,8 @@ def cleanup(self): def cleanup_decorator(cleanup_impl): if self.cleanup_impl: raise OperationAlreadyDefinedError(Op.VIRTUAL_CLEANUP) - self.cleanup_impl = v.check_function(cleanup_impl, - Op.VIRTUAL_CLEANUP) + self.cleanup_impl = v.check_function( + cleanup_impl, Op.VIRTUAL_CLEANUP) return cleanup_impl return cleanup_decorator @@ -309,9 +309,9 @@ def _internal_cleanup(self, request): source_config = SourceConfigDefinition.from_dict( json.loads(request.source_config.parameters.json)) - self.cleanup_impl(repository=repository, - source_config=source_config, - virtual_source=virtual_source) + self.cleanup_impl( + repository=repository, source_config=source_config, + virtual_source=virtual_source) virtual_cleanup_response = platform_pb2.VirtualCleanupResponse() virtual_cleanup_response.return_value.CopyFrom( @@ -690,8 +690,8 @@ def _internal_initialize(self, request): repository = RepositoryDefinition.from_dict( json.loads(request.repository.parameters.json)) - config = self.initialize_impl(repository=repository, - virtual_source=virtual_source) + config = self.initialize_impl( + repository=repository, virtual_source=virtual_source) # Validate that this is a SourceConfigDefinition object. if not isinstance(config, SourceConfigDefinition): diff --git a/platform/src/main/python/dlpx/virtualization/platform/import_validations.py b/platform/src/main/python/dlpx/virtualization/platform/import_validations.py index 12960511..d0249d9a 100644 --- a/platform/src/main/python/dlpx/virtualization/platform/import_validations.py +++ b/platform/src/main/python/dlpx/virtualization/platform/import_validations.py @@ -1,7 +1,8 @@ # -# Copyright (c) 2020 by Delphix. All rights reserved. +# Copyright (c) 2020, 2021 by Delphix. All rights reserved. # import inspect +import six from dlpx.virtualization.platform import exceptions from dlpx.virtualization.platform.import_util import (import_check, @@ -25,7 +26,6 @@ def validate_entry_point(plugin_module): if plugin_module.entry_point is None: raise exceptions.IncorrectPluginCodeError( 'Plugin entry point object is None.') - if not hasattr(plugin_module.module_content, plugin_module.entry_point): raise exceptions.UserError( 'Entry point \'{}:{}\' does not exist. \'{}\' is not a symbol' @@ -83,7 +83,10 @@ def validate_named_args(plugin_module): for op_name_key, op_name in plugin_attrib.__dict__.items(): if op_name is None: continue - actual_args = inspect.getargspec(op_name) + if six.PY2: + actual_args = inspect.getargspec(op_name) + else: + actual_args = inspect.getfullargspec(op_name) warnings.extend( _check_args(method_name=op_name.__name__, expected_args=_lookup_expected_args( diff --git a/platform/src/main/python/dlpx/virtualization/platform/migration_helper.py b/platform/src/main/python/dlpx/virtualization/platform/migration_helper.py index 11119124..92b16337 100644 --- a/platform/src/main/python/dlpx/virtualization/platform/migration_helper.py +++ b/platform/src/main/python/dlpx/virtualization/platform/migration_helper.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # import re @@ -309,6 +309,7 @@ def __get_sorted_impls(migration_id, impl_dict): # if not migration_id: return [] + # # First filter out all ids less than the migration id. We need to do # this because even after sorting, we wouldn't know where in the list diff --git a/platform/src/main/python/dlpx/virtualization/platform/util.py b/platform/src/main/python/dlpx/virtualization/platform/util.py index d80a8718..79037b9e 100644 --- a/platform/src/main/python/dlpx/virtualization/platform/util.py +++ b/platform/src/main/python/dlpx/virtualization/platform/util.py @@ -1,7 +1,8 @@ # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # import dlpx.virtualization.api +from dlpx.virtualization.common.util import to_str def get_virtualization_api_version(): @@ -9,4 +10,4 @@ def get_virtualization_api_version(): :return: version string """ - return dlpx.virtualization.api.__version__ + return to_str(dlpx.virtualization.api.__version__) diff --git a/platform/src/test/python/dlpx/virtualization/fake_generated_definitions.py b/platform/src/test/python/dlpx/virtualization/fake_generated_definitions.py index 90fbcef6..e4ef3419 100644 --- a/platform/src/test/python/dlpx/virtualization/fake_generated_definitions.py +++ b/platform/src/test/python/dlpx/virtualization/fake_generated_definitions.py @@ -1,3 +1,6 @@ +import six + + class Model(object): # swaggerTypes: The key is attribute name and the # value is attribute type. @@ -10,7 +13,7 @@ class Model(object): class RepositoryDefinition(Model): def __init__(self, name): - self.swagger_types = {'name': str} + self.swagger_types = {'name': six.string_types[0]} self.attribute_map = {'name': 'name'} self._name = name @@ -29,7 +32,7 @@ def to_dict(self): class SourceConfigDefinition(Model): def __init__(self, name): - self.swagger_types = {'name': str} + self.swagger_types = {'name': six.string_types[0]} self.attribute_map = {'name': 'name'} self._name = name @@ -48,7 +51,7 @@ def to_dict(self): class LinkedSourceDefinition(Model): def __init__(self, name): - self.swagger_types = {'name': str} + self.swagger_types = {'name': six.string_types[0]} self.attribute_map = {'name': 'name'} self._name = name @@ -64,7 +67,7 @@ def from_dict(input_dict): class VirtualSourceDefinition(Model): def __init__(self, name): - self.swagger_types = {'name': str} + self.swagger_types = {'name': six.string_types[0]} self.attribute_map = {'name': 'name'} self._name = name @@ -83,7 +86,7 @@ def to_dict(self): class SnapshotDefinition(Model): def __init__(self, name): - self.swagger_types = {'name': str} + self.swagger_types = {'name': six.string_types[0]} self.attribute_map = {'name': 'name'} self._name = name diff --git a/platform/src/test/python/dlpx/virtualization/test_plugin.py b/platform/src/test/python/dlpx/virtualization/test_plugin.py index 10740fe9..f69258ca 100755 --- a/platform/src/test/python/dlpx/virtualization/test_plugin.py +++ b/platform/src/test/python/dlpx/virtualization/test_plugin.py @@ -1,23 +1,24 @@ from __future__ import absolute_import # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # import json import pytest +import re +import six from dlpx.virtualization.api import common_pb2, platform_pb2 -from dlpx.virtualization.common import (RemoteConnection, RemoteEnvironment, - RemoteHost, RemoteUser) +from dlpx.virtualization.common import ( + RemoteConnection, RemoteEnvironment, RemoteHost, RemoteUser) from dlpx.virtualization.platform.exceptions import ( IncorrectReturnTypeError, IncorrectUpgradeObjectTypeError, OperationAlreadyDefinedError, PluginRuntimeError) from mock import MagicMock, patch from . import fake_generated_definitions -from .fake_generated_definitions import (RepositoryDefinition, - SnapshotDefinition, - SourceConfigDefinition) +from .fake_generated_definitions import ( + RepositoryDefinition, SnapshotDefinition, SourceConfigDefinition) TEST_BINARY_PATH = '/binary/path' TEST_SCRATCH_PATH = '/scratch/path' @@ -64,9 +65,10 @@ })) TEST_POST_UPGRADE_PARAMS = ({ u'obj': - '"{\\"obj\\": {\\"prettyName\\": \\"prettyUpgrade\\", ' - '\\"name\\": \\"upgrade\\", \\"metadata\\": \\"metadata\\"}}"' + '"{\\"obj\\": {\\"name\\": \\"upgrade\\", ' + '\\"prettyName\\": \\"prettyUpgrade\\", \\"metadata\\": \\"metadata\\"}}"' }) + MIGRATION_IDS = ('2020.1.1', '2020.2.2') @@ -93,8 +95,8 @@ def configure_impl(): with pytest.raises(OperationAlreadyDefinedError): - @my_plugin.virtual.configure() - def configure_impl(): + @my_plugin.virtual.configure() # noqa F811 + def configure_impl(): # noqa F811 pass class NotModel1: @@ -468,10 +470,16 @@ def virtual_configure_impl(virtual_source, repository, snapshot): my_plugin.virtual._internal_configure(configure_request) message = err_info.value.message - assert message == ( - "The returned object for the virtual.configure() operation was" - " type 'unicode' but should be of class 'dlpx.virtualization." - "fake_generated_definitions.SourceConfigDefinition'.") + if six.PY2: + assert message == ( + "The returned object for the virtual.configure() operation was" + " type 'unicode' but should be of class 'dlpx.virtualization." + "fake_generated_definitions.SourceConfigDefinition'.") + else: + assert message == ( + "The returned object for the virtual.configure() operation was" + " class 'str' but should be of class 'dlpx.virtualization." + "fake_generated_definitions.SourceConfigDefinition'.") @staticmethod def test_virtual_unconfigure(my_plugin, virtual_source, repository, @@ -527,14 +535,11 @@ def virtual_reconfigure_impl(virtual_source, repository, source_config, assert config.parameters.json == expected_source_config @staticmethod - def test_virtual_reconfigure_return_incorrect_type(my_plugin, - virtual_source, - repository, - source_config, - snapshot): + def test_virtual_reconfigure_return_incorrect_type( + my_plugin, virtual_source, repository, source_config, snapshot): @my_plugin.virtual.reconfigure() - def virtual_reconfigure_impl(virtual_source, repository, source_config, - snapshot): + def virtual_reconfigure_impl( + virtual_source, repository, source_config, snapshot): TestPlugin.assert_plugin_args(virtual_source=virtual_source, source_config=source_config, repository=repository, @@ -554,17 +559,21 @@ def virtual_reconfigure_impl(virtual_source, repository, source_config, my_plugin.virtual._internal_reconfigure(reconfigure_request) message = err_info.value.message - assert message == ( - "The returned object for the virtual.reconfigure() operation was" - " type 'unicode' but should be of class 'dlpx.virtualization." - "fake_generated_definitions.SourceConfigDefinition'.") - - @staticmethod - def test_virtual_cleanup(my_plugin, virtual_source, repository, - source_config): + if six.PY2: + assert message == ( + "The returned object for the virtual.reconfigure() operation was" + " type 'unicode' but should be of class 'dlpx.virtualization." + "fake_generated_definitions.SourceConfigDefinition'.") + else: + assert message == ( + "The returned object for the virtual.reconfigure() operation was" + " class 'str' but should be of class 'dlpx.virtualization." + "fake_generated_definitions.SourceConfigDefinition'.") + + @staticmethod + def test_virtual_cleanup(my_plugin, virtual_source, repository, source_config): @my_plugin.virtual.cleanup() - def virtual_cleanup_impl(virtual_source, repository, - source_config): + def virtual_cleanup_impl(virtual_source, repository, source_config): TestPlugin.assert_plugin_args(virtual_source=virtual_source, repository=repository, source_config=source_config) @@ -586,8 +595,7 @@ def virtual_cleanup_impl(virtual_source, repository, assert virtual_cleanup_response.return_value == expected_result @staticmethod - def test_virtual_start(my_plugin, virtual_source, repository, - source_config): + def test_virtual_start(my_plugin, virtual_source, repository, source_config): @my_plugin.virtual.start() def virtual_start_impl(virtual_source, repository, source_config): TestPlugin.assert_plugin_args(virtual_source=virtual_source, @@ -722,8 +730,6 @@ def virtual_initialize_impl(virtual_source, repository): TestPlugin.setup_request(request=initialize_request, virtual_source=virtual_source, repository=repository) - - expected_source_config = TEST_REPOSITORY_JSON initialize_response = my_plugin.virtual._internal_initialize( initialize_request) @@ -749,10 +755,16 @@ def virtual_initialize_impl(virtual_source, repository): with pytest.raises(IncorrectReturnTypeError) as err_info: my_plugin.virtual._internal_initialize(initialize_request) message = err_info.value.message - assert message == ( - "The returned object for the virtual.initialize() operation was" - " type 'NoneType' but should be of class 'dlpx.virtualization." - "fake_generated_definitions.SourceConfigDefinition'.") + if six.PY2: + assert message == ( + "The returned object for the virtual.initialize() operation was" + " type 'NoneType' but should be of class 'dlpx.virtualization." + "fake_generated_definitions.SourceConfigDefinition'.") + else: + assert message == ( + "The returned object for the virtual.initialize() operation was" + " class 'NoneType' but should be of class 'dlpx.virtualization." + "fake_generated_definitions.SourceConfigDefinition'.") @staticmethod def test_virtual_mount_spec(my_plugin, virtual_source, repository): @@ -829,12 +841,20 @@ def repository_discovery_impl(source_connection): repository_discovery_request) message = err_info.value.message - assert message == ( - "The returned object for the discovery.repository() operation was" - " a list of [type 'str', class 'dlpx.virtualization" - ".fake_generated_definitions.RepositoryDefinition'] but should" - " be of type 'list of dlpx.virtualization" - ".fake_generated_definitions.RepositoryDefinition'.") + if six.PY2: + assert message == ( + "The returned object for the discovery.repository() operation was" + " a list of [type 'str', class 'dlpx.virtualization" + ".fake_generated_definitions.RepositoryDefinition'] but should" + " be of type 'list of dlpx.virtualization" + ".fake_generated_definitions.RepositoryDefinition'.") + else: + assert message == ( + "The returned object for the discovery.repository() operation was" + " a list of [class 'str', class 'dlpx.virtualization" + ".fake_generated_definitions.RepositoryDefinition'] but should" + " be of type 'list of dlpx.virtualization" + ".fake_generated_definitions.RepositoryDefinition'.") @staticmethod def test_source_config_discovery(my_plugin, connection, repository): @@ -894,8 +914,9 @@ def mock_direct_pre_snapshot(direct_source, repository, source_config, def test_direct_pre_snapshot_null_snapparams(my_plugin, direct_source, repository, source_config): @my_plugin.linked.pre_snapshot() - def mock_direct_pre_snapshot(direct_source, repository, source_config, - optional_snapshot_parameters): + def mock_direct_pre_snapshot( + direct_source, repository, source_config, + optional_snapshot_parameters): TestPlugin.assert_direct_source(direct_source) TestPlugin.assert_repository(repository) TestPlugin.assert_source_config(source_config) @@ -951,11 +972,11 @@ def direct_post_snapshot_impl(direct_source, repository, source_config, assert snapshot.parameters.json == expected_snapshot @staticmethod - def test_direct_post_snapshot_null_snapparams(my_plugin, direct_source, - repository, source_config): + def test_direct_post_snapshot_null_snapparams( + my_plugin, direct_source, repository, source_config): @my_plugin.linked.post_snapshot() def direct_post_snapshot_impl(direct_source, repository, source_config, - optional_snapshot_parameters): + optional_snapshot_parameters): TestPlugin.assert_direct_source(direct_source) TestPlugin.assert_repository(repository) TestPlugin.assert_source_config(source_config) @@ -1014,7 +1035,7 @@ def test_staged_pre_snapshot_null_snapparams(my_plugin, staged_source, repository, source_config): @my_plugin.linked.pre_snapshot() def staged_pre_snapshot_impl(staged_source, repository, source_config, - optional_snapshot_parameters): + optional_snapshot_parameters): TestPlugin.assert_staged_source(staged_source) TestPlugin.assert_repository(repository) TestPlugin.assert_source_config(source_config) @@ -1072,7 +1093,7 @@ def test_staged_post_snapshot_null_snapparams(my_plugin, staged_source, repository, source_config): @my_plugin.linked.post_snapshot() def staged_post_snapshot_impl(staged_source, repository, source_config, - optional_snapshot_parameters): + optional_snapshot_parameters): TestPlugin.assert_staged_source(staged_source) TestPlugin.assert_repository(repository) TestPlugin.assert_source_config(source_config) @@ -1262,8 +1283,8 @@ def test_upgrade_repository_success(my_plugin): def upgrade_repository(old_repository): return TEST_POST_MIGRATION_METADATA_1 - @my_plugin.upgrade.repository('2020.2.2') - def upgrade_repository(old_repository): + @my_plugin.upgrade.repository('2020.2.2') # noqa F811 + def upgrade_repository(old_repository): # noqa F811 return TEST_POST_MIGRATION_METADATA_2 upgrade_request = platform_pb2.UpgradeRequest() @@ -1278,7 +1299,17 @@ def upgrade_repository(old_repository): expected_response.return_value.post_upgrade_parameters\ .update(TEST_POST_UPGRADE_PARAMS) - assert expected_response == upgrade_response + pat = re.compile( + 'return_value{post_upgrade_parameters{key:"obj"value:""{"obj":{("' + 'prettyName":"prettyUpgrade"|"name":"upgrade"),("prettyName":"' + 'prettyUpgrade"|"name":"upgrade"),"metadata":"metadata"}}""}}' + ) + assert re.match( + pat, str(expected_response).replace( + "\n", "").replace("\\", "").replace(" ", "")) + assert re.match( + pat, str(upgrade_response).replace( + "\n", "").replace("\\", "").replace(" ", "")) @staticmethod def test_upgrade_source_config_success(my_plugin): @@ -1286,8 +1317,8 @@ def test_upgrade_source_config_success(my_plugin): def upgrade_source_config(old_source_config): return TEST_POST_MIGRATION_METADATA_1 - @my_plugin.upgrade.source_config('2020.2.2') - def upgrade_source_config(old_source_config): + @my_plugin.upgrade.source_config('2020.2.2') # noqa F811 + def upgrade_source_config(old_source_config): # noqa F811 return TEST_POST_MIGRATION_METADATA_2 upgrade_request = platform_pb2.UpgradeRequest() @@ -1302,7 +1333,17 @@ def upgrade_source_config(old_source_config): expected_response.return_value.post_upgrade_parameters \ .update(TEST_POST_UPGRADE_PARAMS) - assert expected_response == upgrade_response + pat = re.compile( + 'return_value{post_upgrade_parameters{key:"obj"value:""{"obj":' + '{("prettyName":"prettyUpgrade"|"name":"upgrade"),("prettyName":"' + 'prettyUpgrade"|"name":"upgrade"),"metadata":"metadata"}}""}}' + ) + assert re.match( + pat, str(expected_response).replace( + "\n", "").replace("\\", "").replace(" ", "")) + assert re.match( + pat, str(upgrade_response).replace( + "\n", "").replace("\\", "").replace(" ", "")) @staticmethod def test_upgrade_linked_source_success(my_plugin): @@ -1310,8 +1351,8 @@ def test_upgrade_linked_source_success(my_plugin): def upgrade_linked_source(old_linked_source): return TEST_POST_MIGRATION_METADATA_1 - @my_plugin.upgrade.linked_source('2020.2.2') - def upgrade_linked_source(old_linked_source): + @my_plugin.upgrade.linked_source('2020.2.2') # noqa F811 + def upgrade_linked_source(old_linked_source): # noqa F811 return TEST_POST_MIGRATION_METADATA_2 upgrade_request = platform_pb2.UpgradeRequest() @@ -1326,7 +1367,17 @@ def upgrade_linked_source(old_linked_source): expected_response.return_value.post_upgrade_parameters \ .update(TEST_POST_UPGRADE_PARAMS) - assert expected_response == upgrade_response + pat = re.compile( + 'return_value{post_upgrade_parameters{key:"obj"value:""{"obj":{("' + 'prettyName":"prettyUpgrade"|"name":"upgrade"),("prettyName":"' + 'prettyUpgrade"|"name":"upgrade"),"metadata":"metadata"}}""}}' + ) + assert re.match( + pat, str(expected_response).replace( + "\n", "").replace("\\", "").replace(" ", "")) + assert re.match( + pat, str(upgrade_response).replace( + "\n", "").replace("\\", "").replace(" ", "")) @staticmethod def test_upgrade_virtual_source_success(my_plugin): @@ -1334,8 +1385,8 @@ def test_upgrade_virtual_source_success(my_plugin): def upgrade_virtual_source(old_virtual_source): return TEST_POST_MIGRATION_METADATA_1 - @my_plugin.upgrade.virtual_source('2020.2.2') - def upgrade_virtual_source(old_virtual_source): + @my_plugin.upgrade.virtual_source('2020.2.2') # noqa F811 + def upgrade_virtual_source(old_virtual_source): # noqa F811 return TEST_POST_MIGRATION_METADATA_2 upgrade_request = platform_pb2.UpgradeRequest() @@ -1350,7 +1401,17 @@ def upgrade_virtual_source(old_virtual_source): expected_response.return_value.post_upgrade_parameters \ .update(TEST_POST_UPGRADE_PARAMS) - assert expected_response == upgrade_response + pat = re.compile( + 'return_value{post_upgrade_parameters{key:"obj"value:""{"obj":{("name":' + '"upgrade"|"prettyName":"prettyUpgrade"),("name":"upgrade"|"prettyName":' + '"prettyUpgrade"),"metadata":"metadata"}}""}}' + ) + assert re.match( + pat, str(expected_response).replace( + "\n", "").replace("\\", "").replace(" ", "")) + assert re.match( + pat, str(upgrade_response).replace( + "\n", "").replace("\\", "").replace(" ", "")) @staticmethod def test_upgrade_snapshot_success(my_plugin): @@ -1358,8 +1419,8 @@ def test_upgrade_snapshot_success(my_plugin): def upgrade_snapshot(old_snapshot): return TEST_POST_MIGRATION_METADATA_1 - @my_plugin.upgrade.snapshot('2020.2.2') - def upgrade_snapshot(old_snapshot): + @my_plugin.upgrade.snapshot('2020.2.2') # noqa F811 + def upgrade_snapshot(old_snapshot): # noqa F811 return TEST_POST_MIGRATION_METADATA_2 upgrade_request = platform_pb2.UpgradeRequest() @@ -1374,7 +1435,17 @@ def upgrade_snapshot(old_snapshot): expected_response.return_value.post_upgrade_parameters \ .update(TEST_POST_UPGRADE_PARAMS) - assert expected_response == upgrade_response + pat = re.compile( + 'return_value{post_upgrade_parameters{key:"obj"value:""{"obj":{("' + 'prettyName":"prettyUpgrade"|"name":"upgrade"),("name":"' + 'upgrade"|"prettyName":"prettyUpgrade"),"metadata":"metadata"}}""}}' + ) + assert re.match( + pat, str(expected_response).replace( + "\n", "").replace("\\", "").replace(" ", "")) + assert re.match( + pat, str(upgrade_response).replace( + "\n", "").replace("\\", "").replace(" ", "")) @staticmethod def test_upgrade_repository_incorrect_upgrade_object_type(my_plugin): @@ -1442,8 +1513,8 @@ def test_upgrade_snapshot_fail_with_runtime_error(my_plugin): def upgrade_snapshot(old_snapshot): raise RuntimeError('RuntimeError in snapshot migration') - @my_plugin.upgrade.snapshot('2020.2.2') - def upgrade_snapshot(old_snapshot): + @my_plugin.upgrade.snapshot('2020.2.2') # noqa F811 + def upgrade_snapshot(old_snapshot): # noqa F811 raise RuntimeError('RuntimeError in snapshot migration') upgrade_request = platform_pb2.UpgradeRequest() diff --git a/platform/src/test/python/dlpx/virtualization/test_plugin_classes.py b/platform/src/test/python/dlpx/virtualization/test_plugin_classes.py index c1e5215a..fb12496a 100644 --- a/platform/src/test/python/dlpx/virtualization/test_plugin_classes.py +++ b/platform/src/test/python/dlpx/virtualization/test_plugin_classes.py @@ -1,8 +1,9 @@ # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # import pytest +import six from dlpx.virtualization.common._common_classes import (RemoteEnvironment, RemoteHost) from dlpx.virtualization.common.exceptions import IncorrectTypeError @@ -40,17 +41,27 @@ def test_init_mount_bad_remote_env(): def test_init_mount_bad_mount_path(remote_environment): with pytest.raises(IncorrectTypeError) as err_info: Mount(remote_environment, 10000, 'shared_path') - assert err_info.value.message == ( - "Mount's parameter 'mount_path' was type 'int' but should" - " be of type 'basestring'.") + if six.PY2: + assert err_info.value.message == ( + "Mount's parameter 'mount_path' was type 'int' but should" + " be of type 'basestring'.") + else: + assert err_info.value.message == ( + "Mount's parameter 'mount_path' was class 'int' but should" + " be of class 'str'.") @staticmethod def test_init_mount_bad_shared_path(remote_environment): with pytest.raises(IncorrectTypeError) as err_info: Mount(remote_environment, 'mount_path', 10000) - assert err_info.value.message == ( - "Mount's parameter 'shared_path' was type 'int' but should" - " be of type 'basestring' if defined.") + if six.PY2: + assert err_info.value.message == ( + "Mount's parameter 'shared_path' was type 'int' but should" + " be of type 'basestring' if defined.") + else: + assert err_info.value.message == ( + "Mount's parameter 'shared_path' was class 'int' but should" + " be of class 'str' if defined.") @staticmethod def test_init_ownership_spec(): @@ -60,17 +71,27 @@ def test_init_ownership_spec(): def test_init_ownership_spec_bad_uid(): with pytest.raises(IncorrectTypeError) as err_info: OwnershipSpecification('10', 10) - assert err_info.value.message == ( - "OwnershipSpecification's parameter 'uid' was type 'str' but" - " should be of type 'int'.") + if six.PY2: + assert err_info.value.message == ( + "OwnershipSpecification's parameter 'uid' was type 'str' but" + " should be of type 'int'.") + else: + assert err_info.value.message == ( + "OwnershipSpecification's parameter 'uid' was class 'str' but" + " should be of class 'int'.") @staticmethod def test_init_ownership_spec_bad_gid(): with pytest.raises(IncorrectTypeError) as err_info: OwnershipSpecification(10, '10') - assert err_info.value.message == ( - "OwnershipSpecification's parameter 'gid' was type 'str' but" - " should be of type 'int'.") + if six.PY2: + assert err_info.value.message == ( + "OwnershipSpecification's parameter 'gid' was type 'str' but" + " should be of type 'int'.") + else: + assert err_info.value.message == ( + "OwnershipSpecification's parameter 'gid' was class 'str' but" + " should be of class 'int'.") @staticmethod def test_init_mount_spec(remote_environment): @@ -108,29 +129,48 @@ def test_init_mount_incorrect_format_reference_string(reference_string): def test_init_mount_invalid_reference_type(reference): with pytest.raises(IncorrectTypeError) as err_info: Mount(reference, 'mount_path', 'shared_path') - assert err_info.value.message == ( - "Mount's parameter 'remote_environment' was type '{}' but " - "should be of any one of the following types: " - "'['dlpx.virtualization.common._common_classes.RemoteEnvironment'," - " 'basestring']'.".format(type(reference).__name__)) + if six.PY2: + assert err_info.value.message == ( + "Mount's parameter 'remote_environment' was type '{}' but " + "should be of any one of the following types: " + "'['dlpx.virtualization.common._common_classes.RemoteEnvironment'," + " 'basestring']'.".format(type(reference).__name__)) + else: + assert err_info.value.message == ( + "Mount's parameter 'remote_environment' was class '{}' but " + "should be of any one of the following types: " + "'['dlpx.virtualization.common._common_classes.RemoteEnvironment'," + " 'str']'.".format(type(reference).__name__)) @staticmethod def test_init_mount_spec_mounts_not_list(): with pytest.raises(IncorrectTypeError) as err_info: MountSpecification('string', OwnershipSpecification(10, 10)) - assert err_info.value.message == ( - "MountSpecification's parameter 'mounts' was type 'str' but" - " should be of type 'list of dlpx.virtualization.platform" - "._plugin_classes.Mount'.") + if six.PY2: + assert err_info.value.message == ( + "MountSpecification's parameter 'mounts' was type 'str' but" + " should be of type 'list of dlpx.virtualization.platform" + "._plugin_classes.Mount'.") + else: + assert err_info.value.message == ( + "MountSpecification's parameter 'mounts' was class 'str' but" + " should be of type 'list of dlpx.virtualization.platform" + "._plugin_classes.Mount'.") @staticmethod def test_init_mount_spec_bad_mounts(): with pytest.raises(IncorrectTypeError) as err_info: MountSpecification(['string'], OwnershipSpecification(10, 10)) - assert err_info.value.message == ( - "MountSpecification's parameter 'mounts' was a list of" - " [type 'str'] but should be of type 'list of dlpx.virtualization" - ".platform._plugin_classes.Mount'.") + if six.PY2: + assert err_info.value.message == ( + "MountSpecification's parameter 'mounts' was a list of" + " [type 'str'] but should be of type 'list of dlpx.virtualization" + ".platform._plugin_classes.Mount'.") + else: + assert err_info.value.message == ( + "MountSpecification's parameter 'mounts' was a list of" + " [class 'str'] but should be of type 'list of dlpx.virtualization" + ".platform._plugin_classes.Mount'.") @staticmethod def test_init_mount_spec_bad_owner_spec(remote_environment): @@ -138,8 +178,15 @@ def test_init_mount_spec_bad_owner_spec(remote_environment): with pytest.raises(IncorrectTypeError) as err_info: MountSpecification([mount], 'string') - assert err_info.value.message == ( - "MountSpecification's parameter 'ownership_specification' was" - " type 'str' but should be of class 'dlpx.virtualization" - ".platform._plugin_classes.OwnershipSpecification'" - " if defined.") + if six.PY2: + assert err_info.value.message == ( + "MountSpecification's parameter 'ownership_specification' was" + " type 'str' but should be of class 'dlpx.virtualization" + ".platform._plugin_classes.OwnershipSpecification'" + " if defined.") + else: + assert err_info.value.message == ( + "MountSpecification's parameter 'ownership_specification' was" + " class 'str' but should be of class 'dlpx.virtualization" + ".platform._plugin_classes.OwnershipSpecification'" + " if defined.") diff --git a/platform/src/test/python/dlpx/virtualization/test_upgrade.py b/platform/src/test/python/dlpx/virtualization/test_upgrade.py index f5801e28..7990d384 100755 --- a/platform/src/test/python/dlpx/virtualization/test_upgrade.py +++ b/platform/src/test/python/dlpx/virtualization/test_upgrade.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # import copy @@ -254,8 +254,8 @@ def repo_upgrade_one(input_dict): output_dict['migrations'].append('lua repo 1.1') return output_dict - @upgrade_type_decorator('1.2', MigrationType.LUA) - def repo_upgrade_one(input_dict): + @upgrade_type_decorator('1.2', MigrationType.LUA) # noqa F811 + def repo_upgrade_one(input_dict): # noqa F811 output_dict = copy.deepcopy(input_dict) output_dict['migrations'].append('lua repo 1.2') return output_dict @@ -266,14 +266,14 @@ def repo_upgrade_two(input_dict): output_dict['migrations'].append('platform repo 2020.4.2') return output_dict - @upgrade_type_decorator('2020.4.3') - def repo_upgrade_two(input_dict): + @upgrade_type_decorator('2020.4.3') # noqa F811 + def repo_upgrade_two(input_dict): # noqa F811 output_dict = copy.deepcopy(input_dict) output_dict['migrations'].append('platform repo 2020.4.3') return output_dict - @upgrade_type_decorator('2020.4.4') - def repo_upgrade_two(input_dict): + @upgrade_type_decorator('2020.4.4') # noqa F811 + def repo_upgrade_two(input_dict): # noqa F811 output_dict = copy.deepcopy(input_dict) output_dict['migrations'].append('platform repo 2020.4.4') return output_dict diff --git a/tools/.python-version b/tools/.python-version deleted file mode 100644 index 43c4dbe6..00000000 --- a/tools/.python-version +++ /dev/null @@ -1 +0,0 @@ -2.7.17 diff --git a/tools/setup.cfg b/tools/setup.cfg index 690c13de..91544edc 100644 --- a/tools/setup.cfg +++ b/tools/setup.cfg @@ -1,25 +1,25 @@ # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # [metadata] -Metadata-Version: 1.2 -Author: Delphix -Author-email: virtualization-plugins@delphix.com -Home-page: https://developer.delphix.com -Summary: Delphix Virtualization SDK Tools -Long-description: file: README.md -Long-description-content-type: text/markdown -Classifiers: +metadata_version: 1.2 +author: Delphix +author_email: virtualization-plugins@delphix.com +home_page: https://developer.delphix.com +summary: Delphix Virtualization SDK Tools +long_description: file: README.md +long_description_content_type: text/markdown +classifiers: Development Status :: 5 - Production/Stable Programming Language :: Python - Programming Language :: Python :: 2.7 + Programming Language :: Python :: 3.8 License :: OSI Approved :: Apache Software License Operating System :: OS Independent [options] include_package_data = True -Requires-Python: 2.7 +requires_python: 3.8 [options.entry_points] console_scripts = diff --git a/tools/setup.py b/tools/setup.py index b5e0edc1..0e91ae4f 100644 --- a/tools/setup.py +++ b/tools/setup.py @@ -7,15 +7,16 @@ version = version_file.read().strip() install_requires = [ - "click >= 7.1", - "click-configfile == 0.2.3", - "dvp-platform == {}".format(version), - "enum34 >= 1.1.6", - "flake8 >= 3.6", - "jinja2 >= 2.10", - "jsonschema >= 3", - "pyyaml >= 3", - "requests >= 2.21.0", + "click == 7.1.2", + "click-configfile == 0.2.3", + "dvp-platform == {}".format(version), + "enum34 >= 1.1.6", + "flake8 >= 3.6", + "jinja2 >= 2.10", + "jsonschema >= 3", + "pyyaml >= 3", + "requests >= 2.21.0", + "httpretty == 0.9.7", ] setuptools.setup(name='dvp-tools', @@ -23,4 +24,5 @@ install_requires=install_requires, package_dir={'': PYTHON_SRC}, packages=setuptools.find_packages(PYTHON_SRC), + python_requires='>=3.8, <3.9', ) diff --git a/tools/src/main/python/dlpx/virtualization/_internal/cli.py b/tools/src/main/python/dlpx/virtualization/_internal/cli.py index 7d785591..ea7094b5 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/cli.py +++ b/tools/src/main/python/dlpx/virtualization/_internal/cli.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2019, 2020 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # import logging @@ -45,8 +45,7 @@ # adding it to CONTEXT_SETTINGS to avoid any side-effects on other commands. # CONTEXT_SETTINGS_INIT = dict(help_option_names=['-h', '--help'], - obj=click_util.ConfigFileProcessor.read_config(), - token_normalize_func=lambda x: x.encode("ascii")) + obj=click_util.ConfigFileProcessor.read_config()) DVP_CONFIG_MAP = CONTEXT_SETTINGS['obj'] @@ -96,11 +95,11 @@ def delphix_sdk(verbose, quiet): # will be printed to the console until this is executed. # logging_util.add_console_handler(console_logging_level) - - if sys.version_info[:2] != (2, 7): + if sys.version_info[:2] != (3, 8): raise exceptions.UserError( 'Python version check failed.' - 'Supported version is 2.7.x, found {}'.format(sys.version_info)) + 'Supported versions are 2.7.x and 3.8.x, found {}' + .format(sys.version_info)) @delphix_sdk.command(context_settings=CONTEXT_SETTINGS_INIT) diff --git a/tools/src/main/python/dlpx/virtualization/_internal/codegen.py b/tools/src/main/python/dlpx/virtualization/_internal/codegen.py index e137cf4b..cfc2d91d 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/codegen.py +++ b/tools/src/main/python/dlpx/virtualization/_internal/codegen.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2019, 2020 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # import copy @@ -12,6 +12,7 @@ import subprocess from dlpx.virtualization._internal import const, exceptions, file_util +from dlpx.virtualization.common.util import to_str logger = logging.getLogger(__name__) UNKNOWN_ERR = 'UNKNOWN_ERR' @@ -97,7 +98,8 @@ def generate_python(name, source_dir, plugin_config_dir, schema_content): # def _make_url_refs_opaque(json): if isinstance(json, dict): - for key in json: + keys = list(json.keys()) + for key in keys: if key == '$ref' and isinstance(json[key], six.string_types)\ and json[key].startswith('https://delphix.com/platform/api#'): json.pop(key) @@ -182,8 +184,7 @@ def _execute_swagger_codegen(swagger_file, output_dir): output_dir ] - logger.info('Running process with arguments: {!r}'.format( - ' '.join(process_inputs))) + logger.info(f"Running process with arguments: \'{' '.join(process_inputs)}\'") process = subprocess.Popen(process_inputs, stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -192,12 +193,13 @@ def _execute_swagger_codegen(swagger_file, output_dir): raise exceptions.UserError('Swagger python code generation failed.' ' Make sure java is on the PATH.') raise exceptions.UserError( - 'Unable to run {!r} to generate python code.' - '\nError code: {}. Error message: {}'.format( - jar, err.errno, os.strerror(err.errno))) + f"Unable to run '{jar}' to generate python code." + f"\nError code: {err.errno}. Error message: {os.strerror(err.errno)}") # Get the pipes pointed so we have access to them. stdout, stderr = process.communicate() + stdout = to_str(stdout) + stderr = to_str(stderr) # # Wait for the process to end and take the results. If res then we know diff --git a/tools/src/main/python/dlpx/virtualization/_internal/codegen/templates/base_model_.mustache b/tools/src/main/python/dlpx/virtualization/_internal/codegen/templates/base_model_.mustache index d413a47b..ec6631a3 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/codegen/templates/base_model_.mustache +++ b/tools/src/main/python/dlpx/virtualization/_internal/codegen/templates/base_model_.mustache @@ -1,10 +1,10 @@ # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # import pprint - import six + {{^supportPython2}} import typing {{/supportPython2}} @@ -37,7 +37,7 @@ class Model(object): """ result = {} - for attr, _ in six.iteritems(self.swagger_types): + for attr, _ in self.swagger_types.items(): value = getattr(self, attr) if value is None: # Plugins use the JSON schema specification to define their @@ -180,13 +180,16 @@ class GeneratedClassesTypeError(GeneratedClassesError): def _remove_angle_brackets(type_string): return type_string.replace('<', '').replace('>', '') + # In py3 the builtins module will be named 'builtins', in py2 it will + # be '__builtin__'. + builtins = ['__builtin__', 'builtins'] if isinstance(expected_type, list): if len(expected_type) != 1: raise ValueError('The thrown GeneratedClassesTypeError should' ' have had a list of size 1 as the' ' expected_type') single_type = expected_type[0] - if single_type.__module__ != '__builtin__': + if single_type.__module__ not in builtins: type_name = '{}.{}'.format( single_type.__module__, single_type.__name__) else: @@ -198,7 +201,10 @@ class GeneratedClassesTypeError(GeneratedClassesError): ' have had a set of size 1 as the' ' expected_type') single_type = expected_type.pop() - if single_type.__module__ != '__builtin__': + + + st_module = single_type.__module__ + if st_module not in builtins: type_name = '{}.{}'.format( single_type.__module__, single_type.__name__) else: @@ -209,14 +215,14 @@ class GeneratedClassesTypeError(GeneratedClassesError): raise ValueError('The thrown GeneratedClassesTypeError should' ' have had a dict of size 1 as the' ' expected_type') - key_type = expected_type.keys()[0] - value_type = expected_type.values()[0] - if key_type.__module__ != '__builtin__': + key_type = list(expected_type.keys())[0] + value_type = list(expected_type.values())[0] + if key_type.__module__ not in builtins: key_type_name = '{}.{}'.format( key_type.__module__, key_type.__name__) else: key_type_name = key_type.__name__ - if value_type.__module__ != '__builtin__': + if value_type.__module__ not in builtins: value_type_name = '{}.{}'.format( value_type.__module__, value_type.__name__) else: @@ -224,11 +230,11 @@ class GeneratedClassesTypeError(GeneratedClassesError): expected = "type 'dict of {}:{}'".format( key_type_name, value_type_name) else: - expected = _remove_angle_brackets(str(expected_type)) + expected = _remove_angle_brackets(six.string_types[0](expected_type)) if isinstance(actual_type, list): actual = 'a list of [{}]'.format( - ', '.join(_remove_angle_brackets(str(single_type)) + ', '.join(_remove_angle_brackets(six.string_types[0](single_type)) for single_type in actual_type)) elif isinstance(actual_type, set): # @@ -241,19 +247,19 @@ class GeneratedClassesTypeError(GeneratedClassesError): for type_tuple in actual_type)): actual = 'a dict with keys of {}{}{}'.format( '{', - ', '.join(_remove_angle_brackets(str(single_type)) + ', '.join(_remove_angle_brackets(six.string_types[0](single_type)) for single_type in actual_type), '}') else: actual = 'a dict of {}{}{}'.format( '{', ', '.join(['{0}:{1}'.format( - _remove_angle_brackets(str(k)), - _remove_angle_brackets(str(v))) for k, v in actual_type]), + _remove_angle_brackets(six.string_types[0](k)), + _remove_angle_brackets(six.string_types[0](v))) for k, v in actual_type]), '}') else: - actual = _remove_angle_brackets(str(actual_type)) + actual = _remove_angle_brackets(six.string_types[0](actual_type)) return actual, expected @@ -276,14 +282,15 @@ class GeneratedClassesTypeError(GeneratedClassesError): if not required and parameter is None: return None # Now check if the types are incorrect. + num_types = (float, complex, int) if expected_type == float: - if not isinstance(parameter, (float, int, long, complex)): + if not isinstance(parameter, num_types): return GeneratedClassesTypeError(object_type, parameter_name, type(parameter), float, required) - elif expected_type == str: + elif expected_type == six.string_types[0]: if not isinstance(parameter, six.string_types): return GeneratedClassesTypeError(object_type, parameter_name, @@ -300,7 +307,7 @@ class GeneratedClassesTypeError(GeneratedClassesError): required) if element_type == float: - check = all(isinstance(elem, (float, int, long, complex)) + check = all(isinstance(elem, num_types) for elem in parameter) else: check = all(isinstance(elem, element_type) @@ -336,7 +343,7 @@ class GeneratedClassesTypeError(GeneratedClassesError): if element_type: if element_type == float: value_check = all(isinstance(v, - (float, int, long, complex)) + num_types) for v in parameter.values()) else: value_check = all(isinstance(v, element_type) @@ -350,7 +357,7 @@ class GeneratedClassesTypeError(GeneratedClassesError): {six.string_types[0]: element_type}, required) else: - if not all(isinstance(k, six.string_types[0]) for k in parameter.keys()): + if not all(isinstance(k, six.string_types) for k in parameter.keys()): return GeneratedClassesTypeError( object_type, parameter_name, diff --git a/tools/src/main/python/dlpx/virtualization/_internal/codegen/templates/util.mustache b/tools/src/main/python/dlpx/virtualization/_internal/codegen/templates/util.mustache index f61af5c3..b6a59aa2 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/codegen/templates/util.mustache +++ b/tools/src/main/python/dlpx/virtualization/_internal/codegen/templates/util.mustache @@ -1,7 +1,9 @@ +# +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. +# import datetime import pydoc import re - import six @@ -33,7 +35,7 @@ def get_contained_type(type_string): for pattern in patterns: match = re.search(pattern, type_string) if match and match.group(1) != 'ERRORUNKNOWN': - # Convert the type to str here. + # Convert the type to string here. if match.group(1) == 'str': return six.string_types[0] return pydoc.locate(match.group(1)) @@ -53,7 +55,7 @@ def deserialize_model(data, klass): if not instance.swagger_types: return data - for attr, attr_type in six.iteritems(instance.swagger_types): + for attr, attr_type in instance.swagger_types.items(): if (data is not None and instance.attribute_map[attr] in data and isinstance(data, dict)): value = data[instance.attribute_map[attr]] @@ -73,7 +75,8 @@ def _deserialize(data, klass): if data is None: return None - if issubclass(klass, (int, float, long, complex, six.string_types[0], bool)): + primitive_types = {float, complex, bool, int, six.text_type, six.binary_type} + if issubclass(klass, tuple(primitive_types)): return _deserialize_primitive(data, klass) elif klass == datetime.date: return deserialize_date(data) @@ -99,15 +102,14 @@ def _deserialize_primitive(data, klass): try: value = klass(data) except UnicodeEncodeError: - if isinstance(data, str): + if isinstance(data, six.binary_type): # # Ignore errors even if the string is not proper UTF-8 or has - # broken marker bytes. The builtin unicode function can do this. + # broken marker bytes. # - value = unicode(data, 'utf-8', errors='ignore') + value = str(data, "utf-8", errors='ignore') else: - # Assume the value object has proper __unicode__() method. - value = unicode(data) + value = six.text_type(data) except TypeError: value = data return value @@ -166,4 +168,4 @@ def _deserialize_dict(data): :return: deserialized dict. :rtype: dict """ - return {k: _deserialize(v, type(v)) for k, v in six.iteritems(data)} + return {k: _deserialize(v, type(v)) for k, v in data.items()} diff --git a/tools/src/main/python/dlpx/virtualization/_internal/commands/build.py b/tools/src/main/python/dlpx/virtualization/_internal/commands/build.py index 30eff7ab..38c46d02 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/commands/build.py +++ b/tools/src/main/python/dlpx/virtualization/_internal/commands/build.py @@ -1,19 +1,20 @@ # -# Copyright (c) 2019, 2020 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # import base64 import compileall import copy +import io import json import logging import os -import StringIO import zipfile from dlpx.virtualization._internal import (codegen, exceptions, file_util, package_util, plugin_dependency_util, plugin_util) +from dlpx.virtualization.common.util import to_str logger = logging.getLogger(__name__) @@ -56,8 +57,16 @@ def build(plugin_config, 'Build parameters include plugin_config: %s, upload_artifact: %s,' ' generate_only: %s', plugin_config, upload_artifact, generate_only) + # Click handles the conversions for us, so we need not run inputs through to_str in + # the cli build function. However, this function may be called from places other + # than its corresponding cli function, such as unit tests. As such, we should ensure + # all appropriate inputs at this point are properly converted to unicode strings as + # soon as they enter the program. + plugin_config = to_str(plugin_config) + upload_artifact = to_str(upload_artifact) + if local_vsdk_root: - local_vsdk_root = os.path.expanduser(local_vsdk_root) + local_vsdk_root = to_str(os.path.expanduser(local_vsdk_root)) # Read content of the plugin config file provided and perform validations logger.info('Validating plugin config file %s', plugin_config) @@ -100,7 +109,6 @@ def build(plugin_config, os.path.dirname(plugin_config), schemas) except exceptions.UserError as err: raise exceptions.BuildFailedError(err) - if generate_only: # # If the generate_only flag is set then just return after generation @@ -140,6 +148,10 @@ def build(plugin_config, # Install dependencies in the plugin's source root in the build directory. plugin_dependency_util.install_deps(build_src_dir, local_vsdk_root=local_vsdk_root) + virtualization_dir = os.path.join(build_src_dir, "dlpx", "virtualization") + + for pkg in ['api', 'common', 'libs', 'platform']: + plugin_dependency_util.compile_py_files(os.path.join(virtualization_dir, pkg)) # Patch dependencies. patch_dependencies(build_src_dir) @@ -160,7 +172,7 @@ def build(plugin_config, logger.info('Successfully generated artifact file at %s.', upload_artifact) - logger.warn('\nBUILD SUCCESSFUL.') + logger.warning('\nBUILD SUCCESSFUL.') def patch_dependencies(build_src_dir): @@ -174,7 +186,7 @@ def patch_dependencies(build_src_dir): json_format_path = os.path.join( build_src_dir, 'google', 'protobuf', 'json_format.py') with open(json_format_path, 'r') as f: - json_format_text = f.read() + json_format_text = to_str(f.read()) json_format_text = json_format_text\ .replace(UNPAIRED_SURROGATE_DEFINITION, '')\ .replace(UNPAIRED_SURROGATE_SEARCH, '') @@ -187,6 +199,7 @@ def prepare_upload_artifact(plugin_config_content, src_dir, schemas, manifest): # This is the output dictionary that will be written # to the upload_artifact. # + artifact = { # Hard code the type to a set default. 'type': @@ -198,7 +211,7 @@ def prepare_upload_artifact(plugin_config_content, src_dir, schemas, manifest): # set default value of locale to en-us 'defaultLocale': plugin_config_content.get('defaultLocale', LOCALE_DEFAULT), - # set default value of language to PYTHON27 + # set default value of language to PYTHON38 'language': plugin_config_content['language'], 'hostTypes': @@ -252,7 +265,7 @@ def prepare_upload_artifact(plugin_config_content, src_dir, schemas, manifest): artifact['minimumLuaVersion'] = plugin_config_content[ 'minimumLuaVersion'] - return artifact + return to_str(artifact) def get_linked_source_definition_type(plugin_config_content): @@ -332,7 +345,7 @@ def zip_and_encode_source_files(source_code_dir): Jython creates a class loader to import .py files which the security manager prohibits. """ - + source_code_dir = to_str(source_code_dir) # # The contents of the zip should have relative and not absolute paths or # else the imports won't work as expected. @@ -348,7 +361,8 @@ def zip_and_encode_source_files(source_code_dir): raise exceptions.UserError( 'Failed to compile source code in the directory {}.'.format( source_code_dir)) - out_file = StringIO.StringIO() + + out_file = io.BytesIO() with zipfile.ZipFile(out_file, 'w', zipfile.ZIP_DEFLATED) as zip_file: for root, _, files in os.walk('.'): for filename in files: diff --git a/tools/src/main/python/dlpx/virtualization/_internal/commands/download_logs.py b/tools/src/main/python/dlpx/virtualization/_internal/commands/download_logs.py index 7000b701..b29f5e17 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/commands/download_logs.py +++ b/tools/src/main/python/dlpx/virtualization/_internal/commands/download_logs.py @@ -1,10 +1,11 @@ # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # import logging from dlpx.virtualization._internal import delphix_client, package_util +from dlpx.virtualization.common.util import to_str logger = logging.getLogger(__name__) @@ -34,6 +35,17 @@ def download_logs(engine, plugin_config, user, password, directory): logger.info('Downloading plugin logs from {} to: {}'.format( engine, directory)) + # Click handles the conversions for us, so we need not run inputs through to_str in + # the cli download_logs function. However, to be safe, this function may be called + # from places other than its corresponding cli function, such as unit tests. As + # such, we should ensure all appropriate inputs at this point are properly + # converted to unicode strings as soon as they enter the program. + engine = to_str(engine) + plugin_config = to_str(plugin_config) + user = to_str(user) + password = to_str(password) + directory = to_str(directory) + # Create a new delphix session. client = delphix_client.DelphixClient(engine) client.login(package_util.get_engine_api_version(), user, password) diff --git a/tools/src/main/python/dlpx/virtualization/_internal/commands/initialize.py b/tools/src/main/python/dlpx/virtualization/_internal/commands/initialize.py index 7d6d04b5..0975d822 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/commands/initialize.py +++ b/tools/src/main/python/dlpx/virtualization/_internal/commands/initialize.py @@ -1,10 +1,11 @@ # -# Copyright (c) 2019, 2020 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # import logging import os import shutil +import six import uuid from collections import OrderedDict @@ -12,6 +13,7 @@ import yaml from dlpx.virtualization._internal import (codegen, const, exceptions, file_util, plugin_util) +from dlpx.virtualization.common.util import to_bytes, to_str logger = logging.getLogger(__name__) @@ -60,6 +62,16 @@ def init(root, ingestion_strategy, name, host_type): 'Host Types': host_type }) + # Click handles the conversions for us, so we need not run inputs through to_str in + # the cli init function. However, this function may be called from places other + # than its corresponding cli function, such as unit tests. As such, we should ensure + # all appropriate inputs at this point are properly converted to unicode strings as + # soon as they enter the program. + root = to_str(root) + ingestion_strategy = to_str(ingestion_strategy) + name = to_str(name) + host_type = to_str(host_type) + # Files paths based on 'root' to be used throughout src_dir_path = os.path.join(root, DEFAULT_SRC_DIRECTORY) config_file_path = os.path.join(root, DEFAULT_PLUGIN_CONFIG_FILE) @@ -207,17 +219,30 @@ def _get_default_plugin_config(plugin_id, ingestion_strategy, name, OrderedDict: A valid plugin configuration roughly ordered from most interesting to a new plugin author to least interesting. """ - # Ensure values are type 'str'. If they are type unicode yaml prints - # them with '!!python/unicode' prepended to the value. - config = OrderedDict([('id', plugin_id.encode('utf-8')), - ('name', name.encode('utf-8')), - ('language', 'PYTHON27'), ('hostTypes', ['UNIX']), - ('pluginType', ingestion_strategy.encode('utf-8')), - ('entryPoint', entry_point.encode('utf-8')), - ('srcDir', src_dir_path.encode('utf-8')), - ('schemaFile', schema_file_path.encode('utf-8')), - ('hostTypes', [host_type.encode('utf-8')]), - ('buildNumber', default_build_number.encode('utf-8')) + if six.PY2: + # + # Ensure values are type 'str'. If they are type unicode yaml prints + # them with '!!python/unicode' prepended to the value. + # + # In Py3 yaml will print bytes with `!!binary |` prepended to them, so we + # should leave the as strings. + # + plugin_id = to_bytes(plugin_id) + name = to_bytes(name) + ingestion_strategy = to_bytes(ingestion_strategy) + entry_point = to_bytes(entry_point) + src_dir_path = to_bytes(src_dir_path) + schema_file_path = to_bytes(schema_file_path) + host_type = to_bytes(host_type) + default_build_number = to_bytes(default_build_number) + config = OrderedDict([('id', plugin_id), + ('name', name), + ('language', 'PYTHON38'), ('hostTypes', ['UNIX']), + ('pluginType', ingestion_strategy), + ('entryPoint', entry_point), + ('srcDir', src_dir_path), + ('schemaFile', schema_file_path), + ('hostTypes', [host_type]), + ('buildNumber', default_build_number) ]) - return config diff --git a/tools/src/main/python/dlpx/virtualization/_internal/commands/upload.py b/tools/src/main/python/dlpx/virtualization/_internal/commands/upload.py index 1395f4ae..fbfa58a3 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/commands/upload.py +++ b/tools/src/main/python/dlpx/virtualization/_internal/commands/upload.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # import errno @@ -8,6 +8,7 @@ import os from dlpx.virtualization._internal import delphix_client, exceptions +from dlpx.virtualization.common.util import to_str logger = logging.getLogger(__name__) UNKNOWN_ERR = 'UNKNOWN_ERR' @@ -36,6 +37,18 @@ def upload(engine, user, upload_artifact, password, wait): ' wait: {}'.format(engine, user, upload_artifact, wait)) logger.info('Uploading plugin artifact {} ...'.format(upload_artifact)) + # + # Click handles the conversions for us, so we need not run inputs through to_str in + # the cli upload function. However, this function may be called from places other + # than its corresponding cli function, such as unit tests. As such, we should ensure + # all appropriate inputs at this point are properly converted to unicode strings as + # soon as they enter the program. + # + engine = to_str(engine) + user = to_str(user) + upload_artifact = to_str(upload_artifact) + password = to_str(password) + # Read content of upload artifact try: with open(upload_artifact, 'rb') as f: diff --git a/tools/src/main/python/dlpx/virtualization/_internal/delphix_client.py b/tools/src/main/python/dlpx/virtualization/_internal/delphix_client.py index 03b5799e..55d941f0 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/delphix_client.py +++ b/tools/src/main/python/dlpx/virtualization/_internal/delphix_client.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # import json @@ -9,6 +9,7 @@ import requests from dlpx.virtualization._internal import exceptions, plugin_util +from dlpx.virtualization.common.util import to_bytes, to_str logger = logging.getLogger(__name__) @@ -36,12 +37,11 @@ def login(self, engine_api, user, password): Takes in the engine_api, user, and password and attempts to login to the engine. Can raise HttpPostError and UnexpectedError. """ - logger.info('Logging onto the Delphix Engine {!r}.'.format( - self.__engine)) + logger.info(f"Logging onto the Delphix Engine '{self.__engine}'.") self.__post('delphix/session', data={ - 'type': 'APISession', - 'version': engine_api + 'version': engine_api, + 'type': 'APISession' }) logger.debug('Session started successfully.') self.__post('delphix/login', @@ -50,7 +50,7 @@ def login(self, engine_api, user, password): 'username': user, 'password': password }) - logger.info('Successfully logged in as {!r}.'.format(user)) + logger.info(f"Successfully logged in as '{user}'.") @staticmethod def get_engine_api(artifact_content): @@ -70,7 +70,7 @@ def get_engine_api(artifact_content): json.dumps(engine_api))) return engine_api logger.debug( - 'engineApi found but malformed: {!r}'.format(engine_api)) + f"engineApi found but malformed: '{engine_api}'") raise exceptions.InvalidArtifactError() def __post(self, resource, content_type='application/json', data=None): @@ -93,7 +93,8 @@ def __post(self, resource, content_type='application/json', data=None): # Issue post request that was passed in, if data is a dict then convert # it to a json string. # - if data is not None and not isinstance(data, (str, bytes, unicode)): + if data is not None and not isinstance(data, (str, bytes)): + data = to_str(data) data = json.dumps(data) try: response = requests.post(url=url, data=data, headers=headers) @@ -211,7 +212,7 @@ def __download_logs(self, plugin_name, token, directory): "dlpx-plugin-logs-{}-{}.tar.gz".format(plugin_name, token)) with open(download_zip_name, "wb") as f: for chunk in download_zip_data: - f.write(chunk) + f.write(to_bytes(chunk)) def upload_plugin(self, name, content, wait): """ @@ -223,9 +224,9 @@ def upload_plugin(self, name, content, wait): logger.debug('Getting token to do upload.') response = self.__post('delphix/toolkit/requestUploadToken') token = response['result']['token'] - logger.debug('Got token {!r} successfully.'.format(token)) + logger.debug(f"Got token '{token}' successfully.") - logger.info('Uploading plugin {!r}.'.format(name)) + logger.info(f"Uploading plugin '{name}'.") # Encode plugin content. upload_response = self.__post('delphix/data/upload', content_type=self.__UPLOAD_CONTENT, @@ -306,8 +307,8 @@ def download_plugin_logs(self, directory, plugin_config): } response = self.__post('delphix/service/support/bundle/generate', data=data) - token = response['result'].encode('utf-8').strip() - logger.debug('Got token {!r} successfully.'.format(token)) + token = to_str(response['result'].encode('utf-8').strip()) + logger.debug(f"Got token '{token}' successfully.") self.__download_logs(plugin_name, token, directory) diff --git a/tools/src/main/python/dlpx/virtualization/_internal/exceptions.py b/tools/src/main/python/dlpx/virtualization/_internal/exceptions.py index 5013e2d5..decff5cd 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/exceptions.py +++ b/tools/src/main/python/dlpx/virtualization/_internal/exceptions.py @@ -1,11 +1,13 @@ # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # import collections import json import re +from dlpx.virtualization.common.util import to_str + class SDKToolingError(Exception): """ @@ -255,14 +257,15 @@ def __format_error(err): 'type': 'object'} """ # - # Validation error message could be unicode encoded string. Strip out - # any leading unicode characters for proper display and logging. + # Validation error message could be byte string. Strip out + # any leading byte characters for proper display and logging. # - err_msg = re.compile(r'\bu\b', re.IGNORECASE) + err_msg = re.compile(r'\bb\b', re.IGNORECASE) err_msg = err_msg.sub("", err.message) + map_func = to_str error_string = 'Error: {} on {}'.format( - err_msg, map(str, list(err.schema_path))) + err_msg, list(map(map_func, list(err.schema_path)))) return error_string diff --git a/tools/src/main/python/dlpx/virtualization/_internal/package_util.py b/tools/src/main/python/dlpx/virtualization/_internal/package_util.py index 1eb75725..298256e5 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/package_util.py +++ b/tools/src/main/python/dlpx/virtualization/_internal/package_util.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # import functools @@ -38,7 +38,7 @@ def _get_settings(): This assumes that the settings file is in the root of dlpx.virtualization._internal. """ - parser = configparser.SafeConfigParser() + parser = configparser.ConfigParser() parser.read(os.path.join(get_internal_package_root(), SETTINGS_FILE_NAME)) return parser diff --git a/tools/src/main/python/dlpx/virtualization/_internal/plugin_dependency_util.py b/tools/src/main/python/dlpx/virtualization/_internal/plugin_dependency_util.py index 1a90a154..4ae8ea88 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/plugin_dependency_util.py +++ b/tools/src/main/python/dlpx/virtualization/_internal/plugin_dependency_util.py @@ -1,14 +1,17 @@ # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # +import compileall import logging import os +import py_compile import subprocess import sys from dlpx.virtualization._internal import file_util, package_util from dlpx.virtualization._internal.exceptions import SubprocessFailedError +from dlpx.virtualization.common.util import to_str logger = logging.getLogger(__name__) @@ -110,12 +113,12 @@ def _execute_pip(pip_args): """ args = [sys.executable, '-m', 'pip'] args.extend(pip_args) - logger.debug('Executing %s', ' '.join(args)) proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) all_output, _ = proc.communicate() + all_output = to_str(all_output) exit_code = proc.wait() # @@ -168,9 +171,29 @@ def _build_wheel(package_root, target_dir=None): cwd=package_root) all_output, _ = proc.communicate() + all_output = to_str(all_output) exit_code = proc.wait() if exit_code != 0: raise SubprocessFailedError(' '.join(args), exit_code, all_output) else: logger.debug(all_output) + + +def compile_py_files(dpath): + """ + Compiles the python files in the given directory, generating the pyc files in the + same directory. + """ + compileall.compile_dir(dpath, force=True, quiet=1, legacy=True, ddir=".") + + +def compile_py_file(fpath: str): + """ + Compiles the python file at the given path, generating the pyc file in the + same directory. + + :param fpath: + :return: + """ + py_compile.compile(fpath, cfile=f"{fpath}c") diff --git a/tools/src/main/python/dlpx/virtualization/_internal/plugin_importer.py b/tools/src/main/python/dlpx/virtualization/_internal/plugin_importer.py index 501708f0..858ef5a9 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/plugin_importer.py +++ b/tools/src/main/python/dlpx/virtualization/_internal/plugin_importer.py @@ -1,7 +1,6 @@ # -# Copyright (c) 2019, 2020 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # -import importlib import logging import os import sys @@ -11,6 +10,7 @@ import yaml from dlpx.virtualization._internal import const, exceptions from dlpx.virtualization.platform import import_util +from dlpx.virtualization._internal import plugin_dependency_util logger = logging.getLogger(__name__) @@ -97,7 +97,6 @@ def __internal_import(self): self.__plugin_entry_point, self.__src_dir, err)) warnings['exception'].append(exception_msg) - return plugin_manifest, warnings @staticmethod @@ -161,7 +160,7 @@ def __run_checks(self, warnings): # warning_msg = exceptions.ValidationFailedError( warnings).message - logger.warn(warning_msg) + logger.warning(warning_msg) def __check_for_required_methods(self): """ @@ -189,8 +188,6 @@ def _import_module_and_get_manifest(queue, src_dir, module, entry_point, """ Imports the plugin module, runs validations and returns the manifest. """ - module_content = None - try: module_content = _import_helper(queue, src_dir, module) except exceptions.UserError: @@ -248,10 +245,29 @@ def _import_helper(queue, src_dir, module): exceptions. """ module_content = None - sys.path.append(src_dir) + try: + # + # Compile the plugin_runner module to be copied over to the docker container's + # plugin dir. + # The module comes in the format + # + appended_paths = _add_dirs_to_sys_path(src_dir) + plugin_dependency_util.compile_py_files(src_dir) + except FileNotFoundError: + # + # If the module couldn't be found, the user's entry point has the incorrect + # module name specified. + # + error = exceptions.UserError("No module named {}".format(module)) + queue.put({'exception': error}) try: - module_content = importlib.import_module(module) + # + # Module comes in the format pkg.[subpkg1.]*module. Since we've added the + # necessary paths to sys.path, we can import the module directly to retrieve + # its contents + # + module_content = __import__(module.split(".")[-1]) except (ImportError, TypeError) as err: queue.put({'exception': err}) except Exception as err: @@ -274,14 +290,32 @@ def _import_helper(queue, src_dir, module): error = exceptions.SDKToolingError(str(err)) queue.put({'sdk exception': error}) finally: - sys.path.remove(src_dir) + for p in appended_paths: + sys.path.remove(p) if not module_content: raise exceptions.UserError("Plugin module content is None") - return module_content +def _add_dirs_to_sys_path(src_dir, appended_paths=None): + if appended_paths is None: + appended_paths = [] + if src_dir not in sys.path: + sys.path.append(src_dir) + appended_paths.append(src_dir) + for root, dirs, _ in os.walk(src_dir): + for dir_name in dirs: + if dir_name in ["__pycache__"]: + continue + src_dir = os.path.sep.join([root, dir_name]) + if src_dir not in sys.path: + sys.path.append(src_dir) + appended_paths.append(src_dir) + _add_dirs_to_sys_path(src_dir, appended_paths=appended_paths) + return appended_paths + + def _process_warnings(queue, warnings): for warning in warnings: queue.put({'exception': warning}) diff --git a/tools/src/main/python/dlpx/virtualization/_internal/plugin_validator.py b/tools/src/main/python/dlpx/virtualization/_internal/plugin_validator.py index 9608a2db..821b3587 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/plugin_validator.py +++ b/tools/src/main/python/dlpx/virtualization/_internal/plugin_validator.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # import json @@ -12,6 +12,7 @@ from dlpx.virtualization._internal.codegen import CODEGEN_PACKAGE from flake8.api import legacy as flake8 from jsonschema import Draft7Validator +from dlpx.virtualization.common.util import to_bytes, to_str logger = logging.getLogger(__name__) @@ -33,6 +34,10 @@ def __init__(self, plugin_config, plugin_config_schema, plugin_config_content=None): + plugin_config = to_str(plugin_config) + plugin_config_schema = to_str(plugin_config_schema) + if plugin_config_content is not None: + plugin_config_content = to_str(plugin_config_content) self.__plugin_config = plugin_config self.__plugin_config_schema = plugin_config_schema self.__plugin_config_content = plugin_config_content @@ -83,7 +88,7 @@ def __read_plugin_config_file(self): try: with open(self.__plugin_config, 'rb') as f: try: - return yaml.safe_load(f) + return to_str(yaml.safe_load(f)) except yaml.YAMLError as err: if hasattr(err, 'problem_mark'): mark = err.problem_mark @@ -126,7 +131,7 @@ def __validate_plugin_config_content(self): schemaFile: the file containing defined schemas in the plugin manualDiscovery whether or not manual discovery is supported pluginType whether the plugin is DIRECT or STAGED - language language of the source code(ex: PYTHON27 for python2.7) + language language of the source code(ex: PYTHON38 for python3.8) Args: plugin_config_content (dict): A dictionary representing a plugin @@ -140,7 +145,7 @@ def __validate_plugin_config_content(self): try: with open(self.__plugin_config_schema, 'r') as f: try: - plugin_schema = json.load(f) + plugin_schema = to_str(json.load(f)) except ValueError as err: raise exceptions.UserError( 'Failed to load schemas because {} is not a ' @@ -155,8 +160,8 @@ def __validate_plugin_config_content(self): os.strerror(err.errno))) # Convert plugin config content to json - plugin_config_json = json.loads( - json.dumps(self.__plugin_config_content)) + plugin_config_json = to_str(json.loads( + to_bytes(json.dumps(self.__plugin_config_content)))) # Validate the plugin config against the schema v = Draft7Validator(plugin_schema) diff --git a/tools/src/main/python/dlpx/virtualization/_internal/schema_validator.py b/tools/src/main/python/dlpx/virtualization/_internal/schema_validator.py index 46354fce..75388f4b 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/schema_validator.py +++ b/tools/src/main/python/dlpx/virtualization/_internal/schema_validator.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # import json @@ -8,6 +8,7 @@ from collections import namedtuple from dlpx.virtualization._internal import exceptions +from dlpx.virtualization.common.util import to_str from jsonschema import Draft7Validator logger = logging.getLogger(__name__) @@ -25,9 +26,9 @@ class SchemaValidator: back. """ def __init__(self, schema_file, plugin_meta_schema, schemas=None): - self.__schema_file = schema_file - self.__plugin_meta_schema = plugin_meta_schema - self.__plugin_schemas = schemas + self.__schema_file = to_str(schema_file) + self.__plugin_meta_schema = to_str(plugin_meta_schema) + self.__plugin_schemas = to_str(schemas) @property def result(self): @@ -55,7 +56,7 @@ def __read_schema_file(self): try: with open(self.__schema_file, 'r') as f: try: - return json.load(f) + return to_str(json.load(f)) except ValueError as err: raise exceptions.UserError( 'Failed to load schemas because \'{}\' is not a ' @@ -77,7 +78,7 @@ def __validate_schemas(self): try: with open(self.__plugin_meta_schema, 'r') as f: try: - plugin_meta_schema = json.load(f) + plugin_meta_schema = to_str(json.load(f)) except ValueError as err: raise exceptions.UserError( 'Failed to load schemas because \'{}\' is not a ' @@ -98,8 +99,12 @@ def __validate_schemas(self): # This will do lazy validation so that we can consolidate all the # validation errors and report everything wrong with the schema. # - validation_errors = sorted(v.iter_errors(self.__plugin_schemas), - key=lambda e: e.path) + # In Python 3.8, we are using jsonschema 4.X.X. This version of jsonschema + # breaks when we pass a dictionary to Draft7Validator.iter_errors(). + # Instead it expects a list. + # + errors = v.iter_errors(self.__plugin_schemas) + validation_errors = sorted(errors, key=lambda e: e.path) if validation_errors: raise exceptions.SchemaValidationError(self.__schema_file, diff --git a/tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_config_schema.json b/tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_config_schema.json index 15f80871..4c6f3f3f 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_config_schema.json +++ b/tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_config_schema.json @@ -38,7 +38,7 @@ }, "language": { "type": "string", - "enum": ["PYTHON27"] + "enum": ["PYTHON38"] }, "rootSquashEnabled": { "type": "boolean" diff --git a/tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_schema.json b/tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_schema.json index b8be1836..edbfeab6 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_schema.json +++ b/tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_schema.json @@ -37,10 +37,6 @@ }, "type": ["object", "boolean"], "properties": { - "$id": { - "type": "string", - "format": "uri-reference" - }, "$schema": { "type": "string", "format": "uri" diff --git a/tools/src/test/python/dlpx/virtualization/_internal/commands/test_build.py b/tools/src/test/python/dlpx/virtualization/_internal/commands/test_build.py index bcc5a9ca..084e914e 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/commands/test_build.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/commands/test_build.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2019, 2020 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # import json @@ -35,7 +35,9 @@ class TestBuild: @mock.patch( 'dlpx.virtualization._internal.plugin_dependency_util.install_deps') @mock.patch('os.path.isabs', return_value=False) - def test_build_success(mock_relative_path, mock_install_deps, + @mock.patch( + 'dlpx.virtualization._internal.plugin_dependency_util.compile_py_files') + def test_build_success(mock_compile_py_files, mock_relative_path, mock_install_deps, mock_generate_python, mock_plugin_manifest, mock_patch_dependencies, plugin_config_file, artifact_file, artifact_content, @@ -72,12 +74,13 @@ def test_build_success(mock_relative_path, mock_install_deps, @mock.patch( 'dlpx.virtualization._internal.plugin_dependency_util.install_deps') @mock.patch('os.path.isabs', return_value=False) + @mock.patch( + 'dlpx.virtualization._internal.plugin_dependency_util.compile_py_files') def test_build_success_with_patched_dependencies( - mock_relative_path, mock_install_deps, + mock_compile_py_files, mock_relative_path, mock_install_deps, mock_generate_python, mock_plugin_manifest, plugin_config_file, artifact_file, codegen_gen_py_inputs, json_format_file_patched, json_format_content): - build.build(plugin_config_file, artifact_file, False, False) with open(json_format_file_patched, 'r') as f: @@ -95,10 +98,12 @@ def test_build_success_with_patched_dependencies( @mock.patch( 'dlpx.virtualization._internal.plugin_dependency_util.install_deps') @mock.patch('os.path.isabs', return_value=False) - def test_build_success_from_init(mock_relative_path, mock_install_deps, - mock_patch_dependencies, - tmpdir, ingestion_strategy, host_type, - plugin_name, artifact_file): + @mock.patch( + 'dlpx.virtualization._internal.plugin_dependency_util.compile_py_files') + def test_build_success_from_init( + mock_relative_path, mock_compile_py_files, mock_install_deps, + mock_patch_dependencies, tmpdir, ingestion_strategy, host_type, + plugin_name, artifact_file): # Initialize an empty directory. init.init(tmpdir.strpath, ingestion_strategy, plugin_name, host_type) plugin_config_file = os.path.join( @@ -124,11 +129,13 @@ def test_build_success_from_init(mock_relative_path, mock_install_deps, @mock.patch( 'dlpx.virtualization._internal.plugin_dependency_util.install_deps') @mock.patch('os.path.isabs', return_value=False) + @mock.patch( + 'dlpx.virtualization._internal.plugin_dependency_util.compile_py_files') def test_build_success_non_default_output_file( - mock_relative_path, mock_install_deps, mock_generate_python, - mock_import_plugin, mock_patch_dependencies, - plugin_config_file, artifact_file, - artifact_content, codegen_gen_py_inputs): + mock_relative_path, mock_compile_py_files, mock_install_deps, + mock_generate_python, mock_import_plugin, mock_patch_dependencies, + plugin_config_file, artifact_file, artifact_content, + codegen_gen_py_inputs): gen_py = codegen_gen_py_inputs # Before running build assert that the artifact file does not exist. @@ -236,13 +243,13 @@ def test_build_manifest_fail(mock_relative_path, mock_install_deps, @mock.patch( 'dlpx.virtualization._internal.plugin_dependency_util.install_deps') @mock.patch('os.path.isabs', return_value=False) - def test_build_prepare_artifact_fail(mock_relative_path, mock_install_deps, - mock_generate_python, - mock_plugin_manifest, - mock_patch_dependencies, - mock_prep_artifact, - plugin_config_file, artifact_file, - codegen_gen_py_inputs): + @mock.patch( + 'dlpx.virtualization._internal.plugin_dependency_util.compile_py_files') + def test_build_prepare_artifact_fail( + mock_compile_py_files, mock_relative_path, mock_install_deps, + mock_generate_python, mock_plugin_manifest, mock_patch_dependencies, + mock_prep_artifact, plugin_config_file, artifact_file, + codegen_gen_py_inputs): gen_py = codegen_gen_py_inputs # Before running build assert that the artifact file does not exist. @@ -281,11 +288,13 @@ def test_build_prepare_artifact_fail(mock_relative_path, mock_install_deps, @mock.patch( 'dlpx.virtualization._internal.plugin_dependency_util.install_deps') @mock.patch('os.path.isabs', return_value=False) + @mock.patch( + 'dlpx.virtualization._internal.plugin_dependency_util.compile_py_files') def test_build_generate_artifact_fail( - mock_relative_path, mock_install_deps, mock_generate_python, - mock_plugin_manifest, mock_patch_dependencies, - mock_gen_artifact, plugin_config_file, - artifact_file, codegen_gen_py_inputs): + mock_compile_py_files, mock_relative_path, mock_install_deps, + mock_generate_python, mock_plugin_manifest, mock_patch_dependencies, + mock_gen_artifact, plugin_config_file, artifact_file, + codegen_gen_py_inputs): gen_py = codegen_gen_py_inputs # Before running build assert that the artifact file does not exist. @@ -419,10 +428,12 @@ def test_zip_and_encode_source_files_encode_fail(mock_encode, src_dir): @mock.patch( 'dlpx.virtualization._internal.plugin_dependency_util.install_deps') @mock.patch('os.path.isabs', return_value=False) + @mock.patch( + 'dlpx.virtualization._internal.plugin_dependency_util.compile_py_files') @pytest.mark.parametrize('plugin_id', ['77f18ce4-4425-4cd6-b9a7-23653254d660']) - def test_id_validation_positive(mock_relative_path, mock_install_deps, - mock_patch_dependencies, + def test_id_validation_positive(mock_compile_py_files, mock_relative_path, + mock_install_deps, mock_patch_dependencies, mock_import_plugin, plugin_config_file, artifact_file): build.build(plugin_config_file, artifact_file, False) @@ -497,7 +508,7 @@ def test_plugin_bad_language(mock_generate_python, plugin_config_file, build.build(plugin_config_file, artifact_file, False, False) message = err_info.value.message - assert "'BAD_LANGUAGE' is not one of ['PYTHON27']" in message + assert "'BAD_LANGUAGE' is not one of ['PYTHON38']" in message assert not mock_generate_python.called @@ -603,9 +614,9 @@ def test_schema_bad_format(mock_generate_python, plugin_config_file, message = err_info.value.message assert ( - 'Failed to load schemas because \'{}\' is not a valid json file.' - ' Error: Extra data: line 2 column 1 - line 2 column 9' - ' (char 19 - 27)'.format(schema_file)) in message + "Failed to load schemas because '{}' is not a valid json file." + " Error: Extra data: line 2 column 1 (char 19) \n\nBUILD" + " FAILED.").format(schema_file) in message assert not mock_generate_python.called @@ -829,5 +840,5 @@ def test_non_existing_entry_file(mock_relative_path, plugin_config_file, build.build(plugin_config_file, artifact_file, False, False) message = err_info.value.message - exp_message = "No module named {module}".format(module=entry_module) + exp_message = "No module named \'{module}\'".format(module=entry_module) assert exp_message in message diff --git a/tools/src/test/python/dlpx/virtualization/_internal/commands/test_codegen.py b/tools/src/test/python/dlpx/virtualization/_internal/commands/test_codegen.py index ca6e3f84..b0fce144 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/commands/test_codegen.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/commands/test_codegen.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2019, 2020 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # import errno @@ -287,9 +287,9 @@ def test_execute_swagger_codegen_jar_issue(tmpdir, schema_content, codegen._execute_swagger_codegen(swagger_file, tmpdir.strpath) message = err_info.value.message - assert message == ('Unable to run {!r} to generate python code.' - '\nError code: 23. Error message: Too many open' - ' files in system'.format(popen_helper.jar)) + assert message == ( + f"Unable to run '{popen_helper.jar}' to generate python code.\nError" + " code: 23. Error message: Too many open files in system") assert popen_helper.stdout_input == subprocess.PIPE assert popen_helper.stderr_input == subprocess.PIPE diff --git a/tools/src/test/python/dlpx/virtualization/_internal/commands/test_delphix_client.py b/tools/src/test/python/dlpx/virtualization/_internal/commands/test_delphix_client.py index 8396089a..e21fb584 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/commands/test_delphix_client.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/commands/test_delphix_client.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # import json @@ -7,6 +7,7 @@ import requests from dlpx.virtualization._internal import delphix_client, exceptions +from dlpx.virtualization.common.util import to_str import httpretty import mock @@ -403,11 +404,11 @@ def test_delphix_client_unknown_error(engine_api): assert err_info.value.status_code == 404 assert err_info.value.response == ( - '{\n "status": "UNKNOWN", \n "blob": "Unknown"\n}') + '{\n "blob": "Unknown",\n "status": "UNKNOWN"\n}') + assert err_info.value.message == ( 'Received an unexpected error with HTTP Status 404,\nDumping full' - ' response:\n{\n "status": "UNKNOWN", \n "blob": "Unknown"\n}') - + ' response:\n{\n "blob": "Unknown",\n "status": "UNKNOWN"\n}') history = httpretty.HTTPretty.latest_requests assert history[-1].path == u'/resources/json/delphix/session' @@ -471,17 +472,18 @@ def test_delphix_client_wrong_login_no_detail(engine_api): message = err_info.value.message assert err_info.value.status_code == 401 - assert message == ('API request failed with HTTP Status 401' - '\nUnable to parse details of error.' - ' Dumping full response: {' - '\n "commandOutput": null, ' - '\n "diagnoses": [], ' - '\n "type": "APIError", ' - '\n "id": "exception.webservices.login.failed", ' - '\n "error": "Not a real error: Invalid username' - ' or password. Try with a different set of' - ' credentials."' - '\n}') + expected_message = ('API request failed with HTTP Status 401' + '\nUnable to parse details of error.' + ' Dumping full response: {' + '\n "type": "APIError",' + '\n "error": "Not a real error: Invalid username' + ' or password. Try with a different set of' + ' credentials.",' + '\n "id": "exception.webservices.login.failed",' + '\n "commandOutput": null,' + '\n "diagnoses": []' + '\n}') + assert message == expected_message history = httpretty.HTTPretty.latest_requests assert history[-1].path == u'/resources/json/delphix/login' @@ -726,10 +728,12 @@ def test_delphix_client_download_success(engine_api, src_dir, dc.download_plugin_logs(src_dir, plugin_config_file) history = httpretty.HTTPretty.latest_requests - assert (history[-1].path == + to_str(history[-1].__dict__) + + assert (to_str(history[-1].path) == u'/resources/json/delphix/data/downloadOutputStream' u'?token=5d6d5bb8-0f71-4304-8922-49c4c95c2387') - assert history[-2].path == ( + assert to_str(history[-2].path) == ( u'/resources/json/delphix/service/support/bundle/generate') assert history[-3].path == u'/resources/json/delphix/toolkit' assert history[-4].path == u'/resources/json/delphix/login' @@ -760,14 +764,15 @@ def test_validate_fail(artifact_content): delphix_client.DelphixClient.get_engine_api(artifact_content) message = err_info.value.message - assert message == ( + expected_message = ( 'The engineApi field is either missing or malformed.' ' The field must be of the form:' '\n{' '\n "type": "APIVersion",' - ' \n "major": 1,' - ' \n "minor": 7,' - ' \n "micro": 0' + '\n "major": 1,' + '\n "minor": 7,' + '\n "micro": 0' '\n}' '\nVerify that the artifact passed in was generated' ' by the build function.') + assert message == expected_message diff --git a/tools/src/test/python/dlpx/virtualization/_internal/commands/test_initialize.py b/tools/src/test/python/dlpx/virtualization/_internal/commands/test_initialize.py index 62fa9dfe..fe54a998 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/commands/test_initialize.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/commands/test_initialize.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2019, 2020 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # import ast @@ -12,6 +12,7 @@ from dlpx.virtualization._internal import (const, exceptions, plugin_util, plugin_validator, schema_validator) from dlpx.virtualization._internal.commands import initialize as init +from dlpx.virtualization.common.util import to_str @pytest.fixture @@ -63,6 +64,29 @@ def format_template(plugin_name, ingestion_strategy, host_type): raise RuntimeError( 'Got unrecognized ingestion strategy: {}'.format( ingestion_strategy)) + + # + # PY2 VS PY3 STRING/BYTES/UNICODE REPRESENTATIONS + # + # repr(("")) + # | PY2 | PY3 | + # -------------------------------------------------------- + # repr(str("1")) | "'1'" | '1' | + # -------------------------------------------------------- + # repr(bytes("1", "utf-8")) | N/A | b'1' | + # -------------------------------------------------------- + # repr(bytes("1")) | '1' | N/A | + # -------------------------------------------------------- + # repr(unicode("1")) | u'1' | N/A | + # -------------------------------------------------------- + # + # Looking at the above table, we need to do one of the following to get a + # representation that is not prepended with `u` or `b`: + # 1) if six.PY2: plugin_name = to_bytes(plugin_name) + # 2) if six.PY3: plugin_name = to_str(plugin_name) (to_str will return a + # unicode string in PY2) + # + plugin_name = to_str(plugin_name) return template.render(name=repr(plugin_name), linked_operations=operations, default_mount_path=default_mount_path) @@ -110,9 +134,10 @@ def test_init(tmpdir, ingestion_strategy, host_type, schema_template, entry_file_path = os.path.join(tmpdir.strpath, config['srcDir'], entry_file) with open(entry_file_path, 'r') as f: - contents = f.read() - assert contents == format_entry_point_template( - config['id'], ingestion_strategy, host_type) + contents = to_str(f.read()) + expected_contents = to_str(format_entry_point_template( + config['id'], ingestion_strategy, host_type)) + assert contents == expected_contents @staticmethod def test_init_with_relative_path(tmpdir): @@ -217,9 +242,8 @@ def test_init_calls_cleanup_on_failure(mock_cleanup, mock_yaml_dump, @staticmethod def test_default_schema_definition(schema_template): validator = schema_validator.SchemaValidator(None, const.PLUGIN_SCHEMA, - schema_template) + schemas=schema_template) validator.validate() - # Validate the repository schema only has the 'name' property. assert len(schema_template['repositoryDefinition'] ['properties']) == 1, json.dumps( diff --git a/tools/src/test/python/dlpx/virtualization/_internal/commands/test_templates.py b/tools/src/test/python/dlpx/virtualization/_internal/commands/test_templates.py index bb7e9d89..7da59ca3 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/commands/test_templates.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/commands/test_templates.py @@ -1,15 +1,17 @@ # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # import importlib import itertools import json import os +import re import subprocess import sys from dlpx.virtualization._internal import codegen +from dlpx.virtualization.common.util import to_bytes, to_str import pytest @@ -35,7 +37,7 @@ def module(tmp_factory, schema_content): # create the config file to point to the tmpdir config_dict = {'packageName': tmpdir.name} config_file = basedir.joinpath('codegen-config.json') - config_file.write_bytes(json.dumps(config_dict, indent=2)) + config_file.write_bytes(to_bytes(json.dumps(config_dict, indent=2))) execute_swagger_codegen(swagger_file, str(config_file), str(basedir)) return importlib.import_module('.definitions', package=tmpdir.name) @@ -60,6 +62,8 @@ def execute_swagger_codegen(swagger_file, config_file, output_dir): # Get the pipes pointed so we have access to them. stdout, stderr = process.communicate() + stdout = to_str(stdout) + stderr = to_str(stderr) # # Wait for the process to end and take the results. If res then we know @@ -109,14 +113,14 @@ def schema_content(): 'type': 'object', 'additionalProperties': False, 'properties': { + 'requiredStringProperty': { + 'type': 'string', + 'pattern': "^test.*" + }, 'stringProperty': { 'type': 'string', 'minLength': 5, 'maxLength': 10 - }, - 'requiredStringProperty': { - 'type': 'string', - 'pattern': "^test.*" } }, 'required': ['requiredStringProperty'] @@ -198,7 +202,7 @@ def test_required_param_missing(module): @staticmethod def test_required_param_missing_setter(module): - test_object = module.TestDefinition('test string') + test_object = module.TestDefinition(required_string_property='test string') with pytest.raises(module.GeneratedClassesError) as err_info: test_object.required_string_property = None @@ -209,16 +213,17 @@ def test_required_param_missing_setter(module): @staticmethod def test_not_string(module): with pytest.raises(module.GeneratedClassesTypeError) as err_info: - module.TestDefinition('test string', 10) + module.TestDefinition( + required_string_property='test string', string_property=10) message = err_info.value.message assert message == ( "TestDefinition's parameter 'string_property' was" - " type 'int' but should be of type 'basestring' if defined.") + " class 'int' but should be of class 'str' if defined.") @staticmethod def test_not_string_setter(module): - test_object = module.TestDefinition('test string') + test_object = module.TestDefinition(required_string_property='test string') with pytest.raises(module.GeneratedClassesTypeError) as err_info: test_object.string_property = 10 @@ -226,12 +231,13 @@ def test_not_string_setter(module): message = err_info.value.message assert message == ( "TestDefinition's parameter 'string_property' was" - " type 'int' but should be of type 'basestring' if defined.") + " class 'int' but should be of class 'str' if defined.") @staticmethod def test_min_length(module): with pytest.raises(module.GeneratedClassesError) as err_info: - module.TestDefinition('test string', 'test') + module.TestDefinition( + required_string_property='test string', string_property='test') message = err_info.value.message assert message == ("Invalid value for 'string_property', length was 4" @@ -239,7 +245,7 @@ def test_min_length(module): @staticmethod def test_min_length_setter(module): - test_object = module.TestDefinition('test string') + test_object = module.TestDefinition(required_string_property='test string') with pytest.raises(module.GeneratedClassesError) as err_info: test_object.string_property = 'test' @@ -251,7 +257,9 @@ def test_min_length_setter(module): @staticmethod def test_max_length(module): with pytest.raises(module.GeneratedClassesError) as err_info: - module.TestDefinition('test string', 'test too long of string') + module.TestDefinition( + required_string_property='test string', + string_property='test too long of string') message = err_info.value.message assert message == ("Invalid value for 'string_property', length was 23" @@ -259,7 +267,7 @@ def test_max_length(module): @staticmethod def test_max_length_setter(module): - test_object = module.TestDefinition('test string') + test_object = module.TestDefinition(required_string_property='test string') with pytest.raises(module.GeneratedClassesError) as err_info: test_object.string_property = 'test too long of string' @@ -271,7 +279,7 @@ def test_max_length_setter(module): @staticmethod def test_bad_pattern(module): with pytest.raises(module.GeneratedClassesError) as err_info: - module.TestDefinition('bad test string') + module.TestDefinition(required_string_property='bad test string') message = err_info.value.message assert message == ("Invalid value for 'required_string_property'," @@ -280,7 +288,7 @@ def test_bad_pattern(module): @staticmethod def test_bad_pattern_setter(module): - test_object = module.TestDefinition('test string') + test_object = module.TestDefinition(required_string_property='test string') with pytest.raises(module.GeneratedClassesError) as err_info: test_object.required_string_property = 'bad test string' @@ -327,7 +335,9 @@ def schema_content(): @staticmethod def test_success(module): - test_object = module.TestDefinition(200.5, None, -50) + test_object = module.TestDefinition( + required_number_property=200.5, number_property=None, + required_integer_property=-50) assert test_object.required_number_property == 200.5 assert not test_object.number_property @@ -344,7 +354,9 @@ def test_success(module): @staticmethod def test_success_setter(module): - test_object = module.TestDefinition(200.5, None, -50) + test_object = module.TestDefinition( + required_number_property=200.5, number_property=None, + required_integer_property=-50) test_object.number_property = 13.5 test_object.integer_property = 18 @@ -365,7 +377,9 @@ def test_success_setter(module): @staticmethod def test_success_number_is_int(module): - test_object = module.TestDefinition(200, 13, -50, 18) + test_object = module.TestDefinition( + required_number_property=200, number_property=13, + required_integer_property=-50, integer_property=18) assert test_object.required_number_property == 200 assert test_object.number_property == 13 @@ -375,12 +389,13 @@ def test_success_number_is_int(module): @staticmethod def test_int_passed_in_as_float(module): with pytest.raises(module.GeneratedClassesTypeError) as err_info: - module.TestDefinition(200.5, 13.5, -50.5, 18.5) + module.TestDefinition( + required_number_property=200.5, required_integer_property=2.2) message = err_info.value.message assert message == ( "TestDefinition's parameter 'required_integer_property' was" - " type 'float' but should be of type 'int'.") + " class 'float' but should be of class 'int'.") @staticmethod def test_required_param_missing(module): @@ -392,21 +407,25 @@ def test_required_param_missing(module): " must not be 'None'.") with pytest.raises(module.GeneratedClassesError) as err_info: - module.TestDefinition(200.5) + module.TestDefinition(required_number_property=200.5) message = err_info.value.message assert message == ("The required parameter 'required_integer_property'" " must not be 'None'.") with pytest.raises(module.GeneratedClassesError) as err_info: - module.TestDefinition(None, 13.5, -50, 18) + module.TestDefinition( + required_number_property=None, number_property=13.5, + required_integer_property=-50, integer_property=18) message = err_info.value.message assert message == ("The required parameter 'required_number_property'" " must not be 'None'.") with pytest.raises(module.GeneratedClassesError) as err_info: - module.TestDefinition(200.5, 13.5, None, 18) + module.TestDefinition( + required_number_property=200.5, number_property=13.5, + required_integer_property=None, integer_property=18) message = err_info.value.message assert message == ("The required parameter 'required_integer_property'" @@ -414,7 +433,9 @@ def test_required_param_missing(module): @staticmethod def test_required_param_missing_setter(module): - test_object = module.TestDefinition(200.5, None, -50) + test_object = module.TestDefinition( + required_number_property=200.5, number_property=None, + required_integer_property=-50) with pytest.raises(module.GeneratedClassesError) as err_info: test_object.required_number_property = None @@ -432,45 +453,53 @@ def test_required_param_missing_setter(module): @staticmethod def test_not_number(module): with pytest.raises(module.GeneratedClassesTypeError) as err_info: - module.TestDefinition('string', None, -50) + module.TestDefinition( + required_number_property='string', number_property=None, + required_integer_property=-50) message = err_info.value.message assert message == ( "TestDefinition's parameter 'required_number_property' was" - " type 'str' but should be of type 'float'.") + " class 'str' but should be of class 'float'.") with pytest.raises(module.GeneratedClassesTypeError) as err_info: - module.TestDefinition(200.5, None, 'string') + module.TestDefinition( + required_number_property=200.5, number_property=None, + required_integer_property='string') message = err_info.value.message assert message == ( - "TestDefinition's parameter 'required_integer_property' was" - " type 'str' but should be of type 'int'.") + "TestDefinition's parameter 'required_integer_property' was" + " class 'str' but should be of class 'int'.") @staticmethod def test_not_number_setter(module): - test_object = module.TestDefinition(200.5, None, -50) + test_object = module.TestDefinition( + required_number_property=200.5, number_property=None, + required_integer_property=-50) with pytest.raises(module.GeneratedClassesTypeError) as err_info: test_object.number_property = 'string' message = err_info.value.message assert message == ( - "TestDefinition's parameter 'number_property' was" - " type 'str' but should be of type 'float' if defined.") + "TestDefinition's parameter 'number_property' was" + " class 'str' but should be of class 'float' if defined.") with pytest.raises(module.GeneratedClassesTypeError) as err_info: test_object.integer_property = 'string' message = err_info.value.message assert message == ( - "TestDefinition's parameter 'integer_property' was" - " type 'str' but should be of type 'int' if defined.") + "TestDefinition's parameter 'integer_property' was" + " class 'str' but should be of class 'int' if defined.") @staticmethod def test_minimum(module): with pytest.raises(module.GeneratedClassesError) as err_info: - module.TestDefinition(200.5, 1.0, -50) + module.TestDefinition( + required_number_property=200.5, number_property=1.0, + required_integer_property=-50) message = err_info.value.message assert message == ("Invalid value for 'number_property', value was 1.0" @@ -478,7 +507,9 @@ def test_minimum(module): @staticmethod def test_minimum_setter(module): - test_object = module.TestDefinition(200.5, None, -50) + test_object = module.TestDefinition( + required_number_property=200.5, number_property=None, + required_integer_property=-50) with pytest.raises(module.GeneratedClassesError) as err_info: test_object.number_property = 1.0 @@ -490,7 +521,9 @@ def test_minimum_setter(module): @staticmethod def test_maximum(module): with pytest.raises(module.GeneratedClassesError) as err_info: - module.TestDefinition(200.5, 13.5, -50, 21) + module.TestDefinition( + required_number_property=200.5, number_property=13.5, + required_integer_property=-50, integer_property=21) message = err_info.value.message assert message == ("Invalid value for 'integer_property', value was 21" @@ -498,7 +531,9 @@ def test_maximum(module): @staticmethod def test_maximum_setter(module): - test_object = module.TestDefinition(200.5, None, -50, None) + test_object = module.TestDefinition( + required_number_property=200.5, number_property=None, + required_integer_property=-50, integer_property=None) with pytest.raises(module.GeneratedClassesError) as err_info: test_object.integer_property = 21 @@ -510,7 +545,9 @@ def test_maximum_setter(module): @staticmethod def test_exclusive_minimum(module): with pytest.raises(module.GeneratedClassesError) as err_info: - module.TestDefinition(2.0, 13.5, -50) + module.TestDefinition( + required_number_property=2.0, number_property=13.5, + required_integer_property=-50) message = err_info.value.message assert message == ("Invalid value for 'required_number_property'," @@ -518,7 +555,9 @@ def test_exclusive_minimum(module): @staticmethod def test_exclusive_minimum_setter(module): - test_object = module.TestDefinition(200.5, None, -50) + test_object = module.TestDefinition( + required_number_property=200.5, number_property=None, + required_integer_property=-50) with pytest.raises(module.GeneratedClassesError) as err_info: test_object.required_number_property = 2.0 @@ -530,7 +569,7 @@ def test_exclusive_minimum_setter(module): @staticmethod def test_exclusive_maximum(module): with pytest.raises(module.GeneratedClassesError) as err_info: - module.TestDefinition(200.5, 13.5, 100) + module.TestDefinition(200.5, 13.5, required_integer_property=100) message = err_info.value.message assert message == ("Invalid value for 'required_integer_property'," @@ -538,7 +577,9 @@ def test_exclusive_maximum(module): @staticmethod def test_exclusive_maximum_setter(module): - test_object = module.TestDefinition(200.5, None, -50) + test_object = module.TestDefinition( + required_number_property=200.5, number_property=None, + required_integer_property=-50) with pytest.raises(module.GeneratedClassesError) as err_info: test_object.required_integer_property = 100 @@ -698,8 +739,8 @@ def test_object_is_string(module): message = err_info.value.message assert message == ( - "TestDefinition's parameter 'required_object_property' was type" - " 'str' but should be of a dict with keys type 'basestring'.") + "TestDefinition's parameter 'required_object_property' was class" + " 'str' but should be of a dict with keys type 'str'.") @staticmethod def test_object_is_string_setter(module): @@ -711,8 +752,8 @@ def test_object_is_string_setter(module): message = err_info.value.message assert message == ( - "TestDefinition's parameter 'required_object_property' was type" - " 'str' but should be of a dict with keys type 'basestring'.") + "TestDefinition's parameter 'required_object_property' was class" + " 'str' but should be of a dict with keys type 'str'.") @staticmethod def test_object_is_array(module): @@ -722,8 +763,8 @@ def test_object_is_array(module): message = err_info.value.message assert message == ( - "TestDefinition's parameter 'required_object_property' was type" - " 'list' but should be of a dict with keys type 'basestring'.") + "TestDefinition's parameter 'required_object_property' was class" + " 'list' but should be of a dict with keys type 'str'.") @staticmethod def test_object_is_array_setter(module): @@ -735,8 +776,8 @@ def test_object_is_array_setter(module): message = err_info.value.message assert message == ( - "TestDefinition's parameter 'required_object_property' was type" - " 'list' but should be of a dict with keys type 'basestring'.") + "TestDefinition's parameter 'required_object_property' was class" + " 'list' but should be of a dict with keys type 'str'.") @staticmethod def test_object_dict_with_bad_key_type(module): @@ -748,9 +789,9 @@ def test_object_dict_with_bad_key_type(module): }) expected_msg_template = ( - "TestDefinition's parameter 'required_object_property' was a dict" - " with keys of {{{}}} but should be" - " of a dict with keys type 'basestring'.") + "TestDefinition's parameter 'required_object_property' was a dict" + " with keys of {{{}}} but should be" + " of a dict with keys type 'str'.") possible_messages = create_possible_expected_messages( expected_msg_template, [str, int, bool]) @@ -774,9 +815,10 @@ def test_object_dict_with_bad_key_type_setter(module): } expected_msg_template = ( - "TestDefinition's parameter 'required_object_property' was a dict" - " with keys of {{{}}} but should be" - " of a dict with keys type 'basestring'.") + "TestDefinition's parameter 'required_object_property' was a dict" + " with keys of {{{}}} but should be" + " of a dict with keys type 'str'.") + possible_messages = create_possible_expected_messages( expected_msg_template, [str, int, bool]) @@ -795,9 +837,9 @@ def test_semi_defined_dict_not_bool_value_type(module): message = err_info.value.message assert message == ( - "TestDefinition's parameter 'boolean_dict_property' was a" - " dict of {type 'str':type 'str'} but should be of type 'dict of" - " basestring:bool' if defined.") + "TestDefinition's parameter 'boolean_dict_property' was a" + " dict of {class 'str':class 'str'} but should be of type 'dict of" + " str:bool' if defined.") @staticmethod def test_semi_defined_dict_not_bool_value_type_setter(module): @@ -809,9 +851,9 @@ def test_semi_defined_dict_not_bool_value_type_setter(module): message = err_info.value.message assert message == ( - "TestDefinition's parameter 'boolean_dict_property' was a" - " dict of {type 'str':type 'str'} but should be of type 'dict of" - " basestring:bool' if defined.") + "TestDefinition's parameter 'boolean_dict_property' was a" + " dict of {class 'str':class 'str'} but should be of type 'dict of" + " str:bool' if defined.") @staticmethod def test_semi_defined_dict_not_number_value_type(module): @@ -822,8 +864,8 @@ def test_semi_defined_dict_not_number_value_type(module): message = err_info.value.message assert message == ( "TestDefinition's parameter 'number_dict_property' was a" - " dict of {type 'str':type 'str'} but should be of type 'dict of" - " basestring:float' if defined.") + " dict of {class 'str':class 'str'} but should be of type 'dict of" + " str:float' if defined.") @staticmethod def test_semi_defined_dict_not_number_value_type_setter(module): @@ -836,8 +878,8 @@ def test_semi_defined_dict_not_number_value_type_setter(module): message = err_info.value.message assert message == ( "TestDefinition's parameter 'number_dict_property' was a" - " dict of {type 'str':type 'str'} but should be of type 'dict of" - " basestring:float' if defined.") + " dict of {class 'str':class 'str'} but should be of type 'dict of" + " str:float' if defined.") @staticmethod def test_internal_class_is_string(module): @@ -847,7 +889,7 @@ def test_internal_class_is_string(module): message = err_info.value.message assert message == ( - "TestDefinition's parameter 'defined_object_property' was type" + "TestDefinition's parameter 'defined_object_property' was class" " 'str' but should be of class" " '{}.TestDefinitionDefinedObjectProperty' if defined.".format( module.TestDefinitionDefinedObjectProperty.__module__)) @@ -862,7 +904,7 @@ def test_internal_class_is_string_setter(module): message = err_info.value.message assert message == ( - "TestDefinition's parameter 'defined_object_property' was type" + "TestDefinition's parameter 'defined_object_property' was class" " 'str' but should be of class" " '{}.TestDefinitionDefinedObjectProperty' if defined.".format( module.TestDefinitionDefinedObjectProperty.__module__)) @@ -878,8 +920,10 @@ class TestOtherClass: message = err_info.value.message assert message == ( - "TestDefinition's parameter 'defined_object_property' was type" - " 'instance' but should be of class" + "TestDefinition's parameter 'defined_object_property' was class" + " 'dlpx.virtualization._internal.commands.test_templates" + ".TestTemplateObjectProperty.test_internal_class_is_other_class" + ".locals.TestOtherClass' but should be of class" " '{}.TestDefinitionDefinedObjectProperty' if defined.".format( module.TestDefinitionDefinedObjectProperty.__module__)) @@ -896,9 +940,12 @@ class TestOtherClass: message = err_info.value.message assert message == ( - "TestDefinition's parameter 'defined_object_property' was type" - " 'instance' but should be of class" - " '{}.TestDefinitionDefinedObjectProperty' if defined.".format( + "TestDefinition's parameter 'defined_object_property' was class" + " 'dlpx.virtualization._internal.commands.test_templates" + ".TestTemplateObjectProperty" + ".test_internal_class_is_other_class_setter.locals.TestOtherClass' " + "but should be of class '{}.TestDefinitionDefinedObjectProperty' " + "if defined.".format( module.TestDefinitionDefinedObjectProperty.__module__)) @@ -999,8 +1046,8 @@ def test_array_is_string(module): message = err_info.value.message assert message == ( - "TestDefinition's parameter 'required_array_property' was type" - " 'str' but should be of type 'list of float'.") + "TestDefinition's parameter 'required_array_property' was class" + " 'str' but should be of type 'list of float'.") @staticmethod def test_array_is_string_setter(module): @@ -1011,8 +1058,8 @@ def test_array_is_string_setter(module): message = err_info.value.message assert message == ( - "TestDefinition's parameter 'required_array_property' was type" - " 'str' but should be of type 'list of float'.") + "TestDefinition's parameter 'required_array_property' was class" + " 'str' but should be of type 'list of float'.") @staticmethod def test_array_is_object(module): @@ -1025,8 +1072,8 @@ def test_array_is_object(module): message = err_info.value.message assert message == ( - "TestDefinition's parameter 'array_property' was type" - " 'dict' but should be of type 'list' if defined.") + "TestDefinition's parameter 'array_property' was class" + " 'dict' but should be of class 'list' if defined.") @staticmethod def test_array_is_object_setter(module): @@ -1037,8 +1084,8 @@ def test_array_is_object_setter(module): message = err_info.value.message assert message == ( - "TestDefinition's parameter 'array_property' was type" - " 'dict' but should be of type 'list' if defined.") + "TestDefinition's parameter 'array_property' was class" + " 'dict' but should be of class 'list' if defined.") @staticmethod def test_number_array_wrong_elem_types(module): @@ -1047,9 +1094,9 @@ def test_number_array_wrong_elem_types(module): message = err_info.value.message assert message == ( - "TestDefinition's parameter 'required_array_property' was a list" - " of [type 'str', type 'bool'] but should be of type 'list of" - " float'.") + "TestDefinition's parameter 'required_array_property' was a list" + " of [class 'str', class 'bool'] but should be of type 'list of" + " float'.") @staticmethod def test_number_array_wrong_elem_types_setter(module): @@ -1060,9 +1107,9 @@ def test_number_array_wrong_elem_types_setter(module): message = err_info.value.message assert message == ( - "TestDefinition's parameter 'required_array_property' was a list" - " of [type 'str', type 'bool'] but should be of type 'list of" - " float'.") + "TestDefinition's parameter 'required_array_property' was a list" + " of [class 'str', class 'bool'] but should be of type 'list of" + " float'.") @staticmethod def test_string_array_wrong_elem_types(module): @@ -1073,8 +1120,8 @@ def test_string_array_wrong_elem_types(module): message = err_info.value.message assert message == ( "TestDefinition's parameter 'string_array_property' was a list of" - " [type 'str', type 'int', type 'float'] but should be of type" - " 'list of basestring' if defined.") + " [class 'str', class 'int', class 'float'] but should be of type" + " 'list of str' if defined.") @staticmethod def test_string_array_wrong_elem_types_setter(module): @@ -1085,9 +1132,9 @@ def test_string_array_wrong_elem_types_setter(module): message = err_info.value.message assert message == ( - "TestDefinition's parameter 'string_array_property' was a list of" - " [type 'str', type 'int', type 'float'] but should be of type" - " 'list of basestring' if defined.") + "TestDefinition's parameter 'string_array_property' was a list of" + " [class 'str', class 'int', class 'float'] but should be of type" + " 'list of str' if defined.") class TestTemplateBooleanProperty: @@ -1171,8 +1218,8 @@ def test_boolean_is_string(module): message = err_info.value.message assert message == ( - "TestDefinition's parameter 'required_boolean_property' was type" - " 'str' but should be of type 'bool'.") + "TestDefinition's parameter 'required_boolean_property' was class" + " 'str' but should be of class 'bool'.") @staticmethod def test_boolean_is_string_setter(module): @@ -1183,8 +1230,8 @@ def test_boolean_is_string_setter(module): message = err_info.value.message assert message == ( - "TestDefinition's parameter 'required_boolean_property' was type" - " 'str' but should be of type 'bool'.") + "TestDefinition's parameter 'required_boolean_property' was class" + " 'str' but should be of class 'bool'.") class TestTemplateEnumProperty: @@ -1334,8 +1381,8 @@ def test_required_string_not_string(module): message = err_info.value.message assert message == ( - "TestDefinition's parameter 'required_string_property' was type" - " 'int' but should be of type 'basestring'.") + "TestDefinition's parameter 'required_string_property' was class" + " 'int' but should be of class 'str'.") @staticmethod def test_required_string_not_string_setter(module): @@ -1348,8 +1395,8 @@ def test_required_string_not_string_setter(module): message = err_info.value.message assert message == ( - "TestDefinition's parameter 'required_string_property' was type" - " 'int' but should be of type 'basestring'.") + "TestDefinition's parameter 'required_string_property' was class" + " 'int' but should be of class 'str'.") @staticmethod def test_string_incorrect_enum(module): @@ -1387,8 +1434,8 @@ def test_string_not_string(module): message = err_info.value.message assert message == ( - "TestDefinition's parameter 'string_property' was type 'int' but" - " should be of type 'basestring' if defined.") + "TestDefinition's parameter 'string_property' was class 'int' but" + " should be of class 'str' if defined.") @staticmethod def test_string_not_string_setter(module): @@ -1401,8 +1448,8 @@ def test_string_not_string_setter(module): message = err_info.value.message assert message == ( - "TestDefinition's parameter 'string_property' was type 'int' but" - " should be of type 'basestring' if defined.") + "TestDefinition's parameter 'string_property' was class 'int' but" + " should be of class 'str' if defined.") @staticmethod def test_required_object_incorrect_enum(module): @@ -1439,8 +1486,8 @@ def test_required_object_not_object(module): message = err_info.value.message assert message == ( - "TestDefinition's parameter 'required_object_property' was type" - " 'str' but should be of a dict with keys type 'basestring'.") + "TestDefinition's parameter 'required_object_property' was class" + " 'str' but should be of a dict with keys type 'str'.") @staticmethod def test_required_object_not_object_setter(module): @@ -1453,8 +1500,8 @@ def test_required_object_not_object_setter(module): message = err_info.value.message assert message == ( - "TestDefinition's parameter 'required_object_property' was type" - " 'str' but should be of a dict with keys type 'basestring'.") + "TestDefinition's parameter 'required_object_property' was class" + " 'str' but should be of a dict with keys type 'str'.") @staticmethod def test_object_incorrect_enum(module): @@ -1493,8 +1540,8 @@ def test_object_not_object(module): message = err_info.value.message assert message == ( - "TestDefinition's parameter 'object_property' was type 'str' but" - " should be of a dict with keys type 'basestring' if defined.") + "TestDefinition's parameter 'object_property' was class 'str' but" + " should be of a dict with keys type 'str' if defined.") @staticmethod def test_object_not_object_setter(module): @@ -1507,8 +1554,8 @@ def test_object_not_object_setter(module): message = err_info.value.message assert message == ( - "TestDefinition's parameter 'object_property' was type 'str' but" - " should be of a dict with keys type 'basestring' if defined.") + "TestDefinition's parameter 'object_property' was class 'str' but" + " should be of a dict with keys type 'str' if defined.") @staticmethod def test_required_array_incorrect_enum(module): @@ -1518,9 +1565,12 @@ def test_required_array_incorrect_enum(module): required_array_property=['FA', 'SO']) message = err_info.value.message - assert message == ( - "Invalid values for 'required_array_property'. Was [FA, SO]" - " but must be a subset of [DO, RE, MI].") + # In Py3, the "FA" and "SO" are not always listed in the same order. Use a + # regex to check test success regardless of this ordering. + pattern = re.compile( + "Invalid values for 'required_array_property'. Was \\[(FA, SO|SO, FA)\\] " + "but must be a subset of \\[DO, RE, MI\\].") + assert re.match(pattern, message) is not None @staticmethod def test_required_array_incorrect_enum_setter(module): @@ -1532,9 +1582,10 @@ def test_required_array_incorrect_enum_setter(module): test_object.required_array_property = ['FA', 'SO'] message = err_info.value.message - assert message == ( - "Invalid values for 'required_array_property'. Was [FA, SO]" - " but must be a subset of [DO, RE, MI].") + pattern = re.compile( + "Invalid values for 'required_array_property'. Was \\[(FA, SO|SO, FA)\\] " + "but must be a subset of \\[DO, RE, MI\\].") + assert re.match(pattern, message) is not None @staticmethod def test_required_array_not_array(module): @@ -1545,8 +1596,8 @@ def test_required_array_not_array(module): message = err_info.value.message assert message == ( - "TestDefinition's parameter 'required_array_property' was type" - " 'str' but should be of type 'list of basestring'.") + "TestDefinition's parameter 'required_array_property' was class" + " 'str' but should be of type 'list of str'.") @staticmethod def test_required_array_not_array_setter(module): @@ -1559,8 +1610,8 @@ def test_required_array_not_array_setter(module): message = err_info.value.message assert message == ( - "TestDefinition's parameter 'required_array_property' was type" - " 'str' but should be of type 'list of basestring'.") + "TestDefinition's parameter 'required_array_property' was class" + " 'str' but should be of type 'list of str'.") @staticmethod def test_array_incorrect_enum(module): @@ -1571,8 +1622,10 @@ def test_array_incorrect_enum(module): array_property=['FA', 'SO']) message = err_info.value.message - assert message == ("Invalid values for 'array_property'. Was [FA, SO]" - " but must be a subset of [DO, RE, MI] if defined.") + pattern = re.compile( + "Invalid values for 'array_property'. Was \\[(FA, SO|SO, FA)\\] but must" + " be a subset of \\[DO, RE, MI\\] if defined.") + assert re.match(pattern, message) is not None @staticmethod def test_array_incorrect_enum_setter(module): @@ -1584,8 +1637,11 @@ def test_array_incorrect_enum_setter(module): test_object.array_property = ['FA', 'SO'] message = err_info.value.message - assert message == ("Invalid values for 'array_property'. Was [FA, SO]" - " but must be a subset of [DO, RE, MI] if defined.") + pattern = re.compile( + "Invalid values for 'array_property'. Was \\[(FA, SO|SO, FA)\\] but must" + " be a subset of \\[DO, RE, MI\\] if defined.") + + assert re.match(pattern, message) is not None @staticmethod def test_array_not_array(module): @@ -1597,8 +1653,8 @@ def test_array_not_array(module): message = err_info.value.message assert message == ( - "TestDefinition's parameter 'array_property' was type 'str' but" - " should be of type 'list of basestring' if defined.") + "TestDefinition's parameter 'array_property' was class 'str' but" + " should be of type 'list of str' if defined.") @staticmethod def test_array_not_array_setter(module): @@ -1611,5 +1667,5 @@ def test_array_not_array_setter(module): message = err_info.value.message assert message == ( - "TestDefinition's parameter 'array_property' was type 'str' but" - " should be of type 'list of basestring' if defined.") + "TestDefinition's parameter 'array_property' was class 'str' but" + " should be of type 'list of str' if defined.") diff --git a/tools/src/test/python/dlpx/virtualization/_internal/conftest.py b/tools/src/test/python/dlpx/virtualization/_internal/conftest.py index 3b9a42ee..43c43602 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/conftest.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/conftest.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2019, 2020 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # import configparser @@ -7,9 +7,11 @@ import json import os +from importlib import reload import yaml from dlpx.virtualization._internal import cli, click_util, const, package_util from dlpx.virtualization._internal.commands import build +from dlpx.virtualization.common.util import to_bytes, to_str import pytest @@ -39,7 +41,7 @@ def plugin_config_file(tmpdir, plugin_config_filename, plugin_config_content): f = tmpdir.join(plugin_config_filename) if plugin_config_content: - f.write(plugin_config_content) + f.write(to_str(plugin_config_content)) return f.strpath @@ -113,8 +115,8 @@ def _write_dvp_config_file(tmpdir, if dev_config_properties: parser['dev'] = dev_config_properties - with open(dvp_config_filepath, 'wb') as config_file: - parser.write(config_file) + with open(dvp_config_filepath, 'w') as config_file: + parser.write(to_bytes(config_file)) # # Add temp_dir to list of config files the ConfigFileProcessor will @@ -156,7 +158,7 @@ def artifact_file(tmpdir, artifact_content, artifact_filename, # Only write the artifact if we want to actually create it. if isinstance(artifact_content, dict): artifact_content = json.dumps(artifact_content, indent=4) - f.write(artifact_content) + f.write(to_bytes(artifact_content)) return f.strpath @@ -249,7 +251,7 @@ def external_version(): @pytest.fixture def language(): - return 'PYTHON27' + return 'PYTHON38' @pytest.fixture @@ -644,7 +646,7 @@ def artifact_content(engine_api, virtual_source_definition, 'name': 'python_vfiles', 'externalVersion': '2.0.0', 'defaultLocale': 'en-us', - 'language': 'PYTHON27', + 'language': 'PYTHON38', 'hostTypes': ['UNIX'], 'entryPoint': 'python_vfiles:vfiles', 'buildApi': package_util.get_build_api_version(), diff --git a/tools/src/test/python/dlpx/virtualization/_internal/test_cli.py b/tools/src/test/python/dlpx/virtualization/_internal/test_cli.py index 4e8e215a..49c673bf 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/test_cli.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/test_cli.py @@ -1,8 +1,9 @@ # -# Copyright (c) 2019, 2020 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # import os +import re import click.testing as click_testing import yaml @@ -172,7 +173,7 @@ def test_blank_ingestion_strategy(plugin_name): ['init', '-n', plugin_name, '-s', '']) assert result.exit_code != 0 - assert "invalid choice" in result.output + assert "Invalid value" in result.output @staticmethod def test_non_existent_root_dir(plugin_name): @@ -214,7 +215,7 @@ def test_multiple_host_types(): ]) assert result.exit_code != 0 - assert "invalid choice" in result.output + assert "Invalid value" in result.output @staticmethod @mock.patch('dlpx.virtualization._internal.commands.initialize.init') @@ -235,7 +236,7 @@ def test_invalid_host_type(): result = runner.invoke(cli.delphix_sdk, ['init', '-t', 'UNI']) assert result.exit_code != 0 - assert "invalid choice" in result.output + assert "Invalid value" in result.output class TestBuildCli: @@ -576,12 +577,10 @@ def test_with_config_file_fail(artifact_file): os.chdir(cwd) assert result.exit_code == 2 - assert result.output == (u'Usage: delphix-sdk upload [OPTIONS]\n' - u'\n' - u'Error: Invalid value for \'-e\' / ' - u'\'--engine\': Option is required ' - u'and must be specified via the command line.' - u'\n') + output = result.output.replace("\n", "") + pattern = re.compile( + r"Usage: delphix-sdk upload \[OPTIONS\].*Error: Invalid value for '-e.*") + assert re.match(pattern, output) is not None class TestDownloadCli: @@ -640,13 +639,11 @@ def test_missing_params(): ]) assert result.exit_code == 2 - assert result.output == ( - u"Usage: delphix-sdk download-logs [OPTIONS]\n" - u"\n" - u"Error: Invalid value for '-e' / " - u"'--engine': Option is required " - u"and must be specified via the command line." - u"\n") + output = result.output.replace("\n", "") + pattern = re.compile( + r"Usage: delphix-sdk download-logs \[OPTIONS\].*" + r"Error: Invalid value for '-e.*") + assert re.match(pattern, output) is not None @staticmethod @mock.patch( @@ -789,10 +786,9 @@ def test_with_config_file_fail(plugin_config_file, dvp_config_file): os.chdir(cwd) assert result.exit_code == 2 - assert result.output == ( - u"Usage: delphix-sdk download-logs [OPTIONS]\n" - u"\n" - u"Error: Invalid value for '-e' / " - u"'--engine': Option is required " - u"and must be specified via the command line." - u"\n") + + output = result.output.replace("\n", "") + pattern = re.compile( + r"Usage: delphix-sdk download-logs \[OPTIONS\].*" + r"Error: Invalid value for '-e.*") + assert re.match(pattern, output) is not None diff --git a/tools/src/test/python/dlpx/virtualization/_internal/test_plugin_importer.py b/tools/src/test/python/dlpx/virtualization/_internal/test_plugin_importer.py index e54ad838..1d20ee9c 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/test_plugin_importer.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/test_plugin_importer.py @@ -1,14 +1,14 @@ # -# Copyright (c) 2019, 2020 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # -import exceptions import os +import re import uuid from collections import OrderedDict from multiprocessing import Queue -from dlpx.virtualization._internal import (file_util, plugin_util, - plugin_validator, plugin_importer) +from dlpx.virtualization._internal import ( + file_util, plugin_util, plugin_validator, plugin_importer, exceptions) from dlpx.virtualization._internal.plugin_importer import PluginImporter import mock @@ -146,7 +146,6 @@ def test_successful_validation(mock_file_util, plugin_config_file, def test_multiple_warnings(mock_file_util, plugin_config_file, fake_src_dir, expected_errors): mock_file_util.return_value = fake_src_dir - with pytest.raises(exceptions.UserError) as err_info: importer = get_plugin_importer(plugin_config_file) importer.validate_plugin_module() @@ -233,12 +232,12 @@ def test_plugin_info_warn_mode(mock_import, mock_relative_path, plugin_config_file, src_dir, plugin_module_content): plugin_config_content = OrderedDict([ - ('id', str(uuid.uuid4())), ('name', 'staged'.encode('utf-8')), - ('version', '0.1.0'), ('language', 'PYTHON27'), - ('hostTypes', ['UNIX']), ('pluginType', 'STAGED'.encode('utf-8')), + ('id', str(uuid.uuid4())), ('name', 'staged'), + ('version', '0.1.0'), ('language', 'PYTHON38'), + ('hostTypes', ['UNIX']), ('pluginType', 'STAGED'), ('manualDiscovery', True), - ('entryPoint', 'staged_plugin:staged'.encode('utf-8')), - ('srcDir', src_dir), ('schemaFile', 'schema.json'.encode('utf-8')) + ('entryPoint', 'staged_plugin:staged'), + ('srcDir', src_dir), ('schemaFile', 'schema.json') ]) mock_import.return_value = plugin_module_content try: @@ -282,7 +281,8 @@ def test_import_error(mock_file_util, plugin_config_file, importer.validate_plugin_module() message = err_info.value.message - assert expected_error in message + pattern = re.compile("^Error: No module named (')?dlpxxx(')?.*") + assert re.match(pattern, message) is not None @staticmethod @pytest.mark.parametrize( @@ -309,7 +309,6 @@ def test_bad_syntax(mock_file_util, plugin_config_file, def test_undefined_name_error(mock_file_util, plugin_config_file, fake_src_dir, expected_error): mock_file_util.return_value = fake_src_dir - with pytest.raises(exceptions.SDKToolingError) as err_info: importer = get_plugin_importer(plugin_config_file) importer.validate_plugin_module() diff --git a/tools/src/test/python/dlpx/virtualization/_internal/test_plugin_validator.py b/tools/src/test/python/dlpx/virtualization/_internal/test_plugin_validator.py index e81ce7da..4b419768 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/test_plugin_validator.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/test_plugin_validator.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2019, 2020 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # import json @@ -24,8 +24,8 @@ def test_plugin_bad_schema(plugin_config_file, plugin_config_content, message = err_info.value.message assert ('Failed to load schemas because {} is not a valid json file.' - ' Error: Extra data: line 2 column 1 - line 2 column 9' - ' (char 19 - 27)'.format(schema_file)) in message + ' Error: Extra data: line 2 column 1 (char 19)' + .format(schema_file)) in message @staticmethod @pytest.mark.parametrize('plugin_config_file', ['/dir/plugin_config.yml']) diff --git a/tools/src/test/python/dlpx/virtualization/_internal/test_schema_validator.py b/tools/src/test/python/dlpx/virtualization/_internal/test_schema_validator.py index 2901c0cf..5f735a79 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/test_schema_validator.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/test_schema_validator.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2019, 2020 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # import json @@ -22,8 +22,8 @@ def test_bad_meta_schema(schema_file, tmpdir, schema_filename): message = err_info.value.message assert ("Failed to load schemas because '{}' is not a valid json file." - " Error: Extra data: line 2 column 1 - line 2 column 9" - " (char 19 - 27)".format(schema_file)) in message + " Error: Extra data: line 2 column 1 (char 19)" + .format(schema_file)) in message @staticmethod def test_bad_schema_file(schema_file):