diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 8213c309..c3164d8b 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 3.1.0 +current_version = 4.0.4 commit = False tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+)(?P\d+))? diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index ab5d1efe..3aa35023 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -3,13 +3,48 @@ name: Pre-commit actions for Delphix Virtualization SDK on: [pull_request] jobs: - pytest: - name: Test ${{ matrix.package }} on ${{ matrix.os }} using pytest + pytest27: + name: Test ${{ matrix.package }} on ${{ matrix.os }} using pytest (Python 2.7) runs-on: ${{ matrix.os }} strategy: max-parallel: 4 matrix: python-version: [2.7] + os: [ubuntu-latest, macos-10.15, windows-latest] + package: [common, libs, platform] + + steps: + - name: Checkout ${{ matrix.package }} project + uses: actions/checkout@v3 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + + - name: Install ${{ matrix.package }} dependencies + working-directory: ${{ matrix.package }} + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt --find-links https://test.pypi.org/simple/dvp-api/ + + - name: Install ${{ matrix.package }} project + working-directory: ${{ matrix.package }} + run: | + pip install . --find-links https://test.pypi.org/simple/dvp-api/ + + - name: Test ${{ matrix.package }} project with pytest + working-directory: ${{ matrix.package }} + run: | + python -m pytest src/test/python + + pytest38: + name: Test ${{ matrix.package }} on ${{ matrix.os }} using pytest (Python 3.8) + runs-on: ${{ matrix.os }} + strategy: + max-parallel: 4 + matrix: + python-version: [3.8] os: [ubuntu-latest, macos-latest, windows-latest] package: [common, libs, platform, tools] @@ -38,33 +73,64 @@ jobs: run: | python -m pytest src/test/python - lint: - name: Lint ${{ matrix.package }} + lintpython27: + name: Lint ${{ matrix.package }} - Python27 runs-on: ubuntu-latest strategy: max-parallel: 4 matrix: - package: [tools] - + package: [common, libs, platform] + steps: - name: Checkout ${{ matrix.package }} uses: actions/checkout@v1 - + - name: Set up Python 2.7 uses: actions/setup-python@v1 with: python-version: 2.7 - + + - name: Install flake8 + run: | + python -m pip install --upgrade pip + pip install flake8 + + - name: Run flake8 on src directory + working-directory: ${{ matrix.package }} + run: python -m flake8 src/main/python --max-line-length 88 + + - name: Run flake8 on test directory + working-directory: ${{ matrix.package }} + run: python -m flake8 src/test/python --max-line-length 88 + + lintpython38: + name: Lint ${{ matrix.package }} - Python38 + + runs-on: ubuntu-latest + strategy: + max-parallel: 4 + matrix: + package: [common, libs, platform, tools] + + steps: + - name: Checkout ${{ matrix.package }} + uses: actions/checkout@v1 + + - name: Set up Python 3.8 + uses: actions/setup-python@v1 + with: + python-version: 3.8 + - name: Install flake8 run: | python -m pip install --upgrade pip pip install flake8 - + - name: Run flake8 on src directory working-directory: ${{ matrix.package }} run: python -m flake8 src/main/python --max-line-length 88 - + - name: Run flake8 on test directory working-directory: ${{ matrix.package }} run: python -m flake8 src/test/python --max-line-length 88 diff --git a/.github/workflows/publish-docs.yml b/.github/workflows/publish-docs.yml index a7165fef..12938703 100644 --- a/.github/workflows/publish-docs.yml +++ b/.github/workflows/publish-docs.yml @@ -75,7 +75,7 @@ jobs: if: ${{ github.repository == matrix.repository }} uses: peaceiris/actions-gh-pages@v3 with: - github_token: ${{ secrets.GITHUB_TOKEN }} + deploy_key: ${{ secrets.SDK_DEPLOY_KEY }} publish_dir: docs/site commit_message: Deploy to gh-pages 🚀 user_name: "github-actions[bot]" diff --git a/.gitignore b/.gitignore index c73fd091..b1fe796c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # # IntelliJ config files @@ -35,3 +35,8 @@ venv/ # Python cache __pycache__ +# .python-version files +*.python-version + +# OSX DS Store files +*.DS_Store diff --git a/README-dev.md b/README-dev.md index 34540f17..65f3ce74 100644 --- a/README-dev.md +++ b/README-dev.md @@ -99,6 +99,8 @@ If you want to bump the major/minor/patch version, run `bumpversion [major|minor If you want to get rid of the dev label (bump from `1.1.0.dev7` to `1.1.0`), run `bumpversion release`. +Note: After bumpversion the tools unit test will need to be manually updated to test for the new version. + ## Testing Currently, there are three types of SDK testing: unit, manual, and functional (blackbox). @@ -122,7 +124,9 @@ all the standard workflows. The same workflows will be exercised by functional ( ### Functional (blackbox) testing To run blackbox tests, follow these steps: 1. Push your code to a branch in the forked repository on Github. Let's say the branch is called `my-feature` in repository called `/virtualization-sdk`. -2. Navigate to the app-gate directory and start tests using `git blackbox`. For the guide on which test suite to use, +2. If you bumped the version (one of major, minor, or micro, not the dev version part), then QA will have to createa a new branch (qa-appdata-toolkits branch sdk-3-2-0 for example with version 3.2.0) and update their map before you can run the blackbox tests: +* automation/regression/BlackBox/blackbox/appdata/virtualization_sdk/dvp_settings.py +3. Navigate to the app-gate directory and start tests using `git blackbox`. For the guide on which test suite to use, see the next sections. At a minimum, each pull request should pass `appdata_python_samples` and `appdata_basic` tests with a direct or staged plugin. diff --git a/README.md b/README.md index 9299bf8b..9e1229f0 100644 --- a/README.md +++ b/README.md @@ -10,9 +10,10 @@ The latest user documentation can be found [here](https://developer.delphix.com) ### Prerequisites - macOS 10.14+, Ubuntu 16.04+, or Windows 10 -- Python 2.7 (Python 3 is not supported) +- Python 2.7 (vSDK 3.1.0 and earlier) +- Python 3.8 (vSDK 4.0.0 and later) - Java 7+ -- Delphix Engine 5.3.5.0 or above +- A Delphix Engine of an [appropriate version](/References/Version_Compatibility.md) ### Installing diff --git a/common/.python-version b/common/.python-version deleted file mode 100644 index 43c4dbe6..00000000 --- a/common/.python-version +++ /dev/null @@ -1 +0,0 @@ -2.7.17 diff --git a/common/setup.cfg b/common/setup.cfg index 8199f265..a482f5f1 100644 --- a/common/setup.cfg +++ b/common/setup.cfg @@ -1,20 +1,21 @@ # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # [metadata] -Metadata-Version: 1.2 -Author: Delphix -Author-email: virtualization-plugins@delphix.com -Home-page: https://developer.delphix.com -Long-description: file: README.md -Long-description-content-type: text/markdown -Classifiers: +metadata_version: 1.2 +author: Delphix +author_email: virtualization-plugins@delphix.com +home_page: https://developer.delphix.com +long_description: file: README.md +long_description_content_type: text/markdown +classifiers: Development Status :: 5 - Production/Stable Programming Language :: Python Programming Language :: Python :: 2.7 + Programming Language :: Python :: 3.8 License :: OSI Approved :: Apache Software License Operating System :: OS Independent [options] -Requires-Python: 2.7 +requires_python: >=2.7, <=3.8, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*, !=3.7.* diff --git a/common/setup.py b/common/setup.py index 56d61c75..d3593d72 100644 --- a/common/setup.py +++ b/common/setup.py @@ -4,7 +4,7 @@ PYTHON_SRC = 'src/main/python' install_requires = [ - "dvp-api == 1.5.0", + "dvp-api == 1.6.3", ] with open(os.path.join(PYTHON_SRC, 'dlpx/virtualization/common/VERSION')) as version_file: @@ -15,4 +15,5 @@ install_requires=install_requires, package_dir={'': PYTHON_SRC}, packages=setuptools.find_packages(PYTHON_SRC), + python_requires='>=2.7, <3.9, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*, !=3.7.*', ) diff --git a/common/src/main/python/dlpx/virtualization/common/VERSION b/common/src/main/python/dlpx/virtualization/common/VERSION index fd2a0186..c5106e6d 100644 --- a/common/src/main/python/dlpx/virtualization/common/VERSION +++ b/common/src/main/python/dlpx/virtualization/common/VERSION @@ -1 +1 @@ -3.1.0 +4.0.4 diff --git a/common/src/main/python/dlpx/virtualization/common/__init__.py b/common/src/main/python/dlpx/virtualization/common/__init__.py index b4692226..c4c8dcd6 100644 --- a/common/src/main/python/dlpx/virtualization/common/__init__.py +++ b/common/src/main/python/dlpx/virtualization/common/__init__.py @@ -1,7 +1,7 @@ # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # __path__ = __import__('pkgutil').extend_path(__path__, __name__) -from dlpx.virtualization.common._common_classes import * +from dlpx.virtualization.common._common_classes import * # noqa diff --git a/common/src/main/python/dlpx/virtualization/common/_common_classes.py b/common/src/main/python/dlpx/virtualization/common/_common_classes.py index 2ad05804..57a698f4 100644 --- a/common/src/main/python/dlpx/virtualization/common/_common_classes.py +++ b/common/src/main/python/dlpx/virtualization/common/_common_classes.py @@ -1,11 +1,11 @@ # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # from abc import ABCMeta +import six from dlpx.virtualization.api import common_pb2, libs_pb2 from dlpx.virtualization.common.exceptions import IncorrectTypeError -from enum import IntEnum """Classes used for Plugin Operations @@ -101,19 +101,19 @@ class RemoteEnvironment(object): """ def __init__(self, name, reference, host): - if not isinstance(name, basestring): + if not isinstance(name, six.string_types): raise IncorrectTypeError( RemoteEnvironment, 'name', type(name), - basestring) + six.string_types[0]) self.__name = name - if not isinstance(reference, basestring): + if not isinstance(reference, six.string_types): raise IncorrectTypeError( RemoteEnvironment, 'reference', type(reference), - basestring) + six.string_types[0]) self.__reference = reference if isinstance(host, RemoteHost): @@ -134,7 +134,9 @@ def reference(self): return self.__reference def to_proto(self): - """Converts plugin class RemoteEnvironment to protobuf class common_pb2.RemoteEnvironment + """ + Converts plugin class RemoteEnvironment to protobuf + class common_pb2.RemoteEnvironment """ remote_environment = common_pb2.RemoteEnvironment() remote_environment.name = self.name @@ -144,7 +146,9 @@ def to_proto(self): @staticmethod def from_proto(environment): - """Converts protobuf class common_pb2.RemoteEnvironment to plugin class RemoteEnvironment + """ + Converts protobuf class common_pb2.RemoteEnvironment to plugin + class RemoteEnvironment """ if not isinstance(environment, common_pb2.RemoteEnvironment): raise IncorrectTypeError( @@ -172,33 +176,33 @@ class RemoteHost(object): """ def __init__(self, name, reference, binary_path, scratch_path): - if not isinstance(name, basestring): + if not isinstance(name, six.string_types): raise IncorrectTypeError( RemoteHost, 'name', type(name), - basestring) + six.string_types[0]) self.__name = name - if not isinstance(reference, basestring): + if not isinstance(reference, six.string_types): raise IncorrectTypeError( RemoteHost, 'reference', type(reference), - basestring) + six.string_types[0]) self.__reference = reference - if not isinstance(binary_path, basestring): + if not isinstance(binary_path, six.string_types): raise IncorrectTypeError( RemoteHost, 'binary_path', type(binary_path), - basestring) + six.string_types[0]) self.__binary_path = binary_path - if not isinstance(scratch_path, basestring): + if not isinstance(scratch_path, six.string_types): raise IncorrectTypeError( RemoteHost, 'scratch_path', type(scratch_path), - basestring) + six.string_types[0]) self.__scratch_path = scratch_path @property @@ -255,19 +259,19 @@ class RemoteUser(object): reference: Reference of the RemoteUser. """ def __init__(self, name, reference): - if not isinstance(name, basestring): + if not isinstance(name, six.string_types): raise IncorrectTypeError( RemoteUser, 'name', type(name), - basestring) + six.string_types[0]) self.__name = name - if not isinstance(reference, basestring): + if not isinstance(reference, six.string_types): raise IncorrectTypeError( RemoteUser, 'reference', type(reference), - basestring) + six.string_types[0]) self.__reference = reference @property @@ -311,12 +315,12 @@ class Credentials(object): username: User name. """ def __init__(self, username): - if not isinstance(username, basestring): + if not isinstance(username, six.string_types): raise IncorrectTypeError( Credentials, 'username', type(username), - basestring) + six.string_types[0]) self.__username = username __metaclass__ = ABCMeta @@ -339,12 +343,12 @@ class PasswordCredentials(Credentials): """ def __init__(self, username, password): super(PasswordCredentials, self).__init__(username) - if not isinstance(password, basestring): + if not isinstance(password, six.string_types): raise IncorrectTypeError( PasswordCredentials, 'password', type(password), - basestring) + six.string_types[0]) self.__password = password @property @@ -353,7 +357,9 @@ def password(self): @staticmethod def from_proto(credentials_result): - """Converts protobuf class libs_pb2.CredentialsResult to plugin class PasswordCredentials + """ + Converts protobuf class libs_pb2.CredentialsResult to plugin + class PasswordCredentials """ if not isinstance(credentials_result, libs_pb2.CredentialsResult): raise IncorrectTypeError( @@ -375,23 +381,24 @@ class KeyPairCredentials(Credentials): Args: username (str): User name. private_key (str): Private key. - public_key (str): Public key corresponding to private key. Empty string if not present. + public_key (str): Public key corresponding to private key. Empty string if + not present. """ def __init__(self, username, private_key, public_key): super(KeyPairCredentials, self).__init__(username) - if not isinstance(private_key, basestring): + if not isinstance(private_key, six.string_types): raise IncorrectTypeError( KeyPairCredentials, 'private_key', type(private_key), - basestring) + six.string_types[0]) self.__private_key = private_key - if not isinstance(public_key, basestring): + if not isinstance(public_key, six.string_types): raise IncorrectTypeError( KeyPairCredentials, 'public_key', type(public_key), - basestring) + six.string_types[0]) self.__public_key = public_key @property diff --git a/common/src/main/python/dlpx/virtualization/common/exceptions.py b/common/src/main/python/dlpx/virtualization/common/exceptions.py index d1032a81..3ecef88e 100644 --- a/common/src/main/python/dlpx/virtualization/common/exceptions.py +++ b/common/src/main/python/dlpx/virtualization/common/exceptions.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # @@ -77,7 +77,12 @@ def _remove_angle_brackets(type_string): return type_string.replace('<', '').replace('>', '') def _get_type_name(type_object): - if type_object.__module__ != '__builtin__': + # + # In py3 the builtins module will be named 'builtins', in py2 it will + # be '__builtin__'. + # + builtins = ['__builtin__', 'builtins'] + if type_object.__module__ not in builtins: type_name = '{}.{}'.format( type_object.__module__, type_object.__name__) else: @@ -99,9 +104,13 @@ def _get_type_name(type_object): if len(expected_type) > 1: for index in range(0, len(expected_type)): expected_type[index] = _get_type_name(expected_type[index]) - expected_type[index] = _remove_angle_brackets(str(expected_type[index])) + expected_type[index] = _remove_angle_brackets( + str(expected_type[index])) expected = "any one of the following types: '{}'".format(expected_type) + elif len(expected_type) == 0: + raise PlatformError('The thrown TypeError should have had a list' + ' of size >= 1 as the expected_type') else: single_type = expected_type[0] type_name = _get_type_name(single_type) @@ -112,8 +121,8 @@ def _get_type_name(type_object): raise PlatformError('The thrown TypeError should have had a' ' dict of size 1 as the expected_type') - key_type = expected_type.keys()[0] - value_type = expected_type.values()[0] + key_type = list(expected_type.keys())[0] + value_type = list(expected_type.values())[0] key_type_name = _get_type_name(key_type) value_type_name = _get_type_name(value_type) @@ -139,7 +148,7 @@ def _get_type_name(type_object): raise PlatformError('The thrown TypeError should have had a' ' set of tuples to represent a dict') actual = 'a dict of {{{}}}'.format(', '.join(['{0}:{1}'.format( - _remove_angle_brackets(str(k)), + _remove_angle_brackets(str(k)), _remove_angle_brackets(str(v))) for k, v in actual_type])) else: actual = _remove_angle_brackets(str(actual_type)) @@ -164,12 +173,8 @@ class IncorrectTypeError(PluginRuntimeError): """ def __init__( - self, - object_type, - parameter_name, - actual_type, - expected_type, - required=True): + self, object_type, parameter_name, actual_type, expected_type, + required=True): actual, expected = self.get_actual_and_expected_type( actual_type, expected_type) @@ -179,4 +184,4 @@ def __init__( actual, expected, (' if defined', '')[required])) - super(IncorrectTypeError, self).__init__(message) \ No newline at end of file + super(IncorrectTypeError, self).__init__(message) diff --git a/common/src/main/python/dlpx/virtualization/common/util.py b/common/src/main/python/dlpx/virtualization/common/util.py new file mode 100644 index 00000000..2ff42f6f --- /dev/null +++ b/common/src/main/python/dlpx/virtualization/common/util.py @@ -0,0 +1,125 @@ +# +# Copyright (c) 2021 by Delphix. All rights reserved. +# + + +""" +Utility functions to convert between unicode and bytes. +""" + +import six + + +def to_bytes(string, encoding="utf-8"): + """ + Converts the given object to binary object, bytes (Py3) or str (Py2). + + :param string: The string like object to convert to bytes + :type string: ``object`` + :param encoding: The encoding to encode the string with. + :type encoding: ``str`` + :returns: The encoded string. + :rtype: ``bytes`` + """ + if string is None: + return + + if isinstance(string, dict): + for k, v in string.items(): + string[k] = to_bytes(v, encoding=encoding) + return string + + if isinstance(string, list): + return [to_bytes(i, encoding=encoding) for i in string] + + if isinstance(string, set): + return {to_bytes(i, encoding=encoding) for i in string} + + if isinstance(string, str): + return _to_bytes(string, encoding) + + return string + + +def _to_bytes(string, encoding): + if six.PY3: + if isinstance(string, str): + return string.encode(encoding) + else: + return bytes(string) + else: + if isinstance(string, unicode): # noqa + return string.encode(encoding) + else: + return str(string) + + +def to_str(b, encoding="utf-8"): + """ + Converts the given object to a text object, unicode (Py2) or str (Py3). + + :param b: The object to convert + :type b: ``object`` + :param encoding: The encoding to encode the string with. + :type encoding: ``str`` + :returns: The decoded string. + :rtype: ``str`` + """ + if b is None: + return + + if isinstance(b, dict): + for k, v in b.items(): + b[k] = to_str(v, encoding=encoding) + return b + + if isinstance(b, list): + return [to_str(i, encoding=encoding) for i in b] + + if isinstance(b, set): + return {to_str(i, encoding=encoding) for i in b} + + if isinstance(b, bytes): + return _to_str(b, encoding=encoding) + return b + + +def _to_str(b, encoding): + if six.PY3: + if isinstance(b, bytes): + try: + return str(b, encoding) + except UnicodeDecodeError: + pass + raise UnicodeError( + "Could not decode value with encoding {}".format(encoding) + ) + else: + return b + else: + if isinstance(b, str): + try: + return b.decode(encoding) + except UnicodeDecodeError: + pass + raise UnicodeError( + "Could not decode value with encoding {}".format(encoding) + ) + return b + + +def response_to_str(response): + """ + The response_to_str function ensures all relevant properties of the given + response are unicode (py2) / str (py3). Should be called on a response as + soon as it is received. + + Args: + response (RunPowerShellResponse or RunBashResponse or RunExpectResponse): + Response received by run_bash or run_powershell or run_expect + """ + if response.HasField("return_value"): + if hasattr(response.return_value, "stdout"): + response.return_value.stdout = to_str(response.return_value.stdout) + if hasattr(response.return_value, "stderr"): + response.return_value.stderr = to_str(response.return_value.stderr) diff --git a/common/src/test/python/dlpx/virtualization/common/__init__.py b/common/src/test/python/dlpx/virtualization/common/__init__.py index c7fd3fc1..e9e178ce 100644 --- a/common/src/test/python/dlpx/virtualization/common/__init__.py +++ b/common/src/test/python/dlpx/virtualization/common/__init__.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # -__path__ = __import__('pkgutil').extend_path(__path__, __name__) \ No newline at end of file +__path__ = __import__('pkgutil').extend_path(__path__, __name__) diff --git a/common/src/test/python/dlpx/virtualization/common/test_common_classes.py b/common/src/test/python/dlpx/virtualization/common/test_common_classes.py index d0ede0f7..16d6373d 100644 --- a/common/src/test/python/dlpx/virtualization/common/test_common_classes.py +++ b/common/src/test/python/dlpx/virtualization/common/test_common_classes.py @@ -1,11 +1,16 @@ # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # import pytest +import six from dlpx.virtualization.api import common_pb2 -from dlpx.virtualization.common._common_classes import (RemoteConnection, RemoteEnvironment, RemoteHost, RemoteUser) -from dlpx.virtualization.common.exceptions import IncorrectTypeError +from dlpx.virtualization.common._common_classes import ( + KeyPairCredentials, PasswordCredentials, RemoteConnection, RemoteEnvironment, + RemoteHost, RemoteUser) +from dlpx.virtualization.common.exceptions import ( + IncorrectTypeError, PluginRuntimeError, PlatformError) + @pytest.fixture def remote_user(): @@ -31,19 +36,31 @@ def test_init_remote_connection_success(remote_user, remote_environment): def test_init_remote_connection_incorrect_environment(remote_user): with pytest.raises(IncorrectTypeError) as err_info: RemoteConnection('', remote_user) - assert err_info.value.message == ( - "RemoteConnection's parameter 'environment' was" - " type 'str' but should be of class 'dlpx.virtualization" - ".common._common_classes.RemoteEnvironment'.") + if six.PY2: + assert err_info.value.message == ( + "RemoteConnection's parameter 'environment' was" + " type 'str' but should be of class 'dlpx.virtualization" + ".common._common_classes.RemoteEnvironment'.") + else: + assert err_info.value.message == ( + "RemoteConnection's parameter 'environment' was" + " class 'str' but should be of class 'dlpx.virtualization" + ".common._common_classes.RemoteEnvironment'.") @staticmethod def test_init_remote_connection_incorrect_user(remote_environment): with pytest.raises(IncorrectTypeError) as err_info: RemoteConnection(remote_environment, '') - assert err_info.value.message == ( - "RemoteConnection's parameter 'user' was" - " type 'str' but should be of class 'dlpx.virtualization" - ".common._common_classes.RemoteUser'.") + if six.PY2: + assert err_info.value.message == ( + "RemoteConnection's parameter 'user' was" + " type 'str' but should be of class 'dlpx.virtualization" + ".common._common_classes.RemoteUser'.") + else: + assert err_info.value.message == ( + "RemoteConnection's parameter 'user' was" + " class 'str' but should be of class 'dlpx.virtualization" + ".common._common_classes.RemoteUser'.") @staticmethod def test_remote_connection_to_proto(remote_user, remote_environment): @@ -61,10 +78,16 @@ def test_remote_connection_from_proto_success(): def test_remote_connection_from_proto_fail(): with pytest.raises(IncorrectTypeError) as err_info: RemoteConnection.from_proto('') - assert err_info.value.message == ( - "RemoteConnection's parameter 'connection' was" - " type 'str' but should be of class 'dlpx.virtualization.api" - ".common_pb2.RemoteConnection'.") + if six.PY2: + assert err_info.value.message == ( + "RemoteConnection's parameter 'connection' was" + " type 'str' but should be of class 'dlpx.virtualization.api" + ".common_pb2.RemoteConnection'.") + else: + assert err_info.value.message == ( + "RemoteConnection's parameter 'connection' was" + " class 'str' but should be of class 'dlpx.virtualization.api" + ".common_pb2.RemoteConnection'.") class TestRemoteEnvironment: @@ -76,26 +99,42 @@ def test_init_remote_environment_success(remote_host): def test_init_remote_environment_incorrect_name(remote_host): with pytest.raises(IncorrectTypeError) as err_info: RemoteEnvironment(1, '', remote_host) - assert err_info.value.message == ( - "RemoteEnvironment's parameter 'name' was" - " type 'int' but should be of type 'basestring'.") + if six.PY2: + assert err_info.value.message == ( + "RemoteEnvironment's parameter 'name' was" + " type 'int' but should be of type 'basestring'.") + else: + assert err_info.value.message == ( + "RemoteEnvironment's parameter 'name' was" + " class 'int' but should be of class 'str'.") @staticmethod def test_init_remote_environment_incorrect_reference(remote_host): with pytest.raises(IncorrectTypeError) as err_info: RemoteEnvironment('', 1, remote_host) - assert err_info.value.message == ( - "RemoteEnvironment's parameter 'reference' was" - " type 'int' but should be of type 'basestring'.") + if six.PY2: + assert err_info.value.message == ( + "RemoteEnvironment's parameter 'reference' was" + " type 'int' but should be of type 'basestring'.") + else: + assert err_info.value.message == ( + "RemoteEnvironment's parameter 'reference' was" + " class 'int' but should be of class 'str'.") @staticmethod def test_init_remote_environment_incorrect_host(): with pytest.raises(IncorrectTypeError) as err_info: RemoteEnvironment('', '', '') - assert err_info.value.message == ( - "RemoteEnvironment's parameter 'host' was" - " type 'str' but should be of class 'dlpx.virtualization" - ".common._common_classes.RemoteHost'.") + if six.PY2: + assert err_info.value.message == ( + "RemoteEnvironment's parameter 'host' was" + " type 'str' but should be of class 'dlpx.virtualization" + ".common._common_classes.RemoteHost'.") + else: + assert err_info.value.message == ( + "RemoteEnvironment's parameter 'host' was" + " class 'str' but should be of class 'dlpx.virtualization" + ".common._common_classes.RemoteHost'.") @staticmethod def test_remote_environment_to_proto(remote_host): @@ -113,10 +152,16 @@ def test_remote_environment_from_proto_success(): def test_remote_environment_from_proto_fail(): with pytest.raises(IncorrectTypeError) as err_info: RemoteEnvironment.from_proto('') - assert err_info.value.message == ( - "RemoteEnvironment's parameter 'environment' was" - " type 'str' but should be of class 'dlpx.virtualization.api" - ".common_pb2.RemoteEnvironment'.") + if six.PY2: + assert err_info.value.message == ( + "RemoteEnvironment's parameter 'environment' was" + " type 'str' but should be of class 'dlpx.virtualization.api" + ".common_pb2.RemoteEnvironment'.") + else: + assert err_info.value.message == ( + "RemoteEnvironment's parameter 'environment' was" + " class 'str' but should be of class 'dlpx.virtualization.api" + ".common_pb2.RemoteEnvironment'.") class TestRemoteHost: @@ -131,33 +176,53 @@ def test_init_remote_host_success(): def test_init_remote_host_incorrect_name(): with pytest.raises(IncorrectTypeError) as err_info: RemoteHost(1, '', '', '') - assert err_info.value.message == ( - "RemoteHost's parameter 'name' was" - " type 'int' but should be of type 'basestring'.") + if six.PY2: + assert err_info.value.message == ( + "RemoteHost's parameter 'name' was" + " type 'int' but should be of type 'basestring'.") + else: + assert err_info.value.message == ( + "RemoteHost's parameter 'name' was" + " class 'int' but should be of class 'str'.") @staticmethod def test_init_remote_host_incorrect_reference(): with pytest.raises(IncorrectTypeError) as err_info: RemoteHost('', 1, '', '') - assert err_info.value.message == ( - "RemoteHost's parameter 'reference' was" - " type 'int' but should be of type 'basestring'.") + if six.PY2: + assert err_info.value.message == ( + "RemoteHost's parameter 'reference' was" + " type 'int' but should be of type 'basestring'.") + else: + assert err_info.value.message == ( + "RemoteHost's parameter 'reference' was" + " class 'int' but should be of class 'str'.") @staticmethod def test_init_remote_host_incorrect_binary_path(): with pytest.raises(IncorrectTypeError) as err_info: RemoteHost('', '', 1, '') - assert err_info.value.message == ( - "RemoteHost's parameter 'binary_path' was" - " type 'int' but should be of type 'basestring'.") + if six.PY2: + assert err_info.value.message == ( + "RemoteHost's parameter 'binary_path' was" + " type 'int' but should be of type 'basestring'.") + else: + assert err_info.value.message == ( + "RemoteHost's parameter 'binary_path' was" + " class 'int' but should be of class 'str'.") @staticmethod def test_init_remote_host_incorrect_scratch_path(): with pytest.raises(IncorrectTypeError) as err_info: RemoteHost('', '', '', 1) - assert err_info.value.message == ( - "RemoteHost's parameter 'scratch_path' was" - " type 'int' but should be of type 'basestring'.") + if six.PY2: + assert err_info.value.message == ( + "RemoteHost's parameter 'scratch_path' was" + " type 'int' but should be of type 'basestring'.") + else: + assert err_info.value.message == ( + "RemoteHost's parameter 'scratch_path' was" + " class 'int' but should be of class 'str'.") @staticmethod def test_remote_host_to_proto_non_default(): @@ -175,10 +240,16 @@ def test_remote_host_from_proto_success(): def test_remote_host_from_proto_fail(): with pytest.raises(IncorrectTypeError) as err_info: RemoteHost.from_proto('') - assert err_info.value.message == ( - "RemoteHost's parameter 'host' was" - " type 'str' but should be of class 'dlpx.virtualization.api" - ".common_pb2.RemoteHost'.") + if six.PY2: + assert err_info.value.message == ( + "RemoteHost's parameter 'host' was" + " type 'str' but should be of class 'dlpx.virtualization.api" + ".common_pb2.RemoteHost'.") + else: + assert err_info.value.message == ( + "RemoteHost's parameter 'host' was" + " class 'str' but should be of class 'dlpx.virtualization.api" + ".common_pb2.RemoteHost'.") class TestRemoteUser: @@ -190,17 +261,27 @@ def test_init_remote_user_success(): def test_init_remote_user_incorrect_name(): with pytest.raises(IncorrectTypeError) as err_info: RemoteUser(1, '') - assert err_info.value.message == ( - "RemoteUser's parameter 'name' was" - " type 'int' but should be of type 'basestring'.") + if six.PY2: + assert err_info.value.message == ( + "RemoteUser's parameter 'name' was" + " type 'int' but should be of type 'basestring'.") + else: + assert err_info.value.message == ( + "RemoteUser's parameter 'name' was" + " class 'int' but should be of class 'str'.") @staticmethod def test_init_remote_user_incorrect_reference(): with pytest.raises(IncorrectTypeError) as err_info: RemoteUser('', 1) - assert err_info.value.message == ( - "RemoteUser's parameter 'reference' was" - " type 'int' but should be of type 'basestring'.") + if six.PY2: + assert err_info.value.message == ( + "RemoteUser's parameter 'reference' was" + " type 'int' but should be of type 'basestring'.") + else: + assert err_info.value.message == ( + "RemoteUser's parameter 'reference' was" + " class 'int' but should be of class 'str'.") @staticmethod def test_remote_user_to_proto_non_default(): @@ -218,7 +299,133 @@ def test_remote_user_from_proto_success(): def test_remote_user_from_proto_fail(): with pytest.raises(IncorrectTypeError) as err_info: RemoteUser.from_proto('') + if six.PY2: + assert err_info.value.message == ( + "RemoteUser's parameter 'user' was" + " type 'str' but should be of class 'dlpx.virtualization.api" + ".common_pb2.RemoteUser'.") + else: + assert err_info.value.message == ( + "RemoteUser's parameter 'user' was" + " class 'str' but should be of class 'dlpx.virtualization.api" + ".common_pb2.RemoteUser'.") + + +class TestCredentials: + @staticmethod + def test_init_credentials_incorrect_username_type(): + with pytest.raises(IncorrectTypeError) as err_info: + KeyPairCredentials(RemoteUser("user1", "reference1"), "1234", "5678") + if six.PY2: + assert err_info.value.message == ( + "Credentials's parameter 'username' was class " + "'dlpx.virtualization.common._common_classes.RemoteUser' but should be " + "of type 'basestring'.") + else: + assert err_info.value.message == ( + "Credentials's parameter 'username' was class " + "'dlpx.virtualization.common._common_classes.RemoteUser' but should be " + "of class 'str'.") + + +class TestKeyPairCredentials: + @staticmethod + def test_init_key_pair_credentials_incorrect_public_key_type(): + with pytest.raises(IncorrectTypeError) as err_info: + KeyPairCredentials("user1", "1234", 1) + if six.PY2: + assert err_info.value.message == ( + "KeyPairCredentials's parameter 'public_key' was type 'int' " + "but should be of type 'basestring'.") + else: + assert err_info.value.message == ( + "KeyPairCredentials's parameter 'public_key' was class 'int' " + "but should be of class 'str'.") + + @staticmethod + def test_init_key_pair_credentials_incorrect_private_key_type(): + with pytest.raises(IncorrectTypeError) as err_info: + KeyPairCredentials("user1", 1, "1234") + if six.PY2: + assert err_info.value.message == ( + "KeyPairCredentials's parameter 'private_key' was type 'int' " + "but should be of type 'basestring'.") + else: + assert err_info.value.message == ( + "KeyPairCredentials's parameter 'private_key' was class 'int' " + "but should be of class 'str'.") + + +class TestPasswordCredentials: + @staticmethod + def test_init_password_credentials_incorrect_password_type(): + with pytest.raises(IncorrectTypeError) as err_info: + PasswordCredentials("user1", 12345) + if six.PY2: + assert err_info.value.message == ( + "PasswordCredentials's parameter 'password' was type 'int' but " + "should be of type 'basestring'.") + else: + assert err_info.value.message == ( + "PasswordCredentials's parameter 'password' was class 'int' but " + "should be of class 'str'.") + + +class TestPluginRuntimeError: + @staticmethod + def test_plugin_runtime_error_get_actual_and_expected_type(): + actual, expected = PluginRuntimeError.get_actual_and_expected_type( + list([str]), dict({str: dict})) + if six.PY2: + assert actual == "a list of [type 'str']" + else: + assert actual == "a list of [class 'str']" + assert expected == "type 'dict of str:dict'" + + @staticmethod + def test_plugin_runtime_error_get_actual_and_expected_type_multi_expected_types(): + actual, expected = PluginRuntimeError.get_actual_and_expected_type( + list([str]), list([str, int, dict, bool])) + if six.PY2: + assert actual == "a list of [type 'str']" + else: + assert actual == "a list of [class 'str']" + assert expected == ( + "any one of the following types: '['str', 'int', 'dict', 'bool']'") + + @staticmethod + def test_plugin_runtime_error_get_actual_and_expected_type_single_item_list_expected_types(): # noqa + actual, expected = PluginRuntimeError.get_actual_and_expected_type( + list([str]), list([str])) + if six.PY2: + assert actual == "a list of [type 'str']" + else: + assert actual == "a list of [class 'str']" + assert expected == "type 'list of str'" + + @staticmethod + def test_plugin_runtime_error_get_actual_and_expected_type_empty_dict_expected_type(): # noqa + with pytest.raises(PlatformError) as err_info: + PluginRuntimeError.get_actual_and_expected_type(list(), dict()) + + assert err_info.value.message == ( + "The thrown TypeError should have had a dict of size 1 as the " + "expected_type") + + @staticmethod + def test_plugin_runtime_error_get_actual_and_expected_type_empty_list_expected_type(): # noqa + with pytest.raises(PlatformError) as err_info: + PluginRuntimeError.get_actual_and_expected_type(list(), list()) + + assert err_info.value.message == ( + "The thrown TypeError should have had a list of size >= 1 as the " + "expected_type") + + @staticmethod + def test_plugin_runtime_error_get_actual_and_expected_type_list_with_duplicate_expected_type(): # noqa + with pytest.raises(PlatformError) as err_info: + PluginRuntimeError.get_actual_and_expected_type(list(), list([str, str])) + assert err_info.value.message == ( - "RemoteUser's parameter 'user' was" - " type 'str' but should be of class 'dlpx.virtualization.api" - ".common_pb2.RemoteUser'.") + "The thrown TypeError should have had a list of size 1 as the " + "expected_type") diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index 9db47fd6..1e44ce46 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -70,7 +70,7 @@ We have two goals: Provide the best documentation we can for our customers, and 1. Learn Markdown or use a really good IDE. It's easy to use, but there are complex topics like tables, admonishments, links, and images that you may need some practice with. Look at the other docs in the repo for inspiration and tutelage. 2. Test everything in mkdocs locally. Best practice is to always have mkdocs running in one terminal tab. It auto-refreshes when you make changes, so you can make sure that nothing breaks, and that your content looks good. -3. Do not create new directories (nav categories) in /docs without working with Jaspal Sumal (jaspal.sumal@delphix.com) +3. Do not create new directories (nav categories) in /docs without working with Ryan Fowler (ryan.fowler@delphix.com) 4. Place all screenshots in the local media/ directory of the category you're editing in. For example, if you're editing a page in docs/Getting_Started, put any screenshots you're going to use in docs/Getting_Started/media 5. Use relative links to reference screenshots (./media/image.png) and other pages (../Getting_Started/pagename/) 6. Beware the .pages file. .pages is a hidden file in every folder that provides page order. Any pages not listed in .pages will be alphabetically ordered _after_ the pages that have been listed. If you have a typo in this file or specify a renamed/deleted page, it will break mkdocs. @@ -84,16 +84,16 @@ We have two goals: Provide the best documentation we can for our customers, and 1. The diff can usually provide what you need for reviewing changes. However, use mkdocs to review locally whenever possible to ensure good formatting and no breaks to mkdocs. 2. For minor corrections, leave a general comment on the review and vote to ship it so the author can fix it and push. -3. For major docs projects (e.g. whole new sections of docs or large batches of changes), coordinate with Jas. It is possible we'd be better off using another approach to review (e.g. track notes via Google Sheets) -4. If you're a reviewer that is not hooked into reviewboard, and unable to get set up to use it, work with Jas on an alternative approach (e.g. track notes via Google Sheets) +3. For major docs projects (e.g. whole new sections of docs or large batches of changes), coordinate with Ryan. It is possible we'd be better off using another approach to review (e.g. track notes via Google Sheets) +4. If you're a reviewer that is not hooked into reviewboard, and unable to get set up to use it, work with Ryan on an alternative approach (e.g. track notes via Google Sheets) 5. If there are issues in production docs, the current procedure is to post the issue in the #docs channel. ## Publishing Publishing is currently a manual process that will be automated into the release process at a future point in time. The publishing workflow follows these steps: -1. After the git repo is frozen, Jas begins review and adjustments. -2. If there are technical questions or issues, Jas will take back to engineering for review. +1. After the git repo is frozen, Ryan begins review and adjustments. +2. If there are technical questions or issues, Ryan will take back to engineering for review. 3. The publish process will run. This process will: * Pull the appropriate branch to a build machine * Run "mkdocs build clean" to compile documentation to HTML diff --git a/docs/docs/Best_Practices/.pages b/docs/docs/Best_Practices/.pages index 9a05e0d3..ceea0305 100644 --- a/docs/docs/Best_Practices/.pages +++ b/docs/docs/Best_Practices/.pages @@ -4,6 +4,8 @@ arrange: - Managing_Scripts_For_Remote_Execution.md - User_Visible_Errors.md - Sensitive_Data.md - - Unicode_Data.md + - Strings.md + - Runtime_Environment.md - Working_with_Powershell.md - Scratch_Paths.md + - Message_Limits.md diff --git a/docs/docs/Best_Practices/Code_Sharing.md b/docs/docs/Best_Practices/Code_Sharing.md index 5640e6a2..1dafe9bb 100644 --- a/docs/docs/Best_Practices/Code_Sharing.md +++ b/docs/docs/Best_Practices/Code_Sharing.md @@ -2,7 +2,7 @@ All Python modules inside of `srcDir` can be imported just as they would be if the plugin was executing locally. When a plugin operation is executed `srcDir` is the current working directory so all imports need to be relative to `srcDir` regardless of the path of the module doing the import. -Please refer to Python's [documentation on modules](https://docs.python.org/2/tutorial/modules.html#modules) to learn more about modules and imports. +Please refer to Python's [documentation on modules](https://docs.python.org/3.8/tutorial/modules.html#modules) to learn more about modules and imports. ## Example @@ -30,10 +30,12 @@ postgres Any module in the plugin could import `execution_util.py` with `from utils import execution_util`. !!! warning "Gotcha" - Since the platform uses Python 2.7, every directory needs to have an `__init__.py` file in it otherwise the modules and resources in the folder will not be found at runtime. For more information on `__init__.py` files refer to Python's [documentation on packages](https://docs.python.org/2/tutorial/modules.html#packages). - + When using a vSDK version that was built on Python 2.7, every directory needs to have an `__init__.py` file in it otherwise the modules and resources in the folder will not be found at runtime. For more information on `__init__.py` files refer to Python's [documentation on packages](https://docs.python.org/2/tutorial/modules.html#packages). + Note that the `srcDir` in the plugin config file (`src` in this example) does _not_ need an `__init__.py` file. + For information on which vSDK versions run on Python 2.7, visit the [Version Compatibility Page](/References/Version_Compatibility/#virtualization-sdk-and-python-compatibility-map). + Assume `schema.json` contains: ``` @@ -117,21 +119,21 @@ def find_schemas(source_connection, repository): return [SourceConfigDefinition(name=name) for name in schema_names] ``` !!! note - Even though `discovery.py` is in the `operations` package, the import for `execution_util` is still relative to the `srcDir` specified in the plugin config file. `execution_util` is in the `utils` package so it is imported with `from utils import execution_util`. - + Even though `discovery.py` is in the `operations` package, the import for `execution_util` is still relative to the `srcDir` specified in the plugin config file. `execution_util` is in the `utils` package so it is imported with `from utils import execution_util`. + ### execution_util.py `execution_util.py ` has two methods `execute_sql` and `execute_shell`. `execute_sql` takes the name of a SQL script in `resources/` and executes it with `resources/execute_sql.sh`. `execute_shell` takes the name of a shell script in `resources/` and executes it. ```python -import pkgutil +from importlib import resources from dlpx.virtualization import libs def execute_sql(source_connection, install_name, script_name): - psql_script = pkgutil.get_data("resources", "execute_sql.sh") - sql_script = pkgutil.get_data("resources", script_name) + psql_script = resources.read_text("resources", "execute_sql.sh") + sql_script = resources.read_text("resources", script_name) result = libs.run_bash( source_connection, psql_script, variables={"SCRIPT": sql_script}, check=True @@ -140,11 +142,16 @@ def execute_sql(source_connection, install_name, script_name): def execute_shell(source_connection, script_name): - script = pkgutil.get_data("resources", script_name) + script = resources.read_text("resources", script_name) result = libs.run_bash(source_connection, script, check=True) return result.stdout ``` +!!! warning + If developing a plugin in Python 2.7, you will need to use `pkgutil.get_data` rather than `importlib.resources.read_text`. + + See [Managing Scripts For Remote Execution](/Best_Practices/Managing_Scripts_For_Remote_Execution.md) for more info. + !!! note - Both `execute_sql` and `execute_shell` use the `check` parameter which will cause an error to be raised if the exit code is non-zero. For more information refer to the `run_bash` [documentation](/References/Platform_Libraries.md#run_bash). \ No newline at end of file + Both `execute_sql` and `execute_shell` use the `check` parameter which will cause an error to be raised if the exit code is non-zero. For more information refer to the `run_bash` [documentation](/References/Platform_Libraries.md#run_bash). diff --git a/docs/docs/Best_Practices/Managing_Scripts_For_Remote_Execution.md b/docs/docs/Best_Practices/Managing_Scripts_For_Remote_Execution.md index ef63e26c..853f0f56 100644 --- a/docs/docs/Best_Practices/Managing_Scripts_For_Remote_Execution.md +++ b/docs/docs/Best_Practices/Managing_Scripts_For_Remote_Execution.md @@ -1,9 +1,11 @@ # Managing Scripts for Remote Execution -To execute a PowerShell or Bash script or Expect script on a remote host, you must provide the script as a string to `run_powershell` or `run_bash` or `run_expect`. While you can keep these strings as literals in your Python code, best practice is to keep them as resource files in your source directory and access them with `pkgutil`. +To execute a PowerShell or Bash script or Expect script on a remote host, you must provide the script as a string to `run_powershell` or `run_bash` or `run_expect`. While you can keep these strings as literals in your Python code, best practice is to keep them as resource files in your source directory and access them with `pkgutil` or `importlib`, depending on your plugin language. [pkgutil](https://docs.python.org/2/library/pkgutil.html) is part of the standard Python library. The method that is applicable to resources is [pkgutil.get_data](https://docs.python.org/2/library/pkgutil.html#pkgutil.get_data). +When developing a plugin in Python3, it is instead suggested to use the newer `importlib.resources`. This package is part of the standard Python 3 library. The method that is applicable to resources is [resources.read_text](https://docs.python.org/3.8/library/importlib.html#importlib.resources.read_text), which accepts the same arguments as `pkgutil.get_data`. + ### Basic Usage Given the following plugin structure: @@ -66,7 +68,7 @@ def post_snapshot(direct_source, repository, source_config): raise UserError( 'Failed to get date', 'Make sure the user has the required permissions', - '{}\n{}'.format(response.stdout, rsponse.stderr)) + '{}\n{}'.format(response.stdout, response.stderr)) return SnapshotDefinition(name='Snapshot', date=response.stdout) ``` @@ -75,7 +77,9 @@ def post_snapshot(direct_source, repository, source_config): This assumes that `src/` is Python's current working directory. This is the behavior of the Virtualization Platform. !!! warning "Resources need to be in a Python module" - `pkgutil.get_data` cannot retrieve the contents of a resource that is not in a Python package. This means that a resource that is in the first level of your source directory will not be retrievable with `pkgutil`. Resources must be in a subdirectory of your source directory, and that subdirectory must contain an `__init__.py` file. + `pkgutil.get_data` cannot retrieve the contents of a resource that is not in a Python package. When developing with Python 2.7, this means that a resource that is in the first level of your source directory will not be retrievable with `pkgutil`. Resources must be in a subdirectory of your source directory, and that subdirectory must contain an `__init__.py` file. + + Python 3 does not have this requirement. ### Multi-level Packages @@ -101,3 +105,9 @@ The contents of `src/resources/platform/get_date.sh` can be retrieved with: ```python script_content = pkgutil.get_data('resources.platform', 'get_date.sh') ``` + +In a Python 3.8 plugin, the suggested approach is: + +```python +script_content = resources.read_text('resources.platform', 'get_date.sh') +``` diff --git a/docs/docs/Best_Practices/Message_Limits.md b/docs/docs/Best_Practices/Message_Limits.md new file mode 100644 index 00000000..0389b1c5 --- /dev/null +++ b/docs/docs/Best_Practices/Message_Limits.md @@ -0,0 +1,44 @@ +# Message Limits + +There are limits on how much data can be sent back and forth between the plugin and engine at a time. There are five scenarios where this comes into play: + +1. Inputs sent from the engine to the plugin, as arguments to a [Plugin Operation](/References/Plugin_Operations.md). For example, the schema-defined `Repository` object that is provided as input to plugin operations. + +2. Outputs sent back from the plugin to the engine, as the return values from plugin operations. + +3. Exception messages and call stacks thrown by plugin code. For example, the `message` field within [User Visible Errors](/Best_Practices/User_Visible_Errors.md). + +4. Inputs sent from the plugin to the engine, as arguments to a [Platform library](/References/Platform_Libraries.md) function. For example, the `message` field that is passed to `logger.debug`. + +5. Outputs sent back from the engine to the plugin, as the return values from Platform Library functions. For example, the `stdout` resulting from a call to `libs.run_bash`. + +For case 1 and 2, the total size of data must be less than 4 mebibytes (4 MiB). + +For case 3, the total size of data must be less than 128 kibibytes (128 KiB). + +For case 4 and 5, the total size of data must be less than 192 mebibytes (192 MiB). + +The actual size of this information at runtime is dependent on how the Python interpreter chooses to represent the information, so it's not always possible to know ahead of time what the exact size will be. + +Here are some examples of where problems may occur: + +1. Using `libs.run_bash` to print the entire contents of a large file to stdout. + +2. Using a single `logger` command with many pages of output. + +3. Throwing an exception with a large message or stack trace. + +4. Large amount of metadata in a plugin defined schema like `Repository` or `Virtual Source`. + +## How to tell if the message size was exceeded + +The plugin operation or platform library callback will fail with a RPC error. The exception will look like: + +``` +Error +Discovery of "my_plugin" failed: Plugin operation "Repository Discovery" got a RPC error for plugin "my_plugin". UNAVAILABLE: Network closed for unknown reason +``` + +## What to do if the maximum metadata or message size is exceeded + +Reach out to us via the [Virtualization SDK GitHub repository](https://github.com/delphix/virtualization-sdk/) for guidance. diff --git a/docs/docs/Best_Practices/Runtime_Environment.md b/docs/docs/Best_Practices/Runtime_Environment.md new file mode 100644 index 00000000..aadaf9cf --- /dev/null +++ b/docs/docs/Best_Practices/Runtime_Environment.md @@ -0,0 +1,63 @@ +# Plugin Runtime Environment + +## Process Lifetime +Plugin code runs inside of a Python interpreter process on the Delphix Engine. + +A fair question to ask is "What is the lifetime of this interpreter process?" After all, if the interpreter +process runs for a long time, then the plugin might be able to store things in memory for later access. + +Unfortunately, **there are no guarantees about process lifetime**. Your interpreter process could last two years, or it could last 400 microseconds. There is no way to know or predict this ahead of time. + +So, do not make any assumptions about interpreter process lifetime in your plugin code. + + +## Available Modules +Our Python 2.7 runtime environment only contains the [Python Standard Library](https://docs.python.org/2/library/). No additional Python modules/libraries are available. + +If you want to use some Python module that is not part of the standard library, you might be able to do so. +You would need to include that library as part of your plugin. That would involve downloading the source +code for that module, and copying it into your source directory. For more information on how to lay out code in your source directory, see [Code Sharing](/Best_Practices/Code_Sharing.md). + +### Warnings +There are two major things to watch out for if you decide to incorporate a 3rd-party library. + +1) Make sure you're legally allowed to do so! The licensing agreement on the module will decide if, and +under what circumstances, you're allowed to make copies of, and redistribute the module. Some modules will +allow this, some will disallow this, and some will allow this for a fee. + +2) Some Python libraries include native code (often written in C or C++). There is no support for using +such libraries with plugin code. The reason for this is that native code needs to be +specially compiled and built for the machine that it the library will be running on. And, unfortunately, +the machine your plugin runs on (the Delphix Engine) is likely very different from the machine you use +to develop and build your plugin. + +## Network Access +As of Delphix Engine version 6.0.11.0, plugin code is able to use the network directly. No network access is +possible in earlier versions. + +For example, suppose your plugin wants to talk to some DBMS running on some remote host. +If the DBMS supports it, your plugin code might be able to connect to the DBMS server and talk to the +DBMS directly. This can avoid the need to do DBMS operations via running Bash/Powershell code on the remote host. + + +### Example +```python +import httplib +import json + +dbms_port = 5432 + +# Directly contact our DBMS's REST server to get a list of databases +def list_databases(remote_ip): + cx = httplib.HTTPConnection(remote_ip, dbms_port) + cx.request("GET", "/databases") + response = cx.getresponse() + return json.loads(response.read()) +``` + +What your plugin can access depends entirely on the customer. Some customers will set up their Delphix +Engines such that plugins have full access to the entire internet. Some will completely restrict the network +so that the plugin can only access a small handful of remote hosts. + +If your plugin has any specific network requirements, it's recommended to try, in your code, to confirm that these requirements are met. For example, the plugin could make such a check in the +`discovery.repository()` operation, and throw an error if the check fails. Like any other requirement, this should of course be documented. diff --git a/docs/docs/Best_Practices/Strings.md b/docs/docs/Best_Practices/Strings.md new file mode 100644 index 00000000..1dfb5f5c --- /dev/null +++ b/docs/docs/Best_Practices/Strings.md @@ -0,0 +1,90 @@ +# Working With Strings + +Unfortunately, Python 2.7 makes it very easy to accidentally write string-related code that will sometimes work, but sometimes fail (especially for people who are not using English). Read on for some tips for how to avoid this. + +## The Two String Types +Python 2.7 has two different types that are both called "strings". One represents +a sequence of **bytes**, and the other represents a sequence of **characters**. + +```python +# The default string (aka 'str object') represents bytes +my_bytes = "This string is a sequence of bytes" + +# A 'Unicode object' represents characters (note the u just before the quote) +my_characters = u"This string is a sequence of characters" +``` + +## Unicode Strings Are Preferred + +There are a couple of reasons to prefer the "unicode object" over the "str object". + +First, in most cases, we care about characters, and we're not particularly interested in which bytes +are used to represent those characters. That is, we might care that we have a "letter H" followed by a "letter I", but it's usually irrelevant to us what byte values happen to be used. + +Second, there are lots of different schemes available which give rules for how to represent characters as bytes. These schemes are called "encodings"; some examples include "ASCII", "UTF-8", "Shift-JIS", and "UCS-2". Each encoding uses different rules about which characters are represented by which bytes. + +A "str object" doesn't know anything about encodings... it is just a sequence of bytes. So, when a programmer is working with one of these byte strings, they have to know which encoding rules are in play. + +In order to avoid problems, **we recommend using Unicode strings everywhere** in your plugin code. + +## Delphix I/O + +Your plugin will sometimes need to send strings back and forth to Delphix code. There are two supported formats for doing this. Any time you receive a string from Delphix, it will be in one of the two following forms. This includes arguments to your plugin operations, and return values from "Delphix Libs" functions. Likewise, any time you send a string to Delphix, it must be in one of these two forms. + +Acceptable forms: + +1. A Unicode string (recommended) +2. A "str object" (byte string) that uses the UTF-8 encoding + +## Converting Between Types + +Sometimes (hopefully rarely!), you might find yourself needing to convert back and forth between byte strings and character strings. For example, you might need to read or write a file on a remote system that is required to use some specific encoding. Here's how to do that: + +```python +# Converting from a character string ("unicode") to a byte string ("str") +my_utf8_byte_string = my_character_string.encode("utf-8") +my_utf16_byte_string = my_character_string.encode("utf-16") + +# Converting from a byte string to a character string +my_character_string1 = my_utf8_byte_string.decode("utf-8") +my_character_string2 = my_utf16_byte_string.decode("utf-16") +my_character_string3 = my_ascii_byte_string.decode("ascii") +``` + +Things to note: + +- `encode` goes from characters to bytes. `decode` goes from bytes to characters. +- If you try to `encode` a character string using the `ascii` encoding, but your character string contains non-ascii characters, you'll get an error. More generally: some encodings will error out with some characters. +- If you don't specify an encoding, Python will supply a default. But, there's a good chance the default will be wrong for your use case. So, always specify the encoding! +- Don't try to `encode` a byte string. If you do this, Python will "helpfully" insert an implicit `decode` first, which tends to cause very confusing error messages. Likewise, don't try to `decode` a character string. +- `utf-8` is likely the best encoding to use for most situations. It accepts all characters, does not have issues with byte ordering, and is understood by most systems. This is not true of most other encodings. + +## Using Non-ASCII characters in Python files + +Python 2.7 source code files are assumed to use the "ASCII" encoding, unless told otherwise. Unfortunately, ASCII is an obsolete encoding that only knows how to deal with a small number of characters, and only really supports American English. + +In order to include non-ASCII characters in your source code, you need to use a different encoding than ASCII, and you need to tell the Python interpreter which encoding you're using. In Python 2.7, this is done with a "magic" comment at the very top of each file. + +Here is an example of the first line of a Python file that uses the UTF-8 encoding: +```python +# -*- coding: utf-8 -*- +``` + +If you do not specify an encoding, and the source code contains any non-ASCII characters, you will get errors + when building the plugin using [dvp build](/References/CLI.md#build) or during the execution of a plugin operation. + +### Example + +```python +# -*- coding: utf-8 -*- +from dlpx.virtualization.platform import Plugin +from dlpx.virtualization import libs +from generated.definitions import RepositoryDefinition + +plugin = Plugin() + +@plugin.discovery.repository() +def repository_discovery(source_connection): + # Create a repository with name that uses non-ASCII characters + return [RepositoryDefinition(name=u"Théâtre")] +``` diff --git a/docs/docs/Best_Practices/Unicode_Data.md b/docs/docs/Best_Practices/Unicode_Data.md deleted file mode 100644 index 06b6f88a..00000000 --- a/docs/docs/Best_Practices/Unicode_Data.md +++ /dev/null @@ -1,29 +0,0 @@ -# Working with Unicode Data - -To use unicode characters in the plugin code, the following lines should be included at top of the plugin code: - -```python -#!/usr/bin/env python -# -*- coding: utf-8 -*- -``` - -Otherwise, there may be errors when building the plugin using [dvp build](/References/CLI.md#build) or during the execution of a plugin operation. - -## Example - -```python -#!/usr/bin/env python -# -*- coding: utf-8 -*- -from dlpx.virtualization.platform import Plugin -from dlpx.virtualization import libs -from generated.definitions import RepositoryDefinition - -plugin = Plugin() - -@plugin.discovery.repository() -def repository_discovery(source_connection): - # Create a repository with name ☃ - command = 'echo ☃' - result = libs.run_bash(source_connection, command) - return [RepositoryDefinition(name=result.stdout)] -``` \ No newline at end of file diff --git a/docs/docs/Best_Practices/User_Visible_Errors.md b/docs/docs/Best_Practices/User_Visible_Errors.md index db3476b5..e8ce3a41 100644 --- a/docs/docs/Best_Practices/User_Visible_Errors.md +++ b/docs/docs/Best_Practices/User_Visible_Errors.md @@ -10,11 +10,13 @@ message | String | Description of the failure to show the end user. action | String | **Optional**. List of actions that the end user could take to fix the problem. If not provided, it defaults to `Contact the plugin author to correct the error.` output | String | **Optional**. Output or stack trace from the failure to give the end user more information so that they can self diagnose. If not provided, it defaults to the stack trace of the failure. +!!! warning + There is a limit to how much data can be stored within the fields of a `UserError`. See [Message Limits](/Best_Practices/Message_Limits.md) for details. ## Example ```python -import pkgutil +from importlib import resources from dlpx.virtualization.platform import Plugin from generated.definitions import SourceConfigDefinition from dlpx.virtualization.platform.exceptions import UserError @@ -23,7 +25,7 @@ plugin = Plugin() @plugin.virtual.start() def start(virtual_source, repository, source_config): - script_content = pkgutil.get_data('resources', 'start_database.sh') + script_content = resources.read_text('resources', 'start_database.sh') response = libs.run_bash(virtual_source.connection, script_content) @@ -35,6 +37,11 @@ def start(virtual_source, repository, source_config): '{}\n{}'.format(response.stdout, response.stderr)) ``` +!!! warning + If developing a plugin in Python 2.7, you will need to use `pkgutil.get_data` rather than `importlib.resources.read_text`. + + See [Managing Scripts For Remote Execution](/Best_Practices/Managing_Scripts_For_Remote_Execution.md) for more info. + The UI would show the end user if the plugin operation above fails: ![Screenshot](images/UserError_Start.png) \ No newline at end of file diff --git a/docs/docs/Building_Your_First_Plugin/Initial_Setup.md b/docs/docs/Building_Your_First_Plugin/Initial_Setup.md index d2b0424e..f3bcaa19 100644 --- a/docs/docs/Building_Your_First_Plugin/Initial_Setup.md +++ b/docs/docs/Building_Your_First_Plugin/Initial_Setup.md @@ -53,7 +53,7 @@ To start, we will create a new directory where our new plugin code will live. Now that we are in our new plugin directory, we can use the `dvp` tool to create a plugin for us. This plugin will be a mere skeleton -- it will not do anything useful until we modify it in the subsequent pages. ``` -(venv) first_plugin$ dvp init -n first_plugin -s STAGED -t WINDOWS +(venv) first_plugin$ dvp init -n first_plugin -s STAGED -t UNIX ``` The `-n` argument here means "plugin name." We are using the name `first_plugin`. @@ -111,4 +111,4 @@ You will be prompted for a password. Once the upload is finished, you can verify the installation from the Manage > Toolkits screen in the Delphix Engine UI. -![Screenshot](images/PostUpload.png) \ No newline at end of file +![Screenshot](images/PostUpload.png) diff --git a/docs/docs/Getting_Started.md b/docs/docs/Getting_Started.md index 83d08838..ff884118 100644 --- a/docs/docs/Getting_Started.md +++ b/docs/docs/Getting_Started.md @@ -12,9 +12,14 @@ The platform and libs modules expose objects and methods needed to develop a plu ## Requirements - macOS 10.14+, Ubuntu 16.04+, or Windows 10 -- Python 2.7 (Python 3 is not supported) +- Python 2.7 (vSDK 3.1.0 and earlier) +- Python 3.8 (vSDK 4.0.0 and later) - Java 7+ -- Delphix Engine 6.0.3.0 or above +- A Delphix Engine of an [appropriate version](/References/Version_Compatibility.md) +- An active internet connection to download packages from [PyPI](https://pypi.org/) + +!!! tip "Use proxy server" + Pip recommends setting up a proxy server in case of restricted internet access. Please follow the [guidelines](https://pip.pypa.io/en/stable/user_guide/#using-a-proxy-server) from Pip on how to set up a proxy server. ## Installation To install the latest version of the SDK run: @@ -26,9 +31,13 @@ $ pip install dvp !!! tip "Use a Virtual Environment" We highly recommended that you develop plugins inside of a virtual environment. To learn more about virtual environments, refer to [Virtualenv's documentation](https://virtualenv.pypa.io/en/latest/). - The virtual environment needs to use Python 2.7. This is configured when creating the virtualenv: + If using vSDK 3.1.0 or earlier, the virtual environment needs to use Python 2.7. + + If using vSDK 4.0.0 or earlier, the virtual environment needs to use Python 3.8. + + This is configured when creating the virtualenv: - ```$ virtualenv -p /path/to/python2.7/binary ENV``` + ```$ virtualenv -p /path/to/python2.7/binary ENV``` or ```$ virtualenv -p /path/to/python3.8/binary ENV``` To install a specific version of the SDK run: diff --git a/docs/docs/References/Decorators.md b/docs/docs/References/Decorators.md index 038cf01c..4abb7223 100644 --- a/docs/docs/References/Decorators.md +++ b/docs/docs/References/Decorators.md @@ -13,7 +13,7 @@ plugin = Plugin() # Use the decorator to annotate the function that corresponds to the "Virtual Source Start" Plugin Operation @plugin.virtual_source.start() def my_start(virtual_source, repository, source_config): - print "running start" + print("running start") ``` !!! info @@ -38,6 +38,7 @@ Plugin Operation | Decorator [Virtual Source Initialize](Plugin_Operations.md#virtual-source-initialize) | `@plugin.virtual.initialize()` [Virtual Source Unconfigure](Plugin_Operations.md#virtual-source-unconfigure) | `@plugin.virtual.unconfigure()` [Virtual Source Reconfigure](Plugin_Operations.md#virtual-source-reconfigure) | `@plugin.virtual.reconfigure()` +[Virtual Source Cleanup](Plugin_Operations.md#virtual-source-cleanup) | `@plugin.virtual.cleanup()` [Virtual Source Start](Plugin_Operations.md#virtual-source-start) | `@plugin.virtual.start()` [Virtual Source Stop](Plugin_Operations.md#virtual-source-stop) | `@plugin.virtual.stop()` [VirtualSource Pre-Snapshot](Plugin_Operations.md#virtualsource-pre-snapshot) | `@plugin.virtual.pre_snapshot()` diff --git a/docs/docs/References/Logging.md b/docs/docs/References/Logging.md index d4e09439..180833b7 100644 --- a/docs/docs/References/Logging.md +++ b/docs/docs/References/Logging.md @@ -6,10 +6,10 @@ The Virtualization Platform keeps plugin-specific log files. A plugin can, at an ## Overview -The Virtualization Platform integrates with Python's built-in [logging framework](https://docs.python.org/2/library/logging.html). A special [Handler](https://docs.python.org/2/library/logging.html#handler-objects) is exposed by the platform at `dlpx.virtualization.libs.PlatformHandler`. This handler needs to be added to the Python logger your plugin creates. Logging statements made through Python's logging framework will then be routed to the platform. +The Virtualization Platform integrates with Python's built-in [logging framework](https://docs.python.org/3.8/library/logging.html). A special [Handler](https://docs.python.org/3.8/library/logging.html#handler-objects) is exposed by the platform at `dlpx.virtualization.libs.PlatformHandler`. This handler needs to be added to the Python logger your plugin creates. Logging statements made through Python's logging framework will then be routed to the platform. ## Basic Setup - Below is the absolute minimum needed to setup logging for the platform. Please refer to Python's [logging documentation](https://docs.python.org/2/library/logging.html) and the [example below](#customized-example) to better understand how it can be customized. + Below is the absolute minimum needed to setup logging for the platform. Please refer to Python's [logging documentation](https://docs.python.org/3.8/library/logging.html) and the [example below](#customized-example) to better understand how it can be customized. ```python import logging @@ -36,9 +36,11 @@ logger.setLevel(logging.DEBUG) To avoid this complexity, add the `PlatformHandler` to the root logger. The root logger can be retrieved with `logging.getLogger()`. +!!! warning + There is a limit to how much data can be stored within a log message. See [Message Limits](/Best_Practices/Message_Limits.md) for details. ## Usage -Once the `PlatformHandler` has been added to the logger, logging is done with Python's [Logger](https://docs.python.org/2/library/logging.html#logger-objects) object. Below is a simple example including the basic setup code used above: +Once the `PlatformHandler` has been added to the logger, logging is done with Python's [Logger](https://docs.python.org/3.8/library/logging.html#logger-objects) object. Below is a simple example including the basic setup code used above: ```python import logging @@ -66,7 +68,7 @@ Imagine you notice that your plugin is taking a very long time to do discovery. Suppose your plugin has a source config discovery operation that looks like this (code is abbreviated to be easier to follow): ```python -import pkgutil +from importlib import resources from dlpx.virtualization import libs from dlpx.virtualization.platform import Plugin @@ -81,10 +83,10 @@ def repository_discovery(source_connection): @plugin.discovery.source_config() def source_config_discovery(source_connection, repository): - version_result = libs.run_bash(source_connection, pkgutil.get_data('resources', 'get_db_version.sh')) - users_result = libs.run_bash(source_connection, pkgutil.get_data('resources', 'get_db_users.sh')) - db_results = libs.run_bash(source_connection, pkgutil.get_data('resources', 'get_databases.sh')) - status_result = libs.run_bash(source_connection, pkgutil.get_data('resources', 'get_database_statuses.sh')) + version_result = libs.run_bash(source_connection, resources.read_text('resources', 'get_db_version.sh')) + users_result = libs.run_bash(source_connection, resources.read_text('resources', 'get_db_users.sh')) + db_results = libs.run_bash(source_connection, resources.read_text('resources', 'get_databases.sh')) + status_result = libs.run_bash(source_connection, resources.read_text('resources', 'get_database_statuses.sh')) # Return an empty list for simplicity. In reality # something would be done with the results above. @@ -92,10 +94,15 @@ def source_config_discovery(source_connection, repository): ``` +!!! warning + If developing a plugin in Python 2.7, you will need to use `pkgutil.get_data` rather than `importlib.resources.read_text`. + + See [Managing Scripts For Remote Execution](/Best_Practices/Managing_Scripts_For_Remote_Execution.md) for more info. + Now, imagine that you notice that it's taking a long time to do discovery, and you'd like to try to figure out why. One thing that might help is to add logging, like this: ```python import logging -import pkgutil +from importlib import resources from dlpx.virtualization import libs from dlpx.virtualization.platform import Plugin @@ -141,13 +148,13 @@ def repository_discovery(source_connection): @plugin.discovery.source_config() def source_config_discovery(source_connection, repository): logger.debug('About to get DB version') - version_result = libs.run_bash(source_connection, pkgutil.get_data('resources', 'get_db_version.sh')) + version_result = libs.run_bash(source_connection, resources.read_text('resources', 'get_db_version.sh')) logger.debug('About to get DB users') - users_result = libs.run_bash(source_connection, pkgutil.get_data('resources', 'get_db_users.sh')) + users_result = libs.run_bash(source_connection, resources.read_text('resources', 'get_db_users.sh')) logger.debug('About to get databases') - db_results = libs.run_bash(source_connection, pkgutil.get_data('resources', 'get_databases.sh')) + db_results = libs.run_bash(source_connection, resources.read_text('resources', 'get_databases.sh')) logger.debug('About to get DB statuses') - status_result = libs.run_bash(source_connection, pkgutil.get_data('resources', 'get_database_statuses.sh')) + status_result = libs.run_bash(source_connection, resources.read_text('resources', 'get_database_statuses.sh')) logger.debug('Done collecting data') # Return an empty list for simplicity. In reality @@ -174,7 +181,7 @@ Download a support bundle by going to **Help** > **Support Logs** and select ** ## Logging Levels -Python has a number of [preset logging levels](https://docs.python.org/2/library/logging.html#logging-levels) and allows for custom ones as well. Since logging on the Virtualization Platform uses the `logging` framework, log statements of all levels are supported. +Python has a number of [preset logging levels](https://docs.python.org/3.8/library/logging.html#logging-levels) and allows for custom ones as well. Since logging on the Virtualization Platform uses the `logging` framework, log statements of all levels are supported. However, the Virtualization Platform will map all logging levels into three files: `debug.log`, `info.log`, and `error.log` in the following way: diff --git a/docs/docs/References/Platform_Libraries.md b/docs/docs/References/Platform_Libraries.md index 39301549..82392d48 100644 --- a/docs/docs/References/Platform_Libraries.md +++ b/docs/docs/References/Platform_Libraries.md @@ -1,5 +1,5 @@ # Platform Libraries -Set of functions that plugins can use these for executing remote commands, etc. +Delphix provides a set of functions that plugins can use for executing remote commands, etc. ## retrieve_credentials @@ -56,7 +56,7 @@ Argument | Type | Description -------- | ---- | ----------- remote_connection | [RemoteConnection](Classes.md#remoteconnection) | Connection associated with the remote host to run the command on. command | String | Command to run on the host. -variables | dict[String, String] | **Optional**. Environement variables to set when running the command. +variables | dict[String, String] | **Optional**. Environment variables to set when running the command. use_login_shell | boolean | **Optional**. Whether to use a login shell. check | boolean | **Optional**. Whether or not to raise an exception if the `exit_code` in the `RunBashResponse` is non-zero. @@ -71,7 +71,7 @@ stderr | String | Stderr from the command. ### Examples -Calling bash with an inline command. +##### Calling bash with an inline command. ```python from dlpx.virtualization import libs @@ -86,7 +86,7 @@ print response.stdout print response.stderr ``` -Using parameters to construct a bash command. +##### Using parameters to construct a bash command. ```python from dlpx.virtualization import libs @@ -98,10 +98,11 @@ command = "mysqldump -u {} -p {}".format(name,port) response = libs.run_bash(connection, command) ``` -Running a bash script that is saved in a directory. +##### Running a bash script that is saved in a directory. +###### Python 2.7 recommended approach ```python - + import pkgutil from dlpx.virtualization import libs @@ -110,6 +111,17 @@ Running a bash script that is saved in a directory. # Execute script on remote host response = libs.run_bash(direct_source.connection, script_content) ``` +###### Python 3.8 recommended approach +```python + + from importlib import resources + from dlpx.virtualization import libs + + script_content = resources.read_text('resources', 'get_date.sh') + + # Execute script on remote host + response = libs.run_bash(direct_source.connection, script_content) +``` For more information please go to [Managing Scripts for Remote Execution](/Best_Practices/Managing_Scripts_For_Remote_Execution.md) section. ## run_expect @@ -126,7 +138,7 @@ Argument | Type | Description -------- | ---- | ----------- remote_connection | [RemoteConnection](Classes.md#remoteconnection) | Connection associated with the remote host to run the command on. command | String | Expect(Tcl) command to run. -variables | dict[String, String] | **Optional**. Environement variables to set when running the command. +variables | dict[String, String] | **Optional**. Environment variables to set when running the command. ### Returns An object of `RunExpectResponse` @@ -168,7 +180,7 @@ Argument | Type | Description -------- | ---- | ----------- remote_connection | [RemoteConnection](Classes.md#remoteconnection) | Connection associated with the remote host to run the command on. command | String | Command to run to the remote host. -variables | dict[String, String] | **Optional**. Environement variables to set when running the command. +variables | dict[String, String] | **Optional**. Environment variables to set when running the command. check | boolean | **Optional**. Whether or not to raise an exception if the `exit_code` in the `RunPowershellResponse` is non-zero. ### Returns diff --git a/docs/docs/References/Plugin_Config.md b/docs/docs/References/Plugin_Config.md index 1626d6f8..0351ff78 100644 --- a/docs/docs/References/Plugin_Config.md +++ b/docs/docs/References/Plugin_Config.md @@ -17,7 +17,7 @@ The name of the file can be specified during the build. By default, the build lo |entryPoint|Y|string|A fully qualified Python symbol that points to the `dlpx.virtualization.platform.Plugin` object that defines the plugin.

It must be in the form `importable.module:object_name` where `importable.module` is in `srcDir`.| |manualDiscovery|N|boolean|True if the plugin supports manual discovery of source config objects. The default value is `true`.| |pluginType|Y|enum|The ingestion strategy of the plugin. Can be either `STAGED` or `DIRECT`.| -|language|Y|enum|Must be `PYTHON27`.| +|language|Y|enum|Must be `PYTHON38`.| |defaultLocale|N|enum|The locale to be used by the plugin if the Delphix user does not specify one. Plugin messages will be displayed in this locale by default. The default value is `en-us`.| |rootSquashEnabled|N|boolean|This dictates whether "root squash" is enabled on NFS mounts for the plugin (i.e. whether the `root` user on remote hosts has access to the NFS mounts). Setting this to `false` allows processes usually run as `root`, like Docker daemons, access to the NFS mounts. The default value is `true`. This field only applies to Unix hosts.| |extendedStartStopHooks|N|boolean|This controls whether the user's pre-start and post-start hooks will run during enable operations (and, likewise, whether pre-stop and post-stop hooks will run during disable operations). The default value is `false`.| @@ -46,14 +46,13 @@ This is a valid plugin config for the plugin: ```yaml id: 7cf830f2-82f3-4d5d-a63c-7bbe50c22b32 name: MongoDB -version: 2.0.0 hostTypes: - UNIX entryPoint: mongo_runner:mongodb srcDir: src/ schemaFile: schema.json pluginType: DIRECT -language: PYTHON27 +language: PYTHON38 buildNumber: 0.1.0 ``` This is a valid plugin config for the plugin with `manualDiscovery` set to `false` and an `externalVersion` set: @@ -68,7 +67,7 @@ srcDir: src/ schemaFile: schema.json manualDiscovery: false pluginType: DIRECT -language: PYTHON27 +language: PYTHON38 externalVersion: "MongoDB 1.0" buildNumber: "1" ``` diff --git a/docs/docs/References/Plugin_Operations.md b/docs/docs/References/Plugin_Operations.md index e02e3773..a66e1590 100644 --- a/docs/docs/References/Plugin_Operations.md +++ b/docs/docs/References/Plugin_Operations.md @@ -8,7 +8,6 @@ For each operation, the argument names must match exactly. For example, the Repository Discovery operation must have a single argument named `source_connection`. - Plugin Operation | **Required** | Decorator | Delphix Engine Operations ---------------- | -------- | --------- | ------------------------- [Repository
Discovery](#repository-discovery) | **Yes** |`discovery.repository()` | [Environment Discovery](Workflows.md#environment-discovery-refresh)
[Environment Refresh](Workflows.md#environment-discovery-refresh) @@ -26,6 +25,7 @@ Plugin Operation | **Required** | Decorator | Delphix Engine Operations [Virtual Source
Configure](#virtual-source-configure) | **Yes** | `virtual.configure()` | [Virtual Source Provision](Workflows.md#virtual-source-provision)
[Virtual Source Refresh](Workflows.md#virtual-source-refresh) [Virtual Source
Unconfigure](#virtual-source-unconfigure) | **No** | `virtual.unconfigure()` | [Virtual Source Refresh](Workflows.md#virtual-source-refresh)
[Virtual Source Delete](Workflows.md#virtual-source-delete) [Virtual Source
Reconfigure](#virtual-source-reconfigure) | **Yes** | `virtual.reconfigure()` | [Virtual Source Rollback](Workflows.md#virtual-source-rollback)
[Virtual Source Enable](Workflows.md#virtual-source-enable) +[Virtual Source
Cleanup](#virtual-source-cleanup) | **No** | `virtual.cleanup()` | [Virtual Source Delete](Workflows.md#virtual-source-delete) [Virtual Source
Start](#virtual-source-start) | **No** | `virtual.start()` | [Virtual Source Start](Workflows.md#virtual-source-start) [Virtual Source
Stop](#virtual-source-stop) | **No** | `virtual.stop()` | [Virtual Source Stop](Workflows.md#virtual-source-stop) [Virtual Source
Pre-Snapshot](#virtual-source-pre-snapshot) | **No** | `virtual.pre_snapshot()` | [Virtual Source Snapshot](Workflows.md#virtual-source-snapshot) @@ -824,6 +824,50 @@ def reconfigure(virtual_source, repository, source_config, snapshot): } ``` +## Virtual Source Cleanup + +Intended to allow a final cleanup during a delete operation, unlike unconfigure which can be used to signal a temporary dissassociation with a database. + +Cleanup is called during the delete flow after unconfigure. + +### Required / Optional +**Optional.** + +### Delphix Engine Operations + +* [Virtual Source Delete](Workflows.md#virtual-source-delete) + +### Signature + +`def cleanup(virtual_source, repository, source_config)` + +### Decorator + +`virtual.cleanup()` + +### Arguments + +Argument | Type | Description +-------- | ---- | ----------- +virtual_source | [VirtualSource](Classes.md#virtualsource) | The source associated with this operation. +repository | [RepositoryDefinition](Schemas_and_Autogenerated_Classes.md#repositorydefinition-class) | The repository associated with this source. +source_config | [SourceConfigDefinition](Schemas_and_Autogenerated_Classes.md#sourceconfigdefinition-class) | The source config associated with this source. + +### Returns +None + +### Example + +```python +from dlpx.virtualization.platform import Plugin + +plugin = Plugin() + +@plugin.virtual.cleanup() +def cleanup(virtual_source, repository, source_config): + pass +``` + ## Virtual Source Start Executed whenever the data should be placed in a "running" state. diff --git a/docs/docs/References/Schemas.md b/docs/docs/References/Schemas.md index 2f2a11a5..88c765b1 100644 --- a/docs/docs/References/Schemas.md +++ b/docs/docs/References/Schemas.md @@ -24,6 +24,8 @@ Delphix Object | Schema | Autogenerated Class [Snapshot](Glossary.md#linked-source) | [SnapshotDefinition](Schemas_and_Autogenerated_Classes.md#snapshotdefinition-schema) | [SnapshotDefinition](Schemas_and_Autogenerated_Classes.md#snapshotdefinition-class) [Snapshot Parameters](Glossary.md#snapshot-parameters) | [SnapshotParametersDefinition](Schemas_and_Autogenerated_Classes.md#snapshotparametersdefinition-schema) | [SnapshotParametersDefinition](Schemas_and_Autogenerated_Classes.md#snapshotparametersdefinition-class) +!!! warning + There are limits to how much data can be stored within a plugin defined schema. See [Message Limits](/Best_Practices/Message_Limits.md) for details. ## JSON Schemas @@ -116,7 +118,6 @@ For much more detail on JSON schemas, including which keywords are available, wh !!! info Be careful when using the JSON schema keyword `default`. This keyword is commonly misunderstood. Specifically, it does not mean "If the user does not provide a value, then this default value will be auto-substituted instead", as you might expect. In fact, in JSON schemas, `default` has no semantic meaning at all! Currently, the only thing the Delphix Engine will use this for is to pre-fill widgets on the UI. - ### Delphix-specific Extensions to JSON Schema The JSON schema vocabulary is designed to be extensible for special uses, and Delphix has taken advantage of this to add some new Delphix-specific keywords. diff --git a/docs/docs/References/Version_Compatibility.md b/docs/docs/References/Version_Compatibility.md new file mode 100644 index 00000000..1075cefb --- /dev/null +++ b/docs/docs/References/Version_Compatibility.md @@ -0,0 +1,28 @@ +# Virtualization SDK Version Compatibility + +## Virtualization SDK and Delphix Engine (DE) Compatibility Map + +|vSDK Version|Earliest Supported DE Version|Latest Supported DE Version| +|------------|:---------------------------:|:-------------------------:| +|4.0.4|6.0.12.0|[Latest Release](https://docs.delphix.com/docs/release-notes/)| +|4.0.2|6.0.12.0|[Latest Release](https://docs.delphix.com/docs/release-notes/)| +|3.1.0|6.0.7.0|[Latest Release](https://docs.delphix.com/docs/release-notes/)| +|3.0.0|6.0.6.0|[Latest Release](https://docs.delphix.com/docs/release-notes/)| +|2.1.0|6.0.3.0|[Latest Release](https://docs.delphix.com/docs/release-notes/)| +|2.0.0|6.0.2.0|[Latest Release](https://docs.delphix.com/docs/release-notes/)| +|1.0.0|6.0.2.0|[Latest Release](https://docs.delphix.com/docs/release-notes/)| +|0.4.0|5.3.5.0|6.0.1.0| + +## Virtualization SDK and Python Compatibility Map + +|vSDK Version|Python Version| +|------------|:------------:| +|4.0.4|3.8| +|4.0.2|3.8| +|3.1.0|2.7| +|3.0.0|2.7| +|2.1.0|2.7| +|2.0.0|2.7| +|1.0.0|2.7| +|0.4.0|2.7| + diff --git a/docs/docs/References/html/VirtualSourceDelete.html b/docs/docs/References/html/VirtualSourceDelete.html index 9e42c12e..7953f998 100644 --- a/docs/docs/References/html/VirtualSourceDelete.html +++ b/docs/docs/References/html/VirtualSourceDelete.html @@ -1,12 +1,12 @@ + - + -Draw.io Diagram - +VirtualSourceDelete.html -
- +
+ diff --git a/docs/docs/References/images/VirtualSourceDelete.png b/docs/docs/References/images/VirtualSourceDelete.png index ab8f2cff..9e1c005c 100644 Binary files a/docs/docs/References/images/VirtualSourceDelete.png and b/docs/docs/References/images/VirtualSourceDelete.png differ diff --git a/docs/docs/Release_Notes/4.0.0/.pages b/docs/docs/Release_Notes/4.0.0/.pages new file mode 100644 index 00000000..baf0835d --- /dev/null +++ b/docs/docs/Release_Notes/4.0.0/.pages @@ -0,0 +1,3 @@ +arrange: + - 4.0.0.md + - 4.0.0_Breaking_Changes.md diff --git a/docs/docs/Release_Notes/4.0.0/4.0.0.md b/docs/docs/Release_Notes/4.0.0/4.0.0.md new file mode 100644 index 00000000..7d339fe2 --- /dev/null +++ b/docs/docs/Release_Notes/4.0.0/4.0.0.md @@ -0,0 +1,11 @@ +# Release - v4.0.0 + +To install or upgrade the SDK, refer to instructions [here](/Getting_Started.md#installation). + +## New & Improved + +* Added support for plugins written in Python 3.8. + +## Breaking Changes + +* The CLI now requires Python 3.8 for installation and usage. \ No newline at end of file diff --git a/docs/docs/Release_Notes/4.0.0/4.0.0_Breaking_Changes.md b/docs/docs/Release_Notes/4.0.0/4.0.0_Breaking_Changes.md new file mode 100644 index 00000000..51125e56 --- /dev/null +++ b/docs/docs/Release_Notes/4.0.0/4.0.0_Breaking_Changes.md @@ -0,0 +1,4 @@ +# Breaking Changes - v.4.0.0 + +## New Language Requirement +The vSDK now requires Python 3.8. diff --git a/docs/docs/Release_Notes/4.0.4/4.0.4.md b/docs/docs/Release_Notes/4.0.4/4.0.4.md new file mode 100644 index 00000000..53cf9fef --- /dev/null +++ b/docs/docs/Release_Notes/4.0.4/4.0.4.md @@ -0,0 +1,15 @@ +# Release - v4.0.4 + +To install or upgrade the SDK, refer to instructions [here](/Getting_Started.md#installation). + +## New & Improved + +* Updated Delphix logo in [developer.delphix.com](https://developer.delphix.com/). +* Internet connectivity requirements added to [getting started](/Getting_Started.md#Requirements) page. +* __hash__() default implementation added for auto generated classes. +* Support added for symlinks folder within src folder. +* Handled empty system password during Lua to Platform upgrade. + +## Breaking Changes + +* No breaking changes in this release! \ No newline at end of file diff --git a/docs/docs/images/.DS_Store b/docs/docs/images/.DS_Store deleted file mode 100644 index 5008ddfc..00000000 Binary files a/docs/docs/images/.DS_Store and /dev/null differ diff --git a/docs/docs/images/delphix-logo-white.png b/docs/docs/images/delphix-logo-white.png deleted file mode 100644 index 80102867..00000000 Binary files a/docs/docs/images/delphix-logo-white.png and /dev/null differ diff --git a/docs/docs/images/favicon_logo.png b/docs/docs/images/favicon_logo.png new file mode 100644 index 00000000..3461ef44 Binary files /dev/null and b/docs/docs/images/favicon_logo.png differ diff --git a/docs/docs/images/header_logo.png b/docs/docs/images/header_logo.png new file mode 100644 index 00000000..0b935412 Binary files /dev/null and b/docs/docs/images/header_logo.png differ diff --git a/docs/docs/images/logo.png b/docs/docs/images/logo.png deleted file mode 100755 index b65b41f1..00000000 Binary files a/docs/docs/images/logo.png and /dev/null differ diff --git a/docs/material/.DS_Store b/docs/material/.DS_Store deleted file mode 100644 index fc4645d0..00000000 Binary files a/docs/material/.DS_Store and /dev/null differ diff --git a/docs/material/assets/stylesheets/application.fbb7f3af.css b/docs/material/assets/stylesheets/application.fbb7f3af.css index 398e092a..499e7df5 100644 --- a/docs/material/assets/stylesheets/application.fbb7f3af.css +++ b/docs/material/assets/stylesheets/application.fbb7f3af.css @@ -577,7 +577,7 @@ hr { .md-header-nav { padding: 0 0.4rem; } .md-header-nav a.md-header-nav__button.md-logo { - padding: 1.4rem; } + padding: 0; } .md-header-nav__button { position: relative; transition: opacity 0.25s; diff --git a/docs/material/partials/header.html b/docs/material/partials/header.html index f7aeae68..9339e7dc 100644 --- a/docs/material/partials/header.html +++ b/docs/material/partials/header.html @@ -8,7 +8,7 @@ {% elif config.theme.logo.startswith("http") %} {% else %} - + {% endif %} diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index d1799bc9..7391352a 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -1,18 +1,18 @@ -site_name: Delphix Virtualization SDK 3.0.0 +site_name: Delphix Virtualization SDK 4.0.0 theme: name: material custom_dir: 'material/' palette: primary: accent: - logo: 'images/delphix-logo-white.png' - favicon: 'images/logo.png' + logo: 'images/header_logo.png' + favicon: 'images/favicon_logo.png' font: text: Helvetica Neue code: Ubuntu Mono feature: -copyright: Copyright © 2021 Delphix Corp. +copyright: Copyright © 2022 Delphix Corp. google_analytics: - 'UA-35429885-3' diff --git a/docs/readme.md b/docs/readme.md index ece74f2f..89fa73bf 100644 --- a/docs/readme.md +++ b/docs/readme.md @@ -2,6 +2,13 @@ This is the Markdown-based documentation for the Virtualization SDK. +## Important Note On Building Docs + +As of this writing, the rest of the Virtualization SDK codebase is based on Python 2. +However, our docs infrastructure is based on Python 3! So, **all of the below commands +must be run in a Python 3 environment**. It's recommended to use a totally separate +virtual environment for docs work than the one you use in the rest of the SDK codebase. + ## Local Testing Install dependencies for building documentation and run `pipenv run mkdocs serve` @@ -13,8 +20,8 @@ To activate this project's virtualenv, run pipenv shell. Alternatively, run a command inside the virtualenv with pipenv run. $ pipenv run mkdocs serve -INFO - Building documentation... -INFO - Cleaning site directory +INFO - Building documentation... +INFO - Cleaning site directory [I 200424 15:54:06 server:292] Serving on http://127.0.0.1:8000 [I 200424 15:54:06 handlers:59] Start watching changes [I 200424 15:54:06 handlers:61] Start detecting changes @@ -59,7 +66,7 @@ Install `setuptools==45` to get around a deprecated API in version 46. $ pip install setuptools==45 Collecting setuptools==45 Downloading setuptools-45.0.0-py2.py3-none-any.whl (583 kB) - |████████████████████████████████| 583 kB 2.7 MB/s + |████████████████████████████████| 583 kB 2.7 MB/s Installing collected packages: setuptools Attempting uninstall: setuptools Found existing installation: setuptools 46.1.3 @@ -85,13 +92,13 @@ This will generate the `site` directory which will contain all the gererated doc 5. Go to your individual virtualization-sdk repo's settings, scroll to the bottom and verify under the GitHub Pages section the `Source` is set to `gh-pages branch`. 6. Right above this will be a link explaining where your docs are published. -You can also utilize the GitHub workflow for publishing docs (`.github/workflows/publish-docs.yml`) associated with a pull request. +You can also utilize the GitHub workflow for publishing docs (`.github/workflows/publish-docs.yml`) associated with a pull request. The workflow is present on the `develop` branch. Create a branch called `docs/x.y.z` off `develop` on your fork of the repository to ensure that your docs branch triggers the workflow. If you have more than one `docs/x.y.z` branch in your fork, you have to push your doc changes to the docs branch with the latest `x.y.z` version. Otherwise, the workflow won't run. You also have to make sure to choose `gh-pages` branch on your fork as the [publishing source](https://help.github.com/en/github/working-with-github-pages/configuring-a-publishing-source-for-your-github-pages-site#choosing-a-publishing-source). Once you push doc changes to the `docs/.x.y.z` branch, the docs site should be available under -`.github.io/virtualization-sdk` shortly after. You can see the status of publishing under +`.github.io/virtualization-sdk` shortly after. You can see the status of publishing under `https://github.com//virtualization-sdk/actions`. This is a fast way to give a preview of your changes in a pull request. diff --git a/dvp/.python-version b/dvp/.python-version index 43c4dbe6..d20cc2bf 100644 --- a/dvp/.python-version +++ b/dvp/.python-version @@ -1 +1 @@ -2.7.17 +3.8.10 diff --git a/dvp/setup.cfg b/dvp/setup.cfg index 776cfe6a..48fb5749 100644 --- a/dvp/setup.cfg +++ b/dvp/setup.cfg @@ -1,22 +1,22 @@ # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # [metadata] -Metadata-Version: 1.2 -Author: Delphix -Author-email: virtualization-plugins@delphix.com -Home-page: https://developer.delphix.com -Summary: Delphix Virtualization Platform SDK -Long-description: file: README.md -Long-description-content-type: text/markdown -Keywords: virtualization plugin -Classifiers: +metadata_version: 1.2 +author: Delphix +author_email: virtualization-plugins@delphix.com +home_page: https://developer.delphix.com +summary: Delphix Virtualization Platform SDK +long_description: file: README.md +long_description_content_type: text/markdown +keywords: virtualization plugin +classifiers: Development Status :: 5 - Production/Stable Programming Language :: Python - Programming Language :: Python :: 2.7 + Programming Language :: Python :: 3.8 License :: OSI Approved :: Apache Software License Operating System :: OS Independent [options] -Requires-Python: 2.7 +requires_python: 3.8 diff --git a/dvp/setup.py b/dvp/setup.py index b530d493..e7f78c7b 100644 --- a/dvp/setup.py +++ b/dvp/setup.py @@ -18,4 +18,5 @@ install_requires=install_requires, package_dir={'': PYTHON_SRC}, packages=setuptools.find_packages(PYTHON_SRC), + python_requires='>=3.8, <3.9', ) diff --git a/dvp/src/main/python/dlpx/virtualization/VERSION b/dvp/src/main/python/dlpx/virtualization/VERSION index fd2a0186..c5106e6d 100644 --- a/dvp/src/main/python/dlpx/virtualization/VERSION +++ b/dvp/src/main/python/dlpx/virtualization/VERSION @@ -1 +1 @@ -3.1.0 +4.0.4 diff --git a/libs/.python-version b/libs/.python-version deleted file mode 100644 index 43c4dbe6..00000000 --- a/libs/.python-version +++ /dev/null @@ -1 +0,0 @@ -2.7.17 diff --git a/libs/setup.cfg b/libs/setup.cfg index 3e46329a..09cd385a 100644 --- a/libs/setup.cfg +++ b/libs/setup.cfg @@ -1,21 +1,22 @@ # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # [metadata] -Metadata-Version: 1.2 -Author: Delphix -Author-email: virtualization-plugins@delphix.com -Home-page: https://developer.delphix.com -Summary: Delphix Virtualization Platform Libraries -Long-description: file: README.md -Long-description-content-type: text/markdown -Classifiers: +metadata_version: 1.2 +author: Delphix +author_email: virtualization-plugins@delphix.com +home_page: https://developer.delphix.com +summary: Delphix Virtualization Platform Libraries +long_description: file: README.md +long_description_content_type: text/markdown +classifiers: Development Status :: 5 - Production/Stable Programming Language :: Python Programming Language :: Python :: 2.7 + Programming Language :: Python :: 3.8 License :: OSI Approved :: Apache Software License Operating System :: OS Independent [options] -Requires-Python: 2.7 +requires_python: >=2.7, <=3.8, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*, !=3.7.* diff --git a/libs/setup.py b/libs/setup.py index 907b103d..f443840a 100644 --- a/libs/setup.py +++ b/libs/setup.py @@ -7,7 +7,7 @@ version = version_file.read().strip() install_requires = [ - "dvp-api == 1.5.0", + "dvp-api == 1.6.3", "dvp-common == {}".format(version) ] @@ -16,4 +16,5 @@ install_requires=install_requires, package_dir={'': PYTHON_SRC}, packages=setuptools.find_packages(PYTHON_SRC), + python_requires='>=2.7, <3.9, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*, !=3.7.*', ) diff --git a/libs/src/main/python/dlpx/virtualization/libs/VERSION b/libs/src/main/python/dlpx/virtualization/libs/VERSION index fd2a0186..c5106e6d 100644 --- a/libs/src/main/python/dlpx/virtualization/libs/VERSION +++ b/libs/src/main/python/dlpx/virtualization/libs/VERSION @@ -1 +1 @@ -3.1.0 +4.0.4 diff --git a/libs/src/main/python/dlpx/virtualization/libs/__init__.py b/libs/src/main/python/dlpx/virtualization/libs/__init__.py index 6af6f0e5..040a606b 100644 --- a/libs/src/main/python/dlpx/virtualization/libs/__init__.py +++ b/libs/src/main/python/dlpx/virtualization/libs/__init__.py @@ -1,8 +1,8 @@ # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # __path__ = __import__('pkgutil').extend_path(__path__, __name__) -from dlpx.virtualization.libs.libs import * -from dlpx.virtualization.libs._logging import * +from dlpx.virtualization.libs.libs import * # noqa +from dlpx.virtualization.libs._logging import * # noqa diff --git a/libs/src/main/python/dlpx/virtualization/libs/_logging.py b/libs/src/main/python/dlpx/virtualization/libs/_logging.py index ec377932..0e97eb7e 100644 --- a/libs/src/main/python/dlpx/virtualization/libs/_logging.py +++ b/libs/src/main/python/dlpx/virtualization/libs/_logging.py @@ -1,10 +1,11 @@ # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # from logging import Handler from dlpx.virtualization.libs import libs +from dlpx.virtualization.common.util import to_str __all__ = [ "PlatformHandler" @@ -17,4 +18,5 @@ class PlatformHandler(Handler): """ def emit(self, record): msg = self.format(record) + msg = to_str(msg) libs._log_request(msg, record.levelno) diff --git a/libs/src/main/python/dlpx/virtualization/libs/exceptions.py b/libs/src/main/python/dlpx/virtualization/libs/exceptions.py index fc4a7c11..f2d4c3d9 100644 --- a/libs/src/main/python/dlpx/virtualization/libs/exceptions.py +++ b/libs/src/main/python/dlpx/virtualization/libs/exceptions.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # import sys @@ -66,22 +66,16 @@ class IncorrectArgumentTypeError(PluginRuntimeError): message (str): A user-readable message describing the exception. """ - def __init__( - self, - parameter_name, - actual_type, - expected_type, - required=True): - actual, expected = self.get_actual_and_expected_type( - actual_type, expected_type) + def __init__(self, parameter_name, actual_type, expected_type, required=True): + actual, expected = self.get_actual_and_expected_type(actual_type, expected_type) - # Get the name of the function that is throwning this error. + # Get the name of the function that is throwing this error. func_name = sys._getframe(1).f_code.co_name message = ("The function {}'s argument '{}' was {} but should" " be of {}{}.".format( - func_name, - parameter_name, - actual, - expected, - (' if defined', '')[required])) + func_name, + parameter_name, + actual, + expected, + (' if defined', '')[required])) super(IncorrectArgumentTypeError, self).__init__(message) diff --git a/libs/src/main/python/dlpx/virtualization/libs/libs.py b/libs/src/main/python/dlpx/virtualization/libs/libs.py index 69355a4b..90f3ce2b 100644 --- a/libs/src/main/python/dlpx/virtualization/libs/libs.py +++ b/libs/src/main/python/dlpx/virtualization/libs/libs.py @@ -1,4 +1,4 @@ -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # # -*- coding: utf-8 -*- @@ -33,10 +33,12 @@ from dlpx.virtualization.common._common_classes import (RemoteConnection, PasswordCredentials, KeyPairCredentials) +from dlpx.virtualization.common.util import response_to_str, to_str from google.protobuf import json_format from google.protobuf.struct_pb2 import Struct import logging +import six __all__ = [ @@ -74,23 +76,24 @@ def _handle_response(response): def _check_exit_code(response, check): - """ - This functions checks the exitcode received in response and throws PluginScriptError - if check is True. - - Args: - response (RunPowerShellResponse or RunBashResponse or RunExpectResponse): Response received by run_bash or - run_powershell or run_expect - check (bool): if True and non-zero exitcode is received in response, raise PluginScriptError - """ - if (check and response.HasField('return_value') - and response.return_value.exit_code != 0): - raise PluginScriptError('The script failed with exit code {}.' - ' stdout : {} and ' - ' stderr : {}'.format( - response.return_value.exit_code, - response.return_value.stdout, - response.return_value.stderr)) + """ + This functions checks the exitcode received in response and throws + PluginScriptError if check is True. + + Args: + response (RunPowerShellResponse or RunBashResponse or RunExpectResponse): Response + received by run_bash or run_powershell or run_expect + check (bool): if True and non-zero exitcode is received in response, raise + PluginScriptError + """ + if (check and response.HasField('return_value') + and response.return_value.exit_code != 0): + raise PluginScriptError('The script failed with exit code {}.' + ' stdout : {} and ' + ' stderr : {}'.format( + response.return_value.exit_code, + response.return_value.stdout, + response.return_value.stderr)) def run_bash(remote_connection, command, variables=None, use_login_shell=False, @@ -128,6 +131,8 @@ def run_bash(remote_connection, command, variables=None, use_login_shell=False, if variables is None: variables = {} + command = to_str(command) + variables = to_str(variables) # Validate all the arguments passed in are the right types based on docs. if not isinstance(remote_connection, RemoteConnection): @@ -135,23 +140,23 @@ def run_bash(remote_connection, command, variables=None, use_login_shell=False, 'remote_connection', type(remote_connection), RemoteConnection) - if not isinstance(command, basestring): - raise IncorrectArgumentTypeError('command', type(command), basestring) + if not isinstance(command, six.string_types): + raise IncorrectArgumentTypeError('command', type(command), six.string_types[0]) if variables and not isinstance(variables, dict): raise IncorrectArgumentTypeError( 'variables', type(variables), - {basestring: basestring}, + {six.string_types[0]: six.string_types[0]}, False) - if (variables and (not all(isinstance(variable, basestring) + if (variables and (not all(isinstance(variable, six.string_types) for variable in variables.keys()) or - not all(isinstance(value, basestring) + not all(isinstance(value, six.string_types) for value in variables.values()))): raise IncorrectArgumentTypeError( 'variables', {(type(variable), type(value)) for variable, value in variables.items()}, - {basestring: basestring}, + {six.string_types[0]: six.string_types[0]}, False) if use_login_shell and not isinstance(use_login_shell, bool): raise IncorrectArgumentTypeError( @@ -165,6 +170,7 @@ def run_bash(remote_connection, command, variables=None, use_login_shell=False, run_bash_request.variables[variable] = value run_bash_response = internal_libs.run_bash(run_bash_request) + response_to_str(run_bash_response) _check_exit_code(run_bash_response, check) return _handle_response(run_bash_response) @@ -192,46 +198,54 @@ def run_sync(remote_connection, source_directory, rsync_user=None, from dlpx.virtualization._engine import libs as internal_libs + source_directory = to_str(source_directory) + if rsync_user is not None: + rsync_user = to_str(rsync_user) + if exclude_paths is not None: + exclude_paths = to_str(exclude_paths) + if sym_links_to_follow is not None: + sym_links_to_follow = to_str(sym_links_to_follow) + # Validate all the arguments passed in are the right types based on docs. if not isinstance(remote_connection, RemoteConnection): raise IncorrectArgumentTypeError( 'remote_connection', type(remote_connection), RemoteConnection) - if not isinstance(source_directory, basestring): + if not isinstance(source_directory, six.string_types): raise IncorrectArgumentTypeError( - 'source_directory', type(source_directory), basestring) - if rsync_user and not isinstance(rsync_user, basestring): + 'source_directory', type(source_directory), six.string_types[0]) + if rsync_user and not isinstance(rsync_user, six.string_types): raise IncorrectArgumentTypeError( 'rsync_user', type(rsync_user), - basestring, + six.string_types[0], False) if exclude_paths and not isinstance(exclude_paths, list): raise IncorrectArgumentTypeError( 'exclude_paths', type(exclude_paths), - [basestring], + [six.string_types[0]], False) if (exclude_paths and not all(isinstance( - path, basestring) for path in exclude_paths)): + path, six.string_types) for path in exclude_paths)): raise IncorrectArgumentTypeError( 'exclude_paths', [type(path) for path in exclude_paths], - [basestring], + [six.string_types[0]], False) if sym_links_to_follow and not isinstance(sym_links_to_follow, list): raise IncorrectArgumentTypeError( 'sym_links_to_follow', type(sym_links_to_follow), - [basestring], + [six.string_types[0]], False) - if (sym_links_to_follow and not all(isinstance(link, basestring) + if (sym_links_to_follow and not all(isinstance(link, six.string_types) for link in sym_links_to_follow)): raise IncorrectArgumentTypeError( 'sym_links_to_follow', [type(link) for link in sym_links_to_follow], - [basestring], + [six.string_types[0]], False) run_sync_request = libs_pb2.RunSyncRequest() @@ -245,6 +259,7 @@ def run_sync(remote_connection, source_directory, rsync_user=None, run_sync_request.sym_links_to_follow.extend(sym_links_to_follow) response = internal_libs.run_sync(run_sync_request) + response_to_str(response) _handle_response(response) @@ -282,29 +297,32 @@ def run_powershell(remote_connection, command, variables=None, check=False): if variables is None: variables = {} + command = to_str(command) + variables = to_str(variables) + # Validate all the arguments passed in are the right types based on docs. if not isinstance(remote_connection, RemoteConnection): raise IncorrectArgumentTypeError( 'remote_connection', type(remote_connection), RemoteConnection) - if not isinstance(command, basestring): - raise IncorrectArgumentTypeError('command', type(command), basestring) + if not isinstance(command, six.string_types): + raise IncorrectArgumentTypeError('command', type(command), six.string_types[0]) if variables and not isinstance(variables, dict): raise IncorrectArgumentTypeError( 'variables', type(variables), - {basestring: basestring}, + {six.string_types[0]: six.string_types[0]}, False) - if (variables and (not all(isinstance(variable, basestring) + if (variables and (not all(isinstance(variable, six.string_types) for variable in variables.keys()) or - not all(isinstance(value, basestring) + not all(isinstance(value, six.string_types) for value in variables.values()))): raise IncorrectArgumentTypeError( 'variables', {(type(variable), type(value)) for variable, value in variables.items()}, - {basestring: basestring}, + {six.string_types[0]: six.string_types[0]}, False) run_powershell_request = libs_pb2.RunPowerShellRequest() @@ -314,6 +332,7 @@ def run_powershell(remote_connection, command, variables=None, check=False): run_powershell_request.variables[variable] = value run_powershell_response = internal_libs.run_powershell( run_powershell_request) + response_to_str(run_powershell_response) _check_exit_code(run_powershell_response, check) return _handle_response(run_powershell_response) @@ -343,33 +362,35 @@ def run_expect(remote_connection, command, variables=None, check=False): # scope to allow unit testing of this module. # from dlpx.virtualization._engine import libs as internal_libs - if variables is None: variables = {} + command = to_str(command) + variables = to_str(variables) + # Validate all the arguments passed in are the right types based on docs. if not isinstance(remote_connection, RemoteConnection): raise IncorrectArgumentTypeError( 'remote_connection', type(remote_connection), RemoteConnection) - if not isinstance(command, basestring): - raise IncorrectArgumentTypeError('command', type(command), basestring) + if not isinstance(command, six.string_types): + raise IncorrectArgumentTypeError('command', type(command), six.string_types[0]) if variables and not isinstance(variables, dict): raise IncorrectArgumentTypeError( 'variables', type(variables), - {basestring: basestring}, + {six.string_types[0]: six.string_types[0]}, False) - if (variables and (not all(isinstance(variable, basestring) + if (variables and (not all(isinstance(variable, six.string_types) for variable in variables.keys()) or - not all(isinstance(value, basestring) + not all(isinstance(value, six.string_types) for value in variables.values()))): raise IncorrectArgumentTypeError( 'variables', {(type(variable), type(value)) for variable, value in variables.items()}, - {basestring: basestring}, + {six.string_types[0]: six.string_types[0]}, False) run_expect_request = libs_pb2.RunExpectRequest() @@ -379,6 +400,7 @@ def run_expect(remote_connection, command, variables=None, check=False): run_expect_request.variables[variable] = value run_expect_response = internal_libs.run_expect(run_expect_request) + response_to_str(run_expect_response) _check_exit_code(run_expect_response, check) return _handle_response(run_expect_response) @@ -400,6 +422,7 @@ def _log_request(message, log_level): """ from dlpx.virtualization._engine import libs as internal_libs + message = to_str(message) log_request = libs_pb2.LogRequest() log_request.message = message @@ -415,23 +438,27 @@ def _log_request(message, log_level): log_request.level = libs_pb2.LogRequest.ERROR response = internal_libs.log(log_request) + response_to_str(response) _handle_response(response) def retrieve_credentials(credentials_supplier): - """This is an internal wrapper around the Virtualization library's credentials retrieval API. - Given a supplier provided by Virtualization, retrieves the credentials from that supplier. + """ + This is an internal wrapper around the Virtualization library's credentials + retrieval API. Given a supplier provided by Virtualization, retrieves the + credentials from that supplier. Args: credentials_supplier (dict): Properties that make up a supplier of credentials. Return: - Subclass of Credentials retrieved from supplier. Either a PasswordCredentials or a KeyPairCredentials - from dlpx.virtualization.common._common_classes. + Subclass of Credentials retrieved from supplier. Either a PasswordCredentials + or a KeyPairCredentials from dlpx.virtualization.common._common_classes. """ from dlpx.virtualization._engine import libs as internal_libs if not isinstance(credentials_supplier, dict): - raise IncorrectArgumentTypeError('credentials_supplier', type(credentials_supplier), dict) + raise IncorrectArgumentTypeError( + 'credentials_supplier', type(credentials_supplier), dict) credentials_request = libs_pb2.CredentialsRequest() credentials_struct = Struct() @@ -439,10 +466,21 @@ def retrieve_credentials(credentials_supplier): credentials_request.credentials_supplier.CopyFrom(credentials_struct) response = internal_libs.retrieve_credentials(credentials_request) - + response_to_str(response) credentials_result = _handle_response(response) - if credentials_result.password != "": - return PasswordCredentials(credentials_result.username, credentials_result.password) + # + # As protobuf definition of credentials_result object has all the + # attributes private_key, public_key and password irrespective of + # whether it is a keypair or a password credential type, we consider the + # object to be password credential type if private_key and public_key is + # not set. + # + if ( + not credentials_result.key_pair.private_key and + not credentials_result.key_pair.public_key + ): + return PasswordCredentials( + credentials_result.username, credentials_result.password) return KeyPairCredentials( credentials_result.username, credentials_result.key_pair.private_key, @@ -450,22 +488,26 @@ def retrieve_credentials(credentials_supplier): def upgrade_password(password, username=None): - """This is an internal wrapper around Virtualization's credentials-supplier conversion API. - It is intended for use during plugin upgrade when a plugin needs to transform a password - value into a more generic credentials supplier object. + """ + This is an internal wrapper around Virtualization's credentials-supplier conversion + API. It is intended for use during plugin upgrade when a plugin needs to transform + a password value into a more generic credentials supplier object. Args: - password (basestring): Plain password string. - username (basestring, defaults to None): User name contained in the password credential supplier to return. + password (str): Plain password string. + username (str, defaults to None): User name contained in the password + credential supplier to return. Return: Credentials supplier (dict) that supplies the given password and username. """ from dlpx.virtualization._engine import libs as internal_libs - if not isinstance(password, basestring): - raise IncorrectArgumentTypeError('password', type(password), basestring) - if username and not isinstance(username, basestring): - raise IncorrectArgumentTypeError('username', type(username), basestring, required=False) + if not isinstance(password, six.string_types): + raise IncorrectArgumentTypeError( + 'password', type(password), six.string_types[0]) + if username and not isinstance(username, six.string_types): + raise IncorrectArgumentTypeError( + 'username', type(username), six.string_types[0], required=False) upgrade_password_request = libs_pb2.UpgradePasswordRequest() upgrade_password_request.password = password @@ -473,6 +515,6 @@ def upgrade_password(password, username=None): upgrade_password_request.username = username response = internal_libs.upgrade_password(upgrade_password_request) - + response_to_str(response) upgrade_password_result = _handle_response(response) return json_format.MessageToDict(upgrade_password_result.credentials_supplier) diff --git a/libs/src/test/python/dlpx/virtualization/_engine/libs.py b/libs/src/test/python/dlpx/virtualization/_engine/libs.py index 99356f49..bb0d58dc 100644 --- a/libs/src/test/python/dlpx/virtualization/_engine/libs.py +++ b/libs/src/test/python/dlpx/virtualization/_engine/libs.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # """ @@ -25,4 +25,4 @@ def run_expect(run_expect_request): def log(log_debug_request): - pass \ No newline at end of file + pass diff --git a/libs/src/test/python/dlpx/virtualization/conftest.py b/libs/src/test/python/dlpx/virtualization/conftest.py index 4c805eb0..52e16d2b 100644 --- a/libs/src/test/python/dlpx/virtualization/conftest.py +++ b/libs/src/test/python/dlpx/virtualization/conftest.py @@ -1,9 +1,10 @@ # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # import pytest -from dlpx.virtualization.common._common_classes import RemoteUser, RemoteHost, RemoteEnvironment, RemoteConnection +from dlpx.virtualization.common._common_classes import ( + RemoteUser, RemoteHost, RemoteEnvironment, RemoteConnection) @pytest.fixture diff --git a/libs/src/test/python/dlpx/virtualization/test_libs.py b/libs/src/test/python/dlpx/virtualization/test_libs.py index 4e94d375..abf8eba0 100644 --- a/libs/src/test/python/dlpx/virtualization/test_libs.py +++ b/libs/src/test/python/dlpx/virtualization/test_libs.py @@ -1,16 +1,16 @@ # -# Copyright (c) 2019, 2020 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # import mock import pytest +import six from dlpx.virtualization.api import libs_pb2 from dlpx.virtualization import libs from dlpx.virtualization.libs.exceptions import ( IncorrectArgumentTypeError, LibraryError, PluginScriptError) from google.protobuf import json_format -from dlpx.virtualization.common._common_classes import (PasswordCredentials) class TestLibsRunBash: @@ -53,10 +53,10 @@ def mock_run_bash(actual_run_bash_request): @staticmethod def test_run_bash_check_true_success_exitcode(remote_connection): - expected_run_bash_response = libs_pb2.RunBashResponse() - expected_run_bash_response.return_value.exit_code = 0 - expected_run_bash_response.return_value.stdout = "stdout" - expected_run_bash_response.return_value.stderr = "stderr" + expected_response = libs_pb2.RunBashResponse() + expected_response.return_value.exit_code = 0 + expected_response.return_value.stdout = "stdout" + expected_response.return_value.stderr = "stderr" expected_command = "command" expected_variables = None @@ -73,19 +73,17 @@ def mock_run_bash(actual_run_bash_request): actual_run_bash_request.remote_connection.environment.reference == remote_connection.environment.reference ) - return expected_run_bash_response + return expected_response with mock.patch("dlpx.virtualization._engine.libs.run_bash", side_effect=mock_run_bash, create=True): - actual_run_bash_result = libs.run_bash(remote_connection, - expected_command, - expected_variables, - expected_use_login_shell, - check=True) + actual_result = libs.run_bash( + remote_connection, expected_command, expected_variables, + expected_use_login_shell, check=True) - assert actual_run_bash_result.exit_code == expected_run_bash_response.return_value.exit_code - assert actual_run_bash_result.stdout == expected_run_bash_response.return_value.stdout - assert actual_run_bash_result.stderr == expected_run_bash_response.return_value.stderr + assert actual_result.exit_code == expected_response.return_value.exit_code + assert actual_result.stdout == expected_response.return_value.stdout + assert actual_result.stderr == expected_response.return_value.stderr @staticmethod def test_run_bash_with_check_true_failed_exitcode(remote_connection): @@ -143,11 +141,16 @@ def test_run_bash_bad_remote_connection(): with pytest.raises(IncorrectArgumentTypeError) as err_info: libs.run_bash(connection, command, variables, use_login_shell) - - assert err_info.value.message == ( - "The function run_bash's argument 'remote_connection' was" - " type 'str' but should be of" - " class 'dlpx.virtualization.common._common_classes.RemoteConnection'.") + if six.PY2: + assert err_info.value.message == ( + "The function run_bash's argument 'remote_connection' was" + " type 'str' but should be of" + " class 'dlpx.virtualization.common._common_classes.RemoteConnection'.") + else: + assert err_info.value.message == ( + "The function run_bash's argument 'remote_connection' was" + " class 'str' but should be of" + " class 'dlpx.virtualization.common._common_classes.RemoteConnection'.") @staticmethod def test_run_bash_bad_command(remote_connection): @@ -158,10 +161,14 @@ def test_run_bash_bad_command(remote_connection): with pytest.raises(IncorrectArgumentTypeError) as err_info: libs.run_bash(remote_connection, command, variables, use_login_shell) - - assert err_info.value.message == ( - "The function run_bash's argument 'command' was" - " type 'int' but should be of type 'basestring'.") + if six.PY2: + assert err_info.value.message == ( + "The function run_bash's argument 'command' was" + " type 'int' but should be of type 'basestring'.") + else: + assert err_info.value.message == ( + "The function run_bash's argument 'command' was" + " class 'int' but should be of class 'str'.") @staticmethod def test_run_bash_variables_not_dict(remote_connection): @@ -172,11 +179,16 @@ def test_run_bash_variables_not_dict(remote_connection): with pytest.raises(IncorrectArgumentTypeError) as err_info: libs.run_bash(remote_connection, command, variables, use_login_shell) - - assert err_info.value.message == ( - "The function run_bash's argument 'variables' was" - " type 'str' but should be of" - " type 'dict of basestring:basestring' if defined.") + if six.PY2: + assert err_info.value.message == ( + "The function run_bash's argument 'variables' was" + " type 'unicode' but should be of" + " type 'dict of basestring:basestring' if defined.") + else: + assert err_info.value.message == ( + "The function run_bash's argument 'variables' was" + " class 'str' but should be of" + " type 'dict of str:str' if defined.") @staticmethod def test_run_bash_bad_variables(remote_connection): @@ -190,13 +202,20 @@ def test_run_bash_bad_variables(remote_connection): with pytest.raises(IncorrectArgumentTypeError) as err_info: libs.run_bash(remote_connection, command, variables, use_login_shell) - - message = ("The function run_bash's argument 'variables' was" - " a dict of {{type 'str':type '{}', type 'str':type '{}'}}" - " but should be of" - " type 'dict of basestring:basestring' if defined.") - assert (err_info.value.message == message.format('int', 'str') or - err_info.value.message == message.format('str', 'int')) + if six.PY2: + message = ("The function run_bash's argument 'variables' was" + " a dict of {{type 'str':type '{}', type 'str':type '{}'}}" + " but should be of" + " type 'dict of basestring:basestring' if defined.") + assert (err_info.value.message == message.format('int', 'unicode') or + err_info.value.message == message.format('unicode', 'int')) + else: + message = ("The function run_bash's argument 'variables' was" + " a dict of {{class 'str':class '{}', class 'str':class '{}'}}" + " but should be of" + " type 'dict of str:str' if defined.") + assert (err_info.value.message == message.format('int', 'str') or + err_info.value.message == message.format('str', 'int')) @staticmethod def test_run_bash_bad_use_login_shell(remote_connection): @@ -207,10 +226,14 @@ def test_run_bash_bad_use_login_shell(remote_connection): with pytest.raises(IncorrectArgumentTypeError) as err_info: libs.run_bash(remote_connection, command, variables, use_login_shell) - - assert err_info.value.message == ( - "The function run_bash's argument 'use_login_shell' was" - " type 'str' but should be of type 'bool' if defined.") + if six.PY2: + assert err_info.value.message == ( + "The function run_bash's argument 'use_login_shell' was" + " type 'str' but should be of type 'bool' if defined.") + else: + assert err_info.value.message == ( + "The function run_bash's argument 'use_login_shell' was" + " class 'str' but should be of class 'bool' if defined.") class TestLibsRunSync: @@ -296,11 +319,16 @@ def test_run_sync_bad_remote_connection(): rsync_user, exclude_paths, sym_links_to_follow) - - assert err_info.value.message == ( - "The function run_sync's argument 'remote_connection' was" - " type 'str' but should be of" - " class 'dlpx.virtualization.common._common_classes.RemoteConnection'.") + if six.PY2: + assert err_info.value.message == ( + "The function run_sync's argument 'remote_connection' was" + " type 'str' but should be of" + " class 'dlpx.virtualization.common._common_classes.RemoteConnection'.") + else: + assert err_info.value.message == ( + "The function run_sync's argument 'remote_connection' was" + " class 'str' but should be of" + " class 'dlpx.virtualization.common._common_classes.RemoteConnection'.") @staticmethod def test_run_sync_bad_source_directory(remote_connection): @@ -317,10 +345,14 @@ def test_run_sync_bad_source_directory(remote_connection): rsync_user, exclude_paths, sym_links_to_follow) - - assert err_info.value.message == ( - "The function run_sync's argument 'source_directory' was" - " type 'int' but should be of type 'basestring'.") + if six.PY2: + assert err_info.value.message == ( + "The function run_sync's argument 'source_directory' was" + " type 'int' but should be of type 'basestring'.") + else: + assert err_info.value.message == ( + "The function run_sync's argument 'source_directory' was" + " class 'int' but should be of class 'str'.") @staticmethod def test_run_sync_bad_rsync_user(remote_connection): @@ -337,10 +369,14 @@ def test_run_sync_bad_rsync_user(remote_connection): rsync_user, exclude_paths, sym_links_to_follow) - - assert err_info.value.message == ( - "The function run_sync's argument 'rsync_user' was" - " type 'int' but should be of type 'basestring' if defined.") + if six.PY2: + assert err_info.value.message == ( + "The function run_sync's argument 'rsync_user' was" + " type 'int' but should be of type 'basestring' if defined.") + else: + assert err_info.value.message == ( + "The function run_sync's argument 'rsync_user' was" + " class 'int' but should be of class 'str' if defined.") @staticmethod def test_run_sync_exclude_paths_not_list(remote_connection): @@ -357,11 +393,16 @@ def test_run_sync_exclude_paths_not_list(remote_connection): rsync_user, exclude_paths, sym_links_to_follow) - - assert err_info.value.message == ( - "The function run_sync's argument 'exclude_paths' was" - " type 'str' but should be of" - " type 'list of basestring' if defined.") + if six.PY2: + assert err_info.value.message == ( + "The function run_sync's argument 'exclude_paths' was" + " type 'unicode' but should be of" + " type 'list of basestring' if defined.") + else: + assert err_info.value.message == ( + "The function run_sync's argument 'exclude_paths' was" + " class 'str' but should be of" + " type 'list of str' if defined.") @staticmethod def test_run_sync_bad_exclude_paths(remote_connection): @@ -378,11 +419,16 @@ def test_run_sync_bad_exclude_paths(remote_connection): rsync_user, exclude_paths, sym_links_to_follow) - - assert err_info.value.message == ( - "The function run_sync's argument 'exclude_paths' was a list of" - " [type 'str', type 'int'] but should be of" - " type 'list of basestring' if defined.") + if six.PY2: + assert err_info.value.message == ( + "The function run_sync's argument 'exclude_paths' was a list of" + " [type 'unicode', type 'int'] but should be of" + " type 'list of basestring' if defined.") + else: + assert err_info.value.message == ( + "The function run_sync's argument 'exclude_paths' was a list of" + " [class 'str', class 'int'] but should be of" + " type 'list of str' if defined.") @staticmethod def test_run_sync_sym_links_to_follow_not_list(remote_connection): @@ -399,11 +445,16 @@ def test_run_sync_sym_links_to_follow_not_list(remote_connection): rsync_user, exclude_paths, sym_links_to_follow) - - assert err_info.value.message == ( - "The function run_sync's argument 'sym_links_to_follow' was" - " type 'str' but should be of" - " type 'list of basestring' if defined.") + if six.PY2: + assert err_info.value.message == ( + "The function run_sync's argument 'sym_links_to_follow' was" + " type 'unicode' but should be of" + " type 'list of basestring' if defined.") + else: + assert err_info.value.message == ( + "The function run_sync's argument 'sym_links_to_follow' was" + " class 'str' but should be of" + " type 'list of str' if defined.") @staticmethod def test_run_sync_bad_sym_links_to_follow(remote_connection): @@ -420,11 +471,18 @@ def test_run_sync_bad_sym_links_to_follow(remote_connection): rsync_user, exclude_paths, sym_links_to_follow) - - assert err_info.value.message == ( - "The function run_sync's argument 'sym_links_to_follow' was" - " a list of [type 'str', type 'int'] but should be of" - " type 'list of basestring' if defined.") + if six.PY2: + assert err_info.value.message == ( + "The function run_sync's argument 'sym_links_to_follow' was" + " a list of [type 'unicode', type 'int'] but should be of" + " type 'list of " + "basestring' if defined.") + else: + assert err_info.value.message == ( + "The function run_sync's argument 'sym_links_to_follow' was" + " a list of [class 'str', class 'int'] but should be of" + " type 'list of " + "str' if defined.") class TestLibsRunPowershell: @@ -463,35 +521,35 @@ def mock_run_powershell(actual_run_powershell_request): @staticmethod def test_run_powershell_check_true_exitcode_success(remote_connection): - expected_run_powershell_response = libs_pb2.RunPowerShellResponse() - expected_run_powershell_response.return_value.exit_code = 0 - expected_run_powershell_response.return_value.stdout = "stdout" - expected_run_powershell_response.return_value.stderr = "stderr" + expected_response = libs_pb2.RunPowerShellResponse() + expected_response.return_value.exit_code = 0 + expected_response.return_value.stdout = "stdout" + expected_response.return_value.stderr = "stderr" expected_command = "command" expected_variables = None - def mock_run_powershell(actual_run_powershell_request): - assert actual_run_powershell_request.command == expected_command + def mock_run_powershell(actual_request): + assert actual_request.command == expected_command assert ( - actual_run_powershell_request.remote_connection.environment.name + actual_request.remote_connection.environment.name == remote_connection.environment.name ) assert ( - actual_run_powershell_request.remote_connection.environment.reference + actual_request.remote_connection.environment.reference == remote_connection.environment.reference ) - return expected_run_powershell_response + return expected_response with mock.patch("dlpx.virtualization._engine.libs.run_powershell", side_effect=mock_run_powershell, create=True): - actual_run_powershell_result = libs.run_powershell( + actual_result = libs.run_powershell( remote_connection, expected_command, expected_variables, check=True) - assert actual_run_powershell_result.exit_code == expected_run_powershell_response.return_value.exit_code - assert actual_run_powershell_result.stdout == expected_run_powershell_response.return_value.stdout - assert actual_run_powershell_result.stderr == expected_run_powershell_response.return_value.stderr + assert actual_result.exit_code == expected_response.return_value.exit_code + assert actual_result.stdout == expected_response.return_value.stdout + assert actual_result.stderr == expected_response.return_value.stderr @staticmethod def test_run_powershell_check_true_exitcode_failed(remote_connection): @@ -508,8 +566,8 @@ def test_run_powershell_check_true_exitcode_failed(remote_connection): with mock.patch("dlpx.virtualization._engine.libs.run_powershell", return_value=response, create=True): with pytest.raises(PluginScriptError) as info: - response = libs.run_powershell(remote_connection, "test_command", - check=True) + libs.run_powershell(remote_connection, "test_command", + check=True) assert info.value.message == expected_message @staticmethod @@ -549,11 +607,16 @@ def test_run_powershell_bad_remote_connection(): with pytest.raises(IncorrectArgumentTypeError) as err_info: libs.run_powershell(connection, command, variables) - - assert err_info.value.message == ( - "The function run_powershell's argument 'remote_connection' was" - " type 'str' but should be of" - " class 'dlpx.virtualization.common._common_classes.RemoteConnection'.") + if six.PY2: + assert err_info.value.message == ( + "The function run_powershell's argument 'remote_connection' was" + " type 'str' but should be of" + " class 'dlpx.virtualization.common._common_classes.RemoteConnection'.") + else: + assert err_info.value.message == ( + "The function run_powershell's argument 'remote_connection' was" + " class 'str' but should be of" + " class 'dlpx.virtualization.common._common_classes.RemoteConnection'.") @staticmethod def test_run_powershell_bad_command(remote_connection): @@ -563,10 +626,14 @@ def test_run_powershell_bad_command(remote_connection): with pytest.raises(IncorrectArgumentTypeError) as err_info: libs.run_powershell(remote_connection, command, variables) - - assert err_info.value.message == ( - "The function run_powershell's argument 'command' was" - " type 'int' but should be of type 'basestring'.") + if six.PY2: + assert err_info.value.message == ( + "The function run_powershell's argument 'command' was" + " type 'int' but should be of type 'basestring'.") + else: + assert err_info.value.message == ( + "The function run_powershell's argument 'command' was" + " class 'int' but should be of class 'str'.") @staticmethod def test_run_powershell_variables_not_dict(remote_connection): @@ -576,11 +643,16 @@ def test_run_powershell_variables_not_dict(remote_connection): with pytest.raises(IncorrectArgumentTypeError) as err_info: libs.run_powershell(remote_connection, command, variables) - - assert err_info.value.message == ( - "The function run_powershell's argument 'variables' was" - " type 'str' but should be of" - " type 'dict of basestring:basestring' if defined.") + if six.PY2: + assert err_info.value.message == ( + "The function run_powershell's argument 'variables' was" + " type 'unicode' but should be of" + " type 'dict of basestring:basestring' if defined.") + else: + assert err_info.value.message == ( + "The function run_powershell's argument 'variables' was" + " class 'str' but should be of" + " type 'dict of str:str' if defined.") @staticmethod def test_run_powershell_bad_variables(remote_connection): @@ -593,13 +665,20 @@ def test_run_powershell_bad_variables(remote_connection): with pytest.raises(IncorrectArgumentTypeError) as err_info: libs.run_powershell(remote_connection, command, variables) - - message = ("The function run_powershell's argument 'variables' was" - " a dict of {{type 'str':type '{}', type 'str':type '{}'}}" - " but should be of" - " type 'dict of basestring:basestring' if defined.") - assert (err_info.value.message == message.format('int', 'str') or - err_info.value.message == message.format('str', 'int')) + if six.PY2: + message = ("The function run_powershell's argument 'variables' was" + " a dict of {{type 'str':type '{}', type 'str':type '{}'}}" + " but should be of" + " type 'dict of basestring:basestring' if defined.") + assert (err_info.value.message == message.format('int', 'unicode') or + err_info.value.message == message.format('unicode', 'int')) + else: + message = ("The function run_powershell's argument 'variables' was" + " a dict of {{class 'str':class '{}', class 'str':class '{}'}}" + " but should be of" + " type 'dict of str:str' if defined.") + assert (err_info.value.message == message.format('int', 'str') or + err_info.value.message == message.format('str', 'int')) class TestLibsRunExpect: @@ -638,10 +717,10 @@ def mock_run_expect(actual_run_expect_request): @staticmethod def test_run_expect_check_true_exitcode_success(remote_connection): - expected_run_expect_response = libs_pb2.RunPowerShellResponse() - expected_run_expect_response.return_value.exit_code = 0 - expected_run_expect_response.return_value.stdout = "stdout" - expected_run_expect_response.return_value.stderr = "stderr" + expected_response = libs_pb2.RunPowerShellResponse() + expected_response.return_value.exit_code = 0 + expected_response.return_value.stdout = "stdout" + expected_response.return_value.stderr = "stderr" expected_command = "command" expected_variables = None @@ -656,17 +735,17 @@ def mock_run_expect(actual_run_expect_request): actual_run_expect_request.remote_connection.environment.reference == remote_connection.environment.reference ) - return expected_run_expect_response + return expected_response with mock.patch("dlpx.virtualization._engine.libs.run_expect", side_effect=mock_run_expect, create=True): - actual_run_expect_result = libs.run_expect( + actual_result = libs.run_expect( remote_connection, expected_command, expected_variables, check=True) - assert actual_run_expect_result.exit_code == expected_run_expect_response.return_value.exit_code - assert actual_run_expect_result.stdout == expected_run_expect_response.return_value.stdout - assert actual_run_expect_result.stderr == expected_run_expect_response.return_value.stderr + assert actual_result.exit_code == expected_response.return_value.exit_code + assert actual_result.stdout == expected_response.return_value.stdout + assert actual_result.stderr == expected_response.return_value.stderr @staticmethod def test_run_expect_check_true_exitcode_failed(remote_connection): @@ -683,8 +762,7 @@ def test_run_expect_check_true_exitcode_failed(remote_connection): with mock.patch("dlpx.virtualization._engine.libs.run_expect", return_value=response, create=True): with pytest.raises(PluginScriptError) as info: - response = libs.run_expect(remote_connection, "test_command", - check=True) + libs.run_expect(remote_connection, "test_command", check=True) assert info.value.message == expected_message @staticmethod @@ -724,11 +802,16 @@ def test_run_expect_bad_remote_connection(): with pytest.raises(IncorrectArgumentTypeError) as err_info: libs.run_expect(connection, command, variables) - - assert err_info.value.message == ( - "The function run_expect's argument 'remote_connection' was" - " type 'str' but should be of" - " class 'dlpx.virtualization.common._common_classes.RemoteConnection'.") + if six.PY2: + assert err_info.value.message == ( + "The function run_expect's argument 'remote_connection' was" + " type 'str' but should be of" + " class 'dlpx.virtualization.common._common_classes.RemoteConnection'.") + else: + assert err_info.value.message == ( + "The function run_expect's argument 'remote_connection' was" + " class 'str' but should be of" + " class 'dlpx.virtualization.common._common_classes.RemoteConnection'.") @staticmethod def test_run_expect_bad_command(remote_connection): @@ -739,9 +822,14 @@ def test_run_expect_bad_command(remote_connection): with pytest.raises(IncorrectArgumentTypeError) as err_info: libs.run_expect(remote_connection, command, variables) - assert err_info.value.message == ( - "The function run_expect's argument 'command' was" - " type 'int' but should be of type 'basestring'.") + if six.PY2: + assert err_info.value.message == ( + "The function run_expect's argument 'command' was" + " type 'int' but should be of type 'basestring'.") + else: + assert err_info.value.message == ( + "The function run_expect's argument 'command' was" + " class 'int' but should be of class 'str'.") @staticmethod def test_run_expect_variables_not_dict(remote_connection): @@ -751,11 +839,16 @@ def test_run_expect_variables_not_dict(remote_connection): with pytest.raises(IncorrectArgumentTypeError) as err_info: libs.run_expect(remote_connection, command, variables) - - assert err_info.value.message == ( - "The function run_expect's argument 'variables' was" - " type 'str' but should be of" - " type 'dict of basestring:basestring' if defined.") + if six.PY2: + assert err_info.value.message == ( + "The function run_expect's argument 'variables' was" + " type 'unicode' but should be of" + " type 'dict of basestring:basestring' if defined.") + else: + assert err_info.value.message == ( + "The function run_expect's argument 'variables' was" + " class 'str' but should be of" + " type 'dict of str:str' if defined.") @staticmethod def test_run_expect_bad_variables(remote_connection): @@ -769,60 +862,72 @@ def test_run_expect_bad_variables(remote_connection): with pytest.raises(IncorrectArgumentTypeError) as err_info: libs.run_expect(remote_connection, command, variables) - message = ("The function run_expect's argument 'variables' was" - " a dict of {{type 'str':type '{}', type 'str':type '{}'}}" - " but should be of" - " type 'dict of basestring:basestring' if defined.") - assert (err_info.value.message == message.format('int', 'str') or - err_info.value.message == message.format('str', 'int')) + if six.PY2: + message = ("The function run_expect's argument 'variables' was" + " a dict of {{type 'str':type '{}', type 'str':type '{}'}}" + " but should be of" + " type 'dict of basestring:basestring' if defined.") + assert (err_info.value.message == message.format('int', 'unicode') or + err_info.value.message == message.format('unicode', 'int')) + else: + message = ("The function run_expect's argument 'variables' was" + " a dict of {{class 'str':class '{}', class 'str':class '{}'}}" + " but should be of" + " type 'dict of str:str' if defined.") + assert (err_info.value.message == message.format('int', 'str') or + err_info.value.message == message.format('str', 'int')) class TestLibsRetrieveCredentials: @staticmethod def test_retrieve_password_credentials(): - expected_retrieve_credentials_response = libs_pb2.CredentialsResponse() - expected_retrieve_credentials_response.return_value.username = 'some user' - expected_retrieve_credentials_response.return_value.password = 'some password' + expected_response = libs_pb2.CredentialsResponse() + expected_response.return_value.username = 'some user' + expected_response.return_value.password = 'some password' - expected_credentials_supplier = {'some supplier property': 'some supplier value'} + expected_credentials_supplier = { + 'some supplier property': 'some supplier value'} def mock_retrieve_credentials(actual_retrieve_credentials_request): - assert json_format.MessageToDict(actual_retrieve_credentials_request.credentials_supplier) == \ - expected_credentials_supplier + assert json_format.MessageToDict( + actual_retrieve_credentials_request.credentials_supplier + ) == expected_credentials_supplier - return expected_retrieve_credentials_response + return expected_response with mock.patch('dlpx.virtualization._engine.libs.retrieve_credentials', side_effect=mock_retrieve_credentials, create=True): - actual_retrieve_credentials_result = libs.retrieve_credentials(expected_credentials_supplier) + actual_result = libs.retrieve_credentials(expected_credentials_supplier) - expected = expected_retrieve_credentials_response.return_value - assert actual_retrieve_credentials_result.username == expected.username - assert actual_retrieve_credentials_result.password == expected.password + expected = expected_response.return_value + assert actual_result.username == expected.username + assert actual_result.password == expected.password @staticmethod def test_retrieve_keypair_credentials(): - expected_retrieve_credentials_response = libs_pb2.CredentialsResponse() - expected_retrieve_credentials_response.return_value.username = 'some user' - expected_retrieve_credentials_response.return_value.key_pair.private_key = 'some private key' - expected_retrieve_credentials_response.return_value.key_pair.public_key = 'some public key' + expected_response = libs_pb2.CredentialsResponse() + expected_response.return_value.username = 'some user' + expected_response.return_value.key_pair.private_key = 'some private key' + expected_response.return_value.key_pair.public_key = 'some public key' - expected_credentials_supplier = {'some supplier property': 'some supplier value'} + expected_credentials_supplier = { + 'some supplier property': 'some supplier value'} def mock_retrieve_credentials(actual_retrieve_credentials_request): - assert json_format.MessageToDict(actual_retrieve_credentials_request.credentials_supplier) == \ - expected_credentials_supplier + assert json_format.MessageToDict( + actual_retrieve_credentials_request.credentials_supplier + ) == expected_credentials_supplier - return expected_retrieve_credentials_response + return expected_response with mock.patch('dlpx.virtualization._engine.libs.retrieve_credentials', side_effect=mock_retrieve_credentials, create=True): - actual_retrieve_credentials_result = libs.retrieve_credentials(expected_credentials_supplier) + actual_result = libs.retrieve_credentials(expected_credentials_supplier) - expected = expected_retrieve_credentials_response.return_value - assert actual_retrieve_credentials_result.username == expected.username - assert actual_retrieve_credentials_result.private_key == expected.key_pair.private_key - assert actual_retrieve_credentials_result.public_key == expected.key_pair.public_key + expected = expected_response.return_value + assert actual_result.username == expected.username + assert actual_result.private_key == expected.key_pair.private_key + assert actual_result.public_key == expected.key_pair.public_key @staticmethod def test_retrieve_credentials_with_actionable_error(): @@ -836,7 +941,8 @@ def test_retrieve_credentials_with_actionable_error(): with mock.patch('dlpx.virtualization._engine.libs.retrieve_credentials', return_value=response, create=True): with pytest.raises(LibraryError) as err_info: - libs.retrieve_credentials({'some supplier property': 'some supplier value'}) + libs.retrieve_credentials( + {'some supplier property': 'some supplier value'}) assert err_info.value._id == expected_id assert err_info.value.message == expected_message @@ -850,7 +956,8 @@ def test_retrieve_credentials_with_nonactionable_error(): with mock.patch('dlpx.virtualization._engine.libs.retrieve_credentials', return_value=response, create=True): with pytest.raises(SystemExit): - libs.retrieve_credentials({'some supplier property': 'some supplier value'}) + libs.retrieve_credentials( + {'some supplier property': 'some supplier value'}) @staticmethod def test_retrieve_credentials_bad_supplier(): @@ -859,10 +966,14 @@ def test_retrieve_credentials_bad_supplier(): with pytest.raises(IncorrectArgumentTypeError) as err_info: libs.retrieve_credentials(credentials_supplier) - - assert err_info.value.message == ( - "The function retrieve_credentials's argument 'credentials_supplier' was" - " type 'int' but should be of type 'dict'.") + if six.PY2: + assert err_info.value.message == ( + "The function retrieve_credentials's argument 'credentials_supplier' " + "was type 'int' but should be of type 'dict'.") + else: + assert err_info.value.message == ( + "The function retrieve_credentials's argument 'credentials_supplier' " + "was class 'int' but should be of class 'dict'.") class TestLibsUpgradePassword: @@ -870,9 +981,11 @@ class TestLibsUpgradePassword: def test_upgrade_password(): expected_password = 'some password' - expected_credentials_supplier = {'type': 'NamedPasswordCredential', 'password': expected_password} + expected_credentials_supplier = { + 'type': 'NamedPasswordCredential', 'password': expected_password} expected_upgrade_password_response = libs_pb2.UpgradePasswordResponse() - expected_upgrade_password_response.return_value.credentials_supplier.update(expected_credentials_supplier) + expected_upgrade_password_response.return_value.credentials_supplier.update( + expected_credentials_supplier) def mock_upgrade_password(actual_upgrade_password_request): assert actual_upgrade_password_request.password == expected_password @@ -890,9 +1003,12 @@ def test_upgrade_password_with_username(): expected_password = 'some password' expected_username = 'some user name' - expected_credentials_supplier = {'type': 'NamedPasswordCredential', 'password': expected_password, 'username': expected_username} + expected_credentials_supplier = { + 'type': 'NamedPasswordCredential', 'password': expected_password, + 'username': expected_username} expected_upgrade_password_response = libs_pb2.UpgradePasswordResponse() - expected_upgrade_password_response.return_value.credentials_supplier.update(expected_credentials_supplier) + expected_upgrade_password_response.return_value.credentials_supplier.update( + expected_credentials_supplier) def mock_upgrade_password(actual_upgrade_password_request): assert actual_upgrade_password_request.password == expected_password @@ -902,7 +1018,8 @@ def mock_upgrade_password(actual_upgrade_password_request): with mock.patch('dlpx.virtualization._engine.libs.upgrade_password', side_effect=mock_upgrade_password, create=True): - actual_upgrade_password_result = libs.upgrade_password(expected_password, username=expected_username) + actual_upgrade_password_result = libs.upgrade_password( + expected_password, username=expected_username) assert actual_upgrade_password_result == expected_credentials_supplier @@ -913,10 +1030,14 @@ def test_upgrade_password_invalid_password(): with pytest.raises(IncorrectArgumentTypeError) as err_info: libs.upgrade_password(expected_password, username=expected_username) - - assert err_info.value.message == ( - "The function upgrade_password's argument 'password' was" - " type 'int' but should be of type 'basestring'.") + if six.PY2: + assert err_info.value.message == ( + "The function upgrade_password's argument 'password' was" + " type 'int' but should be of type 'basestring'.") + else: + assert err_info.value.message == ( + "The function upgrade_password's argument 'password' was" + " class 'int' but should be of class 'str'.") @staticmethod def test_upgrade_password_invalid_username(): @@ -925,7 +1046,11 @@ def test_upgrade_password_invalid_username(): with pytest.raises(IncorrectArgumentTypeError) as err_info: libs.upgrade_password(expected_password, username=expected_username) - - assert err_info.value.message == ( - "The function upgrade_password's argument 'username' was" - " type 'int' but should be of type 'basestring' if defined.") + if six.PY2: + assert err_info.value.message == ( + "The function upgrade_password's argument 'username' was" + " type 'int' but should be of type 'basestring' if defined.") + else: + assert err_info.value.message == ( + "The function upgrade_password's argument 'username' was" + " class 'int' but should be of class 'str' if defined.") diff --git a/libs/src/test/python/dlpx/virtualization/test_logging.py b/libs/src/test/python/dlpx/virtualization/test_logging.py index 022cffa8..45a4ddc2 100644 --- a/libs/src/test/python/dlpx/virtualization/test_logging.py +++ b/libs/src/test/python/dlpx/virtualization/test_logging.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # import logging @@ -30,7 +30,6 @@ def successful_response(): (logging.ERROR, LogRequest.ERROR), (logging.CRITICAL, LogRequest.ERROR) ]) - @mock.patch("dlpx.virtualization._engine.libs", create=True) def test_levels(mock_internal_libs, py_level, expected_level, successful_response): mock_internal_libs.log.return_value = successful_response @@ -67,7 +66,6 @@ def test_format(mock_internal_libs, successful_response): mock_internal_libs.log.assert_called_with(log_request) - @staticmethod @mock.patch("dlpx.virtualization._engine.libs", create=True) def test_log_non_string(mock_internal_libs, successful_response): diff --git a/platform/.python-version b/platform/.python-version deleted file mode 100644 index 43c4dbe6..00000000 --- a/platform/.python-version +++ /dev/null @@ -1 +0,0 @@ -2.7.17 diff --git a/platform/setup.cfg b/platform/setup.cfg index 1e4a607a..5e67bf3e 100644 --- a/platform/setup.cfg +++ b/platform/setup.cfg @@ -1,21 +1,22 @@ # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # [metadata] -Metadata-Version: 1.2 -Author: Delphix -Author-email: virtualization-plugins@delphix.com -Home-page: https://developer.delphix.com -Summary: Delphix Virtualization Platform APIs -Long-description: file: README.md -Long-description-content-type: text/markdown -Classifiers: +metadata_version: 1.2 +author: Delphix +author_email: virtualization-plugins@delphix.com +home_page: https://developer.delphix.com +summary: Delphix Virtualization Platform APIs +long_description: file: README.md +long_description_content_type: text/markdown +classifiers: Development Status :: 5 - Production/Stable Programming Language :: Python Programming Language :: Python :: 2.7 + Programming Language :: Python :: 3.8 License :: OSI Approved :: Apache Software License Operating System :: OS Independent [options] -Requires-Python: 2.7 +requires_python: >=2.7, <=3.8, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*, !=3.7.* diff --git a/platform/setup.py b/platform/setup.py index c9ec3a5d..6cd76b71 100644 --- a/platform/setup.py +++ b/platform/setup.py @@ -7,7 +7,7 @@ version = version_file.read().strip() install_requires = [ - "dvp-api == 1.5.0", + "dvp-api == 1.6.3", "dvp-common == {}".format(version), "enum34;python_version < '3.4'", ] @@ -17,4 +17,5 @@ install_requires=install_requires, package_dir={'': PYTHON_SRC}, packages=setuptools.find_packages(PYTHON_SRC), + python_requires='>=2.7, <3.9, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*, !=3.7.*', ) diff --git a/platform/src/main/python/dlpx/virtualization/platform/VERSION b/platform/src/main/python/dlpx/virtualization/platform/VERSION index fd2a0186..c5106e6d 100644 --- a/platform/src/main/python/dlpx/virtualization/platform/VERSION +++ b/platform/src/main/python/dlpx/virtualization/platform/VERSION @@ -1 +1 @@ -3.1.0 +4.0.4 diff --git a/platform/src/main/python/dlpx/virtualization/platform/__init__.py b/platform/src/main/python/dlpx/virtualization/platform/__init__.py index 589f76e4..f22e502c 100644 --- a/platform/src/main/python/dlpx/virtualization/platform/__init__.py +++ b/platform/src/main/python/dlpx/virtualization/platform/__init__.py @@ -1,17 +1,17 @@ # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # __path__ = __import__('pkgutil').extend_path(__path__, __name__) -from dlpx.virtualization.platform.validation_util import * -from dlpx.virtualization.platform.migration_helper import * -from dlpx.virtualization.platform._plugin_classes import * -from dlpx.virtualization.platform._discovery import * -from dlpx.virtualization.platform._linked import * -from dlpx.virtualization.platform._upgrade import * -from dlpx.virtualization.platform._virtual import * -from dlpx.virtualization.platform._plugin import * -from dlpx.virtualization.platform.util import * -from dlpx.virtualization.platform.import_util import * -from dlpx.virtualization.platform.import_validations import * +from dlpx.virtualization.platform.validation_util import * # noqa +from dlpx.virtualization.platform.migration_helper import * # noqa +from dlpx.virtualization.platform._plugin_classes import * # noqa +from dlpx.virtualization.platform._discovery import * # noqa +from dlpx.virtualization.platform._linked import * # noqa +from dlpx.virtualization.platform._upgrade import * # noqa +from dlpx.virtualization.platform._virtual import * # noqa +from dlpx.virtualization.platform._plugin import * # noqa +from dlpx.virtualization.platform.util import * # noqa +from dlpx.virtualization.platform.import_util import * # noqa +from dlpx.virtualization.platform.import_validations import * # noqa diff --git a/platform/src/main/python/dlpx/virtualization/platform/_plugin_classes.py b/platform/src/main/python/dlpx/virtualization/platform/_plugin_classes.py index 4d2da24f..97ff49ca 100644 --- a/platform/src/main/python/dlpx/virtualization/platform/_plugin_classes.py +++ b/platform/src/main/python/dlpx/virtualization/platform/_plugin_classes.py @@ -1,10 +1,10 @@ # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # import re +import six from enum import Enum -import six from dlpx.virtualization.common import (RemoteConnection, RemoteEnvironment, RemoteHost) from dlpx.virtualization.common.exceptions import IncorrectTypeError @@ -160,8 +160,8 @@ def __init__(self, remote_environment, mount_path, shared_path=None): plugin writer from attempting to provide parameter values that they won't have access to.""" def __is_correct_reference_format(reference): - unix_format = re.compile("^UNIX_HOST_ENVIRONMENT-\d+$") - win_format = re.compile("^WINDOWS_HOST_ENVIRONMENT-\d+$") + unix_format = re.compile(r"^UNIX_HOST_ENVIRONMENT-\d+$") + win_format = re.compile(r"^WINDOWS_HOST_ENVIRONMENT-\d+$") return (bool(unix_format.match(reference)) or bool(win_format.match(reference))) @@ -199,7 +199,7 @@ def __make_remote_environment_from_reference(reference): six.string_types[0]) self._mount_path = mount_path - if shared_path and not isinstance(shared_path, six.string_types[0]): + if shared_path and not isinstance(shared_path, six.string_types): raise IncorrectTypeError(Mount, 'shared_path', type(shared_path), six.string_types[0], False) self._shared_path = shared_path diff --git a/platform/src/main/python/dlpx/virtualization/platform/_virtual.py b/platform/src/main/python/dlpx/virtualization/platform/_virtual.py index 2b2318a9..02d8826e 100644 --- a/platform/src/main/python/dlpx/virtualization/platform/_virtual.py +++ b/platform/src/main/python/dlpx/virtualization/platform/_virtual.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # # -*- coding: utf-8 -*- @@ -25,6 +25,7 @@ class VirtualOperations(object): def __init__(self): self.configure_impl = None self.unconfigure_impl = None + self.cleanup_impl = None self.reconfigure_impl = None self.start_impl = None self.stop_impl = None @@ -54,6 +55,16 @@ def unconfigure_decorator(unconfigure_impl): return unconfigure_decorator + def cleanup(self): + def cleanup_decorator(cleanup_impl): + if self.cleanup_impl: + raise OperationAlreadyDefinedError(Op.VIRTUAL_CLEANUP) + self.cleanup_impl = v.check_function( + cleanup_impl, Op.VIRTUAL_CLEANUP) + return cleanup_impl + + return cleanup_decorator + def reconfigure(self): def reconfigure_decorator(reconfigure_impl): if self.reconfigure_impl: @@ -255,6 +266,58 @@ def _internal_unconfigure(self, request): platform_pb2.UnconfigureResult()) return unconfigure_response + def _internal_cleanup(self, request): + """Cleanup operation wrapper. + + Executed when deleting an existing virtual source. + This plugin operation is run after unconfigure. + + Args: + request (VirtualCleanupRequest): Cleanup operation arguments. + + Returns: + VirtualCleanupResponse: A response containing VirtualCleanupResult + if successful or PluginErrorResult in case of an error. + """ + # Reasoning for method imports are in this file's docstring. + from generated.definitions import VirtualSourceDefinition + from generated.definitions import RepositoryDefinition + from generated.definitions import SourceConfigDefinition + + # + # While virtual.cleanup() is not a required operation, this should + # not be called if it wasn't implemented. + # + if not self.cleanup_impl: + raise OperationNotDefinedError(Op.VIRTUAL_CLEANUP) + + virtual_source_definition = VirtualSourceDefinition.from_dict( + json.loads(request.virtual_source.parameters.json)) + mounts = [ + VirtualOperations._from_protobuf_single_subset_mount(m) + for m in request.virtual_source.mounts + ] + + virtual_source = VirtualSource(guid=request.virtual_source.guid, + connection=RemoteConnection.from_proto( + request.virtual_source.connection), + parameters=virtual_source_definition, + mounts=mounts) + + repository = RepositoryDefinition.from_dict( + json.loads(request.repository.parameters.json)) + source_config = SourceConfigDefinition.from_dict( + json.loads(request.source_config.parameters.json)) + + self.cleanup_impl( + repository=repository, source_config=source_config, + virtual_source=virtual_source) + + virtual_cleanup_response = platform_pb2.VirtualCleanupResponse() + virtual_cleanup_response.return_value.CopyFrom( + platform_pb2.VirtualCleanupResult()) + return virtual_cleanup_response + def _internal_reconfigure(self, request): """Reconfigure operation wrapper. @@ -627,8 +690,8 @@ def _internal_initialize(self, request): repository = RepositoryDefinition.from_dict( json.loads(request.repository.parameters.json)) - config = self.initialize_impl(repository=repository, - virtual_source=virtual_source) + config = self.initialize_impl( + repository=repository, virtual_source=virtual_source) # Validate that this is a SourceConfigDefinition object. if not isinstance(config, SourceConfigDefinition): diff --git a/platform/src/main/python/dlpx/virtualization/platform/import_validations.py b/platform/src/main/python/dlpx/virtualization/platform/import_validations.py index 12960511..d0249d9a 100644 --- a/platform/src/main/python/dlpx/virtualization/platform/import_validations.py +++ b/platform/src/main/python/dlpx/virtualization/platform/import_validations.py @@ -1,7 +1,8 @@ # -# Copyright (c) 2020 by Delphix. All rights reserved. +# Copyright (c) 2020, 2021 by Delphix. All rights reserved. # import inspect +import six from dlpx.virtualization.platform import exceptions from dlpx.virtualization.platform.import_util import (import_check, @@ -25,7 +26,6 @@ def validate_entry_point(plugin_module): if plugin_module.entry_point is None: raise exceptions.IncorrectPluginCodeError( 'Plugin entry point object is None.') - if not hasattr(plugin_module.module_content, plugin_module.entry_point): raise exceptions.UserError( 'Entry point \'{}:{}\' does not exist. \'{}\' is not a symbol' @@ -83,7 +83,10 @@ def validate_named_args(plugin_module): for op_name_key, op_name in plugin_attrib.__dict__.items(): if op_name is None: continue - actual_args = inspect.getargspec(op_name) + if six.PY2: + actual_args = inspect.getargspec(op_name) + else: + actual_args = inspect.getfullargspec(op_name) warnings.extend( _check_args(method_name=op_name.__name__, expected_args=_lookup_expected_args( diff --git a/platform/src/main/python/dlpx/virtualization/platform/migration_helper.py b/platform/src/main/python/dlpx/virtualization/platform/migration_helper.py index b53e7def..92b16337 100644 --- a/platform/src/main/python/dlpx/virtualization/platform/migration_helper.py +++ b/platform/src/main/python/dlpx/virtualization/platform/migration_helper.py @@ -1,8 +1,9 @@ # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # import re +import six from dlpx.virtualization.platform import validation_util as v from dlpx.virtualization.platform.exceptions import ( @@ -142,7 +143,7 @@ def __add(self, migration_id, impl_name): @staticmethod def __validate_migration_id(migration_id, impl_name): # First validate that the id is a string - if not isinstance(migration_id, basestring): + if not isinstance(migration_id, six.string_types): raise MigrationIdIncorrectTypeError(migration_id, impl_name) # Next check if the id is the right format @@ -260,7 +261,7 @@ def add_snapshot(self, migration_id, snapshot_impl): def __validate_lua_major_minor_version(migration_id, impl_name, decorator_name, impl_getter): # First validate that the major minor version is a string - if not isinstance(migration_id, basestring): + if not isinstance(migration_id, six.string_types): raise MigrationIdIncorrectTypeError(migration_id, impl_name) # Next check if the id already exists in this particular dictionary @@ -308,6 +309,7 @@ def __get_sorted_impls(migration_id, impl_dict): # if not migration_id: return [] + # # First filter out all ids less than the migration id. We need to do # this because even after sorting, we wouldn't know where in the list diff --git a/platform/src/main/python/dlpx/virtualization/platform/operation.py b/platform/src/main/python/dlpx/virtualization/platform/operation.py index 12f1fb6d..87a06c31 100644 --- a/platform/src/main/python/dlpx/virtualization/platform/operation.py +++ b/platform/src/main/python/dlpx/virtualization/platform/operation.py @@ -21,6 +21,7 @@ class Operation(Enum): VIRTUAL_CONFIGURE = 'virtual.configure()' VIRTUAL_UNCONFIGURE = 'virtual.unconfigure()' VIRTUAL_RECONFIGURE = 'virtual.reconfigure()' + VIRTUAL_CLEANUP = 'virtual.cleanup()' VIRTUAL_START = 'virtual.start()' VIRTUAL_STOP = 'virtual.stop()' VIRTUAL_PRE_SNAPSHOT = 'virtual.pre_snapshot()' diff --git a/platform/src/main/python/dlpx/virtualization/platform/util.py b/platform/src/main/python/dlpx/virtualization/platform/util.py index d80a8718..79037b9e 100644 --- a/platform/src/main/python/dlpx/virtualization/platform/util.py +++ b/platform/src/main/python/dlpx/virtualization/platform/util.py @@ -1,7 +1,8 @@ # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # import dlpx.virtualization.api +from dlpx.virtualization.common.util import to_str def get_virtualization_api_version(): @@ -9,4 +10,4 @@ def get_virtualization_api_version(): :return: version string """ - return dlpx.virtualization.api.__version__ + return to_str(dlpx.virtualization.api.__version__) diff --git a/platform/src/test/python/dlpx/virtualization/fake_generated_definitions.py b/platform/src/test/python/dlpx/virtualization/fake_generated_definitions.py index 90fbcef6..e4ef3419 100644 --- a/platform/src/test/python/dlpx/virtualization/fake_generated_definitions.py +++ b/platform/src/test/python/dlpx/virtualization/fake_generated_definitions.py @@ -1,3 +1,6 @@ +import six + + class Model(object): # swaggerTypes: The key is attribute name and the # value is attribute type. @@ -10,7 +13,7 @@ class Model(object): class RepositoryDefinition(Model): def __init__(self, name): - self.swagger_types = {'name': str} + self.swagger_types = {'name': six.string_types[0]} self.attribute_map = {'name': 'name'} self._name = name @@ -29,7 +32,7 @@ def to_dict(self): class SourceConfigDefinition(Model): def __init__(self, name): - self.swagger_types = {'name': str} + self.swagger_types = {'name': six.string_types[0]} self.attribute_map = {'name': 'name'} self._name = name @@ -48,7 +51,7 @@ def to_dict(self): class LinkedSourceDefinition(Model): def __init__(self, name): - self.swagger_types = {'name': str} + self.swagger_types = {'name': six.string_types[0]} self.attribute_map = {'name': 'name'} self._name = name @@ -64,7 +67,7 @@ def from_dict(input_dict): class VirtualSourceDefinition(Model): def __init__(self, name): - self.swagger_types = {'name': str} + self.swagger_types = {'name': six.string_types[0]} self.attribute_map = {'name': 'name'} self._name = name @@ -83,7 +86,7 @@ def to_dict(self): class SnapshotDefinition(Model): def __init__(self, name): - self.swagger_types = {'name': str} + self.swagger_types = {'name': six.string_types[0]} self.attribute_map = {'name': 'name'} self._name = name diff --git a/platform/src/test/python/dlpx/virtualization/test_migration_helper.py b/platform/src/test/python/dlpx/virtualization/test_migration_helper.py index de90d246..ac5ba436 100644 --- a/platform/src/test/python/dlpx/virtualization/test_migration_helper.py +++ b/platform/src/test/python/dlpx/virtualization/test_migration_helper.py @@ -1,9 +1,10 @@ +from __future__ import absolute_import # # Copyright (c) 2019 by Delphix. All rights reserved. # import pytest -import conftest +from . import conftest from dlpx.virtualization.platform import migration_helper as m from dlpx.virtualization.platform.exceptions import ( MigrationIdAlreadyUsedError, MigrationIdIncorrectFormatError, diff --git a/platform/src/test/python/dlpx/virtualization/test_plugin.py b/platform/src/test/python/dlpx/virtualization/test_plugin.py index 8cfde728..f69258ca 100755 --- a/platform/src/test/python/dlpx/virtualization/test_plugin.py +++ b/platform/src/test/python/dlpx/virtualization/test_plugin.py @@ -1,22 +1,24 @@ +from __future__ import absolute_import # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # import json import pytest +import re +import six from dlpx.virtualization.api import common_pb2, platform_pb2 -from dlpx.virtualization.common import (RemoteConnection, RemoteEnvironment, - RemoteHost, RemoteUser) +from dlpx.virtualization.common import ( + RemoteConnection, RemoteEnvironment, RemoteHost, RemoteUser) from dlpx.virtualization.platform.exceptions import ( IncorrectReturnTypeError, IncorrectUpgradeObjectTypeError, OperationAlreadyDefinedError, PluginRuntimeError) from mock import MagicMock, patch -import fake_generated_definitions -from fake_generated_definitions import (RepositoryDefinition, - SnapshotDefinition, - SourceConfigDefinition) +from . import fake_generated_definitions +from .fake_generated_definitions import ( + RepositoryDefinition, SnapshotDefinition, SourceConfigDefinition) TEST_BINARY_PATH = '/binary/path' TEST_SCRATCH_PATH = '/scratch/path' @@ -63,9 +65,10 @@ })) TEST_POST_UPGRADE_PARAMS = ({ u'obj': - '"{\\"obj\\": {\\"prettyName\\": \\"prettyUpgrade\\", ' - '\\"name\\": \\"upgrade\\", \\"metadata\\": \\"metadata\\"}}"' + '"{\\"obj\\": {\\"name\\": \\"upgrade\\", ' + '\\"prettyName\\": \\"prettyUpgrade\\", \\"metadata\\": \\"metadata\\"}}"' }) + MIGRATION_IDS = ('2020.1.1', '2020.2.2') @@ -92,8 +95,8 @@ def configure_impl(): with pytest.raises(OperationAlreadyDefinedError): - @my_plugin.virtual.configure() - def configure_impl(): + @my_plugin.virtual.configure() # noqa F811 + def configure_impl(): # noqa F811 pass class NotModel1: @@ -467,10 +470,16 @@ def virtual_configure_impl(virtual_source, repository, snapshot): my_plugin.virtual._internal_configure(configure_request) message = err_info.value.message - assert message == ( - "The returned object for the virtual.configure() operation was" - " type 'unicode' but should be of class 'dlpx.virtualization." - "fake_generated_definitions.SourceConfigDefinition'.") + if six.PY2: + assert message == ( + "The returned object for the virtual.configure() operation was" + " type 'unicode' but should be of class 'dlpx.virtualization." + "fake_generated_definitions.SourceConfigDefinition'.") + else: + assert message == ( + "The returned object for the virtual.configure() operation was" + " class 'str' but should be of class 'dlpx.virtualization." + "fake_generated_definitions.SourceConfigDefinition'.") @staticmethod def test_virtual_unconfigure(my_plugin, virtual_source, repository, @@ -526,14 +535,11 @@ def virtual_reconfigure_impl(virtual_source, repository, source_config, assert config.parameters.json == expected_source_config @staticmethod - def test_virtual_reconfigure_return_incorrect_type(my_plugin, - virtual_source, - repository, - source_config, - snapshot): + def test_virtual_reconfigure_return_incorrect_type( + my_plugin, virtual_source, repository, source_config, snapshot): @my_plugin.virtual.reconfigure() - def virtual_reconfigure_impl(virtual_source, repository, source_config, - snapshot): + def virtual_reconfigure_impl( + virtual_source, repository, source_config, snapshot): TestPlugin.assert_plugin_args(virtual_source=virtual_source, source_config=source_config, repository=repository, @@ -553,14 +559,43 @@ def virtual_reconfigure_impl(virtual_source, repository, source_config, my_plugin.virtual._internal_reconfigure(reconfigure_request) message = err_info.value.message - assert message == ( - "The returned object for the virtual.reconfigure() operation was" - " type 'unicode' but should be of class 'dlpx.virtualization." - "fake_generated_definitions.SourceConfigDefinition'.") + if six.PY2: + assert message == ( + "The returned object for the virtual.reconfigure() operation was" + " type 'unicode' but should be of class 'dlpx.virtualization." + "fake_generated_definitions.SourceConfigDefinition'.") + else: + assert message == ( + "The returned object for the virtual.reconfigure() operation was" + " class 'str' but should be of class 'dlpx.virtualization." + "fake_generated_definitions.SourceConfigDefinition'.") + + @staticmethod + def test_virtual_cleanup(my_plugin, virtual_source, repository, source_config): + @my_plugin.virtual.cleanup() + def virtual_cleanup_impl(virtual_source, repository, source_config): + TestPlugin.assert_plugin_args(virtual_source=virtual_source, + repository=repository, + source_config=source_config) + return + + virtual_cleanup_request = platform_pb2.VirtualCleanupRequest() + TestPlugin.setup_request(request=virtual_cleanup_request, + virtual_source=virtual_source, + repository=repository, + source_config=source_config) + + expected_result = platform_pb2.VirtualCleanupResult() + + virtual_cleanup_response = my_plugin.virtual._internal_cleanup( + virtual_cleanup_request) + + # Check that the response's oneof is set to return_value and not error + assert virtual_cleanup_response.WhichOneof('result') == 'return_value' + assert virtual_cleanup_response.return_value == expected_result @staticmethod - def test_virtual_start(my_plugin, virtual_source, repository, - source_config): + def test_virtual_start(my_plugin, virtual_source, repository, source_config): @my_plugin.virtual.start() def virtual_start_impl(virtual_source, repository, source_config): TestPlugin.assert_plugin_args(virtual_source=virtual_source, @@ -695,8 +730,6 @@ def virtual_initialize_impl(virtual_source, repository): TestPlugin.setup_request(request=initialize_request, virtual_source=virtual_source, repository=repository) - - expected_source_config = TEST_REPOSITORY_JSON initialize_response = my_plugin.virtual._internal_initialize( initialize_request) @@ -722,10 +755,16 @@ def virtual_initialize_impl(virtual_source, repository): with pytest.raises(IncorrectReturnTypeError) as err_info: my_plugin.virtual._internal_initialize(initialize_request) message = err_info.value.message - assert message == ( - "The returned object for the virtual.initialize() operation was" - " type 'NoneType' but should be of class 'dlpx.virtualization." - "fake_generated_definitions.SourceConfigDefinition'.") + if six.PY2: + assert message == ( + "The returned object for the virtual.initialize() operation was" + " type 'NoneType' but should be of class 'dlpx.virtualization." + "fake_generated_definitions.SourceConfigDefinition'.") + else: + assert message == ( + "The returned object for the virtual.initialize() operation was" + " class 'NoneType' but should be of class 'dlpx.virtualization." + "fake_generated_definitions.SourceConfigDefinition'.") @staticmethod def test_virtual_mount_spec(my_plugin, virtual_source, repository): @@ -802,12 +841,20 @@ def repository_discovery_impl(source_connection): repository_discovery_request) message = err_info.value.message - assert message == ( - "The returned object for the discovery.repository() operation was" - " a list of [type 'str', class 'dlpx.virtualization" - ".fake_generated_definitions.RepositoryDefinition'] but should" - " be of type 'list of dlpx.virtualization" - ".fake_generated_definitions.RepositoryDefinition'.") + if six.PY2: + assert message == ( + "The returned object for the discovery.repository() operation was" + " a list of [type 'str', class 'dlpx.virtualization" + ".fake_generated_definitions.RepositoryDefinition'] but should" + " be of type 'list of dlpx.virtualization" + ".fake_generated_definitions.RepositoryDefinition'.") + else: + assert message == ( + "The returned object for the discovery.repository() operation was" + " a list of [class 'str', class 'dlpx.virtualization" + ".fake_generated_definitions.RepositoryDefinition'] but should" + " be of type 'list of dlpx.virtualization" + ".fake_generated_definitions.RepositoryDefinition'.") @staticmethod def test_source_config_discovery(my_plugin, connection, repository): @@ -867,8 +914,9 @@ def mock_direct_pre_snapshot(direct_source, repository, source_config, def test_direct_pre_snapshot_null_snapparams(my_plugin, direct_source, repository, source_config): @my_plugin.linked.pre_snapshot() - def mock_direct_pre_snapshot(direct_source, repository, source_config, - optional_snapshot_parameters): + def mock_direct_pre_snapshot( + direct_source, repository, source_config, + optional_snapshot_parameters): TestPlugin.assert_direct_source(direct_source) TestPlugin.assert_repository(repository) TestPlugin.assert_source_config(source_config) @@ -924,11 +972,11 @@ def direct_post_snapshot_impl(direct_source, repository, source_config, assert snapshot.parameters.json == expected_snapshot @staticmethod - def test_direct_post_snapshot_null_snapparams(my_plugin, direct_source, - repository, source_config): + def test_direct_post_snapshot_null_snapparams( + my_plugin, direct_source, repository, source_config): @my_plugin.linked.post_snapshot() def direct_post_snapshot_impl(direct_source, repository, source_config, - optional_snapshot_parameters): + optional_snapshot_parameters): TestPlugin.assert_direct_source(direct_source) TestPlugin.assert_repository(repository) TestPlugin.assert_source_config(source_config) @@ -987,7 +1035,7 @@ def test_staged_pre_snapshot_null_snapparams(my_plugin, staged_source, repository, source_config): @my_plugin.linked.pre_snapshot() def staged_pre_snapshot_impl(staged_source, repository, source_config, - optional_snapshot_parameters): + optional_snapshot_parameters): TestPlugin.assert_staged_source(staged_source) TestPlugin.assert_repository(repository) TestPlugin.assert_source_config(source_config) @@ -1045,7 +1093,7 @@ def test_staged_post_snapshot_null_snapparams(my_plugin, staged_source, repository, source_config): @my_plugin.linked.post_snapshot() def staged_post_snapshot_impl(staged_source, repository, source_config, - optional_snapshot_parameters): + optional_snapshot_parameters): TestPlugin.assert_staged_source(staged_source) TestPlugin.assert_repository(repository) TestPlugin.assert_source_config(source_config) @@ -1235,8 +1283,8 @@ def test_upgrade_repository_success(my_plugin): def upgrade_repository(old_repository): return TEST_POST_MIGRATION_METADATA_1 - @my_plugin.upgrade.repository('2020.2.2') - def upgrade_repository(old_repository): + @my_plugin.upgrade.repository('2020.2.2') # noqa F811 + def upgrade_repository(old_repository): # noqa F811 return TEST_POST_MIGRATION_METADATA_2 upgrade_request = platform_pb2.UpgradeRequest() @@ -1251,7 +1299,17 @@ def upgrade_repository(old_repository): expected_response.return_value.post_upgrade_parameters\ .update(TEST_POST_UPGRADE_PARAMS) - assert expected_response == upgrade_response + pat = re.compile( + 'return_value{post_upgrade_parameters{key:"obj"value:""{"obj":{("' + 'prettyName":"prettyUpgrade"|"name":"upgrade"),("prettyName":"' + 'prettyUpgrade"|"name":"upgrade"),"metadata":"metadata"}}""}}' + ) + assert re.match( + pat, str(expected_response).replace( + "\n", "").replace("\\", "").replace(" ", "")) + assert re.match( + pat, str(upgrade_response).replace( + "\n", "").replace("\\", "").replace(" ", "")) @staticmethod def test_upgrade_source_config_success(my_plugin): @@ -1259,8 +1317,8 @@ def test_upgrade_source_config_success(my_plugin): def upgrade_source_config(old_source_config): return TEST_POST_MIGRATION_METADATA_1 - @my_plugin.upgrade.source_config('2020.2.2') - def upgrade_source_config(old_source_config): + @my_plugin.upgrade.source_config('2020.2.2') # noqa F811 + def upgrade_source_config(old_source_config): # noqa F811 return TEST_POST_MIGRATION_METADATA_2 upgrade_request = platform_pb2.UpgradeRequest() @@ -1275,7 +1333,17 @@ def upgrade_source_config(old_source_config): expected_response.return_value.post_upgrade_parameters \ .update(TEST_POST_UPGRADE_PARAMS) - assert expected_response == upgrade_response + pat = re.compile( + 'return_value{post_upgrade_parameters{key:"obj"value:""{"obj":' + '{("prettyName":"prettyUpgrade"|"name":"upgrade"),("prettyName":"' + 'prettyUpgrade"|"name":"upgrade"),"metadata":"metadata"}}""}}' + ) + assert re.match( + pat, str(expected_response).replace( + "\n", "").replace("\\", "").replace(" ", "")) + assert re.match( + pat, str(upgrade_response).replace( + "\n", "").replace("\\", "").replace(" ", "")) @staticmethod def test_upgrade_linked_source_success(my_plugin): @@ -1283,8 +1351,8 @@ def test_upgrade_linked_source_success(my_plugin): def upgrade_linked_source(old_linked_source): return TEST_POST_MIGRATION_METADATA_1 - @my_plugin.upgrade.linked_source('2020.2.2') - def upgrade_linked_source(old_linked_source): + @my_plugin.upgrade.linked_source('2020.2.2') # noqa F811 + def upgrade_linked_source(old_linked_source): # noqa F811 return TEST_POST_MIGRATION_METADATA_2 upgrade_request = platform_pb2.UpgradeRequest() @@ -1299,7 +1367,17 @@ def upgrade_linked_source(old_linked_source): expected_response.return_value.post_upgrade_parameters \ .update(TEST_POST_UPGRADE_PARAMS) - assert expected_response == upgrade_response + pat = re.compile( + 'return_value{post_upgrade_parameters{key:"obj"value:""{"obj":{("' + 'prettyName":"prettyUpgrade"|"name":"upgrade"),("prettyName":"' + 'prettyUpgrade"|"name":"upgrade"),"metadata":"metadata"}}""}}' + ) + assert re.match( + pat, str(expected_response).replace( + "\n", "").replace("\\", "").replace(" ", "")) + assert re.match( + pat, str(upgrade_response).replace( + "\n", "").replace("\\", "").replace(" ", "")) @staticmethod def test_upgrade_virtual_source_success(my_plugin): @@ -1307,8 +1385,8 @@ def test_upgrade_virtual_source_success(my_plugin): def upgrade_virtual_source(old_virtual_source): return TEST_POST_MIGRATION_METADATA_1 - @my_plugin.upgrade.virtual_source('2020.2.2') - def upgrade_virtual_source(old_virtual_source): + @my_plugin.upgrade.virtual_source('2020.2.2') # noqa F811 + def upgrade_virtual_source(old_virtual_source): # noqa F811 return TEST_POST_MIGRATION_METADATA_2 upgrade_request = platform_pb2.UpgradeRequest() @@ -1323,7 +1401,17 @@ def upgrade_virtual_source(old_virtual_source): expected_response.return_value.post_upgrade_parameters \ .update(TEST_POST_UPGRADE_PARAMS) - assert expected_response == upgrade_response + pat = re.compile( + 'return_value{post_upgrade_parameters{key:"obj"value:""{"obj":{("name":' + '"upgrade"|"prettyName":"prettyUpgrade"),("name":"upgrade"|"prettyName":' + '"prettyUpgrade"),"metadata":"metadata"}}""}}' + ) + assert re.match( + pat, str(expected_response).replace( + "\n", "").replace("\\", "").replace(" ", "")) + assert re.match( + pat, str(upgrade_response).replace( + "\n", "").replace("\\", "").replace(" ", "")) @staticmethod def test_upgrade_snapshot_success(my_plugin): @@ -1331,8 +1419,8 @@ def test_upgrade_snapshot_success(my_plugin): def upgrade_snapshot(old_snapshot): return TEST_POST_MIGRATION_METADATA_1 - @my_plugin.upgrade.snapshot('2020.2.2') - def upgrade_snapshot(old_snapshot): + @my_plugin.upgrade.snapshot('2020.2.2') # noqa F811 + def upgrade_snapshot(old_snapshot): # noqa F811 return TEST_POST_MIGRATION_METADATA_2 upgrade_request = platform_pb2.UpgradeRequest() @@ -1347,7 +1435,17 @@ def upgrade_snapshot(old_snapshot): expected_response.return_value.post_upgrade_parameters \ .update(TEST_POST_UPGRADE_PARAMS) - assert expected_response == upgrade_response + pat = re.compile( + 'return_value{post_upgrade_parameters{key:"obj"value:""{"obj":{("' + 'prettyName":"prettyUpgrade"|"name":"upgrade"),("name":"' + 'upgrade"|"prettyName":"prettyUpgrade"),"metadata":"metadata"}}""}}' + ) + assert re.match( + pat, str(expected_response).replace( + "\n", "").replace("\\", "").replace(" ", "")) + assert re.match( + pat, str(upgrade_response).replace( + "\n", "").replace("\\", "").replace(" ", "")) @staticmethod def test_upgrade_repository_incorrect_upgrade_object_type(my_plugin): @@ -1415,8 +1513,8 @@ def test_upgrade_snapshot_fail_with_runtime_error(my_plugin): def upgrade_snapshot(old_snapshot): raise RuntimeError('RuntimeError in snapshot migration') - @my_plugin.upgrade.snapshot('2020.2.2') - def upgrade_snapshot(old_snapshot): + @my_plugin.upgrade.snapshot('2020.2.2') # noqa F811 + def upgrade_snapshot(old_snapshot): # noqa F811 raise RuntimeError('RuntimeError in snapshot migration') upgrade_request = platform_pb2.UpgradeRequest() diff --git a/platform/src/test/python/dlpx/virtualization/test_plugin_classes.py b/platform/src/test/python/dlpx/virtualization/test_plugin_classes.py index c1e5215a..fb12496a 100644 --- a/platform/src/test/python/dlpx/virtualization/test_plugin_classes.py +++ b/platform/src/test/python/dlpx/virtualization/test_plugin_classes.py @@ -1,8 +1,9 @@ # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # import pytest +import six from dlpx.virtualization.common._common_classes import (RemoteEnvironment, RemoteHost) from dlpx.virtualization.common.exceptions import IncorrectTypeError @@ -40,17 +41,27 @@ def test_init_mount_bad_remote_env(): def test_init_mount_bad_mount_path(remote_environment): with pytest.raises(IncorrectTypeError) as err_info: Mount(remote_environment, 10000, 'shared_path') - assert err_info.value.message == ( - "Mount's parameter 'mount_path' was type 'int' but should" - " be of type 'basestring'.") + if six.PY2: + assert err_info.value.message == ( + "Mount's parameter 'mount_path' was type 'int' but should" + " be of type 'basestring'.") + else: + assert err_info.value.message == ( + "Mount's parameter 'mount_path' was class 'int' but should" + " be of class 'str'.") @staticmethod def test_init_mount_bad_shared_path(remote_environment): with pytest.raises(IncorrectTypeError) as err_info: Mount(remote_environment, 'mount_path', 10000) - assert err_info.value.message == ( - "Mount's parameter 'shared_path' was type 'int' but should" - " be of type 'basestring' if defined.") + if six.PY2: + assert err_info.value.message == ( + "Mount's parameter 'shared_path' was type 'int' but should" + " be of type 'basestring' if defined.") + else: + assert err_info.value.message == ( + "Mount's parameter 'shared_path' was class 'int' but should" + " be of class 'str' if defined.") @staticmethod def test_init_ownership_spec(): @@ -60,17 +71,27 @@ def test_init_ownership_spec(): def test_init_ownership_spec_bad_uid(): with pytest.raises(IncorrectTypeError) as err_info: OwnershipSpecification('10', 10) - assert err_info.value.message == ( - "OwnershipSpecification's parameter 'uid' was type 'str' but" - " should be of type 'int'.") + if six.PY2: + assert err_info.value.message == ( + "OwnershipSpecification's parameter 'uid' was type 'str' but" + " should be of type 'int'.") + else: + assert err_info.value.message == ( + "OwnershipSpecification's parameter 'uid' was class 'str' but" + " should be of class 'int'.") @staticmethod def test_init_ownership_spec_bad_gid(): with pytest.raises(IncorrectTypeError) as err_info: OwnershipSpecification(10, '10') - assert err_info.value.message == ( - "OwnershipSpecification's parameter 'gid' was type 'str' but" - " should be of type 'int'.") + if six.PY2: + assert err_info.value.message == ( + "OwnershipSpecification's parameter 'gid' was type 'str' but" + " should be of type 'int'.") + else: + assert err_info.value.message == ( + "OwnershipSpecification's parameter 'gid' was class 'str' but" + " should be of class 'int'.") @staticmethod def test_init_mount_spec(remote_environment): @@ -108,29 +129,48 @@ def test_init_mount_incorrect_format_reference_string(reference_string): def test_init_mount_invalid_reference_type(reference): with pytest.raises(IncorrectTypeError) as err_info: Mount(reference, 'mount_path', 'shared_path') - assert err_info.value.message == ( - "Mount's parameter 'remote_environment' was type '{}' but " - "should be of any one of the following types: " - "'['dlpx.virtualization.common._common_classes.RemoteEnvironment'," - " 'basestring']'.".format(type(reference).__name__)) + if six.PY2: + assert err_info.value.message == ( + "Mount's parameter 'remote_environment' was type '{}' but " + "should be of any one of the following types: " + "'['dlpx.virtualization.common._common_classes.RemoteEnvironment'," + " 'basestring']'.".format(type(reference).__name__)) + else: + assert err_info.value.message == ( + "Mount's parameter 'remote_environment' was class '{}' but " + "should be of any one of the following types: " + "'['dlpx.virtualization.common._common_classes.RemoteEnvironment'," + " 'str']'.".format(type(reference).__name__)) @staticmethod def test_init_mount_spec_mounts_not_list(): with pytest.raises(IncorrectTypeError) as err_info: MountSpecification('string', OwnershipSpecification(10, 10)) - assert err_info.value.message == ( - "MountSpecification's parameter 'mounts' was type 'str' but" - " should be of type 'list of dlpx.virtualization.platform" - "._plugin_classes.Mount'.") + if six.PY2: + assert err_info.value.message == ( + "MountSpecification's parameter 'mounts' was type 'str' but" + " should be of type 'list of dlpx.virtualization.platform" + "._plugin_classes.Mount'.") + else: + assert err_info.value.message == ( + "MountSpecification's parameter 'mounts' was class 'str' but" + " should be of type 'list of dlpx.virtualization.platform" + "._plugin_classes.Mount'.") @staticmethod def test_init_mount_spec_bad_mounts(): with pytest.raises(IncorrectTypeError) as err_info: MountSpecification(['string'], OwnershipSpecification(10, 10)) - assert err_info.value.message == ( - "MountSpecification's parameter 'mounts' was a list of" - " [type 'str'] but should be of type 'list of dlpx.virtualization" - ".platform._plugin_classes.Mount'.") + if six.PY2: + assert err_info.value.message == ( + "MountSpecification's parameter 'mounts' was a list of" + " [type 'str'] but should be of type 'list of dlpx.virtualization" + ".platform._plugin_classes.Mount'.") + else: + assert err_info.value.message == ( + "MountSpecification's parameter 'mounts' was a list of" + " [class 'str'] but should be of type 'list of dlpx.virtualization" + ".platform._plugin_classes.Mount'.") @staticmethod def test_init_mount_spec_bad_owner_spec(remote_environment): @@ -138,8 +178,15 @@ def test_init_mount_spec_bad_owner_spec(remote_environment): with pytest.raises(IncorrectTypeError) as err_info: MountSpecification([mount], 'string') - assert err_info.value.message == ( - "MountSpecification's parameter 'ownership_specification' was" - " type 'str' but should be of class 'dlpx.virtualization" - ".platform._plugin_classes.OwnershipSpecification'" - " if defined.") + if six.PY2: + assert err_info.value.message == ( + "MountSpecification's parameter 'ownership_specification' was" + " type 'str' but should be of class 'dlpx.virtualization" + ".platform._plugin_classes.OwnershipSpecification'" + " if defined.") + else: + assert err_info.value.message == ( + "MountSpecification's parameter 'ownership_specification' was" + " class 'str' but should be of class 'dlpx.virtualization" + ".platform._plugin_classes.OwnershipSpecification'" + " if defined.") diff --git a/platform/src/test/python/dlpx/virtualization/test_upgrade.py b/platform/src/test/python/dlpx/virtualization/test_upgrade.py index f5801e28..7990d384 100755 --- a/platform/src/test/python/dlpx/virtualization/test_upgrade.py +++ b/platform/src/test/python/dlpx/virtualization/test_upgrade.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # import copy @@ -254,8 +254,8 @@ def repo_upgrade_one(input_dict): output_dict['migrations'].append('lua repo 1.1') return output_dict - @upgrade_type_decorator('1.2', MigrationType.LUA) - def repo_upgrade_one(input_dict): + @upgrade_type_decorator('1.2', MigrationType.LUA) # noqa F811 + def repo_upgrade_one(input_dict): # noqa F811 output_dict = copy.deepcopy(input_dict) output_dict['migrations'].append('lua repo 1.2') return output_dict @@ -266,14 +266,14 @@ def repo_upgrade_two(input_dict): output_dict['migrations'].append('platform repo 2020.4.2') return output_dict - @upgrade_type_decorator('2020.4.3') - def repo_upgrade_two(input_dict): + @upgrade_type_decorator('2020.4.3') # noqa F811 + def repo_upgrade_two(input_dict): # noqa F811 output_dict = copy.deepcopy(input_dict) output_dict['migrations'].append('platform repo 2020.4.3') return output_dict - @upgrade_type_decorator('2020.4.4') - def repo_upgrade_two(input_dict): + @upgrade_type_decorator('2020.4.4') # noqa F811 + def repo_upgrade_two(input_dict): # noqa F811 output_dict = copy.deepcopy(input_dict) output_dict['migrations'].append('platform repo 2020.4.4') return output_dict diff --git a/tools/.python-version b/tools/.python-version deleted file mode 100644 index 43c4dbe6..00000000 --- a/tools/.python-version +++ /dev/null @@ -1 +0,0 @@ -2.7.17 diff --git a/tools/setup.cfg b/tools/setup.cfg index 690c13de..91544edc 100644 --- a/tools/setup.cfg +++ b/tools/setup.cfg @@ -1,25 +1,25 @@ # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # [metadata] -Metadata-Version: 1.2 -Author: Delphix -Author-email: virtualization-plugins@delphix.com -Home-page: https://developer.delphix.com -Summary: Delphix Virtualization SDK Tools -Long-description: file: README.md -Long-description-content-type: text/markdown -Classifiers: +metadata_version: 1.2 +author: Delphix +author_email: virtualization-plugins@delphix.com +home_page: https://developer.delphix.com +summary: Delphix Virtualization SDK Tools +long_description: file: README.md +long_description_content_type: text/markdown +classifiers: Development Status :: 5 - Production/Stable Programming Language :: Python - Programming Language :: Python :: 2.7 + Programming Language :: Python :: 3.8 License :: OSI Approved :: Apache Software License Operating System :: OS Independent [options] include_package_data = True -Requires-Python: 2.7 +requires_python: 3.8 [options.entry_points] console_scripts = diff --git a/tools/setup.py b/tools/setup.py index b5e0edc1..0e91ae4f 100644 --- a/tools/setup.py +++ b/tools/setup.py @@ -7,15 +7,16 @@ version = version_file.read().strip() install_requires = [ - "click >= 7.1", - "click-configfile == 0.2.3", - "dvp-platform == {}".format(version), - "enum34 >= 1.1.6", - "flake8 >= 3.6", - "jinja2 >= 2.10", - "jsonschema >= 3", - "pyyaml >= 3", - "requests >= 2.21.0", + "click == 7.1.2", + "click-configfile == 0.2.3", + "dvp-platform == {}".format(version), + "enum34 >= 1.1.6", + "flake8 >= 3.6", + "jinja2 >= 2.10", + "jsonschema >= 3", + "pyyaml >= 3", + "requests >= 2.21.0", + "httpretty == 0.9.7", ] setuptools.setup(name='dvp-tools', @@ -23,4 +24,5 @@ install_requires=install_requires, package_dir={'': PYTHON_SRC}, packages=setuptools.find_packages(PYTHON_SRC), + python_requires='>=3.8, <3.9', ) diff --git a/tools/src/main/python/dlpx/virtualization/_internal/VERSION b/tools/src/main/python/dlpx/virtualization/_internal/VERSION index fd2a0186..c5106e6d 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/VERSION +++ b/tools/src/main/python/dlpx/virtualization/_internal/VERSION @@ -1 +1 @@ -3.1.0 +4.0.4 diff --git a/tools/src/main/python/dlpx/virtualization/_internal/cli.py b/tools/src/main/python/dlpx/virtualization/_internal/cli.py index 7d785591..ea7094b5 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/cli.py +++ b/tools/src/main/python/dlpx/virtualization/_internal/cli.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2019, 2020 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # import logging @@ -45,8 +45,7 @@ # adding it to CONTEXT_SETTINGS to avoid any side-effects on other commands. # CONTEXT_SETTINGS_INIT = dict(help_option_names=['-h', '--help'], - obj=click_util.ConfigFileProcessor.read_config(), - token_normalize_func=lambda x: x.encode("ascii")) + obj=click_util.ConfigFileProcessor.read_config()) DVP_CONFIG_MAP = CONTEXT_SETTINGS['obj'] @@ -96,11 +95,11 @@ def delphix_sdk(verbose, quiet): # will be printed to the console until this is executed. # logging_util.add_console_handler(console_logging_level) - - if sys.version_info[:2] != (2, 7): + if sys.version_info[:2] != (3, 8): raise exceptions.UserError( 'Python version check failed.' - 'Supported version is 2.7.x, found {}'.format(sys.version_info)) + 'Supported versions are 2.7.x and 3.8.x, found {}' + .format(sys.version_info)) @delphix_sdk.command(context_settings=CONTEXT_SETTINGS_INIT) diff --git a/tools/src/main/python/dlpx/virtualization/_internal/codegen.py b/tools/src/main/python/dlpx/virtualization/_internal/codegen.py index 30b1529c..cfc2d91d 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/codegen.py +++ b/tools/src/main/python/dlpx/virtualization/_internal/codegen.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2019, 2020 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # import copy @@ -8,9 +8,11 @@ import logging import os import shutil +import six import subprocess from dlpx.virtualization._internal import const, exceptions, file_util +from dlpx.virtualization.common.util import to_str logger = logging.getLogger(__name__) UNKNOWN_ERR = 'UNKNOWN_ERR' @@ -96,8 +98,9 @@ def generate_python(name, source_dir, plugin_config_dir, schema_content): # def _make_url_refs_opaque(json): if isinstance(json, dict): - for key in json: - if key == '$ref' and isinstance(json[key], basestring)\ + keys = list(json.keys()) + for key in keys: + if key == '$ref' and isinstance(json[key], six.string_types)\ and json[key].startswith('https://delphix.com/platform/api#'): json.pop(key) json['type'] = 'object' @@ -181,8 +184,7 @@ def _execute_swagger_codegen(swagger_file, output_dir): output_dir ] - logger.info('Running process with arguments: {!r}'.format( - ' '.join(process_inputs))) + logger.info(f"Running process with arguments: \'{' '.join(process_inputs)}\'") process = subprocess.Popen(process_inputs, stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -191,12 +193,13 @@ def _execute_swagger_codegen(swagger_file, output_dir): raise exceptions.UserError('Swagger python code generation failed.' ' Make sure java is on the PATH.') raise exceptions.UserError( - 'Unable to run {!r} to generate python code.' - '\nError code: {}. Error message: {}'.format( - jar, err.errno, os.strerror(err.errno))) + f"Unable to run '{jar}' to generate python code." + f"\nError code: {err.errno}. Error message: {os.strerror(err.errno)}") # Get the pipes pointed so we have access to them. stdout, stderr = process.communicate() + stdout = to_str(stdout) + stderr = to_str(stderr) # # Wait for the process to end and take the results. If res then we know diff --git a/tools/src/main/python/dlpx/virtualization/_internal/codegen/templates/base_model_.mustache b/tools/src/main/python/dlpx/virtualization/_internal/codegen/templates/base_model_.mustache index 80f49d0b..cf30e9de 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/codegen/templates/base_model_.mustache +++ b/tools/src/main/python/dlpx/virtualization/_internal/codegen/templates/base_model_.mustache @@ -1,10 +1,12 @@ # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # import pprint - import six +import hashlib +import json + {{^supportPython2}} import typing {{/supportPython2}} @@ -37,7 +39,7 @@ class Model(object): """ result = {} - for attr, _ in six.iteritems(self.swagger_types): + for attr, _ in self.swagger_types.items(): value = getattr(self, attr) if value is None: # Plugins use the JSON schema specification to define their @@ -95,6 +97,11 @@ class Model(object): """Returns true if both objects are equal""" return self.to_dict() == other.to_dict() + def __hash__(self): + bytes_val = json.dumps(self.to_dict(), sort_keys=True, ensure_ascii=True, default=str) + hash_val = hashlib.sha1(bytes_val.encode()).hexdigest() + return int(hash_val, 16) + def __ne__(self, other): """Returns true if both objects are not equal""" return not self == other @@ -170,7 +177,7 @@ class GeneratedClassesTypeError(GeneratedClassesError): container then we assume there is one element in it and that type is the expected type of the container. (For dicts this is the key) ie: if expected_type = {str} then the returned expected string with - be something like "type 'dict with key basestring'" + be something like "type 'dict with key str'" Returns: tuple (str, str): the actual and expected strings used for the @@ -180,13 +187,16 @@ class GeneratedClassesTypeError(GeneratedClassesError): def _remove_angle_brackets(type_string): return type_string.replace('<', '').replace('>', '') + # In py3 the builtins module will be named 'builtins', in py2 it will + # be '__builtin__'. + builtins = ['__builtin__', 'builtins'] if isinstance(expected_type, list): if len(expected_type) != 1: raise ValueError('The thrown GeneratedClassesTypeError should' ' have had a list of size 1 as the' ' expected_type') single_type = expected_type[0] - if single_type.__module__ != '__builtin__': + if single_type.__module__ not in builtins: type_name = '{}.{}'.format( single_type.__module__, single_type.__name__) else: @@ -198,7 +208,10 @@ class GeneratedClassesTypeError(GeneratedClassesError): ' have had a set of size 1 as the' ' expected_type') single_type = expected_type.pop() - if single_type.__module__ != '__builtin__': + + + st_module = single_type.__module__ + if st_module not in builtins: type_name = '{}.{}'.format( single_type.__module__, single_type.__name__) else: @@ -209,14 +222,14 @@ class GeneratedClassesTypeError(GeneratedClassesError): raise ValueError('The thrown GeneratedClassesTypeError should' ' have had a dict of size 1 as the' ' expected_type') - key_type = expected_type.keys()[0] - value_type = expected_type.values()[0] - if key_type.__module__ != '__builtin__': + key_type = list(expected_type.keys())[0] + value_type = list(expected_type.values())[0] + if key_type.__module__ not in builtins: key_type_name = '{}.{}'.format( key_type.__module__, key_type.__name__) else: key_type_name = key_type.__name__ - if value_type.__module__ != '__builtin__': + if value_type.__module__ not in builtins: value_type_name = '{}.{}'.format( value_type.__module__, value_type.__name__) else: @@ -224,11 +237,11 @@ class GeneratedClassesTypeError(GeneratedClassesError): expected = "type 'dict of {}:{}'".format( key_type_name, value_type_name) else: - expected = _remove_angle_brackets(str(expected_type)) + expected = _remove_angle_brackets(six.string_types[0](expected_type)) if isinstance(actual_type, list): actual = 'a list of [{}]'.format( - ', '.join(_remove_angle_brackets(str(single_type)) + ', '.join(_remove_angle_brackets(six.string_types[0](single_type)) for single_type in actual_type)) elif isinstance(actual_type, set): # @@ -241,19 +254,19 @@ class GeneratedClassesTypeError(GeneratedClassesError): for type_tuple in actual_type)): actual = 'a dict with keys of {}{}{}'.format( '{', - ', '.join(_remove_angle_brackets(str(single_type)) + ', '.join(_remove_angle_brackets(six.string_types[0](single_type)) for single_type in actual_type), '}') else: actual = 'a dict of {}{}{}'.format( '{', ', '.join(['{0}:{1}'.format( - _remove_angle_brackets(str(k)), - _remove_angle_brackets(str(v))) for k, v in actual_type]), + _remove_angle_brackets(six.string_types[0](k)), + _remove_angle_brackets(six.string_types[0](v))) for k, v in actual_type]), '}') else: - actual = _remove_angle_brackets(str(actual_type)) + actual = _remove_angle_brackets(six.string_types[0](actual_type)) return actual, expected @@ -276,19 +289,20 @@ class GeneratedClassesTypeError(GeneratedClassesError): if not required and parameter is None: return None # Now check if the types are incorrect. + num_types = (float, complex, int) if expected_type == float: - if not isinstance(parameter, (float, int, long, complex)): + if not isinstance(parameter, num_types): return GeneratedClassesTypeError(object_type, parameter_name, type(parameter), float, required) - elif expected_type == str: - if not isinstance(parameter, basestring): + elif expected_type == six.string_types[0]: + if not isinstance(parameter, six.string_types): return GeneratedClassesTypeError(object_type, parameter_name, type(parameter), - basestring, + six.string_types[0], required) elif expected_type == list: if element_type: @@ -300,7 +314,7 @@ class GeneratedClassesTypeError(GeneratedClassesError): required) if element_type == float: - check = all(isinstance(elem, (float, int, long, complex)) + check = all(isinstance(elem, num_types) for elem in parameter) else: check = all(isinstance(elem, element_type) @@ -324,7 +338,7 @@ class GeneratedClassesTypeError(GeneratedClassesError): return GeneratedClassesTypeError(object_type, parameter_name, type(parameter), - {basestring}, + {six.string_types[0]}, required) # @@ -336,26 +350,26 @@ class GeneratedClassesTypeError(GeneratedClassesError): if element_type: if element_type == float: value_check = all(isinstance(v, - (float, int, long, complex)) + num_types) for v in parameter.values()) else: value_check = all(isinstance(v, element_type) for v in parameter.values()) - if (not all(isinstance(k, basestring) + if (not all(isinstance(k, six.string_types) for k in parameter.keys()) or not value_check): return GeneratedClassesTypeError( object_type, parameter_name, {(type(k), type(v)) for k, v in parameter.items()}, - {basestring: element_type}, + {six.string_types[0]: element_type}, required) else: - if not all(isinstance(k, basestring) for k in parameter.keys()): + if not all(isinstance(k, six.string_types) for k in parameter.keys()): return GeneratedClassesTypeError( object_type, parameter_name, {type(k) for k in parameter.keys()}, - {basestring}, + {six.string_types[0]}, required) else: diff --git a/tools/src/main/python/dlpx/virtualization/_internal/codegen/templates/util.mustache b/tools/src/main/python/dlpx/virtualization/_internal/codegen/templates/util.mustache index 008528ae..b6a59aa2 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/codegen/templates/util.mustache +++ b/tools/src/main/python/dlpx/virtualization/_internal/codegen/templates/util.mustache @@ -1,7 +1,9 @@ +# +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. +# import datetime import pydoc import re - import six @@ -33,9 +35,9 @@ def get_contained_type(type_string): for pattern in patterns: match = re.search(pattern, type_string) if match and match.group(1) != 'ERRORUNKNOWN': - # Convert the type to basestring here. + # Convert the type to string here. if match.group(1) == 'str': - return basestring + return six.string_types[0] return pydoc.locate(match.group(1)) return None @@ -53,7 +55,7 @@ def deserialize_model(data, klass): if not instance.swagger_types: return data - for attr, attr_type in six.iteritems(instance.swagger_types): + for attr, attr_type in instance.swagger_types.items(): if (data is not None and instance.attribute_map[attr] in data and isinstance(data, dict)): value = data[instance.attribute_map[attr]] @@ -73,7 +75,8 @@ def _deserialize(data, klass): if data is None: return None - if issubclass(klass, (int, float, long, complex, basestring, bool)): + primitive_types = {float, complex, bool, int, six.text_type, six.binary_type} + if issubclass(klass, tuple(primitive_types)): return _deserialize_primitive(data, klass) elif klass == datetime.date: return deserialize_date(data) @@ -93,21 +96,20 @@ def _deserialize_primitive(data, klass): :param data: data to deserialize. :param klass: class literal. - :return: int, float, long, complex, basestring, bool. - :rtype: int | float | long | complex | basestring | bool + :return: int, float, long, complex, str, bool. + :rtype: int | float | long | complex | str | bool """ try: value = klass(data) except UnicodeEncodeError: - if isinstance(data, str): + if isinstance(data, six.binary_type): # # Ignore errors even if the string is not proper UTF-8 or has - # broken marker bytes. The builtin unicode function can do this. + # broken marker bytes. # - value = unicode(data, 'utf-8', errors='ignore') + value = str(data, "utf-8", errors='ignore') else: - # Assume the value object has proper __unicode__() method. - value = unicode(data) + value = six.text_type(data) except TypeError: value = data return value @@ -166,4 +168,4 @@ def _deserialize_dict(data): :return: deserialized dict. :rtype: dict """ - return {k: _deserialize(v, type(v)) for k, v in six.iteritems(data)} + return {k: _deserialize(v, type(v)) for k, v in data.items()} diff --git a/tools/src/main/python/dlpx/virtualization/_internal/commands/build.py b/tools/src/main/python/dlpx/virtualization/_internal/commands/build.py index 30eff7ab..83a16a14 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/commands/build.py +++ b/tools/src/main/python/dlpx/virtualization/_internal/commands/build.py @@ -1,19 +1,20 @@ # -# Copyright (c) 2019, 2020 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # import base64 import compileall import copy +import io import json import logging import os -import StringIO import zipfile from dlpx.virtualization._internal import (codegen, exceptions, file_util, package_util, plugin_dependency_util, plugin_util) +from dlpx.virtualization.common.util import to_str logger = logging.getLogger(__name__) @@ -56,8 +57,16 @@ def build(plugin_config, 'Build parameters include plugin_config: %s, upload_artifact: %s,' ' generate_only: %s', plugin_config, upload_artifact, generate_only) + # Click handles the conversions for us, so we need not run inputs through to_str in + # the cli build function. However, this function may be called from places other + # than its corresponding cli function, such as unit tests. As such, we should ensure + # all appropriate inputs at this point are properly converted to unicode strings as + # soon as they enter the program. + plugin_config = to_str(plugin_config) + upload_artifact = to_str(upload_artifact) + if local_vsdk_root: - local_vsdk_root = os.path.expanduser(local_vsdk_root) + local_vsdk_root = to_str(os.path.expanduser(local_vsdk_root)) # Read content of the plugin config file provided and perform validations logger.info('Validating plugin config file %s', plugin_config) @@ -100,7 +109,6 @@ def build(plugin_config, os.path.dirname(plugin_config), schemas) except exceptions.UserError as err: raise exceptions.BuildFailedError(err) - if generate_only: # # If the generate_only flag is set then just return after generation @@ -137,9 +145,23 @@ def build(plugin_config, # Copy everything from the source directory into the build directory. file_util.clean_copy(src_dir, build_src_dir) + # + # At this point, build folder only consists of files and folders from src directory. + # Also, the symlinks folder in src directory are copied as normal folder to build + # directory. When we create the zip files, we copy all the .pyc files (skip the .py + # files) from the build directory. We are compiling the build folder with + # legacy=true to make sure pyc files are created within the same directory + # as py files. + # + plugin_dependency_util.compile_py_files(build_src_dir) + # Install dependencies in the plugin's source root in the build directory. plugin_dependency_util.install_deps(build_src_dir, local_vsdk_root=local_vsdk_root) + virtualization_dir = os.path.join(build_src_dir, "dlpx", "virtualization") + + for pkg in ['api', 'common', 'libs', 'platform']: + plugin_dependency_util.compile_py_files(os.path.join(virtualization_dir, pkg)) # Patch dependencies. patch_dependencies(build_src_dir) @@ -160,7 +182,7 @@ def build(plugin_config, logger.info('Successfully generated artifact file at %s.', upload_artifact) - logger.warn('\nBUILD SUCCESSFUL.') + logger.warning('\nBUILD SUCCESSFUL.') def patch_dependencies(build_src_dir): @@ -174,7 +196,7 @@ def patch_dependencies(build_src_dir): json_format_path = os.path.join( build_src_dir, 'google', 'protobuf', 'json_format.py') with open(json_format_path, 'r') as f: - json_format_text = f.read() + json_format_text = to_str(f.read()) json_format_text = json_format_text\ .replace(UNPAIRED_SURROGATE_DEFINITION, '')\ .replace(UNPAIRED_SURROGATE_SEARCH, '') @@ -187,6 +209,7 @@ def prepare_upload_artifact(plugin_config_content, src_dir, schemas, manifest): # This is the output dictionary that will be written # to the upload_artifact. # + artifact = { # Hard code the type to a set default. 'type': @@ -198,7 +221,7 @@ def prepare_upload_artifact(plugin_config_content, src_dir, schemas, manifest): # set default value of locale to en-us 'defaultLocale': plugin_config_content.get('defaultLocale', LOCALE_DEFAULT), - # set default value of language to PYTHON27 + # set default value of language to PYTHON38 'language': plugin_config_content['language'], 'hostTypes': @@ -252,7 +275,7 @@ def prepare_upload_artifact(plugin_config_content, src_dir, schemas, manifest): artifact['minimumLuaVersion'] = plugin_config_content[ 'minimumLuaVersion'] - return artifact + return to_str(artifact) def get_linked_source_definition_type(plugin_config_content): @@ -332,7 +355,7 @@ def zip_and_encode_source_files(source_code_dir): Jython creates a class loader to import .py files which the security manager prohibits. """ - + source_code_dir = to_str(source_code_dir) # # The contents of the zip should have relative and not absolute paths or # else the imports won't work as expected. @@ -348,7 +371,8 @@ def zip_and_encode_source_files(source_code_dir): raise exceptions.UserError( 'Failed to compile source code in the directory {}.'.format( source_code_dir)) - out_file = StringIO.StringIO() + + out_file = io.BytesIO() with zipfile.ZipFile(out_file, 'w', zipfile.ZIP_DEFLATED) as zip_file: for root, _, files in os.walk('.'): for filename in files: diff --git a/tools/src/main/python/dlpx/virtualization/_internal/commands/download_logs.py b/tools/src/main/python/dlpx/virtualization/_internal/commands/download_logs.py index 7000b701..b29f5e17 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/commands/download_logs.py +++ b/tools/src/main/python/dlpx/virtualization/_internal/commands/download_logs.py @@ -1,10 +1,11 @@ # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # import logging from dlpx.virtualization._internal import delphix_client, package_util +from dlpx.virtualization.common.util import to_str logger = logging.getLogger(__name__) @@ -34,6 +35,17 @@ def download_logs(engine, plugin_config, user, password, directory): logger.info('Downloading plugin logs from {} to: {}'.format( engine, directory)) + # Click handles the conversions for us, so we need not run inputs through to_str in + # the cli download_logs function. However, to be safe, this function may be called + # from places other than its corresponding cli function, such as unit tests. As + # such, we should ensure all appropriate inputs at this point are properly + # converted to unicode strings as soon as they enter the program. + engine = to_str(engine) + plugin_config = to_str(plugin_config) + user = to_str(user) + password = to_str(password) + directory = to_str(directory) + # Create a new delphix session. client = delphix_client.DelphixClient(engine) client.login(package_util.get_engine_api_version(), user, password) diff --git a/tools/src/main/python/dlpx/virtualization/_internal/commands/initialize.py b/tools/src/main/python/dlpx/virtualization/_internal/commands/initialize.py index 7d6d04b5..0975d822 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/commands/initialize.py +++ b/tools/src/main/python/dlpx/virtualization/_internal/commands/initialize.py @@ -1,10 +1,11 @@ # -# Copyright (c) 2019, 2020 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # import logging import os import shutil +import six import uuid from collections import OrderedDict @@ -12,6 +13,7 @@ import yaml from dlpx.virtualization._internal import (codegen, const, exceptions, file_util, plugin_util) +from dlpx.virtualization.common.util import to_bytes, to_str logger = logging.getLogger(__name__) @@ -60,6 +62,16 @@ def init(root, ingestion_strategy, name, host_type): 'Host Types': host_type }) + # Click handles the conversions for us, so we need not run inputs through to_str in + # the cli init function. However, this function may be called from places other + # than its corresponding cli function, such as unit tests. As such, we should ensure + # all appropriate inputs at this point are properly converted to unicode strings as + # soon as they enter the program. + root = to_str(root) + ingestion_strategy = to_str(ingestion_strategy) + name = to_str(name) + host_type = to_str(host_type) + # Files paths based on 'root' to be used throughout src_dir_path = os.path.join(root, DEFAULT_SRC_DIRECTORY) config_file_path = os.path.join(root, DEFAULT_PLUGIN_CONFIG_FILE) @@ -207,17 +219,30 @@ def _get_default_plugin_config(plugin_id, ingestion_strategy, name, OrderedDict: A valid plugin configuration roughly ordered from most interesting to a new plugin author to least interesting. """ - # Ensure values are type 'str'. If they are type unicode yaml prints - # them with '!!python/unicode' prepended to the value. - config = OrderedDict([('id', plugin_id.encode('utf-8')), - ('name', name.encode('utf-8')), - ('language', 'PYTHON27'), ('hostTypes', ['UNIX']), - ('pluginType', ingestion_strategy.encode('utf-8')), - ('entryPoint', entry_point.encode('utf-8')), - ('srcDir', src_dir_path.encode('utf-8')), - ('schemaFile', schema_file_path.encode('utf-8')), - ('hostTypes', [host_type.encode('utf-8')]), - ('buildNumber', default_build_number.encode('utf-8')) + if six.PY2: + # + # Ensure values are type 'str'. If they are type unicode yaml prints + # them with '!!python/unicode' prepended to the value. + # + # In Py3 yaml will print bytes with `!!binary |` prepended to them, so we + # should leave the as strings. + # + plugin_id = to_bytes(plugin_id) + name = to_bytes(name) + ingestion_strategy = to_bytes(ingestion_strategy) + entry_point = to_bytes(entry_point) + src_dir_path = to_bytes(src_dir_path) + schema_file_path = to_bytes(schema_file_path) + host_type = to_bytes(host_type) + default_build_number = to_bytes(default_build_number) + config = OrderedDict([('id', plugin_id), + ('name', name), + ('language', 'PYTHON38'), ('hostTypes', ['UNIX']), + ('pluginType', ingestion_strategy), + ('entryPoint', entry_point), + ('srcDir', src_dir_path), + ('schemaFile', schema_file_path), + ('hostTypes', [host_type]), + ('buildNumber', default_build_number) ]) - return config diff --git a/tools/src/main/python/dlpx/virtualization/_internal/commands/upload.py b/tools/src/main/python/dlpx/virtualization/_internal/commands/upload.py index 1395f4ae..fbfa58a3 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/commands/upload.py +++ b/tools/src/main/python/dlpx/virtualization/_internal/commands/upload.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # import errno @@ -8,6 +8,7 @@ import os from dlpx.virtualization._internal import delphix_client, exceptions +from dlpx.virtualization.common.util import to_str logger = logging.getLogger(__name__) UNKNOWN_ERR = 'UNKNOWN_ERR' @@ -36,6 +37,18 @@ def upload(engine, user, upload_artifact, password, wait): ' wait: {}'.format(engine, user, upload_artifact, wait)) logger.info('Uploading plugin artifact {} ...'.format(upload_artifact)) + # + # Click handles the conversions for us, so we need not run inputs through to_str in + # the cli upload function. However, this function may be called from places other + # than its corresponding cli function, such as unit tests. As such, we should ensure + # all appropriate inputs at this point are properly converted to unicode strings as + # soon as they enter the program. + # + engine = to_str(engine) + user = to_str(user) + upload_artifact = to_str(upload_artifact) + password = to_str(password) + # Read content of upload artifact try: with open(upload_artifact, 'rb') as f: diff --git a/tools/src/main/python/dlpx/virtualization/_internal/delphix_client.py b/tools/src/main/python/dlpx/virtualization/_internal/delphix_client.py index 03b5799e..55d941f0 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/delphix_client.py +++ b/tools/src/main/python/dlpx/virtualization/_internal/delphix_client.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # import json @@ -9,6 +9,7 @@ import requests from dlpx.virtualization._internal import exceptions, plugin_util +from dlpx.virtualization.common.util import to_bytes, to_str logger = logging.getLogger(__name__) @@ -36,12 +37,11 @@ def login(self, engine_api, user, password): Takes in the engine_api, user, and password and attempts to login to the engine. Can raise HttpPostError and UnexpectedError. """ - logger.info('Logging onto the Delphix Engine {!r}.'.format( - self.__engine)) + logger.info(f"Logging onto the Delphix Engine '{self.__engine}'.") self.__post('delphix/session', data={ - 'type': 'APISession', - 'version': engine_api + 'version': engine_api, + 'type': 'APISession' }) logger.debug('Session started successfully.') self.__post('delphix/login', @@ -50,7 +50,7 @@ def login(self, engine_api, user, password): 'username': user, 'password': password }) - logger.info('Successfully logged in as {!r}.'.format(user)) + logger.info(f"Successfully logged in as '{user}'.") @staticmethod def get_engine_api(artifact_content): @@ -70,7 +70,7 @@ def get_engine_api(artifact_content): json.dumps(engine_api))) return engine_api logger.debug( - 'engineApi found but malformed: {!r}'.format(engine_api)) + f"engineApi found but malformed: '{engine_api}'") raise exceptions.InvalidArtifactError() def __post(self, resource, content_type='application/json', data=None): @@ -93,7 +93,8 @@ def __post(self, resource, content_type='application/json', data=None): # Issue post request that was passed in, if data is a dict then convert # it to a json string. # - if data is not None and not isinstance(data, (str, bytes, unicode)): + if data is not None and not isinstance(data, (str, bytes)): + data = to_str(data) data = json.dumps(data) try: response = requests.post(url=url, data=data, headers=headers) @@ -211,7 +212,7 @@ def __download_logs(self, plugin_name, token, directory): "dlpx-plugin-logs-{}-{}.tar.gz".format(plugin_name, token)) with open(download_zip_name, "wb") as f: for chunk in download_zip_data: - f.write(chunk) + f.write(to_bytes(chunk)) def upload_plugin(self, name, content, wait): """ @@ -223,9 +224,9 @@ def upload_plugin(self, name, content, wait): logger.debug('Getting token to do upload.') response = self.__post('delphix/toolkit/requestUploadToken') token = response['result']['token'] - logger.debug('Got token {!r} successfully.'.format(token)) + logger.debug(f"Got token '{token}' successfully.") - logger.info('Uploading plugin {!r}.'.format(name)) + logger.info(f"Uploading plugin '{name}'.") # Encode plugin content. upload_response = self.__post('delphix/data/upload', content_type=self.__UPLOAD_CONTENT, @@ -306,8 +307,8 @@ def download_plugin_logs(self, directory, plugin_config): } response = self.__post('delphix/service/support/bundle/generate', data=data) - token = response['result'].encode('utf-8').strip() - logger.debug('Got token {!r} successfully.'.format(token)) + token = to_str(response['result'].encode('utf-8').strip()) + logger.debug(f"Got token '{token}' successfully.") self.__download_logs(plugin_name, token, directory) diff --git a/tools/src/main/python/dlpx/virtualization/_internal/exceptions.py b/tools/src/main/python/dlpx/virtualization/_internal/exceptions.py index 5013e2d5..decff5cd 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/exceptions.py +++ b/tools/src/main/python/dlpx/virtualization/_internal/exceptions.py @@ -1,11 +1,13 @@ # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # import collections import json import re +from dlpx.virtualization.common.util import to_str + class SDKToolingError(Exception): """ @@ -255,14 +257,15 @@ def __format_error(err): 'type': 'object'} """ # - # Validation error message could be unicode encoded string. Strip out - # any leading unicode characters for proper display and logging. + # Validation error message could be byte string. Strip out + # any leading byte characters for proper display and logging. # - err_msg = re.compile(r'\bu\b', re.IGNORECASE) + err_msg = re.compile(r'\bb\b', re.IGNORECASE) err_msg = err_msg.sub("", err.message) + map_func = to_str error_string = 'Error: {} on {}'.format( - err_msg, map(str, list(err.schema_path))) + err_msg, list(map(map_func, list(err.schema_path)))) return error_string diff --git a/tools/src/main/python/dlpx/virtualization/_internal/package_util.py b/tools/src/main/python/dlpx/virtualization/_internal/package_util.py index 1eb75725..298256e5 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/package_util.py +++ b/tools/src/main/python/dlpx/virtualization/_internal/package_util.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # import functools @@ -38,7 +38,7 @@ def _get_settings(): This assumes that the settings file is in the root of dlpx.virtualization._internal. """ - parser = configparser.SafeConfigParser() + parser = configparser.ConfigParser() parser.read(os.path.join(get_internal_package_root(), SETTINGS_FILE_NAME)) return parser diff --git a/tools/src/main/python/dlpx/virtualization/_internal/plugin_dependency_util.py b/tools/src/main/python/dlpx/virtualization/_internal/plugin_dependency_util.py index 1a90a154..4ae8ea88 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/plugin_dependency_util.py +++ b/tools/src/main/python/dlpx/virtualization/_internal/plugin_dependency_util.py @@ -1,14 +1,17 @@ # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # +import compileall import logging import os +import py_compile import subprocess import sys from dlpx.virtualization._internal import file_util, package_util from dlpx.virtualization._internal.exceptions import SubprocessFailedError +from dlpx.virtualization.common.util import to_str logger = logging.getLogger(__name__) @@ -110,12 +113,12 @@ def _execute_pip(pip_args): """ args = [sys.executable, '-m', 'pip'] args.extend(pip_args) - logger.debug('Executing %s', ' '.join(args)) proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) all_output, _ = proc.communicate() + all_output = to_str(all_output) exit_code = proc.wait() # @@ -168,9 +171,29 @@ def _build_wheel(package_root, target_dir=None): cwd=package_root) all_output, _ = proc.communicate() + all_output = to_str(all_output) exit_code = proc.wait() if exit_code != 0: raise SubprocessFailedError(' '.join(args), exit_code, all_output) else: logger.debug(all_output) + + +def compile_py_files(dpath): + """ + Compiles the python files in the given directory, generating the pyc files in the + same directory. + """ + compileall.compile_dir(dpath, force=True, quiet=1, legacy=True, ddir=".") + + +def compile_py_file(fpath: str): + """ + Compiles the python file at the given path, generating the pyc file in the + same directory. + + :param fpath: + :return: + """ + py_compile.compile(fpath, cfile=f"{fpath}c") diff --git a/tools/src/main/python/dlpx/virtualization/_internal/plugin_importer.py b/tools/src/main/python/dlpx/virtualization/_internal/plugin_importer.py index f3ad2119..858ef5a9 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/plugin_importer.py +++ b/tools/src/main/python/dlpx/virtualization/_internal/plugin_importer.py @@ -1,7 +1,6 @@ # -# Copyright (c) 2019, 2020 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # -import importlib import logging import os import sys @@ -11,6 +10,7 @@ import yaml from dlpx.virtualization._internal import const, exceptions from dlpx.virtualization.platform import import_util +from dlpx.virtualization._internal import plugin_dependency_util logger = logging.getLogger(__name__) @@ -97,7 +97,6 @@ def __internal_import(self): self.__plugin_entry_point, self.__src_dir, err)) warnings['exception'].append(exception_msg) - return plugin_manifest, warnings @staticmethod @@ -161,7 +160,7 @@ def __run_checks(self, warnings): # warning_msg = exceptions.ValidationFailedError( warnings).message - logger.warn(warning_msg) + logger.warning(warning_msg) def __check_for_required_methods(self): """ @@ -189,8 +188,6 @@ def _import_module_and_get_manifest(queue, src_dir, module, entry_point, """ Imports the plugin module, runs validations and returns the manifest. """ - module_content = None - try: module_content = _import_helper(queue, src_dir, module) except exceptions.UserError: @@ -248,10 +245,29 @@ def _import_helper(queue, src_dir, module): exceptions. """ module_content = None - sys.path.append(src_dir) + try: + # + # Compile the plugin_runner module to be copied over to the docker container's + # plugin dir. + # The module comes in the format + # + appended_paths = _add_dirs_to_sys_path(src_dir) + plugin_dependency_util.compile_py_files(src_dir) + except FileNotFoundError: + # + # If the module couldn't be found, the user's entry point has the incorrect + # module name specified. + # + error = exceptions.UserError("No module named {}".format(module)) + queue.put({'exception': error}) try: - module_content = importlib.import_module(module) + # + # Module comes in the format pkg.[subpkg1.]*module. Since we've added the + # necessary paths to sys.path, we can import the module directly to retrieve + # its contents + # + module_content = __import__(module.split(".")[-1]) except (ImportError, TypeError) as err: queue.put({'exception': err}) except Exception as err: @@ -274,14 +290,32 @@ def _import_helper(queue, src_dir, module): error = exceptions.SDKToolingError(str(err)) queue.put({'sdk exception': error}) finally: - sys.path.remove(src_dir) + for p in appended_paths: + sys.path.remove(p) if not module_content: raise exceptions.UserError("Plugin module content is None") - return module_content +def _add_dirs_to_sys_path(src_dir, appended_paths=None): + if appended_paths is None: + appended_paths = [] + if src_dir not in sys.path: + sys.path.append(src_dir) + appended_paths.append(src_dir) + for root, dirs, _ in os.walk(src_dir): + for dir_name in dirs: + if dir_name in ["__pycache__"]: + continue + src_dir = os.path.sep.join([root, dir_name]) + if src_dir not in sys.path: + sys.path.append(src_dir) + appended_paths.append(src_dir) + _add_dirs_to_sys_path(src_dir, appended_paths=appended_paths) + return appended_paths + + def _process_warnings(queue, warnings): for warning in warnings: queue.put({'exception': warning}) @@ -330,6 +364,8 @@ def _prepare_manifest(entry_point, module_content): bool(plugin_object.virtual.unconfigure_impl), 'hasVirtualReconfigure': bool(plugin_object.virtual.reconfigure_impl), + 'hasVirtualCleanup': + bool(plugin_object.virtual.cleanup_impl), 'hasVirtualStart': bool(plugin_object.virtual.start_impl), 'hasVirtualStop': diff --git a/tools/src/main/python/dlpx/virtualization/_internal/plugin_validator.py b/tools/src/main/python/dlpx/virtualization/_internal/plugin_validator.py index 9608a2db..821b3587 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/plugin_validator.py +++ b/tools/src/main/python/dlpx/virtualization/_internal/plugin_validator.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # import json @@ -12,6 +12,7 @@ from dlpx.virtualization._internal.codegen import CODEGEN_PACKAGE from flake8.api import legacy as flake8 from jsonschema import Draft7Validator +from dlpx.virtualization.common.util import to_bytes, to_str logger = logging.getLogger(__name__) @@ -33,6 +34,10 @@ def __init__(self, plugin_config, plugin_config_schema, plugin_config_content=None): + plugin_config = to_str(plugin_config) + plugin_config_schema = to_str(plugin_config_schema) + if plugin_config_content is not None: + plugin_config_content = to_str(plugin_config_content) self.__plugin_config = plugin_config self.__plugin_config_schema = plugin_config_schema self.__plugin_config_content = plugin_config_content @@ -83,7 +88,7 @@ def __read_plugin_config_file(self): try: with open(self.__plugin_config, 'rb') as f: try: - return yaml.safe_load(f) + return to_str(yaml.safe_load(f)) except yaml.YAMLError as err: if hasattr(err, 'problem_mark'): mark = err.problem_mark @@ -126,7 +131,7 @@ def __validate_plugin_config_content(self): schemaFile: the file containing defined schemas in the plugin manualDiscovery whether or not manual discovery is supported pluginType whether the plugin is DIRECT or STAGED - language language of the source code(ex: PYTHON27 for python2.7) + language language of the source code(ex: PYTHON38 for python3.8) Args: plugin_config_content (dict): A dictionary representing a plugin @@ -140,7 +145,7 @@ def __validate_plugin_config_content(self): try: with open(self.__plugin_config_schema, 'r') as f: try: - plugin_schema = json.load(f) + plugin_schema = to_str(json.load(f)) except ValueError as err: raise exceptions.UserError( 'Failed to load schemas because {} is not a ' @@ -155,8 +160,8 @@ def __validate_plugin_config_content(self): os.strerror(err.errno))) # Convert plugin config content to json - plugin_config_json = json.loads( - json.dumps(self.__plugin_config_content)) + plugin_config_json = to_str(json.loads( + to_bytes(json.dumps(self.__plugin_config_content)))) # Validate the plugin config against the schema v = Draft7Validator(plugin_schema) diff --git a/tools/src/main/python/dlpx/virtualization/_internal/schema_validator.py b/tools/src/main/python/dlpx/virtualization/_internal/schema_validator.py index 46354fce..75388f4b 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/schema_validator.py +++ b/tools/src/main/python/dlpx/virtualization/_internal/schema_validator.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # import json @@ -8,6 +8,7 @@ from collections import namedtuple from dlpx.virtualization._internal import exceptions +from dlpx.virtualization.common.util import to_str from jsonschema import Draft7Validator logger = logging.getLogger(__name__) @@ -25,9 +26,9 @@ class SchemaValidator: back. """ def __init__(self, schema_file, plugin_meta_schema, schemas=None): - self.__schema_file = schema_file - self.__plugin_meta_schema = plugin_meta_schema - self.__plugin_schemas = schemas + self.__schema_file = to_str(schema_file) + self.__plugin_meta_schema = to_str(plugin_meta_schema) + self.__plugin_schemas = to_str(schemas) @property def result(self): @@ -55,7 +56,7 @@ def __read_schema_file(self): try: with open(self.__schema_file, 'r') as f: try: - return json.load(f) + return to_str(json.load(f)) except ValueError as err: raise exceptions.UserError( 'Failed to load schemas because \'{}\' is not a ' @@ -77,7 +78,7 @@ def __validate_schemas(self): try: with open(self.__plugin_meta_schema, 'r') as f: try: - plugin_meta_schema = json.load(f) + plugin_meta_schema = to_str(json.load(f)) except ValueError as err: raise exceptions.UserError( 'Failed to load schemas because \'{}\' is not a ' @@ -98,8 +99,12 @@ def __validate_schemas(self): # This will do lazy validation so that we can consolidate all the # validation errors and report everything wrong with the schema. # - validation_errors = sorted(v.iter_errors(self.__plugin_schemas), - key=lambda e: e.path) + # In Python 3.8, we are using jsonschema 4.X.X. This version of jsonschema + # breaks when we pass a dictionary to Draft7Validator.iter_errors(). + # Instead it expects a list. + # + errors = v.iter_errors(self.__plugin_schemas) + validation_errors = sorted(errors, key=lambda e: e.path) if validation_errors: raise exceptions.SchemaValidationError(self.__schema_file, diff --git a/tools/src/main/python/dlpx/virtualization/_internal/settings.cfg b/tools/src/main/python/dlpx/virtualization/_internal/settings.cfg index 700afb91..d71b6a5d 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/settings.cfg +++ b/tools/src/main/python/dlpx/virtualization/_internal/settings.cfg @@ -20,7 +20,7 @@ # versions in those packages until they are shipped out of band. # [General] -engine_api_version = 1.11.6 +engine_api_version = 1.11.11 distribution_name = dvp-tools package_author = Delphix namespace_package = dlpx diff --git a/tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_config_schema.json b/tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_config_schema.json index 15f80871..4c6f3f3f 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_config_schema.json +++ b/tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_config_schema.json @@ -38,7 +38,7 @@ }, "language": { "type": "string", - "enum": ["PYTHON27"] + "enum": ["PYTHON38"] }, "rootSquashEnabled": { "type": "boolean" diff --git a/tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_importer.yaml b/tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_importer.yaml index d769cd62..b04e88fc 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_importer.yaml +++ b/tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_importer.yaml @@ -52,6 +52,10 @@ EXPECTED_STAGED_ARGS_BY_OP: - repository - source_config - snapshot + cleanup_impl: + - virtual_source + - repository + - source_config start_impl: - virtual_source - repository @@ -112,6 +116,10 @@ EXPECTED_DIRECT_ARGS_BY_OP: - repository - source_config - snapshot + cleanup_impl: + - virtual_source + - repository + - source_config start_impl: - virtual_source - repository diff --git a/tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_schema.json b/tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_schema.json index b8be1836..edbfeab6 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_schema.json +++ b/tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_schema.json @@ -37,10 +37,6 @@ }, "type": ["object", "boolean"], "properties": { - "$id": { - "type": "string", - "format": "uri-reference" - }, "$schema": { "type": "string", "format": "uri" diff --git a/tools/src/test/python/dlpx/virtualization/_internal/commands/test_build.py b/tools/src/test/python/dlpx/virtualization/_internal/commands/test_build.py index bcc5a9ca..d611fc8b 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/commands/test_build.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/commands/test_build.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2019, 2020 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # import json @@ -35,7 +35,9 @@ class TestBuild: @mock.patch( 'dlpx.virtualization._internal.plugin_dependency_util.install_deps') @mock.patch('os.path.isabs', return_value=False) - def test_build_success(mock_relative_path, mock_install_deps, + @mock.patch( + 'dlpx.virtualization._internal.plugin_dependency_util.compile_py_files') + def test_build_success(mock_compile_py_files, mock_relative_path, mock_install_deps, mock_generate_python, mock_plugin_manifest, mock_patch_dependencies, plugin_config_file, artifact_file, artifact_content, @@ -65,6 +67,8 @@ def test_build_success(mock_relative_path, mock_install_deps, assert content == artifact_content @staticmethod + @mock.patch( + 'dlpx.virtualization._internal.commands.build.patch_dependencies') @mock.patch( 'dlpx.virtualization._internal.plugin_util.get_plugin_manifest', return_value={}) @@ -72,12 +76,57 @@ def test_build_success(mock_relative_path, mock_install_deps, @mock.patch( 'dlpx.virtualization._internal.plugin_dependency_util.install_deps') @mock.patch('os.path.isabs', return_value=False) + def test_build_success_with_symlink( + mock_relative_path, mock_install_deps, mock_generate_python, + mock_plugin_manifest, mock_patch_dependencies, plugin_config_file, + artifact_file, artifact_content, codegen_gen_py_inputs, + add_symlink_folder_to_src_dir): + gen_py = codegen_gen_py_inputs + # Check if the symlink folder is created. + assert os.path.islink(add_symlink_folder_to_src_dir) + # Before running build assert that the artifact file does not exist. + assert not os.path.exists(artifact_file) + + build.build(plugin_config_file, artifact_file, False, False) + + mock_generate_python.assert_called_once_with(gen_py.name, + gen_py.source_dir, + gen_py.plugin_content_dir, + gen_py.schema_dict) + mock_plugin_manifest.assert_called() + mock_install_deps.assert_called() + mock_relative_path.assert_called() + mock_patch_dependencies.assert_called() + + # After running build this file should now exist. + assert os.path.exists(artifact_file) + + with open(artifact_file, 'rb') as f: + content = json.load(f) + + # Assert that source code changed because of symlink folder. + assert len(content.keys() - artifact_content) == 0 + assert len(artifact_content.keys() - content) == 0 + difference = [key for key in content.keys() + & artifact_content if content[key] != artifact_content[key]] + assert len(difference) == 1 + assert difference.__getitem__(0) == "sourceCode" + + @staticmethod + @mock.patch( + 'dlpx.virtualization._internal.plugin_util.get_plugin_manifest', + return_value={}) + @mock.patch('dlpx.virtualization._internal.codegen.generate_python') + @mock.patch( + 'dlpx.virtualization._internal.plugin_dependency_util.install_deps') + @mock.patch('os.path.isabs', return_value=False) + @mock.patch( + 'dlpx.virtualization._internal.plugin_dependency_util.compile_py_files') def test_build_success_with_patched_dependencies( - mock_relative_path, mock_install_deps, + mock_compile_py_files, mock_relative_path, mock_install_deps, mock_generate_python, mock_plugin_manifest, plugin_config_file, artifact_file, codegen_gen_py_inputs, json_format_file_patched, json_format_content): - build.build(plugin_config_file, artifact_file, False, False) with open(json_format_file_patched, 'r') as f: @@ -95,10 +144,12 @@ def test_build_success_with_patched_dependencies( @mock.patch( 'dlpx.virtualization._internal.plugin_dependency_util.install_deps') @mock.patch('os.path.isabs', return_value=False) - def test_build_success_from_init(mock_relative_path, mock_install_deps, - mock_patch_dependencies, - tmpdir, ingestion_strategy, host_type, - plugin_name, artifact_file): + @mock.patch( + 'dlpx.virtualization._internal.plugin_dependency_util.compile_py_files') + def test_build_success_from_init( + mock_relative_path, mock_compile_py_files, mock_install_deps, + mock_patch_dependencies, tmpdir, ingestion_strategy, host_type, + plugin_name, artifact_file): # Initialize an empty directory. init.init(tmpdir.strpath, ingestion_strategy, plugin_name, host_type) plugin_config_file = os.path.join( @@ -124,11 +175,13 @@ def test_build_success_from_init(mock_relative_path, mock_install_deps, @mock.patch( 'dlpx.virtualization._internal.plugin_dependency_util.install_deps') @mock.patch('os.path.isabs', return_value=False) + @mock.patch( + 'dlpx.virtualization._internal.plugin_dependency_util.compile_py_files') def test_build_success_non_default_output_file( - mock_relative_path, mock_install_deps, mock_generate_python, - mock_import_plugin, mock_patch_dependencies, - plugin_config_file, artifact_file, - artifact_content, codegen_gen_py_inputs): + mock_relative_path, mock_compile_py_files, mock_install_deps, + mock_generate_python, mock_import_plugin, mock_patch_dependencies, + plugin_config_file, artifact_file, artifact_content, + codegen_gen_py_inputs): gen_py = codegen_gen_py_inputs # Before running build assert that the artifact file does not exist. @@ -236,13 +289,13 @@ def test_build_manifest_fail(mock_relative_path, mock_install_deps, @mock.patch( 'dlpx.virtualization._internal.plugin_dependency_util.install_deps') @mock.patch('os.path.isabs', return_value=False) - def test_build_prepare_artifact_fail(mock_relative_path, mock_install_deps, - mock_generate_python, - mock_plugin_manifest, - mock_patch_dependencies, - mock_prep_artifact, - plugin_config_file, artifact_file, - codegen_gen_py_inputs): + @mock.patch( + 'dlpx.virtualization._internal.plugin_dependency_util.compile_py_files') + def test_build_prepare_artifact_fail( + mock_compile_py_files, mock_relative_path, mock_install_deps, + mock_generate_python, mock_plugin_manifest, mock_patch_dependencies, + mock_prep_artifact, plugin_config_file, artifact_file, + codegen_gen_py_inputs): gen_py = codegen_gen_py_inputs # Before running build assert that the artifact file does not exist. @@ -281,11 +334,13 @@ def test_build_prepare_artifact_fail(mock_relative_path, mock_install_deps, @mock.patch( 'dlpx.virtualization._internal.plugin_dependency_util.install_deps') @mock.patch('os.path.isabs', return_value=False) + @mock.patch( + 'dlpx.virtualization._internal.plugin_dependency_util.compile_py_files') def test_build_generate_artifact_fail( - mock_relative_path, mock_install_deps, mock_generate_python, - mock_plugin_manifest, mock_patch_dependencies, - mock_gen_artifact, plugin_config_file, - artifact_file, codegen_gen_py_inputs): + mock_compile_py_files, mock_relative_path, mock_install_deps, + mock_generate_python, mock_plugin_manifest, mock_patch_dependencies, + mock_gen_artifact, plugin_config_file, artifact_file, + codegen_gen_py_inputs): gen_py = codegen_gen_py_inputs # Before running build assert that the artifact file does not exist. @@ -419,10 +474,12 @@ def test_zip_and_encode_source_files_encode_fail(mock_encode, src_dir): @mock.patch( 'dlpx.virtualization._internal.plugin_dependency_util.install_deps') @mock.patch('os.path.isabs', return_value=False) + @mock.patch( + 'dlpx.virtualization._internal.plugin_dependency_util.compile_py_files') @pytest.mark.parametrize('plugin_id', ['77f18ce4-4425-4cd6-b9a7-23653254d660']) - def test_id_validation_positive(mock_relative_path, mock_install_deps, - mock_patch_dependencies, + def test_id_validation_positive(mock_compile_py_files, mock_relative_path, + mock_install_deps, mock_patch_dependencies, mock_import_plugin, plugin_config_file, artifact_file): build.build(plugin_config_file, artifact_file, False) @@ -497,7 +554,7 @@ def test_plugin_bad_language(mock_generate_python, plugin_config_file, build.build(plugin_config_file, artifact_file, False, False) message = err_info.value.message - assert "'BAD_LANGUAGE' is not one of ['PYTHON27']" in message + assert "'BAD_LANGUAGE' is not one of ['PYTHON38']" in message assert not mock_generate_python.called @@ -603,9 +660,9 @@ def test_schema_bad_format(mock_generate_python, plugin_config_file, message = err_info.value.message assert ( - 'Failed to load schemas because \'{}\' is not a valid json file.' - ' Error: Extra data: line 2 column 1 - line 2 column 9' - ' (char 19 - 27)'.format(schema_file)) in message + "Failed to load schemas because '{}' is not a valid json file." + " Error: Extra data: line 2 column 1 (char 19) \n\nBUILD" + " FAILED.").format(schema_file) in message assert not mock_generate_python.called @@ -829,5 +886,5 @@ def test_non_existing_entry_file(mock_relative_path, plugin_config_file, build.build(plugin_config_file, artifact_file, False, False) message = err_info.value.message - exp_message = "No module named {module}".format(module=entry_module) + exp_message = "No module named \'{module}\'".format(module=entry_module) assert exp_message in message diff --git a/tools/src/test/python/dlpx/virtualization/_internal/commands/test_codegen.py b/tools/src/test/python/dlpx/virtualization/_internal/commands/test_codegen.py index ca6e3f84..b0fce144 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/commands/test_codegen.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/commands/test_codegen.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2019, 2020 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # import errno @@ -287,9 +287,9 @@ def test_execute_swagger_codegen_jar_issue(tmpdir, schema_content, codegen._execute_swagger_codegen(swagger_file, tmpdir.strpath) message = err_info.value.message - assert message == ('Unable to run {!r} to generate python code.' - '\nError code: 23. Error message: Too many open' - ' files in system'.format(popen_helper.jar)) + assert message == ( + f"Unable to run '{popen_helper.jar}' to generate python code.\nError" + " code: 23. Error message: Too many open files in system") assert popen_helper.stdout_input == subprocess.PIPE assert popen_helper.stderr_input == subprocess.PIPE diff --git a/tools/src/test/python/dlpx/virtualization/_internal/commands/test_delphix_client.py b/tools/src/test/python/dlpx/virtualization/_internal/commands/test_delphix_client.py index 8396089a..e21fb584 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/commands/test_delphix_client.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/commands/test_delphix_client.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # import json @@ -7,6 +7,7 @@ import requests from dlpx.virtualization._internal import delphix_client, exceptions +from dlpx.virtualization.common.util import to_str import httpretty import mock @@ -403,11 +404,11 @@ def test_delphix_client_unknown_error(engine_api): assert err_info.value.status_code == 404 assert err_info.value.response == ( - '{\n "status": "UNKNOWN", \n "blob": "Unknown"\n}') + '{\n "blob": "Unknown",\n "status": "UNKNOWN"\n}') + assert err_info.value.message == ( 'Received an unexpected error with HTTP Status 404,\nDumping full' - ' response:\n{\n "status": "UNKNOWN", \n "blob": "Unknown"\n}') - + ' response:\n{\n "blob": "Unknown",\n "status": "UNKNOWN"\n}') history = httpretty.HTTPretty.latest_requests assert history[-1].path == u'/resources/json/delphix/session' @@ -471,17 +472,18 @@ def test_delphix_client_wrong_login_no_detail(engine_api): message = err_info.value.message assert err_info.value.status_code == 401 - assert message == ('API request failed with HTTP Status 401' - '\nUnable to parse details of error.' - ' Dumping full response: {' - '\n "commandOutput": null, ' - '\n "diagnoses": [], ' - '\n "type": "APIError", ' - '\n "id": "exception.webservices.login.failed", ' - '\n "error": "Not a real error: Invalid username' - ' or password. Try with a different set of' - ' credentials."' - '\n}') + expected_message = ('API request failed with HTTP Status 401' + '\nUnable to parse details of error.' + ' Dumping full response: {' + '\n "type": "APIError",' + '\n "error": "Not a real error: Invalid username' + ' or password. Try with a different set of' + ' credentials.",' + '\n "id": "exception.webservices.login.failed",' + '\n "commandOutput": null,' + '\n "diagnoses": []' + '\n}') + assert message == expected_message history = httpretty.HTTPretty.latest_requests assert history[-1].path == u'/resources/json/delphix/login' @@ -726,10 +728,12 @@ def test_delphix_client_download_success(engine_api, src_dir, dc.download_plugin_logs(src_dir, plugin_config_file) history = httpretty.HTTPretty.latest_requests - assert (history[-1].path == + to_str(history[-1].__dict__) + + assert (to_str(history[-1].path) == u'/resources/json/delphix/data/downloadOutputStream' u'?token=5d6d5bb8-0f71-4304-8922-49c4c95c2387') - assert history[-2].path == ( + assert to_str(history[-2].path) == ( u'/resources/json/delphix/service/support/bundle/generate') assert history[-3].path == u'/resources/json/delphix/toolkit' assert history[-4].path == u'/resources/json/delphix/login' @@ -760,14 +764,15 @@ def test_validate_fail(artifact_content): delphix_client.DelphixClient.get_engine_api(artifact_content) message = err_info.value.message - assert message == ( + expected_message = ( 'The engineApi field is either missing or malformed.' ' The field must be of the form:' '\n{' '\n "type": "APIVersion",' - ' \n "major": 1,' - ' \n "minor": 7,' - ' \n "micro": 0' + '\n "major": 1,' + '\n "minor": 7,' + '\n "micro": 0' '\n}' '\nVerify that the artifact passed in was generated' ' by the build function.') + assert message == expected_message diff --git a/tools/src/test/python/dlpx/virtualization/_internal/commands/test_initialize.py b/tools/src/test/python/dlpx/virtualization/_internal/commands/test_initialize.py index 62fa9dfe..fe54a998 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/commands/test_initialize.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/commands/test_initialize.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2019, 2020 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # import ast @@ -12,6 +12,7 @@ from dlpx.virtualization._internal import (const, exceptions, plugin_util, plugin_validator, schema_validator) from dlpx.virtualization._internal.commands import initialize as init +from dlpx.virtualization.common.util import to_str @pytest.fixture @@ -63,6 +64,29 @@ def format_template(plugin_name, ingestion_strategy, host_type): raise RuntimeError( 'Got unrecognized ingestion strategy: {}'.format( ingestion_strategy)) + + # + # PY2 VS PY3 STRING/BYTES/UNICODE REPRESENTATIONS + # + # repr(("")) + # | PY2 | PY3 | + # -------------------------------------------------------- + # repr(str("1")) | "'1'" | '1' | + # -------------------------------------------------------- + # repr(bytes("1", "utf-8")) | N/A | b'1' | + # -------------------------------------------------------- + # repr(bytes("1")) | '1' | N/A | + # -------------------------------------------------------- + # repr(unicode("1")) | u'1' | N/A | + # -------------------------------------------------------- + # + # Looking at the above table, we need to do one of the following to get a + # representation that is not prepended with `u` or `b`: + # 1) if six.PY2: plugin_name = to_bytes(plugin_name) + # 2) if six.PY3: plugin_name = to_str(plugin_name) (to_str will return a + # unicode string in PY2) + # + plugin_name = to_str(plugin_name) return template.render(name=repr(plugin_name), linked_operations=operations, default_mount_path=default_mount_path) @@ -110,9 +134,10 @@ def test_init(tmpdir, ingestion_strategy, host_type, schema_template, entry_file_path = os.path.join(tmpdir.strpath, config['srcDir'], entry_file) with open(entry_file_path, 'r') as f: - contents = f.read() - assert contents == format_entry_point_template( - config['id'], ingestion_strategy, host_type) + contents = to_str(f.read()) + expected_contents = to_str(format_entry_point_template( + config['id'], ingestion_strategy, host_type)) + assert contents == expected_contents @staticmethod def test_init_with_relative_path(tmpdir): @@ -217,9 +242,8 @@ def test_init_calls_cleanup_on_failure(mock_cleanup, mock_yaml_dump, @staticmethod def test_default_schema_definition(schema_template): validator = schema_validator.SchemaValidator(None, const.PLUGIN_SCHEMA, - schema_template) + schemas=schema_template) validator.validate() - # Validate the repository schema only has the 'name' property. assert len(schema_template['repositoryDefinition'] ['properties']) == 1, json.dumps( diff --git a/tools/src/test/python/dlpx/virtualization/_internal/commands/test_templates.py b/tools/src/test/python/dlpx/virtualization/_internal/commands/test_templates.py index bb7e9d89..7da59ca3 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/commands/test_templates.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/commands/test_templates.py @@ -1,15 +1,17 @@ # -# Copyright (c) 2019 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # import importlib import itertools import json import os +import re import subprocess import sys from dlpx.virtualization._internal import codegen +from dlpx.virtualization.common.util import to_bytes, to_str import pytest @@ -35,7 +37,7 @@ def module(tmp_factory, schema_content): # create the config file to point to the tmpdir config_dict = {'packageName': tmpdir.name} config_file = basedir.joinpath('codegen-config.json') - config_file.write_bytes(json.dumps(config_dict, indent=2)) + config_file.write_bytes(to_bytes(json.dumps(config_dict, indent=2))) execute_swagger_codegen(swagger_file, str(config_file), str(basedir)) return importlib.import_module('.definitions', package=tmpdir.name) @@ -60,6 +62,8 @@ def execute_swagger_codegen(swagger_file, config_file, output_dir): # Get the pipes pointed so we have access to them. stdout, stderr = process.communicate() + stdout = to_str(stdout) + stderr = to_str(stderr) # # Wait for the process to end and take the results. If res then we know @@ -109,14 +113,14 @@ def schema_content(): 'type': 'object', 'additionalProperties': False, 'properties': { + 'requiredStringProperty': { + 'type': 'string', + 'pattern': "^test.*" + }, 'stringProperty': { 'type': 'string', 'minLength': 5, 'maxLength': 10 - }, - 'requiredStringProperty': { - 'type': 'string', - 'pattern': "^test.*" } }, 'required': ['requiredStringProperty'] @@ -198,7 +202,7 @@ def test_required_param_missing(module): @staticmethod def test_required_param_missing_setter(module): - test_object = module.TestDefinition('test string') + test_object = module.TestDefinition(required_string_property='test string') with pytest.raises(module.GeneratedClassesError) as err_info: test_object.required_string_property = None @@ -209,16 +213,17 @@ def test_required_param_missing_setter(module): @staticmethod def test_not_string(module): with pytest.raises(module.GeneratedClassesTypeError) as err_info: - module.TestDefinition('test string', 10) + module.TestDefinition( + required_string_property='test string', string_property=10) message = err_info.value.message assert message == ( "TestDefinition's parameter 'string_property' was" - " type 'int' but should be of type 'basestring' if defined.") + " class 'int' but should be of class 'str' if defined.") @staticmethod def test_not_string_setter(module): - test_object = module.TestDefinition('test string') + test_object = module.TestDefinition(required_string_property='test string') with pytest.raises(module.GeneratedClassesTypeError) as err_info: test_object.string_property = 10 @@ -226,12 +231,13 @@ def test_not_string_setter(module): message = err_info.value.message assert message == ( "TestDefinition's parameter 'string_property' was" - " type 'int' but should be of type 'basestring' if defined.") + " class 'int' but should be of class 'str' if defined.") @staticmethod def test_min_length(module): with pytest.raises(module.GeneratedClassesError) as err_info: - module.TestDefinition('test string', 'test') + module.TestDefinition( + required_string_property='test string', string_property='test') message = err_info.value.message assert message == ("Invalid value for 'string_property', length was 4" @@ -239,7 +245,7 @@ def test_min_length(module): @staticmethod def test_min_length_setter(module): - test_object = module.TestDefinition('test string') + test_object = module.TestDefinition(required_string_property='test string') with pytest.raises(module.GeneratedClassesError) as err_info: test_object.string_property = 'test' @@ -251,7 +257,9 @@ def test_min_length_setter(module): @staticmethod def test_max_length(module): with pytest.raises(module.GeneratedClassesError) as err_info: - module.TestDefinition('test string', 'test too long of string') + module.TestDefinition( + required_string_property='test string', + string_property='test too long of string') message = err_info.value.message assert message == ("Invalid value for 'string_property', length was 23" @@ -259,7 +267,7 @@ def test_max_length(module): @staticmethod def test_max_length_setter(module): - test_object = module.TestDefinition('test string') + test_object = module.TestDefinition(required_string_property='test string') with pytest.raises(module.GeneratedClassesError) as err_info: test_object.string_property = 'test too long of string' @@ -271,7 +279,7 @@ def test_max_length_setter(module): @staticmethod def test_bad_pattern(module): with pytest.raises(module.GeneratedClassesError) as err_info: - module.TestDefinition('bad test string') + module.TestDefinition(required_string_property='bad test string') message = err_info.value.message assert message == ("Invalid value for 'required_string_property'," @@ -280,7 +288,7 @@ def test_bad_pattern(module): @staticmethod def test_bad_pattern_setter(module): - test_object = module.TestDefinition('test string') + test_object = module.TestDefinition(required_string_property='test string') with pytest.raises(module.GeneratedClassesError) as err_info: test_object.required_string_property = 'bad test string' @@ -327,7 +335,9 @@ def schema_content(): @staticmethod def test_success(module): - test_object = module.TestDefinition(200.5, None, -50) + test_object = module.TestDefinition( + required_number_property=200.5, number_property=None, + required_integer_property=-50) assert test_object.required_number_property == 200.5 assert not test_object.number_property @@ -344,7 +354,9 @@ def test_success(module): @staticmethod def test_success_setter(module): - test_object = module.TestDefinition(200.5, None, -50) + test_object = module.TestDefinition( + required_number_property=200.5, number_property=None, + required_integer_property=-50) test_object.number_property = 13.5 test_object.integer_property = 18 @@ -365,7 +377,9 @@ def test_success_setter(module): @staticmethod def test_success_number_is_int(module): - test_object = module.TestDefinition(200, 13, -50, 18) + test_object = module.TestDefinition( + required_number_property=200, number_property=13, + required_integer_property=-50, integer_property=18) assert test_object.required_number_property == 200 assert test_object.number_property == 13 @@ -375,12 +389,13 @@ def test_success_number_is_int(module): @staticmethod def test_int_passed_in_as_float(module): with pytest.raises(module.GeneratedClassesTypeError) as err_info: - module.TestDefinition(200.5, 13.5, -50.5, 18.5) + module.TestDefinition( + required_number_property=200.5, required_integer_property=2.2) message = err_info.value.message assert message == ( "TestDefinition's parameter 'required_integer_property' was" - " type 'float' but should be of type 'int'.") + " class 'float' but should be of class 'int'.") @staticmethod def test_required_param_missing(module): @@ -392,21 +407,25 @@ def test_required_param_missing(module): " must not be 'None'.") with pytest.raises(module.GeneratedClassesError) as err_info: - module.TestDefinition(200.5) + module.TestDefinition(required_number_property=200.5) message = err_info.value.message assert message == ("The required parameter 'required_integer_property'" " must not be 'None'.") with pytest.raises(module.GeneratedClassesError) as err_info: - module.TestDefinition(None, 13.5, -50, 18) + module.TestDefinition( + required_number_property=None, number_property=13.5, + required_integer_property=-50, integer_property=18) message = err_info.value.message assert message == ("The required parameter 'required_number_property'" " must not be 'None'.") with pytest.raises(module.GeneratedClassesError) as err_info: - module.TestDefinition(200.5, 13.5, None, 18) + module.TestDefinition( + required_number_property=200.5, number_property=13.5, + required_integer_property=None, integer_property=18) message = err_info.value.message assert message == ("The required parameter 'required_integer_property'" @@ -414,7 +433,9 @@ def test_required_param_missing(module): @staticmethod def test_required_param_missing_setter(module): - test_object = module.TestDefinition(200.5, None, -50) + test_object = module.TestDefinition( + required_number_property=200.5, number_property=None, + required_integer_property=-50) with pytest.raises(module.GeneratedClassesError) as err_info: test_object.required_number_property = None @@ -432,45 +453,53 @@ def test_required_param_missing_setter(module): @staticmethod def test_not_number(module): with pytest.raises(module.GeneratedClassesTypeError) as err_info: - module.TestDefinition('string', None, -50) + module.TestDefinition( + required_number_property='string', number_property=None, + required_integer_property=-50) message = err_info.value.message assert message == ( "TestDefinition's parameter 'required_number_property' was" - " type 'str' but should be of type 'float'.") + " class 'str' but should be of class 'float'.") with pytest.raises(module.GeneratedClassesTypeError) as err_info: - module.TestDefinition(200.5, None, 'string') + module.TestDefinition( + required_number_property=200.5, number_property=None, + required_integer_property='string') message = err_info.value.message assert message == ( - "TestDefinition's parameter 'required_integer_property' was" - " type 'str' but should be of type 'int'.") + "TestDefinition's parameter 'required_integer_property' was" + " class 'str' but should be of class 'int'.") @staticmethod def test_not_number_setter(module): - test_object = module.TestDefinition(200.5, None, -50) + test_object = module.TestDefinition( + required_number_property=200.5, number_property=None, + required_integer_property=-50) with pytest.raises(module.GeneratedClassesTypeError) as err_info: test_object.number_property = 'string' message = err_info.value.message assert message == ( - "TestDefinition's parameter 'number_property' was" - " type 'str' but should be of type 'float' if defined.") + "TestDefinition's parameter 'number_property' was" + " class 'str' but should be of class 'float' if defined.") with pytest.raises(module.GeneratedClassesTypeError) as err_info: test_object.integer_property = 'string' message = err_info.value.message assert message == ( - "TestDefinition's parameter 'integer_property' was" - " type 'str' but should be of type 'int' if defined.") + "TestDefinition's parameter 'integer_property' was" + " class 'str' but should be of class 'int' if defined.") @staticmethod def test_minimum(module): with pytest.raises(module.GeneratedClassesError) as err_info: - module.TestDefinition(200.5, 1.0, -50) + module.TestDefinition( + required_number_property=200.5, number_property=1.0, + required_integer_property=-50) message = err_info.value.message assert message == ("Invalid value for 'number_property', value was 1.0" @@ -478,7 +507,9 @@ def test_minimum(module): @staticmethod def test_minimum_setter(module): - test_object = module.TestDefinition(200.5, None, -50) + test_object = module.TestDefinition( + required_number_property=200.5, number_property=None, + required_integer_property=-50) with pytest.raises(module.GeneratedClassesError) as err_info: test_object.number_property = 1.0 @@ -490,7 +521,9 @@ def test_minimum_setter(module): @staticmethod def test_maximum(module): with pytest.raises(module.GeneratedClassesError) as err_info: - module.TestDefinition(200.5, 13.5, -50, 21) + module.TestDefinition( + required_number_property=200.5, number_property=13.5, + required_integer_property=-50, integer_property=21) message = err_info.value.message assert message == ("Invalid value for 'integer_property', value was 21" @@ -498,7 +531,9 @@ def test_maximum(module): @staticmethod def test_maximum_setter(module): - test_object = module.TestDefinition(200.5, None, -50, None) + test_object = module.TestDefinition( + required_number_property=200.5, number_property=None, + required_integer_property=-50, integer_property=None) with pytest.raises(module.GeneratedClassesError) as err_info: test_object.integer_property = 21 @@ -510,7 +545,9 @@ def test_maximum_setter(module): @staticmethod def test_exclusive_minimum(module): with pytest.raises(module.GeneratedClassesError) as err_info: - module.TestDefinition(2.0, 13.5, -50) + module.TestDefinition( + required_number_property=2.0, number_property=13.5, + required_integer_property=-50) message = err_info.value.message assert message == ("Invalid value for 'required_number_property'," @@ -518,7 +555,9 @@ def test_exclusive_minimum(module): @staticmethod def test_exclusive_minimum_setter(module): - test_object = module.TestDefinition(200.5, None, -50) + test_object = module.TestDefinition( + required_number_property=200.5, number_property=None, + required_integer_property=-50) with pytest.raises(module.GeneratedClassesError) as err_info: test_object.required_number_property = 2.0 @@ -530,7 +569,7 @@ def test_exclusive_minimum_setter(module): @staticmethod def test_exclusive_maximum(module): with pytest.raises(module.GeneratedClassesError) as err_info: - module.TestDefinition(200.5, 13.5, 100) + module.TestDefinition(200.5, 13.5, required_integer_property=100) message = err_info.value.message assert message == ("Invalid value for 'required_integer_property'," @@ -538,7 +577,9 @@ def test_exclusive_maximum(module): @staticmethod def test_exclusive_maximum_setter(module): - test_object = module.TestDefinition(200.5, None, -50) + test_object = module.TestDefinition( + required_number_property=200.5, number_property=None, + required_integer_property=-50) with pytest.raises(module.GeneratedClassesError) as err_info: test_object.required_integer_property = 100 @@ -698,8 +739,8 @@ def test_object_is_string(module): message = err_info.value.message assert message == ( - "TestDefinition's parameter 'required_object_property' was type" - " 'str' but should be of a dict with keys type 'basestring'.") + "TestDefinition's parameter 'required_object_property' was class" + " 'str' but should be of a dict with keys type 'str'.") @staticmethod def test_object_is_string_setter(module): @@ -711,8 +752,8 @@ def test_object_is_string_setter(module): message = err_info.value.message assert message == ( - "TestDefinition's parameter 'required_object_property' was type" - " 'str' but should be of a dict with keys type 'basestring'.") + "TestDefinition's parameter 'required_object_property' was class" + " 'str' but should be of a dict with keys type 'str'.") @staticmethod def test_object_is_array(module): @@ -722,8 +763,8 @@ def test_object_is_array(module): message = err_info.value.message assert message == ( - "TestDefinition's parameter 'required_object_property' was type" - " 'list' but should be of a dict with keys type 'basestring'.") + "TestDefinition's parameter 'required_object_property' was class" + " 'list' but should be of a dict with keys type 'str'.") @staticmethod def test_object_is_array_setter(module): @@ -735,8 +776,8 @@ def test_object_is_array_setter(module): message = err_info.value.message assert message == ( - "TestDefinition's parameter 'required_object_property' was type" - " 'list' but should be of a dict with keys type 'basestring'.") + "TestDefinition's parameter 'required_object_property' was class" + " 'list' but should be of a dict with keys type 'str'.") @staticmethod def test_object_dict_with_bad_key_type(module): @@ -748,9 +789,9 @@ def test_object_dict_with_bad_key_type(module): }) expected_msg_template = ( - "TestDefinition's parameter 'required_object_property' was a dict" - " with keys of {{{}}} but should be" - " of a dict with keys type 'basestring'.") + "TestDefinition's parameter 'required_object_property' was a dict" + " with keys of {{{}}} but should be" + " of a dict with keys type 'str'.") possible_messages = create_possible_expected_messages( expected_msg_template, [str, int, bool]) @@ -774,9 +815,10 @@ def test_object_dict_with_bad_key_type_setter(module): } expected_msg_template = ( - "TestDefinition's parameter 'required_object_property' was a dict" - " with keys of {{{}}} but should be" - " of a dict with keys type 'basestring'.") + "TestDefinition's parameter 'required_object_property' was a dict" + " with keys of {{{}}} but should be" + " of a dict with keys type 'str'.") + possible_messages = create_possible_expected_messages( expected_msg_template, [str, int, bool]) @@ -795,9 +837,9 @@ def test_semi_defined_dict_not_bool_value_type(module): message = err_info.value.message assert message == ( - "TestDefinition's parameter 'boolean_dict_property' was a" - " dict of {type 'str':type 'str'} but should be of type 'dict of" - " basestring:bool' if defined.") + "TestDefinition's parameter 'boolean_dict_property' was a" + " dict of {class 'str':class 'str'} but should be of type 'dict of" + " str:bool' if defined.") @staticmethod def test_semi_defined_dict_not_bool_value_type_setter(module): @@ -809,9 +851,9 @@ def test_semi_defined_dict_not_bool_value_type_setter(module): message = err_info.value.message assert message == ( - "TestDefinition's parameter 'boolean_dict_property' was a" - " dict of {type 'str':type 'str'} but should be of type 'dict of" - " basestring:bool' if defined.") + "TestDefinition's parameter 'boolean_dict_property' was a" + " dict of {class 'str':class 'str'} but should be of type 'dict of" + " str:bool' if defined.") @staticmethod def test_semi_defined_dict_not_number_value_type(module): @@ -822,8 +864,8 @@ def test_semi_defined_dict_not_number_value_type(module): message = err_info.value.message assert message == ( "TestDefinition's parameter 'number_dict_property' was a" - " dict of {type 'str':type 'str'} but should be of type 'dict of" - " basestring:float' if defined.") + " dict of {class 'str':class 'str'} but should be of type 'dict of" + " str:float' if defined.") @staticmethod def test_semi_defined_dict_not_number_value_type_setter(module): @@ -836,8 +878,8 @@ def test_semi_defined_dict_not_number_value_type_setter(module): message = err_info.value.message assert message == ( "TestDefinition's parameter 'number_dict_property' was a" - " dict of {type 'str':type 'str'} but should be of type 'dict of" - " basestring:float' if defined.") + " dict of {class 'str':class 'str'} but should be of type 'dict of" + " str:float' if defined.") @staticmethod def test_internal_class_is_string(module): @@ -847,7 +889,7 @@ def test_internal_class_is_string(module): message = err_info.value.message assert message == ( - "TestDefinition's parameter 'defined_object_property' was type" + "TestDefinition's parameter 'defined_object_property' was class" " 'str' but should be of class" " '{}.TestDefinitionDefinedObjectProperty' if defined.".format( module.TestDefinitionDefinedObjectProperty.__module__)) @@ -862,7 +904,7 @@ def test_internal_class_is_string_setter(module): message = err_info.value.message assert message == ( - "TestDefinition's parameter 'defined_object_property' was type" + "TestDefinition's parameter 'defined_object_property' was class" " 'str' but should be of class" " '{}.TestDefinitionDefinedObjectProperty' if defined.".format( module.TestDefinitionDefinedObjectProperty.__module__)) @@ -878,8 +920,10 @@ class TestOtherClass: message = err_info.value.message assert message == ( - "TestDefinition's parameter 'defined_object_property' was type" - " 'instance' but should be of class" + "TestDefinition's parameter 'defined_object_property' was class" + " 'dlpx.virtualization._internal.commands.test_templates" + ".TestTemplateObjectProperty.test_internal_class_is_other_class" + ".locals.TestOtherClass' but should be of class" " '{}.TestDefinitionDefinedObjectProperty' if defined.".format( module.TestDefinitionDefinedObjectProperty.__module__)) @@ -896,9 +940,12 @@ class TestOtherClass: message = err_info.value.message assert message == ( - "TestDefinition's parameter 'defined_object_property' was type" - " 'instance' but should be of class" - " '{}.TestDefinitionDefinedObjectProperty' if defined.".format( + "TestDefinition's parameter 'defined_object_property' was class" + " 'dlpx.virtualization._internal.commands.test_templates" + ".TestTemplateObjectProperty" + ".test_internal_class_is_other_class_setter.locals.TestOtherClass' " + "but should be of class '{}.TestDefinitionDefinedObjectProperty' " + "if defined.".format( module.TestDefinitionDefinedObjectProperty.__module__)) @@ -999,8 +1046,8 @@ def test_array_is_string(module): message = err_info.value.message assert message == ( - "TestDefinition's parameter 'required_array_property' was type" - " 'str' but should be of type 'list of float'.") + "TestDefinition's parameter 'required_array_property' was class" + " 'str' but should be of type 'list of float'.") @staticmethod def test_array_is_string_setter(module): @@ -1011,8 +1058,8 @@ def test_array_is_string_setter(module): message = err_info.value.message assert message == ( - "TestDefinition's parameter 'required_array_property' was type" - " 'str' but should be of type 'list of float'.") + "TestDefinition's parameter 'required_array_property' was class" + " 'str' but should be of type 'list of float'.") @staticmethod def test_array_is_object(module): @@ -1025,8 +1072,8 @@ def test_array_is_object(module): message = err_info.value.message assert message == ( - "TestDefinition's parameter 'array_property' was type" - " 'dict' but should be of type 'list' if defined.") + "TestDefinition's parameter 'array_property' was class" + " 'dict' but should be of class 'list' if defined.") @staticmethod def test_array_is_object_setter(module): @@ -1037,8 +1084,8 @@ def test_array_is_object_setter(module): message = err_info.value.message assert message == ( - "TestDefinition's parameter 'array_property' was type" - " 'dict' but should be of type 'list' if defined.") + "TestDefinition's parameter 'array_property' was class" + " 'dict' but should be of class 'list' if defined.") @staticmethod def test_number_array_wrong_elem_types(module): @@ -1047,9 +1094,9 @@ def test_number_array_wrong_elem_types(module): message = err_info.value.message assert message == ( - "TestDefinition's parameter 'required_array_property' was a list" - " of [type 'str', type 'bool'] but should be of type 'list of" - " float'.") + "TestDefinition's parameter 'required_array_property' was a list" + " of [class 'str', class 'bool'] but should be of type 'list of" + " float'.") @staticmethod def test_number_array_wrong_elem_types_setter(module): @@ -1060,9 +1107,9 @@ def test_number_array_wrong_elem_types_setter(module): message = err_info.value.message assert message == ( - "TestDefinition's parameter 'required_array_property' was a list" - " of [type 'str', type 'bool'] but should be of type 'list of" - " float'.") + "TestDefinition's parameter 'required_array_property' was a list" + " of [class 'str', class 'bool'] but should be of type 'list of" + " float'.") @staticmethod def test_string_array_wrong_elem_types(module): @@ -1073,8 +1120,8 @@ def test_string_array_wrong_elem_types(module): message = err_info.value.message assert message == ( "TestDefinition's parameter 'string_array_property' was a list of" - " [type 'str', type 'int', type 'float'] but should be of type" - " 'list of basestring' if defined.") + " [class 'str', class 'int', class 'float'] but should be of type" + " 'list of str' if defined.") @staticmethod def test_string_array_wrong_elem_types_setter(module): @@ -1085,9 +1132,9 @@ def test_string_array_wrong_elem_types_setter(module): message = err_info.value.message assert message == ( - "TestDefinition's parameter 'string_array_property' was a list of" - " [type 'str', type 'int', type 'float'] but should be of type" - " 'list of basestring' if defined.") + "TestDefinition's parameter 'string_array_property' was a list of" + " [class 'str', class 'int', class 'float'] but should be of type" + " 'list of str' if defined.") class TestTemplateBooleanProperty: @@ -1171,8 +1218,8 @@ def test_boolean_is_string(module): message = err_info.value.message assert message == ( - "TestDefinition's parameter 'required_boolean_property' was type" - " 'str' but should be of type 'bool'.") + "TestDefinition's parameter 'required_boolean_property' was class" + " 'str' but should be of class 'bool'.") @staticmethod def test_boolean_is_string_setter(module): @@ -1183,8 +1230,8 @@ def test_boolean_is_string_setter(module): message = err_info.value.message assert message == ( - "TestDefinition's parameter 'required_boolean_property' was type" - " 'str' but should be of type 'bool'.") + "TestDefinition's parameter 'required_boolean_property' was class" + " 'str' but should be of class 'bool'.") class TestTemplateEnumProperty: @@ -1334,8 +1381,8 @@ def test_required_string_not_string(module): message = err_info.value.message assert message == ( - "TestDefinition's parameter 'required_string_property' was type" - " 'int' but should be of type 'basestring'.") + "TestDefinition's parameter 'required_string_property' was class" + " 'int' but should be of class 'str'.") @staticmethod def test_required_string_not_string_setter(module): @@ -1348,8 +1395,8 @@ def test_required_string_not_string_setter(module): message = err_info.value.message assert message == ( - "TestDefinition's parameter 'required_string_property' was type" - " 'int' but should be of type 'basestring'.") + "TestDefinition's parameter 'required_string_property' was class" + " 'int' but should be of class 'str'.") @staticmethod def test_string_incorrect_enum(module): @@ -1387,8 +1434,8 @@ def test_string_not_string(module): message = err_info.value.message assert message == ( - "TestDefinition's parameter 'string_property' was type 'int' but" - " should be of type 'basestring' if defined.") + "TestDefinition's parameter 'string_property' was class 'int' but" + " should be of class 'str' if defined.") @staticmethod def test_string_not_string_setter(module): @@ -1401,8 +1448,8 @@ def test_string_not_string_setter(module): message = err_info.value.message assert message == ( - "TestDefinition's parameter 'string_property' was type 'int' but" - " should be of type 'basestring' if defined.") + "TestDefinition's parameter 'string_property' was class 'int' but" + " should be of class 'str' if defined.") @staticmethod def test_required_object_incorrect_enum(module): @@ -1439,8 +1486,8 @@ def test_required_object_not_object(module): message = err_info.value.message assert message == ( - "TestDefinition's parameter 'required_object_property' was type" - " 'str' but should be of a dict with keys type 'basestring'.") + "TestDefinition's parameter 'required_object_property' was class" + " 'str' but should be of a dict with keys type 'str'.") @staticmethod def test_required_object_not_object_setter(module): @@ -1453,8 +1500,8 @@ def test_required_object_not_object_setter(module): message = err_info.value.message assert message == ( - "TestDefinition's parameter 'required_object_property' was type" - " 'str' but should be of a dict with keys type 'basestring'.") + "TestDefinition's parameter 'required_object_property' was class" + " 'str' but should be of a dict with keys type 'str'.") @staticmethod def test_object_incorrect_enum(module): @@ -1493,8 +1540,8 @@ def test_object_not_object(module): message = err_info.value.message assert message == ( - "TestDefinition's parameter 'object_property' was type 'str' but" - " should be of a dict with keys type 'basestring' if defined.") + "TestDefinition's parameter 'object_property' was class 'str' but" + " should be of a dict with keys type 'str' if defined.") @staticmethod def test_object_not_object_setter(module): @@ -1507,8 +1554,8 @@ def test_object_not_object_setter(module): message = err_info.value.message assert message == ( - "TestDefinition's parameter 'object_property' was type 'str' but" - " should be of a dict with keys type 'basestring' if defined.") + "TestDefinition's parameter 'object_property' was class 'str' but" + " should be of a dict with keys type 'str' if defined.") @staticmethod def test_required_array_incorrect_enum(module): @@ -1518,9 +1565,12 @@ def test_required_array_incorrect_enum(module): required_array_property=['FA', 'SO']) message = err_info.value.message - assert message == ( - "Invalid values for 'required_array_property'. Was [FA, SO]" - " but must be a subset of [DO, RE, MI].") + # In Py3, the "FA" and "SO" are not always listed in the same order. Use a + # regex to check test success regardless of this ordering. + pattern = re.compile( + "Invalid values for 'required_array_property'. Was \\[(FA, SO|SO, FA)\\] " + "but must be a subset of \\[DO, RE, MI\\].") + assert re.match(pattern, message) is not None @staticmethod def test_required_array_incorrect_enum_setter(module): @@ -1532,9 +1582,10 @@ def test_required_array_incorrect_enum_setter(module): test_object.required_array_property = ['FA', 'SO'] message = err_info.value.message - assert message == ( - "Invalid values for 'required_array_property'. Was [FA, SO]" - " but must be a subset of [DO, RE, MI].") + pattern = re.compile( + "Invalid values for 'required_array_property'. Was \\[(FA, SO|SO, FA)\\] " + "but must be a subset of \\[DO, RE, MI\\].") + assert re.match(pattern, message) is not None @staticmethod def test_required_array_not_array(module): @@ -1545,8 +1596,8 @@ def test_required_array_not_array(module): message = err_info.value.message assert message == ( - "TestDefinition's parameter 'required_array_property' was type" - " 'str' but should be of type 'list of basestring'.") + "TestDefinition's parameter 'required_array_property' was class" + " 'str' but should be of type 'list of str'.") @staticmethod def test_required_array_not_array_setter(module): @@ -1559,8 +1610,8 @@ def test_required_array_not_array_setter(module): message = err_info.value.message assert message == ( - "TestDefinition's parameter 'required_array_property' was type" - " 'str' but should be of type 'list of basestring'.") + "TestDefinition's parameter 'required_array_property' was class" + " 'str' but should be of type 'list of str'.") @staticmethod def test_array_incorrect_enum(module): @@ -1571,8 +1622,10 @@ def test_array_incorrect_enum(module): array_property=['FA', 'SO']) message = err_info.value.message - assert message == ("Invalid values for 'array_property'. Was [FA, SO]" - " but must be a subset of [DO, RE, MI] if defined.") + pattern = re.compile( + "Invalid values for 'array_property'. Was \\[(FA, SO|SO, FA)\\] but must" + " be a subset of \\[DO, RE, MI\\] if defined.") + assert re.match(pattern, message) is not None @staticmethod def test_array_incorrect_enum_setter(module): @@ -1584,8 +1637,11 @@ def test_array_incorrect_enum_setter(module): test_object.array_property = ['FA', 'SO'] message = err_info.value.message - assert message == ("Invalid values for 'array_property'. Was [FA, SO]" - " but must be a subset of [DO, RE, MI] if defined.") + pattern = re.compile( + "Invalid values for 'array_property'. Was \\[(FA, SO|SO, FA)\\] but must" + " be a subset of \\[DO, RE, MI\\] if defined.") + + assert re.match(pattern, message) is not None @staticmethod def test_array_not_array(module): @@ -1597,8 +1653,8 @@ def test_array_not_array(module): message = err_info.value.message assert message == ( - "TestDefinition's parameter 'array_property' was type 'str' but" - " should be of type 'list of basestring' if defined.") + "TestDefinition's parameter 'array_property' was class 'str' but" + " should be of type 'list of str' if defined.") @staticmethod def test_array_not_array_setter(module): @@ -1611,5 +1667,5 @@ def test_array_not_array_setter(module): message = err_info.value.message assert message == ( - "TestDefinition's parameter 'array_property' was type 'str' but" - " should be of type 'list of basestring' if defined.") + "TestDefinition's parameter 'array_property' was class 'str' but" + " should be of type 'list of str' if defined.") diff --git a/tools/src/test/python/dlpx/virtualization/_internal/conftest.py b/tools/src/test/python/dlpx/virtualization/_internal/conftest.py index 39a4460f..de6c79eb 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/conftest.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/conftest.py @@ -1,15 +1,18 @@ # -# Copyright (c) 2019, 2020 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # import configparser import copy import json import os +import shutil +from importlib import reload import yaml from dlpx.virtualization._internal import cli, click_util, const, package_util from dlpx.virtualization._internal.commands import build +from dlpx.virtualization.common.util import to_bytes, to_str import pytest @@ -39,7 +42,7 @@ def plugin_config_file(tmpdir, plugin_config_filename, plugin_config_content): f = tmpdir.join(plugin_config_filename) if plugin_config_content: - f.write(plugin_config_content) + f.write(to_str(plugin_config_content)) return f.strpath @@ -113,8 +116,8 @@ def _write_dvp_config_file(tmpdir, if dev_config_properties: parser['dev'] = dev_config_properties - with open(dvp_config_filepath, 'wb') as config_file: - parser.write(config_file) + with open(dvp_config_filepath, 'w') as config_file: + parser.write(to_bytes(config_file)) # # Add temp_dir to list of config files the ConfigFileProcessor will @@ -156,7 +159,7 @@ def artifact_file(tmpdir, artifact_content, artifact_filename, # Only write the artifact if we want to actually create it. if isinstance(artifact_content, dict): artifact_content = json.dumps(artifact_content, indent=4) - f.write(artifact_content) + f.write(to_bytes(artifact_content)) return f.strpath @@ -249,7 +252,7 @@ def external_version(): @pytest.fixture def language(): - return 'PYTHON27' + return 'PYTHON38' @pytest.fixture @@ -404,6 +407,7 @@ def mount_specification(virtual_source, repository): virtual.mount_specification_impl = mount_specification virtual.status_impl = None virtual.initialize_impl = None + virtual.cleanup_impl = None return virtual @@ -447,7 +451,8 @@ def plugin_manifest(upgrade_operation): 'hasVirtualMountSpecification': True, 'hasVirtualStatus': False, 'hasInitialize': False, - 'migrationIdList': upgrade_operation.migration_id_list + 'migrationIdList': upgrade_operation.migration_id_list, + 'hasVirtualCleanup': False, } return manifest @@ -626,6 +631,16 @@ def additional_definition(): return None +@pytest.fixture +def add_symlink_folder_to_src_dir(tmpdir, src_dir): + dummy_folder = os.path.join(tmpdir, "dummy_folder") + os.mkdir(dummy_folder) + shutil.copy2(os.path.join(os.path.dirname(__file__), "__init__.py"), dummy_folder) + destination = os.path.join(src_dir, "dummy_folder") + os.symlink(dummy_folder, destination) + return destination + + @pytest.fixture def artifact_content(engine_api, virtual_source_definition, linked_source_definition, discovery_definition, @@ -642,7 +657,7 @@ def artifact_content(engine_api, virtual_source_definition, 'name': 'python_vfiles', 'externalVersion': '2.0.0', 'defaultLocale': 'en-us', - 'language': 'PYTHON27', + 'language': 'PYTHON38', 'hostTypes': ['UNIX'], 'entryPoint': 'python_vfiles:vfiles', 'buildApi': package_util.get_build_api_version(), @@ -691,7 +706,7 @@ def artifact_content(engine_api, virtual_source_definition, @pytest.fixture def engine_api(): - return {'type': 'APIVersion', 'major': 1, 'minor': 11, 'micro': 6} + return {'type': 'APIVersion', 'major': 1, 'minor': 11, 'micro': 11} @pytest.fixture diff --git a/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/multiple_warnings.py b/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/multiple_warnings.py index 56f4d299..1ee0d307 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/multiple_warnings.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/multiple_warnings.py @@ -75,6 +75,11 @@ def unconfigure(repository, source_config, virtual_source): pass +@vfiles.virtual.cleanup() +def cleanup(repository, source_config, virtual_source, bad_arg): + pass + + @vfiles.upgrade.repository('2019.10.30') def repo_upgrade(old_repository): return old_repository diff --git a/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/successful.py b/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/successful.py index cb355933..da568732 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/successful.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/successful.py @@ -81,6 +81,11 @@ def unconfigure(repository, source_config, virtual_source): pass +@direct.virtual.cleanup() +def cleanup(repository, source_config, virtual_source): + pass + + @direct.upgrade.repository('1.3', MigrationType.LUA) def repo_upgrade(old_repository): return old_repository diff --git a/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/upgrade_warnings.py b/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/upgrade_warnings.py index 0046fe49..0f018bf2 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/upgrade_warnings.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/upgrade_warnings.py @@ -74,6 +74,11 @@ def unconfigure(repository, source_config, virtual_source): pass +@direct.virtual.cleanup() +def cleanup(repository, source_config, virtual_source): + pass + + @direct.upgrade.repository('2019.11.20') def repo_upgrade(old_repository): return old_repository diff --git a/tools/src/test/python/dlpx/virtualization/_internal/test_cli.py b/tools/src/test/python/dlpx/virtualization/_internal/test_cli.py index 4e8e215a..49c673bf 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/test_cli.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/test_cli.py @@ -1,8 +1,9 @@ # -# Copyright (c) 2019, 2020 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # import os +import re import click.testing as click_testing import yaml @@ -172,7 +173,7 @@ def test_blank_ingestion_strategy(plugin_name): ['init', '-n', plugin_name, '-s', '']) assert result.exit_code != 0 - assert "invalid choice" in result.output + assert "Invalid value" in result.output @staticmethod def test_non_existent_root_dir(plugin_name): @@ -214,7 +215,7 @@ def test_multiple_host_types(): ]) assert result.exit_code != 0 - assert "invalid choice" in result.output + assert "Invalid value" in result.output @staticmethod @mock.patch('dlpx.virtualization._internal.commands.initialize.init') @@ -235,7 +236,7 @@ def test_invalid_host_type(): result = runner.invoke(cli.delphix_sdk, ['init', '-t', 'UNI']) assert result.exit_code != 0 - assert "invalid choice" in result.output + assert "Invalid value" in result.output class TestBuildCli: @@ -576,12 +577,10 @@ def test_with_config_file_fail(artifact_file): os.chdir(cwd) assert result.exit_code == 2 - assert result.output == (u'Usage: delphix-sdk upload [OPTIONS]\n' - u'\n' - u'Error: Invalid value for \'-e\' / ' - u'\'--engine\': Option is required ' - u'and must be specified via the command line.' - u'\n') + output = result.output.replace("\n", "") + pattern = re.compile( + r"Usage: delphix-sdk upload \[OPTIONS\].*Error: Invalid value for '-e.*") + assert re.match(pattern, output) is not None class TestDownloadCli: @@ -640,13 +639,11 @@ def test_missing_params(): ]) assert result.exit_code == 2 - assert result.output == ( - u"Usage: delphix-sdk download-logs [OPTIONS]\n" - u"\n" - u"Error: Invalid value for '-e' / " - u"'--engine': Option is required " - u"and must be specified via the command line." - u"\n") + output = result.output.replace("\n", "") + pattern = re.compile( + r"Usage: delphix-sdk download-logs \[OPTIONS\].*" + r"Error: Invalid value for '-e.*") + assert re.match(pattern, output) is not None @staticmethod @mock.patch( @@ -789,10 +786,9 @@ def test_with_config_file_fail(plugin_config_file, dvp_config_file): os.chdir(cwd) assert result.exit_code == 2 - assert result.output == ( - u"Usage: delphix-sdk download-logs [OPTIONS]\n" - u"\n" - u"Error: Invalid value for '-e' / " - u"'--engine': Option is required " - u"and must be specified via the command line." - u"\n") + + output = result.output.replace("\n", "") + pattern = re.compile( + r"Usage: delphix-sdk download-logs \[OPTIONS\].*" + r"Error: Invalid value for '-e.*") + assert re.match(pattern, output) is not None diff --git a/tools/src/test/python/dlpx/virtualization/_internal/test_package_util.py b/tools/src/test/python/dlpx/virtualization/_internal/test_package_util.py index 87f63c08..49671c5e 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/test_package_util.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/test_package_util.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2019, 2020 by Delphix. All rights reserved. +# Copyright (c) 2019, 2022 by Delphix. All rights reserved. # import os from dlpx.virtualization._internal import package_util @@ -10,23 +10,23 @@ class TestPackageUtil: @staticmethod def test_get_version(): - assert package_util.get_version() == '3.1.0' + assert package_util.get_version() == '4.0.4' @staticmethod def test_get_virtualization_api_version(): - assert package_util.get_virtualization_api_version() == '1.5.0' + assert package_util.get_virtualization_api_version() == '1.6.3' @staticmethod def test_get_engine_api_version(): - assert package_util.get_engine_api_version_from_settings() == '1.11.6' + assert package_util.get_engine_api_version_from_settings() == '1.11.11' @staticmethod def test_get_build_api_version_json(): build_api_version = { 'type': 'APIVersion', 'major': 1, - 'minor': 5, - 'micro': 0 + 'minor': 6, + 'micro': 3 } assert package_util.get_build_api_version() == build_api_version @@ -36,7 +36,7 @@ def test_get_engine_api_version_json(): 'type': 'APIVersion', 'major': 1, 'minor': 11, - 'micro': 6 + 'micro': 11 } assert package_util.get_engine_api_version() == engine_api_version diff --git a/tools/src/test/python/dlpx/virtualization/_internal/test_plugin_importer.py b/tools/src/test/python/dlpx/virtualization/_internal/test_plugin_importer.py index a8795765..1d20ee9c 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/test_plugin_importer.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/test_plugin_importer.py @@ -1,14 +1,14 @@ # -# Copyright (c) 2019, 2020 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # -import exceptions import os +import re import uuid from collections import OrderedDict from multiprocessing import Queue -from dlpx.virtualization._internal import (file_util, plugin_util, - plugin_validator, plugin_importer) +from dlpx.virtualization._internal import ( + file_util, plugin_util, plugin_validator, plugin_importer, exceptions) from dlpx.virtualization._internal.plugin_importer import PluginImporter import mock @@ -137,14 +137,15 @@ def test_successful_validation(mock_file_util, plugin_config_file, ('multiple_warnings:vfiles', 'DIRECT', [ 'Error: Number of arguments do not match in method status', 'Error: Named argument mismatch in method status', + 'Error: Number of arguments do not match in method cleanup', + 'Error: Named argument mismatch in method cleanup', 'Warning: Implementation missing for required method' - ' virtual.reconfigure().', '1 Warning(s). 2 Error(s).' + ' virtual.reconfigure().', '1 Warning(s). 4 Error(s).' ])]) @mock.patch('dlpx.virtualization._internal.file_util.get_src_dir_path') def test_multiple_warnings(mock_file_util, plugin_config_file, fake_src_dir, expected_errors): mock_file_util.return_value = fake_src_dir - with pytest.raises(exceptions.UserError) as err_info: importer = get_plugin_importer(plugin_config_file) importer.validate_plugin_module() @@ -231,12 +232,12 @@ def test_plugin_info_warn_mode(mock_import, mock_relative_path, plugin_config_file, src_dir, plugin_module_content): plugin_config_content = OrderedDict([ - ('id', str(uuid.uuid4())), ('name', 'staged'.encode('utf-8')), - ('version', '0.1.0'), ('language', 'PYTHON27'), - ('hostTypes', ['UNIX']), ('pluginType', 'STAGED'.encode('utf-8')), + ('id', str(uuid.uuid4())), ('name', 'staged'), + ('version', '0.1.0'), ('language', 'PYTHON38'), + ('hostTypes', ['UNIX']), ('pluginType', 'STAGED'), ('manualDiscovery', True), - ('entryPoint', 'staged_plugin:staged'.encode('utf-8')), - ('srcDir', src_dir), ('schemaFile', 'schema.json'.encode('utf-8')) + ('entryPoint', 'staged_plugin:staged'), + ('srcDir', src_dir), ('schemaFile', 'schema.json') ]) mock_import.return_value = plugin_module_content try: @@ -280,7 +281,8 @@ def test_import_error(mock_file_util, plugin_config_file, importer.validate_plugin_module() message = err_info.value.message - assert expected_error in message + pattern = re.compile("^Error: No module named (')?dlpxxx(')?.*") + assert re.match(pattern, message) is not None @staticmethod @pytest.mark.parametrize( @@ -307,7 +309,6 @@ def test_bad_syntax(mock_file_util, plugin_config_file, def test_undefined_name_error(mock_file_util, plugin_config_file, fake_src_dir, expected_error): mock_file_util.return_value = fake_src_dir - with pytest.raises(exceptions.SDKToolingError) as err_info: importer = get_plugin_importer(plugin_config_file) importer.validate_plugin_module() diff --git a/tools/src/test/python/dlpx/virtualization/_internal/test_plugin_validator.py b/tools/src/test/python/dlpx/virtualization/_internal/test_plugin_validator.py index e81ce7da..4b419768 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/test_plugin_validator.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/test_plugin_validator.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2019, 2020 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # import json @@ -24,8 +24,8 @@ def test_plugin_bad_schema(plugin_config_file, plugin_config_content, message = err_info.value.message assert ('Failed to load schemas because {} is not a valid json file.' - ' Error: Extra data: line 2 column 1 - line 2 column 9' - ' (char 19 - 27)'.format(schema_file)) in message + ' Error: Extra data: line 2 column 1 (char 19)' + .format(schema_file)) in message @staticmethod @pytest.mark.parametrize('plugin_config_file', ['/dir/plugin_config.yml']) diff --git a/tools/src/test/python/dlpx/virtualization/_internal/test_schema_validator.py b/tools/src/test/python/dlpx/virtualization/_internal/test_schema_validator.py index 2901c0cf..5f735a79 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/test_schema_validator.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/test_schema_validator.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2019, 2020 by Delphix. All rights reserved. +# Copyright (c) 2019, 2021 by Delphix. All rights reserved. # import json @@ -22,8 +22,8 @@ def test_bad_meta_schema(schema_file, tmpdir, schema_filename): message = err_info.value.message assert ("Failed to load schemas because '{}' is not a valid json file." - " Error: Extra data: line 2 column 1 - line 2 column 9" - " (char 19 - 27)".format(schema_file)) in message + " Error: Extra data: line 2 column 1 (char 19)" + .format(schema_file)) in message @staticmethod def test_bad_schema_file(schema_file):