diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 9e9dedd4..1807b5b5 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 3.0.0 +current_version = 3.1.0.dev2 commit = False tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+)(?P\d+))? diff --git a/common/requirements.txt b/common/requirements.txt index 33673b1b..3b2c337b 100644 --- a/common/requirements.txt +++ b/common/requirements.txt @@ -1,5 +1,6 @@ bump2version==0.5.11 contextlib2==0.6.0.post1 ; python_version < '3' +enum34==1.1.6 funcsigs==1.0.2 ; python_version < '3.0' importlib-metadata==0.23 ; python_version < '3.8' more-itertools==5.0.0 ; python_version <= '2.7' diff --git a/common/setup.py b/common/setup.py index 7d9f0244..6b5dff76 100644 --- a/common/setup.py +++ b/common/setup.py @@ -4,7 +4,7 @@ PYTHON_SRC = 'src/main/python' install_requires = [ - "dvp-api == 1.4.0", + "dvp-api == 1.5.0.dev1", ] with open(os.path.join(PYTHON_SRC, 'dlpx/virtualization/common/VERSION')) as version_file: diff --git a/common/src/main/python/dlpx/virtualization/common/VERSION b/common/src/main/python/dlpx/virtualization/common/VERSION index 56fea8a0..a8eefdb0 100644 --- a/common/src/main/python/dlpx/virtualization/common/VERSION +++ b/common/src/main/python/dlpx/virtualization/common/VERSION @@ -1 +1 @@ -3.0.0 \ No newline at end of file +3.1.0.dev2 diff --git a/common/src/main/python/dlpx/virtualization/common/_common_classes.py b/common/src/main/python/dlpx/virtualization/common/_common_classes.py index c5d71fe3..2ad05804 100644 --- a/common/src/main/python/dlpx/virtualization/common/_common_classes.py +++ b/common/src/main/python/dlpx/virtualization/common/_common_classes.py @@ -2,8 +2,10 @@ # Copyright (c) 2019 by Delphix. All rights reserved. # -from dlpx.virtualization.api import common_pb2 +from abc import ABCMeta +from dlpx.virtualization.api import common_pb2, libs_pb2 from dlpx.virtualization.common.exceptions import IncorrectTypeError +from enum import IntEnum """Classes used for Plugin Operations @@ -17,7 +19,10 @@ "RemoteConnection", "RemoteEnvironment", "RemoteHost", - "RemoteUser"] + "RemoteUser", + "Credentials", + "PasswordCredentials", + "KeyPairCredentials"] class RemoteConnection(object): @@ -293,3 +298,121 @@ def from_proto(user): common_pb2.RemoteUser) remote_user = RemoteUser(name=user.name, reference=user.reference) return remote_user + + +class Credentials(object): + """Plugin base class for CredentialsResult to be used for plugin operations + and library functions. + + Plugin authors should use this instead of the corresponding protobuf generated + class. + + Args: + username: User name. + """ + def __init__(self, username): + if not isinstance(username, basestring): + raise IncorrectTypeError( + Credentials, + 'username', + type(username), + basestring) + self.__username = username + + __metaclass__ = ABCMeta + + @property + def username(self): + return self.__username + + +class PasswordCredentials(Credentials): + """Plugin class for CredentialsResult with password to be used for plugin operations + and library functions. + + Plugin authors should use this instead of the corresponding protobuf generated + class. + + Args: + username: User name. + password: Password. + """ + def __init__(self, username, password): + super(PasswordCredentials, self).__init__(username) + if not isinstance(password, basestring): + raise IncorrectTypeError( + PasswordCredentials, + 'password', + type(password), + basestring) + self.__password = password + + @property + def password(self): + return self.__password + + @staticmethod + def from_proto(credentials_result): + """Converts protobuf class libs_pb2.CredentialsResult to plugin class PasswordCredentials + """ + if not isinstance(credentials_result, libs_pb2.CredentialsResult): + raise IncorrectTypeError( + PasswordCredentials, + 'credentials_result', + type(credentials_result), + libs_pb2.CredentialsResult) + return PasswordCredentials( + username=credentials_result.username, password=credentials_result.pasword) + + +class KeyPairCredentials(Credentials): + """Plugin class for CredentialsResult with key pair to be used for plugin operations + and library functions. + + Plugin authors should use this instead of the corresponding protobuf generated + class. + + Args: + username (str): User name. + private_key (str): Private key. + public_key (str): Public key corresponding to private key. Empty string if not present. + """ + def __init__(self, username, private_key, public_key): + super(KeyPairCredentials, self).__init__(username) + if not isinstance(private_key, basestring): + raise IncorrectTypeError( + KeyPairCredentials, + 'private_key', + type(private_key), + basestring) + self.__private_key = private_key + if not isinstance(public_key, basestring): + raise IncorrectTypeError( + KeyPairCredentials, + 'public_key', + type(public_key), + basestring) + self.__public_key = public_key + + @property + def private_key(self): + return self.__private_key + + @property + def public_key(self): + return self.__public_key + + @staticmethod + def from_proto(credentials_result): + """Converts protobuf class libs_pb2.CredentialsResult to plugin class KeyPairCredentials + """ + if not isinstance(credentials_result, libs_pb2.CredentialsResult): + raise IncorrectTypeError( + KeyPairCredentials, + 'credentials_result', + type(credentials_result), + libs_pb2.CredentialsResult) + return KeyPairCredentials( + username=credentials_result.username, + private_key=credentials_result.key_pair.private_key, + public_key=credentials_result.key_pair.public_key) diff --git a/dvp/src/main/python/dlpx/virtualization/VERSION b/dvp/src/main/python/dlpx/virtualization/VERSION index 56fea8a0..a8eefdb0 100644 --- a/dvp/src/main/python/dlpx/virtualization/VERSION +++ b/dvp/src/main/python/dlpx/virtualization/VERSION @@ -1 +1 @@ -3.0.0 \ No newline at end of file +3.1.0.dev2 diff --git a/libs/requirements.txt b/libs/requirements.txt index 275195fe..a8a53e05 100644 --- a/libs/requirements.txt +++ b/libs/requirements.txt @@ -1,6 +1,7 @@ ./../common bump2version==0.5.11 contextlib2==0.6.0.post1 ; python_version < '3' +enum34==1.1.6 funcsigs==1.0.2 ; python_version < '3.3' importlib-metadata==1.3.0 ; python_version < '3.8' mock==3.0.5 diff --git a/libs/setup.py b/libs/setup.py index d7c601f8..17cea43b 100644 --- a/libs/setup.py +++ b/libs/setup.py @@ -7,7 +7,7 @@ version = version_file.read().strip() install_requires = [ - "dvp-api == 1.4.0", + "dvp-api == 1.5.0.dev1", "dvp-common == {}".format(version) ] diff --git a/libs/src/main/python/dlpx/virtualization/libs/VERSION b/libs/src/main/python/dlpx/virtualization/libs/VERSION index 56fea8a0..a8eefdb0 100644 --- a/libs/src/main/python/dlpx/virtualization/libs/VERSION +++ b/libs/src/main/python/dlpx/virtualization/libs/VERSION @@ -1 +1 @@ -3.0.0 \ No newline at end of file +3.1.0.dev2 diff --git a/libs/src/main/python/dlpx/virtualization/libs/libs.py b/libs/src/main/python/dlpx/virtualization/libs/libs.py index 2e0fb4bd..69355a4b 100644 --- a/libs/src/main/python/dlpx/virtualization/libs/libs.py +++ b/libs/src/main/python/dlpx/virtualization/libs/libs.py @@ -30,7 +30,11 @@ from dlpx.virtualization.libs.exceptions import (IncorrectArgumentTypeError, LibraryError, PluginScriptError) -from dlpx.virtualization.common._common_classes import RemoteConnection +from dlpx.virtualization.common._common_classes import (RemoteConnection, + PasswordCredentials, + KeyPairCredentials) +from google.protobuf import json_format +from google.protobuf.struct_pb2 import Struct import logging @@ -39,7 +43,9 @@ "run_bash", "run_sync", "run_powershell", - "run_expect" + "run_expect", + "retrieve_credentials", + "upgrade_password" ] @@ -410,3 +416,63 @@ def _log_request(message, log_level): response = internal_libs.log(log_request) _handle_response(response) + + +def retrieve_credentials(credentials_supplier): + """This is an internal wrapper around the Virtualization library's credentials retrieval API. + Given a supplier provided by Virtualization, retrieves the credentials from that supplier. + + Args: + credentials_supplier (dict): Properties that make up a supplier of credentials. + Return: + Subclass of Credentials retrieved from supplier. Either a PasswordCredentials or a KeyPairCredentials + from dlpx.virtualization.common._common_classes. + """ + from dlpx.virtualization._engine import libs as internal_libs + + if not isinstance(credentials_supplier, dict): + raise IncorrectArgumentTypeError('credentials_supplier', type(credentials_supplier), dict) + + credentials_request = libs_pb2.CredentialsRequest() + credentials_struct = Struct() + credentials_struct.update(credentials_supplier) + credentials_request.credentials_supplier.CopyFrom(credentials_struct) + + response = internal_libs.retrieve_credentials(credentials_request) + + credentials_result = _handle_response(response) + if credentials_result.password != "": + return PasswordCredentials(credentials_result.username, credentials_result.password) + return KeyPairCredentials( + credentials_result.username, + credentials_result.key_pair.private_key, + credentials_result.key_pair.public_key) + + +def upgrade_password(password, username=None): + """This is an internal wrapper around Virtualization's credentials-supplier conversion API. + It is intended for use during plugin upgrade when a plugin needs to transform a password + value into a more generic credentials supplier object. + + Args: + password (basestring): Plain password string. + username (basestring, defaults to None): User name contained in the password credential supplier to return. + Return: + Credentials supplier (dict) that supplies the given password and username. + """ + from dlpx.virtualization._engine import libs as internal_libs + + if not isinstance(password, basestring): + raise IncorrectArgumentTypeError('password', type(password), basestring) + if username and not isinstance(username, basestring): + raise IncorrectArgumentTypeError('username', type(username), basestring, required=False) + + upgrade_password_request = libs_pb2.UpgradePasswordRequest() + upgrade_password_request.password = password + if username: + upgrade_password_request.username = username + + response = internal_libs.upgrade_password(upgrade_password_request) + + upgrade_password_result = _handle_response(response) + return json_format.MessageToDict(upgrade_password_result.credentials_supplier) diff --git a/libs/src/test/python/dlpx/virtualization/test_libs.py b/libs/src/test/python/dlpx/virtualization/test_libs.py index 83364932..4e94d375 100644 --- a/libs/src/test/python/dlpx/virtualization/test_libs.py +++ b/libs/src/test/python/dlpx/virtualization/test_libs.py @@ -9,6 +9,8 @@ from dlpx.virtualization import libs from dlpx.virtualization.libs.exceptions import ( IncorrectArgumentTypeError, LibraryError, PluginScriptError) +from google.protobuf import json_format +from dlpx.virtualization.common._common_classes import (PasswordCredentials) class TestLibsRunBash: @@ -773,3 +775,157 @@ def test_run_expect_bad_variables(remote_connection): " type 'dict of basestring:basestring' if defined.") assert (err_info.value.message == message.format('int', 'str') or err_info.value.message == message.format('str', 'int')) + + +class TestLibsRetrieveCredentials: + @staticmethod + def test_retrieve_password_credentials(): + expected_retrieve_credentials_response = libs_pb2.CredentialsResponse() + expected_retrieve_credentials_response.return_value.username = 'some user' + expected_retrieve_credentials_response.return_value.password = 'some password' + + expected_credentials_supplier = {'some supplier property': 'some supplier value'} + + def mock_retrieve_credentials(actual_retrieve_credentials_request): + assert json_format.MessageToDict(actual_retrieve_credentials_request.credentials_supplier) == \ + expected_credentials_supplier + + return expected_retrieve_credentials_response + + with mock.patch('dlpx.virtualization._engine.libs.retrieve_credentials', + side_effect=mock_retrieve_credentials, create=True): + actual_retrieve_credentials_result = libs.retrieve_credentials(expected_credentials_supplier) + + expected = expected_retrieve_credentials_response.return_value + assert actual_retrieve_credentials_result.username == expected.username + assert actual_retrieve_credentials_result.password == expected.password + + @staticmethod + def test_retrieve_keypair_credentials(): + expected_retrieve_credentials_response = libs_pb2.CredentialsResponse() + expected_retrieve_credentials_response.return_value.username = 'some user' + expected_retrieve_credentials_response.return_value.key_pair.private_key = 'some private key' + expected_retrieve_credentials_response.return_value.key_pair.public_key = 'some public key' + + expected_credentials_supplier = {'some supplier property': 'some supplier value'} + + def mock_retrieve_credentials(actual_retrieve_credentials_request): + assert json_format.MessageToDict(actual_retrieve_credentials_request.credentials_supplier) == \ + expected_credentials_supplier + + return expected_retrieve_credentials_response + + with mock.patch('dlpx.virtualization._engine.libs.retrieve_credentials', + side_effect=mock_retrieve_credentials, create=True): + actual_retrieve_credentials_result = libs.retrieve_credentials(expected_credentials_supplier) + + expected = expected_retrieve_credentials_response.return_value + assert actual_retrieve_credentials_result.username == expected.username + assert actual_retrieve_credentials_result.private_key == expected.key_pair.private_key + assert actual_retrieve_credentials_result.public_key == expected.key_pair.public_key + + @staticmethod + def test_retrieve_credentials_with_actionable_error(): + expected_id = 15 + expected_message = 'Some message' + + response = libs_pb2.CredentialsResponse() + response.error.actionable_error.id = expected_id + response.error.actionable_error.message = expected_message + + with mock.patch('dlpx.virtualization._engine.libs.retrieve_credentials', + return_value=response, create=True): + with pytest.raises(LibraryError) as err_info: + libs.retrieve_credentials({'some supplier property': 'some supplier value'}) + + assert err_info.value._id == expected_id + assert err_info.value.message == expected_message + + @staticmethod + def test_retrieve_credentials_with_nonactionable_error(): + response = libs_pb2.CredentialsResponse() + na_error = libs_pb2.NonActionableLibraryError() + response.error.non_actionable_error.CopyFrom(na_error) + + with mock.patch('dlpx.virtualization._engine.libs.retrieve_credentials', + return_value=response, create=True): + with pytest.raises(SystemExit): + libs.retrieve_credentials({'some supplier property': 'some supplier value'}) + + @staticmethod + def test_retrieve_credentials_bad_supplier(): + # Set the supplier be an int instead of a dictionary. + credentials_supplier = 10 + + with pytest.raises(IncorrectArgumentTypeError) as err_info: + libs.retrieve_credentials(credentials_supplier) + + assert err_info.value.message == ( + "The function retrieve_credentials's argument 'credentials_supplier' was" + " type 'int' but should be of type 'dict'.") + + +class TestLibsUpgradePassword: + @staticmethod + def test_upgrade_password(): + expected_password = 'some password' + + expected_credentials_supplier = {'type': 'NamedPasswordCredential', 'password': expected_password} + expected_upgrade_password_response = libs_pb2.UpgradePasswordResponse() + expected_upgrade_password_response.return_value.credentials_supplier.update(expected_credentials_supplier) + + def mock_upgrade_password(actual_upgrade_password_request): + assert actual_upgrade_password_request.password == expected_password + + return expected_upgrade_password_response + + with mock.patch('dlpx.virtualization._engine.libs.upgrade_password', + side_effect=mock_upgrade_password, create=True): + actual_upgrade_password_result = libs.upgrade_password(expected_password) + + assert actual_upgrade_password_result == expected_credentials_supplier + + @staticmethod + def test_upgrade_password_with_username(): + expected_password = 'some password' + expected_username = 'some user name' + + expected_credentials_supplier = {'type': 'NamedPasswordCredential', 'password': expected_password, 'username': expected_username} + expected_upgrade_password_response = libs_pb2.UpgradePasswordResponse() + expected_upgrade_password_response.return_value.credentials_supplier.update(expected_credentials_supplier) + + def mock_upgrade_password(actual_upgrade_password_request): + assert actual_upgrade_password_request.password == expected_password + assert actual_upgrade_password_request.username == expected_username + + return expected_upgrade_password_response + + with mock.patch('dlpx.virtualization._engine.libs.upgrade_password', + side_effect=mock_upgrade_password, create=True): + actual_upgrade_password_result = libs.upgrade_password(expected_password, username=expected_username) + + assert actual_upgrade_password_result == expected_credentials_supplier + + @staticmethod + def test_upgrade_password_invalid_password(): + expected_password = 10 + expected_username = 'some user name' + + with pytest.raises(IncorrectArgumentTypeError) as err_info: + libs.upgrade_password(expected_password, username=expected_username) + + assert err_info.value.message == ( + "The function upgrade_password's argument 'password' was" + " type 'int' but should be of type 'basestring'.") + + @staticmethod + def test_upgrade_password_invalid_username(): + expected_password = 'some password' + expected_username = 10 + + with pytest.raises(IncorrectArgumentTypeError) as err_info: + libs.upgrade_password(expected_password, username=expected_username) + + assert err_info.value.message == ( + "The function upgrade_password's argument 'username' was" + " type 'int' but should be of type 'basestring' if defined.") diff --git a/platform/setup.py b/platform/setup.py index 5c63a776..2e58111e 100644 --- a/platform/setup.py +++ b/platform/setup.py @@ -7,7 +7,7 @@ version = version_file.read().strip() install_requires = [ - "dvp-api == 1.4.0", + "dvp-api == 1.5.0.dev1", "dvp-common == {}".format(version), "enum34;python_version < '3.4'", ] diff --git a/platform/src/main/python/dlpx/virtualization/platform/VERSION b/platform/src/main/python/dlpx/virtualization/platform/VERSION index 56fea8a0..a8eefdb0 100644 --- a/platform/src/main/python/dlpx/virtualization/platform/VERSION +++ b/platform/src/main/python/dlpx/virtualization/platform/VERSION @@ -1 +1 @@ -3.0.0 \ No newline at end of file +3.1.0.dev2 diff --git a/tools/src/main/python/dlpx/virtualization/_internal/VERSION b/tools/src/main/python/dlpx/virtualization/_internal/VERSION index 56fea8a0..a8eefdb0 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/VERSION +++ b/tools/src/main/python/dlpx/virtualization/_internal/VERSION @@ -1 +1 @@ -3.0.0 \ No newline at end of file +3.1.0.dev2 diff --git a/tools/src/main/python/dlpx/virtualization/_internal/codegen.py b/tools/src/main/python/dlpx/virtualization/_internal/codegen.py index 77a34397..30b1529c 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/codegen.py +++ b/tools/src/main/python/dlpx/virtualization/_internal/codegen.py @@ -90,10 +90,37 @@ def generate_python(name, source_dir, plugin_config_dir, schema_content): _copy_generated_to_dir(output_dir, source_dir) +# +# Makes all references to platform json schema definitions "opaque" by replacing +# them with plain object types. +# +def _make_url_refs_opaque(json): + if isinstance(json, dict): + for key in json: + if key == '$ref' and isinstance(json[key], basestring)\ + and json[key].startswith('https://delphix.com/platform/api#'): + json.pop(key) + json['type'] = 'object' + else: + _make_url_refs_opaque(json[key]) + elif isinstance(json, list): + for element in json: + _make_url_refs_opaque(element) + + def _write_swagger_file(name, schema_dict, output_dir): swagger_json = copy.deepcopy(SWAGGER_JSON_FORMAT) swagger_json['info']['title'] = name - swagger_json['definitions'] = schema_dict + swagger_json['definitions'] = copy.deepcopy(schema_dict) + # + # When resolving URL $refs in JSON schemas, Swagger tries to actually access those + # URLs. But URL $refs are only meant to be identifiers. Also, specifically, plugin + # developers shouldn't have to manipulate credentials objects, which are passed to + # plugins by Virtualization and only intended for passing in call-backs to + # Virtualization. Thus, we replace $refs for plugin API objects with generic, opaque + # objects. + # + _make_url_refs_opaque(swagger_json['definitions']) swagger_file = os.path.join(output_dir, SWAGGER_FILE_NAME) logger.info('Writing swagger file to {}'.format(swagger_file)) diff --git a/tools/src/main/python/dlpx/virtualization/_internal/commands/build.py b/tools/src/main/python/dlpx/virtualization/_internal/commands/build.py index 5a1fec2a..30eff7ab 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/commands/build.py +++ b/tools/src/main/python/dlpx/virtualization/_internal/commands/build.py @@ -27,6 +27,14 @@ BUILD_DIR_NAME = 'build' +UNPAIRED_SURROGATE_DEFINITION = '''_UNPAIRED_SURROGATE_PATTERN = re.compile(six.u( + r\'[\\ud800-\\udbff](?![\\udc00-\\udfff])|(?