diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 9e9dedd4..1807b5b5 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 3.0.0 +current_version = 3.1.0.dev2 commit = False tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+)(?P\d+))? diff --git a/.github/workflows/publish-docs.yml b/.github/workflows/publish-docs.yml index 03af1009..a7165fef 100644 --- a/.github/workflows/publish-docs.yml +++ b/.github/workflows/publish-docs.yml @@ -9,6 +9,7 @@ on: - 'docs/**' paths: - 'docs/**' + - '.github/workflows/publish-docs.yml' jobs: publish: @@ -17,6 +18,7 @@ jobs: matrix: python-version: [3.7] package: [docs] + repository: ['delphix/virtualization-sdk'] steps: - uses: actions/checkout@v2 @@ -69,7 +71,18 @@ jobs: working-directory: ${{ matrix.package }} run: | pipenv run mkdocs build --clean - - name: Deploy the contents of docs/site to gh-pages 🚀 + - name: Deploy the contents of docs/site to gh-pages (developer.delphix.com) 🚀 + if: ${{ github.repository == matrix.repository }} + uses: peaceiris/actions-gh-pages@v3 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + publish_dir: docs/site + commit_message: Deploy to gh-pages 🚀 + user_name: "github-actions[bot]" + user_email: "github-actions[bot]@users.noreply.github.com" + cname: developer.delphix.com + - name: Deploy the contents of docs/site to personal gh-pages 🚀 + if: ${{ github.repository != matrix.repository }} uses: peaceiris/actions-gh-pages@v3 with: github_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/common/requirements.txt b/common/requirements.txt index 33673b1b..3b2c337b 100644 --- a/common/requirements.txt +++ b/common/requirements.txt @@ -1,5 +1,6 @@ bump2version==0.5.11 contextlib2==0.6.0.post1 ; python_version < '3' +enum34==1.1.6 funcsigs==1.0.2 ; python_version < '3.0' importlib-metadata==0.23 ; python_version < '3.8' more-itertools==5.0.0 ; python_version <= '2.7' diff --git a/common/setup.py b/common/setup.py index 7d9f0244..6b5dff76 100644 --- a/common/setup.py +++ b/common/setup.py @@ -4,7 +4,7 @@ PYTHON_SRC = 'src/main/python' install_requires = [ - "dvp-api == 1.4.0", + "dvp-api == 1.5.0.dev1", ] with open(os.path.join(PYTHON_SRC, 'dlpx/virtualization/common/VERSION')) as version_file: diff --git a/common/src/main/python/dlpx/virtualization/common/VERSION b/common/src/main/python/dlpx/virtualization/common/VERSION index 56fea8a0..a8eefdb0 100644 --- a/common/src/main/python/dlpx/virtualization/common/VERSION +++ b/common/src/main/python/dlpx/virtualization/common/VERSION @@ -1 +1 @@ -3.0.0 \ No newline at end of file +3.1.0.dev2 diff --git a/common/src/main/python/dlpx/virtualization/common/_common_classes.py b/common/src/main/python/dlpx/virtualization/common/_common_classes.py index c5d71fe3..2ad05804 100644 --- a/common/src/main/python/dlpx/virtualization/common/_common_classes.py +++ b/common/src/main/python/dlpx/virtualization/common/_common_classes.py @@ -2,8 +2,10 @@ # Copyright (c) 2019 by Delphix. All rights reserved. # -from dlpx.virtualization.api import common_pb2 +from abc import ABCMeta +from dlpx.virtualization.api import common_pb2, libs_pb2 from dlpx.virtualization.common.exceptions import IncorrectTypeError +from enum import IntEnum """Classes used for Plugin Operations @@ -17,7 +19,10 @@ "RemoteConnection", "RemoteEnvironment", "RemoteHost", - "RemoteUser"] + "RemoteUser", + "Credentials", + "PasswordCredentials", + "KeyPairCredentials"] class RemoteConnection(object): @@ -293,3 +298,121 @@ def from_proto(user): common_pb2.RemoteUser) remote_user = RemoteUser(name=user.name, reference=user.reference) return remote_user + + +class Credentials(object): + """Plugin base class for CredentialsResult to be used for plugin operations + and library functions. + + Plugin authors should use this instead of the corresponding protobuf generated + class. + + Args: + username: User name. + """ + def __init__(self, username): + if not isinstance(username, basestring): + raise IncorrectTypeError( + Credentials, + 'username', + type(username), + basestring) + self.__username = username + + __metaclass__ = ABCMeta + + @property + def username(self): + return self.__username + + +class PasswordCredentials(Credentials): + """Plugin class for CredentialsResult with password to be used for plugin operations + and library functions. + + Plugin authors should use this instead of the corresponding protobuf generated + class. + + Args: + username: User name. + password: Password. + """ + def __init__(self, username, password): + super(PasswordCredentials, self).__init__(username) + if not isinstance(password, basestring): + raise IncorrectTypeError( + PasswordCredentials, + 'password', + type(password), + basestring) + self.__password = password + + @property + def password(self): + return self.__password + + @staticmethod + def from_proto(credentials_result): + """Converts protobuf class libs_pb2.CredentialsResult to plugin class PasswordCredentials + """ + if not isinstance(credentials_result, libs_pb2.CredentialsResult): + raise IncorrectTypeError( + PasswordCredentials, + 'credentials_result', + type(credentials_result), + libs_pb2.CredentialsResult) + return PasswordCredentials( + username=credentials_result.username, password=credentials_result.pasword) + + +class KeyPairCredentials(Credentials): + """Plugin class for CredentialsResult with key pair to be used for plugin operations + and library functions. + + Plugin authors should use this instead of the corresponding protobuf generated + class. + + Args: + username (str): User name. + private_key (str): Private key. + public_key (str): Public key corresponding to private key. Empty string if not present. + """ + def __init__(self, username, private_key, public_key): + super(KeyPairCredentials, self).__init__(username) + if not isinstance(private_key, basestring): + raise IncorrectTypeError( + KeyPairCredentials, + 'private_key', + type(private_key), + basestring) + self.__private_key = private_key + if not isinstance(public_key, basestring): + raise IncorrectTypeError( + KeyPairCredentials, + 'public_key', + type(public_key), + basestring) + self.__public_key = public_key + + @property + def private_key(self): + return self.__private_key + + @property + def public_key(self): + return self.__public_key + + @staticmethod + def from_proto(credentials_result): + """Converts protobuf class libs_pb2.CredentialsResult to plugin class KeyPairCredentials + """ + if not isinstance(credentials_result, libs_pb2.CredentialsResult): + raise IncorrectTypeError( + KeyPairCredentials, + 'credentials_result', + type(credentials_result), + libs_pb2.CredentialsResult) + return KeyPairCredentials( + username=credentials_result.username, + private_key=credentials_result.key_pair.private_key, + public_key=credentials_result.key_pair.public_key) diff --git a/docs/docs/Building_Your_First_Plugin/Data_Ingestion.md b/docs/docs/Building_Your_First_Plugin/Data_Ingestion.md index a3aaf6d3..67dcd0cb 100644 --- a/docs/docs/Building_Your_First_Plugin/Data_Ingestion.md +++ b/docs/docs/Building_Your_First_Plugin/Data_Ingestion.md @@ -21,7 +21,7 @@ quite limiting. For our first plugin, we will be using the more flexible [staging](/References/Glossary.md#staged-linkingsyncing) strategy. With this strategy, the Delphix Engine uses NFS for Unix environments (or iSCSI on Windows environments) to mount storage onto a [staging environment](/References/Glossary.md#staging-environment). Our plugin will then be in full control of how to get data from the source environment onto this storage mount. -With the staging strategy, there are two types of syncs: sync and resync. A `sync` is used to ingest incremental changes while a `resync` is used to re-ingest all the data for the dSource. For databases, this could mean re-ingesting from a full database backup to reset the dSource. A `sync` and a `resync` will execute the same plugin operations. To differentiate a `sync` from a `resync`, simply add a boolean property (i.e. `resync`) in the plugin's [snapshot parameters definition](References/Schemas_and_Autogenerated_Classes.md#snapshotparametersdefinition-schema). Once `sync` or `resync` is selected, the property will be passed into [linked.pre_snapshot](/References/Plugin_Operations.md#staged-linked-source-pre-snapshot) and [linked.post_snapshot](/References/Plugin_Operations.md#staged-linked-source-post-snapshot) as a [snapshot parameter](/References/Glossary.md#snapshot-parameters). +With the staging strategy, there are two types of syncs: sync and resync. A `sync` is used to ingest incremental changes while a `resync` is used to re-ingest all the data for the dSource. For databases, this could mean re-ingesting from a full database backup to reset the dSource. A `sync` and a `resync` will execute the same plugin operations. To differentiate a `sync` from a `resync`, simply add a boolean property (i.e. `resync`) in the plugin's [snapshot parameters definition](/References/Schemas_and_Autogenerated_Classes.md#snapshotparametersdefinition-schema). Once `sync` or `resync` is selected, the property will be passed into [linked.pre_snapshot](/References/Plugin_Operations.md#staged-linked-source-pre-snapshot) and [linked.post_snapshot](/References/Plugin_Operations.md#staged-linked-source-post-snapshot) as a [snapshot parameter](/References/Glossary.md#snapshot-parameters). A regular `sync` is the default and is executed as part of policy driven syncs. A `resync` is only executed during initial ingestion or if the Delphix user manually starts one. The customer can manually trigger a `resync` via the UI by selecting the dSource, going to more options and selecting **Resynchronize dSource**. ![Screenshot](images/Resync.png) diff --git a/docs/docs/Release_Notes/3.0.0/3.0.0.md b/docs/docs/Release_Notes/3.0.0/3.0.0.md index e8c41e89..c86f0998 100644 --- a/docs/docs/Release_Notes/3.0.0/3.0.0.md +++ b/docs/docs/Release_Notes/3.0.0/3.0.0.md @@ -7,9 +7,14 @@ To install or upgrade the SDK, refer to instructions [here](/Getting_Started.md# * Provide end-users with configurable options prior to taking a snapshot. * The options selected are provided as input to pre/post-snapshot functions. +* Added support to initialize an [empty VDB](/References/Glossary#empty-vdb). + * Create an empty virtual dataset within Delphix, instead of creating it externally and ingesting it. + * Utilize this functionality by implementing the [initialize](/References/Plugin_Operations#virtual-source-initialize) operation. + * Added a `scratch_path` property on the [RemoteHost](/References/Classes/#remotehost) object which can be used as: * A location to store small amounts of persistent data. * A location to mount VDB data. + More details about `scratch_path` can be found [here](/Best_Practices/Scratch_Paths.md) ## Breaking Changes diff --git a/dvp/src/main/python/dlpx/virtualization/VERSION b/dvp/src/main/python/dlpx/virtualization/VERSION index 56fea8a0..a8eefdb0 100644 --- a/dvp/src/main/python/dlpx/virtualization/VERSION +++ b/dvp/src/main/python/dlpx/virtualization/VERSION @@ -1 +1 @@ -3.0.0 \ No newline at end of file +3.1.0.dev2 diff --git a/libs/requirements.txt b/libs/requirements.txt index 275195fe..a8a53e05 100644 --- a/libs/requirements.txt +++ b/libs/requirements.txt @@ -1,6 +1,7 @@ ./../common bump2version==0.5.11 contextlib2==0.6.0.post1 ; python_version < '3' +enum34==1.1.6 funcsigs==1.0.2 ; python_version < '3.3' importlib-metadata==1.3.0 ; python_version < '3.8' mock==3.0.5 diff --git a/libs/setup.py b/libs/setup.py index d7c601f8..17cea43b 100644 --- a/libs/setup.py +++ b/libs/setup.py @@ -7,7 +7,7 @@ version = version_file.read().strip() install_requires = [ - "dvp-api == 1.4.0", + "dvp-api == 1.5.0.dev1", "dvp-common == {}".format(version) ] diff --git a/libs/src/main/python/dlpx/virtualization/libs/VERSION b/libs/src/main/python/dlpx/virtualization/libs/VERSION index 56fea8a0..a8eefdb0 100644 --- a/libs/src/main/python/dlpx/virtualization/libs/VERSION +++ b/libs/src/main/python/dlpx/virtualization/libs/VERSION @@ -1 +1 @@ -3.0.0 \ No newline at end of file +3.1.0.dev2 diff --git a/libs/src/main/python/dlpx/virtualization/libs/libs.py b/libs/src/main/python/dlpx/virtualization/libs/libs.py index 2e0fb4bd..69355a4b 100644 --- a/libs/src/main/python/dlpx/virtualization/libs/libs.py +++ b/libs/src/main/python/dlpx/virtualization/libs/libs.py @@ -30,7 +30,11 @@ from dlpx.virtualization.libs.exceptions import (IncorrectArgumentTypeError, LibraryError, PluginScriptError) -from dlpx.virtualization.common._common_classes import RemoteConnection +from dlpx.virtualization.common._common_classes import (RemoteConnection, + PasswordCredentials, + KeyPairCredentials) +from google.protobuf import json_format +from google.protobuf.struct_pb2 import Struct import logging @@ -39,7 +43,9 @@ "run_bash", "run_sync", "run_powershell", - "run_expect" + "run_expect", + "retrieve_credentials", + "upgrade_password" ] @@ -410,3 +416,63 @@ def _log_request(message, log_level): response = internal_libs.log(log_request) _handle_response(response) + + +def retrieve_credentials(credentials_supplier): + """This is an internal wrapper around the Virtualization library's credentials retrieval API. + Given a supplier provided by Virtualization, retrieves the credentials from that supplier. + + Args: + credentials_supplier (dict): Properties that make up a supplier of credentials. + Return: + Subclass of Credentials retrieved from supplier. Either a PasswordCredentials or a KeyPairCredentials + from dlpx.virtualization.common._common_classes. + """ + from dlpx.virtualization._engine import libs as internal_libs + + if not isinstance(credentials_supplier, dict): + raise IncorrectArgumentTypeError('credentials_supplier', type(credentials_supplier), dict) + + credentials_request = libs_pb2.CredentialsRequest() + credentials_struct = Struct() + credentials_struct.update(credentials_supplier) + credentials_request.credentials_supplier.CopyFrom(credentials_struct) + + response = internal_libs.retrieve_credentials(credentials_request) + + credentials_result = _handle_response(response) + if credentials_result.password != "": + return PasswordCredentials(credentials_result.username, credentials_result.password) + return KeyPairCredentials( + credentials_result.username, + credentials_result.key_pair.private_key, + credentials_result.key_pair.public_key) + + +def upgrade_password(password, username=None): + """This is an internal wrapper around Virtualization's credentials-supplier conversion API. + It is intended for use during plugin upgrade when a plugin needs to transform a password + value into a more generic credentials supplier object. + + Args: + password (basestring): Plain password string. + username (basestring, defaults to None): User name contained in the password credential supplier to return. + Return: + Credentials supplier (dict) that supplies the given password and username. + """ + from dlpx.virtualization._engine import libs as internal_libs + + if not isinstance(password, basestring): + raise IncorrectArgumentTypeError('password', type(password), basestring) + if username and not isinstance(username, basestring): + raise IncorrectArgumentTypeError('username', type(username), basestring, required=False) + + upgrade_password_request = libs_pb2.UpgradePasswordRequest() + upgrade_password_request.password = password + if username: + upgrade_password_request.username = username + + response = internal_libs.upgrade_password(upgrade_password_request) + + upgrade_password_result = _handle_response(response) + return json_format.MessageToDict(upgrade_password_result.credentials_supplier) diff --git a/libs/src/test/python/dlpx/virtualization/test_libs.py b/libs/src/test/python/dlpx/virtualization/test_libs.py index 83364932..4e94d375 100644 --- a/libs/src/test/python/dlpx/virtualization/test_libs.py +++ b/libs/src/test/python/dlpx/virtualization/test_libs.py @@ -9,6 +9,8 @@ from dlpx.virtualization import libs from dlpx.virtualization.libs.exceptions import ( IncorrectArgumentTypeError, LibraryError, PluginScriptError) +from google.protobuf import json_format +from dlpx.virtualization.common._common_classes import (PasswordCredentials) class TestLibsRunBash: @@ -773,3 +775,157 @@ def test_run_expect_bad_variables(remote_connection): " type 'dict of basestring:basestring' if defined.") assert (err_info.value.message == message.format('int', 'str') or err_info.value.message == message.format('str', 'int')) + + +class TestLibsRetrieveCredentials: + @staticmethod + def test_retrieve_password_credentials(): + expected_retrieve_credentials_response = libs_pb2.CredentialsResponse() + expected_retrieve_credentials_response.return_value.username = 'some user' + expected_retrieve_credentials_response.return_value.password = 'some password' + + expected_credentials_supplier = {'some supplier property': 'some supplier value'} + + def mock_retrieve_credentials(actual_retrieve_credentials_request): + assert json_format.MessageToDict(actual_retrieve_credentials_request.credentials_supplier) == \ + expected_credentials_supplier + + return expected_retrieve_credentials_response + + with mock.patch('dlpx.virtualization._engine.libs.retrieve_credentials', + side_effect=mock_retrieve_credentials, create=True): + actual_retrieve_credentials_result = libs.retrieve_credentials(expected_credentials_supplier) + + expected = expected_retrieve_credentials_response.return_value + assert actual_retrieve_credentials_result.username == expected.username + assert actual_retrieve_credentials_result.password == expected.password + + @staticmethod + def test_retrieve_keypair_credentials(): + expected_retrieve_credentials_response = libs_pb2.CredentialsResponse() + expected_retrieve_credentials_response.return_value.username = 'some user' + expected_retrieve_credentials_response.return_value.key_pair.private_key = 'some private key' + expected_retrieve_credentials_response.return_value.key_pair.public_key = 'some public key' + + expected_credentials_supplier = {'some supplier property': 'some supplier value'} + + def mock_retrieve_credentials(actual_retrieve_credentials_request): + assert json_format.MessageToDict(actual_retrieve_credentials_request.credentials_supplier) == \ + expected_credentials_supplier + + return expected_retrieve_credentials_response + + with mock.patch('dlpx.virtualization._engine.libs.retrieve_credentials', + side_effect=mock_retrieve_credentials, create=True): + actual_retrieve_credentials_result = libs.retrieve_credentials(expected_credentials_supplier) + + expected = expected_retrieve_credentials_response.return_value + assert actual_retrieve_credentials_result.username == expected.username + assert actual_retrieve_credentials_result.private_key == expected.key_pair.private_key + assert actual_retrieve_credentials_result.public_key == expected.key_pair.public_key + + @staticmethod + def test_retrieve_credentials_with_actionable_error(): + expected_id = 15 + expected_message = 'Some message' + + response = libs_pb2.CredentialsResponse() + response.error.actionable_error.id = expected_id + response.error.actionable_error.message = expected_message + + with mock.patch('dlpx.virtualization._engine.libs.retrieve_credentials', + return_value=response, create=True): + with pytest.raises(LibraryError) as err_info: + libs.retrieve_credentials({'some supplier property': 'some supplier value'}) + + assert err_info.value._id == expected_id + assert err_info.value.message == expected_message + + @staticmethod + def test_retrieve_credentials_with_nonactionable_error(): + response = libs_pb2.CredentialsResponse() + na_error = libs_pb2.NonActionableLibraryError() + response.error.non_actionable_error.CopyFrom(na_error) + + with mock.patch('dlpx.virtualization._engine.libs.retrieve_credentials', + return_value=response, create=True): + with pytest.raises(SystemExit): + libs.retrieve_credentials({'some supplier property': 'some supplier value'}) + + @staticmethod + def test_retrieve_credentials_bad_supplier(): + # Set the supplier be an int instead of a dictionary. + credentials_supplier = 10 + + with pytest.raises(IncorrectArgumentTypeError) as err_info: + libs.retrieve_credentials(credentials_supplier) + + assert err_info.value.message == ( + "The function retrieve_credentials's argument 'credentials_supplier' was" + " type 'int' but should be of type 'dict'.") + + +class TestLibsUpgradePassword: + @staticmethod + def test_upgrade_password(): + expected_password = 'some password' + + expected_credentials_supplier = {'type': 'NamedPasswordCredential', 'password': expected_password} + expected_upgrade_password_response = libs_pb2.UpgradePasswordResponse() + expected_upgrade_password_response.return_value.credentials_supplier.update(expected_credentials_supplier) + + def mock_upgrade_password(actual_upgrade_password_request): + assert actual_upgrade_password_request.password == expected_password + + return expected_upgrade_password_response + + with mock.patch('dlpx.virtualization._engine.libs.upgrade_password', + side_effect=mock_upgrade_password, create=True): + actual_upgrade_password_result = libs.upgrade_password(expected_password) + + assert actual_upgrade_password_result == expected_credentials_supplier + + @staticmethod + def test_upgrade_password_with_username(): + expected_password = 'some password' + expected_username = 'some user name' + + expected_credentials_supplier = {'type': 'NamedPasswordCredential', 'password': expected_password, 'username': expected_username} + expected_upgrade_password_response = libs_pb2.UpgradePasswordResponse() + expected_upgrade_password_response.return_value.credentials_supplier.update(expected_credentials_supplier) + + def mock_upgrade_password(actual_upgrade_password_request): + assert actual_upgrade_password_request.password == expected_password + assert actual_upgrade_password_request.username == expected_username + + return expected_upgrade_password_response + + with mock.patch('dlpx.virtualization._engine.libs.upgrade_password', + side_effect=mock_upgrade_password, create=True): + actual_upgrade_password_result = libs.upgrade_password(expected_password, username=expected_username) + + assert actual_upgrade_password_result == expected_credentials_supplier + + @staticmethod + def test_upgrade_password_invalid_password(): + expected_password = 10 + expected_username = 'some user name' + + with pytest.raises(IncorrectArgumentTypeError) as err_info: + libs.upgrade_password(expected_password, username=expected_username) + + assert err_info.value.message == ( + "The function upgrade_password's argument 'password' was" + " type 'int' but should be of type 'basestring'.") + + @staticmethod + def test_upgrade_password_invalid_username(): + expected_password = 'some password' + expected_username = 10 + + with pytest.raises(IncorrectArgumentTypeError) as err_info: + libs.upgrade_password(expected_password, username=expected_username) + + assert err_info.value.message == ( + "The function upgrade_password's argument 'username' was" + " type 'int' but should be of type 'basestring' if defined.") diff --git a/platform/setup.py b/platform/setup.py index 5c63a776..2e58111e 100644 --- a/platform/setup.py +++ b/platform/setup.py @@ -7,7 +7,7 @@ version = version_file.read().strip() install_requires = [ - "dvp-api == 1.4.0", + "dvp-api == 1.5.0.dev1", "dvp-common == {}".format(version), "enum34;python_version < '3.4'", ] diff --git a/platform/src/main/python/dlpx/virtualization/platform/VERSION b/platform/src/main/python/dlpx/virtualization/platform/VERSION index 56fea8a0..a8eefdb0 100644 --- a/platform/src/main/python/dlpx/virtualization/platform/VERSION +++ b/platform/src/main/python/dlpx/virtualization/platform/VERSION @@ -1 +1 @@ -3.0.0 \ No newline at end of file +3.1.0.dev2 diff --git a/tools/src/main/python/dlpx/virtualization/_internal/VERSION b/tools/src/main/python/dlpx/virtualization/_internal/VERSION index 56fea8a0..a8eefdb0 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/VERSION +++ b/tools/src/main/python/dlpx/virtualization/_internal/VERSION @@ -1 +1 @@ -3.0.0 \ No newline at end of file +3.1.0.dev2 diff --git a/tools/src/main/python/dlpx/virtualization/_internal/codegen.py b/tools/src/main/python/dlpx/virtualization/_internal/codegen.py index 77a34397..30b1529c 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/codegen.py +++ b/tools/src/main/python/dlpx/virtualization/_internal/codegen.py @@ -90,10 +90,37 @@ def generate_python(name, source_dir, plugin_config_dir, schema_content): _copy_generated_to_dir(output_dir, source_dir) +# +# Makes all references to platform json schema definitions "opaque" by replacing +# them with plain object types. +# +def _make_url_refs_opaque(json): + if isinstance(json, dict): + for key in json: + if key == '$ref' and isinstance(json[key], basestring)\ + and json[key].startswith('https://delphix.com/platform/api#'): + json.pop(key) + json['type'] = 'object' + else: + _make_url_refs_opaque(json[key]) + elif isinstance(json, list): + for element in json: + _make_url_refs_opaque(element) + + def _write_swagger_file(name, schema_dict, output_dir): swagger_json = copy.deepcopy(SWAGGER_JSON_FORMAT) swagger_json['info']['title'] = name - swagger_json['definitions'] = schema_dict + swagger_json['definitions'] = copy.deepcopy(schema_dict) + # + # When resolving URL $refs in JSON schemas, Swagger tries to actually access those + # URLs. But URL $refs are only meant to be identifiers. Also, specifically, plugin + # developers shouldn't have to manipulate credentials objects, which are passed to + # plugins by Virtualization and only intended for passing in call-backs to + # Virtualization. Thus, we replace $refs for plugin API objects with generic, opaque + # objects. + # + _make_url_refs_opaque(swagger_json['definitions']) swagger_file = os.path.join(output_dir, SWAGGER_FILE_NAME) logger.info('Writing swagger file to {}'.format(swagger_file)) diff --git a/tools/src/main/python/dlpx/virtualization/_internal/commands/build.py b/tools/src/main/python/dlpx/virtualization/_internal/commands/build.py index 5a1fec2a..30eff7ab 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/commands/build.py +++ b/tools/src/main/python/dlpx/virtualization/_internal/commands/build.py @@ -27,6 +27,14 @@ BUILD_DIR_NAME = 'build' +UNPAIRED_SURROGATE_DEFINITION = '''_UNPAIRED_SURROGATE_PATTERN = re.compile(six.u( + r\'[\\ud800-\\udbff](?![\\udc00-\\udfff])|(?