diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 98ef68c6..bf9cf6b8 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.2.0.dev1 +current_version = 3.0.0.dev2 commit = False tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+)(?P\d+))? diff --git a/.dependabot/config.yaml b/.dependabot/config.yaml index 2502bd72..df419c78 100644 --- a/.dependabot/config.yaml +++ b/.dependabot/config.yaml @@ -3,21 +3,21 @@ update_configs: # Keep requirements.txt files up-to-date in each package. - package_manager: "python" directory: "/dvp" - update_schedule: "daily" + update_schedule: "monthly" target_branch: "develop" - package_manager: "python" directory: "/common" - update_schedule: "daily" + update_schedule: "monthly" target_branch: "develop" - package_manager: "python" directory: "/platform" - update_schedule: "daily" + update_schedule: "monthly" target_branch: "develop" - package_manager: "python" directory: "/libs" - update_schedule: "daily" + update_schedule: "monthly" target_branch: "develop" - package_manager: "python" directory: "/tools" - update_schedule: "daily" + update_schedule: "monthly" target_branch: "develop" diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index b7f9d4da..b9a5cc1a 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1 +1 @@ -* @ankursarin @fdrozdowski @nhlien93 @crystalplumage +* @ankursarin @nhlien93 @mothslaw diff --git a/README-dev.md b/README-dev.md index 61c23ce7..34540f17 100644 --- a/README-dev.md +++ b/README-dev.md @@ -125,17 +125,17 @@ To run blackbox tests, follow these steps: 2. Navigate to the app-gate directory and start tests using `git blackbox`. For the guide on which test suite to use, see the next sections. -At a minimum, each pull request should pass `appdata_python_samples` and `appdata_sanity` tests with a direct or staged plugin. +At a minimum, each pull request should pass `appdata_python_samples` and `appdata_basic` tests with a direct or staged plugin. See the section below for the description of each test suite. #### Blackbox tests targeting wrappers (mostly Delphix Engine workflows) * appdata_python_samples (sample plugins from the app-gate): `git blackbox -s appdata_python_samples --extra-params="-p virt-sdk-repo=https://github.com//virtualization-sdk.git -p virt-sdk-branch=my-feature"`, -* appdata_sanity with a direct Python plugin on CentOS 7.3: `git blackbox -s appdata_sanity -c APPDATA_PYTHON_DIRECT_CENTOS73 -a --extra-params="-p virt-sdk-repo=https://github.com//virtualization-sdk.git -p virt-sdk-branch=my-feature"`, -* appdata_sanity with a staged Python plugin on CentOS 7.3: `git blackbox -s appdata_sanity -c APPDATA_PYTHON_STAGED_CENTOS73 -a --extra-params="-p virt-sdk-repo=https://github.com//virtualization-sdk.git -p virt-sdk-branch=my-feature"`. +* appdata_basic with a direct Python plugin on CentOS 7.3: `git blackbox -s appdata_basic -c APPDATA_PYTHON_DIRECT_CENTOS73 -a --extra-params="-p virt-sdk-repo=https://github.com//virtualization-sdk.git -p virt-sdk-branch=my-feature"`, +* appdata_basic with a staged Python plugin on CentOS 7.3: `git blackbox -s appdata_basic -c APPDATA_PYTHON_STAGED_CENTOS73 -a --extra-params="-p virt-sdk-repo=https://github.com//virtualization-sdk.git -p virt-sdk-branch=my-feature"`. #### Blackbox tests targeting the CLI (~80% CLI tests) * virtualization_sdk (installs and tests a direct Python plugin on Ubuntu 18): `git blackbox -s virtualization_sdk -c APPDATA_SDK_UBUNTU18_DIRECT_CENTOS73 --extra-params="-p virt-sdk-repo=https://github.com//virtualization-sdk.git -p virt-sdk-branch=my-feature"`, * virtualization_sdk (installs and tests a staged Python plugin on Ubuntu 18): -`git blackbox -s virtualization_sdk -c APPDATA_SDK_UBUNTU18_STAGED_CENTOS73 --extra-params="-p virt-sdk-repo=https://github.com//virtualization-sdk.git -p virt-sdk-branch=my-feature"`. \ No newline at end of file +`git blackbox -s virtualization_sdk -c APPDATA_SDK_UBUNTU18_STAGED_CENTOS73 --extra-params="-p virt-sdk-repo=https://github.com//virtualization-sdk.git -p virt-sdk-branch=my-feature"`. diff --git a/common/requirements.txt b/common/requirements.txt index 33673b1b..3b2c337b 100644 --- a/common/requirements.txt +++ b/common/requirements.txt @@ -1,5 +1,6 @@ bump2version==0.5.11 contextlib2==0.6.0.post1 ; python_version < '3' +enum34==1.1.6 funcsigs==1.0.2 ; python_version < '3.0' importlib-metadata==0.23 ; python_version < '3.8' more-itertools==5.0.0 ; python_version <= '2.7' diff --git a/common/setup.py b/common/setup.py index 64e5643c..c95b7890 100644 --- a/common/setup.py +++ b/common/setup.py @@ -4,7 +4,7 @@ PYTHON_SRC = 'src/main/python' install_requires = [ - "dvp-api == 1.3.0", + "dvp-api == 1.4.0.dev13", ] with open(os.path.join(PYTHON_SRC, 'dlpx/virtualization/common/VERSION')) as version_file: diff --git a/common/src/main/python/dlpx/virtualization/common/VERSION b/common/src/main/python/dlpx/virtualization/common/VERSION index f1855a71..a8693131 100644 --- a/common/src/main/python/dlpx/virtualization/common/VERSION +++ b/common/src/main/python/dlpx/virtualization/common/VERSION @@ -1 +1 @@ -2.2.0.dev1 \ No newline at end of file +3.0.0.dev2 diff --git a/common/src/main/python/dlpx/virtualization/common/_common_classes.py b/common/src/main/python/dlpx/virtualization/common/_common_classes.py index c5d71fe3..2ad05804 100644 --- a/common/src/main/python/dlpx/virtualization/common/_common_classes.py +++ b/common/src/main/python/dlpx/virtualization/common/_common_classes.py @@ -2,8 +2,10 @@ # Copyright (c) 2019 by Delphix. All rights reserved. # -from dlpx.virtualization.api import common_pb2 +from abc import ABCMeta +from dlpx.virtualization.api import common_pb2, libs_pb2 from dlpx.virtualization.common.exceptions import IncorrectTypeError +from enum import IntEnum """Classes used for Plugin Operations @@ -17,7 +19,10 @@ "RemoteConnection", "RemoteEnvironment", "RemoteHost", - "RemoteUser"] + "RemoteUser", + "Credentials", + "PasswordCredentials", + "KeyPairCredentials"] class RemoteConnection(object): @@ -293,3 +298,121 @@ def from_proto(user): common_pb2.RemoteUser) remote_user = RemoteUser(name=user.name, reference=user.reference) return remote_user + + +class Credentials(object): + """Plugin base class for CredentialsResult to be used for plugin operations + and library functions. + + Plugin authors should use this instead of the corresponding protobuf generated + class. + + Args: + username: User name. + """ + def __init__(self, username): + if not isinstance(username, basestring): + raise IncorrectTypeError( + Credentials, + 'username', + type(username), + basestring) + self.__username = username + + __metaclass__ = ABCMeta + + @property + def username(self): + return self.__username + + +class PasswordCredentials(Credentials): + """Plugin class for CredentialsResult with password to be used for plugin operations + and library functions. + + Plugin authors should use this instead of the corresponding protobuf generated + class. + + Args: + username: User name. + password: Password. + """ + def __init__(self, username, password): + super(PasswordCredentials, self).__init__(username) + if not isinstance(password, basestring): + raise IncorrectTypeError( + PasswordCredentials, + 'password', + type(password), + basestring) + self.__password = password + + @property + def password(self): + return self.__password + + @staticmethod + def from_proto(credentials_result): + """Converts protobuf class libs_pb2.CredentialsResult to plugin class PasswordCredentials + """ + if not isinstance(credentials_result, libs_pb2.CredentialsResult): + raise IncorrectTypeError( + PasswordCredentials, + 'credentials_result', + type(credentials_result), + libs_pb2.CredentialsResult) + return PasswordCredentials( + username=credentials_result.username, password=credentials_result.pasword) + + +class KeyPairCredentials(Credentials): + """Plugin class for CredentialsResult with key pair to be used for plugin operations + and library functions. + + Plugin authors should use this instead of the corresponding protobuf generated + class. + + Args: + username (str): User name. + private_key (str): Private key. + public_key (str): Public key corresponding to private key. Empty string if not present. + """ + def __init__(self, username, private_key, public_key): + super(KeyPairCredentials, self).__init__(username) + if not isinstance(private_key, basestring): + raise IncorrectTypeError( + KeyPairCredentials, + 'private_key', + type(private_key), + basestring) + self.__private_key = private_key + if not isinstance(public_key, basestring): + raise IncorrectTypeError( + KeyPairCredentials, + 'public_key', + type(public_key), + basestring) + self.__public_key = public_key + + @property + def private_key(self): + return self.__private_key + + @property + def public_key(self): + return self.__public_key + + @staticmethod + def from_proto(credentials_result): + """Converts protobuf class libs_pb2.CredentialsResult to plugin class KeyPairCredentials + """ + if not isinstance(credentials_result, libs_pb2.CredentialsResult): + raise IncorrectTypeError( + KeyPairCredentials, + 'credentials_result', + type(credentials_result), + libs_pb2.CredentialsResult) + return KeyPairCredentials( + username=credentials_result.username, + private_key=credentials_result.key_pair.private_key, + public_key=credentials_result.key_pair.public_key) diff --git a/docs/docs/References/Decorators.md b/docs/docs/References/Decorators.md index 20158783..038cf01c 100644 --- a/docs/docs/References/Decorators.md +++ b/docs/docs/References/Decorators.md @@ -13,13 +13,13 @@ plugin = Plugin() # Use the decorator to annotate the function that corresponds to the "Virtual Source Start" Plugin Operation @plugin.virtual_source.start() def my_start(virtual_source, repository, source_config): - print "running start" + print "running start" ``` !!! info Decorators exposed by the Virtualization SDK are inherently python function calls and needs parentheses `()` appended at the end. -Assuming the name of the object, is `plugin` as above, the table below lists the corresponding decorators for each plugin operation. +Assuming the name of the object is `plugin` as above, the table below lists the corresponding decorators for each plugin operation. Plugin Operation | Decorator ---------------- | -------- @@ -35,6 +35,7 @@ Plugin Operation | Decorator [Staged Linked Source Worker](Plugin_Operations.md#staged-linked-source-worker) | `@plugin.linked.worker()` [Staged Linked Source Mount Specification](Plugin_Operations.md#staged-linked-source-mount-specification) | `@plugin.linked.mount_specification()` [Virtual Source Configure](Plugin_Operations.md#virtual-source-configure) | `@plugin.virtual.configure()` +[Virtual Source Initialize](Plugin_Operations.md#virtual-source-initialize) | `@plugin.virtual.initialize()` [Virtual Source Unconfigure](Plugin_Operations.md#virtual-source-unconfigure) | `@plugin.virtual.unconfigure()` [Virtual Source Reconfigure](Plugin_Operations.md#virtual-source-reconfigure) | `@plugin.virtual.reconfigure()` [Virtual Source Start](Plugin_Operations.md#virtual-source-start) | `@plugin.virtual.start()` diff --git a/docs/docs/References/Glossary.md b/docs/docs/References/Glossary.md index a1038684..3d439050 100644 --- a/docs/docs/References/Glossary.md +++ b/docs/docs/References/Glossary.md @@ -28,6 +28,13 @@ The process by which the Delphix Engine learns about how a particular environmen ## dSource See [Linked Dataset](#linked-dataset) +## Empty VDB +A VDB that is created from scratch, without provisioning from another dataset. Users can create empty VDBs when they want to construct a brand-new dataset from within Delphix, instead of creating it externally and then ingesting it. + +This "empty" VDB, of course, will typically not stay empty for long. Data will be added as users work with the new dataset. + +A plugin can support this functionality by implementing the [initialize](Plugin_Operations.md#virtual-source-initialize) operation. + ## Environment A remote system that the Delphix Engine can interact with. An environment can be used as a [source](#source-environment), [staging](#staging-environment) or [target](#target-environment) environment (or any combination of those). For example, a Linux machine that the Delphix Engine can connect to is an environment. @@ -105,7 +112,7 @@ The process by which the Delphix Engine ingests data from a dataset on a [source An [environment](#environment) on which Delphix-provided virtualized datasets can be used. ## Lua Toolkit -Legacy model for writing "plugins" in Lua, with limited documentation and support for writing, building and uploading toolkits. This was the predecessor to the Virtualization SDK. +Legacy model for writing "plugins" in Lua, with limited documentation and limited support for writing, building and uploading toolkits. This was the predecessor to the Virtualization SDK. ## Upgrade Operation A special plugin operation that takes data produced by an older version of a plugin, and transforms it into the format expected by the new version of the plugin. diff --git a/docs/docs/References/Plugin_Operations.md b/docs/docs/References/Plugin_Operations.md index c0321ac2..2fe76756 100644 --- a/docs/docs/References/Plugin_Operations.md +++ b/docs/docs/References/Plugin_Operations.md @@ -22,6 +22,7 @@ Plugin Operation | **Required** | Decorator | Delphix Engine Operations [Staged Linked Source
Status](#staged-linked-source-status) | **No** |`linked.status()` | N/A [Staged Linked Source
Worker](#staged-linked-source-worker) | **No** |`linked.worker()` | N/A [Staged Linked Source
Mount Specification](#staged-linked-source-mount-specification) | **Yes** | `linked.mount_specification()` | [Linked Source Sync](Workflows.md#linked-source-sync)
[Linked Source Enable](Workflows.md#linked-source-enable) +[Virtual Source
Initialize](#virtual-source-initialize) | **No** | `virtual.initialize()` | [Virtual Source Create Empty VDB](Workflows.md#virtual-source-create-empty-vdb) [Virtual Source
Configure](#virtual-source-configure) | **Yes** | `virtual.configure()` | [Virtual Source Provision](Workflows.md#virtual-source-provision)
[Virtual Source Refresh](Workflows.md#virtual-source-refresh) [Virtual Source
Unconfigure](#virtual-source-unconfigure) | **No** | `virtual.unconfigure()` | [Virtual Source Refresh](Workflows.md#virtual-source-refresh)
[Virtual Source Delete](Workflows.md#virtual-source-delete) [Virtual Source
Reconfigure](#virtual-source-reconfigure) | **Yes** | `virtual.reconfigure()` | [Virtual Source Rollback](Workflows.md#virtual-source-rollback)
[Virtual Source Enable](Workflows.md#virtual-source-enable) @@ -593,6 +594,68 @@ def linked_mount_specification(staged_source, repository): return MountSpecification([mount], ownership_spec) ``` +## Virtual Source Initialize + +Initializes a brand-new [empty VDB](Glossary.md#empty-vdb). As with all VDBs, this new dataset will have access to mounted Delphix Engine storage, but of course there will be no data there at first. + +The job of the plugin is to do whatever is necessary to set up a new dataset from scratch. For example, this might involve running a `CREATE DATABASE` command. This is an optional operation -- users will not be allowed to create empty VDBs for plugins that choose not to implement this operation. + +As with the `configure` operation, this `initialize` operation must return source config parameters that represent the new dataset. + +### Required / Optional +**Optional.** + +### Delphix Engine Operations + +* [Virtual Source Create Empty VDB](Workflows.md#virtual-source-create-empty-vdb) + +### Signature + +`def initialize(virtual_source, repository)` + +### Decorator + +`virtual.initialize()` + +### Arguments + +Argument | Type | Description +-------- | ---- | ----------- +virtual_source | [VirtualSource](Classes.md#virtualsource) | The source associated with this operation. +repository | [RepositoryDefinition](Schemas_and_Autogenerated_Classes.md#repositorydefinition-class) | The repository associated with this source. + +### Returns +[SourceConfigDefinition](Schemas_and_Autogenerated_Classes.md#sourceconfigdefinition-class) + +### Example + +```python +from dlpx.virtualization.platform import Plugin +from generated.defintions import SourceConfigDefinition + +plugin = Plugin() + +@plugin.virtual.initialize() +def initialize(virtual_source, repository): + source_config = SourceConfigDefinition(name="config_name") + return source_config +``` + +> The above command assumes a [SourceConfig Schema](Schemas_and_Autogenerated_Classes.md#sourceconfig-schema) defined as: + +```json +{ + "type": "object", + "required": ["name"], + "additionalProperties": false, + "properties": { + "name": { "type": "string" } + }, + "identityFields": ["name"], + "nameField": ["name"] +} +``` + ## Virtual Source Configure Configures the data in a particular snapshot to be usable on a target environment. For database data files, this may mean recovering from a crash consistent format or backup. For application files, this may mean reconfiguring XML files or rewriting hostnames and symlinks. diff --git a/docs/docs/References/Schemas.md b/docs/docs/References/Schemas.md index c2a4ad54..8411e39e 100644 --- a/docs/docs/References/Schemas.md +++ b/docs/docs/References/Schemas.md @@ -112,6 +112,9 @@ Here is a JSON object that conforms to the above schema: For much more detail on JSON schemas, including which keywords are available, what they mean, and where you can use them, see . +!!! info + Be careful when using the JSON schema keyword `default`. This keyword is commonly misunderstood. Specifically, it does not mean "If the user does not provide a value, then this default value will be auto-substituted instead", as you might expect. In fact, in JSON schemas, `default` has no semantic meaning at all! Currently, the only thing the Delphix Engine will use this for is to pre-fill widgets on the UI. + ### Delphix-specific Extensions to JSON Schema @@ -309,7 +312,7 @@ This will allow the plugin author to ask the user to select [environments](/Refe "type": "string", "format": "reference", "referenceType": "HOST_USER", - "matches”: "env" + "matches": "env" } } ``` @@ -337,7 +340,7 @@ The `referenceType` keyword is used to specify the [reference](#reference) type. "type": "string", "format": "reference", "referenceType": "HOST_USER", - "matches”: "env" + "matches": "env" } } ``` @@ -362,7 +365,7 @@ The `matches` keyword is used to map an [environment user](/References/Glossary. "type": "string", "format": "reference", "referenceType": "HOST_USER", - "matches”: "env" + "matches": "env" } } ``` diff --git a/docs/docs/References/Workflows.md b/docs/docs/References/Workflows.md index 8bf50999..d58ed815 100644 --- a/docs/docs/References/Workflows.md +++ b/docs/docs/References/Workflows.md @@ -32,6 +32,10 @@ ![Screenshot](images/VirtualSourceSnapshot.png) +## Virtual Source Create Empty VDB + +![Screenshot](images/VirtualSourceCreateEmpty.png) + ## Virtual Source Refresh ![Screenshot](images/VirtualSourceRefresh.png) diff --git a/docs/docs/References/html/VirtualSourceCreateEmpty.html b/docs/docs/References/html/VirtualSourceCreateEmpty.html new file mode 100644 index 00000000..0a039f91 --- /dev/null +++ b/docs/docs/References/html/VirtualSourceCreateEmpty.html @@ -0,0 +1,12 @@ + + + +Draw.io Diagram + + + + +
+ + + diff --git a/docs/docs/References/images/VirtualSourceCreateEmpty.png b/docs/docs/References/images/VirtualSourceCreateEmpty.png new file mode 100644 index 00000000..911b1072 Binary files /dev/null and b/docs/docs/References/images/VirtualSourceCreateEmpty.png differ diff --git a/dvp/src/main/python/dlpx/virtualization/VERSION b/dvp/src/main/python/dlpx/virtualization/VERSION index f1855a71..a8693131 100644 --- a/dvp/src/main/python/dlpx/virtualization/VERSION +++ b/dvp/src/main/python/dlpx/virtualization/VERSION @@ -1 +1 @@ -2.2.0.dev1 \ No newline at end of file +3.0.0.dev2 diff --git a/libs/requirements.txt b/libs/requirements.txt index 275195fe..a8a53e05 100644 --- a/libs/requirements.txt +++ b/libs/requirements.txt @@ -1,6 +1,7 @@ ./../common bump2version==0.5.11 contextlib2==0.6.0.post1 ; python_version < '3' +enum34==1.1.6 funcsigs==1.0.2 ; python_version < '3.3' importlib-metadata==1.3.0 ; python_version < '3.8' mock==3.0.5 diff --git a/libs/setup.py b/libs/setup.py index d3cff739..cc474b90 100644 --- a/libs/setup.py +++ b/libs/setup.py @@ -7,7 +7,7 @@ version = version_file.read().strip() install_requires = [ - "dvp-api == 1.3.0", + "dvp-api == 1.4.0.dev13", "dvp-common == {}".format(version) ] diff --git a/libs/src/main/python/dlpx/virtualization/libs/VERSION b/libs/src/main/python/dlpx/virtualization/libs/VERSION index f1855a71..a8693131 100644 --- a/libs/src/main/python/dlpx/virtualization/libs/VERSION +++ b/libs/src/main/python/dlpx/virtualization/libs/VERSION @@ -1 +1 @@ -2.2.0.dev1 \ No newline at end of file +3.0.0.dev2 diff --git a/libs/src/main/python/dlpx/virtualization/libs/libs.py b/libs/src/main/python/dlpx/virtualization/libs/libs.py index 2e0fb4bd..4e531d87 100644 --- a/libs/src/main/python/dlpx/virtualization/libs/libs.py +++ b/libs/src/main/python/dlpx/virtualization/libs/libs.py @@ -30,7 +30,10 @@ from dlpx.virtualization.libs.exceptions import (IncorrectArgumentTypeError, LibraryError, PluginScriptError) -from dlpx.virtualization.common._common_classes import RemoteConnection +from dlpx.virtualization.common._common_classes import (RemoteConnection, + PasswordCredentials, + KeyPairCredentials) +from google.protobuf.struct_pb2 import Struct import logging @@ -39,10 +42,12 @@ "run_bash", "run_sync", "run_powershell", - "run_expect" + "run_expect", + "retrieve_credentials" ] + def _handle_response(response): """This function handles callback responses. It proceeds differently based on what the response reported... @@ -410,3 +415,34 @@ def _log_request(message, log_level): response = internal_libs.log(log_request) _handle_response(response) + + +def retrieve_credentials(credentials_supplier): + """This is an internal wrapper around the Virtualization library's credentials retrieval API. + Given a supplier provided by Virtualization, retrieves the credentials from that supplier. + + Args: + credentials_supplier (dict): Properties that make up a supplier of credentials. + Return: + Subclass of Credentials retrieved from supplier. Either a PasswordCredentials or a KeyPairCredentials + from dlpx.virtualization.common._common_classes. + """ + from dlpx.virtualization._engine import libs as internal_libs + + if not isinstance(credentials_supplier, dict): + raise IncorrectArgumentTypeError('credentials_supplier', type(credentials_supplier), dict) + + credentials_request = libs_pb2.CredentialsRequest() + credentials_struct = Struct() + credentials_struct.update(credentials_supplier) + credentials_request.credentials_supplier.CopyFrom(credentials_struct) + + response = internal_libs.retrieve_credentials(credentials_request) + + credentials_result = _handle_response(response) + if credentials_result.password != "": + return PasswordCredentials(credentials_result.username, credentials_result.password) + return KeyPairCredentials( + credentials_result.username, + credentials_result.key_pair.private_key, + credentials_result.key_pair.public_key) diff --git a/libs/src/test/python/dlpx/virtualization/test_libs.py b/libs/src/test/python/dlpx/virtualization/test_libs.py index 83364932..7bb18307 100644 --- a/libs/src/test/python/dlpx/virtualization/test_libs.py +++ b/libs/src/test/python/dlpx/virtualization/test_libs.py @@ -9,6 +9,7 @@ from dlpx.virtualization import libs from dlpx.virtualization.libs.exceptions import ( IncorrectArgumentTypeError, LibraryError, PluginScriptError) +from google.protobuf import json_format class TestLibsRunBash: @@ -773,3 +774,91 @@ def test_run_expect_bad_variables(remote_connection): " type 'dict of basestring:basestring' if defined.") assert (err_info.value.message == message.format('int', 'str') or err_info.value.message == message.format('str', 'int')) + + +class TestLibsRetrieveCredentials: + @staticmethod + def test_retrieve_password_credentials(): + expected_retrieve_credentials_response = libs_pb2.CredentialsResponse() + expected_retrieve_credentials_response.return_value.username = 'some user' + expected_retrieve_credentials_response.return_value.password = 'some password' + + expected_credentials_supplier = {'some supplier property': 'some supplier value'} + + def mock_retrieve_credentials(actual_retrieve_credentials_request): + assert json_format.MessageToDict(actual_retrieve_credentials_request.credentials_supplier) == \ + expected_credentials_supplier + + return expected_retrieve_credentials_response + + with mock.patch('dlpx.virtualization._engine.libs.retrieve_credentials', + side_effect=mock_retrieve_credentials, create=True): + actual_retrieve_credentials_result = libs.retrieve_credentials(expected_credentials_supplier) + + expected = expected_retrieve_credentials_response.return_value + assert actual_retrieve_credentials_result.username == expected.username + assert actual_retrieve_credentials_result.password == expected.password + + @staticmethod + def test_retrieve_keypair_credentials(): + expected_retrieve_credentials_response = libs_pb2.CredentialsResponse() + expected_retrieve_credentials_response.return_value.username = 'some user' + expected_retrieve_credentials_response.return_value.key_pair.private_key = 'some private key' + expected_retrieve_credentials_response.return_value.key_pair.public_key = 'some public key' + + expected_credentials_supplier = {'some supplier property': 'some supplier value'} + + def mock_retrieve_credentials(actual_retrieve_credentials_request): + assert json_format.MessageToDict(actual_retrieve_credentials_request.credentials_supplier) == \ + expected_credentials_supplier + + return expected_retrieve_credentials_response + + with mock.patch('dlpx.virtualization._engine.libs.retrieve_credentials', + side_effect=mock_retrieve_credentials, create=True): + actual_retrieve_credentials_result = libs.retrieve_credentials(expected_credentials_supplier) + + expected = expected_retrieve_credentials_response.return_value + assert actual_retrieve_credentials_result.username == expected.username + assert actual_retrieve_credentials_result.private_key == expected.key_pair.private_key + assert actual_retrieve_credentials_result.public_key == expected.key_pair.public_key + + @staticmethod + def test_retrieve_credentials_with_actionable_error(): + expected_id = 15 + expected_message = 'Some message' + + response = libs_pb2.CredentialsResponse() + response.error.actionable_error.id = expected_id + response.error.actionable_error.message = expected_message + + with mock.patch('dlpx.virtualization._engine.libs.retrieve_credentials', + return_value=response, create=True): + with pytest.raises(LibraryError) as err_info: + libs.retrieve_credentials({'some supplier property': 'some supplier value'}) + + assert err_info.value._id == expected_id + assert err_info.value.message == expected_message + + @staticmethod + def test_retrieve_credentials_with_nonactionable_error(): + response = libs_pb2.CredentialsResponse() + na_error = libs_pb2.NonActionableLibraryError() + response.error.non_actionable_error.CopyFrom(na_error) + + with mock.patch('dlpx.virtualization._engine.libs.retrieve_credentials', + return_value=response, create=True): + with pytest.raises(SystemExit): + libs.retrieve_credentials({'some supplier property': 'some supplier value'}) + + @staticmethod + def test_retrieve_credentials_bad_supplier(): + # Set the supplier be an int instead of a dictionary. + credentials_supplier = 10 + + with pytest.raises(IncorrectArgumentTypeError) as err_info: + libs.retrieve_credentials(credentials_supplier) + + assert err_info.value.message == ( + "The function retrieve_credentials's argument 'credentials_supplier' was" + " type 'int' but should be of type 'dict'.") diff --git a/platform/setup.py b/platform/setup.py index b4a46018..1fd835e7 100644 --- a/platform/setup.py +++ b/platform/setup.py @@ -7,7 +7,7 @@ version = version_file.read().strip() install_requires = [ - "dvp-api == 1.3.0", + "dvp-api == 1.4.0.dev13", "dvp-common == {}".format(version), "enum34;python_version < '3.4'", ] diff --git a/platform/src/main/python/dlpx/virtualization/platform/VERSION b/platform/src/main/python/dlpx/virtualization/platform/VERSION index f1855a71..a8693131 100644 --- a/platform/src/main/python/dlpx/virtualization/platform/VERSION +++ b/platform/src/main/python/dlpx/virtualization/platform/VERSION @@ -1 +1 @@ -2.2.0.dev1 \ No newline at end of file +3.0.0.dev2 diff --git a/platform/src/main/python/dlpx/virtualization/platform/_linked.py b/platform/src/main/python/dlpx/virtualization/platform/_linked.py index d6e8c656..4e6c07ee 100644 --- a/platform/src/main/python/dlpx/virtualization/platform/_linked.py +++ b/platform/src/main/python/dlpx/virtualization/platform/_linked.py @@ -121,6 +121,7 @@ def _internal_direct_pre_snapshot(self, request): from generated.definitions import RepositoryDefinition from generated.definitions import LinkedSourceDefinition from generated.definitions import SourceConfigDefinition + from generated.definitions import SnapshotParametersDefinition # # While linked.pre_snapshot() is not a required operation, this should @@ -141,10 +142,21 @@ def _internal_direct_pre_snapshot(self, request): json.loads(request.repository.parameters.json)) source_config = SourceConfigDefinition.from_dict( json.loads(request.source_config.parameters.json)) + snap_params = json.loads(request.snapshot_parameters.parameters.json) + # + # The snapshot_parameters object should be set to None if the json from + # the protobuf is None to differentiate no snapshot parameters vs empty + # snapshot parameters. + # + snapshot_parameters = ( + None if snap_params is None else + SnapshotParametersDefinition.from_dict(snap_params)) - self.pre_snapshot_impl(direct_source=direct_source, - repository=repository, - source_config=source_config) + self.pre_snapshot_impl( + direct_source=direct_source, + repository=repository, + source_config=source_config, + optional_snapshot_parameters=snapshot_parameters) direct_pre_snapshot_response = platform_pb2.DirectPreSnapshotResponse() direct_pre_snapshot_response.return_value.CopyFrom( @@ -173,6 +185,7 @@ def _internal_direct_post_snapshot(self, request): from generated.definitions import LinkedSourceDefinition from generated.definitions import SourceConfigDefinition from generated.definitions import SnapshotDefinition + from generated.definitions import SnapshotParametersDefinition def to_protobuf(snapshot): parameters = common_pb2.PluginDefinedObject() @@ -196,10 +209,21 @@ def to_protobuf(snapshot): json.loads(request.repository.parameters.json)) source_config = SourceConfigDefinition.from_dict( json.loads(request.source_config.parameters.json)) + snap_params = json.loads(request.snapshot_parameters.parameters.json) + # + # The snapshot_parameters object should be set to None if the json from + # the protobuf is None to differentiate no snapshot parameters vs empty + # snapshot parameters. + # + snapshot_parameters = ( + None if snap_params is None else + SnapshotParametersDefinition.from_dict(snap_params)) - snapshot = self.post_snapshot_impl(direct_source=direct_source, - repository=repository, - source_config=source_config) + snapshot = self.post_snapshot_impl( + direct_source=direct_source, + repository=repository, + source_config=source_config, + optional_snapshot_parameters=snapshot_parameters) # Validate that this is a SnapshotDefinition object if not isinstance(snapshot, SnapshotDefinition): @@ -263,13 +287,21 @@ def _internal_staged_pre_snapshot(self, request): json.loads(request.repository.parameters.json)) source_config = SourceConfigDefinition.from_dict( json.loads(request.source_config.parameters.json)) - snapshot_parameters = SnapshotParametersDefinition.from_dict( - json.loads(request.snapshot_parameters.parameters.json)) + snap_params = json.loads(request.snapshot_parameters.parameters.json) + # + # The snapshot_parameters object should be set to None if the json from + # the protobuf is None to differentiate no snapshot parameters vs empty + # snapshot parameters. + # + snapshot_parameters = ( + None if snap_params is None else + SnapshotParametersDefinition.from_dict(snap_params)) - self.pre_snapshot_impl(staged_source=staged_source, - repository=repository, - source_config=source_config, - snapshot_parameters=snapshot_parameters) + self.pre_snapshot_impl( + staged_source=staged_source, + repository=repository, + source_config=source_config, + optional_snapshot_parameters=snapshot_parameters) response = platform_pb2.StagedPreSnapshotResponse() response.return_value.CopyFrom(platform_pb2.StagedPreSnapshotResult()) @@ -330,14 +362,21 @@ def to_protobuf(snapshot): json.loads(request.repository.parameters.json)) source_config = SourceConfigDefinition.from_dict( json.loads(request.source_config.parameters.json)) - snapshot_parameters = SnapshotParametersDefinition.from_dict( - json.loads(request.snapshot_parameters.parameters.json)) + snap_params = json.loads(request.snapshot_parameters.parameters.json) + # + # The snapshot_parameters object should be set to None if the json from + # the protobuf is None to differentiate no snapshot parameters vs empty + # snapshot parameters. + # + snapshot_parameters = ( + None if snap_params is None else + SnapshotParametersDefinition.from_dict(snap_params)) snapshot = self.post_snapshot_impl( staged_source=staged_source, repository=repository, source_config=source_config, - snapshot_parameters=snapshot_parameters) + optional_snapshot_parameters=snapshot_parameters) # Validate that this is a SnapshotDefinition object if not isinstance(snapshot, SnapshotDefinition): diff --git a/platform/src/main/python/dlpx/virtualization/platform/_virtual.py b/platform/src/main/python/dlpx/virtualization/platform/_virtual.py index 6c048c46..2b2318a9 100644 --- a/platform/src/main/python/dlpx/virtualization/platform/_virtual.py +++ b/platform/src/main/python/dlpx/virtualization/platform/_virtual.py @@ -626,15 +626,18 @@ def _internal_initialize(self, request): repository = RepositoryDefinition.from_dict( json.loads(request.repository.parameters.json)) - source_config = SourceConfigDefinition.from_dict( - json.loads(request.source_config.parameters.json)) - self.initialize_impl(repository=repository, - source_config=source_config, + config = self.initialize_impl(repository=repository, virtual_source=virtual_source) + + # Validate that this is a SourceConfigDefinition object. + if not isinstance(config, SourceConfigDefinition): + raise IncorrectReturnTypeError(Op.VIRTUAL_INITIALIZE, type(config), + SourceConfigDefinition) + initialize_response = platform_pb2.InitializeResponse() - initialize_response.return_value.CopyFrom( - platform_pb2.InitializeResult()) + initialize_response.return_value.source_config.parameters.json = ( + json.dumps(config.to_dict())) return initialize_response def _internal_mount_specification(self, request): diff --git a/platform/src/test/python/dlpx/virtualization/fake_generated_definitions.py b/platform/src/test/python/dlpx/virtualization/fake_generated_definitions.py index 481c04a5..90fbcef6 100644 --- a/platform/src/test/python/dlpx/virtualization/fake_generated_definitions.py +++ b/platform/src/test/python/dlpx/virtualization/fake_generated_definitions.py @@ -101,14 +101,6 @@ def to_dict(self): class SnapshotParametersDefinition(Model): - """ - The appdata snapshot parameter will eventually be customizable but for now - this just follows the old appdata parameter where the delphix user can decide - if resync is true or not. This will now go into pre and post snapshot - operations rather than the resync operation. The main point is customers will - set this to be true is that this means the operation is a "hard" resync and - that all data should be refreshed. - """ def __init__(self, resync): self.swagger_types = {'resync': bool} diff --git a/platform/src/test/python/dlpx/virtualization/test_plugin.py b/platform/src/test/python/dlpx/virtualization/test_plugin.py index 7a2060f8..8cfde728 100755 --- a/platform/src/test/python/dlpx/virtualization/test_plugin.py +++ b/platform/src/test/python/dlpx/virtualization/test_plugin.py @@ -686,25 +686,46 @@ def virtual_status_impl(virtual_source, repository, source_config): def test_virtual_initialize(my_plugin, virtual_source, repository, source_config): @my_plugin.virtual.initialize() - def virtual_initialize_impl(virtual_source, repository, source_config): + def virtual_initialize_impl(virtual_source, repository): TestPlugin.assert_plugin_args(virtual_source=virtual_source, - repository=repository, - source_config=source_config) - return + repository=repository) + return SourceConfigDefinition(repository.name) initialize_request = platform_pb2.InitializeRequest() TestPlugin.setup_request(request=initialize_request, virtual_source=virtual_source, - repository=repository, - source_config=source_config) + repository=repository) - expected_result = platform_pb2.InitializeResult() + expected_source_config = TEST_REPOSITORY_JSON initialize_response = my_plugin.virtual._internal_initialize( initialize_request) # Check that the response's oneof is set to return_value and not error assert initialize_response.WhichOneof('result') == 'return_value' - assert initialize_response.return_value == expected_result + actual_source_config = initialize_response.return_value.source_config + assert actual_source_config.parameters.json == TEST_REPOSITORY_JSON + + @staticmethod + def test_virtual_initialize_return_none(my_plugin, virtual_source, + repository, source_config): + @my_plugin.virtual.initialize() + def virtual_initialize_impl(virtual_source, repository): + TestPlugin.assert_plugin_args(virtual_source=virtual_source, + repository=repository) + # Will return none. + + initialize_request = platform_pb2.InitializeRequest() + TestPlugin.setup_request(request=initialize_request, + virtual_source=virtual_source, + repository=repository) + + with pytest.raises(IncorrectReturnTypeError) as err_info: + my_plugin.virtual._internal_initialize(initialize_request) + message = err_info.value.message + assert message == ( + "The returned object for the virtual.initialize() operation was" + " type 'NoneType' but should be of class 'dlpx.virtualization." + "fake_generated_definitions.SourceConfigDefinition'.") @staticmethod def test_virtual_mount_spec(my_plugin, virtual_source, repository): @@ -814,19 +835,55 @@ def source_config_discovery_impl(source_connection, repository): @staticmethod def test_direct_pre_snapshot(my_plugin, direct_source, repository, - source_config): + source_config, snapshot_parameters): @my_plugin.linked.pre_snapshot() - def mock_direct_pre_snapshot(direct_source, repository, source_config): - TestPlugin.assert_plugin_args(direct_source=direct_source, - repository=repository, - source_config=source_config) + def mock_direct_pre_snapshot(direct_source, repository, source_config, + optional_snapshot_parameters): + TestPlugin.assert_plugin_args( + direct_source=direct_source, + repository=repository, + source_config=source_config, + snapshot_parameters=optional_snapshot_parameters) return direct_pre_snapshot_request = platform_pb2.DirectPreSnapshotRequest() TestPlugin.setup_request(request=direct_pre_snapshot_request, direct_source=direct_source, repository=repository, - source_config=source_config) + source_config=source_config, + snapshot_parameters=snapshot_parameters) + + expected_result = platform_pb2.DirectPreSnapshotResult() + direct_pre_snapshot_response = ( + my_plugin.linked._internal_direct_pre_snapshot( + direct_pre_snapshot_request)) + + # Check that the response's oneof is set to return_value and not error + assert direct_pre_snapshot_response.WhichOneof( + 'result') == 'return_value' + assert direct_pre_snapshot_response.return_value == expected_result + + @staticmethod + def test_direct_pre_snapshot_null_snapparams(my_plugin, direct_source, + repository, source_config): + @my_plugin.linked.pre_snapshot() + def mock_direct_pre_snapshot(direct_source, repository, source_config, + optional_snapshot_parameters): + TestPlugin.assert_direct_source(direct_source) + TestPlugin.assert_repository(repository) + TestPlugin.assert_source_config(source_config) + assert not optional_snapshot_parameters + return + + snapshot_parameters = common_pb2.SnapshotParameters() + snapshot_parameters.parameters.json = 'null' + + direct_pre_snapshot_request = platform_pb2.DirectPreSnapshotRequest() + TestPlugin.setup_request(request=direct_pre_snapshot_request, + direct_source=direct_source, + repository=repository, + source_config=source_config, + snapshot_parameters=snapshot_parameters) expected_result = platform_pb2.DirectPreSnapshotResult() direct_pre_snapshot_response = ( @@ -840,20 +897,54 @@ def mock_direct_pre_snapshot(direct_source, repository, source_config): @staticmethod def test_direct_post_snapshot(my_plugin, direct_source, repository, - source_config): + source_config, snapshot_parameters): @my_plugin.linked.post_snapshot() - def direct_post_snapshot_impl(direct_source, repository, - source_config): - TestPlugin.assert_plugin_args(direct_source=direct_source, - repository=repository, - source_config=source_config) + def direct_post_snapshot_impl(direct_source, repository, source_config, + optional_snapshot_parameters): + TestPlugin.assert_plugin_args( + direct_source=direct_source, + repository=repository, + source_config=source_config, + snapshot_parameters=optional_snapshot_parameters) return SnapshotDefinition(TEST_SNAPSHOT) direct_post_snapshot_request = platform_pb2.DirectPostSnapshotRequest() - TestPlugin.setup_request(request=direct_post_snapshot_request, - direct_source=direct_source, - repository=repository, - source_config=source_config) + TestPlugin.setup_request( + request=direct_post_snapshot_request, + direct_source=direct_source, + repository=repository, + source_config=source_config, + snapshot_parameters=snapshot_parameters) + + direct_post_snapshot_response = ( + my_plugin.linked._internal_direct_post_snapshot( + direct_post_snapshot_request)) + expected_snapshot = TEST_SNAPSHOT_JSON + snapshot = direct_post_snapshot_response.return_value.snapshot + assert snapshot.parameters.json == expected_snapshot + + @staticmethod + def test_direct_post_snapshot_null_snapparams(my_plugin, direct_source, + repository, source_config): + @my_plugin.linked.post_snapshot() + def direct_post_snapshot_impl(direct_source, repository, source_config, + optional_snapshot_parameters): + TestPlugin.assert_direct_source(direct_source) + TestPlugin.assert_repository(repository) + TestPlugin.assert_source_config(source_config) + assert not optional_snapshot_parameters + return SnapshotDefinition(TEST_SNAPSHOT) + + snapshot_parameters = common_pb2.SnapshotParameters() + snapshot_parameters.parameters.json = 'null' + + direct_post_snapshot_request = platform_pb2.DirectPostSnapshotRequest() + TestPlugin.setup_request( + request=direct_post_snapshot_request, + direct_source=direct_source, + repository=repository, + source_config=source_config, + snapshot_parameters=snapshot_parameters) direct_post_snapshot_response = ( my_plugin.linked._internal_direct_post_snapshot( @@ -867,20 +958,52 @@ def test_staged_pre_snapshot(my_plugin, staged_source, repository, source_config, snapshot_parameters): @my_plugin.linked.pre_snapshot() def staged_pre_snapshot_impl(staged_source, repository, source_config, - snapshot_parameters): + optional_snapshot_parameters): TestPlugin.assert_plugin_args( staged_source=staged_source, repository=repository, source_config=source_config, - snapshot_parameters=snapshot_parameters) + snapshot_parameters=optional_snapshot_parameters) return staged_pre_snapshot_request = platform_pb2.StagedPreSnapshotRequest() - TestPlugin.setup_request(request=staged_pre_snapshot_request, - staged_source=staged_source, - repository=repository, - source_config=source_config, - snapshot_parameters=snapshot_parameters) + TestPlugin.setup_request( + request=staged_pre_snapshot_request, + staged_source=staged_source, + repository=repository, + source_config=source_config, + snapshot_parameters=snapshot_parameters) + + expected_result = platform_pb2.StagedPreSnapshotResult() + response = my_plugin.linked._internal_staged_pre_snapshot( + staged_pre_snapshot_request) + + # Check that the response's oneof is set to return_value and not error + assert response.WhichOneof('result') == 'return_value' + assert (response.return_value == expected_result) + + @staticmethod + def test_staged_pre_snapshot_null_snapparams(my_plugin, staged_source, + repository, source_config): + @my_plugin.linked.pre_snapshot() + def staged_pre_snapshot_impl(staged_source, repository, source_config, + optional_snapshot_parameters): + TestPlugin.assert_staged_source(staged_source) + TestPlugin.assert_repository(repository) + TestPlugin.assert_source_config(source_config) + assert not optional_snapshot_parameters + return + + snapshot_parameters = common_pb2.SnapshotParameters() + snapshot_parameters.parameters.json = 'null' + + staged_pre_snapshot_request = platform_pb2.StagedPreSnapshotRequest() + TestPlugin.setup_request( + request=staged_pre_snapshot_request, + staged_source=staged_source, + repository=repository, + source_config=source_config, + snapshot_parameters=snapshot_parameters) expected_result = platform_pb2.StagedPreSnapshotResult() response = my_plugin.linked._internal_staged_pre_snapshot( @@ -895,20 +1018,50 @@ def test_staged_post_snapshot(my_plugin, staged_source, repository, source_config, snapshot_parameters): @my_plugin.linked.post_snapshot() def staged_post_snapshot_impl(staged_source, repository, source_config, - snapshot_parameters): + optional_snapshot_parameters): TestPlugin.assert_plugin_args( staged_source=staged_source, repository=repository, source_config=source_config, - snapshot_parameters=snapshot_parameters) + snapshot_parameters=optional_snapshot_parameters) return SnapshotDefinition(TEST_SNAPSHOT) staged_post_snapshot_request = platform_pb2.StagedPostSnapshotRequest() - TestPlugin.setup_request(request=staged_post_snapshot_request, - staged_source=staged_source, - repository=repository, - source_config=source_config, - snapshot_parameters=snapshot_parameters) + TestPlugin.setup_request( + request=staged_post_snapshot_request, + staged_source=staged_source, + repository=repository, + source_config=source_config, + snapshot_parameters=snapshot_parameters) + + response = my_plugin.linked._internal_staged_post_snapshot( + staged_post_snapshot_request) + expected = TEST_SNAPSHOT_JSON + + assert response.return_value.snapshot.parameters.json == expected + + @staticmethod + def test_staged_post_snapshot_null_snapparams(my_plugin, staged_source, + repository, source_config): + @my_plugin.linked.post_snapshot() + def staged_post_snapshot_impl(staged_source, repository, source_config, + optional_snapshot_parameters): + TestPlugin.assert_staged_source(staged_source) + TestPlugin.assert_repository(repository) + TestPlugin.assert_source_config(source_config) + assert not optional_snapshot_parameters + return SnapshotDefinition(TEST_SNAPSHOT) + + snapshot_parameters = common_pb2.SnapshotParameters() + snapshot_parameters.parameters.json = 'null' + + staged_post_snapshot_request = platform_pb2.StagedPostSnapshotRequest() + TestPlugin.setup_request( + request=staged_post_snapshot_request, + staged_source=staged_source, + repository=repository, + source_config=source_config, + snapshot_parameters=snapshot_parameters) response = my_plugin.linked._internal_staged_post_snapshot( staged_post_snapshot_request) diff --git a/tools/src/main/python/dlpx/virtualization/_internal/VERSION b/tools/src/main/python/dlpx/virtualization/_internal/VERSION index f1855a71..a8693131 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/VERSION +++ b/tools/src/main/python/dlpx/virtualization/_internal/VERSION @@ -1 +1 @@ -2.2.0.dev1 \ No newline at end of file +3.0.0.dev2 diff --git a/tools/src/main/python/dlpx/virtualization/_internal/codegen.py b/tools/src/main/python/dlpx/virtualization/_internal/codegen.py index 9eda39b7..30b1529c 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/codegen.py +++ b/tools/src/main/python/dlpx/virtualization/_internal/codegen.py @@ -26,24 +26,6 @@ 'definitions': {} } -# -# The default snapshot params we currently support. This is so that in the -# future when we want to support plugin author defined schemas for snapshot -# params the upgrade case will be relatively simple. -# - -SNAPSHOT_PARAMS_JSON = { - 'snapshotParametersDefinition': { - 'type': 'object', - 'additionalProperties': False, - 'properties': { - 'resync': { - 'type': 'boolean' - } - } - } -} - SWAGGER_FILE_NAME = 'swagger.json' CODEGEN_PACKAGE = 'generated' CODEGEN_MODULE = 'definitions' @@ -108,12 +90,37 @@ def generate_python(name, source_dir, plugin_config_dir, schema_content): _copy_generated_to_dir(output_dir, source_dir) +# +# Makes all references to platform json schema definitions "opaque" by replacing +# them with plain object types. +# +def _make_url_refs_opaque(json): + if isinstance(json, dict): + for key in json: + if key == '$ref' and isinstance(json[key], basestring)\ + and json[key].startswith('https://delphix.com/platform/api#'): + json.pop(key) + json['type'] = 'object' + else: + _make_url_refs_opaque(json[key]) + elif isinstance(json, list): + for element in json: + _make_url_refs_opaque(element) + + def _write_swagger_file(name, schema_dict, output_dir): swagger_json = copy.deepcopy(SWAGGER_JSON_FORMAT) swagger_json['info']['title'] = name swagger_json['definitions'] = copy.deepcopy(schema_dict) - # Add in the snapshot param definition - swagger_json['definitions'].update(SNAPSHOT_PARAMS_JSON) + # + # When resolving URL $refs in JSON schemas, Swagger tries to actually access those + # URLs. But URL $refs are only meant to be identifiers. Also, specifically, plugin + # developers shouldn't have to manipulate credentials objects, which are passed to + # plugins by Virtualization and only intended for passing in call-backs to + # Virtualization. Thus, we replace $refs for plugin API objects with generic, opaque + # objects. + # + _make_url_refs_opaque(swagger_json['definitions']) swagger_file = os.path.join(output_dir, SWAGGER_FILE_NAME) logger.info('Writing swagger file to {}'.format(swagger_file)) diff --git a/tools/src/main/python/dlpx/virtualization/_internal/commands/build.py b/tools/src/main/python/dlpx/virtualization/_internal/commands/build.py index e56bc7f5..b0dbdcc6 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/commands/build.py +++ b/tools/src/main/python/dlpx/virtualization/_internal/commands/build.py @@ -23,6 +23,7 @@ DISCOVERY_DEFINITION_TYPE = 'PluginDiscoveryDefinition' STAGED_LINKED_SOURCE_TYPE = 'PluginLinkedStagedSourceDefinition' DIRECT_LINKED_SOURCE_TYPE = 'PluginLinkedDirectSourceDefinition' +SNAPSHOT_PARAMETERS_DEFINITION_TYPE = 'PluginSnapshotParametersDefinition' BUILD_DIR_NAME = 'build' @@ -207,6 +208,10 @@ def prepare_upload_artifact(plugin_config_content, src_dir, schemas, manifest): prepare_discovery_definition(plugin_config_content, schemas), 'snapshotSchema': schemas['snapshotDefinition'], + 'snapshotParametersDefinition': { + 'type': SNAPSHOT_PARAMETERS_DEFINITION_TYPE, + 'schema': schemas['snapshotParametersDefinition'] + }, 'manifest': manifest } diff --git a/tools/src/main/python/dlpx/virtualization/_internal/commands/plugin_template/direct_operations.py.template b/tools/src/main/python/dlpx/virtualization/_internal/commands/plugin_template/direct_operations.py.template index 88b5a416..164e512a 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/commands/plugin_template/direct_operations.py.template +++ b/tools/src/main/python/dlpx/virtualization/_internal/commands/plugin_template/direct_operations.py.template @@ -1,3 +1,4 @@ @plugin.linked.post_snapshot() -def linked_post_snapshot(direct_source, repository, source_config): +def linked_post_snapshot(direct_source, repository, source_config, + optional_snapshot_parameters): return SnapshotDefinition() \ No newline at end of file diff --git a/tools/src/main/python/dlpx/virtualization/_internal/commands/plugin_template/schema_template.json b/tools/src/main/python/dlpx/virtualization/_internal/commands/plugin_template/schema_template.json index 261d2e15..8dd02073 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/commands/plugin_template/schema_template.json +++ b/tools/src/main/python/dlpx/virtualization/_internal/commands/plugin_template/schema_template.json @@ -31,5 +31,10 @@ "type" : "object", "additionalProperties" : false, "properties" : {} + }, + "snapshotParametersDefinition": { + "type" : "object", + "additionalProperties" : false, + "properties" : {} } } diff --git a/tools/src/main/python/dlpx/virtualization/_internal/commands/plugin_template/staged_operations.py.template b/tools/src/main/python/dlpx/virtualization/_internal/commands/plugin_template/staged_operations.py.template index f23cb76f..d5879ec4 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/commands/plugin_template/staged_operations.py.template +++ b/tools/src/main/python/dlpx/virtualization/_internal/commands/plugin_template/staged_operations.py.template @@ -2,7 +2,7 @@ def linked_post_snapshot(staged_source, repository, source_config, - snapshot_parameters): + optional_snapshot_parameters): return SnapshotDefinition() diff --git a/tools/src/main/python/dlpx/virtualization/_internal/settings.cfg b/tools/src/main/python/dlpx/virtualization/_internal/settings.cfg index c12e9e3b..93477cdc 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/settings.cfg +++ b/tools/src/main/python/dlpx/virtualization/_internal/settings.cfg @@ -20,7 +20,7 @@ # versions in those packages until they are shipped out of band. # [General] -engine_api_version = 1.11.3 +engine_api_version = 1.12.0 distribution_name = dvp-tools package_author = Delphix namespace_package = dlpx diff --git a/tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_importer.yaml b/tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_importer.yaml index bb95c585..d769cd62 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_importer.yaml +++ b/tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_importer.yaml @@ -13,12 +13,12 @@ EXPECTED_STAGED_ARGS_BY_OP: - staged_source - repository - source_config - - snapshot_parameters + - optional_snapshot_parameters post_snapshot_impl: - staged_source - repository - source_config - - snapshot_parameters + - optional_snapshot_parameters start_staging_impl: - staged_source - repository @@ -75,7 +75,6 @@ EXPECTED_STAGED_ARGS_BY_OP: initialize_impl: - virtual_source - repository - - source_config mount_specification_impl: - virtual_source - repository @@ -93,10 +92,12 @@ EXPECTED_DIRECT_ARGS_BY_OP: - direct_source - repository - source_config + - optional_snapshot_parameters post_snapshot_impl: - direct_source - repository - source_config + - optional_snapshot_parameters VirtualOperations: configure_impl: - virtual_source @@ -134,7 +135,6 @@ EXPECTED_DIRECT_ARGS_BY_OP: initialize_impl: - virtual_source - repository - - source_config mount_specification_impl: - virtual_source - repository diff --git a/tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_schema.json b/tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_schema.json index 2d29ee37..b8be1836 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_schema.json +++ b/tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_schema.json @@ -179,7 +179,8 @@ "sourceConfigDefinition", "virtualSourceDefinition", "linkedSourceDefinition", - "snapshotDefinition" + "snapshotDefinition", + "snapshotParametersDefinition" ], "properties": { "repositoryDefinition": { @@ -210,6 +211,9 @@ }, "snapshotDefinition": { "$ref": "#/definitions/jsonSchema" + }, + "snapshotParametersDefinition": { + "$ref": "#/definitions/jsonSchema" } }, "additionalProperties": false diff --git a/tools/src/test/python/dlpx/virtualization/_internal/commands/test_build.py b/tools/src/test/python/dlpx/virtualization/_internal/commands/test_build.py index a207b31e..351d627a 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/commands/test_build.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/commands/test_build.py @@ -8,6 +8,7 @@ import yaml from dlpx.virtualization._internal import const, exceptions from dlpx.virtualization._internal.commands import build +from dlpx.virtualization._internal.commands import initialize as init from dlpx.virtualization._internal.plugin_importer import PluginImporter import mock @@ -48,6 +49,9 @@ def test_build_success(mock_relative_path, mock_install_deps, gen_py.plugin_content_dir, gen_py.schema_dict) mock_plugin_manifest.assert_called() + mock_install_deps.assert_called() + mock_relative_path.assert_called() + # After running build this file should now exist. assert os.path.exists(artifact_file) @@ -56,6 +60,31 @@ def test_build_success(mock_relative_path, mock_install_deps, assert content == artifact_content + @staticmethod + @pytest.mark.parametrize('ingestion_strategy', + [const.DIRECT_TYPE, const.STAGED_TYPE]) + @pytest.mark.parametrize('host_type', + [const.UNIX_HOST_TYPE, const.WINDOWS_HOST_TYPE]) + @mock.patch( + 'dlpx.virtualization._internal.plugin_dependency_util.install_deps') + @mock.patch('os.path.isabs', return_value=False) + def test_build_success_from_init(mock_relative_path, mock_install_deps, + tmpdir, ingestion_strategy, host_type, + plugin_name, artifact_file): + # Initialize an empty directory. + init.init(tmpdir.strpath, ingestion_strategy, plugin_name, host_type) + plugin_config_file = os.path.join( + tmpdir.strpath, init.DEFAULT_PLUGIN_CONFIG_FILE) + # Before running build assert that the artifact file does not exist. + assert not os.path.exists(artifact_file) + + build.build(plugin_config_file, artifact_file, False, False) + + mock_relative_path.assert_called() + mock_install_deps.assert_called() + + assert os.path.exists(artifact_file) + @staticmethod @pytest.mark.parametrize('artifact_filename', ['somefile.json']) @mock.patch.object(PluginImporter, @@ -284,13 +313,13 @@ def test_prepare_discovery_definition(plugin_config_content, assert actual_discovery_definition == discovery_definition @staticmethod - def test_prepare_upload_artifact_success(basic_artifact_content, + def test_prepare_upload_artifact_success(artifact_content, plugin_config_content, src_dir, schema_content): upload_artifact = build.prepare_upload_artifact( plugin_config_content, src_dir, schema_content, {}) - assert upload_artifact == basic_artifact_content + assert upload_artifact == artifact_content @staticmethod def test_generate_upload_artifact_success(tmpdir, artifact_content): @@ -697,3 +726,81 @@ def test_minimum_lua_version_parameter(plugin_config_content, src_dir, upload_artifact = build.prepare_upload_artifact( plugin_config_content, src_dir, schema_content, {}) assert expected == upload_artifact.get('minimumLuaVersion') + + @staticmethod + @pytest.mark.parametrize('build_number', ['1.0.1']) + def test_build_change_and_build_again(plugin_config_content, src_dir, + schema_content): + upload_artifact = build.prepare_upload_artifact( + plugin_config_content, src_dir, schema_content, {}) + assert plugin_config_content['buildNumber'] == upload_artifact['buildNumber'] + changed_build_number = '7.2.12' + changed_host_type = ['WINDOWS'] + plugin_config_content['buildNumber'] = changed_build_number + plugin_config_content['hostTypes'] = changed_host_type + upload_artifact_2 = build.prepare_upload_artifact( + plugin_config_content, src_dir, schema_content, {}) + assert changed_build_number == upload_artifact_2.get('buildNumber') + assert changed_host_type == upload_artifact_2.get('hostTypes') + + @staticmethod + @pytest.mark.parametrize('repository_definition', + [{ + 'type': 'object', + 'properties': { + 'name': { + 'type': 'badDataType' + } + }, + 'nameField': 'name', + 'identityFields': ['name'] + }]) + @mock.patch('dlpx.virtualization._internal.codegen.generate_python') + def test_bad_data_type_in_schema(mock_generate_python, + plugin_config_file, + artifact_file): + with pytest.raises(exceptions.UserError) as err_info: + build.build(plugin_config_file, artifact_file, False, False) + + message = err_info.value.message + exp_error = "'badDataType' is not valid under any of the given schemas" + assert exp_error in message + + assert not mock_generate_python.called + + @staticmethod + @pytest.mark.parametrize('host_types', ['']) + @mock.patch('dlpx.virtualization._internal.codegen.generate_python') + def test_empty_host_type(mock_generate_python, plugin_config_file, + artifact_file): + with pytest.raises(exceptions.UserError) as err_info: + build.build(plugin_config_file, artifact_file, False, False) + + message = err_info.value.message + assert "Validation failed" in message + assert not mock_generate_python.called + + @staticmethod + @pytest.mark.parametrize('plugin_name', ['']) + @mock.patch('dlpx.virtualization._internal.codegen.generate_python') + def test_empty_plugin_name(mock_generate_python, plugin_config_file, + artifact_file): + with pytest.raises(exceptions.UserError) as err_info: + build.build(plugin_config_file, artifact_file, False, False) + + message = err_info.value.message + assert "Validation failed" in message + assert not mock_generate_python.called + + @staticmethod + @mock.patch('os.path.isabs', return_value=False) + def test_non_existing_entry_file(mock_relative_path, plugin_config_file, + plugin_config_content, artifact_file): + entry_module, _ = plugin_config_content['entryPoint'].split(':') + + with pytest.raises(exceptions.UserError) as err_info: + build.build(plugin_config_file, artifact_file, False, False) + + message = err_info.value.message + exp_message = "No module named {module}".format(module=entry_module) + assert exp_message in message diff --git a/tools/src/test/python/dlpx/virtualization/_internal/commands/test_codegen.py b/tools/src/test/python/dlpx/virtualization/_internal/commands/test_codegen.py index 383e9189..ca6e3f84 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/commands/test_codegen.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/commands/test_codegen.py @@ -175,8 +175,7 @@ def test_get_build_dir_fail(): " Error message: No such file or directory") @staticmethod - def test_write_swagger_file(tmpdir, schema_content, - swagger_schema_content): + def test_write_swagger_file(tmpdir, schema_content): name = 'test' expected_file = tmpdir.join(codegen.SWAGGER_FILE_NAME).strpath codegen._write_swagger_file(name, schema_content, tmpdir.strpath) @@ -186,7 +185,26 @@ def test_write_swagger_file(tmpdir, schema_content, with open(expected_file, 'rb') as f: content = json.load(f) - assert content['definitions'] == swagger_schema_content + assert content['definitions'] == schema_content + assert content['info']['title'] == name + + @staticmethod + def test_write_swagger_file_with_delphix_refs( + tmpdir, schema_content, linked_source_definition_with_refs, + linked_source_definition_with_opaque_refs): + name = 'test' + expected_file = tmpdir.join(codegen.SWAGGER_FILE_NAME).strpath + schema_content['linkedSourceDefinition'] = linked_source_definition_with_refs + codegen._write_swagger_file(name, schema_content, tmpdir.strpath) + assert os.path.exists(expected_file) + assert os.path.isfile(expected_file) + + with open(expected_file, 'rb') as f: + content = json.load(f) + + schema_content['linkedSourceDefinition'] = \ + linked_source_definition_with_opaque_refs + assert content['definitions'] == schema_content assert content['info']['title'] == name @staticmethod diff --git a/tools/src/test/python/dlpx/virtualization/_internal/commands/test_initialize.py b/tools/src/test/python/dlpx/virtualization/_internal/commands/test_initialize.py index 5486eaa6..62fa9dfe 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/commands/test_initialize.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/commands/test_initialize.py @@ -114,6 +114,20 @@ def test_init(tmpdir, ingestion_strategy, host_type, schema_template, assert contents == format_entry_point_template( config['id'], ingestion_strategy, host_type) + @staticmethod + def test_init_with_relative_path(tmpdir): + os.chdir(tmpdir.strpath) + init.init(".", const.DIRECT_TYPE, "", const.UNIX_HOST_TYPE) + + result = plugin_util.validate_plugin_config_file( + os.path.join(tmpdir.strpath, init.DEFAULT_PLUGIN_CONFIG_FILE), + True) + + config = result.plugin_config_content + + # Validate that the plugin name is equal to plugin id + assert config['name'] == config['id'] + @staticmethod def test_init_without_plugin_name(tmpdir): init.init(tmpdir.strpath, const.DIRECT_TYPE, "", const.UNIX_HOST_TYPE) diff --git a/tools/src/test/python/dlpx/virtualization/_internal/conftest.py b/tools/src/test/python/dlpx/virtualization/_internal/conftest.py index 0f95743c..b5bff3c2 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/conftest.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/conftest.py @@ -37,7 +37,8 @@ def plugin_config_file(tmpdir, plugin_config_filename, plugin_config_content): default_flow_style=False) f = tmpdir.join(plugin_config_filename) - f.write(plugin_config_content) + if plugin_config_content: + f.write(plugin_config_content) return f.strpath @@ -453,7 +454,8 @@ def plugin_manifest(upgrade_operation): @pytest.fixture def schema_content(repository_definition, source_config_definition, virtual_source_definition, linked_source_definition, - snapshot_definition, additional_definition): + snapshot_definition, snapshot_parameters_definition, + additional_definition): schema = {} @@ -472,19 +474,12 @@ def schema_content(repository_definition, source_config_definition, if snapshot_definition: schema['snapshotDefinition'] = snapshot_definition - if additional_definition: - schema['additionalDefinition'] = additional_definition - - return schema - - -@pytest.fixture -def swagger_schema_content(schema_content, snapshot_parameters_definition): - - schema = schema_content if snapshot_parameters_definition: schema['snapshotParametersDefinition'] = snapshot_parameters_definition + if additional_definition: + schema['additionalDefinition'] = additional_definition + return schema @@ -525,7 +520,7 @@ def source_config_definition(): def virtual_source_definition(): return { 'type': 'object', - 'additionalProperties': False, + 'additionalProperties': True, 'properties': { 'path': { 'type': 'string' @@ -539,6 +534,66 @@ def linked_source_definition(): return {'type': 'object', 'additionalProperties': False, 'properties': {}} +@pytest.fixture +def linked_source_definition_with_refs(): + return { + 'type': 'object', + 'additionalProperties': True, + 'properties': { + 'path': { + 'type': 'string' + }, + 'credentials': { + '$ref': 'https://delphix.com/platform/api#credentialsSupplier' + }, + 'credentialsContainer': { + 'type': 'object', + 'properties': { + 'nestedCredentials': { + '$ref': 'https://delphix.com/platform/api#credentialsSupplier' + }, + } + }, + 'credentialsArray': { + 'type': 'array', + 'items': [ + {'$ref': 'https://delphix.com/platform/api#credentialsSupplier'} + ] + } + } + } + + +@pytest.fixture +def linked_source_definition_with_opaque_refs(): + return { + 'type': 'object', + 'additionalProperties': True, + 'properties': { + 'path': { + 'type': 'string' + }, + 'credentials': { + 'type': 'object' + }, + 'credentialsContainer': { + 'type': 'object', + 'properties': { + 'nestedCredentials': { + 'type': 'object' + }, + } + }, + 'credentialsArray': { + 'type': 'array', + 'items': [ + {'type': 'object'} + ] + } + } + } + + @pytest.fixture def snapshot_definition(): return { @@ -570,54 +625,10 @@ def additional_definition(): return None -@pytest.fixture -def basic_artifact_content(engine_api, virtual_source_definition, - linked_source_definition, discovery_definition, - snapshot_definition): - artifact = { - 'type': 'Plugin', - 'name': '16bef554-9470-11e9-b2e3-8c8590d4a42c', - 'prettyName': 'python_vfiles', - 'externalVersion': '2.0.0', - 'defaultLocale': 'en-us', - 'language': 'PYTHON27', - 'hostTypes': ['UNIX'], - 'entryPoint': 'python_vfiles:vfiles', - 'buildApi': package_util.get_build_api_version(), - 'engineApi': engine_api, - 'rootSquashEnabled': True, - 'buildNumber': '2', - 'luaName': 'lua-toolkit-1', - 'extendedStartStopHooks': False, - 'minimumLuaVersion': '2.3', - 'sourceCode': 'UEsFBgAAAAAAAAAAAAAAAAAAAAAAAA==', - 'manifest': {} - } - if virtual_source_definition: - artifact['virtualSourceDefinition'] = { - 'type': 'PluginVirtualSourceDefinition', - 'parameters': virtual_source_definition - } - - if linked_source_definition: - artifact['linkedSourceDefinition'] = { - 'type': 'PluginLinkedDirectSourceDefinition', - 'parameters': linked_source_definition - } - - if discovery_definition: - artifact['discoveryDefinition'] = discovery_definition - - if snapshot_definition: - artifact['snapshotSchema'] = snapshot_definition - - return artifact - - @pytest.fixture def artifact_content(engine_api, virtual_source_definition, linked_source_definition, discovery_definition, - snapshot_definition): + snapshot_definition, snapshot_parameters_definition): """ This fixture creates base artifact that was generated from build and used in upload. If any fields besides engine_api needs to be changed, @@ -664,12 +675,22 @@ def artifact_content(engine_api, virtual_source_definition, if snapshot_definition: artifact['snapshotSchema'] = snapshot_definition + if snapshot_parameters_definition: + artifact['snapshotParametersDefinition'] = ( + snapshot_parameters_definition) + + if snapshot_parameters_definition: + artifact['snapshotParametersDefinition'] = { + 'type': 'PluginSnapshotParametersDefinition', + 'schema': snapshot_parameters_definition, + } + return artifact @pytest.fixture def engine_api(): - return {'type': 'APIVersion', 'major': 1, 'minor': 11, 'micro': 3} + return {'type': 'APIVersion', 'major': 1, 'minor': 12, 'micro': 0} @pytest.fixture diff --git a/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/bad_syntax.py b/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/bad_syntax.py new file mode 100644 index 00000000..c0024806 --- /dev/null +++ b/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/bad_syntax.py @@ -0,0 +1,17 @@ +# +# Copyright (c) 2020 by Delphix. All rights reserved. +# +# flake8: noqa +from dlpx.virtualization.platform import Plugin + +plugin = Plugin() + + +@plugin.discovery.repository() +def repository_discovery(source_connection) + return None + + +@plugin.discovery.source_config() +def source_config_discovery(source_connection, repository) + return None diff --git a/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/import_error.py b/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/import_error.py new file mode 100644 index 00000000..6ec26e2a --- /dev/null +++ b/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/import_error.py @@ -0,0 +1,17 @@ +# +# Copyright (c) 2020 by Delphix. All rights reserved. +# +# flake8: noqa +from dlpxxx.virtualization.platform import Plugin + +plugin = Plugin() + + +@plugin.discovery.repository() +def repository_discovery(source_connection): + return None + + +@plugin.discovery.source_config() +def source_config_discovery(source_connection, repository): + return None diff --git a/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/multiple_warnings.py b/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/multiple_warnings.py index c0a031bc..56f4d299 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/multiple_warnings.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/multiple_warnings.py @@ -18,12 +18,14 @@ def source_config_discovery(source_connection, repository): @vfiles.linked.pre_snapshot() -def direct_pre_snapshot(direct_source, repository, source_config): +def direct_pre_snapshot(direct_source, repository, source_config, + optional_snapshot_parameters): return @vfiles.linked.post_snapshot() -def direct_post_snapshot(direct_source, repository, source_config): +def direct_post_snapshot(direct_source, repository, source_config, + optional_snapshot_parameters): return None @@ -40,12 +42,12 @@ def mount_specification(repository, virtual_source): @vfiles.virtual.post_snapshot() -def postSnapshot(repository, source_config, virtual_source): +def post_snapshot(repository, source_config, virtual_source): return None @vfiles.virtual.pre_snapshot() -def preSnapshot(repository, source_config, virtual_source): +def pre_snapshot(repository, source_config, virtual_source): pass diff --git a/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/successful.py b/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/successful.py index 24d57a34..cb355933 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/successful.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/successful.py @@ -18,12 +18,14 @@ def source_config_discovery(source_connection, repository): @direct.linked.pre_snapshot() -def direct_pre_snapshot(direct_source, repository, source_config): +def direct_pre_snapshot(direct_source, repository, source_config, + optional_snapshot_parameters): return @direct.linked.post_snapshot() -def direct_post_snapshot(direct_source, repository, source_config): +def direct_post_snapshot(direct_source, repository, source_config, + optional_snapshot_parameters): return None @@ -33,6 +35,11 @@ def configure(virtual_source, repository, snapshot): name = "VDB mounted to " + path return None +@direct.virtual.initialize() +def initialize(virtual_source, repository): + path = virtual_source.parameters.path + name = "VDB mounted to " + path + return None @direct.virtual.mount_specification() def mount_specification(repository, virtual_source): @@ -40,12 +47,12 @@ def mount_specification(repository, virtual_source): @direct.virtual.post_snapshot() -def postSnapshot(repository, source_config, virtual_source): +def post_snapshot(repository, source_config, virtual_source): return None @direct.virtual.pre_snapshot() -def preSnapshot(repository, source_config, virtual_source): +def pre_snapshot(repository, source_config, virtual_source): pass diff --git a/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/undefined_name.py b/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/undefined_name.py new file mode 100644 index 00000000..db4ec97a --- /dev/null +++ b/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/undefined_name.py @@ -0,0 +1,17 @@ +# +# Copyright (c) 2020 by Delphix. All rights reserved. +# +# flake8: noqa +from dlpx.virtualization.platform import Plugin + +plugin = Plugin() + + +@directplugin.discovery.repository() +def repository_discovery(source_connection): + return None + + +@plugin.discovery.source_config() +def source_config_discovery(source_connection, repository): + return None diff --git a/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/upgrade_warnings.py b/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/upgrade_warnings.py index 68ecd5b2..0046fe49 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/upgrade_warnings.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/upgrade_warnings.py @@ -18,12 +18,14 @@ def source_config_discovery(source_connection, repository): @direct.linked.pre_snapshot() -def direct_pre_snapshot(direct_source, repository, source_config): +def direct_pre_snapshot(direct_source, repository, source_config, + optional_snapshot_parameters): return @direct.linked.post_snapshot() -def direct_post_snapshot(direct_source, repository, source_config): +def direct_post_snapshot(direct_source, repository, source_config, + optional_snapshot_parameters): return None diff --git a/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/staged/multiple_warnings.py b/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/staged/multiple_warnings.py index 094c1dde..5f3a6c81 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/staged/multiple_warnings.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/staged/multiple_warnings.py @@ -32,13 +32,13 @@ def staged_mount_specification(staged_source, repository): @staged.linked.pre_snapshot() def staged_pre_snapshot(repository, source_config, staged_source, - snapshot_parameters): + optional_snapshot_parameters): pass @staged.linked.post_snapshot() def staged_post_snapshot(repository, source_config, staged_source, - snapshot_parameters): + optional_snapshot_parameters): return None diff --git a/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/staged/successful.py b/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/staged/successful.py index 31ae1151..bea862cf 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/staged/successful.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/staged/successful.py @@ -31,13 +31,13 @@ def staged_mount_specification(staged_source, repository): @staged.linked.pre_snapshot() def staged_pre_snapshot(repository, source_config, staged_source, - snapshot_parameters): + optional_snapshot_parameters): pass @staged.linked.post_snapshot() def staged_post_snapshot(repository, source_config, staged_source, - snapshot_parameters): + optional_snapshot_parameters): return None @@ -65,6 +65,9 @@ def staged_worker(repository, source_config, staged_source): def configure(virtual_source, repository, snapshot): return None +@staged.virtual.initialize() +def initialize(virtual_source, repository): + return None @staged.virtual.mount_specification() def mount_specification(virtual_source, repository): diff --git a/tools/src/test/python/dlpx/virtualization/_internal/test_cli.py b/tools/src/test/python/dlpx/virtualization/_internal/test_cli.py index f9a98dc7..9ef87f31 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/test_cli.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/test_cli.py @@ -163,6 +163,39 @@ def test_invalid_ingestion_strategy(plugin_name): assert result.exit_code != 0 + @staticmethod + def test_blank_ingestion_strategy(plugin_name): + runner = click_testing.CliRunner() + + result = runner.invoke( + cli.delphix_sdk, + ['init', '-n', plugin_name, '-s', '']) + + assert result.exit_code != 0 + assert "invalid choice" in result.output + + @staticmethod + def test_non_existent_root_dir(plugin_name): + runner = click_testing.CliRunner() + + result = runner.invoke( + cli.delphix_sdk, + ['init', '-n', plugin_name, '-r', '/file/does/not/exist']) + + assert result.exit_code != 0 + assert "'/file/does/not/exist' does not exist" in result.output + + @staticmethod + def test_empty_root_dir(plugin_name): + runner = click_testing.CliRunner() + + result = runner.invoke( + cli.delphix_sdk, + ['init', '-n', plugin_name, '-r', '']) + + assert result.exit_code != 0 + assert "Invalid value for '-r'" in result.output + @staticmethod def test_name_required(): runner = click_testing.CliRunner() @@ -385,6 +418,35 @@ def test_with_dev_fail(mock_build, plugin_config_file, artifact_file, assert result.exit_code == 2 assert not mock_build.called, 'build should not have been called' + @staticmethod + @pytest.mark.parametrize('plugin_config_file', + ['']) + def test_empty_config_file(plugin_config_file): + runner = click_testing.CliRunner() + result = runner.invoke(cli.delphix_sdk, + ['build', '-c', plugin_config_file]) + + assert result.exit_code == 2 + expected_error_strings = ("Error: Invalid value for '-c'", + "is a directory") + for expected_error in expected_error_strings: + assert expected_error in result.output + + @staticmethod + @pytest.mark.parametrize('artifact_file', + ['']) + def test_empty_artifact_file(plugin_config_file, artifact_file): + runner = click_testing.CliRunner() + result = runner.invoke(cli.delphix_sdk, + ['build', '-c', plugin_config_file, + '-a', artifact_file]) + + assert result.exit_code == 2 + expected_error_strings = ("Error: Invalid value for '-a'", + "is a directory") + for expected_error in expected_error_strings: + assert expected_error in result.output + class TestUploadCli: @staticmethod diff --git a/tools/src/test/python/dlpx/virtualization/_internal/test_package_util.py b/tools/src/test/python/dlpx/virtualization/_internal/test_package_util.py index fe978f7b..001e0403 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/test_package_util.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/test_package_util.py @@ -10,22 +10,22 @@ class TestPackageUtil: @staticmethod def test_get_version(): - assert package_util.get_version() == '2.2.0.dev1' + assert package_util.get_version() == '3.0.0.dev2' @staticmethod def test_get_virtualization_api_version(): - assert package_util.get_virtualization_api_version() == '1.3.0' + assert package_util.get_virtualization_api_version() == '1.4.0' @staticmethod def test_get_engine_api_version(): - assert package_util.get_engine_api_version_from_settings() == '1.11.3' + assert package_util.get_engine_api_version_from_settings() == '1.12.0' @staticmethod def test_get_build_api_version_json(): build_api_version = { 'type': 'APIVersion', 'major': 1, - 'minor': 3, + 'minor': 4, 'micro': 0 } assert package_util.get_build_api_version() == build_api_version @@ -35,8 +35,8 @@ def test_get_engine_api_version_json(): engine_api_version = { 'type': 'APIVersion', 'major': 1, - 'minor': 11, - 'micro': 3 + 'minor': 12, + 'micro': 0 } assert package_util.get_engine_api_version() == engine_api_version diff --git a/tools/src/test/python/dlpx/virtualization/_internal/test_plugin_importer.py b/tools/src/test/python/dlpx/virtualization/_internal/test_plugin_importer.py index 7a1ce1ab..a8795765 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/test_plugin_importer.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/test_plugin_importer.py @@ -244,3 +244,73 @@ def test_plugin_info_warn_mode(mock_import, mock_relative_path, plugin_config_content, False) except Exception: raise AssertionError() + + @staticmethod + @pytest.mark.parametrize( + 'entry_point,plugin_type,expected_errors', + [('successful:ne_symbol', 'DIRECT', [ + "Error: Entry point 'successful:ne_symbol' does not exist.", + "'ne_symbol' is not a symbol in module ", + ])]) + @mock.patch('dlpx.virtualization._internal.file_util.get_src_dir_path') + def test_non_existing_symbol_in_module(mock_file_util, plugin_config_file, + fake_src_dir, expected_errors): + mock_file_util.return_value = fake_src_dir + + with pytest.raises(exceptions.UserError) as err_info: + importer = get_plugin_importer(plugin_config_file) + importer.validate_plugin_module() + + message = err_info.value.message + for error in expected_errors: + assert error in message + + @staticmethod + @pytest.mark.parametrize( + 'entry_point,expected_error', + [('import_error:plugin', + "Error: No module named dlpxxx.virtualization.platform")]) + @mock.patch('dlpx.virtualization._internal.file_util.get_src_dir_path') + def test_import_error(mock_file_util, plugin_config_file, + fake_src_dir, expected_error): + mock_file_util.return_value = fake_src_dir + + with pytest.raises(exceptions.UserError) as err_info: + importer = get_plugin_importer(plugin_config_file) + importer.validate_plugin_module() + + message = err_info.value.message + assert expected_error in message + + @staticmethod + @pytest.mark.parametrize( + 'entry_point,expected_error', + [('bad_syntax:plugin', "SDK Error: invalid syntax")]) + @mock.patch('dlpx.virtualization._internal.file_util.get_src_dir_path') + def test_bad_syntax(mock_file_util, plugin_config_file, + fake_src_dir, expected_error): + mock_file_util.return_value = fake_src_dir + + with pytest.raises(exceptions.SDKToolingError) as err_info: + importer = get_plugin_importer(plugin_config_file) + importer.validate_plugin_module() + + message = err_info.value.message + assert expected_error in message + + @staticmethod + @pytest.mark.parametrize( + 'entry_point,expected_error', + [('undefined_name:plugin', + "SDK Error: name 'directplugin' is not defined")]) + @mock.patch('dlpx.virtualization._internal.file_util.get_src_dir_path') + def test_undefined_name_error(mock_file_util, plugin_config_file, + fake_src_dir, expected_error): + mock_file_util.return_value = fake_src_dir + + with pytest.raises(exceptions.SDKToolingError) as err_info: + importer = get_plugin_importer(plugin_config_file) + importer.validate_plugin_module() + + message = err_info.value.message + assert expected_error in message diff --git a/tools/src/test/python/dlpx/virtualization/_internal/test_plugin_validator.py b/tools/src/test/python/dlpx/virtualization/_internal/test_plugin_validator.py index 893e6e41..e81ce7da 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/test_plugin_validator.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/test_plugin_validator.py @@ -118,6 +118,7 @@ def test_plugin_additional_properties(plugin_config_file, @staticmethod @pytest.mark.parametrize('host_types', [['xxx']]) @pytest.mark.parametrize('src_dir', [None]) + @pytest.mark.parametrize('plugin_type', ['INDIRECT']) def test_multiple_validation_errors(plugin_config_file, plugin_config_content): with pytest.raises(exceptions.SchemaValidationError) as err_info: @@ -128,6 +129,7 @@ def test_multiple_validation_errors(plugin_config_file, message = err_info.value.message assert "'srcDir' is a required property" in message assert "'xxx' is not one of ['UNIX', 'WINDOWS']" in message + assert "'INDIRECT' is not one of ['DIRECT', 'STAGED']" in message @staticmethod @mock.patch('os.path.isabs', return_value=False)