From 2dfbee8203680589ecdc7db57ead041f20094b6b Mon Sep 17 00:00:00 2001 From: Lindsey Nguyen Date: Fri, 25 Sep 2020 16:16:54 -0700 Subject: [PATCH 01/12] Update SDK to add SnapshotParametersDefinition in uploaded artifact (#255) --- .bumpversion.cfg | 2 +- README-dev.md | 8 +-- common/setup.py | 2 +- .../python/dlpx/virtualization/common/VERSION | 2 +- .../main/python/dlpx/virtualization/VERSION | 2 +- libs/setup.py | 2 +- .../python/dlpx/virtualization/libs/VERSION | 2 +- platform/setup.py | 2 +- .../dlpx/virtualization/platform/VERSION | 2 +- .../dlpx/virtualization/platform/_linked.py | 33 ++++++--- .../fake_generated_definitions.py | 8 --- .../python/dlpx/virtualization/test_plugin.py | 70 +++++++++++-------- .../dlpx/virtualization/_internal/VERSION | 2 +- .../dlpx/virtualization/_internal/codegen.py | 22 +----- .../_internal/commands/build.py | 2 + .../direct_operations.py.template | 3 +- .../plugin_template/schema_template.json | 5 ++ .../staged_operations.py.template | 2 +- .../validation_schemas/plugin_importer.yaml | 6 +- .../validation_schemas/plugin_schema.json | 6 +- .../_internal/commands/test_build.py | 33 ++++++++- .../_internal/commands/test_codegen.py | 5 +- .../dlpx/virtualization/_internal/conftest.py | 69 ++++-------------- .../fake_plugin/direct/multiple_warnings.py | 10 +-- .../fake_plugin/direct/successful.py | 10 +-- .../fake_plugin/direct/upgrade_warnings.py | 6 +- .../fake_plugin/staged/multiple_warnings.py | 4 +- .../fake_plugin/staged/successful.py | 4 +- .../_internal/test_package_util.py | 2 +- 29 files changed, 161 insertions(+), 165 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 98ef68c6..879c212f 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.2.0.dev1 +current_version = 3.0.0.dev0 commit = False tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+)(?P\d+))? diff --git a/README-dev.md b/README-dev.md index 61c23ce7..34540f17 100644 --- a/README-dev.md +++ b/README-dev.md @@ -125,17 +125,17 @@ To run blackbox tests, follow these steps: 2. Navigate to the app-gate directory and start tests using `git blackbox`. For the guide on which test suite to use, see the next sections. -At a minimum, each pull request should pass `appdata_python_samples` and `appdata_sanity` tests with a direct or staged plugin. +At a minimum, each pull request should pass `appdata_python_samples` and `appdata_basic` tests with a direct or staged plugin. See the section below for the description of each test suite. #### Blackbox tests targeting wrappers (mostly Delphix Engine workflows) * appdata_python_samples (sample plugins from the app-gate): `git blackbox -s appdata_python_samples --extra-params="-p virt-sdk-repo=https://github.com//virtualization-sdk.git -p virt-sdk-branch=my-feature"`, -* appdata_sanity with a direct Python plugin on CentOS 7.3: `git blackbox -s appdata_sanity -c APPDATA_PYTHON_DIRECT_CENTOS73 -a --extra-params="-p virt-sdk-repo=https://github.com//virtualization-sdk.git -p virt-sdk-branch=my-feature"`, -* appdata_sanity with a staged Python plugin on CentOS 7.3: `git blackbox -s appdata_sanity -c APPDATA_PYTHON_STAGED_CENTOS73 -a --extra-params="-p virt-sdk-repo=https://github.com//virtualization-sdk.git -p virt-sdk-branch=my-feature"`. +* appdata_basic with a direct Python plugin on CentOS 7.3: `git blackbox -s appdata_basic -c APPDATA_PYTHON_DIRECT_CENTOS73 -a --extra-params="-p virt-sdk-repo=https://github.com//virtualization-sdk.git -p virt-sdk-branch=my-feature"`, +* appdata_basic with a staged Python plugin on CentOS 7.3: `git blackbox -s appdata_basic -c APPDATA_PYTHON_STAGED_CENTOS73 -a --extra-params="-p virt-sdk-repo=https://github.com//virtualization-sdk.git -p virt-sdk-branch=my-feature"`. #### Blackbox tests targeting the CLI (~80% CLI tests) * virtualization_sdk (installs and tests a direct Python plugin on Ubuntu 18): `git blackbox -s virtualization_sdk -c APPDATA_SDK_UBUNTU18_DIRECT_CENTOS73 --extra-params="-p virt-sdk-repo=https://github.com//virtualization-sdk.git -p virt-sdk-branch=my-feature"`, * virtualization_sdk (installs and tests a staged Python plugin on Ubuntu 18): -`git blackbox -s virtualization_sdk -c APPDATA_SDK_UBUNTU18_STAGED_CENTOS73 --extra-params="-p virt-sdk-repo=https://github.com//virtualization-sdk.git -p virt-sdk-branch=my-feature"`. \ No newline at end of file +`git blackbox -s virtualization_sdk -c APPDATA_SDK_UBUNTU18_STAGED_CENTOS73 --extra-params="-p virt-sdk-repo=https://github.com//virtualization-sdk.git -p virt-sdk-branch=my-feature"`. diff --git a/common/setup.py b/common/setup.py index a5e92681..985b4ba3 100644 --- a/common/setup.py +++ b/common/setup.py @@ -4,7 +4,7 @@ PYTHON_SRC = 'src/main/python' install_requires = [ - "dvp-api == 1.4.0.dev10", + "dvp-api == 1.4.0.dev12", ] with open(os.path.join(PYTHON_SRC, 'dlpx/virtualization/common/VERSION')) as version_file: diff --git a/common/src/main/python/dlpx/virtualization/common/VERSION b/common/src/main/python/dlpx/virtualization/common/VERSION index f1855a71..4e0e00e1 100644 --- a/common/src/main/python/dlpx/virtualization/common/VERSION +++ b/common/src/main/python/dlpx/virtualization/common/VERSION @@ -1 +1 @@ -2.2.0.dev1 \ No newline at end of file +3.0.0.dev0 \ No newline at end of file diff --git a/dvp/src/main/python/dlpx/virtualization/VERSION b/dvp/src/main/python/dlpx/virtualization/VERSION index f1855a71..4e0e00e1 100644 --- a/dvp/src/main/python/dlpx/virtualization/VERSION +++ b/dvp/src/main/python/dlpx/virtualization/VERSION @@ -1 +1 @@ -2.2.0.dev1 \ No newline at end of file +3.0.0.dev0 \ No newline at end of file diff --git a/libs/setup.py b/libs/setup.py index 1713aa99..e59da92d 100644 --- a/libs/setup.py +++ b/libs/setup.py @@ -7,7 +7,7 @@ version = version_file.read().strip() install_requires = [ - "dvp-api == 1.4.0.dev10", + "dvp-api == 1.4.0.dev12", "dvp-common == {}".format(version) ] diff --git a/libs/src/main/python/dlpx/virtualization/libs/VERSION b/libs/src/main/python/dlpx/virtualization/libs/VERSION index f1855a71..4e0e00e1 100644 --- a/libs/src/main/python/dlpx/virtualization/libs/VERSION +++ b/libs/src/main/python/dlpx/virtualization/libs/VERSION @@ -1 +1 @@ -2.2.0.dev1 \ No newline at end of file +3.0.0.dev0 \ No newline at end of file diff --git a/platform/setup.py b/platform/setup.py index 3d9daf3c..5944d1e5 100644 --- a/platform/setup.py +++ b/platform/setup.py @@ -7,7 +7,7 @@ version = version_file.read().strip() install_requires = [ - "dvp-api == 1.4.0.dev10", + "dvp-api == 1.4.0.dev12", "dvp-common == {}".format(version), "enum34;python_version < '3.4'", ] diff --git a/platform/src/main/python/dlpx/virtualization/platform/VERSION b/platform/src/main/python/dlpx/virtualization/platform/VERSION index f1855a71..4e0e00e1 100644 --- a/platform/src/main/python/dlpx/virtualization/platform/VERSION +++ b/platform/src/main/python/dlpx/virtualization/platform/VERSION @@ -1 +1 @@ -2.2.0.dev1 \ No newline at end of file +3.0.0.dev0 \ No newline at end of file diff --git a/platform/src/main/python/dlpx/virtualization/platform/_linked.py b/platform/src/main/python/dlpx/virtualization/platform/_linked.py index d6e8c656..ade3ea30 100644 --- a/platform/src/main/python/dlpx/virtualization/platform/_linked.py +++ b/platform/src/main/python/dlpx/virtualization/platform/_linked.py @@ -121,6 +121,7 @@ def _internal_direct_pre_snapshot(self, request): from generated.definitions import RepositoryDefinition from generated.definitions import LinkedSourceDefinition from generated.definitions import SourceConfigDefinition + from generated.definitions import SnapshotParametersDefinition # # While linked.pre_snapshot() is not a required operation, this should @@ -141,10 +142,14 @@ def _internal_direct_pre_snapshot(self, request): json.loads(request.repository.parameters.json)) source_config = SourceConfigDefinition.from_dict( json.loads(request.source_config.parameters.json)) + snapshot_parameters = SnapshotParametersDefinition.from_dict( + json.loads(request.snapshot_parameters.parameters.json)) - self.pre_snapshot_impl(direct_source=direct_source, - repository=repository, - source_config=source_config) + self.pre_snapshot_impl( + direct_source=direct_source, + repository=repository, + source_config=source_config, + optional_snapshot_parameters=snapshot_parameters) direct_pre_snapshot_response = platform_pb2.DirectPreSnapshotResponse() direct_pre_snapshot_response.return_value.CopyFrom( @@ -173,6 +178,7 @@ def _internal_direct_post_snapshot(self, request): from generated.definitions import LinkedSourceDefinition from generated.definitions import SourceConfigDefinition from generated.definitions import SnapshotDefinition + from generated.definitions import SnapshotParametersDefinition def to_protobuf(snapshot): parameters = common_pb2.PluginDefinedObject() @@ -196,10 +202,14 @@ def to_protobuf(snapshot): json.loads(request.repository.parameters.json)) source_config = SourceConfigDefinition.from_dict( json.loads(request.source_config.parameters.json)) + snapshot_parameters = SnapshotParametersDefinition.from_dict( + json.loads(request.snapshot_parameters.parameters.json)) - snapshot = self.post_snapshot_impl(direct_source=direct_source, - repository=repository, - source_config=source_config) + snapshot = self.post_snapshot_impl( + direct_source=direct_source, + repository=repository, + source_config=source_config, + optional_snapshot_parameters=snapshot_parameters) # Validate that this is a SnapshotDefinition object if not isinstance(snapshot, SnapshotDefinition): @@ -266,10 +276,11 @@ def _internal_staged_pre_snapshot(self, request): snapshot_parameters = SnapshotParametersDefinition.from_dict( json.loads(request.snapshot_parameters.parameters.json)) - self.pre_snapshot_impl(staged_source=staged_source, - repository=repository, - source_config=source_config, - snapshot_parameters=snapshot_parameters) + self.pre_snapshot_impl( + staged_source=staged_source, + repository=repository, + source_config=source_config, + optional_snapshot_parameters=snapshot_parameters) response = platform_pb2.StagedPreSnapshotResponse() response.return_value.CopyFrom(platform_pb2.StagedPreSnapshotResult()) @@ -337,7 +348,7 @@ def to_protobuf(snapshot): staged_source=staged_source, repository=repository, source_config=source_config, - snapshot_parameters=snapshot_parameters) + optional_snapshot_parameters=snapshot_parameters) # Validate that this is a SnapshotDefinition object if not isinstance(snapshot, SnapshotDefinition): diff --git a/platform/src/test/python/dlpx/virtualization/fake_generated_definitions.py b/platform/src/test/python/dlpx/virtualization/fake_generated_definitions.py index 481c04a5..90fbcef6 100644 --- a/platform/src/test/python/dlpx/virtualization/fake_generated_definitions.py +++ b/platform/src/test/python/dlpx/virtualization/fake_generated_definitions.py @@ -101,14 +101,6 @@ def to_dict(self): class SnapshotParametersDefinition(Model): - """ - The appdata snapshot parameter will eventually be customizable but for now - this just follows the old appdata parameter where the delphix user can decide - if resync is true or not. This will now go into pre and post snapshot - operations rather than the resync operation. The main point is customers will - set this to be true is that this means the operation is a "hard" resync and - that all data should be refreshed. - """ def __init__(self, resync): self.swagger_types = {'resync': bool} diff --git a/platform/src/test/python/dlpx/virtualization/test_plugin.py b/platform/src/test/python/dlpx/virtualization/test_plugin.py index 5f90bdaa..0d0fa660 100755 --- a/platform/src/test/python/dlpx/virtualization/test_plugin.py +++ b/platform/src/test/python/dlpx/virtualization/test_plugin.py @@ -813,19 +813,23 @@ def source_config_discovery_impl(source_connection, repository): @staticmethod def test_direct_pre_snapshot(my_plugin, direct_source, repository, - source_config): + source_config, snapshot_parameters): @my_plugin.linked.pre_snapshot() - def mock_direct_pre_snapshot(direct_source, repository, source_config): - TestPlugin.assert_plugin_args(direct_source=direct_source, - repository=repository, - source_config=source_config) + def mock_direct_pre_snapshot(direct_source, repository, source_config, + optional_snapshot_parameters): + TestPlugin.assert_plugin_args( + direct_source=direct_source, + repository=repository, + source_config=source_config, + snapshot_parameters=optional_snapshot_parameters) return direct_pre_snapshot_request = platform_pb2.DirectPreSnapshotRequest() TestPlugin.setup_request(request=direct_pre_snapshot_request, direct_source=direct_source, repository=repository, - source_config=source_config) + source_config=source_config, + snapshot_parameters=snapshot_parameters) expected_result = platform_pb2.DirectPreSnapshotResult() direct_pre_snapshot_response = ( @@ -839,20 +843,24 @@ def mock_direct_pre_snapshot(direct_source, repository, source_config): @staticmethod def test_direct_post_snapshot(my_plugin, direct_source, repository, - source_config): + source_config, snapshot_parameters): @my_plugin.linked.post_snapshot() - def direct_post_snapshot_impl(direct_source, repository, - source_config): - TestPlugin.assert_plugin_args(direct_source=direct_source, - repository=repository, - source_config=source_config) + def direct_post_snapshot_impl(direct_source, repository, source_config, + optional_snapshot_parameters): + TestPlugin.assert_plugin_args( + direct_source=direct_source, + repository=repository, + source_config=source_config, + snapshot_parameters=optional_snapshot_parameters) return SnapshotDefinition(TEST_SNAPSHOT) direct_post_snapshot_request = platform_pb2.DirectPostSnapshotRequest() - TestPlugin.setup_request(request=direct_post_snapshot_request, - direct_source=direct_source, - repository=repository, - source_config=source_config) + TestPlugin.setup_request( + request=direct_post_snapshot_request, + direct_source=direct_source, + repository=repository, + source_config=source_config, + snapshot_parameters=snapshot_parameters) direct_post_snapshot_response = ( my_plugin.linked._internal_direct_post_snapshot( @@ -866,20 +874,21 @@ def test_staged_pre_snapshot(my_plugin, staged_source, repository, source_config, snapshot_parameters): @my_plugin.linked.pre_snapshot() def staged_pre_snapshot_impl(staged_source, repository, source_config, - snapshot_parameters): + optional_snapshot_parameters): TestPlugin.assert_plugin_args( staged_source=staged_source, repository=repository, source_config=source_config, - snapshot_parameters=snapshot_parameters) + snapshot_parameters=optional_snapshot_parameters) return staged_pre_snapshot_request = platform_pb2.StagedPreSnapshotRequest() - TestPlugin.setup_request(request=staged_pre_snapshot_request, - staged_source=staged_source, - repository=repository, - source_config=source_config, - snapshot_parameters=snapshot_parameters) + TestPlugin.setup_request( + request=staged_pre_snapshot_request, + staged_source=staged_source, + repository=repository, + source_config=source_config, + snapshot_parameters=snapshot_parameters) expected_result = platform_pb2.StagedPreSnapshotResult() response = my_plugin.linked._internal_staged_pre_snapshot( @@ -894,20 +903,21 @@ def test_staged_post_snapshot(my_plugin, staged_source, repository, source_config, snapshot_parameters): @my_plugin.linked.post_snapshot() def staged_post_snapshot_impl(staged_source, repository, source_config, - snapshot_parameters): + optional_snapshot_parameters): TestPlugin.assert_plugin_args( staged_source=staged_source, repository=repository, source_config=source_config, - snapshot_parameters=snapshot_parameters) + snapshot_parameters=optional_snapshot_parameters) return SnapshotDefinition(TEST_SNAPSHOT) staged_post_snapshot_request = platform_pb2.StagedPostSnapshotRequest() - TestPlugin.setup_request(request=staged_post_snapshot_request, - staged_source=staged_source, - repository=repository, - source_config=source_config, - snapshot_parameters=snapshot_parameters) + TestPlugin.setup_request( + request=staged_post_snapshot_request, + staged_source=staged_source, + repository=repository, + source_config=source_config, + snapshot_parameters=snapshot_parameters) response = my_plugin.linked._internal_staged_post_snapshot( staged_post_snapshot_request) diff --git a/tools/src/main/python/dlpx/virtualization/_internal/VERSION b/tools/src/main/python/dlpx/virtualization/_internal/VERSION index f1855a71..4e0e00e1 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/VERSION +++ b/tools/src/main/python/dlpx/virtualization/_internal/VERSION @@ -1 +1 @@ -2.2.0.dev1 \ No newline at end of file +3.0.0.dev0 \ No newline at end of file diff --git a/tools/src/main/python/dlpx/virtualization/_internal/codegen.py b/tools/src/main/python/dlpx/virtualization/_internal/codegen.py index 9eda39b7..77a34397 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/codegen.py +++ b/tools/src/main/python/dlpx/virtualization/_internal/codegen.py @@ -26,24 +26,6 @@ 'definitions': {} } -# -# The default snapshot params we currently support. This is so that in the -# future when we want to support plugin author defined schemas for snapshot -# params the upgrade case will be relatively simple. -# - -SNAPSHOT_PARAMS_JSON = { - 'snapshotParametersDefinition': { - 'type': 'object', - 'additionalProperties': False, - 'properties': { - 'resync': { - 'type': 'boolean' - } - } - } -} - SWAGGER_FILE_NAME = 'swagger.json' CODEGEN_PACKAGE = 'generated' CODEGEN_MODULE = 'definitions' @@ -111,9 +93,7 @@ def generate_python(name, source_dir, plugin_config_dir, schema_content): def _write_swagger_file(name, schema_dict, output_dir): swagger_json = copy.deepcopy(SWAGGER_JSON_FORMAT) swagger_json['info']['title'] = name - swagger_json['definitions'] = copy.deepcopy(schema_dict) - # Add in the snapshot param definition - swagger_json['definitions'].update(SNAPSHOT_PARAMS_JSON) + swagger_json['definitions'] = schema_dict swagger_file = os.path.join(output_dir, SWAGGER_FILE_NAME) logger.info('Writing swagger file to {}'.format(swagger_file)) diff --git a/tools/src/main/python/dlpx/virtualization/_internal/commands/build.py b/tools/src/main/python/dlpx/virtualization/_internal/commands/build.py index e56bc7f5..b9185b2a 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/commands/build.py +++ b/tools/src/main/python/dlpx/virtualization/_internal/commands/build.py @@ -207,6 +207,8 @@ def prepare_upload_artifact(plugin_config_content, src_dir, schemas, manifest): prepare_discovery_definition(plugin_config_content, schemas), 'snapshotSchema': schemas['snapshotDefinition'], + 'snapshotParametersDefinition': + schemas['snapshotParametersDefinition'], 'manifest': manifest } diff --git a/tools/src/main/python/dlpx/virtualization/_internal/commands/plugin_template/direct_operations.py.template b/tools/src/main/python/dlpx/virtualization/_internal/commands/plugin_template/direct_operations.py.template index 88b5a416..164e512a 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/commands/plugin_template/direct_operations.py.template +++ b/tools/src/main/python/dlpx/virtualization/_internal/commands/plugin_template/direct_operations.py.template @@ -1,3 +1,4 @@ @plugin.linked.post_snapshot() -def linked_post_snapshot(direct_source, repository, source_config): +def linked_post_snapshot(direct_source, repository, source_config, + optional_snapshot_parameters): return SnapshotDefinition() \ No newline at end of file diff --git a/tools/src/main/python/dlpx/virtualization/_internal/commands/plugin_template/schema_template.json b/tools/src/main/python/dlpx/virtualization/_internal/commands/plugin_template/schema_template.json index 261d2e15..8dd02073 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/commands/plugin_template/schema_template.json +++ b/tools/src/main/python/dlpx/virtualization/_internal/commands/plugin_template/schema_template.json @@ -31,5 +31,10 @@ "type" : "object", "additionalProperties" : false, "properties" : {} + }, + "snapshotParametersDefinition": { + "type" : "object", + "additionalProperties" : false, + "properties" : {} } } diff --git a/tools/src/main/python/dlpx/virtualization/_internal/commands/plugin_template/staged_operations.py.template b/tools/src/main/python/dlpx/virtualization/_internal/commands/plugin_template/staged_operations.py.template index f23cb76f..d5879ec4 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/commands/plugin_template/staged_operations.py.template +++ b/tools/src/main/python/dlpx/virtualization/_internal/commands/plugin_template/staged_operations.py.template @@ -2,7 +2,7 @@ def linked_post_snapshot(staged_source, repository, source_config, - snapshot_parameters): + optional_snapshot_parameters): return SnapshotDefinition() diff --git a/tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_importer.yaml b/tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_importer.yaml index 98e41c19..d769cd62 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_importer.yaml +++ b/tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_importer.yaml @@ -13,12 +13,12 @@ EXPECTED_STAGED_ARGS_BY_OP: - staged_source - repository - source_config - - snapshot_parameters + - optional_snapshot_parameters post_snapshot_impl: - staged_source - repository - source_config - - snapshot_parameters + - optional_snapshot_parameters start_staging_impl: - staged_source - repository @@ -92,10 +92,12 @@ EXPECTED_DIRECT_ARGS_BY_OP: - direct_source - repository - source_config + - optional_snapshot_parameters post_snapshot_impl: - direct_source - repository - source_config + - optional_snapshot_parameters VirtualOperations: configure_impl: - virtual_source diff --git a/tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_schema.json b/tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_schema.json index 2d29ee37..b8be1836 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_schema.json +++ b/tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_schema.json @@ -179,7 +179,8 @@ "sourceConfigDefinition", "virtualSourceDefinition", "linkedSourceDefinition", - "snapshotDefinition" + "snapshotDefinition", + "snapshotParametersDefinition" ], "properties": { "repositoryDefinition": { @@ -210,6 +211,9 @@ }, "snapshotDefinition": { "$ref": "#/definitions/jsonSchema" + }, + "snapshotParametersDefinition": { + "$ref": "#/definitions/jsonSchema" } }, "additionalProperties": false diff --git a/tools/src/test/python/dlpx/virtualization/_internal/commands/test_build.py b/tools/src/test/python/dlpx/virtualization/_internal/commands/test_build.py index a2077b33..351d627a 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/commands/test_build.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/commands/test_build.py @@ -8,6 +8,7 @@ import yaml from dlpx.virtualization._internal import const, exceptions from dlpx.virtualization._internal.commands import build +from dlpx.virtualization._internal.commands import initialize as init from dlpx.virtualization._internal.plugin_importer import PluginImporter import mock @@ -48,6 +49,9 @@ def test_build_success(mock_relative_path, mock_install_deps, gen_py.plugin_content_dir, gen_py.schema_dict) mock_plugin_manifest.assert_called() + mock_install_deps.assert_called() + mock_relative_path.assert_called() + # After running build this file should now exist. assert os.path.exists(artifact_file) @@ -56,6 +60,31 @@ def test_build_success(mock_relative_path, mock_install_deps, assert content == artifact_content + @staticmethod + @pytest.mark.parametrize('ingestion_strategy', + [const.DIRECT_TYPE, const.STAGED_TYPE]) + @pytest.mark.parametrize('host_type', + [const.UNIX_HOST_TYPE, const.WINDOWS_HOST_TYPE]) + @mock.patch( + 'dlpx.virtualization._internal.plugin_dependency_util.install_deps') + @mock.patch('os.path.isabs', return_value=False) + def test_build_success_from_init(mock_relative_path, mock_install_deps, + tmpdir, ingestion_strategy, host_type, + plugin_name, artifact_file): + # Initialize an empty directory. + init.init(tmpdir.strpath, ingestion_strategy, plugin_name, host_type) + plugin_config_file = os.path.join( + tmpdir.strpath, init.DEFAULT_PLUGIN_CONFIG_FILE) + # Before running build assert that the artifact file does not exist. + assert not os.path.exists(artifact_file) + + build.build(plugin_config_file, artifact_file, False, False) + + mock_relative_path.assert_called() + mock_install_deps.assert_called() + + assert os.path.exists(artifact_file) + @staticmethod @pytest.mark.parametrize('artifact_filename', ['somefile.json']) @mock.patch.object(PluginImporter, @@ -284,13 +313,13 @@ def test_prepare_discovery_definition(plugin_config_content, assert actual_discovery_definition == discovery_definition @staticmethod - def test_prepare_upload_artifact_success(basic_artifact_content, + def test_prepare_upload_artifact_success(artifact_content, plugin_config_content, src_dir, schema_content): upload_artifact = build.prepare_upload_artifact( plugin_config_content, src_dir, schema_content, {}) - assert upload_artifact == basic_artifact_content + assert upload_artifact == artifact_content @staticmethod def test_generate_upload_artifact_success(tmpdir, artifact_content): diff --git a/tools/src/test/python/dlpx/virtualization/_internal/commands/test_codegen.py b/tools/src/test/python/dlpx/virtualization/_internal/commands/test_codegen.py index 383e9189..0548c654 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/commands/test_codegen.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/commands/test_codegen.py @@ -175,8 +175,7 @@ def test_get_build_dir_fail(): " Error message: No such file or directory") @staticmethod - def test_write_swagger_file(tmpdir, schema_content, - swagger_schema_content): + def test_write_swagger_file(tmpdir, schema_content): name = 'test' expected_file = tmpdir.join(codegen.SWAGGER_FILE_NAME).strpath codegen._write_swagger_file(name, schema_content, tmpdir.strpath) @@ -186,7 +185,7 @@ def test_write_swagger_file(tmpdir, schema_content, with open(expected_file, 'rb') as f: content = json.load(f) - assert content['definitions'] == swagger_schema_content + assert content['definitions'] == schema_content assert content['info']['title'] == name @staticmethod diff --git a/tools/src/test/python/dlpx/virtualization/_internal/conftest.py b/tools/src/test/python/dlpx/virtualization/_internal/conftest.py index 0f95743c..3df3afc4 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/conftest.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/conftest.py @@ -37,7 +37,8 @@ def plugin_config_file(tmpdir, plugin_config_filename, plugin_config_content): default_flow_style=False) f = tmpdir.join(plugin_config_filename) - f.write(plugin_config_content) + if plugin_config_content: + f.write(plugin_config_content) return f.strpath @@ -453,7 +454,8 @@ def plugin_manifest(upgrade_operation): @pytest.fixture def schema_content(repository_definition, source_config_definition, virtual_source_definition, linked_source_definition, - snapshot_definition, additional_definition): + snapshot_definition, snapshot_parameters_definition, + additional_definition): schema = {} @@ -472,19 +474,12 @@ def schema_content(repository_definition, source_config_definition, if snapshot_definition: schema['snapshotDefinition'] = snapshot_definition - if additional_definition: - schema['additionalDefinition'] = additional_definition - - return schema - - -@pytest.fixture -def swagger_schema_content(schema_content, snapshot_parameters_definition): - - schema = schema_content if snapshot_parameters_definition: schema['snapshotParametersDefinition'] = snapshot_parameters_definition + if additional_definition: + schema['additionalDefinition'] = additional_definition + return schema @@ -570,54 +565,10 @@ def additional_definition(): return None -@pytest.fixture -def basic_artifact_content(engine_api, virtual_source_definition, - linked_source_definition, discovery_definition, - snapshot_definition): - artifact = { - 'type': 'Plugin', - 'name': '16bef554-9470-11e9-b2e3-8c8590d4a42c', - 'prettyName': 'python_vfiles', - 'externalVersion': '2.0.0', - 'defaultLocale': 'en-us', - 'language': 'PYTHON27', - 'hostTypes': ['UNIX'], - 'entryPoint': 'python_vfiles:vfiles', - 'buildApi': package_util.get_build_api_version(), - 'engineApi': engine_api, - 'rootSquashEnabled': True, - 'buildNumber': '2', - 'luaName': 'lua-toolkit-1', - 'extendedStartStopHooks': False, - 'minimumLuaVersion': '2.3', - 'sourceCode': 'UEsFBgAAAAAAAAAAAAAAAAAAAAAAAA==', - 'manifest': {} - } - if virtual_source_definition: - artifact['virtualSourceDefinition'] = { - 'type': 'PluginVirtualSourceDefinition', - 'parameters': virtual_source_definition - } - - if linked_source_definition: - artifact['linkedSourceDefinition'] = { - 'type': 'PluginLinkedDirectSourceDefinition', - 'parameters': linked_source_definition - } - - if discovery_definition: - artifact['discoveryDefinition'] = discovery_definition - - if snapshot_definition: - artifact['snapshotSchema'] = snapshot_definition - - return artifact - - @pytest.fixture def artifact_content(engine_api, virtual_source_definition, linked_source_definition, discovery_definition, - snapshot_definition): + snapshot_definition, snapshot_parameters_definition): """ This fixture creates base artifact that was generated from build and used in upload. If any fields besides engine_api needs to be changed, @@ -664,6 +615,10 @@ def artifact_content(engine_api, virtual_source_definition, if snapshot_definition: artifact['snapshotSchema'] = snapshot_definition + if snapshot_parameters_definition: + artifact['snapshotParametersDefinition'] = ( + snapshot_parameters_definition) + return artifact diff --git a/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/multiple_warnings.py b/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/multiple_warnings.py index c0a031bc..56f4d299 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/multiple_warnings.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/multiple_warnings.py @@ -18,12 +18,14 @@ def source_config_discovery(source_connection, repository): @vfiles.linked.pre_snapshot() -def direct_pre_snapshot(direct_source, repository, source_config): +def direct_pre_snapshot(direct_source, repository, source_config, + optional_snapshot_parameters): return @vfiles.linked.post_snapshot() -def direct_post_snapshot(direct_source, repository, source_config): +def direct_post_snapshot(direct_source, repository, source_config, + optional_snapshot_parameters): return None @@ -40,12 +42,12 @@ def mount_specification(repository, virtual_source): @vfiles.virtual.post_snapshot() -def postSnapshot(repository, source_config, virtual_source): +def post_snapshot(repository, source_config, virtual_source): return None @vfiles.virtual.pre_snapshot() -def preSnapshot(repository, source_config, virtual_source): +def pre_snapshot(repository, source_config, virtual_source): pass diff --git a/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/successful.py b/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/successful.py index e8ef3d1c..cb355933 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/successful.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/successful.py @@ -18,12 +18,14 @@ def source_config_discovery(source_connection, repository): @direct.linked.pre_snapshot() -def direct_pre_snapshot(direct_source, repository, source_config): +def direct_pre_snapshot(direct_source, repository, source_config, + optional_snapshot_parameters): return @direct.linked.post_snapshot() -def direct_post_snapshot(direct_source, repository, source_config): +def direct_post_snapshot(direct_source, repository, source_config, + optional_snapshot_parameters): return None @@ -45,12 +47,12 @@ def mount_specification(repository, virtual_source): @direct.virtual.post_snapshot() -def postSnapshot(repository, source_config, virtual_source): +def post_snapshot(repository, source_config, virtual_source): return None @direct.virtual.pre_snapshot() -def preSnapshot(repository, source_config, virtual_source): +def pre_snapshot(repository, source_config, virtual_source): pass diff --git a/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/upgrade_warnings.py b/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/upgrade_warnings.py index 68ecd5b2..0046fe49 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/upgrade_warnings.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/upgrade_warnings.py @@ -18,12 +18,14 @@ def source_config_discovery(source_connection, repository): @direct.linked.pre_snapshot() -def direct_pre_snapshot(direct_source, repository, source_config): +def direct_pre_snapshot(direct_source, repository, source_config, + optional_snapshot_parameters): return @direct.linked.post_snapshot() -def direct_post_snapshot(direct_source, repository, source_config): +def direct_post_snapshot(direct_source, repository, source_config, + optional_snapshot_parameters): return None diff --git a/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/staged/multiple_warnings.py b/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/staged/multiple_warnings.py index 094c1dde..5f3a6c81 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/staged/multiple_warnings.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/staged/multiple_warnings.py @@ -32,13 +32,13 @@ def staged_mount_specification(staged_source, repository): @staged.linked.pre_snapshot() def staged_pre_snapshot(repository, source_config, staged_source, - snapshot_parameters): + optional_snapshot_parameters): pass @staged.linked.post_snapshot() def staged_post_snapshot(repository, source_config, staged_source, - snapshot_parameters): + optional_snapshot_parameters): return None diff --git a/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/staged/successful.py b/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/staged/successful.py index 0d6ca414..bea862cf 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/staged/successful.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/staged/successful.py @@ -31,13 +31,13 @@ def staged_mount_specification(staged_source, repository): @staged.linked.pre_snapshot() def staged_pre_snapshot(repository, source_config, staged_source, - snapshot_parameters): + optional_snapshot_parameters): pass @staged.linked.post_snapshot() def staged_post_snapshot(repository, source_config, staged_source, - snapshot_parameters): + optional_snapshot_parameters): return None diff --git a/tools/src/test/python/dlpx/virtualization/_internal/test_package_util.py b/tools/src/test/python/dlpx/virtualization/_internal/test_package_util.py index 4813de52..063fb132 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/test_package_util.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/test_package_util.py @@ -10,7 +10,7 @@ class TestPackageUtil: @staticmethod def test_get_version(): - assert package_util.get_version() == '2.2.0.dev1' + assert package_util.get_version() == '3.0.0.dev0' @staticmethod def test_get_virtualization_api_version(): From e6f18969d939d8cc63a3d6b6106435df74989218 Mon Sep 17 00:00:00 2001 From: Lindsey Nguyen Date: Wed, 14 Oct 2020 23:19:38 -0700 Subject: [PATCH 02/12] Update engine api for snapshot parameters and rewrite params default to none when json is empty (#260) --- .bumpversion.cfg | 2 +- .../python/dlpx/virtualization/common/VERSION | 2 +- .../main/python/dlpx/virtualization/VERSION | 2 +- .../python/dlpx/virtualization/libs/VERSION | 2 +- .../dlpx/virtualization/platform/VERSION | 2 +- .../dlpx/virtualization/platform/_linked.py | 44 +++++-- .../python/dlpx/virtualization/test_plugin.py | 122 ++++++++++++++++++ .../dlpx/virtualization/_internal/VERSION | 2 +- .../_internal/commands/build.py | 7 +- .../virtualization/_internal/settings.cfg | 2 +- .../dlpx/virtualization/_internal/conftest.py | 8 +- .../_internal/test_package_util.py | 8 +- 12 files changed, 181 insertions(+), 22 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 879c212f..cf3ae7f9 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 3.0.0.dev0 +current_version = 3.0.0.dev1 commit = False tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+)(?P\d+))? diff --git a/common/src/main/python/dlpx/virtualization/common/VERSION b/common/src/main/python/dlpx/virtualization/common/VERSION index 4e0e00e1..dd60b157 100644 --- a/common/src/main/python/dlpx/virtualization/common/VERSION +++ b/common/src/main/python/dlpx/virtualization/common/VERSION @@ -1 +1 @@ -3.0.0.dev0 \ No newline at end of file +3.0.0.dev1 \ No newline at end of file diff --git a/dvp/src/main/python/dlpx/virtualization/VERSION b/dvp/src/main/python/dlpx/virtualization/VERSION index 4e0e00e1..dd60b157 100644 --- a/dvp/src/main/python/dlpx/virtualization/VERSION +++ b/dvp/src/main/python/dlpx/virtualization/VERSION @@ -1 +1 @@ -3.0.0.dev0 \ No newline at end of file +3.0.0.dev1 \ No newline at end of file diff --git a/libs/src/main/python/dlpx/virtualization/libs/VERSION b/libs/src/main/python/dlpx/virtualization/libs/VERSION index 4e0e00e1..dd60b157 100644 --- a/libs/src/main/python/dlpx/virtualization/libs/VERSION +++ b/libs/src/main/python/dlpx/virtualization/libs/VERSION @@ -1 +1 @@ -3.0.0.dev0 \ No newline at end of file +3.0.0.dev1 \ No newline at end of file diff --git a/platform/src/main/python/dlpx/virtualization/platform/VERSION b/platform/src/main/python/dlpx/virtualization/platform/VERSION index 4e0e00e1..dd60b157 100644 --- a/platform/src/main/python/dlpx/virtualization/platform/VERSION +++ b/platform/src/main/python/dlpx/virtualization/platform/VERSION @@ -1 +1 @@ -3.0.0.dev0 \ No newline at end of file +3.0.0.dev1 \ No newline at end of file diff --git a/platform/src/main/python/dlpx/virtualization/platform/_linked.py b/platform/src/main/python/dlpx/virtualization/platform/_linked.py index ade3ea30..4e6c07ee 100644 --- a/platform/src/main/python/dlpx/virtualization/platform/_linked.py +++ b/platform/src/main/python/dlpx/virtualization/platform/_linked.py @@ -142,8 +142,15 @@ def _internal_direct_pre_snapshot(self, request): json.loads(request.repository.parameters.json)) source_config = SourceConfigDefinition.from_dict( json.loads(request.source_config.parameters.json)) - snapshot_parameters = SnapshotParametersDefinition.from_dict( - json.loads(request.snapshot_parameters.parameters.json)) + snap_params = json.loads(request.snapshot_parameters.parameters.json) + # + # The snapshot_parameters object should be set to None if the json from + # the protobuf is None to differentiate no snapshot parameters vs empty + # snapshot parameters. + # + snapshot_parameters = ( + None if snap_params is None else + SnapshotParametersDefinition.from_dict(snap_params)) self.pre_snapshot_impl( direct_source=direct_source, @@ -202,8 +209,15 @@ def to_protobuf(snapshot): json.loads(request.repository.parameters.json)) source_config = SourceConfigDefinition.from_dict( json.loads(request.source_config.parameters.json)) - snapshot_parameters = SnapshotParametersDefinition.from_dict( - json.loads(request.snapshot_parameters.parameters.json)) + snap_params = json.loads(request.snapshot_parameters.parameters.json) + # + # The snapshot_parameters object should be set to None if the json from + # the protobuf is None to differentiate no snapshot parameters vs empty + # snapshot parameters. + # + snapshot_parameters = ( + None if snap_params is None else + SnapshotParametersDefinition.from_dict(snap_params)) snapshot = self.post_snapshot_impl( direct_source=direct_source, @@ -273,8 +287,15 @@ def _internal_staged_pre_snapshot(self, request): json.loads(request.repository.parameters.json)) source_config = SourceConfigDefinition.from_dict( json.loads(request.source_config.parameters.json)) - snapshot_parameters = SnapshotParametersDefinition.from_dict( - json.loads(request.snapshot_parameters.parameters.json)) + snap_params = json.loads(request.snapshot_parameters.parameters.json) + # + # The snapshot_parameters object should be set to None if the json from + # the protobuf is None to differentiate no snapshot parameters vs empty + # snapshot parameters. + # + snapshot_parameters = ( + None if snap_params is None else + SnapshotParametersDefinition.from_dict(snap_params)) self.pre_snapshot_impl( staged_source=staged_source, @@ -341,8 +362,15 @@ def to_protobuf(snapshot): json.loads(request.repository.parameters.json)) source_config = SourceConfigDefinition.from_dict( json.loads(request.source_config.parameters.json)) - snapshot_parameters = SnapshotParametersDefinition.from_dict( - json.loads(request.snapshot_parameters.parameters.json)) + snap_params = json.loads(request.snapshot_parameters.parameters.json) + # + # The snapshot_parameters object should be set to None if the json from + # the protobuf is None to differentiate no snapshot parameters vs empty + # snapshot parameters. + # + snapshot_parameters = ( + None if snap_params is None else + SnapshotParametersDefinition.from_dict(snap_params)) snapshot = self.post_snapshot_impl( staged_source=staged_source, diff --git a/platform/src/test/python/dlpx/virtualization/test_plugin.py b/platform/src/test/python/dlpx/virtualization/test_plugin.py index 0d0fa660..078b54e7 100755 --- a/platform/src/test/python/dlpx/virtualization/test_plugin.py +++ b/platform/src/test/python/dlpx/virtualization/test_plugin.py @@ -841,6 +841,38 @@ def mock_direct_pre_snapshot(direct_source, repository, source_config, 'result') == 'return_value' assert direct_pre_snapshot_response.return_value == expected_result + @staticmethod + def test_direct_pre_snapshot_null_snapparams(my_plugin, direct_source, + repository, source_config): + @my_plugin.linked.pre_snapshot() + def mock_direct_pre_snapshot(direct_source, repository, source_config, + optional_snapshot_parameters): + TestPlugin.assert_direct_source(direct_source) + TestPlugin.assert_repository(repository) + TestPlugin.assert_source_config(source_config) + assert not optional_snapshot_parameters + return + + snapshot_parameters = common_pb2.SnapshotParameters() + snapshot_parameters.parameters.json = 'null' + + direct_pre_snapshot_request = platform_pb2.DirectPreSnapshotRequest() + TestPlugin.setup_request(request=direct_pre_snapshot_request, + direct_source=direct_source, + repository=repository, + source_config=source_config, + snapshot_parameters=snapshot_parameters) + + expected_result = platform_pb2.DirectPreSnapshotResult() + direct_pre_snapshot_response = ( + my_plugin.linked._internal_direct_pre_snapshot( + direct_pre_snapshot_request)) + + # Check that the response's oneof is set to return_value and not error + assert direct_pre_snapshot_response.WhichOneof( + 'result') == 'return_value' + assert direct_pre_snapshot_response.return_value == expected_result + @staticmethod def test_direct_post_snapshot(my_plugin, direct_source, repository, source_config, snapshot_parameters): @@ -869,6 +901,36 @@ def direct_post_snapshot_impl(direct_source, repository, source_config, snapshot = direct_post_snapshot_response.return_value.snapshot assert snapshot.parameters.json == expected_snapshot + @staticmethod + def test_direct_post_snapshot_null_snapparams(my_plugin, direct_source, + repository, source_config): + @my_plugin.linked.post_snapshot() + def direct_post_snapshot_impl(direct_source, repository, source_config, + optional_snapshot_parameters): + TestPlugin.assert_direct_source(direct_source) + TestPlugin.assert_repository(repository) + TestPlugin.assert_source_config(source_config) + assert not optional_snapshot_parameters + return SnapshotDefinition(TEST_SNAPSHOT) + + snapshot_parameters = common_pb2.SnapshotParameters() + snapshot_parameters.parameters.json = 'null' + + direct_post_snapshot_request = platform_pb2.DirectPostSnapshotRequest() + TestPlugin.setup_request( + request=direct_post_snapshot_request, + direct_source=direct_source, + repository=repository, + source_config=source_config, + snapshot_parameters=snapshot_parameters) + + direct_post_snapshot_response = ( + my_plugin.linked._internal_direct_post_snapshot( + direct_post_snapshot_request)) + expected_snapshot = TEST_SNAPSHOT_JSON + snapshot = direct_post_snapshot_response.return_value.snapshot + assert snapshot.parameters.json == expected_snapshot + @staticmethod def test_staged_pre_snapshot(my_plugin, staged_source, repository, source_config, snapshot_parameters): @@ -898,6 +960,37 @@ def staged_pre_snapshot_impl(staged_source, repository, source_config, assert response.WhichOneof('result') == 'return_value' assert (response.return_value == expected_result) + @staticmethod + def test_staged_pre_snapshot_null_snapparams(my_plugin, staged_source, + repository, source_config): + @my_plugin.linked.pre_snapshot() + def staged_pre_snapshot_impl(staged_source, repository, source_config, + optional_snapshot_parameters): + TestPlugin.assert_staged_source(staged_source) + TestPlugin.assert_repository(repository) + TestPlugin.assert_source_config(source_config) + assert not optional_snapshot_parameters + return + + snapshot_parameters = common_pb2.SnapshotParameters() + snapshot_parameters.parameters.json = 'null' + + staged_pre_snapshot_request = platform_pb2.StagedPreSnapshotRequest() + TestPlugin.setup_request( + request=staged_pre_snapshot_request, + staged_source=staged_source, + repository=repository, + source_config=source_config, + snapshot_parameters=snapshot_parameters) + + expected_result = platform_pb2.StagedPreSnapshotResult() + response = my_plugin.linked._internal_staged_pre_snapshot( + staged_pre_snapshot_request) + + # Check that the response's oneof is set to return_value and not error + assert response.WhichOneof('result') == 'return_value' + assert (response.return_value == expected_result) + @staticmethod def test_staged_post_snapshot(my_plugin, staged_source, repository, source_config, snapshot_parameters): @@ -925,6 +1018,35 @@ def staged_post_snapshot_impl(staged_source, repository, source_config, assert response.return_value.snapshot.parameters.json == expected + @staticmethod + def test_staged_post_snapshot_null_snapparams(my_plugin, staged_source, + repository, source_config): + @my_plugin.linked.post_snapshot() + def staged_post_snapshot_impl(staged_source, repository, source_config, + optional_snapshot_parameters): + TestPlugin.assert_staged_source(staged_source) + TestPlugin.assert_repository(repository) + TestPlugin.assert_source_config(source_config) + assert not optional_snapshot_parameters + return SnapshotDefinition(TEST_SNAPSHOT) + + snapshot_parameters = common_pb2.SnapshotParameters() + snapshot_parameters.parameters.json = 'null' + + staged_post_snapshot_request = platform_pb2.StagedPostSnapshotRequest() + TestPlugin.setup_request( + request=staged_post_snapshot_request, + staged_source=staged_source, + repository=repository, + source_config=source_config, + snapshot_parameters=snapshot_parameters) + + response = my_plugin.linked._internal_staged_post_snapshot( + staged_post_snapshot_request) + expected = TEST_SNAPSHOT_JSON + + assert response.return_value.snapshot.parameters.json == expected + @staticmethod def test_start_staging(my_plugin, staged_source, repository, source_config): diff --git a/tools/src/main/python/dlpx/virtualization/_internal/VERSION b/tools/src/main/python/dlpx/virtualization/_internal/VERSION index 4e0e00e1..dd60b157 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/VERSION +++ b/tools/src/main/python/dlpx/virtualization/_internal/VERSION @@ -1 +1 @@ -3.0.0.dev0 \ No newline at end of file +3.0.0.dev1 \ No newline at end of file diff --git a/tools/src/main/python/dlpx/virtualization/_internal/commands/build.py b/tools/src/main/python/dlpx/virtualization/_internal/commands/build.py index b9185b2a..b0dbdcc6 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/commands/build.py +++ b/tools/src/main/python/dlpx/virtualization/_internal/commands/build.py @@ -23,6 +23,7 @@ DISCOVERY_DEFINITION_TYPE = 'PluginDiscoveryDefinition' STAGED_LINKED_SOURCE_TYPE = 'PluginLinkedStagedSourceDefinition' DIRECT_LINKED_SOURCE_TYPE = 'PluginLinkedDirectSourceDefinition' +SNAPSHOT_PARAMETERS_DEFINITION_TYPE = 'PluginSnapshotParametersDefinition' BUILD_DIR_NAME = 'build' @@ -207,8 +208,10 @@ def prepare_upload_artifact(plugin_config_content, src_dir, schemas, manifest): prepare_discovery_definition(plugin_config_content, schemas), 'snapshotSchema': schemas['snapshotDefinition'], - 'snapshotParametersDefinition': - schemas['snapshotParametersDefinition'], + 'snapshotParametersDefinition': { + 'type': SNAPSHOT_PARAMETERS_DEFINITION_TYPE, + 'schema': schemas['snapshotParametersDefinition'] + }, 'manifest': manifest } diff --git a/tools/src/main/python/dlpx/virtualization/_internal/settings.cfg b/tools/src/main/python/dlpx/virtualization/_internal/settings.cfg index c12e9e3b..93477cdc 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/settings.cfg +++ b/tools/src/main/python/dlpx/virtualization/_internal/settings.cfg @@ -20,7 +20,7 @@ # versions in those packages until they are shipped out of band. # [General] -engine_api_version = 1.11.3 +engine_api_version = 1.12.0 distribution_name = dvp-tools package_author = Delphix namespace_package = dlpx diff --git a/tools/src/test/python/dlpx/virtualization/_internal/conftest.py b/tools/src/test/python/dlpx/virtualization/_internal/conftest.py index 3df3afc4..92843f61 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/conftest.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/conftest.py @@ -619,12 +619,18 @@ def artifact_content(engine_api, virtual_source_definition, artifact['snapshotParametersDefinition'] = ( snapshot_parameters_definition) + if snapshot_parameters_definition: + artifact['snapshotParametersDefinition'] = { + 'type': 'PluginSnapshotParametersDefinition', + 'schema': snapshot_parameters_definition, + } + return artifact @pytest.fixture def engine_api(): - return {'type': 'APIVersion', 'major': 1, 'minor': 11, 'micro': 3} + return {'type': 'APIVersion', 'major': 1, 'minor': 12, 'micro': 0} @pytest.fixture diff --git a/tools/src/test/python/dlpx/virtualization/_internal/test_package_util.py b/tools/src/test/python/dlpx/virtualization/_internal/test_package_util.py index 063fb132..6785fa9e 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/test_package_util.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/test_package_util.py @@ -10,7 +10,7 @@ class TestPackageUtil: @staticmethod def test_get_version(): - assert package_util.get_version() == '3.0.0.dev0' + assert package_util.get_version() == '3.0.0.dev1' @staticmethod def test_get_virtualization_api_version(): @@ -18,7 +18,7 @@ def test_get_virtualization_api_version(): @staticmethod def test_get_engine_api_version(): - assert package_util.get_engine_api_version_from_settings() == '1.11.3' + assert package_util.get_engine_api_version_from_settings() == '1.12.0' @staticmethod def test_get_build_api_version_json(): @@ -35,8 +35,8 @@ def test_get_engine_api_version_json(): engine_api_version = { 'type': 'APIVersion', 'major': 1, - 'minor': 11, - 'micro': 3 + 'minor': 12, + 'micro': 0 } assert package_util.get_engine_api_version() == engine_api_version From 111176f7681d7be7d9b12a30ac303447032a669e Mon Sep 17 00:00:00 2001 From: Aleksandr Liber Date: Tue, 20 Oct 2020 01:51:35 -0700 Subject: [PATCH 03/12] Fixes #262 Add test_virtual_initialize_return_none --- .../python/dlpx/virtualization/test_plugin.py | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/platform/src/test/python/dlpx/virtualization/test_plugin.py b/platform/src/test/python/dlpx/virtualization/test_plugin.py index 078b54e7..8cfde728 100755 --- a/platform/src/test/python/dlpx/virtualization/test_plugin.py +++ b/platform/src/test/python/dlpx/virtualization/test_plugin.py @@ -705,6 +705,28 @@ def virtual_initialize_impl(virtual_source, repository): actual_source_config = initialize_response.return_value.source_config assert actual_source_config.parameters.json == TEST_REPOSITORY_JSON + @staticmethod + def test_virtual_initialize_return_none(my_plugin, virtual_source, + repository, source_config): + @my_plugin.virtual.initialize() + def virtual_initialize_impl(virtual_source, repository): + TestPlugin.assert_plugin_args(virtual_source=virtual_source, + repository=repository) + # Will return none. + + initialize_request = platform_pb2.InitializeRequest() + TestPlugin.setup_request(request=initialize_request, + virtual_source=virtual_source, + repository=repository) + + with pytest.raises(IncorrectReturnTypeError) as err_info: + my_plugin.virtual._internal_initialize(initialize_request) + message = err_info.value.message + assert message == ( + "The returned object for the virtual.initialize() operation was" + " type 'NoneType' but should be of class 'dlpx.virtualization." + "fake_generated_definitions.SourceConfigDefinition'.") + @staticmethod def test_virtual_mount_spec(my_plugin, virtual_source, repository): From bfa7c6d56c41cec94adbe0ea966aeb2b687869fd Mon Sep 17 00:00:00 2001 From: ankursarin Date: Mon, 2 Nov 2020 12:21:42 -0800 Subject: [PATCH 04/12] Update CODEOWNERS to be the current list of GKs (#264) --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index b7f9d4da..b9a5cc1a 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1 +1 @@ -* @ankursarin @fdrozdowski @nhlien93 @crystalplumage +* @ankursarin @nhlien93 @mothslaw From 8ba21a5ab3e4dddface3de393c8a539af931a114 Mon Sep 17 00:00:00 2001 From: Jeff Ngo <35587243+jeffngo@users.noreply.github.com> Date: Fri, 20 Nov 2020 17:35:03 -0800 Subject: [PATCH 05/12] Rollback engine api version to 1.11.6 for 6.0.6.0 backports #270 (#271) --- .bumpversion.cfg | 2 +- common/src/main/python/dlpx/virtualization/common/VERSION | 2 +- dvp/src/main/python/dlpx/virtualization/VERSION | 2 +- libs/src/main/python/dlpx/virtualization/libs/VERSION | 2 +- .../src/main/python/dlpx/virtualization/platform/VERSION | 2 +- .../src/main/python/dlpx/virtualization/_internal/VERSION | 2 +- .../python/dlpx/virtualization/_internal/settings.cfg | 2 +- .../test/python/dlpx/virtualization/_internal/conftest.py | 2 +- .../dlpx/virtualization/_internal/test_package_util.py | 8 ++++---- 9 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index cf3ae7f9..bf9cf6b8 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 3.0.0.dev1 +current_version = 3.0.0.dev2 commit = False tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+)(?P\d+))? diff --git a/common/src/main/python/dlpx/virtualization/common/VERSION b/common/src/main/python/dlpx/virtualization/common/VERSION index dd60b157..426be043 100644 --- a/common/src/main/python/dlpx/virtualization/common/VERSION +++ b/common/src/main/python/dlpx/virtualization/common/VERSION @@ -1 +1 @@ -3.0.0.dev1 \ No newline at end of file +3.0.0.dev2 \ No newline at end of file diff --git a/dvp/src/main/python/dlpx/virtualization/VERSION b/dvp/src/main/python/dlpx/virtualization/VERSION index dd60b157..426be043 100644 --- a/dvp/src/main/python/dlpx/virtualization/VERSION +++ b/dvp/src/main/python/dlpx/virtualization/VERSION @@ -1 +1 @@ -3.0.0.dev1 \ No newline at end of file +3.0.0.dev2 \ No newline at end of file diff --git a/libs/src/main/python/dlpx/virtualization/libs/VERSION b/libs/src/main/python/dlpx/virtualization/libs/VERSION index dd60b157..426be043 100644 --- a/libs/src/main/python/dlpx/virtualization/libs/VERSION +++ b/libs/src/main/python/dlpx/virtualization/libs/VERSION @@ -1 +1 @@ -3.0.0.dev1 \ No newline at end of file +3.0.0.dev2 \ No newline at end of file diff --git a/platform/src/main/python/dlpx/virtualization/platform/VERSION b/platform/src/main/python/dlpx/virtualization/platform/VERSION index dd60b157..426be043 100644 --- a/platform/src/main/python/dlpx/virtualization/platform/VERSION +++ b/platform/src/main/python/dlpx/virtualization/platform/VERSION @@ -1 +1 @@ -3.0.0.dev1 \ No newline at end of file +3.0.0.dev2 \ No newline at end of file diff --git a/tools/src/main/python/dlpx/virtualization/_internal/VERSION b/tools/src/main/python/dlpx/virtualization/_internal/VERSION index dd60b157..426be043 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/VERSION +++ b/tools/src/main/python/dlpx/virtualization/_internal/VERSION @@ -1 +1 @@ -3.0.0.dev1 \ No newline at end of file +3.0.0.dev2 \ No newline at end of file diff --git a/tools/src/main/python/dlpx/virtualization/_internal/settings.cfg b/tools/src/main/python/dlpx/virtualization/_internal/settings.cfg index 93477cdc..700afb91 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/settings.cfg +++ b/tools/src/main/python/dlpx/virtualization/_internal/settings.cfg @@ -20,7 +20,7 @@ # versions in those packages until they are shipped out of band. # [General] -engine_api_version = 1.12.0 +engine_api_version = 1.11.6 distribution_name = dvp-tools package_author = Delphix namespace_package = dlpx diff --git a/tools/src/test/python/dlpx/virtualization/_internal/conftest.py b/tools/src/test/python/dlpx/virtualization/_internal/conftest.py index 92843f61..66abfe4d 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/conftest.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/conftest.py @@ -630,7 +630,7 @@ def artifact_content(engine_api, virtual_source_definition, @pytest.fixture def engine_api(): - return {'type': 'APIVersion', 'major': 1, 'minor': 12, 'micro': 0} + return {'type': 'APIVersion', 'major': 1, 'minor': 11, 'micro': 6} @pytest.fixture diff --git a/tools/src/test/python/dlpx/virtualization/_internal/test_package_util.py b/tools/src/test/python/dlpx/virtualization/_internal/test_package_util.py index 6785fa9e..36e9669a 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/test_package_util.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/test_package_util.py @@ -10,7 +10,7 @@ class TestPackageUtil: @staticmethod def test_get_version(): - assert package_util.get_version() == '3.0.0.dev1' + assert package_util.get_version() == '3.0.0.dev2' @staticmethod def test_get_virtualization_api_version(): @@ -18,7 +18,7 @@ def test_get_virtualization_api_version(): @staticmethod def test_get_engine_api_version(): - assert package_util.get_engine_api_version_from_settings() == '1.12.0' + assert package_util.get_engine_api_version_from_settings() == '1.11.6' @staticmethod def test_get_build_api_version_json(): @@ -35,8 +35,8 @@ def test_get_engine_api_version_json(): engine_api_version = { 'type': 'APIVersion', 'major': 1, - 'minor': 12, - 'micro': 0 + 'minor': 11, + 'micro': 6 } assert package_util.get_engine_api_version() == engine_api_version From 9ed06b4f8e1585e8fd17acda44a97353f2e1cf75 Mon Sep 17 00:00:00 2001 From: ravi-cm <61520631+ravi-cm@users.noreply.github.com> Date: Fri, 4 Dec 2020 14:55:01 -0800 Subject: [PATCH 06/12] Fixes #53 Rename "name" to "plugin_id" and "prettyName" to "name" (#266) --- .bumpversion.cfg | 2 +- .../python/dlpx/virtualization/common/VERSION | 2 +- .../main/python/dlpx/virtualization/VERSION | 2 +- .../python/dlpx/virtualization/libs/VERSION | 2 +- .../dlpx/virtualization/platform/VERSION | 2 +- .../dlpx/virtualization/_internal/VERSION | 2 +- .../dlpx/virtualization/_internal/cli.py | 9 +-- .../_internal/commands/build.py | 18 ++--- .../_internal/delphix_client.py | 2 +- .../virtualization/_internal/plugin_util.py | 14 ++-- ...plugin_config_schema_no_id_validation.json | 67 ------------------- .../_internal/commands/test_build.py | 24 ++----- .../_internal/commands/test_delphix_client.py | 1 + .../dlpx/virtualization/_internal/conftest.py | 4 +- .../dlpx/virtualization/_internal/test_cli.py | 22 ------ .../_internal/test_package_util.py | 2 +- 16 files changed, 25 insertions(+), 150 deletions(-) delete mode 100644 tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_config_schema_no_id_validation.json diff --git a/.bumpversion.cfg b/.bumpversion.cfg index bf9cf6b8..4146fcf9 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 3.0.0.dev2 +current_version = 3.0.0.dev3 commit = False tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+)(?P\d+))? diff --git a/common/src/main/python/dlpx/virtualization/common/VERSION b/common/src/main/python/dlpx/virtualization/common/VERSION index 426be043..762c5b5a 100644 --- a/common/src/main/python/dlpx/virtualization/common/VERSION +++ b/common/src/main/python/dlpx/virtualization/common/VERSION @@ -1 +1 @@ -3.0.0.dev2 \ No newline at end of file +3.0.0.dev3 \ No newline at end of file diff --git a/dvp/src/main/python/dlpx/virtualization/VERSION b/dvp/src/main/python/dlpx/virtualization/VERSION index 426be043..762c5b5a 100644 --- a/dvp/src/main/python/dlpx/virtualization/VERSION +++ b/dvp/src/main/python/dlpx/virtualization/VERSION @@ -1 +1 @@ -3.0.0.dev2 \ No newline at end of file +3.0.0.dev3 \ No newline at end of file diff --git a/libs/src/main/python/dlpx/virtualization/libs/VERSION b/libs/src/main/python/dlpx/virtualization/libs/VERSION index 426be043..762c5b5a 100644 --- a/libs/src/main/python/dlpx/virtualization/libs/VERSION +++ b/libs/src/main/python/dlpx/virtualization/libs/VERSION @@ -1 +1 @@ -3.0.0.dev2 \ No newline at end of file +3.0.0.dev3 \ No newline at end of file diff --git a/platform/src/main/python/dlpx/virtualization/platform/VERSION b/platform/src/main/python/dlpx/virtualization/platform/VERSION index 426be043..762c5b5a 100644 --- a/platform/src/main/python/dlpx/virtualization/platform/VERSION +++ b/platform/src/main/python/dlpx/virtualization/platform/VERSION @@ -1 +1 @@ -3.0.0.dev2 \ No newline at end of file +3.0.0.dev3 \ No newline at end of file diff --git a/tools/src/main/python/dlpx/virtualization/_internal/VERSION b/tools/src/main/python/dlpx/virtualization/_internal/VERSION index 426be043..762c5b5a 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/VERSION +++ b/tools/src/main/python/dlpx/virtualization/_internal/VERSION @@ -1 +1 @@ -3.0.0.dev2 \ No newline at end of file +3.0.0.dev3 \ No newline at end of file diff --git a/tools/src/main/python/dlpx/virtualization/_internal/cli.py b/tools/src/main/python/dlpx/virtualization/_internal/cli.py index 227c3ff3..7d785591 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/cli.py +++ b/tools/src/main/python/dlpx/virtualization/_internal/cli.py @@ -188,18 +188,12 @@ def init(root, ingestion_strategy, name, host_type): mutually_exclusive=['upload_artifact'], help=('Only generate the Python classes from the schema definitions. ' 'Do not do a full build or create an upload artifact.')) -@click.option('--skip-id-validation', - is_flag=True, - hidden=True, - help=('An internal flag that does not enforce the format ' - 'of the id. Use of this flag is unsupported.')) @click.option('--dev', is_flag=True, hidden=True, help=('An internal flag that installs dev builds of the ' 'wrappers. This should only be used by SDK developers.')) -def build(plugin_config, upload_artifact, generate_only, skip_id_validation, - dev): +def build(plugin_config, upload_artifact, generate_only, dev): """ Build the plugin code and generate upload artifact file using the configuration provided in the plugin config file. @@ -225,7 +219,6 @@ def build(plugin_config, upload_artifact, generate_only, skip_id_validation, build_internal.build(plugin_config, upload_artifact, generate_only, - skip_id_validation, local_vsdk_root=local_vsdk_root) diff --git a/tools/src/main/python/dlpx/virtualization/_internal/commands/build.py b/tools/src/main/python/dlpx/virtualization/_internal/commands/build.py index b0dbdcc6..5a1fec2a 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/commands/build.py +++ b/tools/src/main/python/dlpx/virtualization/_internal/commands/build.py @@ -31,7 +31,6 @@ def build(plugin_config, upload_artifact, generate_only, - skip_id_validation, local_vsdk_root=None): """This builds the plugin using the configurations provided in config yaml file provided as input. It reads schemas and source code from the files @@ -42,7 +41,6 @@ def build(plugin_config, plugin_config: Plugin config file used for building plugin. upload_artifact: The file to which output of build is written to. generate_only: Only generate python classes from schema definitions. - skip_id_validation: Skip validation of the plugin id. local_vsdk_root: The local path to the root of the Virtualization SDK repository. """ @@ -57,7 +55,7 @@ def build(plugin_config, logger.info('Validating plugin config file %s', plugin_config) try: result = plugin_util.validate_plugin_config_file( - plugin_config, not generate_only, skip_id_validation) + plugin_config, not generate_only) except exceptions.UserError as err: raise exceptions.BuildFailedError(err) @@ -111,8 +109,7 @@ def build(plugin_config, try: result = plugin_util.get_plugin_manifest(plugin_config, plugin_config_content, - not generate_only, - skip_id_validation) + not generate_only) except (exceptions.UserError, exceptions.SDKToolingError) as err: raise exceptions.BuildFailedError(err) @@ -164,16 +161,9 @@ def prepare_upload_artifact(plugin_config_content, src_dir, schemas, manifest): # Hard code the type to a set default. 'type': TYPE, - # - # Delphix Engine still accepts only name and prettyName and - # hence name is mapped to id and prettyName to name. - # Delphix Engine does not accept upper case letters for name field, - # so we convert the name to lowercase letters. - # This will be changed as part of POST GA task PYT-628 - # + 'pluginId': + plugin_config_content['id'], 'name': - plugin_config_content['id'].lower(), - 'prettyName': plugin_config_content['name'], # set default value of locale to en-us 'defaultLocale': diff --git a/tools/src/main/python/dlpx/virtualization/_internal/delphix_client.py b/tools/src/main/python/dlpx/virtualization/_internal/delphix_client.py index 2ab521fa..03b5799e 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/delphix_client.py +++ b/tools/src/main/python/dlpx/virtualization/_internal/delphix_client.py @@ -188,7 +188,7 @@ def __get_plugin_ref_from_id(self, plugin_name, plugin_id): # and make sure the plugin hasn't been replicated. # The 'name' field will be converted to 'id' in the future. # - if p['name'] == plugin_id and p['namespace'] is None: + if p['identifier'] == plugin_id and p['namespace'] is None: return p['reference'] raise exceptions.MissingPluginError(plugin_name, self.__engine) diff --git a/tools/src/main/python/dlpx/virtualization/_internal/plugin_util.py b/tools/src/main/python/dlpx/virtualization/_internal/plugin_util.py index 1a210423..ca87f383 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/plugin_util.py +++ b/tools/src/main/python/dlpx/virtualization/_internal/plugin_util.py @@ -43,8 +43,7 @@ def validate_error_handler(plugin_file, validation_mode): def validate_plugin_config_file(plugin_config, - stop_build, - skip_id_validation=False): + stop_build): """ Reads a plugin config file and validates the contents using a pre-defined schema. If stop_build is True, will report exception @@ -55,10 +54,8 @@ def validate_plugin_config_file(plugin_config, """ validation_mode = (ValidationMode.ERROR if stop_build else ValidationMode.WARNING) - plugin_config_schema_file = (const.PLUGIN_CONFIG_SCHEMA_NO_ID_VALIDATION - if skip_id_validation else - const.PLUGIN_CONFIG_SCHEMA) - validator = PluginValidator(plugin_config, plugin_config_schema_file) + + validator = PluginValidator(plugin_config, const.PLUGIN_CONFIG_SCHEMA) with validate_error_handler(plugin_config, validation_mode): validator.validate_plugin_config() @@ -68,8 +65,7 @@ def validate_plugin_config_file(plugin_config, def get_plugin_manifest(plugin_config_file, plugin_config_content, - stop_build, - skip_id_validation=False): + stop_build): """ Validates the given plugin config content using a pre-defined schema. Plugin config file name is used to get the absolute path of plugin source @@ -115,7 +111,7 @@ def get_plugin_config_property(plugin_config_path, prop): """ Returns the value for a specific property from the plugin config file. """ - result = validate_plugin_config_file(plugin_config_path, False, False) + result = validate_plugin_config_file(plugin_config_path, False) return result.plugin_config_content[prop] diff --git a/tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_config_schema_no_id_validation.json b/tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_config_schema_no_id_validation.json deleted file mode 100644 index 32eb611f..00000000 --- a/tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_config_schema_no_id_validation.json +++ /dev/null @@ -1,67 +0,0 @@ -{ - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "name": { - "type": "string" - }, - "externalVersion": { - "type": "string" - }, - "hostTypes": { - "type": "array", - "items": { - "type": "string", - "enum": [ "UNIX", "WINDOWS" ] - }, - "maxItems": 1 - }, - "entryPoint": { - "type": "string", - "pattern": "^[^:]+:[^:]+$" - }, - "srcDir": { - "type": "string" - }, - "schemaFile": { - "type": "string" - }, - "manualDiscovery": { - "type": "boolean" - }, - "pluginType": { - "type": "string", - "enum": ["DIRECT", "STAGED"] - }, - "language": { - "type": "string", - "enum": ["PYTHON27"] - }, - "rootSquashEnabled": { - "type": "boolean" - }, - "defaultLocale": { - "type": "string", - "default": "en-us" - }, - "buildNumber": { - "type": "string", - "pattern": "^([0-9]+\\.)*[0-9]*[1-9][0-9]*(\\.[0-9]+)*$" - }, - "luaName": { - "type": "string", - "pattern": "^[a-z0-9_:-]+$" - }, - "extendedStartStopHooks" : { - "type": "boolean" - }, - "minimumLuaVersion": { - "type": "string", - "pattern": "^([0-9]+)\\.([0-9]+)$" - } - }, - "additionalProperties": false, - "required": ["id", "name", "hostTypes", "entryPoint", "srcDir", "schemaFile", "pluginType", "language", "buildNumber"] -} diff --git a/tools/src/test/python/dlpx/virtualization/_internal/commands/test_build.py b/tools/src/test/python/dlpx/virtualization/_internal/commands/test_build.py index 351d627a..fd88655d 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/commands/test_build.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/commands/test_build.py @@ -377,15 +377,12 @@ def test_zip_and_encode_source_files_encode_fail(mock_encode, src_dir): @mock.patch( 'dlpx.virtualization._internal.plugin_dependency_util.install_deps') @mock.patch('os.path.isabs', return_value=False) - @pytest.mark.parametrize(('plugin_id', 'skip_id_validation'), - [('77f18ce4-4425-4cd6-b9a7-23653254d660', False), - ('77f18ce4-4425-4cd6-b9a7-23653254d660', True), - ('mongo', True)]) + @pytest.mark.parametrize('plugin_id', + ['77f18ce4-4425-4cd6-b9a7-23653254d660']) def test_id_validation_positive(mock_relative_path, mock_install_deps, mock_import_plugin, plugin_config_file, - artifact_file, skip_id_validation): - build.build(plugin_config_file, artifact_file, False, - skip_id_validation) + artifact_file): + build.build(plugin_config_file, artifact_file, False) @staticmethod @mock.patch.object(PluginImporter, @@ -666,19 +663,6 @@ def test_manual_discovery_parameter(plugin_config_content, src_dir, assert expected == upload_artifact['discoveryDefinition'][ 'manualSourceConfigDiscovery'] - @staticmethod - def test_plugin_config_schemas_diff(): - with open(const.PLUGIN_CONFIG_SCHEMA) as f: - config_schema = json.load(f) - - with open(const.PLUGIN_CONFIG_SCHEMA_NO_ID_VALIDATION) as f: - config_schema_no_id = json.load(f) - - # Only the id's pattern should be different so remove it. - config_schema['properties']['id'].pop('pattern') - - assert config_schema == config_schema_no_id - @staticmethod @pytest.mark.parametrize('build_number, expected', [ pytest.param('0.0.1', '0.0.1'), diff --git a/tools/src/test/python/dlpx/virtualization/_internal/commands/test_delphix_client.py b/tools/src/test/python/dlpx/virtualization/_internal/commands/test_delphix_client.py index 6a9e158c..8396089a 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/commands/test_delphix_client.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/commands/test_delphix_client.py @@ -251,6 +251,7 @@ def httpretty_enabled(): '{"type": "Toolkit", "reference": "APPDATA_TOOLKIT-1",' '"namespace": null, "prettyName": "python_vfiles",' '"name": "16bef554-9470-11e9-b2e3-8c8590d4a42c", "language": "LUA",' + '"identifier": "16bef554-9470-11e9-b2e3-8c8590d4a42c",' '"version": "1.0.0", "buildApi": {"type": "APIVersion",' '"major": 1, "minor": 10, "micro": 5}, "hostTypes": [' '"UNIX", "WINDOWS"], "rootSquashEnabled": true, ' diff --git a/tools/src/test/python/dlpx/virtualization/_internal/conftest.py b/tools/src/test/python/dlpx/virtualization/_internal/conftest.py index 66abfe4d..d38a2b74 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/conftest.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/conftest.py @@ -577,8 +577,8 @@ def artifact_content(engine_api, virtual_source_definition, """ artifact = { 'type': 'Plugin', - 'name': '16bef554-9470-11e9-b2e3-8c8590d4a42c', - 'prettyName': 'python_vfiles', + 'pluginId': '16bef554-9470-11e9-b2e3-8c8590d4a42c', + 'name': 'python_vfiles', 'externalVersion': '2.0.0', 'defaultLocale': 'en-us', 'language': 'PYTHON27', diff --git a/tools/src/test/python/dlpx/virtualization/_internal/test_cli.py b/tools/src/test/python/dlpx/virtualization/_internal/test_cli.py index 9ef87f31..4e8e215a 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/test_cli.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/test_cli.py @@ -268,7 +268,6 @@ def test_default_plugin_file_success(mock_build, plugin_config_filename, mock_build.assert_called_once_with(plugin_config_file, artifact_file, False, - False, local_vsdk_root=None) @staticmethod @@ -290,7 +289,6 @@ def test_generate_only_success(mock_build, plugin_config_filename, mock_build.assert_called_once_with(plugin_config_file, None, True, - False, local_vsdk_root=None) @staticmethod @@ -305,7 +303,6 @@ def test_valid_params(mock_build, plugin_config_file, artifact_file): mock_build.assert_called_once_with(plugin_config_file, artifact_file, False, - False, local_vsdk_root=None) @staticmethod @@ -322,24 +319,6 @@ def test_valid_params_new_name(mock_build, plugin_config_file, os.path.join( os.getcwd(), artifact_filename), False, - False, - local_vsdk_root=None) - - @staticmethod - @mock.patch('dlpx.virtualization._internal.commands.build.build') - def test_skip_id_validation(mock_build, plugin_config_file, artifact_file): - runner = click_testing.CliRunner() - - result = runner.invoke(cli.delphix_sdk, [ - 'build', '-c', plugin_config_file, '-a', artifact_file, - '--skip-id-validation' - ]) - - assert result.exit_code == 0, 'Output: {}'.format(result.output) - mock_build.assert_called_once_with(plugin_config_file, - artifact_file, - False, - True, local_vsdk_root=None) @staticmethod @@ -403,7 +382,6 @@ def test_with_dev(mock_build, plugin_config_file, artifact_file, mock_build.assert_called_once_with(plugin_config_file, artifact_file, False, - False, local_vsdk_root='/path/to/vsdk/dir') @staticmethod diff --git a/tools/src/test/python/dlpx/virtualization/_internal/test_package_util.py b/tools/src/test/python/dlpx/virtualization/_internal/test_package_util.py index 36e9669a..e962ea62 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/test_package_util.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/test_package_util.py @@ -10,7 +10,7 @@ class TestPackageUtil: @staticmethod def test_get_version(): - assert package_util.get_version() == '3.0.0.dev2' + assert package_util.get_version() == '3.0.0.dev3' @staticmethod def test_get_virtualization_api_version(): From 707d0de42e843d2f8673d6e27df035e734387bba Mon Sep 17 00:00:00 2001 From: Jeff Ngo <35587243+jeffngo@users.noreply.github.com> Date: Wed, 16 Dec 2020 15:40:21 -0800 Subject: [PATCH 07/12] Publishing docs to GitHub pages fails because set-env has been deprecated (#288) --- .github/workflows/publish-docs.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/publish-docs.yml b/.github/workflows/publish-docs.yml index d9d09d48..03af1009 100644 --- a/.github/workflows/publish-docs.yml +++ b/.github/workflows/publish-docs.yml @@ -33,8 +33,9 @@ jobs: run: | CURRENT_BRANCH_VAR=${GITHUB_REF#refs/heads/} echo "Current branch: $CURRENT_BRANCH_VAR" - # Set CURRENT_BRANCH environment variable to the current branch name. - echo "::set-env name=CURRENT_BRANCH::$(echo $CURRENT_BRANCH_VAR)" + # Set CURRENT_BRANCH environment variable to the current branch name. Refrain from using 'set-env' as + # GitHub has identified the command as a moderate security vulnerability. + echo "CURRENT_BRANCH=$(echo $CURRENT_BRANCH_VAR)" >> $GITHUB_ENV - name: Display all remote branches working-directory: ${{ matrix.package }} run: | @@ -46,8 +47,9 @@ jobs: # Get only docs branches, extract the "docs/x.y.z" part, sort them in descending order, and get the first one. LATEST_DOCS_BRANCH_VAR=$(git branch -r | grep -e ".*\/*docs\/[0-9].[0-9].[0-9]" | sed -n "s/.*\/*\(docs\/[0-9].[0-9].[0-9]\).*/\1/p" | sort -r | head -n 1) echo "Latest docs branch: $LATEST_DOCS_BRANCH_VAR" - # Set the LATEST_DOCS_BRANCH environment variable. - echo "::set-env name=LATEST_DOCS_BRANCH::$(echo $LATEST_DOCS_BRANCH_VAR)" + # Set the LATEST_DOCS_BRANCH environment variable. Refrain from using 'set-env' as GitHub has identified the + # command as a moderate security vulnerability. + echo "LATEST_DOCS_BRANCH=$(echo $LATEST_DOCS_BRANCH_VAR)" >> $GITHUB_ENV - name: Check that the current branch is the latest docs branch, fail otherwise working-directory: ${{ matrix.package }} run: | From 6cd64ef4abf42ed479c7249b5fddf3691512de85 Mon Sep 17 00:00:00 2001 From: Jeff Ngo <35587243+jeffngo@users.noreply.github.com> Date: Wed, 6 Jan 2021 09:33:00 -0800 Subject: [PATCH 08/12] Bump dvp-api and VSDK versions to release formats (#291) --- .bumpversion.cfg | 2 +- common/setup.py | 2 +- common/src/main/python/dlpx/virtualization/common/VERSION | 2 +- dvp/src/main/python/dlpx/virtualization/VERSION | 2 +- libs/setup.py | 2 +- libs/src/main/python/dlpx/virtualization/libs/VERSION | 2 +- platform/setup.py | 2 +- platform/src/main/python/dlpx/virtualization/platform/VERSION | 2 +- tools/src/main/python/dlpx/virtualization/_internal/VERSION | 2 +- .../python/dlpx/virtualization/_internal/test_package_util.py | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 4146fcf9..9e9dedd4 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 3.0.0.dev3 +current_version = 3.0.0 commit = False tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+)(?P\d+))? diff --git a/common/setup.py b/common/setup.py index 985b4ba3..7d9f0244 100644 --- a/common/setup.py +++ b/common/setup.py @@ -4,7 +4,7 @@ PYTHON_SRC = 'src/main/python' install_requires = [ - "dvp-api == 1.4.0.dev12", + "dvp-api == 1.4.0", ] with open(os.path.join(PYTHON_SRC, 'dlpx/virtualization/common/VERSION')) as version_file: diff --git a/common/src/main/python/dlpx/virtualization/common/VERSION b/common/src/main/python/dlpx/virtualization/common/VERSION index 762c5b5a..56fea8a0 100644 --- a/common/src/main/python/dlpx/virtualization/common/VERSION +++ b/common/src/main/python/dlpx/virtualization/common/VERSION @@ -1 +1 @@ -3.0.0.dev3 \ No newline at end of file +3.0.0 \ No newline at end of file diff --git a/dvp/src/main/python/dlpx/virtualization/VERSION b/dvp/src/main/python/dlpx/virtualization/VERSION index 762c5b5a..56fea8a0 100644 --- a/dvp/src/main/python/dlpx/virtualization/VERSION +++ b/dvp/src/main/python/dlpx/virtualization/VERSION @@ -1 +1 @@ -3.0.0.dev3 \ No newline at end of file +3.0.0 \ No newline at end of file diff --git a/libs/setup.py b/libs/setup.py index e59da92d..d7c601f8 100644 --- a/libs/setup.py +++ b/libs/setup.py @@ -7,7 +7,7 @@ version = version_file.read().strip() install_requires = [ - "dvp-api == 1.4.0.dev12", + "dvp-api == 1.4.0", "dvp-common == {}".format(version) ] diff --git a/libs/src/main/python/dlpx/virtualization/libs/VERSION b/libs/src/main/python/dlpx/virtualization/libs/VERSION index 762c5b5a..56fea8a0 100644 --- a/libs/src/main/python/dlpx/virtualization/libs/VERSION +++ b/libs/src/main/python/dlpx/virtualization/libs/VERSION @@ -1 +1 @@ -3.0.0.dev3 \ No newline at end of file +3.0.0 \ No newline at end of file diff --git a/platform/setup.py b/platform/setup.py index 5944d1e5..5c63a776 100644 --- a/platform/setup.py +++ b/platform/setup.py @@ -7,7 +7,7 @@ version = version_file.read().strip() install_requires = [ - "dvp-api == 1.4.0.dev12", + "dvp-api == 1.4.0", "dvp-common == {}".format(version), "enum34;python_version < '3.4'", ] diff --git a/platform/src/main/python/dlpx/virtualization/platform/VERSION b/platform/src/main/python/dlpx/virtualization/platform/VERSION index 762c5b5a..56fea8a0 100644 --- a/platform/src/main/python/dlpx/virtualization/platform/VERSION +++ b/platform/src/main/python/dlpx/virtualization/platform/VERSION @@ -1 +1 @@ -3.0.0.dev3 \ No newline at end of file +3.0.0 \ No newline at end of file diff --git a/tools/src/main/python/dlpx/virtualization/_internal/VERSION b/tools/src/main/python/dlpx/virtualization/_internal/VERSION index 762c5b5a..56fea8a0 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/VERSION +++ b/tools/src/main/python/dlpx/virtualization/_internal/VERSION @@ -1 +1 @@ -3.0.0.dev3 \ No newline at end of file +3.0.0 \ No newline at end of file diff --git a/tools/src/test/python/dlpx/virtualization/_internal/test_package_util.py b/tools/src/test/python/dlpx/virtualization/_internal/test_package_util.py index e962ea62..670eb684 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/test_package_util.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/test_package_util.py @@ -10,7 +10,7 @@ class TestPackageUtil: @staticmethod def test_get_version(): - assert package_util.get_version() == '3.0.0.dev3' + assert package_util.get_version() == '3.0.0' @staticmethod def test_get_virtualization_api_version(): From e297d965a729f9e198dd448429ccca6587fe2ba7 Mon Sep 17 00:00:00 2001 From: Jeff Ngo <35587243+jeffngo@users.noreply.github.com> Date: Tue, 12 Jan 2021 11:07:33 -0800 Subject: [PATCH 09/12] Bump Virtualization SDK to post release format on develop (#304) --- .bumpversion.cfg | 2 +- common/src/main/python/dlpx/virtualization/common/VERSION | 2 +- dvp/src/main/python/dlpx/virtualization/VERSION | 2 +- libs/src/main/python/dlpx/virtualization/libs/VERSION | 2 +- platform/src/main/python/dlpx/virtualization/platform/VERSION | 2 +- tools/src/main/python/dlpx/virtualization/_internal/VERSION | 2 +- .../python/dlpx/virtualization/_internal/test_package_util.py | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 9e9dedd4..0fabc172 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 3.0.0 +current_version = 3.1.0.dev1 commit = False tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+)(?P\d+))? diff --git a/common/src/main/python/dlpx/virtualization/common/VERSION b/common/src/main/python/dlpx/virtualization/common/VERSION index 56fea8a0..e7a40bd7 100644 --- a/common/src/main/python/dlpx/virtualization/common/VERSION +++ b/common/src/main/python/dlpx/virtualization/common/VERSION @@ -1 +1 @@ -3.0.0 \ No newline at end of file +3.1.0.dev1 diff --git a/dvp/src/main/python/dlpx/virtualization/VERSION b/dvp/src/main/python/dlpx/virtualization/VERSION index 56fea8a0..e7a40bd7 100644 --- a/dvp/src/main/python/dlpx/virtualization/VERSION +++ b/dvp/src/main/python/dlpx/virtualization/VERSION @@ -1 +1 @@ -3.0.0 \ No newline at end of file +3.1.0.dev1 diff --git a/libs/src/main/python/dlpx/virtualization/libs/VERSION b/libs/src/main/python/dlpx/virtualization/libs/VERSION index 56fea8a0..e7a40bd7 100644 --- a/libs/src/main/python/dlpx/virtualization/libs/VERSION +++ b/libs/src/main/python/dlpx/virtualization/libs/VERSION @@ -1 +1 @@ -3.0.0 \ No newline at end of file +3.1.0.dev1 diff --git a/platform/src/main/python/dlpx/virtualization/platform/VERSION b/platform/src/main/python/dlpx/virtualization/platform/VERSION index 56fea8a0..e7a40bd7 100644 --- a/platform/src/main/python/dlpx/virtualization/platform/VERSION +++ b/platform/src/main/python/dlpx/virtualization/platform/VERSION @@ -1 +1 @@ -3.0.0 \ No newline at end of file +3.1.0.dev1 diff --git a/tools/src/main/python/dlpx/virtualization/_internal/VERSION b/tools/src/main/python/dlpx/virtualization/_internal/VERSION index 56fea8a0..e7a40bd7 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/VERSION +++ b/tools/src/main/python/dlpx/virtualization/_internal/VERSION @@ -1 +1 @@ -3.0.0 \ No newline at end of file +3.1.0.dev1 diff --git a/tools/src/test/python/dlpx/virtualization/_internal/test_package_util.py b/tools/src/test/python/dlpx/virtualization/_internal/test_package_util.py index 670eb684..225d1b2d 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/test_package_util.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/test_package_util.py @@ -10,7 +10,7 @@ class TestPackageUtil: @staticmethod def test_get_version(): - assert package_util.get_version() == '3.0.0' + assert package_util.get_version() == '3.1.0.dev1' @staticmethod def test_get_virtualization_api_version(): From 6acfa729bbe00515defc233c6ac090fee4275431 Mon Sep 17 00:00:00 2001 From: Jeff Ngo <35587243+jeffngo@users.noreply.github.com> Date: Tue, 12 Jan 2021 16:23:30 -0800 Subject: [PATCH 10/12] Virtualization SDK Docs - New plugin config property snapshotParametersDefinition (#302) --- docs/docs/References/Glossary.md | 6 +-- docs/docs/References/Schemas.md | 1 + .../Schemas_and_Autogenerated_Classes.md | 37 +++++++++++++++++++ docs/docs/Release_Notes/.pages | 1 + docs/docs/Release_Notes/3.0.0/.pages | 3 ++ docs/docs/Release_Notes/3.0.0/3.0.0.md | 13 +++++++ .../3.0.0/3.0.0_Breaking_Changes.md | 36 ++++++++++++++++++ docs/mkdocs.yml | 5 ++- 8 files changed, 97 insertions(+), 5 deletions(-) create mode 100644 docs/docs/Release_Notes/3.0.0/.pages create mode 100644 docs/docs/Release_Notes/3.0.0/3.0.0.md create mode 100644 docs/docs/Release_Notes/3.0.0/3.0.0_Breaking_Changes.md diff --git a/docs/docs/References/Glossary.md b/docs/docs/References/Glossary.md index 3d439050..c6150997 100644 --- a/docs/docs/References/Glossary.md +++ b/docs/docs/References/Glossary.md @@ -88,10 +88,10 @@ Information that represents a set of dependencies that a dataset requires in ord A formal description of a data type. Plugins use JSON format for their [schemas](Schemas_and_Autogenerated_Classes.md#schemas-and-autogenerated-classes). ## Snapshot -A point-in-time read-only copy of a dataset. A snapshot includes associated metadata represented by the [SnapshotDefinition Schema](Schemas_and_Autogenerated_Classes.md#snapshotdefinition) +A point-in-time read-only copy of a dataset. A snapshot includes associated metadata represented by the [SnapshotDefinition Schema](Schemas_and_Autogenerated_Classes.md#snapshotdefinition). -## Snapshot Parameter -User provided parameters for the snapshot operation. Currently the only properties the parameter has is resync. +## Snapshot Parameters +User provided parameters for the snapshot operation which can be defined in a [Snapshot Parameters Definition](Schemas_and_Autogenerated_Classes.md#snapshotparametersdefinition). ## Source Config A collection of information that the Delphix Engine needs to interact with a dataset (whether [linked](#linked-dataset) or [virtual](#virtual-dataset) on an [environment](#environment). diff --git a/docs/docs/References/Schemas.md b/docs/docs/References/Schemas.md index 8411e39e..472d6f32 100644 --- a/docs/docs/References/Schemas.md +++ b/docs/docs/References/Schemas.md @@ -22,6 +22,7 @@ Delphix Object | Schema | Autogenerated Class [Linked Source](Glossary.md#linked-source) | [LinkedSourceDefinition](Schemas_and_Autogenerated_Classes.md#linkedsourcedefinition-schema) | [LinkedSourceDefinition](Schemas_and_Autogenerated_Classes.md#linkedsourcedefinition-class) [Virtual Source](Glossary.md#virtual-source) | [VirtualSourceDefinition](Schemas_and_Autogenerated_Classes.md#virtualsourcedefinition-schema)| [VirtualSourceDefinition](Schemas_and_Autogenerated_Classes.md#virtualsourcedefinition-class) [Snapshot](Glossary.md#linked-source) | [SnapshotDefinition](Schemas_and_Autogenerated_Classes.md#snapshotdefinition-schema) | [SnapshotDefinition](Schemas_and_Autogenerated_Classes.md#snapshotdefinition-class) +[Snapshot Parameters](Glossary.md#snapshot-parameters) | [SnapshotParametersDefinition](Schemas_and_Autogenerated_Classes.md#snapshotparametersdefinition-schema) | [SnapshotParametersDefinition](Schemas_and_Autogenerated_Classes.md#snapshotparametersdefinition-class) ## JSON Schemas diff --git a/docs/docs/References/Schemas_and_Autogenerated_Classes.md b/docs/docs/References/Schemas_and_Autogenerated_Classes.md index f5ac155b..ba0a97e7 100644 --- a/docs/docs/References/Schemas_and_Autogenerated_Classes.md +++ b/docs/docs/References/Schemas_and_Autogenerated_Classes.md @@ -230,3 +230,40 @@ snapshot2.transaction_id = 1500 # A snapshot that omits the optional "transaction_id" property snapshot3 = SnapshotDefinition(version="1.0.0") ``` + +## SnapshotParametersDefinition + +Defines [Snapshot Parameters](Glossary.md#snapshot-parameters) for the snapshot operation. + +### SnapshotParametersDefinition Schema + +```json +{ + "type": "object", + "required": ["resync"], + "additionalProperties": false, + "properties": { + "resync": { "type": "boolean" } + } +} +``` + +### SnapshotParametersDefinition Class + +Autogenerated based on the [Snapshot Parameters Definition Schema](#snapshotparametersdefinition-schema). + +```python +class SnapshotParametersDefinition: + + def __init__(self, resync): + self._inner_dict = { "resync": resync } +``` + +> To use the class: + +```python +from generated.defintions import SnapshotParametersDefinition + +# Since "resync" is required, it must be specified when constructing the object +snapshot_parameter_definition = SnapshotParametersDefinition(resync=True) +``` diff --git a/docs/docs/Release_Notes/.pages b/docs/docs/Release_Notes/.pages index d160c92f..f123a584 100644 --- a/docs/docs/Release_Notes/.pages +++ b/docs/docs/Release_Notes/.pages @@ -1,4 +1,5 @@ arrange: + - 3.0.0 - 2.1.0 - 2.0.0 - 1.0.0 diff --git a/docs/docs/Release_Notes/3.0.0/.pages b/docs/docs/Release_Notes/3.0.0/.pages new file mode 100644 index 00000000..cd4759ab --- /dev/null +++ b/docs/docs/Release_Notes/3.0.0/.pages @@ -0,0 +1,3 @@ +arrange: + - 3.0.0.md + - 3.0.0_Breaking_Changes.md diff --git a/docs/docs/Release_Notes/3.0.0/3.0.0.md b/docs/docs/Release_Notes/3.0.0/3.0.0.md new file mode 100644 index 00000000..f6854791 --- /dev/null +++ b/docs/docs/Release_Notes/3.0.0/3.0.0.md @@ -0,0 +1,13 @@ +# Release - v3.0.0 + +To install or upgrade the SDK, refer to instructions [here](/Getting_Started.md#installation). + +## New & Improved +* Added a `scratch_path` property on the [RemoteHost](/References/Classes/#remotehost) object for storage and debugging purposes. +More details about `scratch_path` can be found [here](/Best_Practices/Scratch_Paths.md) + +## Breaking Changes + +* Added a new required schema [Snapshot Parameters Definition](/References/Schemas_and_Autogenerated_Classes.md#snapshotparametersdefinition-schema). + +[**For more information and detailed steps to detect and make changes.**](/Release_Notes/3.0.0/3.0.0_Breaking_Changes#new-required-schema) \ No newline at end of file diff --git a/docs/docs/Release_Notes/3.0.0/3.0.0_Breaking_Changes.md b/docs/docs/Release_Notes/3.0.0/3.0.0_Breaking_Changes.md new file mode 100644 index 00000000..b0c13762 --- /dev/null +++ b/docs/docs/Release_Notes/3.0.0/3.0.0_Breaking_Changes.md @@ -0,0 +1,36 @@ +# Breaking Changes - v.3.0.0 + +## New required schema +[Snapshot Parameters Definition](/References/Schemas_and_Autogenerated_Classes.md#snapshotparametersdefinition-schema) allows plugin authors to define [Snapshot Parameters] which can be displayed to an end-user whenever a linked source snapshot is taken. + +### What is affected +All plugins built with v2.0.0 or below will be affected. The [Schema](/References/Schemas) must contain a `snapshotParametersDefinition`. + +### How does it fail +[dvp build](/References/CLI.md#build) will fail with the following error message if the [SnapshotParametersDefinition Schema](/References/Schemas_and_Autogenerated_Classes.md#snapshotparametersdefinition-schema) is not added to the schema file: + +```bash +$ dvp build +Error: 'snapshotParametersDefinition' is a required property on ['required'] + +Validation failed on /private/var/tmp/fp/schemas/schema.json. +0 Warning(s). 1 Error(s) + +BUILD FAILED. +``` + +### How to fix it +Add a [SnapshotParametersDefinition Schema](/References/Schemas_and_Autogenerated_Classes.md#snapshotparametersdefinition-schema) to the `schemaFile` defined in the [Plugin Config](/References/Plugin_Config). + +Example: +```python +"snapshotParametersDefinition": { + "additionalProperties": false, + "properties": { + "resync": { + "type": "boolean" + } + }, + "type": "object" +} +``` diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 75fd7ad4..d1799bc9 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -1,4 +1,4 @@ -site_name: Delphix Virtualization SDK 2.1.0 +site_name: Delphix Virtualization SDK 3.0.0 theme: name: material custom_dir: 'material/' @@ -10,8 +10,9 @@ theme: font: text: Helvetica Neue code: Ubuntu Mono + feature: -copyright: Copyright © 2019 Delphix Corp. +copyright: Copyright © 2021 Delphix Corp. google_analytics: - 'UA-35429885-3' From 556d6bad68344e7d9700ec54d29e17942776c38d Mon Sep 17 00:00:00 2001 From: Jeff Ngo <35587243+jeffngo@users.noreply.github.com> Date: Wed, 20 Jan 2021 10:47:52 -0800 Subject: [PATCH 11/12] Revert "Bump Virtualization SDK to post release format on develop (#304)" (#314) --- .bumpversion.cfg | 2 +- common/src/main/python/dlpx/virtualization/common/VERSION | 2 +- dvp/src/main/python/dlpx/virtualization/VERSION | 2 +- libs/src/main/python/dlpx/virtualization/libs/VERSION | 2 +- platform/src/main/python/dlpx/virtualization/platform/VERSION | 2 +- tools/src/main/python/dlpx/virtualization/_internal/VERSION | 2 +- .../python/dlpx/virtualization/_internal/test_package_util.py | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 0fabc172..9e9dedd4 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 3.1.0.dev1 +current_version = 3.0.0 commit = False tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+)(?P\d+))? diff --git a/common/src/main/python/dlpx/virtualization/common/VERSION b/common/src/main/python/dlpx/virtualization/common/VERSION index e7a40bd7..56fea8a0 100644 --- a/common/src/main/python/dlpx/virtualization/common/VERSION +++ b/common/src/main/python/dlpx/virtualization/common/VERSION @@ -1 +1 @@ -3.1.0.dev1 +3.0.0 \ No newline at end of file diff --git a/dvp/src/main/python/dlpx/virtualization/VERSION b/dvp/src/main/python/dlpx/virtualization/VERSION index e7a40bd7..56fea8a0 100644 --- a/dvp/src/main/python/dlpx/virtualization/VERSION +++ b/dvp/src/main/python/dlpx/virtualization/VERSION @@ -1 +1 @@ -3.1.0.dev1 +3.0.0 \ No newline at end of file diff --git a/libs/src/main/python/dlpx/virtualization/libs/VERSION b/libs/src/main/python/dlpx/virtualization/libs/VERSION index e7a40bd7..56fea8a0 100644 --- a/libs/src/main/python/dlpx/virtualization/libs/VERSION +++ b/libs/src/main/python/dlpx/virtualization/libs/VERSION @@ -1 +1 @@ -3.1.0.dev1 +3.0.0 \ No newline at end of file diff --git a/platform/src/main/python/dlpx/virtualization/platform/VERSION b/platform/src/main/python/dlpx/virtualization/platform/VERSION index e7a40bd7..56fea8a0 100644 --- a/platform/src/main/python/dlpx/virtualization/platform/VERSION +++ b/platform/src/main/python/dlpx/virtualization/platform/VERSION @@ -1 +1 @@ -3.1.0.dev1 +3.0.0 \ No newline at end of file diff --git a/tools/src/main/python/dlpx/virtualization/_internal/VERSION b/tools/src/main/python/dlpx/virtualization/_internal/VERSION index e7a40bd7..56fea8a0 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/VERSION +++ b/tools/src/main/python/dlpx/virtualization/_internal/VERSION @@ -1 +1 @@ -3.1.0.dev1 +3.0.0 \ No newline at end of file diff --git a/tools/src/test/python/dlpx/virtualization/_internal/test_package_util.py b/tools/src/test/python/dlpx/virtualization/_internal/test_package_util.py index 225d1b2d..670eb684 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/test_package_util.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/test_package_util.py @@ -10,7 +10,7 @@ class TestPackageUtil: @staticmethod def test_get_version(): - assert package_util.get_version() == '3.1.0.dev1' + assert package_util.get_version() == '3.0.0' @staticmethod def test_get_virtualization_api_version(): From 7d95126cd6891d73fc91a9566437b815b63a349c Mon Sep 17 00:00:00 2001 From: Jeff Ngo <35587243+jeffngo@users.noreply.github.com> Date: Thu, 21 Jan 2021 08:20:59 -0800 Subject: [PATCH 12/12] Virtualization SDK Docs - Final VSDK 3.0.0 docs and release changes (#311) (#312) --- .../Data_Ingestion.md | 6 +- docs/docs/References/Classes.md | 23 ------ docs/docs/References/Plugin_Operations.md | 22 ++--- docs/docs/Release_Notes/3.0.0/3.0.0.md | 18 ++++- .../3.0.0/3.0.0_Breaking_Changes.md | 80 ++++++++++++++++++- 5 files changed, 110 insertions(+), 39 deletions(-) diff --git a/docs/docs/Building_Your_First_Plugin/Data_Ingestion.md b/docs/docs/Building_Your_First_Plugin/Data_Ingestion.md index 87c04dfa..a3aaf6d3 100644 --- a/docs/docs/Building_Your_First_Plugin/Data_Ingestion.md +++ b/docs/docs/Building_Your_First_Plugin/Data_Ingestion.md @@ -21,7 +21,7 @@ quite limiting. For our first plugin, we will be using the more flexible [staging](/References/Glossary.md#staged-linkingsyncing) strategy. With this strategy, the Delphix Engine uses NFS for Unix environments (or iSCSI on Windows environments) to mount storage onto a [staging environment](/References/Glossary.md#staging-environment). Our plugin will then be in full control of how to get data from the source environment onto this storage mount. -With the staging strategy, there are two types of syncs: sync and resync. A `sync` is used to ingestion incremental changes while a `resync` is used to re-ingest all the data for the dSource. For databases, this could mean re-ingesting from a full database backup to reset the dSource. A `sync` and a `resync` execute the same plugin operations and are differentiated by a boolean flag in the [snapshot_parameters](/References/Classes.md#snapshotparametersdefinition) argument passed into [linked.pre_snapshot](/References/Plugin_Operations.md#staged-linked-source-pre-snapshot) and [linked.post_snapshot](/References/Plugin_Operations.md#staged-linked-source-post-snapshot). +With the staging strategy, there are two types of syncs: sync and resync. A `sync` is used to ingest incremental changes while a `resync` is used to re-ingest all the data for the dSource. For databases, this could mean re-ingesting from a full database backup to reset the dSource. A `sync` and a `resync` will execute the same plugin operations. To differentiate a `sync` from a `resync`, simply add a boolean property (i.e. `resync`) in the plugin's [snapshot parameters definition](References/Schemas_and_Autogenerated_Classes.md#snapshotparametersdefinition-schema). Once `sync` or `resync` is selected, the property will be passed into [linked.pre_snapshot](/References/Plugin_Operations.md#staged-linked-source-pre-snapshot) and [linked.post_snapshot](/References/Plugin_Operations.md#staged-linked-source-post-snapshot) as a [snapshot parameter](/References/Glossary.md#snapshot-parameters). A regular `sync` is the default and is executed as part of policy driven syncs. A `resync` is only executed during initial ingestion or if the Delphix user manually starts one. The customer can manually trigger a `resync` via the UI by selecting the dSource, going to more options and selecting **Resynchronize dSource**. ![Screenshot](images/Resync.png) @@ -168,7 +168,7 @@ Next, we'll add a new function: ```python @plugin.linked.pre_snapshot() -def copy_data_from_source(staged_source, repository, source_config, snapshot_parameters): +def copy_data_from_source(staged_source, repository, source_config, optional_snapshot_parameters): stage_mount_path = staged_source.mount.mount_path data_location = "{}@{}:{}".format(staged_source.parameters.username, staged_source.parameters.source_address, @@ -249,7 +249,7 @@ In fact, the default implementation that was generated by `dvp init` will work j def linked_post_snapshot(staged_source, repository, source_config, - snapshot_parameters): + optional_snapshot_parameters): return SnapshotDefinition() ``` diff --git a/docs/docs/References/Classes.md b/docs/docs/References/Classes.md index 07097a22..afcd2850 100644 --- a/docs/docs/References/Classes.md +++ b/docs/docs/References/Classes.md @@ -144,29 +144,6 @@ Field | Type | Description mounts | list[[Mount](#mount)] | The list of mounts to export the data sets to. ownership_specification | [OwnershipSpecification](#ownershipspecification) | **Optional.** Control the ownership attributes for the data set. It defaults to the environment user of the remote environment if it is not specified. -## SnapshotParametersDefinition - -User provided parameters for the snapshot operation. It includes a boolean property named `resync` that can be used to indicate to the plugin whether or not to initiate a full ingestion of the dSource. The parameters are only set during a manual snapshot. When using a sync policy, `resync` defaults to `false`. - -```python -from dlpx.virtualization.platform import Plugin - -plugin = Plugin() - -@plugin.linked.pre_snapshot() -def linked_pre_snapshot(staged_source, repository, source_config, snapshot_parameters): - if snapshot_parameter.resync: - print(snapshot_parameter.resync) -``` - -> This class will be generated during build and is located with the autogenerated classes. As it is passed into the operation, importing it is not neccessary. - -### Fields - -Field | Type | Description ------ | ---- | ----------- -resync | Boolean | Determines if this snapshot should ingest the dSource from scratch. - ## RemoteEnvironment Represents a remote environment. diff --git a/docs/docs/References/Plugin_Operations.md b/docs/docs/References/Plugin_Operations.md index 2fe76756..e02e3773 100644 --- a/docs/docs/References/Plugin_Operations.md +++ b/docs/docs/References/Plugin_Operations.md @@ -177,7 +177,7 @@ Sets up a [dSource](Glossary.md#dsource) to ingest data. Only applies when using ### Signature -`def linked_pre_snapshot(direct_source, repository, source_config)` +`def linked_pre_snapshot(direct_source, repository, source_config, optional_snapshot_parameters)` ### Decorator @@ -190,6 +190,7 @@ Argument | Type | Description direct_source | [DirectSource](Classes.md#directsource) | The source associated with this operation. repository | [RepositoryDefinition](Schemas_and_Autogenerated_Classes.md#repositorydefinition-class) | The repository associated with this source. source_config | [SourceConfigDefinition](Schemas_and_Autogenerated_Classes.md#sourceconfigdefinition-class) | The source config associated with this source. +optional_snapshot_parameters | [SnapshotParametersDefinition](Classes.md#snapshotparametersdefinition) | The snapshot parameters. The value is `None` when executed during a snapshot policy. ### Returns None @@ -220,7 +221,7 @@ Captures metadata from a [dSource](Glossary.md#dsource) once data has been inges ### Signature -`def linked_post_snapshot(direct_source, repository, source_config)` +`def linked_post_snapshot(direct_source, repository, source_config, optional_snapshot_parameters)` ### Decorator @@ -233,6 +234,7 @@ Argument | Type | Description direct_source | [DirectSource](Classes.md#directsource) | The source associated with this operation. repository | [RepositoryDefinition](Schemas_and_Autogenerated_Classes.md#repositorydefinition-class) | The repository associated with this source. source_config | [SourceConfigDefinition](Schemas_and_Autogenerated_Classes.md#sourceconfigdefinition-class) | The source config associated with this source. +optional_snapshot_parameters | [SnapshotParametersDefinition](Classes.md#snapshotparametersdefinition) | The snapshot parameters. The value is `None` when executed during a snapshot policy. ### Returns [SnapshotDefinition](Schemas_and_Autogenerated_Classes.md#snapshotdefinition-class) @@ -246,7 +248,7 @@ from generated.definitions import SnapshotDefinition plugin = Plugin() @plugin.linked.post_snapshot() -def linked_post_snapshot(direct_source, repository, source_config): +def linked_post_snapshot(direct_source, repository, source_config, optional_snapshot_parameters): snapshot = SnapshotDefinition() snapshot.transaction_id = 1000 return snapshot @@ -277,7 +279,7 @@ Sets up a [dSource](Glossary.md#dsource) to ingest data. Only applies when using ### Signature -`def linked_pre_snapshot(staged_source, repository, source_config, snapshot_parameters)` +`def linked_pre_snapshot(staged_source, repository, source_config, optional_snapshot_parameters)` ### Decorator @@ -290,7 +292,7 @@ Argument | Type | Description staged_source | [StagedSource](Classes.md#stagedsource) | The source associated with this operation. repository | [RepositoryDefinition](Schemas_and_Autogenerated_Classes.md#repositorydefinition-class) | The repository associated with this source. source_config | [SourceConfigDefinition](Schemas_and_Autogenerated_Classes.md#sourceconfigdefinition-class) | The source config associated with this source. -snapshot_parameters | [SnapshotParametersDefinition](Classes.md#snapshotparametersdefinition) | The snapshot parameters. +optional_snapshot_parameters | [SnapshotParametersDefinition](Classes.md#snapshotparametersdefinition) | The snapshot parameters. The value is `None` when executed during a snapshot policy. ### Returns None @@ -303,7 +305,7 @@ from dlpx.virtualization.platform import Plugin plugin = Plugin() @plugin.linked.pre_snapshot() -def linked_pre_snapshot(staged_source, repository, source_config, snapshot_parameters): +def linked_pre_snapshot(staged_source, repository, source_config, optional_snapshot_parameters): pass ``` @@ -320,7 +322,7 @@ Captures metadata from a [dSource](Glossary.md#dsource) once data has been inges ### Signature -`def linked_post_snapshot(staged_source, repository, source_config, snapshot_parameters)` +`def linked_post_snapshot(staged_source, repository, source_config, optional_snapshot_parameters)` ### Decorator @@ -333,7 +335,7 @@ Argument | Type | Description staged_source | [StagedSource](Classes.md#stagedsource) | The source associated with this operation. repository | [RepositoryDefinition](Schemas_and_Autogenerated_Classes.md#repositorydefinition-class) | The repository associated with this source. source_config | [SourceConfigDefinition](Schemas_and_Autogenerated_Classes.md#sourceconfigdefinition-class) | The source config associated with this source. -snapshot_parameters | [SnapshotParametersDefinition](Classes.md#snapshotparametersdefinition) | The snapshot parameters. +optional_snapshot_parameters | [SnapshotParametersDefinition](Classes.md#snapshotparametersdefinition) | The snapshot parameters. The value is `None` when executed during a snapshot policy. ### Returns [SnapshotDefinition](Schemas_and_Autogenerated_Classes.md#snapshotdefinition-class) @@ -347,9 +349,9 @@ from generated.definitions import SnapshotDefinition plugin = Plugin() @plugin.linked.post_snapshot() -def linked_post_snapshot(staged_source, repository, source_config, snapshot_parameters): +def linked_post_snapshot(staged_source, repository, source_config, optional_snapshot_parameters): snapshot = SnapshotDefinition() - if snapshot_parameters.resync: + if optional_snapshot_parameters is not None and optional_snapshot_parameters.resync: snapshot.transaction_id = 1000 else: snapshot.transaction_id = 10 diff --git a/docs/docs/Release_Notes/3.0.0/3.0.0.md b/docs/docs/Release_Notes/3.0.0/3.0.0.md index f6854791..e8c41e89 100644 --- a/docs/docs/Release_Notes/3.0.0/3.0.0.md +++ b/docs/docs/Release_Notes/3.0.0/3.0.0.md @@ -3,11 +3,25 @@ To install or upgrade the SDK, refer to instructions [here](/Getting_Started.md#installation). ## New & Improved -* Added a `scratch_path` property on the [RemoteHost](/References/Classes/#remotehost) object for storage and debugging purposes. +* Added the ability to define snapshot parameters in a [Snapshot Parameters Definition](/References/Schemas_and_Autogenerated_Classes.md#snapshotparametersdefinition-schema). + * Provide end-users with configurable options prior to taking a snapshot. + * The options selected are provided as input to pre/post-snapshot functions. + +* Added a `scratch_path` property on the [RemoteHost](/References/Classes/#remotehost) object which can be used as: + * A location to store small amounts of persistent data. + * A location to mount VDB data. More details about `scratch_path` can be found [here](/Best_Practices/Scratch_Paths.md) ## Breaking Changes * Added a new required schema [Snapshot Parameters Definition](/References/Schemas_and_Autogenerated_Classes.md#snapshotparametersdefinition-schema). -[**For more information and detailed steps to detect and make changes.**](/Release_Notes/3.0.0/3.0.0_Breaking_Changes#new-required-schema) \ No newline at end of file +[**For more information and detailed steps to detect and make changes.**](/Release_Notes/3.0.0/3.0.0_Breaking_Changes#new-required-schema) + +* Added a new parameter to the [Direct Linked Source Pre-Snapshot](/References/Plugin_Operations/#direct-linked-source-pre-snapshot) and [Direct Linked Source Post-Snapshot](/References/Plugin_Operations/#direct-linked-source-pre-snapshot) plugin operations. + +[**For more information and detailed steps to detect and make changes.**](/Release_Notes/3.0.0/3.0.0_Breaking_Changes#new-parameter-in-direct-prepost-snapshot-functions) + +* Renamed a parameter in the [Staged Linked Source Pre-Snapshot](/References/Plugin_Operations/#staged-linked-source-pre-snapshot) and [Staged Linked Source Post-Snapshot](/References/Plugin_Operations/#staged-linked-source-post-snapshot) plugin operations. + +[**For more information and detailed steps to detect and make changes.**](/Release_Notes/3.0.0/3.0.0_Breaking_Changes#parameter-renamed-in-staged-prepost-snapshot-functions) diff --git a/docs/docs/Release_Notes/3.0.0/3.0.0_Breaking_Changes.md b/docs/docs/Release_Notes/3.0.0/3.0.0_Breaking_Changes.md index b0c13762..7bfbddaa 100644 --- a/docs/docs/Release_Notes/3.0.0/3.0.0_Breaking_Changes.md +++ b/docs/docs/Release_Notes/3.0.0/3.0.0_Breaking_Changes.md @@ -1,7 +1,7 @@ # Breaking Changes - v.3.0.0 ## New required schema -[Snapshot Parameters Definition](/References/Schemas_and_Autogenerated_Classes.md#snapshotparametersdefinition-schema) allows plugin authors to define [Snapshot Parameters] which can be displayed to an end-user whenever a linked source snapshot is taken. +[Snapshot Parameters Definition](/References/Schemas_and_Autogenerated_Classes.md#snapshotparametersdefinition-schema) allows plugin authors to define [snapshot parameters](/References/Glossary.md#snapshot-parameters) which can be displayed to an end-user whenever a linked source snapshot is taken. ### What is affected All plugins built with v2.0.0 or below will be affected. The [Schema](/References/Schemas) must contain a `snapshotParametersDefinition`. @@ -34,3 +34,81 @@ Example: "type": "object" } ``` + +## New Parameter in Direct Pre/Post-Snapshot Functions +`optional_snapshot_parameters` has been added as a parameter in [Direct Linked Source Pre-Snapshot](/References/Plugin_Operations/#direct-linked-source-pre-snapshot) and [Direct Linked Source Post-Snapshot](/References/Plugin_Operations/#direct-linked-source-post-snapshot). + +### What is affected +All direct plugins built with v2.1.0 or below will be affected. + +### How does it fail +[dvp build](/References/CLI.md#build) will fail with the following error message if the `optional_snapshot_parameters` is not added: + +```bash +$ dvp build +Error: Named argument mismatch in method linked_post_snapshot. Expected: ['staged_source', 'repository', 'source_config', 'optional_snapshot_parameters'], Found: ['staged_source', 'repository', 'source_config']. + +0 Warning(s). 1 Error(s). + +BUILD FAILED. +``` + +### How to fix it +Add `optional_snapshot_parameters` as a parameter in [Direct Linked Source Pre-Snapshot](/References/Plugin_Operations/#direct-linked-source-pre-snapshot) and [Direct Linked Source Post-Snapshot](/References/Plugin_Operations/#direct-linked-source-post-snapshot). + +* Previous releases + +```python + @plugin.linked.post_snapshot() + def linked_post_snapshot(direct_source, repository, source_config): + return SnapshotDefinition() +``` + +* 3.0.0 + +```python + @plugin.linked.post_snapshot() + def linked_post_snapshot(direct_source, repository, source_config, optional_snapshot_parameters): + return SnapshotDefinition() +``` + +## Parameter Renamed in Staged Pre/Post-Snapshot Functions +The following parameter was renamed in the [Staged Linked Source Pre-Snapshot](/References/Plugin_Operations/#staged-linked-source-pre-snapshot) and [Staged Linked Source Post-Snapshot](/References/Plugin_Operations/#staged-linked-source-post-snapshot) functions: + +| Previous | Updated | +| -------- | ------- | +| `snapshot_parameters` | `optional_snapshot_parameters` | + +### What is affected +All staged plugins built with v2.1.0 or below will be affected. + +### How does it fail +[dvp build](/References/CLI.md#build) will fail with the following error message if the parameter is not renamed from `snapshot_parameters` to `optional_snapshot_parameters`: + +```bash +$ dvp build +Error: Named argument mismatch in method linked_post_snapshot. Expected: ['staged_source', 'repository', 'source_config', 'optional_snapshot_parameters'], Found: ['staged_source', 'repository', 'source_config', 'snapshot_parameters']. + +0 Warning(s). 1 Error(s). + +BUILD FAILED. +``` + +### How to fix it +Rename `snapshot_parameters` to `optional_snapshot_parameters` in [Staged Linked Source Pre-Snapshot](/References/Plugin_Operations/#staged-linked-source-pre-snapshot) and [Staged Linked Source Post-Snapshot](/References/Plugin_Operations/#staged-linked-source-post-snapshot). + +* Previous releases + +```python + @plugin.linked.post_snapshot() + def linked_post_snapshot(staged_source, repository, source_config, snapshot_parameters): + return SnapshotDefinition() +``` + +* 3.0.0 + +```python + @plugin.linked.post_snapshot() + def linked_post_snapshot(staged_source, repository, source_config, optional_snapshot_parameters): + return SnapshotDefinition() +``` \ No newline at end of file