Update SDK to add SnapshotParametersDefinition in uploaded artifact

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 2.2.0.dev1
+current_version = 3.0.0.dev0
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\.(?P<release>[a-z]+)(?P<dev>\d+))?

README-dev.md

@@ -125,17 +125,17 @@ To run blackbox tests, follow these steps:
 2. Navigate to the app-gate directory and start tests using `git blackbox`. For the guide on which test suite to use,
 see the next sections.
 
-At a minimum, each pull request should pass `appdata_python_samples` and `appdata_sanity` tests with a direct or staged plugin.
+At a minimum, each pull request should pass `appdata_python_samples` and `appdata_basic` tests with a direct or staged plugin.
 See the section below for the description of each test suite.
 
 #### Blackbox tests targeting wrappers (mostly Delphix Engine workflows)
 * appdata_python_samples (sample plugins from the app-gate):
 `git blackbox -s appdata_python_samples --extra-params="-p virt-sdk-repo=https://github.com/<username>/virtualization-sdk.git -p virt-sdk-branch=my-feature"`,
-* appdata_sanity with a direct Python plugin on CentOS 7.3: `git blackbox -s appdata_sanity -c APPDATA_PYTHON_DIRECT_CENTOS73 -a --extra-params="-p virt-sdk-repo=https://github.com/<username>/virtualization-sdk.git -p virt-sdk-branch=my-feature"`,
-* appdata_sanity with a staged Python plugin on CentOS 7.3: `git blackbox -s appdata_sanity -c APPDATA_PYTHON_STAGED_CENTOS73 -a --extra-params="-p virt-sdk-repo=https://github.com/<username>/virtualization-sdk.git -p virt-sdk-branch=my-feature"`.
+* appdata_basic with a direct Python plugin on CentOS 7.3: `git blackbox -s appdata_basic -c APPDATA_PYTHON_DIRECT_CENTOS73 -a --extra-params="-p virt-sdk-repo=https://github.com/<username>/virtualization-sdk.git -p virt-sdk-branch=my-feature"`,
+* appdata_basic with a staged Python plugin on CentOS 7.3: `git blackbox -s appdata_basic -c APPDATA_PYTHON_STAGED_CENTOS73 -a --extra-params="-p virt-sdk-repo=https://github.com/<username>/virtualization-sdk.git -p virt-sdk-branch=my-feature"`.
 
 #### Blackbox tests targeting the CLI (~80% CLI tests)
 * virtualization_sdk (installs and tests a direct Python plugin on Ubuntu 18): 
 `git blackbox -s virtualization_sdk -c APPDATA_SDK_UBUNTU18_DIRECT_CENTOS73 --extra-params="-p virt-sdk-repo=https://github.com/<username>/virtualization-sdk.git -p virt-sdk-branch=my-feature"`,
 * virtualization_sdk (installs and tests a staged Python plugin on Ubuntu 18): 
-`git blackbox -s virtualization_sdk -c APPDATA_SDK_UBUNTU18_STAGED_CENTOS73 --extra-params="-p virt-sdk-repo=https://github.com/<username>/virtualization-sdk.git -p virt-sdk-branch=my-feature"`.
+`git blackbox -s virtualization_sdk -c APPDATA_SDK_UBUNTU18_STAGED_CENTOS73 --extra-params="-p virt-sdk-repo=https://github.com/<username>/virtualization-sdk.git -p virt-sdk-branch=my-feature"`.

common/setup.py

@@ -4,7 +4,7 @@
 PYTHON_SRC = 'src/main/python'
 
 install_requires = [
-  "dvp-api == 1.4.0.dev10",
+  "dvp-api == 1.4.0.dev12",
 ]
 
 with open(os.path.join(PYTHON_SRC, 'dlpx/virtualization/common/VERSION')) as version_file:

common/src/main/python/dlpx/virtualization/common/VERSION

@@ -1 +1 @@
-2.2.0.dev1
+3.0.0.dev0

dvp/src/main/python/dlpx/virtualization/VERSION

@@ -1 +1 @@
-2.2.0.dev1
+3.0.0.dev0

libs/setup.py

@@ -7,7 +7,7 @@
     version = version_file.read().strip()
 
 install_requires = [
-  "dvp-api == 1.4.0.dev10",
+  "dvp-api == 1.4.0.dev12",
   "dvp-common == {}".format(version)
 ]
 

libs/src/main/python/dlpx/virtualization/libs/VERSION

@@ -1 +1 @@
-2.2.0.dev1
+3.0.0.dev0

platform/setup.py

@@ -7,7 +7,7 @@
     version = version_file.read().strip()
 
 install_requires = [
-  "dvp-api == 1.4.0.dev10",
+  "dvp-api == 1.4.0.dev12",
   "dvp-common == {}".format(version),
   "enum34;python_version < '3.4'",
 ]

platform/src/main/python/dlpx/virtualization/platform/VERSION

@@ -1 +1 @@
-2.2.0.dev1
+3.0.0.dev0

platform/src/main/python/dlpx/virtualization/platform/_linked.py

@@ -121,6 +121,7 @@ def _internal_direct_pre_snapshot(self, request):
         from generated.definitions import RepositoryDefinition
         from generated.definitions import LinkedSourceDefinition
         from generated.definitions import SourceConfigDefinition
+        from generated.definitions import SnapshotParametersDefinition
 
         #
         # While linked.pre_snapshot() is not a required operation, this should
@@ -141,10 +142,14 @@ def _internal_direct_pre_snapshot(self, request):
             json.loads(request.repository.parameters.json))
         source_config = SourceConfigDefinition.from_dict(
             json.loads(request.source_config.parameters.json))
+        snapshot_parameters = SnapshotParametersDefinition.from_dict(
+            json.loads(request.snapshot_parameters.parameters.json))
 
-        self.pre_snapshot_impl(direct_source=direct_source,
-                               repository=repository,
-                               source_config=source_config)
+        self.pre_snapshot_impl(
+            direct_source=direct_source,
+            repository=repository,
+            source_config=source_config,
+            optional_snapshot_parameters=snapshot_parameters)
 
         direct_pre_snapshot_response = platform_pb2.DirectPreSnapshotResponse()
         direct_pre_snapshot_response.return_value.CopyFrom(
@@ -173,6 +178,7 @@ def _internal_direct_post_snapshot(self, request):
         from generated.definitions import LinkedSourceDefinition
         from generated.definitions import SourceConfigDefinition
         from generated.definitions import SnapshotDefinition
+        from generated.definitions import SnapshotParametersDefinition
 
         def to_protobuf(snapshot):
             parameters = common_pb2.PluginDefinedObject()
@@ -196,10 +202,14 @@ def to_protobuf(snapshot):
             json.loads(request.repository.parameters.json))
         source_config = SourceConfigDefinition.from_dict(
             json.loads(request.source_config.parameters.json))
+        snapshot_parameters = SnapshotParametersDefinition.from_dict(
+            json.loads(request.snapshot_parameters.parameters.json))
 
-        snapshot = self.post_snapshot_impl(direct_source=direct_source,
-                                           repository=repository,
-                                           source_config=source_config)
+        snapshot = self.post_snapshot_impl(
+            direct_source=direct_source,
+            repository=repository,
+            source_config=source_config,
+            optional_snapshot_parameters=snapshot_parameters)
 
         # Validate that this is a SnapshotDefinition object
         if not isinstance(snapshot, SnapshotDefinition):
@@ -266,10 +276,11 @@ def _internal_staged_pre_snapshot(self, request):
         snapshot_parameters = SnapshotParametersDefinition.from_dict(
             json.loads(request.snapshot_parameters.parameters.json))
 
-        self.pre_snapshot_impl(staged_source=staged_source,
-                               repository=repository,
-                               source_config=source_config,
-                               snapshot_parameters=snapshot_parameters)
+        self.pre_snapshot_impl(
+            staged_source=staged_source,
+            repository=repository,
+            source_config=source_config,
+            optional_snapshot_parameters=snapshot_parameters)
 
         response = platform_pb2.StagedPreSnapshotResponse()
         response.return_value.CopyFrom(platform_pb2.StagedPreSnapshotResult())
@@ -337,7 +348,7 @@ def to_protobuf(snapshot):
             staged_source=staged_source,
             repository=repository,
             source_config=source_config,
-            snapshot_parameters=snapshot_parameters)
+            optional_snapshot_parameters=snapshot_parameters)
 
         # Validate that this is a SnapshotDefinition object
         if not isinstance(snapshot, SnapshotDefinition):

platform/src/test/python/dlpx/virtualization/fake_generated_definitions.py

@@ -101,14 +101,6 @@ def to_dict(self):
 
 
 class SnapshotParametersDefinition(Model):
-    """
-  The appdata snapshot parameter will eventually be customizable but for now
-  this just follows the old appdata parameter where the delphix user can decide
-  if resync is true or not. This will now go into pre and post snapshot
-  operations rather than the resync operation. The main point is customers will
-  set this to be true is that this means the operation is a "hard" resync and
-  that all data should be refreshed.
-  """
     def __init__(self, resync):
         self.swagger_types = {'resync': bool}
 

platform/src/test/python/dlpx/virtualization/test_plugin.py

@@ -813,19 +813,23 @@ def source_config_discovery_impl(source_connection, repository):
 
     @staticmethod
     def test_direct_pre_snapshot(my_plugin, direct_source, repository,
-                                 source_config):
+                                 source_config, snapshot_parameters):
         @my_plugin.linked.pre_snapshot()
-        def mock_direct_pre_snapshot(direct_source, repository, source_config):
-            TestPlugin.assert_plugin_args(direct_source=direct_source,
-                                          repository=repository,
-                                          source_config=source_config)
+        def mock_direct_pre_snapshot(direct_source, repository, source_config,
+                                     optional_snapshot_parameters):
+            TestPlugin.assert_plugin_args(
+                direct_source=direct_source,
+                repository=repository,
+                source_config=source_config,
+                snapshot_parameters=optional_snapshot_parameters)
             return
 
         direct_pre_snapshot_request = platform_pb2.DirectPreSnapshotRequest()
         TestPlugin.setup_request(request=direct_pre_snapshot_request,
                                  direct_source=direct_source,
                                  repository=repository,
-                                 source_config=source_config)
+                                 source_config=source_config,
+                                 snapshot_parameters=snapshot_parameters)
 
         expected_result = platform_pb2.DirectPreSnapshotResult()
         direct_pre_snapshot_response = (
@@ -839,20 +843,24 @@ def mock_direct_pre_snapshot(direct_source, repository, source_config):
 
     @staticmethod
     def test_direct_post_snapshot(my_plugin, direct_source, repository,
-                                  source_config):
+                                  source_config, snapshot_parameters):
         @my_plugin.linked.post_snapshot()
-        def direct_post_snapshot_impl(direct_source, repository,
-                                      source_config):
-            TestPlugin.assert_plugin_args(direct_source=direct_source,
-                                          repository=repository,
-                                          source_config=source_config)
+        def direct_post_snapshot_impl(direct_source, repository, source_config,
+                                      optional_snapshot_parameters):
+            TestPlugin.assert_plugin_args(
+                direct_source=direct_source,
+                repository=repository,
+                source_config=source_config,
+                snapshot_parameters=optional_snapshot_parameters)
             return SnapshotDefinition(TEST_SNAPSHOT)
 
         direct_post_snapshot_request = platform_pb2.DirectPostSnapshotRequest()
-        TestPlugin.setup_request(request=direct_post_snapshot_request,
-                                 direct_source=direct_source,
-                                 repository=repository,
-                                 source_config=source_config)
+        TestPlugin.setup_request(
+            request=direct_post_snapshot_request,
+            direct_source=direct_source,
+            repository=repository,
+            source_config=source_config,
+            snapshot_parameters=snapshot_parameters)
 
         direct_post_snapshot_response = (
             my_plugin.linked._internal_direct_post_snapshot(
@@ -866,20 +874,21 @@ def test_staged_pre_snapshot(my_plugin, staged_source, repository,
                                  source_config, snapshot_parameters):
         @my_plugin.linked.pre_snapshot()
         def staged_pre_snapshot_impl(staged_source, repository, source_config,
-                                     snapshot_parameters):
+                                     optional_snapshot_parameters):
             TestPlugin.assert_plugin_args(
                 staged_source=staged_source,
                 repository=repository,
                 source_config=source_config,
-                snapshot_parameters=snapshot_parameters)
+                snapshot_parameters=optional_snapshot_parameters)
             return
 
         staged_pre_snapshot_request = platform_pb2.StagedPreSnapshotRequest()
-        TestPlugin.setup_request(request=staged_pre_snapshot_request,
-                                 staged_source=staged_source,
-                                 repository=repository,
-                                 source_config=source_config,
-                                 snapshot_parameters=snapshot_parameters)
+        TestPlugin.setup_request(
+            request=staged_pre_snapshot_request,
+            staged_source=staged_source,
+            repository=repository,
+            source_config=source_config,
+            snapshot_parameters=snapshot_parameters)
 
         expected_result = platform_pb2.StagedPreSnapshotResult()
         response = my_plugin.linked._internal_staged_pre_snapshot(
@@ -894,20 +903,21 @@ def test_staged_post_snapshot(my_plugin, staged_source, repository,
                                   source_config, snapshot_parameters):
         @my_plugin.linked.post_snapshot()
         def staged_post_snapshot_impl(staged_source, repository, source_config,
-                                      snapshot_parameters):
+                                      optional_snapshot_parameters):
             TestPlugin.assert_plugin_args(
                 staged_source=staged_source,
                 repository=repository,
                 source_config=source_config,
-                snapshot_parameters=snapshot_parameters)
+                snapshot_parameters=optional_snapshot_parameters)
             return SnapshotDefinition(TEST_SNAPSHOT)
 
         staged_post_snapshot_request = platform_pb2.StagedPostSnapshotRequest()
-        TestPlugin.setup_request(request=staged_post_snapshot_request,
-                                 staged_source=staged_source,
-                                 repository=repository,
-                                 source_config=source_config,
-                                 snapshot_parameters=snapshot_parameters)
+        TestPlugin.setup_request(
+            request=staged_post_snapshot_request,
+            staged_source=staged_source,
+            repository=repository,
+            source_config=source_config,
+            snapshot_parameters=snapshot_parameters)
 
         response = my_plugin.linked._internal_staged_post_snapshot(
             staged_post_snapshot_request)

tools/src/main/python/dlpx/virtualization/_internal/VERSION

@@ -1 +1 @@
-2.2.0.dev1
+3.0.0.dev0

tools/src/main/python/dlpx/virtualization/_internal/codegen.py

@@ -26,24 +26,6 @@
     'definitions': {}
 }
 
-#
-# The default snapshot params we currently support. This is so that in the
-# future when we want to support plugin author defined schemas for snapshot
-# params the upgrade case will be relatively simple.
-#
-
-SNAPSHOT_PARAMS_JSON = {
-    'snapshotParametersDefinition': {
-        'type': 'object',
-        'additionalProperties': False,
-        'properties': {
-            'resync': {
-                'type': 'boolean'
-            }
-        }
-    }
-}
-
 SWAGGER_FILE_NAME = 'swagger.json'
 CODEGEN_PACKAGE = 'generated'
 CODEGEN_MODULE = 'definitions'
@@ -111,9 +93,7 @@ def generate_python(name, source_dir, plugin_config_dir, schema_content):
 def _write_swagger_file(name, schema_dict, output_dir):
     swagger_json = copy.deepcopy(SWAGGER_JSON_FORMAT)
     swagger_json['info']['title'] = name
-    swagger_json['definitions'] = copy.deepcopy(schema_dict)
-    # Add in the snapshot param definition
-    swagger_json['definitions'].update(SNAPSHOT_PARAMS_JSON)
+    swagger_json['definitions'] = schema_dict
 
     swagger_file = os.path.join(output_dir, SWAGGER_FILE_NAME)
     logger.info('Writing swagger file to {}'.format(swagger_file))

tools/src/main/python/dlpx/virtualization/_internal/commands/build.py

@@ -207,6 +207,8 @@ def prepare_upload_artifact(plugin_config_content, src_dir, schemas, manifest):
         prepare_discovery_definition(plugin_config_content, schemas),
         'snapshotSchema':
         schemas['snapshotDefinition'],
+        'snapshotParametersDefinition':
+        schemas['snapshotParametersDefinition'],
         'manifest':
         manifest
     }

tools/src/main/python/dlpx/virtualization/_internal/commands/plugin_template/direct_operations.py.template

@@ -1,3 +1,4 @@
 @plugin.linked.post_snapshot()
-def linked_post_snapshot(direct_source, repository, source_config):
+def linked_post_snapshot(direct_source, repository, source_config,
+                         optional_snapshot_parameters):
     return SnapshotDefinition()

tools/src/main/python/dlpx/virtualization/_internal/commands/plugin_template/schema_template.json

@@ -31,5 +31,10 @@
         "type" : "object",
         "additionalProperties" : false,
         "properties" : {}
+    },
+    "snapshotParametersDefinition": {
+        "type" : "object",
+        "additionalProperties" : false,
+        "properties" : {}
     }
 }

tools/src/main/python/dlpx/virtualization/_internal/commands/plugin_template/staged_operations.py.template

@@ -2,7 +2,7 @@
 def linked_post_snapshot(staged_source,
                          repository,
                          source_config,
-                         snapshot_parameters):
+                         optional_snapshot_parameters):
     return SnapshotDefinition()
 
 

tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_importer.yaml

@@ -13,12 +13,12 @@ EXPECTED_STAGED_ARGS_BY_OP:
       - staged_source
       - repository
       - source_config
-      - snapshot_parameters
+      - optional_snapshot_parameters
     post_snapshot_impl:
       - staged_source
       - repository
       - source_config
-      - snapshot_parameters
+      - optional_snapshot_parameters
     start_staging_impl:
       - staged_source
       - repository
@@ -92,10 +92,12 @@ EXPECTED_DIRECT_ARGS_BY_OP:
       - direct_source
       - repository
       - source_config
+      - optional_snapshot_parameters
     post_snapshot_impl:
       - direct_source
       - repository
       - source_config
+      - optional_snapshot_parameters
   VirtualOperations:
     configure_impl:
       - virtual_source