Merge Gitlab's master branch into GitHub's develop branch

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.1.0-internal-7
+current_version = 2.0.0-internal-001
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-(?P<release>[a-z]+)\-(?P<build>\d+))?

.hooksconfig

@@ -1,11 +1,16 @@
 [gate]
     name = virtualization-sdk
     shortname = sdk
-    slack-url = https://hooks.slack.com/services/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
-    slack-color = 1AD6F5 
+    slack-url = https://hooks.slack.com/services/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
+    slack-color = 1AD6F5
     slack-notify = virt-sdk-pushes
     approvers = gatekeepers-virtualization-sdk
     tags-allowed = true
 
  [branch "master"]
     gate-allowed-issuetypes = 1,3,4,5,10001,10302
+
+ [branch "projects/plugin-upgrade"]
+    gate-allowed-issuetypes = 1,3,4,5,10001,10302
+    gate-comment-check = on
+    gate-review-check = on

common/src/main/python/dlpx/virtualization/common/VERSION

@@ -1 +1 @@
-1.1.0-internal-7
+2.0.0-internal-001

dvp/requirements.txt

@@ -1,15 +1,15 @@
 bump2version==0.5.11
 contextlib2==0.6.0.post1 ; python_version < '3'
 funcsigs==1.0.2 ; python_version < '3.0'
-importlib-metadata==0.23 ; python_version < '3.8'
+importlib-metadata==1.3.0 ; python_version < '3.8'
 more-itertools==5.0.0 ; python_version <= '2.7'
-packaging==19.2
-pathlib2==2.3.5 ; python_version < '3.6'
-pluggy==0.13.0
-py==1.8.0
-pyparsing==2.4.5
-pytest==4.6.6
+packaging==20.0
+pathlib2==2.3.5 ; python_version < '3'
+pluggy==0.13.1
+py==1.8.1
+pyparsing==2.4.6
+pytest==4.6.9
 scandir==1.10.0 ; python_version < '3.5'
 six==1.13.0
-wcwidth==0.1.7
+wcwidth==0.1.8
 zipp==0.6.0

dvp/src/main/python/dlpx/virtualization/VERSION

@@ -1 +1 @@
-1.1.0-internal-7
+2.0.0-internal-001

libs/src/main/python/dlpx/virtualization/libs/VERSION

@@ -1 +1 @@
-1.1.0-internal-7
+2.0.0-internal-001

libs/src/test/python/dlpx/virtualization/test_libs.py

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2019 by Delphix. All rights reserved.
+# Copyright (c) 2019, 2020 by Delphix. All rights reserved.
 #
 
 import mock
@@ -189,11 +189,12 @@ def test_run_bash_bad_variables(remote_connection):
         with pytest.raises(IncorrectArgumentTypeError) as err_info:
             libs.run_bash(remote_connection, command, variables, use_login_shell)
 
-        assert err_info.value.message == (
-            "The function run_bash's argument 'variables' was"
-            " a dict of {type 'str':type 'int', type 'str':type 'str'}"
-            " but should be of"
-            " type 'dict of basestring:basestring' if defined.")
+        message = ("The function run_bash's argument 'variables' was"
+                   " a dict of {{type 'str':type '{}', type 'str':type '{}'}}"
+                   " but should be of"
+                   " type 'dict of basestring:basestring' if defined.")
+        assert (err_info.value.message == message.format('int', 'str') or
+                err_info.value.message == message.format('str', 'int'))
 
     @staticmethod
     def test_run_bash_bad_use_login_shell(remote_connection):
@@ -213,7 +214,6 @@ def test_run_bash_bad_use_login_shell(remote_connection):
 class TestLibsRunSync:
     @staticmethod
     def test_run_sync(remote_connection):
-
         expected_run_sync_response = libs_pb2.RunSyncResponse()
 
         expected_source_directory = 'sourceDirectory'
@@ -269,7 +269,6 @@ def test_run_sync_with_actionable_error(remote_connection):
 
     @staticmethod
     def test_run_sync_with_nonactionable_error(remote_connection):
-
         response = libs_pb2.RunSyncResponse()
         na_error = libs_pb2.NonActionableLibraryError()
         response.error.non_actionable_error.CopyFrom(na_error)
@@ -593,11 +592,12 @@ def test_run_powershell_bad_variables(remote_connection):
         with pytest.raises(IncorrectArgumentTypeError) as err_info:
             libs.run_powershell(remote_connection, command, variables)
 
-        assert err_info.value.message == (
-            "The function run_powershell's argument 'variables' was"
-            " a dict of {type 'str':type 'int', type 'str':type 'str'}"
-            " but should be of"
-            " type 'dict of basestring:basestring' if defined.")
+        message = ("The function run_powershell's argument 'variables' was"
+                   " a dict of {{type 'str':type '{}', type 'str':type '{}'}}"
+                   " but should be of"
+                   " type 'dict of basestring:basestring' if defined.")
+        assert (err_info.value.message == message.format('int', 'str') or
+                err_info.value.message == message.format('str', 'int'))
 
 
 class TestLibsRunExpect:
@@ -647,12 +647,12 @@ def test_run_expect_check_true_exitcode_success(remote_connection):
         def mock_run_expect(actual_run_expect_request):
             assert actual_run_expect_request.command == expected_command
             assert (
-                actual_run_expect_request.remote_connection.environment.name
-                == remote_connection.environment.name
+                    actual_run_expect_request.remote_connection.environment.name
+                    == remote_connection.environment.name
             )
             assert (
-                actual_run_expect_request.remote_connection.environment.reference
-                == remote_connection.environment.reference
+                    actual_run_expect_request.remote_connection.environment.reference
+                    == remote_connection.environment.reference
             )
             return expected_run_expect_response
 
@@ -704,7 +704,6 @@ def test_run_expect_with_actionable_error(remote_connection):
 
     @staticmethod
     def test_run_expect_with_nonactionable_error(remote_connection):
-
         response = libs_pb2.RunExpectResponse()
         na_error = libs_pb2.NonActionableLibraryError()
         response.error.non_actionable_error.CopyFrom(na_error)
@@ -768,8 +767,9 @@ def test_run_expect_bad_variables(remote_connection):
         with pytest.raises(IncorrectArgumentTypeError) as err_info:
             libs.run_expect(remote_connection, command, variables)
 
-        assert err_info.value.message == (
-            "The function run_expect's argument 'variables' was"
-            " a dict of {type 'str':type 'int', type 'str':type 'str'}"
-            " but should be of"
-            " type 'dict of basestring:basestring' if defined.")
+        message = ("The function run_expect's argument 'variables' was"
+                   " a dict of {{type 'str':type '{}', type 'str':type '{}'}}"
+                   " but should be of"
+                   " type 'dict of basestring:basestring' if defined.")
+        assert (err_info.value.message == message.format('int', 'str') or
+                err_info.value.message == message.format('str', 'int'))

platform/src/main/python/dlpx/virtualization/platform/VERSION

@@ -1 +1 @@
-1.1.0-internal-7
+2.0.0-internal-001

platform/src/main/python/dlpx/virtualization/platform/__init__.py

@@ -4,6 +4,15 @@
 
 __path__ = __import__('pkgutil').extend_path(__path__, __name__)
 
+
+from dlpx.virtualization.platform.migration_id_set import *
+from dlpx.virtualization.platform.validation_util import *
 from dlpx.virtualization.platform._plugin_classes import *
+from dlpx.virtualization.platform._discovery import *
+from dlpx.virtualization.platform._linked import *
+from dlpx.virtualization.platform._upgrade import *
+from dlpx.virtualization.platform._virtual import *
 from dlpx.virtualization.platform._plugin import *
-from dlpx.virtualization.platform.util import *
+from dlpx.virtualization.platform.util import *
+from dlpx.virtualization.platform.import_util import *
+from dlpx.virtualization.platform.import_validations import *

platform/src/main/python/dlpx/virtualization/platform/_discovery.py

@@ -0,0 +1,163 @@
+#
+# Copyright (c) 2019 by Delphix. All rights reserved.
+#
+
+# -*- coding: utf-8 -*-
+
+"""DiscoveryOperations for the Virtualization Platform
+
+"""
+import json
+from dlpx.virtualization.common import RemoteConnection
+from dlpx.virtualization.api import common_pb2
+from dlpx.virtualization.api import platform_pb2
+from dlpx.virtualization.platform import validation_util as v
+from dlpx.virtualization.platform.operation import Operation as Op
+from dlpx.virtualization.platform.exceptions import (
+    IncorrectReturnTypeError, OperationNotDefinedError,
+    OperationAlreadyDefinedError)
+
+
+__all__ = ['DiscoveryOperations']
+
+
+class DiscoveryOperations(object):
+
+    def __init__(self):
+        self.repository_impl = None
+        self.source_config_impl = None
+
+    def repository(self):
+        def repository_decorator(repository_impl):
+            if self.repository_impl:
+                raise OperationAlreadyDefinedError(Op.DISCOVERY_REPOSITORY)
+
+            self.repository_impl = v.check_function(repository_impl,
+                                                    Op.DISCOVERY_REPOSITORY)
+            return repository_impl
+        return repository_decorator
+
+    def source_config(self):
+        def source_config_decorator(source_config_impl):
+            if self.source_config_impl:
+                raise OperationAlreadyDefinedError(Op.DISCOVERY_SOURCE_CONFIG)
+            self.source_config_impl = v.check_function(
+                source_config_impl, Op.DISCOVERY_SOURCE_CONFIG)
+            return source_config_impl
+        return source_config_decorator
+
+    def _internal_repository(self, request):
+        """Repository discovery wrapper.
+
+        Executed just after adding or refreshing an environment. This plugin
+        operation is run prior to discovering source configs. This plugin
+        operation returns a list of repositories installed on a environment.
+
+        Discover the repositories on an environment given a source connection.
+
+        Args:
+            request (RepositoryDiscoveryRequest): Repository
+            Discovery operation arguments.
+
+        Returns:
+            RepositoryDiscoveryResponse: The return value of repository
+            discovery operation.
+        """
+        from generated.definitions import RepositoryDefinition
+
+        def to_protobuf(repository):
+            parameters = common_pb2.PluginDefinedObject()
+            parameters.json = json.dumps(repository.to_dict())
+            repository_protobuf = common_pb2.Repository()
+            repository_protobuf.parameters.CopyFrom(parameters)
+            return repository_protobuf
+
+        if not self.repository_impl:
+            raise OperationNotDefinedError(Op.DISCOVERY_REPOSITORY)
+
+        repositories = self.repository_impl(
+            source_connection=RemoteConnection.from_proto(request.source_connection))
+
+        # Validate that this is a list of Repository objects
+        if not isinstance(repositories, list):
+            raise IncorrectReturnTypeError(
+                Op.DISCOVERY_REPOSITORY,
+                type(repositories),
+                [RepositoryDefinition])
+
+        if not all(isinstance(repo, RepositoryDefinition)
+                   for repo in repositories):
+            raise IncorrectReturnTypeError(
+                Op.DISCOVERY_REPOSITORY,
+                [type(repo) for repo in repositories],
+                [RepositoryDefinition])
+
+        repository_discovery_response = (
+            platform_pb2.RepositoryDiscoveryResponse())
+        repository_protobuf_list = [to_protobuf(repo) for repo in repositories]
+        repository_discovery_response.return_value.repositories.extend(
+            repository_protobuf_list)
+        return repository_discovery_response
+
+    def _internal_source_config(self, request):
+        """Source config discovery wrapper.
+
+        Executed when adding or refreshing an environment. This plugin
+        operation is run after discovering repositories and before
+        persisting/updating repository and source config data in MDS. This
+        plugin operation returns a list of source configs from a discovered
+        repository.
+
+        Discover the source configs on an environment given a discovered
+        repository.
+
+        Args:
+            request (SourceConfigDiscoveryRequest): Source
+            Config Discovery arguments.
+
+        Returns:
+            SourceConfigDiscoveryResponse: The return value of source config
+            discovery operation.
+        """
+        # Reasoning for method imports are in this file's docstring.
+        from generated.definitions import RepositoryDefinition
+        from generated.definitions import SourceConfigDefinition
+
+        def to_protobuf(source_config):
+            parameters = common_pb2.PluginDefinedObject()
+            parameters.json = json.dumps(source_config.to_dict())
+            source_config_protobuf = common_pb2.SourceConfig()
+            source_config_protobuf.parameters.CopyFrom(parameters)
+            return source_config_protobuf
+
+        if not self.source_config_impl:
+            raise OperationNotDefinedError(Op.DISCOVERY_SOURCE_CONFIG)
+
+        repository_definition = RepositoryDefinition.from_dict(
+            json.loads(request.repository.parameters.json))
+
+        source_configs = self.source_config_impl(
+            source_connection=RemoteConnection.from_proto(request.source_connection),
+            repository=repository_definition)
+
+        # Validate that this is a list of SourceConfigDefinition objects
+        if not isinstance(source_configs, list):
+            raise IncorrectReturnTypeError(
+                Op.DISCOVERY_SOURCE_CONFIG,
+                type(source_configs),
+                [SourceConfigDefinition])
+
+        if not all(isinstance(config, SourceConfigDefinition)
+                   for config in source_configs):
+            raise IncorrectReturnTypeError(
+                Op.DISCOVERY_SOURCE_CONFIG,
+                [type(config) for config in source_configs],
+                [SourceConfigDefinition])
+
+        source_config_discovery_response = (
+            platform_pb2.SourceConfigDiscoveryResponse())
+        source_config_protobuf_list = [to_protobuf(config)
+                                       for config in source_configs]
+        source_config_discovery_response.return_value.source_configs.extend(
+            source_config_protobuf_list)
+        return source_config_discovery_response