CP-3163 Add authentication object to AppData with vault option

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 2.2.0.dev1
+current_version = 3.0.0.dev2
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\.(?P<release>[a-z]+)(?P<dev>\d+))?

.dependabot/config.yaml

@@ -3,21 +3,21 @@ update_configs:
   # Keep requirements.txt files up-to-date in each package.
   - package_manager: "python"
     directory: "/dvp"
-    update_schedule: "daily"
+    update_schedule: "monthly"
     target_branch: "develop"
   - package_manager: "python"
     directory: "/common"
-    update_schedule: "daily"
+    update_schedule: "monthly"
     target_branch: "develop"
   - package_manager: "python"
     directory: "/platform"
-    update_schedule: "daily"
+    update_schedule: "monthly"
     target_branch: "develop"
   - package_manager: "python"
     directory: "/libs"
-    update_schedule: "daily"
+    update_schedule: "monthly"
     target_branch: "develop"
   - package_manager: "python"
     directory: "/tools"
-    update_schedule: "daily"
+    update_schedule: "monthly"
     target_branch: "develop"

.github/CODEOWNERS

@@ -1 +1 @@
-*       @ankursarin @fdrozdowski @nhlien93 @crystalplumage
+*       @ankursarin @nhlien93 @mothslaw

README-dev.md

@@ -125,17 +125,17 @@ To run blackbox tests, follow these steps:
 2. Navigate to the app-gate directory and start tests using `git blackbox`. For the guide on which test suite to use,
 see the next sections.
 
-At a minimum, each pull request should pass `appdata_python_samples` and `appdata_sanity` tests with a direct or staged plugin.
+At a minimum, each pull request should pass `appdata_python_samples` and `appdata_basic` tests with a direct or staged plugin.
 See the section below for the description of each test suite.
 
 #### Blackbox tests targeting wrappers (mostly Delphix Engine workflows)
 * appdata_python_samples (sample plugins from the app-gate):
 `git blackbox -s appdata_python_samples --extra-params="-p virt-sdk-repo=https://github.com/<username>/virtualization-sdk.git -p virt-sdk-branch=my-feature"`,
-* appdata_sanity with a direct Python plugin on CentOS 7.3: `git blackbox -s appdata_sanity -c APPDATA_PYTHON_DIRECT_CENTOS73 -a --extra-params="-p virt-sdk-repo=https://github.com/<username>/virtualization-sdk.git -p virt-sdk-branch=my-feature"`,
-* appdata_sanity with a staged Python plugin on CentOS 7.3: `git blackbox -s appdata_sanity -c APPDATA_PYTHON_STAGED_CENTOS73 -a --extra-params="-p virt-sdk-repo=https://github.com/<username>/virtualization-sdk.git -p virt-sdk-branch=my-feature"`.
+* appdata_basic with a direct Python plugin on CentOS 7.3: `git blackbox -s appdata_basic -c APPDATA_PYTHON_DIRECT_CENTOS73 -a --extra-params="-p virt-sdk-repo=https://github.com/<username>/virtualization-sdk.git -p virt-sdk-branch=my-feature"`,
+* appdata_basic with a staged Python plugin on CentOS 7.3: `git blackbox -s appdata_basic -c APPDATA_PYTHON_STAGED_CENTOS73 -a --extra-params="-p virt-sdk-repo=https://github.com/<username>/virtualization-sdk.git -p virt-sdk-branch=my-feature"`.
 
 #### Blackbox tests targeting the CLI (~80% CLI tests)
 * virtualization_sdk (installs and tests a direct Python plugin on Ubuntu 18): 
 `git blackbox -s virtualization_sdk -c APPDATA_SDK_UBUNTU18_DIRECT_CENTOS73 --extra-params="-p virt-sdk-repo=https://github.com/<username>/virtualization-sdk.git -p virt-sdk-branch=my-feature"`,
 * virtualization_sdk (installs and tests a staged Python plugin on Ubuntu 18): 
-`git blackbox -s virtualization_sdk -c APPDATA_SDK_UBUNTU18_STAGED_CENTOS73 --extra-params="-p virt-sdk-repo=https://github.com/<username>/virtualization-sdk.git -p virt-sdk-branch=my-feature"`.
+`git blackbox -s virtualization_sdk -c APPDATA_SDK_UBUNTU18_STAGED_CENTOS73 --extra-params="-p virt-sdk-repo=https://github.com/<username>/virtualization-sdk.git -p virt-sdk-branch=my-feature"`.

common/requirements.txt

@@ -1,5 +1,6 @@
 bump2version==0.5.11
 contextlib2==0.6.0.post1 ; python_version < '3'
+enum34==1.1.6
 funcsigs==1.0.2 ; python_version < '3.0'
 importlib-metadata==0.23 ; python_version < '3.8'
 more-itertools==5.0.0 ; python_version <= '2.7'

common/setup.py

@@ -4,7 +4,7 @@
 PYTHON_SRC = 'src/main/python'
 
 install_requires = [
-  "dvp-api == 1.3.0",
+  "dvp-api == 1.4.0.dev13",
 ]
 
 with open(os.path.join(PYTHON_SRC, 'dlpx/virtualization/common/VERSION')) as version_file:

common/src/main/python/dlpx/virtualization/common/VERSION

@@ -1 +1 @@
-2.2.0.dev1
+3.0.0.dev2

common/src/main/python/dlpx/virtualization/common/_common_classes.py

@@ -2,8 +2,10 @@
 # Copyright (c) 2019 by Delphix. All rights reserved.
 #
 
-from dlpx.virtualization.api import common_pb2
+from abc import ABCMeta
+from dlpx.virtualization.api import common_pb2, libs_pb2
 from dlpx.virtualization.common.exceptions import IncorrectTypeError
+from enum import IntEnum
 
 """Classes used for Plugin Operations
 
@@ -17,7 +19,10 @@
     "RemoteConnection",
     "RemoteEnvironment",
     "RemoteHost",
-    "RemoteUser"]
+    "RemoteUser",
+    "Credentials",
+    "PasswordCredentials",
+    "KeyPairCredentials"]
 
 
 class RemoteConnection(object):
@@ -293,3 +298,121 @@ def from_proto(user):
                 common_pb2.RemoteUser)
         remote_user = RemoteUser(name=user.name, reference=user.reference)
         return remote_user
+
+
+class Credentials(object):
+    """Plugin base class for CredentialsResult to be used for plugin operations
+    and library functions.
+
+    Plugin authors should use this instead of the corresponding protobuf generated
+    class.
+
+    Args:
+        username: User name.
+    """
+    def __init__(self, username):
+        if not isinstance(username, basestring):
+            raise IncorrectTypeError(
+                Credentials,
+                'username',
+                type(username),
+                basestring)
+        self.__username = username
+
+    __metaclass__ = ABCMeta
+
+    @property
+    def username(self):
+        return self.__username
+
+
+class PasswordCredentials(Credentials):
+    """Plugin class for CredentialsResult with password to be used for plugin operations
+    and library functions.
+
+    Plugin authors should use this instead of the corresponding protobuf generated
+    class.
+
+    Args:
+        username: User name.
+        password: Password.
+    """
+    def __init__(self, username, password):
+        super(PasswordCredentials, self).__init__(username)
+        if not isinstance(password, basestring):
+            raise IncorrectTypeError(
+                PasswordCredentials,
+                'password',
+                type(password),
+                basestring)
+        self.__password = password
+
+    @property
+    def password(self):
+        return self.__password
+
+    @staticmethod
+    def from_proto(credentials_result):
+        """Converts protobuf class libs_pb2.CredentialsResult to plugin class PasswordCredentials
+        """
+        if not isinstance(credentials_result, libs_pb2.CredentialsResult):
+            raise IncorrectTypeError(
+                PasswordCredentials,
+                'credentials_result',
+                type(credentials_result),
+                libs_pb2.CredentialsResult)
+        return PasswordCredentials(
+            username=credentials_result.username, password=credentials_result.pasword)
+
+
+class KeyPairCredentials(Credentials):
+    """Plugin class for CredentialsResult with key pair to be used for plugin operations
+    and library functions.
+
+    Plugin authors should use this instead of the corresponding protobuf generated
+    class.
+
+    Args:
+        username (str): User name.
+        private_key (str): Private key.
+        public_key (str): Public key corresponding to private key. Empty string if not present.
+    """
+    def __init__(self, username, private_key, public_key):
+        super(KeyPairCredentials, self).__init__(username)
+        if not isinstance(private_key, basestring):
+            raise IncorrectTypeError(
+                KeyPairCredentials,
+                'private_key',
+                type(private_key),
+                basestring)
+        self.__private_key = private_key
+        if not isinstance(public_key, basestring):
+            raise IncorrectTypeError(
+                KeyPairCredentials,
+                'public_key',
+                type(public_key),
+                basestring)
+        self.__public_key = public_key
+
+    @property
+    def private_key(self):
+        return self.__private_key
+
+    @property
+    def public_key(self):
+        return self.__public_key
+
+    @staticmethod
+    def from_proto(credentials_result):
+        """Converts protobuf class libs_pb2.CredentialsResult to plugin class KeyPairCredentials
+        """
+        if not isinstance(credentials_result, libs_pb2.CredentialsResult):
+            raise IncorrectTypeError(
+                KeyPairCredentials,
+                'credentials_result',
+                type(credentials_result),
+                libs_pb2.CredentialsResult)
+        return KeyPairCredentials(
+            username=credentials_result.username,
+            private_key=credentials_result.key_pair.private_key,
+            public_key=credentials_result.key_pair.public_key)

docs/docs/References/Decorators.md

@@ -13,13 +13,13 @@ plugin = Plugin()
 # Use the decorator to annotate the function that corresponds to the "Virtual Source Start" Plugin Operation
 @plugin.virtual_source.start()
 def my_start(virtual_source, repository, source_config):
-  print "running start" 
+  print "running start"
 ```
 
 !!! info
     Decorators exposed by the Virtualization SDK are inherently python function calls and needs parentheses `()` appended at the end.
 
-Assuming the name of the object, is `plugin` as above, the table below lists the corresponding decorators for each plugin operation.
+Assuming the name of the object is `plugin` as above, the table below lists the corresponding decorators for each plugin operation.
 
 Plugin Operation | Decorator
 ---------------- |  --------
@@ -35,6 +35,7 @@ Plugin Operation | Decorator
 [Staged Linked Source Worker](Plugin_Operations.md#staged-linked-source-worker) | `@plugin.linked.worker()`
 [Staged Linked Source Mount Specification](Plugin_Operations.md#staged-linked-source-mount-specification) | `@plugin.linked.mount_specification()`
 [Virtual Source Configure](Plugin_Operations.md#virtual-source-configure) | `@plugin.virtual.configure()`
+[Virtual Source Initialize](Plugin_Operations.md#virtual-source-initialize) | `@plugin.virtual.initialize()`
 [Virtual Source Unconfigure](Plugin_Operations.md#virtual-source-unconfigure) | `@plugin.virtual.unconfigure()`
 [Virtual Source Reconfigure](Plugin_Operations.md#virtual-source-reconfigure) | `@plugin.virtual.reconfigure()`
 [Virtual Source Start](Plugin_Operations.md#virtual-source-start) | `@plugin.virtual.start()`

docs/docs/References/Glossary.md

@@ -28,6 +28,13 @@ The process by which the Delphix Engine learns about how a particular environmen
 ## dSource
 See [Linked Dataset](#linked-dataset)
 
+## Empty VDB
+A VDB that is created from scratch, without provisioning from another dataset. Users can create empty VDBs when they want to construct a brand-new dataset from within Delphix, instead of creating it externally and then ingesting it.
+
+This "empty" VDB, of course, will typically not stay empty for long. Data will be added as users work with the new dataset.
+
+A plugin can support this functionality by implementing the [initialize](Plugin_Operations.md#virtual-source-initialize) operation.
+
 ## Environment
 A remote system that the Delphix Engine can interact with. An environment can be used as a [source](#source-environment), [staging](#staging-environment) or [target](#target-environment) environment (or any combination of those).  For example, a Linux machine that the Delphix Engine can connect to is an environment.
 
@@ -105,7 +112,7 @@ The process by which the Delphix Engine ingests data from a dataset on a [source
 An [environment](#environment) on which Delphix-provided virtualized datasets can be used.
 
 ## Lua Toolkit
-Legacy model for writing "plugins" in Lua, with limited documentation and support for writing, building and uploading toolkits. This was the predecessor to the Virtualization SDK. 
+Legacy model for writing "plugins" in Lua, with limited documentation and limited support for writing, building and uploading toolkits. This was the predecessor to the Virtualization SDK.
 
 ## Upgrade Operation
 A special plugin operation that takes data produced by an older version of a plugin, and transforms it into the format expected by the new version of the plugin.

docs/docs/References/Plugin_Operations.md

@@ -22,6 +22,7 @@ Plugin Operation | **Required** | Decorator | Delphix Engine Operations
 [Staged Linked Source<br/>Status](#staged-linked-source-status) | **No** |`linked.status()` | N/A
 [Staged Linked Source<br/>Worker](#staged-linked-source-worker) | **No** |`linked.worker()` | N/A
 [Staged Linked Source<br/>Mount Specification](#staged-linked-source-mount-specification) | **Yes** | `linked.mount_specification()` | [Linked Source Sync](Workflows.md#linked-source-sync)<br/>[Linked Source Enable](Workflows.md#linked-source-enable)
+[Virtual Source<br/>Initialize](#virtual-source-initialize) | **No** | `virtual.initialize()` | [Virtual Source Create Empty VDB](Workflows.md#virtual-source-create-empty-vdb)
 [Virtual Source<br/>Configure](#virtual-source-configure) | **Yes** | `virtual.configure()` | [Virtual Source Provision](Workflows.md#virtual-source-provision)<br/>[Virtual Source Refresh](Workflows.md#virtual-source-refresh)
 [Virtual Source<br/>Unconfigure](#virtual-source-unconfigure) | **No** | `virtual.unconfigure()` | [Virtual Source Refresh](Workflows.md#virtual-source-refresh)<br/>[Virtual Source Delete](Workflows.md#virtual-source-delete)
 [Virtual Source<br/>Reconfigure](#virtual-source-reconfigure) | **Yes** | `virtual.reconfigure()` | [Virtual Source Rollback](Workflows.md#virtual-source-rollback)<br/>[Virtual Source Enable](Workflows.md#virtual-source-enable)
@@ -593,6 +594,68 @@ def linked_mount_specification(staged_source, repository):
   return MountSpecification([mount], ownership_spec)
 ```
 
+## Virtual Source Initialize
+
+Initializes a brand-new [empty VDB](Glossary.md#empty-vdb). As with all VDBs, this new dataset will have access to mounted Delphix Engine storage, but of course there will be no data there at first.
+
+The job of the plugin is to do whatever is necessary to set up a new dataset from scratch. For example, this might involve running a `CREATE DATABASE` command.  This is an optional operation -- users will not be allowed to create empty VDBs for plugins that choose not to implement this operation.
+
+As with the `configure` operation, this `initialize` operation must return source config parameters that represent the new dataset.
+
+### Required / Optional
+**Optional.**
+
+### Delphix Engine Operations
+
+* [Virtual Source Create Empty VDB](Workflows.md#virtual-source-create-empty-vdb)
+
+### Signature
+
+`def initialize(virtual_source, repository)`
+
+### Decorator
+
+`virtual.initialize()`
+
+### Arguments
+
+Argument | Type | Description
+-------- | ---- | -----------
+virtual_source | [VirtualSource](Classes.md#virtualsource) | The source associated with this operation.
+repository | [RepositoryDefinition](Schemas_and_Autogenerated_Classes.md#repositorydefinition-class) | The repository associated with this source.
+
+### Returns
+[SourceConfigDefinition](Schemas_and_Autogenerated_Classes.md#sourceconfigdefinition-class)
+
+### Example
+
+```python
+from dlpx.virtualization.platform import Plugin
+from generated.defintions import SourceConfigDefinition
+
+plugin = Plugin()
+
+@plugin.virtual.initialize()
+def initialize(virtual_source, repository):
+  source_config = SourceConfigDefinition(name="config_name")
+  return source_config
+```
+
+> The above command assumes a [SourceConfig Schema](Schemas_and_Autogenerated_Classes.md#sourceconfig-schema) defined as:
+
+```json
+{
+  "type": "object",
+  "required": ["name"],
+  "additionalProperties": false,
+  "properties": {
+    "name": { "type": "string" }
+  },
+  "identityFields": ["name"],
+  "nameField": ["name"]
+}
+```
+
 ## Virtual Source Configure
 
 Configures the data in a particular snapshot to be usable on a target environment. For database data files, this may mean recovering from a crash consistent format or backup. For application files, this may mean reconfiguring XML files or rewriting hostnames and symlinks.

docs/docs/References/Schemas.md

@@ -112,6 +112,9 @@ Here is a JSON object that conforms to the above schema:
 
 For much more detail on JSON schemas, including which keywords are available, what they mean, and where you can use them, see <https://json-schema.org/understanding-json-schema/>.
 
+!!! info
+    Be careful when using the JSON schema keyword `default`. This keyword is commonly misunderstood. Specifically, it does not mean "If the user does not provide a value, then this default value will be auto-substituted instead", as you might expect. In fact, in JSON schemas, `default` has no semantic meaning at all! Currently, the only thing the Delphix Engine will use this for is to pre-fill widgets on the UI.
+
 
 ### Delphix-specific Extensions to JSON Schema
 
@@ -309,7 +312,7 @@ This will allow the plugin author to ask the user to select [environments](/Refe
     "type": "string",
     "format": "reference",
     "referenceType": "HOST_USER",
-    "matches”: "env"
+    "matches": "env"
   }
 }
 ```
@@ -337,7 +340,7 @@ The `referenceType` keyword is used to specify the [reference](#reference) type.
     "type": "string",
     "format": "reference",
     "referenceType": "HOST_USER",
-    "matches”: "env"
+    "matches": "env"
   }
 }
 ```
@@ -362,7 +365,7 @@ The `matches` keyword is used to map an [environment user](/References/Glossary.
     "type": "string",
     "format": "reference",
     "referenceType": "HOST_USER",
-    "matches”: "env"
+    "matches": "env"
   }
 }
 ```

docs/docs/References/Workflows.md

@@ -32,6 +32,10 @@
 
 ![Screenshot](images/VirtualSourceSnapshot.png)
 
+## Virtual Source Create Empty VDB
+
+![Screenshot](images/VirtualSourceCreateEmpty.png)
+
 ## Virtual Source Refresh
 
 ![Screenshot](images/VirtualSourceRefresh.png)