Fixes #250 Support empty VDBs

common/setup.py

@@ -4,7 +4,7 @@
 PYTHON_SRC = 'src/main/python'
 
 install_requires = [
-  "dvp-api == 1.3.0",
+  "dvp-api == 1.4.0.dev10",
 ]
 
 with open(os.path.join(PYTHON_SRC, 'dlpx/virtualization/common/VERSION')) as version_file:

docs/docs/References/Decorators.md

@@ -13,13 +13,13 @@ plugin = Plugin()
 # Use the decorator to annotate the function that corresponds to the "Virtual Source Start" Plugin Operation
 @plugin.virtual_source.start()
 def my_start(virtual_source, repository, source_config):
-  print "running start" 
+  print "running start"
 ```
 
 !!! info
     Decorators exposed by the Virtualization SDK are inherently python function calls and needs parentheses `()` appended at the end.
 
-Assuming the name of the object, is `plugin` as above, the table below lists the corresponding decorators for each plugin operation.
+Assuming the name of the object is `plugin` as above, the table below lists the corresponding decorators for each plugin operation.
 
 Plugin Operation | Decorator
 ---------------- |  --------
@@ -35,6 +35,7 @@ Plugin Operation | Decorator
 [Staged Linked Source Worker](Plugin_Operations.md#staged-linked-source-worker) | `@plugin.linked.worker()`
 [Staged Linked Source Mount Specification](Plugin_Operations.md#staged-linked-source-mount-specification) | `@plugin.linked.mount_specification()`
 [Virtual Source Configure](Plugin_Operations.md#virtual-source-configure) | `@plugin.virtual.configure()`
+[Virtual Source Initialize](Plugin_Operations.md#virtual-source-initialize) | `@plugin.virtual.initialize()`
 [Virtual Source Unconfigure](Plugin_Operations.md#virtual-source-unconfigure) | `@plugin.virtual.unconfigure()`
 [Virtual Source Reconfigure](Plugin_Operations.md#virtual-source-reconfigure) | `@plugin.virtual.reconfigure()`
 [Virtual Source Start](Plugin_Operations.md#virtual-source-start) | `@plugin.virtual.start()`

docs/docs/References/Glossary.md

@@ -28,6 +28,13 @@ The process by which the Delphix Engine learns about how a particular environmen
 ## dSource
 See [Linked Dataset](#linked-dataset)
 
+## Empty VDB
+A VDB that is created from scratch, without provisioning from another dataset. Users can create empty VDBs when they want to construct a brand-new dataset from within Delphix, instead of creating it externally and then ingesting it.
+
+This "empty" VDB, of course, will typically not stay empty for long. Data will be added as users work with the new dataset.
+
+A plugin can support this functionality by implementing the [initialize](Plugin_Operations.md#virtual-source-initialize) operation.
+
 ## Environment
 A remote system that the Delphix Engine can interact with. An environment can be used as a [source](#source-environment), [staging](#staging-environment) or [target](#target-environment) environment (or any combination of those).  For example, a Linux machine that the Delphix Engine can connect to is an environment.
 
@@ -105,7 +112,7 @@ The process by which the Delphix Engine ingests data from a dataset on a [source
 An [environment](#environment) on which Delphix-provided virtualized datasets can be used.
 
 ## Lua Toolkit
-Legacy model for writing "plugins" in Lua, with limited documentation and support for writing, building and uploading toolkits. This was the predecessor to the Virtualization SDK. 
+Legacy model for writing "plugins" in Lua, with limited documentation and limited support for writing, building and uploading toolkits. This was the predecessor to the Virtualization SDK.
 
 ## Upgrade Operation
 A special plugin operation that takes data produced by an older version of a plugin, and transforms it into the format expected by the new version of the plugin.

docs/docs/References/Plugin_Operations.md

@@ -22,6 +22,7 @@ Plugin Operation | **Required** | Decorator | Delphix Engine Operations
 [Staged Linked Source<br/>Status](#staged-linked-source-status) | **No** |`linked.status()` | N/A
 [Staged Linked Source<br/>Worker](#staged-linked-source-worker) | **No** |`linked.worker()` | N/A
 [Staged Linked Source<br/>Mount Specification](#staged-linked-source-mount-specification) | **Yes** | `linked.mount_specification()` | [Linked Source Sync](Workflows.md#linked-source-sync)<br/>[Linked Source Enable](Workflows.md#linked-source-enable)
+[Virtual Source<br/>Initialize](#virtual-source-initialize) | **No** | `virtual.initialize()` | [Virtual Source Create Empty VDB](Workflows.md#virtual-source-create-empty-vdb)
 [Virtual Source<br/>Configure](#virtual-source-configure) | **Yes** | `virtual.configure()` | [Virtual Source Provision](Workflows.md#virtual-source-provision)<br/>[Virtual Source Refresh](Workflows.md#virtual-source-refresh)
 [Virtual Source<br/>Unconfigure](#virtual-source-unconfigure) | **No** | `virtual.unconfigure()` | [Virtual Source Refresh](Workflows.md#virtual-source-refresh)<br/>[Virtual Source Delete](Workflows.md#virtual-source-delete)
 [Virtual Source<br/>Reconfigure](#virtual-source-reconfigure) | **Yes** | `virtual.reconfigure()` | [Virtual Source Rollback](Workflows.md#virtual-source-rollback)<br/>[Virtual Source Enable](Workflows.md#virtual-source-enable)
@@ -593,6 +594,68 @@ def linked_mount_specification(staged_source, repository):
   return MountSpecification([mount], ownership_spec)
 ```
 
+## Virtual Source Initialize
+
+Initializes a brand-new [empty VDB](Glossary.md#empty-vdb). As with all VDBs, this new dataset will have access to mounted Delphix Engine storage, but of course there will be no data there at first.
+
+The job of the plugin is to do whatever is necessary to set up a new dataset from scratch. For example, this might involve running a `CREATE DATABASE` command.  This is an optional operation -- users will not be allowed to create empty VDBs for plugins that choose not to implement this operation.
+
+As with the `configure` operation, this `initialize` operation must return source config parameters that represent the new dataset.
+
+### Required / Optional
+**Optional.**
+
+### Delphix Engine Operations
+
+* [Virtual Source Create Empty VDB](Workflows.md#virtual-source-create-empty-vdb)
+
+### Signature
+
+`def initialize(virtual_source, repository)`
+
+### Decorator
+
+`virtual.initialize()`
+
+### Arguments
+
+Argument | Type | Description
+-------- | ---- | -----------
+virtual_source | [VirtualSource](Classes.md#virtualsource) | The source associated with this operation.
+repository | [RepositoryDefinition](Schemas_and_Autogenerated_Classes.md#repositorydefinition-class) | The repository associated with this source.
+
+### Returns
+[SourceConfigDefinition](Schemas_and_Autogenerated_Classes.md#sourceconfigdefinition-class)
+
+### Example
+
+```python
+from dlpx.virtualization.platform import Plugin
+from generated.defintions import SourceConfigDefinition
+
+plugin = Plugin()
+
+@plugin.virtual.initialize()
+def initialize(virtual_source, repository):
+  source_config = SourceConfigDefinition(name="config_name")
+  return source_config
+```
+
+> The above command assumes a [SourceConfig Schema](Schemas_and_Autogenerated_Classes.md#sourceconfig-schema) defined as:
+
+```json
+{
+  "type": "object",
+  "required": ["name"],
+  "additionalProperties": false,
+  "properties": {
+    "name": { "type": "string" }
+  },
+  "identityFields": ["name"],
+  "nameField": ["name"]
+}
+```
+
 ## Virtual Source Configure
 
 Configures the data in a particular snapshot to be usable on a target environment. For database data files, this may mean recovering from a crash consistent format or backup. For application files, this may mean reconfiguring XML files or rewriting hostnames and symlinks.

docs/docs/References/Workflows.md

@@ -32,6 +32,10 @@
 
 ![Screenshot](images/VirtualSourceSnapshot.png)
 
+## Virtual Source Create Empty VDB
+
+![Screenshot](images/VirtualSourceCreateEmpty.png)
+
 ## Virtual Source Refresh
 
 ![Screenshot](images/VirtualSourceRefresh.png)

docs/docs/References/html/VirtualSourceCreateEmpty.html

@@ -0,0 +1,12 @@
+<!DOCTYPE html>
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head>
+<title>Draw.io Diagram</title>
+<meta http-equiv="refresh" content="0;URL='https://www.draw.io/?state=%7B%22ids%22:%5B%221HFcufpN6oaA0JSLXy53Q9dzmEO26Iqng%22%5D,%22action%22:%22open%22,%22userId%22:%22110114316315334056404%22%7D#G1HFcufpN6oaA0JSLXy53Q9dzmEO26Iqng'"/>
+<meta charset="utf-8"/>
+</head>
+<body>
+<div class="mxgraph" style="max-width:100%;border:1px solid transparent;" data-mxgraph="{&quot;highlight&quot;:&quot;#0000ff&quot;,&quot;nav&quot;:true,&quot;resize&quot;:true,&quot;xml&quot;:&quot;&lt;mxfile modified=\&quot;2019-03-18T00:59:12.121Z\&quot; host=\&quot;www.draw.io\&quot; agent=\&quot;Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36\&quot; etag=\&quot;SG5n2WI_aCI_iGwCprHQ\&quot; version=\&quot;10.4.9\&quot; type=\&quot;google\&quot;&gt;&lt;diagram id=\&quot;-ZfKtbs5jN34PJB2tePp\&quot; name=\&quot;Page-1\&quot;&gt;7Vjbbts4EP0aA7sPa+hiy9Jj4rRZYFEggNt1+1Qw0kgiSoleio7tfn2HFmmRktKk3tywWAcIOIfkUJxzyBlpEi6r/bUgm/IDz4BNAi/bT8KrSRAkfoz/FXBogVk8a4FC0KyF/A5Y0e+gQU+jW5pB4wyUnDNJNy6Y8rqGVDoYEYLv3GE5Z+6qG1LAAFilhA3RNc1k2aLx3OvwP4EWpVnZ93RPRcxgDTQlyfjOgsJ3k3ApOJdtq9ovganYmbi0897f03t6MAG1fMyEal1dxZ/S9Sf4+Fd6Gf9zs16LP7SXO8K2esOTIGLo77LZkBrbhWr/TYXcEuVsxbciBWzcCH5HG8prM/5WmNErSYSEbNhhEHxG27sOjzyYmDdS8G+ncGOgLktZMWz66rlKslHjqn2hBDfNGd+lJa44lSAqWhPJBQ7blVTCakNSNXaHIxHLKWNLzlQ/LhNmBOI8VS6P61k9URrDbX56NDvGJmCAW9xbkI75NfAKpDjgEN0784N2ij4A80CfiF0nJ9/IqbSkFGmMaAUXJ9cdydjQPP8C58GA81/jd0whV7yGN094nkOUjhKeLZJbPLjPQni88F6Z8HBA+F1L+BRvzZwWWwG//T4gRfBtnUGmWRgLr0USuW0420q4EKm+w49oZ81OYbdp7hM0V3+jJ/L4UzN4LS28/T0RcaFL3MIfOan+CHFm3pMTNx+5nXssQZ1dqDSHVspI09DUJQbDIQ6f0fCmc2N+UaYxrvaOdbCtGxAUdwLCgHsqLV9ofTHLYLvzpAzj6F5imvaq+UluSnS+J6IA+fCFBpmTyoc8WzyOnT+DCWBE0ju3ABjjVq9wwylurZNRvHBlNO+d63bjepadsHuOwpnnXiRJz1EbmIGjo9RO2z5ffdETqO9FBRM+Ui9GWG9EMEHi8pzMzhTMQHlmoy8kmMUrXldnCE0Z/RvuX9xWj72svP+m+F77tjJat9S3NNXNkh3L054WVWVA8U3vgtGiVoJEzpUQnOJzULKYchQ1QhgDxrEqrRDfWEpy+iyJjVZRbp1llUzH1dq2Nw1UkZPySh2Yo1J7pRP42RwWY6VTEi1CEj1Vbdu7YZLFsESajZVIz/Yy4w1oN8VthYGVX5sNpDRHniW+zPxf5iKHnsthFCZDDqOXLHODsa8QbzhxnJ8kjFofzBLhm0oSs8SVzDyJz0sSQS9JRHHP0dlJAs3uU1o7vPseGb77AQ==&lt;/diagram&gt;&lt;/mxfile&gt;&quot;,&quot;toolbar&quot;:&quot;pages zoom layers lightbox&quot;,&quot;page&quot;:0}"></div>
+<a style="position:absolute;top:50%;left:50%;margin-top:-128px;margin-left:-64px;" href="https://www.draw.io/?state=%7B%22ids%22:%5B%221HFcufpN6oaA0JSLXy53Q9dzmEO26Iqng%22%5D,%22action%22:%22open%22,%22userId%22:%22110114316315334056404%22%7D#G1HFcufpN6oaA0JSLXy53Q9dzmEO26Iqng" target="_blank"><img border="0" src="https://www.draw.io/images/drawlogo128.png"/></a>
+</body>
+</html>

libs/setup.py

@@ -7,7 +7,7 @@
     version = version_file.read().strip()
 
 install_requires = [
-  "dvp-api == 1.3.0",
+  "dvp-api == 1.4.0.dev10",
   "dvp-common == {}".format(version)
 ]
 

platform/setup.py

@@ -7,7 +7,7 @@
     version = version_file.read().strip()
 
 install_requires = [
-  "dvp-api == 1.3.0",
+  "dvp-api == 1.4.0.dev10",
   "dvp-common == {}".format(version),
   "enum34;python_version < '3.4'",
 ]

platform/src/main/python/dlpx/virtualization/platform/_virtual.py

@@ -626,15 +626,18 @@ def _internal_initialize(self, request):
 
         repository = RepositoryDefinition.from_dict(
             json.loads(request.repository.parameters.json))
-        source_config = SourceConfigDefinition.from_dict(
-            json.loads(request.source_config.parameters.json))
 
-        self.initialize_impl(repository=repository,
-                             source_config=source_config,
+        config = self.initialize_impl(repository=repository,
                              virtual_source=virtual_source)
+
+        # Validate that this is a SourceConfigDefinition object.
+        if not isinstance(config, SourceConfigDefinition):
+            raise IncorrectReturnTypeError(Op.VIRTUAL_INITIALIZE, type(config),
+                                           SourceConfigDefinition)
+
         initialize_response = platform_pb2.InitializeResponse()
-        initialize_response.return_value.CopyFrom(
-            platform_pb2.InitializeResult())
+        initialize_response.return_value.source_config.parameters.json = (
+            json.dumps(config.to_dict()))
         return initialize_response
 
     def _internal_mount_specification(self, request):

platform/src/test/python/dlpx/virtualization/test_plugin.py

@@ -686,25 +686,24 @@ def virtual_status_impl(virtual_source, repository, source_config):
     def test_virtual_initialize(my_plugin, virtual_source, repository,
                                 source_config):
         @my_plugin.virtual.initialize()
-        def virtual_initialize_impl(virtual_source, repository, source_config):
+        def virtual_initialize_impl(virtual_source, repository):
             TestPlugin.assert_plugin_args(virtual_source=virtual_source,
-                                          repository=repository,
-                                          source_config=source_config)
-            return
+                                          repository=repository)
+            return SourceConfigDefinition(repository.name)
 
         initialize_request = platform_pb2.InitializeRequest()
         TestPlugin.setup_request(request=initialize_request,
                                  virtual_source=virtual_source,
-                                 repository=repository,
-                                 source_config=source_config)
+                                 repository=repository)
 
-        expected_result = platform_pb2.InitializeResult()
+        expected_source_config = TEST_REPOSITORY_JSON
         initialize_response = my_plugin.virtual._internal_initialize(
             initialize_request)
 
         # Check that the response's oneof is set to return_value and not error
         assert initialize_response.WhichOneof('result') == 'return_value'
-        assert initialize_response.return_value == expected_result
+        actual_source_config = initialize_response.return_value.source_config
+        assert actual_source_config.parameters.json == TEST_REPOSITORY_JSON
 
     @staticmethod
     def test_virtual_mount_spec(my_plugin, virtual_source, repository):

tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_importer.yaml

@@ -75,7 +75,6 @@ EXPECTED_STAGED_ARGS_BY_OP:
     initialize_impl:
       - virtual_source
       - repository
-      - source_config
     mount_specification_impl:
       - virtual_source
       - repository
@@ -134,7 +133,6 @@ EXPECTED_DIRECT_ARGS_BY_OP:
     initialize_impl:
       - virtual_source
       - repository
-      - source_config
     mount_specification_impl:
       - virtual_source
       - repository

tools/src/test/python/dlpx/virtualization/_internal/test_package_util.py

@@ -14,7 +14,7 @@ def test_get_version():
 
     @staticmethod
     def test_get_virtualization_api_version():
-        assert package_util.get_virtualization_api_version() == '1.3.0'
+        assert package_util.get_virtualization_api_version() == '1.4.0'
 
     @staticmethod
     def test_get_engine_api_version():
@@ -25,7 +25,7 @@ def test_get_build_api_version_json():
         build_api_version = {
             'type': 'APIVersion',
             'major': 1,
-            'minor': 3,
+            'minor': 4,
             'micro': 0
         }
         assert package_util.get_build_api_version() == build_api_version