diff --git a/common/setup.py b/common/setup.py index 64e5643c..a5e92681 100644 --- a/common/setup.py +++ b/common/setup.py @@ -4,7 +4,7 @@ PYTHON_SRC = 'src/main/python' install_requires = [ - "dvp-api == 1.3.0", + "dvp-api == 1.4.0.dev10", ] with open(os.path.join(PYTHON_SRC, 'dlpx/virtualization/common/VERSION')) as version_file: diff --git a/docs/docs/References/Decorators.md b/docs/docs/References/Decorators.md index 20158783..038cf01c 100644 --- a/docs/docs/References/Decorators.md +++ b/docs/docs/References/Decorators.md @@ -13,13 +13,13 @@ plugin = Plugin() # Use the decorator to annotate the function that corresponds to the "Virtual Source Start" Plugin Operation @plugin.virtual_source.start() def my_start(virtual_source, repository, source_config): - print "running start" + print "running start" ``` !!! info Decorators exposed by the Virtualization SDK are inherently python function calls and needs parentheses `()` appended at the end. -Assuming the name of the object, is `plugin` as above, the table below lists the corresponding decorators for each plugin operation. +Assuming the name of the object is `plugin` as above, the table below lists the corresponding decorators for each plugin operation. Plugin Operation | Decorator ---------------- | -------- @@ -35,6 +35,7 @@ Plugin Operation | Decorator [Staged Linked Source Worker](Plugin_Operations.md#staged-linked-source-worker) | `@plugin.linked.worker()` [Staged Linked Source Mount Specification](Plugin_Operations.md#staged-linked-source-mount-specification) | `@plugin.linked.mount_specification()` [Virtual Source Configure](Plugin_Operations.md#virtual-source-configure) | `@plugin.virtual.configure()` +[Virtual Source Initialize](Plugin_Operations.md#virtual-source-initialize) | `@plugin.virtual.initialize()` [Virtual Source Unconfigure](Plugin_Operations.md#virtual-source-unconfigure) | `@plugin.virtual.unconfigure()` [Virtual Source Reconfigure](Plugin_Operations.md#virtual-source-reconfigure) | `@plugin.virtual.reconfigure()` [Virtual Source Start](Plugin_Operations.md#virtual-source-start) | `@plugin.virtual.start()` diff --git a/docs/docs/References/Glossary.md b/docs/docs/References/Glossary.md index a1038684..3d439050 100644 --- a/docs/docs/References/Glossary.md +++ b/docs/docs/References/Glossary.md @@ -28,6 +28,13 @@ The process by which the Delphix Engine learns about how a particular environmen ## dSource See [Linked Dataset](#linked-dataset) +## Empty VDB +A VDB that is created from scratch, without provisioning from another dataset. Users can create empty VDBs when they want to construct a brand-new dataset from within Delphix, instead of creating it externally and then ingesting it. + +This "empty" VDB, of course, will typically not stay empty for long. Data will be added as users work with the new dataset. + +A plugin can support this functionality by implementing the [initialize](Plugin_Operations.md#virtual-source-initialize) operation. + ## Environment A remote system that the Delphix Engine can interact with. An environment can be used as a [source](#source-environment), [staging](#staging-environment) or [target](#target-environment) environment (or any combination of those). For example, a Linux machine that the Delphix Engine can connect to is an environment. @@ -105,7 +112,7 @@ The process by which the Delphix Engine ingests data from a dataset on a [source An [environment](#environment) on which Delphix-provided virtualized datasets can be used. ## Lua Toolkit -Legacy model for writing "plugins" in Lua, with limited documentation and support for writing, building and uploading toolkits. This was the predecessor to the Virtualization SDK. +Legacy model for writing "plugins" in Lua, with limited documentation and limited support for writing, building and uploading toolkits. This was the predecessor to the Virtualization SDK. ## Upgrade Operation A special plugin operation that takes data produced by an older version of a plugin, and transforms it into the format expected by the new version of the plugin. diff --git a/docs/docs/References/Plugin_Operations.md b/docs/docs/References/Plugin_Operations.md index c0321ac2..2fe76756 100644 --- a/docs/docs/References/Plugin_Operations.md +++ b/docs/docs/References/Plugin_Operations.md @@ -22,6 +22,7 @@ Plugin Operation | **Required** | Decorator | Delphix Engine Operations [Staged Linked Source
Status](#staged-linked-source-status) | **No** |`linked.status()` | N/A [Staged Linked Source
Worker](#staged-linked-source-worker) | **No** |`linked.worker()` | N/A [Staged Linked Source
Mount Specification](#staged-linked-source-mount-specification) | **Yes** | `linked.mount_specification()` | [Linked Source Sync](Workflows.md#linked-source-sync)
[Linked Source Enable](Workflows.md#linked-source-enable) +[Virtual Source
Initialize](#virtual-source-initialize) | **No** | `virtual.initialize()` | [Virtual Source Create Empty VDB](Workflows.md#virtual-source-create-empty-vdb) [Virtual Source
Configure](#virtual-source-configure) | **Yes** | `virtual.configure()` | [Virtual Source Provision](Workflows.md#virtual-source-provision)
[Virtual Source Refresh](Workflows.md#virtual-source-refresh) [Virtual Source
Unconfigure](#virtual-source-unconfigure) | **No** | `virtual.unconfigure()` | [Virtual Source Refresh](Workflows.md#virtual-source-refresh)
[Virtual Source Delete](Workflows.md#virtual-source-delete) [Virtual Source
Reconfigure](#virtual-source-reconfigure) | **Yes** | `virtual.reconfigure()` | [Virtual Source Rollback](Workflows.md#virtual-source-rollback)
[Virtual Source Enable](Workflows.md#virtual-source-enable) @@ -593,6 +594,68 @@ def linked_mount_specification(staged_source, repository): return MountSpecification([mount], ownership_spec) ``` +## Virtual Source Initialize + +Initializes a brand-new [empty VDB](Glossary.md#empty-vdb). As with all VDBs, this new dataset will have access to mounted Delphix Engine storage, but of course there will be no data there at first. + +The job of the plugin is to do whatever is necessary to set up a new dataset from scratch. For example, this might involve running a `CREATE DATABASE` command. This is an optional operation -- users will not be allowed to create empty VDBs for plugins that choose not to implement this operation. + +As with the `configure` operation, this `initialize` operation must return source config parameters that represent the new dataset. + +### Required / Optional +**Optional.** + +### Delphix Engine Operations + +* [Virtual Source Create Empty VDB](Workflows.md#virtual-source-create-empty-vdb) + +### Signature + +`def initialize(virtual_source, repository)` + +### Decorator + +`virtual.initialize()` + +### Arguments + +Argument | Type | Description +-------- | ---- | ----------- +virtual_source | [VirtualSource](Classes.md#virtualsource) | The source associated with this operation. +repository | [RepositoryDefinition](Schemas_and_Autogenerated_Classes.md#repositorydefinition-class) | The repository associated with this source. + +### Returns +[SourceConfigDefinition](Schemas_and_Autogenerated_Classes.md#sourceconfigdefinition-class) + +### Example + +```python +from dlpx.virtualization.platform import Plugin +from generated.defintions import SourceConfigDefinition + +plugin = Plugin() + +@plugin.virtual.initialize() +def initialize(virtual_source, repository): + source_config = SourceConfigDefinition(name="config_name") + return source_config +``` + +> The above command assumes a [SourceConfig Schema](Schemas_and_Autogenerated_Classes.md#sourceconfig-schema) defined as: + +```json +{ + "type": "object", + "required": ["name"], + "additionalProperties": false, + "properties": { + "name": { "type": "string" } + }, + "identityFields": ["name"], + "nameField": ["name"] +} +``` + ## Virtual Source Configure Configures the data in a particular snapshot to be usable on a target environment. For database data files, this may mean recovering from a crash consistent format or backup. For application files, this may mean reconfiguring XML files or rewriting hostnames and symlinks. diff --git a/docs/docs/References/Workflows.md b/docs/docs/References/Workflows.md index 8bf50999..d58ed815 100644 --- a/docs/docs/References/Workflows.md +++ b/docs/docs/References/Workflows.md @@ -32,6 +32,10 @@ ![Screenshot](images/VirtualSourceSnapshot.png) +## Virtual Source Create Empty VDB + +![Screenshot](images/VirtualSourceCreateEmpty.png) + ## Virtual Source Refresh ![Screenshot](images/VirtualSourceRefresh.png) diff --git a/docs/docs/References/html/VirtualSourceCreateEmpty.html b/docs/docs/References/html/VirtualSourceCreateEmpty.html new file mode 100644 index 00000000..0a039f91 --- /dev/null +++ b/docs/docs/References/html/VirtualSourceCreateEmpty.html @@ -0,0 +1,12 @@ + + + +Draw.io Diagram + + + + +
+ + + diff --git a/docs/docs/References/images/VirtualSourceCreateEmpty.png b/docs/docs/References/images/VirtualSourceCreateEmpty.png new file mode 100644 index 00000000..911b1072 Binary files /dev/null and b/docs/docs/References/images/VirtualSourceCreateEmpty.png differ diff --git a/libs/setup.py b/libs/setup.py index d3cff739..1713aa99 100644 --- a/libs/setup.py +++ b/libs/setup.py @@ -7,7 +7,7 @@ version = version_file.read().strip() install_requires = [ - "dvp-api == 1.3.0", + "dvp-api == 1.4.0.dev10", "dvp-common == {}".format(version) ] diff --git a/platform/setup.py b/platform/setup.py index b4a46018..3d9daf3c 100644 --- a/platform/setup.py +++ b/platform/setup.py @@ -7,7 +7,7 @@ version = version_file.read().strip() install_requires = [ - "dvp-api == 1.3.0", + "dvp-api == 1.4.0.dev10", "dvp-common == {}".format(version), "enum34;python_version < '3.4'", ] diff --git a/platform/src/main/python/dlpx/virtualization/platform/_virtual.py b/platform/src/main/python/dlpx/virtualization/platform/_virtual.py index 6c048c46..2b2318a9 100644 --- a/platform/src/main/python/dlpx/virtualization/platform/_virtual.py +++ b/platform/src/main/python/dlpx/virtualization/platform/_virtual.py @@ -626,15 +626,18 @@ def _internal_initialize(self, request): repository = RepositoryDefinition.from_dict( json.loads(request.repository.parameters.json)) - source_config = SourceConfigDefinition.from_dict( - json.loads(request.source_config.parameters.json)) - self.initialize_impl(repository=repository, - source_config=source_config, + config = self.initialize_impl(repository=repository, virtual_source=virtual_source) + + # Validate that this is a SourceConfigDefinition object. + if not isinstance(config, SourceConfigDefinition): + raise IncorrectReturnTypeError(Op.VIRTUAL_INITIALIZE, type(config), + SourceConfigDefinition) + initialize_response = platform_pb2.InitializeResponse() - initialize_response.return_value.CopyFrom( - platform_pb2.InitializeResult()) + initialize_response.return_value.source_config.parameters.json = ( + json.dumps(config.to_dict())) return initialize_response def _internal_mount_specification(self, request): diff --git a/platform/src/test/python/dlpx/virtualization/test_plugin.py b/platform/src/test/python/dlpx/virtualization/test_plugin.py index 7a2060f8..5f90bdaa 100755 --- a/platform/src/test/python/dlpx/virtualization/test_plugin.py +++ b/platform/src/test/python/dlpx/virtualization/test_plugin.py @@ -686,25 +686,24 @@ def virtual_status_impl(virtual_source, repository, source_config): def test_virtual_initialize(my_plugin, virtual_source, repository, source_config): @my_plugin.virtual.initialize() - def virtual_initialize_impl(virtual_source, repository, source_config): + def virtual_initialize_impl(virtual_source, repository): TestPlugin.assert_plugin_args(virtual_source=virtual_source, - repository=repository, - source_config=source_config) - return + repository=repository) + return SourceConfigDefinition(repository.name) initialize_request = platform_pb2.InitializeRequest() TestPlugin.setup_request(request=initialize_request, virtual_source=virtual_source, - repository=repository, - source_config=source_config) + repository=repository) - expected_result = platform_pb2.InitializeResult() + expected_source_config = TEST_REPOSITORY_JSON initialize_response = my_plugin.virtual._internal_initialize( initialize_request) # Check that the response's oneof is set to return_value and not error assert initialize_response.WhichOneof('result') == 'return_value' - assert initialize_response.return_value == expected_result + actual_source_config = initialize_response.return_value.source_config + assert actual_source_config.parameters.json == TEST_REPOSITORY_JSON @staticmethod def test_virtual_mount_spec(my_plugin, virtual_source, repository): diff --git a/tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_importer.yaml b/tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_importer.yaml index bb95c585..98e41c19 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_importer.yaml +++ b/tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_importer.yaml @@ -75,7 +75,6 @@ EXPECTED_STAGED_ARGS_BY_OP: initialize_impl: - virtual_source - repository - - source_config mount_specification_impl: - virtual_source - repository @@ -134,7 +133,6 @@ EXPECTED_DIRECT_ARGS_BY_OP: initialize_impl: - virtual_source - repository - - source_config mount_specification_impl: - virtual_source - repository diff --git a/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/successful.py b/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/successful.py index 24d57a34..e8ef3d1c 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/successful.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/successful.py @@ -33,6 +33,11 @@ def configure(virtual_source, repository, snapshot): name = "VDB mounted to " + path return None +@direct.virtual.initialize() +def initialize(virtual_source, repository): + path = virtual_source.parameters.path + name = "VDB mounted to " + path + return None @direct.virtual.mount_specification() def mount_specification(repository, virtual_source): diff --git a/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/staged/successful.py b/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/staged/successful.py index 31ae1151..0d6ca414 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/staged/successful.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/staged/successful.py @@ -65,6 +65,9 @@ def staged_worker(repository, source_config, staged_source): def configure(virtual_source, repository, snapshot): return None +@staged.virtual.initialize() +def initialize(virtual_source, repository): + return None @staged.virtual.mount_specification() def mount_specification(virtual_source, repository): diff --git a/tools/src/test/python/dlpx/virtualization/_internal/test_package_util.py b/tools/src/test/python/dlpx/virtualization/_internal/test_package_util.py index fe978f7b..4813de52 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/test_package_util.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/test_package_util.py @@ -14,7 +14,7 @@ def test_get_version(): @staticmethod def test_get_virtualization_api_version(): - assert package_util.get_virtualization_api_version() == '1.3.0' + assert package_util.get_virtualization_api_version() == '1.4.0' @staticmethod def test_get_engine_api_version(): @@ -25,7 +25,7 @@ def test_get_build_api_version_json(): build_api_version = { 'type': 'APIVersion', 'major': 1, - 'minor': 3, + 'minor': 4, 'micro': 0 } assert package_util.get_build_api_version() == build_api_version