diff --git a/.dependabot/config.yaml b/.dependabot/config.yaml
index 2502bd72..df419c78 100644
--- a/.dependabot/config.yaml
+++ b/.dependabot/config.yaml
@@ -3,21 +3,21 @@ update_configs:
# Keep requirements.txt files up-to-date in each package.
- package_manager: "python"
directory: "/dvp"
- update_schedule: "daily"
+ update_schedule: "monthly"
target_branch: "develop"
- package_manager: "python"
directory: "/common"
- update_schedule: "daily"
+ update_schedule: "monthly"
target_branch: "develop"
- package_manager: "python"
directory: "/platform"
- update_schedule: "daily"
+ update_schedule: "monthly"
target_branch: "develop"
- package_manager: "python"
directory: "/libs"
- update_schedule: "daily"
+ update_schedule: "monthly"
target_branch: "develop"
- package_manager: "python"
directory: "/tools"
- update_schedule: "daily"
+ update_schedule: "monthly"
target_branch: "develop"
diff --git a/common/setup.py b/common/setup.py
index 3ee5ffbf..64e5643c 100644
--- a/common/setup.py
+++ b/common/setup.py
@@ -4,7 +4,7 @@
PYTHON_SRC = 'src/main/python'
install_requires = [
- "dvp-api == 1.2.0.dev3",
+ "dvp-api == 1.3.0",
]
with open(os.path.join(PYTHON_SRC, 'dlpx/virtualization/common/VERSION')) as version_file:
diff --git a/docs/docs/Best_Practices/Sensitive_Data.md b/docs/docs/Best_Practices/Sensitive_Data.md
index b2183b48..166d15fe 100644
--- a/docs/docs/Best_Practices/Sensitive_Data.md
+++ b/docs/docs/Best_Practices/Sensitive_Data.md
@@ -35,7 +35,7 @@ This tells the Delphix Engine to take special precautions with this password pro
4. Clients of the Delphix Engine's public API will not be able to access the password.
!!! note
- Removing a previously added password property from a field and running a [Data Migration](Glossary.md#data-migration) will expose the password in plaintext. If this is intentional, write a migration to ensure that the new property conforms to the new schema.
+ Removing a previously added password property from a field and running a [Data Migration](/References/Glossary.md#data-migration) will expose the password in plaintext. If this is intentional, write a migration to ensure that the new property conforms to the new schema.
# Using Environment Variables For Remote Data Passing
diff --git a/docs/docs/Building_Your_First_Plugin/Discovery.md b/docs/docs/Building_Your_First_Plugin/Discovery.md
index 67eb9e2a..dcff0993 100644
--- a/docs/docs/Building_Your_First_Plugin/Discovery.md
+++ b/docs/docs/Building_Your_First_Plugin/Discovery.md
@@ -164,7 +164,7 @@ The Delphix Engine will pass us information about the source environment in an a
return [RepositoryDefinition(name='1e87dc30-3cdb-4f0a-9634-07ce017d20d1')]
```
-This creates and returns a Python object that corresponds to the format defined by our repository schema. Because out repository has exactly one string property called `name`, therefore this Python object has one property called `name`.
+This creates and returns a Python object that corresponds to the format defined by our repository schema. Because our repository has exactly one string property called `name`, therefore this Python object has one property called `name`.
Notice that the code generator has filled in the value of `name` with a random string. This results in a plugin operation that works, but which will not be very helpful for the user. We'll change this later.
diff --git a/docs/docs/Building_Your_First_Plugin/Initial_Setup.md b/docs/docs/Building_Your_First_Plugin/Initial_Setup.md
index 5012daa4..d2b0424e 100644
--- a/docs/docs/Building_Your_First_Plugin/Initial_Setup.md
+++ b/docs/docs/Building_Your_First_Plugin/Initial_Setup.md
@@ -4,7 +4,7 @@ Before we begin to start writing plugin code, we will need to do some setup work
The quoted examples in this section assume you're working on a Unix-like system.
-## Sanity check
+## Quick Check
First a reminder that it's highly recommended that you develop your plugin in a [virtual environment](https://virtualenv.pypa.io/en/latest/).
@@ -53,14 +53,14 @@ To start, we will create a new directory where our new plugin code will live.
Now that we are in our new plugin directory, we can use the `dvp` tool to create a plugin for us. This plugin will be a mere skeleton -- it will not do anything useful until we modify it in the subsequent pages.
```
-(venv) first_plugin$ dvp init -n first_plugin -s STAGED -p WINDOWS
+(venv) first_plugin$ dvp init -n first_plugin -s STAGED -t WINDOWS
```
The `-n` argument here means "plugin name." We are using the name `first_plugin`.
The `-s` argument tells which syncing strategy we want to use.
-The `-p` argument tells which host platform our plugin supports.
+The `-t` argument tells which host platform our plugin supports.
You can type `dvp init -h` for more information about the options available.
diff --git a/docs/docs/References/Glossary.md b/docs/docs/References/Glossary.md
index 56e6e0d2..a1038684 100644
--- a/docs/docs/References/Glossary.md
+++ b/docs/docs/References/Glossary.md
@@ -104,6 +104,9 @@ The process by which the Delphix Engine ingests data from a dataset on a [source
## Target Environment
An [environment](#environment) on which Delphix-provided virtualized datasets can be used.
+## Lua Toolkit
+Legacy model for writing "plugins" in Lua, with limited documentation and support for writing, building and uploading toolkits. This was the predecessor to the Virtualization SDK.
+
## Upgrade Operation
A special plugin operation that takes data produced by an older version of a plugin, and transforms it into the format expected by the new version of the plugin.
diff --git a/docs/docs/Release_Notes/2.1.0/2.1.0.md b/docs/docs/Release_Notes/2.1.0/2.1.0.md
index f254d51e..d3e05867 100644
--- a/docs/docs/Release_Notes/2.1.0/2.1.0.md
+++ b/docs/docs/Release_Notes/2.1.0/2.1.0.md
@@ -4,8 +4,12 @@ To install or upgrade the SDK, refer to instructions [here](/Getting_Started.md#
## New & Improved
-* Added the ability to migrate existing Lua toolkits to plugins.
- * [Convert](/Versioning_And_Upgrade/Lua_Toolkit_To_SDK_Plugin_Migration/Converting_Migration_Scripts.md) any existing Lua upgrade scripts to Python migrations by using our new Lua upgrade [decorators](/Versioning_And_Upgrade/Lua_Toolkit_To_SDK_Plugin_Migration/Decorators.md).
- * Added new fields `luaName` and `minimumLuaVersion` to the [Plugin Config](/Versioning_And_Upgrade/Lua_Toolkit_To_SDK_Plugin_Migration/Plugin_Config.md). These properties allow a plugin to specify which Lua toolkit(s) the plugin can upgrade.
+* Added the ability to migrate existing [Lua toolkits](/References/Glossary.md#lua-toolkit) to SDK plugins.
+ * [Convert](/Versioning_And_Upgrade/Lua_Toolkit_To_SDK_Plugin_Migration/Converting_Migration_Scripts.md) any existing Lua upgrade scripts to Python migrations by using new Lua upgrade [decorators](/Versioning_And_Upgrade/Lua_Toolkit_To_SDK_Plugin_Migration/Decorators.md).
+ * Added new optional fields `luaName` and `minimumLuaVersion` to the [Plugin Config](/Versioning_And_Upgrade/Lua_Toolkit_To_SDK_Plugin_Migration/Plugin_Config.md). These properties allow a plugin to specify which Lua toolkit(s) the plugin can migrate.
- More details about Lua Upgrade can be found [here](/Versioning_And_Upgrade/Lua_Toolkit_To_SDK_Plugin_Migration/Overview.md).
\ No newline at end of file
+ More details about Lua toolkit migration can be found [here](/Versioning_And_Upgrade/Lua_Toolkit_To_SDK_Plugin_Migration/Overview.md).
+
+## Breaking Changes
+
+No breaking changes in this release!
\ No newline at end of file
diff --git a/docs/docs/Versioning_And_Upgrade/Lua_Toolkit_To_SDK_Plugin_Migration/Converting_Migration_Scripts.md b/docs/docs/Versioning_And_Upgrade/Lua_Toolkit_To_SDK_Plugin_Migration/Converting_Migration_Scripts.md
index 8f77955e..03c56fea 100644
--- a/docs/docs/Versioning_And_Upgrade/Lua_Toolkit_To_SDK_Plugin_Migration/Converting_Migration_Scripts.md
+++ b/docs/docs/Versioning_And_Upgrade/Lua_Toolkit_To_SDK_Plugin_Migration/Converting_Migration_Scripts.md
@@ -1,8 +1,8 @@
# Converting Lua Upgrade Scripts to Python Data Migrations
-To convert migrations (a.k.a. "upgrade scripts") that were originally written in lua, we need to get the version that the migration upgrades from, the object type the migration is written for, and lastly convert the code into python code using the [decorators](Decorators.md) described previously.
+To convert migrations (a.k.a. "upgrade scripts") that were originally written in Lua, we need to get the version that the migration upgrades from, the object type the migration is written for, and lastly convert the code into Python code using the [decorators](Decorators.md) described previously.
## Example
-Assume there are two versions of a lua toolkit, 1.0.0 and 1.1.0 where the 1.1.0 version is following the basic toolkit directory structure (actually containing all operations):
+Assume there are two versions of a lua toolkit, `1.0.0` and `1.1.0` where the `1.1.0` version is following the basic toolkit directory structure (actually containing all operations):
```
├── main.json
diff --git a/docs/docs/Versioning_And_Upgrade/Lua_Toolkit_To_SDK_Plugin_Migration/Decorators.md b/docs/docs/Versioning_And_Upgrade/Lua_Toolkit_To_SDK_Plugin_Migration/Decorators.md
index df82e3d0..f5eb8d1d 100644
--- a/docs/docs/Versioning_And_Upgrade/Lua_Toolkit_To_SDK_Plugin_Migration/Decorators.md
+++ b/docs/docs/Versioning_And_Upgrade/Lua_Toolkit_To_SDK_Plugin_Migration/Decorators.md
@@ -1,6 +1,6 @@
# Decorators
-The Virtualization SDK exposes [decorators](/References/Decorators.md) as mentioned in the regular documentation. Below we list the additional operations added to suport lua to python migrations. This assumes the name of the `Plugin()` object is `plugin`:
+The Virtualization SDK exposes [decorators](/References/Decorators.md) as mentioned in the regular documentation. Below we list the additional operations added to suport Lua to Python migrations. This assumes `Plugin()` is instantiated as `plugin`:
Plugin Operation | Decorator
---------------- | --------
@@ -11,5 +11,4 @@ Plugin Operation | Decorator
[Lua Snapshot Data Migration](Plugin_Operations.md#lua-snapshot-data-migration) | `@plugin.upgrade.snapshot(lua_version, MigrationType.LUA)`
!!! info "lua_version format"
- The lua_version field in this decorator should be the major minor string of the lua toolkit. This means if in the main.json file the version is set to "1.1.HOTFIX123" the lua_version passed into this decorator should be "1.1".
-
+ The `lua_version` field in this decorator should be the (major,minor) version of the Lua toolkit. This means if the version is set to `1.1.HOTFIX123` in the `main.json` file for the Lua toolkit, the `lua_version` passed into this decorator should be `1.1`.
diff --git a/docs/docs/Versioning_And_Upgrade/Lua_Toolkit_To_SDK_Plugin_Migration/Overview.md b/docs/docs/Versioning_And_Upgrade/Lua_Toolkit_To_SDK_Plugin_Migration/Overview.md
index a48470b0..92293400 100644
--- a/docs/docs/Versioning_And_Upgrade/Lua_Toolkit_To_SDK_Plugin_Migration/Overview.md
+++ b/docs/docs/Versioning_And_Upgrade/Lua_Toolkit_To_SDK_Plugin_Migration/Overview.md
@@ -1,10 +1,10 @@
# Overview
-Before the Virtualization SDK was written, Delphix only supported toolkits written in Lua. There was only limited documentation to help people write, build, and upload toolkits. Multiple toolkits were released and are still used by customers today, so as we move towards only supporting SDK Plugins, there must be a way to get customers off of Lua toolkits and onto SDK plugins.
+Before the Virtualization SDK was written, Delphix only supported [toolkits](/References/Glossary.md#lua-toolkit) written in Lua. There was only limited documentation to help people write, build, and upload toolkits. Multiple toolkits were released and are still used by customers today, so as we move towards only supporting SDK Plugins, there needs to be a way to get customers off of Lua toolkits and onto SDK plugins.
If you are reading this and have no idea what a Lua toolkit is, there is no reason to read further into this section. Everything written in these pages will assume the goal is to write specific code as part of a plugin to convert objects created using Lua toolkits to use the newly uploaded Python plugin.
-In the next few pages, we also make the assuption that you've written both a Lua toolkit and a python plugin before and know some of the terminology already established. If this is not true, please try [building a plugin](/Building_Your_First_Plugin/Overview.md) and [writing some upgrade migrations](/Versioning_And_Upgrade/Upgrade.md) first before coming back here to learn how to add upgrading from Lua toolkits into the mix as described below.
+In the next few pages, we also make the assumption that you've written both a Lua toolkit and a Python plugin before and know some of the terminology already established. If this is not true, please try [building a plugin](/Building_Your_First_Plugin/Overview.md) and [writing some upgrade migrations](/Versioning_And_Upgrade/Upgrade.md) first before coming back here to learn how to add upgrading from Lua toolkits into the mix as described below.
## Basic no-schema Migration
One way to migrate from a Lua toolkit to a plugin is to write an exactly equivalent plugin that does not make any [schema](/References/Schemas.md) changes to the objects that were defined originally in the Lua toolkit. If this is the scenario you are in, then you only need to update the [plugin config](Plugin_Config.md) with a couple of new Lua migration specific fields.
@@ -17,4 +17,4 @@ The other way to migrate from a Lua toolkit to a plugin is to wait and write a p
Having the ability to define Lua upgrade operations in the new plugin code means that older Lua version migration scripts can be [converted](Converting_Migration_Scripts.md), enabling multi-step upgrades from older Lua versions to migrate and become plugins.
!!! warning "New versions of a Lua toolkit is strongly discouraged after Python Plugin is written"
- After having written a Plugin to migrate a specific Lua toolkit, while possible, you should avoid writing new major/minor versions of the toolkit in Lua. Patch releases with no schema changes can still be done. If you need to write a new Lua toolkit version please contact Delphix Support to get help on updating migrations accordingly.
+ After having written a Plugin to migrate a specific Lua toolkit, while possible, you should avoid writing new major/minor versions of the toolkit in Lua. Patch releases with no schema changes can still be done. If you need to write a new Lua toolkit version please contact the Delphix Virtualization SDK Engineering team to get help on updating migrations accordingly.
diff --git a/docs/docs/Versioning_And_Upgrade/Lua_Toolkit_To_SDK_Plugin_Migration/Plugin_Config.md b/docs/docs/Versioning_And_Upgrade/Lua_Toolkit_To_SDK_Plugin_Migration/Plugin_Config.md
index 9b3499ca..dec70fef 100644
--- a/docs/docs/Versioning_And_Upgrade/Lua_Toolkit_To_SDK_Plugin_Migration/Plugin_Config.md
+++ b/docs/docs/Versioning_And_Upgrade/Lua_Toolkit_To_SDK_Plugin_Migration/Plugin_Config.md
@@ -1,12 +1,12 @@
# Plugin Config
-For all regular fields in a plugin config go [here](/References/Plugin_Config.md). The following fields described are the ones needed to migrate lua toolkits to python plugins.
+For all regular fields in a plugin config go [here](/References/Plugin_Config.md). The following fields described are the ones needed to migrate Lua toolkits to Python plugins.
## Fields
|Field Name|Required|Type|Description|
|----------|:------:|:--:|-----------|
-|luaName|N|string|The name of the lua toolkit this plugin should upgrade from. This field is required if the minimumLuaVersion is defined.|
-|minimumLuaVersion|N|string|The lowest major minor version of the lua toolkit that upgrade is supported from. This field is required if the luaName is defined.|
+|luaName|N|string|The name of the Lua toolkit this plugin should upgrade from. This field is required if the minimumLuaVersion is defined.|
+|minimumLuaVersion|N|string|The lowest major minor version of the Lua toolkit that upgrade is supported from. This field is required if the luaName is defined.|
## Example
Assume a lua toolkit with the following `main.json` file:
@@ -139,5 +139,7 @@ buildNumber: 2.0.0
```
!!! info "`id` and `luaName` fields in plugins versus `name` field in toolkits"
- * If the `id` of the plugin being uploaded happens to match the `name` in the toolkit already installed on the engine, the upload will fail regardless of what the `luaName` is. Otherwise, the `luaName` will be used to determine if an already uploaded lua toolkit is considered a lower version of the plugin being uploaded. If the `luaName` is not set then no lua toolkit will be upgraded.
+ * The `luaName` will be used to determine if an already uploaded Lua toolkit is considered a lower version of the Pyhon plugin being uploaded.
+ * If the `luaName` is not set then no Lua toolkit will be upgraded.
+ * If the `id` of the plugin being uploaded happens to match the `name` in the Lua toolkit already installed on the Delphix Engine, the upload will fail regardless of what the `luaName` is.
* When uploading a plugin with the `luaName` set, that `luaName` and `id` pair will be the only pair uploaded successfully. Uploading a new plugin with the same `luaName` but different `id` will fail.
diff --git a/docs/docs/Versioning_And_Upgrade/Lua_Toolkit_To_SDK_Plugin_Migration/Plugin_Operations.md b/docs/docs/Versioning_And_Upgrade/Lua_Toolkit_To_SDK_Plugin_Migration/Plugin_Operations.md
index 99f56cee..3d728f68 100644
--- a/docs/docs/Versioning_And_Upgrade/Lua_Toolkit_To_SDK_Plugin_Migration/Plugin_Operations.md
+++ b/docs/docs/Versioning_And_Upgrade/Lua_Toolkit_To_SDK_Plugin_Migration/Plugin_Operations.md
@@ -1,7 +1,7 @@
# Plugin Operations
## Summary
-Included below are all plugin operations that have to do with lua migrations. For regular operation information go [here](/References/Plugin_Operations.md).
+Plugin operations related to Lua migrations are listed below. Information regarding other operations is [here](/References/Plugin_Operations.md).
Plugin Operation | **Required** | Decorator | Delphix Engine Operations
@@ -15,7 +15,7 @@ Plugin Operation | **Required** | Decorator | Delphix Engine Operations
## Lua Repository Data Migration
-A Lua Repository [Data Migration](/References/Glossary.md#data-migration) migrates repository data from an older [schema](/References/Glossary.md#schema) format defined originally from a lua toolkit to an updated schema format defined in the Python plugin.
+A Lua Repository [Data Migration](/References/Glossary.md#data-migration) migrates repository data from an older [schema](/References/Glossary.md#schema) format defined originally from a Lua toolkit to an updated schema format defined in the Python plugin.
### Required / Optional
**Optional.**
@@ -39,13 +39,13 @@ A Lua Repository [Data Migration](/References/Glossary.md#data-migration) migrat
Argument | Type | Description
-------- | ---- | -----------
-lua_version | String | The lua version of the migration that the upgrade would be migrating from. This is the ID of this migration. The version here is actually just the major and minor version of the lua toolkit therefore each defined migration's lua_version per repository data migration must be unique.
-migration_type | String | This migration type field indicates whether the operation is lua or just a regular data migration. Signify this as LUA so that the major minor version can be used. If not defined, this operation will default to a regular [repository data migration](/References/Plugin_Operations.md#repository-data-migration).
+lua_version | String | The Lua version of the toolkit that this migration would be applicable to. This is the ID of this migration. The version here is actually just the major and minor version of the Lua toolkit. Therefore the `lua_version` for each repository data migration must be unique.
+migration_type | String | This field indicates whether the operation is a Lua migration or just a regular data migration. Specify this as LUA to indicate a Lua migration. If not defined, this operation will default to a regular [repository data migration](/References/Plugin_Operations.md#repository-data-migration).
### Function Arguments
Argument | Type | Description
-------- | ---- | -----------
-old_repository | Dictionary | The plugin-specific data associated with a repository, that conforms to the previous schema defined in lua.
+old_repository | Dictionary | The plugin-specific data associated with a repository, that conforms to the previous schema defined in Lua.
!!! warning
The function argument `old_repository` is a Python dictionary, where each property name appears exactly as described in the previous repository schema. This differs from non-upgrade-related operations, where the function arguments are [autogenerated classes](/References/Schemas_and_Autogenerated_Classes.md) based on the schema.
@@ -62,7 +62,7 @@ from dlpx.virtualization.platform import Plugin
plugin = Plugin()
@plugin.upgrade.repository("1.1", MigrationType.LUA)
-def add_new_flag_to_repo(old_repository):
+def add_new_flag_to_repository(old_repository):
new_repository = dict(old_repository)
new_repository["useNewFeature"] = False
return new_repository
@@ -70,7 +70,7 @@ def add_new_flag_to_repo(old_repository):
## Lua Source Config Data Migration
-A Lua Source Config [Data Migration](/References/Glossary.md#data-migration) migrates source config data from an older [schema](/References/Glossary.md#schema) format defined originally from a lua toolkit to an updated schema format defined in the Python plugin.
+A Lua Source Config [Data Migration](/References/Glossary.md#data-migration) migrates source config data from an older [schema](/References/Glossary.md#schema) format defined originally from a Lua toolkit to an updated schema format defined in the Python plugin.
### Required / Optional
**Optional.**
@@ -94,8 +94,8 @@ A Lua Source Config [Data Migration](/References/Glossary.md#data-migration) mig
Argument | Type | Description
-------- | ---- | -----------
-lua_version | String | The lua version of the migration that the upgrade would be migrating from. This is the ID of this migration. The version here is actually just the major and minor version of the lua toolkit therefore each defined migration's lua_version per source config data migration must be unique.
-migration_type | String | This migration type field indicates whether the operation is lua or just a regular data migration. Signify this as LUA so that the major minor version can be used. If not defined, this operation will default to a regular [source config data migration](/References/Plugin_Operations.md#source-config-data-migration).
+lua_version | String | The Lua version of the toolkit that this migration would be applicable to. This is the ID of this migration. The version here is actually just the major and minor version of the Lua toolkit. Therefore the `lua_version` for each repository data migration must be unique.
+migration_type | String | This field indicates whether the operation is a Lua migration or just a regular data migration. Specify this as LUA to indicate a Lua migration. If not defined, this operation will default to a regular [source config data migration](/References/Plugin_Operations.md#source-config-data-migration).
### Function Arguments
Argument | Type | Description
@@ -124,7 +124,7 @@ def add_new_flag_to_source_config(old_source_config):
```
## Lua Linked Source Data Migration
-A Lua Linked Source [Data Migration](/References/Glossary.md#data-migration) migrates linked source data from an older [schema](/References/Glossary.md#schema) format defined originally from a lua toolkit to an updated schema format defined in the Python plugin.
+A Lua Linked Source [Data Migration](/References/Glossary.md#data-migration) migrates linked source data from an older [schema](/References/Glossary.md#schema) format defined originally from a Lua toolkit to an updated schema format defined in the Python plugin.
### Required / Optional
**Optional.**
@@ -148,8 +148,8 @@ A Lua Linked Source [Data Migration](/References/Glossary.md#data-migration) mig
Argument | Type | Description
-------- | ---- | -----------
-lua_version | String | The lua version of the migration that the upgrade would be migrating from. This is the ID of this migration. The version here is actually just the major and minor version of the lua toolkit therefore each defined migration's lua_version per linked source data migration must be unique.
-migration_type | String | This migration type field indicates whether the operation is lua or just a regular data migration. Signify this as LUA so that the major minor version can be used. If not defined, this operation will default to a regular [linked source data migration](/References/Plugin_Operations.md#linked-source-data-migration).
+lua_version | String | The Lua version of the toolkit that this migration would be applicable to. This is the ID of this migration. The version here is actually just the major and minor version of the Lua toolkit. Therefore the `lua_version` for each repository data migration must be unique.
+migration_type | String | This field indicates whether the operation is a Lua migration or just a regular data migration. Specify this as LUA to indicate a Lua migration. If not defined, this operation will default to a regular [linked source data migration](/References/Plugin_Operations.md#linked-source-data-migration).
### Function Arguments
Argument | Type | Description
@@ -171,14 +171,14 @@ from dlpx.virtualization.platform import Plugin
plugin = Plugin()
@plugin.upgrade.linked_source("1.1", MigrationType.LUA)
-def add_new_flag_to_dsource(old_linked_source):
+def add_new_flag_to_linked_source(old_linked_source):
new_linked_source = dict(old_linked_source)
new_linked_source["useNewFeature"] = False
return new_linked_source
```
## Lua Virtual Source Data Migration
-A Lua Virtual Source [Data Migration](/References/Glossary.md#data-migration) migrates virtual source data from an older [schema](/References/Glossary.md#schema) format defined originally from a lua toolkit to an updated schema format defined in the Python plugin.
+A Lua Virtual Source [Data Migration](/References/Glossary.md#data-migration) migrates virtual source data from an older [schema](/References/Glossary.md#schema) format defined originally from a Lua toolkit to an updated schema format defined in the Python plugin.
### Required / Optional
**Optional.**
@@ -202,8 +202,8 @@ A Lua Virtual Source [Data Migration](/References/Glossary.md#data-migration) mi
Argument | Type | Description
-------- | ---- | -----------
-lua_version | String | The lua version of the migration that the upgrade would be migrating from. This is the ID of this migration. The version here is actually just the major and minor version of the lua toolkit therefore each defined migration's lua_version per virtual source data migration must be unique.
-migration_type | String | This migration type field indicates whether the operation is lua or just a regular data migration. Signify this as LUA so that the major minor version can be used. If not defined, this operation will default to a regular [virtual source data migration](/References/Plugin_Operations.md#virtual-source-data-migration).
+lua_version | String | The Lua version of the toolkit that this migration would be applicable to. This is the ID of this migration. The version here is actually just the major and minor version of the Lua toolkit. Therefore the `lua_version` for each repository data migration must be unique.
+migration_type | String | This field indicates whether the operation is a Lua migration or just a regular data migration. Specify this as LUA to indicate a Lua migration. If not defined, this operation will default to a regular [virtual source data migration](/References/Plugin_Operations.md#virtual-source-data-migration).
### Function Arguments
Argument | Type | Description
@@ -225,14 +225,14 @@ from dlpx.virtualization.platform import Plugin
plugin = Plugin()
@plugin.upgrade.virtual_source("1.1", MigrationType.LUA)
-def add_new_flag_to_vdb(old_virtual_source):
+def add_new_flag_to_virtual_source(old_virtual_source):
new_virtual_source = dict(old_virtual_source)
new_virtual_source["useNewFeature"] = False
return new_virtual_source
```
## Lua Snapshot Data Migration
-A Lua Snapshot [Data Migration](/References/Glossary.md#data-migration) migrates snapshot data from an older [schema](/References/Glossary.md#schema) format defined originally from a lua toolkit to an updated schema format defined in the Python plugin.
+A Lua Snapshot [Data Migration](/References/Glossary.md#data-migration) migrates snapshot data from an older [schema](/References/Glossary.md#schema) format defined originally from a Lua toolkit to an updated schema format defined in the Python plugin.
### Required / Optional
**Optional.**
@@ -256,8 +256,8 @@ A Lua Snapshot [Data Migration](/References/Glossary.md#data-migration) migrates
Argument | Type | Description
-------- | ---- | -----------
-lua_version | String | The lua version of the migration that the upgrade would be migrating from. This is the ID of this migration. The version here is actually just the major and minor version of the lua toolkit therefore each defined migration's lua_version per snapshot data migration must be unique.
-migration_type | String | This migration type field indicates whether the operation is lua or just a regular data migration. Signify this as LUA so that the major minor version can be used. If not defined, this operation will default to a regular [snapshot data migration](/References/Plugin_Operations.md#snapshot-data-migration).
+lua_version | String | The Lua version of the toolkit that this migration would be applicable to. This is the ID of this migration. The version here is actually just the major and minor version of the Lua toolkit. Therefore the `lua_version` for each repository data migration must be unique.
+migration_type | String | This field indicates whether the operation is a Lua migration or just a regular data migration. Specify this as LUA to indicate a Lua migration. If not defined, this operation will default to a regular [snapshot data migration](/References/Plugin_Operations.md#snapshot-data-migration).
### Function Arguments
Argument | Type | Description
diff --git a/libs/setup.py b/libs/setup.py
index 6a5afd94..d3cff739 100644
--- a/libs/setup.py
+++ b/libs/setup.py
@@ -7,7 +7,7 @@
version = version_file.read().strip()
install_requires = [
- "dvp-api == 1.2.0.dev3",
+ "dvp-api == 1.3.0",
"dvp-common == {}".format(version)
]
diff --git a/platform/setup.py b/platform/setup.py
index 87d07aca..b4a46018 100644
--- a/platform/setup.py
+++ b/platform/setup.py
@@ -7,7 +7,7 @@
version = version_file.read().strip()
install_requires = [
- "dvp-api == 1.2.0.dev3",
+ "dvp-api == 1.3.0",
"dvp-common == {}".format(version),
"enum34;python_version < '3.4'",
]
diff --git a/tools/src/test/python/dlpx/virtualization/_internal/commands/test_build.py b/tools/src/test/python/dlpx/virtualization/_internal/commands/test_build.py
index a207b31e..a2077b33 100644
--- a/tools/src/test/python/dlpx/virtualization/_internal/commands/test_build.py
+++ b/tools/src/test/python/dlpx/virtualization/_internal/commands/test_build.py
@@ -697,3 +697,81 @@ def test_minimum_lua_version_parameter(plugin_config_content, src_dir,
upload_artifact = build.prepare_upload_artifact(
plugin_config_content, src_dir, schema_content, {})
assert expected == upload_artifact.get('minimumLuaVersion')
+
+ @staticmethod
+ @pytest.mark.parametrize('build_number', ['1.0.1'])
+ def test_build_change_and_build_again(plugin_config_content, src_dir,
+ schema_content):
+ upload_artifact = build.prepare_upload_artifact(
+ plugin_config_content, src_dir, schema_content, {})
+ assert plugin_config_content['buildNumber'] == upload_artifact['buildNumber']
+ changed_build_number = '7.2.12'
+ changed_host_type = ['WINDOWS']
+ plugin_config_content['buildNumber'] = changed_build_number
+ plugin_config_content['hostTypes'] = changed_host_type
+ upload_artifact_2 = build.prepare_upload_artifact(
+ plugin_config_content, src_dir, schema_content, {})
+ assert changed_build_number == upload_artifact_2.get('buildNumber')
+ assert changed_host_type == upload_artifact_2.get('hostTypes')
+
+ @staticmethod
+ @pytest.mark.parametrize('repository_definition',
+ [{
+ 'type': 'object',
+ 'properties': {
+ 'name': {
+ 'type': 'badDataType'
+ }
+ },
+ 'nameField': 'name',
+ 'identityFields': ['name']
+ }])
+ @mock.patch('dlpx.virtualization._internal.codegen.generate_python')
+ def test_bad_data_type_in_schema(mock_generate_python,
+ plugin_config_file,
+ artifact_file):
+ with pytest.raises(exceptions.UserError) as err_info:
+ build.build(plugin_config_file, artifact_file, False, False)
+
+ message = err_info.value.message
+ exp_error = "'badDataType' is not valid under any of the given schemas"
+ assert exp_error in message
+
+ assert not mock_generate_python.called
+
+ @staticmethod
+ @pytest.mark.parametrize('host_types', [''])
+ @mock.patch('dlpx.virtualization._internal.codegen.generate_python')
+ def test_empty_host_type(mock_generate_python, plugin_config_file,
+ artifact_file):
+ with pytest.raises(exceptions.UserError) as err_info:
+ build.build(plugin_config_file, artifact_file, False, False)
+
+ message = err_info.value.message
+ assert "Validation failed" in message
+ assert not mock_generate_python.called
+
+ @staticmethod
+ @pytest.mark.parametrize('plugin_name', [''])
+ @mock.patch('dlpx.virtualization._internal.codegen.generate_python')
+ def test_empty_plugin_name(mock_generate_python, plugin_config_file,
+ artifact_file):
+ with pytest.raises(exceptions.UserError) as err_info:
+ build.build(plugin_config_file, artifact_file, False, False)
+
+ message = err_info.value.message
+ assert "Validation failed" in message
+ assert not mock_generate_python.called
+
+ @staticmethod
+ @mock.patch('os.path.isabs', return_value=False)
+ def test_non_existing_entry_file(mock_relative_path, plugin_config_file,
+ plugin_config_content, artifact_file):
+ entry_module, _ = plugin_config_content['entryPoint'].split(':')
+
+ with pytest.raises(exceptions.UserError) as err_info:
+ build.build(plugin_config_file, artifact_file, False, False)
+
+ message = err_info.value.message
+ exp_message = "No module named {module}".format(module=entry_module)
+ assert exp_message in message
diff --git a/tools/src/test/python/dlpx/virtualization/_internal/commands/test_initialize.py b/tools/src/test/python/dlpx/virtualization/_internal/commands/test_initialize.py
index 5486eaa6..62fa9dfe 100644
--- a/tools/src/test/python/dlpx/virtualization/_internal/commands/test_initialize.py
+++ b/tools/src/test/python/dlpx/virtualization/_internal/commands/test_initialize.py
@@ -114,6 +114,20 @@ def test_init(tmpdir, ingestion_strategy, host_type, schema_template,
assert contents == format_entry_point_template(
config['id'], ingestion_strategy, host_type)
+ @staticmethod
+ def test_init_with_relative_path(tmpdir):
+ os.chdir(tmpdir.strpath)
+ init.init(".", const.DIRECT_TYPE, "", const.UNIX_HOST_TYPE)
+
+ result = plugin_util.validate_plugin_config_file(
+ os.path.join(tmpdir.strpath, init.DEFAULT_PLUGIN_CONFIG_FILE),
+ True)
+
+ config = result.plugin_config_content
+
+ # Validate that the plugin name is equal to plugin id
+ assert config['name'] == config['id']
+
@staticmethod
def test_init_without_plugin_name(tmpdir):
init.init(tmpdir.strpath, const.DIRECT_TYPE, "", const.UNIX_HOST_TYPE)
diff --git a/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/bad_syntax.py b/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/bad_syntax.py
new file mode 100644
index 00000000..c0024806
--- /dev/null
+++ b/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/bad_syntax.py
@@ -0,0 +1,17 @@
+#
+# Copyright (c) 2020 by Delphix. All rights reserved.
+#
+# flake8: noqa
+from dlpx.virtualization.platform import Plugin
+
+plugin = Plugin()
+
+
+@plugin.discovery.repository()
+def repository_discovery(source_connection)
+ return None
+
+
+@plugin.discovery.source_config()
+def source_config_discovery(source_connection, repository)
+ return None
diff --git a/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/import_error.py b/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/import_error.py
new file mode 100644
index 00000000..6ec26e2a
--- /dev/null
+++ b/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/import_error.py
@@ -0,0 +1,17 @@
+#
+# Copyright (c) 2020 by Delphix. All rights reserved.
+#
+# flake8: noqa
+from dlpxxx.virtualization.platform import Plugin
+
+plugin = Plugin()
+
+
+@plugin.discovery.repository()
+def repository_discovery(source_connection):
+ return None
+
+
+@plugin.discovery.source_config()
+def source_config_discovery(source_connection, repository):
+ return None
diff --git a/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/undefined_name.py b/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/undefined_name.py
new file mode 100644
index 00000000..db4ec97a
--- /dev/null
+++ b/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/undefined_name.py
@@ -0,0 +1,17 @@
+#
+# Copyright (c) 2020 by Delphix. All rights reserved.
+#
+# flake8: noqa
+from dlpx.virtualization.platform import Plugin
+
+plugin = Plugin()
+
+
+@directplugin.discovery.repository()
+def repository_discovery(source_connection):
+ return None
+
+
+@plugin.discovery.source_config()
+def source_config_discovery(source_connection, repository):
+ return None
diff --git a/tools/src/test/python/dlpx/virtualization/_internal/test_cli.py b/tools/src/test/python/dlpx/virtualization/_internal/test_cli.py
index f9a98dc7..9ef87f31 100644
--- a/tools/src/test/python/dlpx/virtualization/_internal/test_cli.py
+++ b/tools/src/test/python/dlpx/virtualization/_internal/test_cli.py
@@ -163,6 +163,39 @@ def test_invalid_ingestion_strategy(plugin_name):
assert result.exit_code != 0
+ @staticmethod
+ def test_blank_ingestion_strategy(plugin_name):
+ runner = click_testing.CliRunner()
+
+ result = runner.invoke(
+ cli.delphix_sdk,
+ ['init', '-n', plugin_name, '-s', ''])
+
+ assert result.exit_code != 0
+ assert "invalid choice" in result.output
+
+ @staticmethod
+ def test_non_existent_root_dir(plugin_name):
+ runner = click_testing.CliRunner()
+
+ result = runner.invoke(
+ cli.delphix_sdk,
+ ['init', '-n', plugin_name, '-r', '/file/does/not/exist'])
+
+ assert result.exit_code != 0
+ assert "'/file/does/not/exist' does not exist" in result.output
+
+ @staticmethod
+ def test_empty_root_dir(plugin_name):
+ runner = click_testing.CliRunner()
+
+ result = runner.invoke(
+ cli.delphix_sdk,
+ ['init', '-n', plugin_name, '-r', ''])
+
+ assert result.exit_code != 0
+ assert "Invalid value for '-r'" in result.output
+
@staticmethod
def test_name_required():
runner = click_testing.CliRunner()
@@ -385,6 +418,35 @@ def test_with_dev_fail(mock_build, plugin_config_file, artifact_file,
assert result.exit_code == 2
assert not mock_build.called, 'build should not have been called'
+ @staticmethod
+ @pytest.mark.parametrize('plugin_config_file',
+ [''])
+ def test_empty_config_file(plugin_config_file):
+ runner = click_testing.CliRunner()
+ result = runner.invoke(cli.delphix_sdk,
+ ['build', '-c', plugin_config_file])
+
+ assert result.exit_code == 2
+ expected_error_strings = ("Error: Invalid value for '-c'",
+ "is a directory")
+ for expected_error in expected_error_strings:
+ assert expected_error in result.output
+
+ @staticmethod
+ @pytest.mark.parametrize('artifact_file',
+ [''])
+ def test_empty_artifact_file(plugin_config_file, artifact_file):
+ runner = click_testing.CliRunner()
+ result = runner.invoke(cli.delphix_sdk,
+ ['build', '-c', plugin_config_file,
+ '-a', artifact_file])
+
+ assert result.exit_code == 2
+ expected_error_strings = ("Error: Invalid value for '-a'",
+ "is a directory")
+ for expected_error in expected_error_strings:
+ assert expected_error in result.output
+
class TestUploadCli:
@staticmethod
diff --git a/tools/src/test/python/dlpx/virtualization/_internal/test_package_util.py b/tools/src/test/python/dlpx/virtualization/_internal/test_package_util.py
index f1cc4849..fe978f7b 100644
--- a/tools/src/test/python/dlpx/virtualization/_internal/test_package_util.py
+++ b/tools/src/test/python/dlpx/virtualization/_internal/test_package_util.py
@@ -14,7 +14,7 @@ def test_get_version():
@staticmethod
def test_get_virtualization_api_version():
- assert package_util.get_virtualization_api_version() == '1.2.0'
+ assert package_util.get_virtualization_api_version() == '1.3.0'
@staticmethod
def test_get_engine_api_version():
@@ -25,7 +25,7 @@ def test_get_build_api_version_json():
build_api_version = {
'type': 'APIVersion',
'major': 1,
- 'minor': 2,
+ 'minor': 3,
'micro': 0
}
assert package_util.get_build_api_version() == build_api_version
diff --git a/tools/src/test/python/dlpx/virtualization/_internal/test_plugin_importer.py b/tools/src/test/python/dlpx/virtualization/_internal/test_plugin_importer.py
index 7a1ce1ab..a8795765 100644
--- a/tools/src/test/python/dlpx/virtualization/_internal/test_plugin_importer.py
+++ b/tools/src/test/python/dlpx/virtualization/_internal/test_plugin_importer.py
@@ -244,3 +244,73 @@ def test_plugin_info_warn_mode(mock_import, mock_relative_path,
plugin_config_content, False)
except Exception:
raise AssertionError()
+
+ @staticmethod
+ @pytest.mark.parametrize(
+ 'entry_point,plugin_type,expected_errors',
+ [('successful:ne_symbol', 'DIRECT', [
+ "Error: Entry point 'successful:ne_symbol' does not exist.",
+ "'ne_symbol' is not a symbol in module ",
+ ])])
+ @mock.patch('dlpx.virtualization._internal.file_util.get_src_dir_path')
+ def test_non_existing_symbol_in_module(mock_file_util, plugin_config_file,
+ fake_src_dir, expected_errors):
+ mock_file_util.return_value = fake_src_dir
+
+ with pytest.raises(exceptions.UserError) as err_info:
+ importer = get_plugin_importer(plugin_config_file)
+ importer.validate_plugin_module()
+
+ message = err_info.value.message
+ for error in expected_errors:
+ assert error in message
+
+ @staticmethod
+ @pytest.mark.parametrize(
+ 'entry_point,expected_error',
+ [('import_error:plugin',
+ "Error: No module named dlpxxx.virtualization.platform")])
+ @mock.patch('dlpx.virtualization._internal.file_util.get_src_dir_path')
+ def test_import_error(mock_file_util, plugin_config_file,
+ fake_src_dir, expected_error):
+ mock_file_util.return_value = fake_src_dir
+
+ with pytest.raises(exceptions.UserError) as err_info:
+ importer = get_plugin_importer(plugin_config_file)
+ importer.validate_plugin_module()
+
+ message = err_info.value.message
+ assert expected_error in message
+
+ @staticmethod
+ @pytest.mark.parametrize(
+ 'entry_point,expected_error',
+ [('bad_syntax:plugin', "SDK Error: invalid syntax")])
+ @mock.patch('dlpx.virtualization._internal.file_util.get_src_dir_path')
+ def test_bad_syntax(mock_file_util, plugin_config_file,
+ fake_src_dir, expected_error):
+ mock_file_util.return_value = fake_src_dir
+
+ with pytest.raises(exceptions.SDKToolingError) as err_info:
+ importer = get_plugin_importer(plugin_config_file)
+ importer.validate_plugin_module()
+
+ message = err_info.value.message
+ assert expected_error in message
+
+ @staticmethod
+ @pytest.mark.parametrize(
+ 'entry_point,expected_error',
+ [('undefined_name:plugin',
+ "SDK Error: name 'directplugin' is not defined")])
+ @mock.patch('dlpx.virtualization._internal.file_util.get_src_dir_path')
+ def test_undefined_name_error(mock_file_util, plugin_config_file,
+ fake_src_dir, expected_error):
+ mock_file_util.return_value = fake_src_dir
+
+ with pytest.raises(exceptions.SDKToolingError) as err_info:
+ importer = get_plugin_importer(plugin_config_file)
+ importer.validate_plugin_module()
+
+ message = err_info.value.message
+ assert expected_error in message
diff --git a/tools/src/test/python/dlpx/virtualization/_internal/test_plugin_validator.py b/tools/src/test/python/dlpx/virtualization/_internal/test_plugin_validator.py
index 893e6e41..e81ce7da 100644
--- a/tools/src/test/python/dlpx/virtualization/_internal/test_plugin_validator.py
+++ b/tools/src/test/python/dlpx/virtualization/_internal/test_plugin_validator.py
@@ -118,6 +118,7 @@ def test_plugin_additional_properties(plugin_config_file,
@staticmethod
@pytest.mark.parametrize('host_types', [['xxx']])
@pytest.mark.parametrize('src_dir', [None])
+ @pytest.mark.parametrize('plugin_type', ['INDIRECT'])
def test_multiple_validation_errors(plugin_config_file,
plugin_config_content):
with pytest.raises(exceptions.SchemaValidationError) as err_info:
@@ -128,6 +129,7 @@ def test_multiple_validation_errors(plugin_config_file,
message = err_info.value.message
assert "'srcDir' is a required property" in message
assert "'xxx' is not one of ['UNIX', 'WINDOWS']" in message
+ assert "'INDIRECT' is not one of ['DIRECT', 'STAGED']" in message
@staticmethod
@mock.patch('os.path.isabs', return_value=False)