Skip to content

Merge from delphix #9

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mothslaw merged 4 commits into from
Feb 10, 2021
Merged

Merge from delphix #9

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
f7b260c
CP-3163 Add authentication object to AppData with vault option
rasantel Aug 13, 2020
23bac75
CP-3829 Upgrade plugin passwords to new credentials object
rasantel Dec 7, 2020
82dd1a3
Merge pull request #330 from rasantel/CP-4097
rasantel Feb 3, 2021
f2c9bfe
Add CNAME to publish-docs (#319) (#336)
jeffngo Feb 9, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .bumpversion.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
[bumpversion]
current_version = 3.0.0
current_version = 3.1.0.dev2
commit = False
tag = False
parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\.(?P<release>[a-z]+)(?P<dev>\d+))?
Expand Down
15 changes: 14 additions & 1 deletion .github/workflows/publish-docs.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ on:
- 'docs/**'
paths:
- 'docs/**'
- '.github/workflows/publish-docs.yml'

jobs:
publish:
Expand All @@ -17,6 +18,7 @@ jobs:
matrix:
python-version: [3.7]
package: [docs]
repository: ['delphix/virtualization-sdk']

steps:
- uses: actions/checkout@v2
Expand Down Expand Up @@ -69,7 +71,18 @@ jobs:
working-directory: ${{ matrix.package }}
run: |
pipenv run mkdocs build --clean
- name: Deploy the contents of docs/site to gh-pages 🚀
- name: Deploy the contents of docs/site to gh-pages (developer.delphix.com) 🚀
if: ${{ github.repository == matrix.repository }}
uses: peaceiris/actions-gh-pages@v3
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
publish_dir: docs/site
commit_message: Deploy to gh-pages 🚀
user_name: "github-actions[bot]"
user_email: "github-actions[bot]@users.noreply.github.com"
cname: developer.delphix.com
- name: Deploy the contents of docs/site to personal gh-pages 🚀
if: ${{ github.repository != matrix.repository }}
uses: peaceiris/actions-gh-pages@v3
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
Expand Down
1 change: 1 addition & 0 deletions common/requirements.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
bump2version==0.5.11
contextlib2==0.6.0.post1 ; python_version < '3'
enum34==1.1.6
funcsigs==1.0.2 ; python_version < '3.0'
importlib-metadata==0.23 ; python_version < '3.8'
more-itertools==5.0.0 ; python_version <= '2.7'
Expand Down
2 changes: 1 addition & 1 deletion common/setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
PYTHON_SRC = 'src/main/python'

install_requires = [
"dvp-api == 1.4.0",
"dvp-api == 1.5.0.dev1",
]

with open(os.path.join(PYTHON_SRC, 'dlpx/virtualization/common/VERSION')) as version_file:
Expand Down
2 changes: 1 addition & 1 deletion common/src/main/python/dlpx/virtualization/common/VERSION
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1 +1 @@
3.0.0
3.1.0.dev2
127 changes: 125 additions & 2 deletions common/src/main/python/dlpx/virtualization/common/_common_classes.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,10 @@
# Copyright (c) 2019 by Delphix. All rights reserved.
#

from dlpx.virtualization.api import common_pb2
from abc import ABCMeta
from dlpx.virtualization.api import common_pb2, libs_pb2
from dlpx.virtualization.common.exceptions import IncorrectTypeError
from enum import IntEnum

"""Classes used for Plugin Operations

Expand All @@ -17,7 +19,10 @@
"RemoteConnection",
"RemoteEnvironment",
"RemoteHost",
"RemoteUser"]
"RemoteUser",
"Credentials",
"PasswordCredentials",
"KeyPairCredentials"]


class RemoteConnection(object):
Expand Down Expand Up @@ -293,3 +298,121 @@ def from_proto(user):
common_pb2.RemoteUser)
remote_user = RemoteUser(name=user.name, reference=user.reference)
return remote_user


class Credentials(object):
"""Plugin base class for CredentialsResult to be used for plugin operations
and library functions.

Plugin authors should use this instead of the corresponding protobuf generated
class.

Args:
username: User name.
"""
def __init__(self, username):
if not isinstance(username, basestring):
raise IncorrectTypeError(
Credentials,
'username',
type(username),
basestring)
self.__username = username

__metaclass__ = ABCMeta

@property
def username(self):
return self.__username


class PasswordCredentials(Credentials):
"""Plugin class for CredentialsResult with password to be used for plugin operations
and library functions.

Plugin authors should use this instead of the corresponding protobuf generated
class.

Args:
username: User name.
password: Password.
"""
def __init__(self, username, password):
super(PasswordCredentials, self).__init__(username)
if not isinstance(password, basestring):
raise IncorrectTypeError(
PasswordCredentials,
'password',
type(password),
basestring)
self.__password = password

@property
def password(self):
return self.__password

@staticmethod
def from_proto(credentials_result):
"""Converts protobuf class libs_pb2.CredentialsResult to plugin class PasswordCredentials
"""
if not isinstance(credentials_result, libs_pb2.CredentialsResult):
raise IncorrectTypeError(
PasswordCredentials,
'credentials_result',
type(credentials_result),
libs_pb2.CredentialsResult)
return PasswordCredentials(
username=credentials_result.username, password=credentials_result.pasword)


class KeyPairCredentials(Credentials):
"""Plugin class for CredentialsResult with key pair to be used for plugin operations
and library functions.

Plugin authors should use this instead of the corresponding protobuf generated
class.

Args:
username (str): User name.
private_key (str): Private key.
public_key (str): Public key corresponding to private key. Empty string if not present.
"""
def __init__(self, username, private_key, public_key):
super(KeyPairCredentials, self).__init__(username)
if not isinstance(private_key, basestring):
raise IncorrectTypeError(
KeyPairCredentials,
'private_key',
type(private_key),
basestring)
self.__private_key = private_key
if not isinstance(public_key, basestring):
raise IncorrectTypeError(
KeyPairCredentials,
'public_key',
type(public_key),
basestring)
self.__public_key = public_key

@property
def private_key(self):
return self.__private_key

@property
def public_key(self):
return self.__public_key

@staticmethod
def from_proto(credentials_result):
"""Converts protobuf class libs_pb2.CredentialsResult to plugin class KeyPairCredentials
"""
if not isinstance(credentials_result, libs_pb2.CredentialsResult):
raise IncorrectTypeError(
KeyPairCredentials,
'credentials_result',
type(credentials_result),
libs_pb2.CredentialsResult)
return KeyPairCredentials(
username=credentials_result.username,
private_key=credentials_result.key_pair.private_key,
public_key=credentials_result.key_pair.public_key)
2 changes: 1 addition & 1 deletion docs/docs/Building_Your_First_Plugin/Data_Ingestion.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ quite limiting.

For our first plugin, we will be using the more flexible [staging](/References/Glossary.md#staged-linkingsyncing) strategy. With this strategy, the Delphix Engine uses NFS for Unix environments (or iSCSI on Windows environments) to mount storage onto a [staging environment](/References/Glossary.md#staging-environment). Our plugin will then be in full control of how to get data from the source environment onto this storage mount.

With the staging strategy, there are two types of syncs: sync and resync. A `sync` is used to ingest incremental changes while a `resync` is used to re-ingest all the data for the dSource. For databases, this could mean re-ingesting from a full database backup to reset the dSource. A `sync` and a `resync` will execute the same plugin operations. To differentiate a `sync` from a `resync`, simply add a boolean property (i.e. `resync`) in the plugin's [snapshot parameters definition](References/Schemas_and_Autogenerated_Classes.md#snapshotparametersdefinition-schema). Once `sync` or `resync` is selected, the property will be passed into [linked.pre_snapshot](/References/Plugin_Operations.md#staged-linked-source-pre-snapshot) and [linked.post_snapshot](/References/Plugin_Operations.md#staged-linked-source-post-snapshot) as a [snapshot parameter](/References/Glossary.md#snapshot-parameters).
With the staging strategy, there are two types of syncs: sync and resync. A `sync` is used to ingest incremental changes while a `resync` is used to re-ingest all the data for the dSource. For databases, this could mean re-ingesting from a full database backup to reset the dSource. A `sync` and a `resync` will execute the same plugin operations. To differentiate a `sync` from a `resync`, simply add a boolean property (i.e. `resync`) in the plugin's [snapshot parameters definition](/References/Schemas_and_Autogenerated_Classes.md#snapshotparametersdefinition-schema). Once `sync` or `resync` is selected, the property will be passed into [linked.pre_snapshot](/References/Plugin_Operations.md#staged-linked-source-pre-snapshot) and [linked.post_snapshot](/References/Plugin_Operations.md#staged-linked-source-post-snapshot) as a [snapshot parameter](/References/Glossary.md#snapshot-parameters).

A regular `sync` is the default and is executed as part of policy driven syncs. A `resync` is only executed during initial ingestion or if the Delphix user manually starts one. The customer can manually trigger a `resync` via the UI by selecting the dSource, going to more options and selecting **Resynchronize dSource**. ![Screenshot](images/Resync.png)

Expand Down
5 changes: 5 additions & 0 deletions docs/docs/Release_Notes/3.0.0/3.0.0.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,14 @@ To install or upgrade the SDK, refer to instructions [here](/Getting_Started.md#
* Provide end-users with configurable options prior to taking a snapshot.
* The options selected are provided as input to pre/post-snapshot functions.

* Added support to initialize an [empty VDB](/References/Glossary#empty-vdb).
* Create an empty virtual dataset within Delphix, instead of creating it externally and ingesting it.
* Utilize this functionality by implementing the [initialize](/References/Plugin_Operations#virtual-source-initialize) operation.

* Added a `scratch_path` property on the [RemoteHost](/References/Classes/#remotehost) object which can be used as:
* A location to store small amounts of persistent data.
* A location to mount VDB data.

More details about `scratch_path` can be found [here](/Best_Practices/Scratch_Paths.md)

## Breaking Changes
Expand Down
2 changes: 1 addition & 1 deletion dvp/src/main/python/dlpx/virtualization/VERSION
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1 +1 @@
3.0.0
3.1.0.dev2
1 change: 1 addition & 0 deletions libs/requirements.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
./../common
bump2version==0.5.11
contextlib2==0.6.0.post1 ; python_version < '3'
enum34==1.1.6
funcsigs==1.0.2 ; python_version < '3.3'
importlib-metadata==1.3.0 ; python_version < '3.8'
mock==3.0.5
Expand Down
2 changes: 1 addition & 1 deletion libs/setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
version = version_file.read().strip()

install_requires = [
"dvp-api == 1.4.0",
"dvp-api == 1.5.0.dev1",
"dvp-common == {}".format(version)
]

Expand Down
2 changes: 1 addition & 1 deletion libs/src/main/python/dlpx/virtualization/libs/VERSION
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1 +1 @@
3.0.0
3.1.0.dev2
70 changes: 68 additions & 2 deletions libs/src/main/python/dlpx/virtualization/libs/libs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,11 @@
from dlpx.virtualization.libs.exceptions import (IncorrectArgumentTypeError,
LibraryError,
PluginScriptError)
from dlpx.virtualization.common._common_classes import RemoteConnection
from dlpx.virtualization.common._common_classes import (RemoteConnection,
PasswordCredentials,
KeyPairCredentials)
from google.protobuf import json_format
from google.protobuf.struct_pb2 import Struct

import logging

Expand All @@ -39,7 +43,9 @@
"run_bash",
"run_sync",
"run_powershell",
"run_expect"
"run_expect",
"retrieve_credentials",
"upgrade_password"
]


Expand Down Expand Up @@ -410,3 +416,63 @@ def _log_request(message, log_level):

response = internal_libs.log(log_request)
_handle_response(response)


def retrieve_credentials(credentials_supplier):
"""This is an internal wrapper around the Virtualization library's credentials retrieval API.
Given a supplier provided by Virtualization, retrieves the credentials from that supplier.

Args:
credentials_supplier (dict): Properties that make up a supplier of credentials.
Return:
Subclass of Credentials retrieved from supplier. Either a PasswordCredentials or a KeyPairCredentials
from dlpx.virtualization.common._common_classes.
"""
from dlpx.virtualization._engine import libs as internal_libs

if not isinstance(credentials_supplier, dict):
raise IncorrectArgumentTypeError('credentials_supplier', type(credentials_supplier), dict)

credentials_request = libs_pb2.CredentialsRequest()
credentials_struct = Struct()
credentials_struct.update(credentials_supplier)
credentials_request.credentials_supplier.CopyFrom(credentials_struct)

response = internal_libs.retrieve_credentials(credentials_request)

credentials_result = _handle_response(response)
if credentials_result.password != "":
return PasswordCredentials(credentials_result.username, credentials_result.password)
return KeyPairCredentials(
credentials_result.username,
credentials_result.key_pair.private_key,
credentials_result.key_pair.public_key)


def upgrade_password(password, username=None):
"""This is an internal wrapper around Virtualization's credentials-supplier conversion API.
It is intended for use during plugin upgrade when a plugin needs to transform a password
value into a more generic credentials supplier object.

Args:
password (basestring): Plain password string.
username (basestring, defaults to None): User name contained in the password credential supplier to return.
Return:
Credentials supplier (dict) that supplies the given password and username.
"""
from dlpx.virtualization._engine import libs as internal_libs

if not isinstance(password, basestring):
raise IncorrectArgumentTypeError('password', type(password), basestring)
if username and not isinstance(username, basestring):
raise IncorrectArgumentTypeError('username', type(username), basestring, required=False)

upgrade_password_request = libs_pb2.UpgradePasswordRequest()
upgrade_password_request.password = password
if username:
upgrade_password_request.username = username

response = internal_libs.upgrade_password(upgrade_password_request)

upgrade_password_result = _handle_response(response)
return json_format.MessageToDict(upgrade_password_result.credentials_supplier)
Loading