From 9ebd2813bda7419045fca0f78de48c6bed4aa6df Mon Sep 17 00:00:00 2001 From: gansach Date: Mon, 24 Mar 2025 16:20:41 +0530 Subject: [PATCH 1/6] feat(datamigration): add support for managed identity auth in Blob to SqlMi migrations --- src/datamigration/HISTORY.rst | 5 +++++ .../vendored_sdks/datamigration/models/_models.py | 8 ++++++++ .../vendored_sdks/datamigration/models/_models_py3.py | 10 ++++++++++ src/datamigration/setup.py | 2 +- 4 files changed, 24 insertions(+), 1 deletion(-) diff --git a/src/datamigration/HISTORY.rst b/src/datamigration/HISTORY.rst index 0fb89deae95..c4cbb59776c 100644 --- a/src/datamigration/HISTORY.rst +++ b/src/datamigration/HISTORY.rst @@ -3,6 +3,11 @@ Release History =============== +======= +1.0.1b1 +++++++ +* [PARAMETER UPDATE] `az datamigration sql-managed-instance create`: `--source-location` now supports Managed Identity for Azure Blob. + ======= 1.0.0b4 ++++++ diff --git a/src/datamigration/azext_datamigration/vendored_sdks/datamigration/models/_models.py b/src/datamigration/azext_datamigration/vendored_sdks/datamigration/models/_models.py index d148f966e60..ea363672700 100644 --- a/src/datamigration/azext_datamigration/vendored_sdks/datamigration/models/_models.py +++ b/src/datamigration/azext_datamigration/vendored_sdks/datamigration/models/_models.py @@ -193,12 +193,18 @@ class AzureBlob(msrest.serialization.Model): :type account_key: str :param blob_container_name: Blob container name where backups are stored. :type blob_container_name: str + :param auth_type: Authentication type for accessing Azure Blob. + :type auth_type: str + :param identity: Identity details for authentication. + :type identity: object """ _attribute_map = { 'storage_account_resource_id': {'key': 'storageAccountResourceId', 'type': 'str'}, 'account_key': {'key': 'accountKey', 'type': 'str'}, 'blob_container_name': {'key': 'blobContainerName', 'type': 'str'}, + 'auth_type': {'key': 'authType', 'type': 'str'}, + 'identity': {'key': 'identity', 'type': 'object'}, } def __init__( @@ -209,6 +215,8 @@ def __init__( self.storage_account_resource_id = kwargs.get('storage_account_resource_id', None) self.account_key = kwargs.get('account_key', None) self.blob_container_name = kwargs.get('blob_container_name', None) + self.auth_type = kwargs.get('auth_type', None) + self.identity = kwargs.get('identity', None) class BackupConfiguration(msrest.serialization.Model): diff --git a/src/datamigration/azext_datamigration/vendored_sdks/datamigration/models/_models_py3.py b/src/datamigration/azext_datamigration/vendored_sdks/datamigration/models/_models_py3.py index 34352e3da03..275f3197451 100644 --- a/src/datamigration/azext_datamigration/vendored_sdks/datamigration/models/_models_py3.py +++ b/src/datamigration/azext_datamigration/vendored_sdks/datamigration/models/_models_py3.py @@ -222,12 +222,18 @@ class AzureBlob(msrest.serialization.Model): :type account_key: str :param blob_container_name: Blob container name where backups are stored. :type blob_container_name: str + :param auth_type: Authentication type for accessing Azure Blob. + :type auth_type: str + :param identity: Identity details for authentication. + :type identity: object """ _attribute_map = { 'storage_account_resource_id': {'key': 'storageAccountResourceId', 'type': 'str'}, 'account_key': {'key': 'accountKey', 'type': 'str'}, 'blob_container_name': {'key': 'blobContainerName', 'type': 'str'}, + 'auth_type': {'key': 'authType', 'type': 'str'}, + 'identity': {'key': 'identity', 'type': 'object'}, } def __init__( @@ -236,12 +242,16 @@ def __init__( storage_account_resource_id: Optional[str] = None, account_key: Optional[str] = None, blob_container_name: Optional[str] = None, + auth_type: Optional[str] = None, + identity: Optional[Dict[str, Dict[str, dict]]] = None, **kwargs ): super(AzureBlob, self).__init__(**kwargs) self.storage_account_resource_id = storage_account_resource_id self.account_key = account_key self.blob_container_name = blob_container_name + self.auth_type = auth_type + self.identity = identity class BackupConfiguration(msrest.serialization.Model): diff --git a/src/datamigration/setup.py b/src/datamigration/setup.py index 77a38395f78..8f910807b16 100644 --- a/src/datamigration/setup.py +++ b/src/datamigration/setup.py @@ -10,7 +10,7 @@ from setuptools import setup, find_packages # HISTORY.rst entry. -VERSION = '1.0.0b4' +VERSION = '1.0.1b1' # The full list of classifiers is available at # https://pypi.python.org/pypi?%3Aaction=list_classifiers From a898af3e465a93d0209767ce9c96fdc507354ee9 Mon Sep 17 00:00:00 2001 From: gansach Date: Wed, 14 May 2025 16:52:15 +0530 Subject: [PATCH 2/6] docs(datamigration): update package versioning --- src/datamigration/HISTORY.rst | 4 ++-- src/datamigration/setup.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/datamigration/HISTORY.rst b/src/datamigration/HISTORY.rst index c4cbb59776c..59f3110a96f 100644 --- a/src/datamigration/HISTORY.rst +++ b/src/datamigration/HISTORY.rst @@ -4,9 +4,9 @@ Release History =============== ======= -1.0.1b1 +1.0.0b5 ++++++ -* [PARAMETER UPDATE] `az datamigration sql-managed-instance create`: `--source-location` now supports Managed Identity for Azure Blob. +* [PARAMETER UPDATE] `az datamigration sql-managed-instance create`: `--source-location` now supports Managed Identity for accessing Azure Blob. ======= 1.0.0b4 diff --git a/src/datamigration/setup.py b/src/datamigration/setup.py index 8f910807b16..1b1dba6bda8 100644 --- a/src/datamigration/setup.py +++ b/src/datamigration/setup.py @@ -10,7 +10,7 @@ from setuptools import setup, find_packages # HISTORY.rst entry. -VERSION = '1.0.1b1' +VERSION = '1.0.0b5' # The full list of classifiers is available at # https://pypi.python.org/pypi?%3Aaction=list_classifiers From 4fc62c6f23460005f127ef15313bc7f7b26cde69 Mon Sep 17 00:00:00 2001 From: gansach Date: Wed, 14 May 2025 17:48:31 +0530 Subject: [PATCH 3/6] fix(datamigration): fix azdev style issues --- .../azext_datamigration/manual/helper.py | 59 +++++++++++-------- 1 file changed, 35 insertions(+), 24 deletions(-) diff --git a/src/datamigration/azext_datamigration/manual/helper.py b/src/datamigration/azext_datamigration/manual/helper.py index 6d58d2d1695..2bfe84382b4 100644 --- a/src/datamigration/azext_datamigration/manual/helper.py +++ b/src/datamigration/azext_datamigration/manual/helper.py @@ -12,21 +12,29 @@ # pylint: disable=line-too-long import base64 +import binascii import ctypes import json import os import platform +import shutil import subprocess +import sys import time import urllib.request import uuid from zipfile import ZipFile -import shutil + import requests +from azure.cli.core.azclierror import (CLIInternalError, FileOperationError, + InvalidArgumentValueError) from knack.util import CLIError -from azure.cli.core.azclierror import CLIInternalError -from azure.cli.core.azclierror import FileOperationError -from azure.cli.core.azclierror import InvalidArgumentValueError + +# Conditionally import winreg for Windows platforms +if sys.platform == "win32": + import winreg +else: + winreg = None # ----------------------------------------------------------------------------------------------------------------- @@ -70,7 +78,7 @@ def is_valid_guid(guid): def is_base64(s): try: return base64.b64encode(base64.b64decode(s)).decode() == s - except Exception: + except (binascii.Error, UnicodeDecodeError): return False @@ -372,7 +380,8 @@ def get_latest_nuget_org_version(package_id): service_index_response = None try: service_index_response = requests.get("https://api.nuget.org/v3/index.json") - except Exception: + service_index_response.raise_for_status() + except requests.exceptions.RequestException: print("Unable to connect to NuGet.org to check for updates.") if (service_index_response is None or @@ -511,27 +520,26 @@ def validate_input(key): # IR key format: IR@{KeyId : Guid}@{PartitionId: Factory Name}@{Factory Region or Factory ServiceEndpoint}@{Base64String} # ----------------------------------------------------------------------------------------------------------------- def is_valid_ir_key_format(key): + if not isinstance(key, str): + return False - try: - keyPrefix = "IR" - separator = '@' - keyParts = 5 + keyPrefix = "IR" + separator = '@' + keyParts = 5 - key_parts = key.split(separator) + key_parts = key.split(separator) - # Length of the split auth key should be 5 and should start with "IR" - if len(key_parts) != keyParts or key.startswith(keyPrefix) is False: - return False - # Check if all parts are not null or empty - for part in key_parts: - if is_string_null_or_empty(part): - return False - # Check pattern of second part as Guid and last part is a base64 string - if not is_valid_guid(key_parts[1]) or not is_base64(key_parts[-1]): + # Length of the split auth key should be 5 and should start with "IR" + if len(key_parts) != keyParts or key.startswith(keyPrefix) is False: + return False + # Check if all parts are not null or empty + for part in key_parts: + if is_string_null_or_empty(part): return False - return True - except Exception: + # Check pattern of second part as Guid and last part is a base64 string + if not is_valid_guid(key_parts[1]) or not is_base64(key_parts[-1]): return False + return True # ----------------------------------------------------------------------------------------------------------------- @@ -539,7 +547,9 @@ def is_valid_ir_key_format(key): # ----------------------------------------------------------------------------------------------------------------- def check_whether_gateway_installed(name): - import winreg + if winreg is None: + return False + # Connecting to key in registry accessRegistry = winreg.ConnectRegistry(None, winreg.HKEY_LOCAL_MACHINE) @@ -626,7 +636,8 @@ def register_ir(key, installed_ir_path=None): # ----------------------------------------------------------------------------------------------------------------- def get_cmd_file_path(): - import winreg + if winreg is None: + raise ModuleNotFoundError("Failed: winreg module not found. Please run this command in Windows environment") try: # Connecting to key in registry accessRegistry = winreg.ConnectRegistry(None, winreg.HKEY_LOCAL_MACHINE) From 97a380c95aea483da4112a71f3348f0aadc85e9b Mon Sep 17 00:00:00 2001 From: gansach Date: Wed, 14 May 2025 18:02:47 +0530 Subject: [PATCH 4/6] fix(datamigration): make os checking more robust --- src/datamigration/azext_datamigration/manual/helper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/datamigration/azext_datamigration/manual/helper.py b/src/datamigration/azext_datamigration/manual/helper.py index 2bfe84382b4..87ccf15828e 100644 --- a/src/datamigration/azext_datamigration/manual/helper.py +++ b/src/datamigration/azext_datamigration/manual/helper.py @@ -31,7 +31,7 @@ from knack.util import CLIError # Conditionally import winreg for Windows platforms -if sys.platform == "win32": +if sys.platform.startswith("win"): import winreg else: winreg = None From 4053bbe5bb125fbf4c77e0d787705d72f53c172d Mon Sep 17 00:00:00 2001 From: gansach Date: Thu, 15 May 2025 11:42:31 +0530 Subject: [PATCH 5/6] fix(datamigration): add pylint exceptions to fix az styledev --- .../azext_datamigration/manual/helper.py | 59 ++++++++----------- 1 file changed, 24 insertions(+), 35 deletions(-) diff --git a/src/datamigration/azext_datamigration/manual/helper.py b/src/datamigration/azext_datamigration/manual/helper.py index 87ccf15828e..cd0c1ca681b 100644 --- a/src/datamigration/azext_datamigration/manual/helper.py +++ b/src/datamigration/azext_datamigration/manual/helper.py @@ -12,29 +12,21 @@ # pylint: disable=line-too-long import base64 -import binascii import ctypes import json import os import platform -import shutil import subprocess -import sys import time import urllib.request import uuid from zipfile import ZipFile - +import shutil import requests -from azure.cli.core.azclierror import (CLIInternalError, FileOperationError, - InvalidArgumentValueError) from knack.util import CLIError - -# Conditionally import winreg for Windows platforms -if sys.platform.startswith("win"): - import winreg -else: - winreg = None +from azure.cli.core.azclierror import CLIInternalError +from azure.cli.core.azclierror import FileOperationError +from azure.cli.core.azclierror import InvalidArgumentValueError # ----------------------------------------------------------------------------------------------------------------- @@ -78,7 +70,7 @@ def is_valid_guid(guid): def is_base64(s): try: return base64.b64encode(base64.b64decode(s)).decode() == s - except (binascii.Error, UnicodeDecodeError): + except Exception: # pylint: disable=broad-except return False @@ -380,8 +372,7 @@ def get_latest_nuget_org_version(package_id): service_index_response = None try: service_index_response = requests.get("https://api.nuget.org/v3/index.json") - service_index_response.raise_for_status() - except requests.exceptions.RequestException: + except Exception: # pylint: disable=broad-except print("Unable to connect to NuGet.org to check for updates.") if (service_index_response is None or @@ -520,26 +511,27 @@ def validate_input(key): # IR key format: IR@{KeyId : Guid}@{PartitionId: Factory Name}@{Factory Region or Factory ServiceEndpoint}@{Base64String} # ----------------------------------------------------------------------------------------------------------------- def is_valid_ir_key_format(key): - if not isinstance(key, str): - return False - keyPrefix = "IR" - separator = '@' - keyParts = 5 + try: + keyPrefix = "IR" + separator = '@' + keyParts = 5 - key_parts = key.split(separator) + key_parts = key.split(separator) - # Length of the split auth key should be 5 and should start with "IR" - if len(key_parts) != keyParts or key.startswith(keyPrefix) is False: - return False - # Check if all parts are not null or empty - for part in key_parts: - if is_string_null_or_empty(part): + # Length of the split auth key should be 5 and should start with "IR" + if len(key_parts) != keyParts or key.startswith(keyPrefix) is False: + return False + # Check if all parts are not null or empty + for part in key_parts: + if is_string_null_or_empty(part): + return False + # Check pattern of second part as Guid and last part is a base64 string + if not is_valid_guid(key_parts[1]) or not is_base64(key_parts[-1]): return False - # Check pattern of second part as Guid and last part is a base64 string - if not is_valid_guid(key_parts[1]) or not is_base64(key_parts[-1]): + return True + except Exception: # pylint: disable=broad-except return False - return True # ----------------------------------------------------------------------------------------------------------------- @@ -547,9 +539,7 @@ def is_valid_ir_key_format(key): # ----------------------------------------------------------------------------------------------------------------- def check_whether_gateway_installed(name): - if winreg is None: - return False - + import winreg # pylint: disable=import-error # Connecting to key in registry accessRegistry = winreg.ConnectRegistry(None, winreg.HKEY_LOCAL_MACHINE) @@ -636,8 +626,7 @@ def register_ir(key, installed_ir_path=None): # ----------------------------------------------------------------------------------------------------------------- def get_cmd_file_path(): - if winreg is None: - raise ModuleNotFoundError("Failed: winreg module not found. Please run this command in Windows environment") + import winreg # pylint: disable=import-error try: # Connecting to key in registry accessRegistry = winreg.ConnectRegistry(None, winreg.HKEY_LOCAL_MACHINE) From 19fdd5f32734bb113a9adec3a3c878c7592fb65f Mon Sep 17 00:00:00 2001 From: gansach Date: Thu, 15 May 2025 15:55:45 +0530 Subject: [PATCH 6/6] docs(datamigration): add examples --- .../azext_datamigration/generated/_help.py | 22 +++++++++++++++++++ .../azext_datamigration/manual/_help.py | 22 +++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/src/datamigration/azext_datamigration/generated/_help.py b/src/datamigration/azext_datamigration/generated/_help.py index 0ed52527e05..3ce0e3a675b 100644 --- a/src/datamigration/azext_datamigration/generated/_help.py +++ b/src/datamigration/azext_datamigration/generated/_help.py @@ -194,6 +194,28 @@ ceGroups/testrg/providers/Microsoft.Sql/managedInstances/instance" --source-database-name "aaa" \ --source-sql-connection authentication="WindowsAuthentication" data-source="aaa" encrypt-connection=true \ password="placeholder" trust-server-certificate=true user-name="bbb" --resource-group "testrg" --target-db-name "db1" + - name: Create or update a Database Migration resource using Azure Blob storage (via System-Assigned Managed Identity) as the backup source. + text: |- + az datamigration sql-managed-instance create --managed-instance-name "managedInstance1" \ +--source-location '{\\"AzureBlob\\":{\\"storageAccountResourceId\\":\\"/subscriptions/1111-2222-3333-4444/resourceGroups/RG/prooviders\ +/Microsoft.Storage/storageAccounts/MyStorage\\",\\"authType\\":\\"ManagedIdentity\\",\\"identity\\":{\\"type\\":\\"SystemAssigned\\"},\\"blobContainerName\\":\\"ContainerName\ +-X\\"}}' --migration-service "/subscriptions/00000000-1111-2222-3333-444444444444/resourceGroups/testrg/providers/Micr\ +osoft.DataMigration/sqlMigrationServices/testagent" --offline-configuration last-backup-name="last_backup_file_name" \ +offline=true --scope "/subscriptions/00000000-1111-2222-3333-444444444444/resourceGroups/testrg/providers/Microsoft.Sql\ +/managedInstances/instance" --source-database-name "aaa" --source-sql-connection authentication="WindowsAuthentication"\ + data-source="aaa" encrypt-connection=true password="placeholder" trust-server-certificate=true user-name="bbb" \ +--resource-group "testrg" --target-db-name "db1" + - name: Create or update a Database Migration resource using Azure Blob storage (via User-Assigned Managed Identity) as the backup source. + text: |- + az datamigration sql-managed-instance create --managed-instance-name "managedInstance1" \ +--source-location '{\\"AzureBlob\\":{\\"storageAccountResourceId\\":\\"/subscriptions/1111-2222-3333-4444/resourceGroups/RG/prooviders\ +/Microsoft.Storage/storageAccounts/MyStorage\\",\\"authType\\":\\"ManagedIdentity\\",\\"identity\\":{\\"type\\":\\"UserAssigned\\",\\"userAssignedIdentities\\":{\\"/subscriptions/00000000-1111-2222-3333-444444444444/resourcegroups/testrg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-uami\":{}}},\\"blobContainerName\\":\\"ContainerName\ +-X\\"}}' --migration-service "/subscriptions/00000000-1111-2222-3333-444444444444/resourceGroups/testrg/providers/Micr\ +osoft.DataMigration/sqlMigrationServices/testagent" --offline-configuration last-backup-name="last_backup_file_name" \ +offline=true --scope "/subscriptions/00000000-1111-2222-3333-444444444444/resourceGroups/testrg/providers/Microsoft.Sql\ +/managedInstances/instance" --source-database-name "aaa" --source-sql-connection authentication="WindowsAuthentication"\ + data-source="aaa" encrypt-connection=true password="placeholder" trust-server-certificate=true user-name="bbb" \ +--resource-group "testrg" --target-db-name "db1" """ helps['datamigration sql-managed-instance cancel'] = """ diff --git a/src/datamigration/azext_datamigration/manual/_help.py b/src/datamigration/azext_datamigration/manual/_help.py index 070a5d70f46..a9648de454d 100644 --- a/src/datamigration/azext_datamigration/manual/_help.py +++ b/src/datamigration/azext_datamigration/manual/_help.py @@ -163,6 +163,28 @@ -X\\"}}' --migration-service "/subscriptions/00000000-1111-2222-3333-444444444444/resourceGroups/testrg/providers/Micr\ osoft.DataMigration/sqlMigrationServices/testagent" --offline-configuration last-backup-name="last_backup_file_name" \ offline=true --scope "/subscriptions/00000000-1111-2222-3333-444444444444/resourceGroups/testrg/providers/Microsoft.Sql\ +/managedInstances/instance" --source-database-name "aaa" --source-sql-connection authentication="WindowsAuthentication"\ + data-source="aaa" encrypt-connection=true password="placeholder" trust-server-certificate=true user-name="bbb" \ +--resource-group "testrg" --target-db-name "db1" + - name: Create or update a Database Migration resource using Azure Blob storage (via System-Assigned Managed Identity) as the backup source. + text: |- + az datamigration sql-managed-instance create --managed-instance-name "managedInstance1" \ +--source-location '{\\"AzureBlob\\":{\\"storageAccountResourceId\\":\\"/subscriptions/1111-2222-3333-4444/resourceGroups/RG/prooviders\ +/Microsoft.Storage/storageAccounts/MyStorage\\",\\"authType\\":\\"ManagedIdentity\\",\\"identity\\":{\\"type\\":\\"SystemAssigned\\"},\\"blobContainerName\\":\\"ContainerName\ +-X\\"}}' --migration-service "/subscriptions/00000000-1111-2222-3333-444444444444/resourceGroups/testrg/providers/Micr\ +osoft.DataMigration/sqlMigrationServices/testagent" --offline-configuration last-backup-name="last_backup_file_name" \ +offline=true --scope "/subscriptions/00000000-1111-2222-3333-444444444444/resourceGroups/testrg/providers/Microsoft.Sql\ +/managedInstances/instance" --source-database-name "aaa" --source-sql-connection authentication="WindowsAuthentication"\ + data-source="aaa" encrypt-connection=true password="placeholder" trust-server-certificate=true user-name="bbb" \ +--resource-group "testrg" --target-db-name "db1" + - name: Create or update a Database Migration resource using Azure Blob storage (via User-Assigned Managed Identity) as the backup source. + text: |- + az datamigration sql-managed-instance create --managed-instance-name "managedInstance1" \ +--source-location '{\\"AzureBlob\\":{\\"storageAccountResourceId\\":\\"/subscriptions/1111-2222-3333-4444/resourceGroups/RG/prooviders\ +/Microsoft.Storage/storageAccounts/MyStorage\\",\\"authType\\":\\"ManagedIdentity\\",\\"identity\\":{\\"type\\":\\"UserAssigned\\",\\"userAssignedIdentities\\":{\\"/subscriptions/00000000-1111-2222-3333-444444444444/resourcegroups/testrg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-uami\":{}}},\\"blobContainerName\\":\\"ContainerName\ +-X\\"}}' --migration-service "/subscriptions/00000000-1111-2222-3333-444444444444/resourceGroups/testrg/providers/Micr\ +osoft.DataMigration/sqlMigrationServices/testagent" --offline-configuration last-backup-name="last_backup_file_name" \ +offline=true --scope "/subscriptions/00000000-1111-2222-3333-444444444444/resourceGroups/testrg/providers/Microsoft.Sql\ /managedInstances/instance" --source-database-name "aaa" --source-sql-connection authentication="WindowsAuthentication"\ data-source="aaa" encrypt-connection=true password="placeholder" trust-server-certificate=true user-name="bbb" \ --resource-group "testrg" --target-db-name "db1"