diff --git a/src/datamigration/HISTORY.rst b/src/datamigration/HISTORY.rst index 4ffc8853932..389e754f3b4 100644 --- a/src/datamigration/HISTORY.rst +++ b/src/datamigration/HISTORY.rst @@ -3,6 +3,11 @@ Release History =============== +0.3.1 +++++++ +* [NEW PARAMETER] `az datamigration register-integration-runtime`: Added parameter `--installed-ir-path` to read the installed location of Microsoft Integration Runtime (SHIR) and use it for registering the Database Migration Service if command is unable to find the installed SHIR path. +* [NEW PARAMETER] `az datamigration performance-data-collection`: Added parameter `--time` to specify the amount of time(in seconds) performance data collection is to be done. After the timeout the process is terminated automatically. + 0.3.0 ++++++ * [BREAKING CHANGE] `az datamigration sql-managed-instance/sql-vm create`: Remove `--provisioing-error` and `--migration-operation-id` as they are unnecessary parameters. diff --git a/src/datamigration/README.md b/src/datamigration/README.md index 16ba265f4df..c2ebcd6efc3 100644 --- a/src/datamigration/README.md +++ b/src/datamigration/README.md @@ -26,7 +26,7 @@ az datamigration performance-data-collection --connection-string "Data Source=La ##### Get-sku-recommendation ##### ``` -az datamigration get-sku-recommendation --output-folder "C:\\PerfCollectionOutput" --database-allow-list AdventureWorks, AdventureWorks2 --display-result --overwrite +az datamigration get-sku-recommendation --output-folder "C:\\PerfCollectionOutput" --database-allow-list AdventureWorks AdventureWorks2 --display-result --overwrite ``` #### datamigration sql-managed-instance #### diff --git a/src/datamigration/azext_datamigration/manual/_help.py b/src/datamigration/azext_datamigration/manual/_help.py index a908fc50c5c..9af90b51197 100644 --- a/src/datamigration/azext_datamigration/manual/_help.py +++ b/src/datamigration/azext_datamigration/manual/_help.py @@ -22,6 +22,9 @@ - name: Run SQL Assessment on given SQL Server using assessment config file. text: |- az datamigration get-assessment --config-file-path "C:\\Users\\user\\document\\config.json" + - name: Run SQL Assessment on multiple SQL Servers in one call using connection string. + text: |- + az datamigration get-assessment --connection-string "Data Source=LabServer1.database.net;Initial Catalog=master;Integrated Security=False;User Id=User;Password=password" "Data Source=LabServer2.database.net;Initial Catalog=master;Integrated Security=False;User Id=User;Password=password" --output-folder "C:\\AssessmentOutput" --overwrite """ helps['datamigration performance-data-collection'] = """ @@ -31,9 +34,15 @@ - name: Collect performance data of a given SQL Server using connection string. text: |- az datamigration performance-data-collection --connection-string "Data Source=LabServer.database.net;Initial Catalog=master;Integrated Security=False;User Id=User;Password=password" --output-folder "C:\\PerfCollectionOutput" --number-of-iteration 5 --perf-query-interval 10 --static-query-interval 60 + - name: Collect performance data of multiple SQL Servers in one call using connection string. + text: |- + az datamigration performance-data-collection --connection-string "Data Source=LabServer1.database.net;Initial Catalog=master;Integrated Security=False;User Id=User;Password=password" "Data Source=LabServer2.database.net;Initial Catalog=master;Integrated Security=False;User Id=User;Password=password" --output-folder "C:\\PerfCollectionOutput" --number-of-iteration 5 --perf-query-interval 10 --static-query-interval 60 - name: Collect performance data of a given SQL Server using assessment config file. text: |- az datamigration performance-data-collection --config-file-path "C:\\Users\\user\\document\\config.json" + - name: Collect performance data of a given SQL Server by specifying a time limit. If the time limit specified is before the complition of a iteration cycle, the process will end without saving the last cycle performance data. + text: |- + az datamigration performance-data-collection --connection-string "Data Source=LabServer.database.net;Initial Catalog=master;Integrated Security=False;User Id=User;Password=password" --output-folder "C:\\PerfCollectionOutput" --number-of-iteration 5 --perf-query-interval 10 --static-query-interval 60 --time 60 """ helps['datamigration get-sku-recommendation'] = """ @@ -42,7 +51,7 @@ examples: - name: Get SKU recommendation for given SQL Server using command line. text: |- - az datamigration get-sku-recommendation --output-folder "C:\\PerfCollectionOutput" --database-allow-list AdventureWorks, AdventureWorks2 --display-result --overwrite + az datamigration get-sku-recommendation --output-folder "C:\\PerfCollectionOutput" --database-allow-list AdventureWorks1 AdventureWorks2 --display-result --overwrite - name: Get SKU recommendation for given SQL Server using assessment config file. text: |- az datamigration get-sku-recommendation --config-file-path "C:\\Users\\user\\document\\config.json" @@ -58,6 +67,9 @@ - name: Install Integration Runtime and register a Sql Migration Service on it. text: |- az datamigration register-integration-runtime --auth-key "IR@00000-0000000-000000-aaaaa-bbbb-cccc" --ir-path "C:\\Users\\user\\Downloads\\IntegrationRuntime.msi" + - name: Read the Integration Runtime from given installation location. + text: |- + az datamigration register-integration-runtime --auth-key "IR@00000-0000000-000000-aaaaa-bbbb-cccc" --installed-ir-path "D:\\My Softwares\\Microsoft Integration Runtime\\5.0" """ helps['datamigration sql-managed-instance create'] = """ diff --git a/src/datamigration/azext_datamigration/manual/_params.py b/src/datamigration/azext_datamigration/manual/_params.py index 44bf7c04634..561cb8392e0 100644 --- a/src/datamigration/azext_datamigration/manual/_params.py +++ b/src/datamigration/azext_datamigration/manual/_params.py @@ -35,6 +35,7 @@ def load_arguments(self, _): c.argument('static_query_interval', type=int, help='Interval at which to query and persist static configuration data, in seconds.') c.argument('number_of_iteration', type=int, help='Number of iterations of performance data collection to perform before persisting to file. For example, with default values, performance data will be persisted every 30 seconds * 20 iterations = 10 minutes. Minimum: 2.') c.argument('config_file_path', type=str, help='Path of the ConfigFile') + c.argument('time', type=int, help='Time after which the command execution automatically stops, in seconds. If this parameter is not specified manual intervention will be required to stop the command execution.') with self.argument_context('datamigration get-sku-recommendation') as c: c.argument('output_folder', type=str, help='Output folder where performance data of the SQL Server is stored. The value here must be the same as the one used in PerfDataCollection') @@ -54,6 +55,7 @@ def load_arguments(self, _): with self.argument_context('datamigration register-integration-runtime') as c: c.argument('auth_key', type=str, help='AuthKey of SQL Migration Service') c.argument('ir_path', type=str, help='Path of Integration Runtime MSI') + c.argument('installed_ir_path', type=str, help='Version folder path in the Integration Runtime installed location. This can be provided when IR is installed but the command is failing to read it. Format: "\\Microsoft Integration Runtime\\"') with self.argument_context('datamigration sql-db create') as c: c.argument('resource_group_name', resource_group_name_type) diff --git a/src/datamigration/azext_datamigration/manual/action.py b/src/datamigration/azext_datamigration/manual/action.py index 7d4ce4a232c..b9d2a1718f2 100644 --- a/src/datamigration/azext_datamigration/manual/action.py +++ b/src/datamigration/azext_datamigration/manual/action.py @@ -22,6 +22,8 @@ from knack.util import CLIError +# Adding new input type for target connection details +# As having type as AddSourceSqlConnection overwrites one of the parameters class AddTargetSqlConnection(argparse.Action): def __call__(self, parser, namespace, values, option_string=None): action = self.get_action(values, option_string) diff --git a/src/datamigration/azext_datamigration/manual/custom.py b/src/datamigration/azext_datamigration/manual/custom.py index 07442e36a4d..f7575bd94fd 100644 --- a/src/datamigration/azext_datamigration/manual/custom.py +++ b/src/datamigration/azext_datamigration/manual/custom.py @@ -12,6 +12,7 @@ # pylint: disable=line-too-long import os +import signal import subprocess from azure.cli.core.azclierror import MutuallyExclusiveArgumentError from azure.cli.core.azclierror import RequiredArgumentMissingError @@ -29,20 +30,31 @@ def datamigration_assessment(connection_string=None, try: + # Setup the console app defaultOutputFolder, exePath = helper.console_app_setup() + # Specifying both parameters is an error if connection_string is not None and config_file_path is not None: raise MutuallyExclusiveArgumentError("Both connection_string and config_file_path are mutually exclusive arguments. Please provide only one of these arguments.") + # When Connection string. if connection_string is not None: + + # Formating for multiple connection string connection_string = " ".join(f"\"{i}\"" for i in connection_string) + + # Joining parameters in a string cmd = f'{exePath} Assess --sqlConnectionStrings {connection_string} ' if output_folder is None else f'{exePath} Assess --sqlConnectionStrings {connection_string} --outputFolder "{output_folder}" ' cmd += '--overwrite False' if overwrite is False else '' subprocess.call(cmd, shell=False) + + # When config file. elif config_file_path is not None: helper.validate_config_file_path(config_file_path, "assess") cmd = f'{exePath} --configFile "{config_file_path}"' subprocess.call(cmd, shell=False) + + # if no parameter is provided. else: raise RequiredArgumentMissingError('No valid parameter set used. Please provide any one of the these prameters: connection_string, config_file_path') @@ -62,32 +74,64 @@ def datamigration_performance_data_collection(connection_string=None, perf_query_interval=30, static_query_interval=3600, number_of_iteration=20, - config_file_path=None): + config_file_path=None, + time=None): try: + # Setup the console app defaultOutputFolder, exePath = helper.console_app_setup() if connection_string is not None and config_file_path is not None: raise MutuallyExclusiveArgumentError("Both sql_connection_string and config_file_path are mutually exclusive arguments. Please provide only one of these arguments.") + # When Connection string. if connection_string is not None: + + # Formating for multiple connection string connection_string = " ".join(f"\"{i}\"" for i in connection_string) + + # parameter set for Perfornace data collection parameterList = { "--outputFolder": output_folder, "--perfQueryIntervalInSec": perf_query_interval, "--staticQueryIntervalInSec": static_query_interval, "--numberOfIterations": number_of_iteration } + + # joining paramaters together in a string cmd = f'{exePath} PerfDataCollection --sqlConnectionStrings {connection_string}' for param in parameterList: if parameterList[param] is not None: cmd += f' {param} "{parameterList[param]}"' - subprocess.call(cmd, shell=False) + + # If time parameter is specified, catch TimeoutExpired exception and terminate the process + if time is None: + subprocess.call(cmd, shell=False) + else: + sp = subprocess.Popen(cmd, shell=False) + try: + outs, errs = sp.communicate(timeout=time) + except subprocess.TimeoutExpired: + sp.send_signal(signal.SIGTERM) + outs, errs = sp.communicate() + + # When Config file. elif config_file_path is not None: helper.validate_config_file_path(config_file_path, "perfdatacollection") cmd = f'{exePath} --configFile "{config_file_path}"' - subprocess.call(cmd, shell=False) + + # If time parameter is specified, catch TimeoutExpired exception and terminate the process + if time is None: + subprocess.call(cmd, shell=False) + else: + sp = subprocess.Popen(cmd, shell=False) + try: + outs, errs = sp.communicate(timeout=time) + except subprocess.TimeoutExpired: + sp.send_signal(signal.SIGTERM) + outs, errs = sp.communicate() + else: raise RequiredArgumentMissingError('No valid parameter set used. Please provide any one of the these prameters: sql_connection_string, config_file_path') @@ -117,16 +161,23 @@ def datamigration_get_sku_recommendation(output_folder=None, config_file_path=None): try: + + # Setup Console app defaultOutputFolder, exePath = helper.console_app_setup() if output_folder is not None and config_file_path is not None: raise MutuallyExclusiveArgumentError("Both output_folder and config_file_path are mutually exclusive arguments. Please provide only one of these arguments.") + # When Config file - Handling this case first to allow no parameter to be specified (runs non config file scenario) if config_file_path is not None: helper.validate_config_file_path(config_file_path, "getskurecommendation") cmd = f'{exePath} --configFile "{config_file_path}"' subprocess.call(cmd, shell=False) + + # When non-config file else: + + # parameter set for Sku recommendation parameterList = { "--outputFolder": output_folder, "--targetPlatform": target_platform, @@ -142,9 +193,13 @@ def datamigration_get_sku_recommendation(output_folder=None, "--databaseDenyList": database_deny_list } cmd = f'{exePath} GetSkuRecommendation' + + # formating the parameter list into a string for param in parameterList: if parameterList[param] is not None and not param.__contains__("List"): cmd += f' {param} "{parameterList[param]}"' + + # in case the parameter input is list format it accordingly elif param.__contains__("List") and parameterList[param] is not None: parameterList[param] = " ".join(f"\"{i}\"" for i in parameterList[param]) cmd += f' {param} {parameterList[param]}' @@ -162,14 +217,19 @@ def datamigration_get_sku_recommendation(output_folder=None, # Register Sql Migration Service on IR command Implementation. # ----------------------------------------------------------------------------------------------------------------- def datamigration_register_ir(auth_key, - ir_path=None): + ir_path=None, + installed_ir_path=None): helper.validate_os_env() + # This command can only be run as admin and in windows if not helper.is_user_admin(): raise UnclassifiedUserFault("Failed: You do not have Administrator rights to run this command. Please re-run this command as an Administrator!") helper.validate_input(auth_key) + + # Run installation if ir_path is provided if ir_path is not None: helper.install_gateway(ir_path) - helper.register_ir(auth_key) + # register or re-register Dms on ir + helper.register_ir(auth_key, installed_ir_path) diff --git a/src/datamigration/azext_datamigration/manual/helper.py b/src/datamigration/azext_datamigration/manual/helper.py index ba953b9343b..ba9c33dbd54 100644 --- a/src/datamigration/azext_datamigration/manual/helper.py +++ b/src/datamigration/azext_datamigration/manual/helper.py @@ -41,7 +41,7 @@ def validate_config_file_path(path, action): if not os.path.exists(path): raise InvalidArgumentValueError(f'Invalid config file path: {path}. Please provide a valid config file path.') - # JSON file + # JSON file read and validation of value in action with open(path, "r", encoding=None) as f: configJson = json.loads(f.read()) try: @@ -159,6 +159,7 @@ def check_whether_gateway_installed(name): # Get the path of Installed softwares accessKey = winreg.OpenKey(accessRegistry, r"SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall") + # Check if any software has name as given name for i in range(0, winreg.QueryInfoKey(accessKey)[0]): installedSoftware = winreg.EnumKey(accessKey, i) installedSoftwareKey = winreg.OpenKey(accessKey, installedSoftware) @@ -169,7 +170,7 @@ def check_whether_gateway_installed(name): except FileNotFoundError: pass - # Adding this try to look for Installed IR in Program files (Assumes the IR is always installed there) + # Adding this try to look for Installed IR in Program files (Assumes the IR is always installed there) to tackle x32 bit python try: diaCmdPath = get_cmd_file_path_static() if os.path.exists(diaCmdPath): @@ -185,17 +186,21 @@ def check_whether_gateway_installed(name): # ----------------------------------------------------------------------------------------------------------------- def install_gateway(path): + # check if gateway is installaed. If yes don't do the installation again if check_whether_gateway_installed("Microsoft Integration Runtime"): print("Microsoft Integration Runtime is already installed") return + # validate path of IR MSI validate_ir_extension(path) + # Check for IR path existance if not os.path.exists(path): raise InvalidArgumentValueError(f"Invalid Integration Runtime MSI path : {path}. Please provide a valid Integration Runtime MSI path") print("Start Integration Runtime installation") + # Installed MSI installCmd = f'msiexec.exe /i "{path}" /quiet /passive' subprocess.call(installCmd, shell=False) time.sleep(30) @@ -206,18 +211,26 @@ def install_gateway(path): # ----------------------------------------------------------------------------------------------------------------- # Helper function to register Sql Migration Service on IR # ----------------------------------------------------------------------------------------------------------------- -def register_ir(key): +def register_ir(key, installed_ir_path=None): print(f"Start to register IR with key: {key}") - cmdFilePath = get_cmd_file_path() + # get SHIR installation location - using registry or user provided + if installed_ir_path is None: + cmdFilePath = get_cmd_file_path() + else: + cmdFilePath = get_cmd_file_path_from_input(installed_ir_path) + # extract the dmgcmd.exe and RegisterIntegrationRuntime.ps1 Script path. directoryPath = os.path.dirname(cmdFilePath) parentDirPath = os.path.dirname(directoryPath) dmgCmdPath = os.path.join(directoryPath, "dmgcmd.exe") regIRScriptPath = os.path.join(parentDirPath, "PowerShellScript", "RegisterIntegrationRuntime.ps1") + # Open Intranet Port (Necessary for Re-Register. Service has to be running for Re-Register to work.) portCmd = f'{dmgCmdPath} -EnableRemoteAccess 8060' + + # Register/ Re-register IR irCmd = f'powershell -command "& \'{regIRScriptPath}\' -gatewayKey {key}"' subprocess.call(portCmd, shell=False) @@ -225,7 +238,7 @@ def register_ir(key): # ----------------------------------------------------------------------------------------------------------------- -# Helper function to get SHIR script path +# Helper function to get SHIR script path from windows registry # ----------------------------------------------------------------------------------------------------------------- def get_cmd_file_path(): @@ -239,18 +252,20 @@ def get_cmd_file_path(): accessValue = winreg.QueryValueEx(accessKey, r"DiacmdPath")[0] return accessValue + + # Handling the case for x32 Python as x64 software like SHIR in registry are not found by x32 Python. except FileNotFoundError: try: diaCmdPath = get_cmd_file_path_static() return diaCmdPath except FileNotFoundError as e: - raise FileOperationError("Failed: No installed IR found or installed IR is not present in Program Files. Please install Integration Runtime in default location and re-run this command") from e + raise FileOperationError("Failed: No installed IR found or installed IR is not present in Program Files. Please install Integration Runtime in default location and re-run this command or use --installed-ir-path parameter to provide the installed IR location") from e except IndexError as e: raise FileOperationError("IR is not properly installed. Please re-install it and re-run this command") from e # ----------------------------------------------------------------------------------------------------------------- -# Helper function to get DiaCmdPath with Static Paths. This function assumes that IR is always installed in program files +# Helper function to get DiaCmdPath with Static Paths. This function assumes that IR is always installed in program files. This function is for handling the case with x32 Python # ----------------------------------------------------------------------------------------------------------------- def get_cmd_file_path_static(): @@ -274,3 +289,20 @@ def get_cmd_file_path_static(): raise FileNotFoundError(f"The system cannot find the path specified: {diaCmdPath}") return diaCmdPath + + +# ----------------------------------------------------------------------------------------------------------------- +# Helper function to get DiaCmdPath using the installed IR path user has given +# ----------------------------------------------------------------------------------------------------------------- +def get_cmd_file_path_from_input(installed_ir_path): + + if not os.path.exists(installed_ir_path): + raise FileNotFoundError(f"The system cannot find the path specified: {installed_ir_path}") + + # Create diaCmd default path and check if it is valid or not. + diaCmdPath = os.path.join(installed_ir_path, "Shared", "diacmd.exe") + + if not os.path.exists(diaCmdPath): + raise FileNotFoundError(f"The system cannot find the path specified: {diaCmdPath}") + + return diaCmdPath diff --git a/src/datamigration/setup.py b/src/datamigration/setup.py index 5a090250d28..315bfa127af 100644 --- a/src/datamigration/setup.py +++ b/src/datamigration/setup.py @@ -10,7 +10,7 @@ from setuptools import setup, find_packages # HISTORY.rst entry. -VERSION = '0.3.0' +VERSION = '0.3.1' try: from azext_datamigration.manual.version import VERSION except ImportError: