diff --git a/src/image-copy/azext_imagecopy/__init__.py b/src/image-copy/azext_imagecopy/__init__.py new file mode 100644 index 00000000000..6b540870c11 --- /dev/null +++ b/src/image-copy/azext_imagecopy/__init__.py @@ -0,0 +1,27 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +from azure.cli.core.help_files import helps +from azure.cli.core.sdk.util import ParametersContext + +helps['image copy'] = """ + type: command + short-summary: Allows to copy a managed image (or vm) to other regions. Keep in mind that it requires the source disk to be available. +""" + +def load_params(_): + with ParametersContext('image copy') as c: + c.register('source_resource_group_name', '--source-resource-group', help='Name of the resource group of the source resource') + c.register('source_object_name', '--source-object-name', help='The name of the image or vm resource') + c.register('target_location', '--target-location', nargs='+', help='Space separated location list to create the image in (use location short codes like westeurope etc.)') + c.register('source_type', '--source-type', default='image', choices=['image', 'vm'], help='image or vm') + c.register('target_resource_group_name', '--target-resource-group', help='Name of the resource group to create images in') + c.register('parallel_degree', '--parallel-degree', type=int, default=-1, help='Number of parallel copy operations') + c.register('cleanup', '--cleanup', action='store_true', default=False, \ + help='Include this switch to delete temporary resources upon completion') + +def load_commands(): + from azure.cli.core.commands import cli_command + cli_command(__name__, 'image copy', 'azext_imagecopy.custom#imagecopy') diff --git a/src/image-copy/azext_imagecopy/azext_metadata.json b/src/image-copy/azext_imagecopy/azext_metadata.json new file mode 100644 index 00000000000..7a73a41bfdf --- /dev/null +++ b/src/image-copy/azext_imagecopy/azext_metadata.json @@ -0,0 +1,2 @@ +{ +} \ No newline at end of file diff --git a/src/image-copy/azext_imagecopy/cli_utils.py b/src/image-copy/azext_imagecopy/cli_utils.py new file mode 100644 index 00000000000..32276cfa29a --- /dev/null +++ b/src/image-copy/azext_imagecopy/cli_utils.py @@ -0,0 +1,44 @@ +import sys +import json + +from subprocess import check_output, STDOUT, CalledProcessError +from azure.cli.core.util import CLIError +import azure.cli.core.azlogging as azlogging + +logger = azlogging.get_az_logger(__name__) + + +def run_cli_command(cmd, return_as_json=False): + try: + cmd_output = check_output(cmd, stderr=STDOUT, universal_newlines=True) + logger.debug('command: %s ended with output: %s', cmd, cmd_output) + + if return_as_json is True: + if cmd_output: + json_output = json.loads(cmd_output) + return json_output + else: + raise CLIError("Command returned an unexpected empty string.") + else: + return cmd_output + except CalledProcessError as ex: + logger.error('command failed: %s', cmd) + logger.error('output: %s', ex.output) + raise ex + except: + logger.error('command ended with an error: %s', cmd) + raise + +def prepare_cli_command(cmd, output_as_json=True): + full_cmd = [sys.executable, '-m', 'azure.cli'] + cmd + + if output_as_json: + full_cmd += ['--output', 'json'] + else: + full_cmd += ['--output', 'tsv'] + + # tag newly created resources, containers don't have tags + if 'create' in cmd and ('container' not in cmd): + full_cmd += ['--tags', 'created_by=image-copy-extension'] + + return full_cmd diff --git a/src/image-copy/azext_imagecopy/create_target.py b/src/image-copy/azext_imagecopy/create_target.py new file mode 100644 index 00000000000..e81e45fda90 --- /dev/null +++ b/src/image-copy/azext_imagecopy/create_target.py @@ -0,0 +1,166 @@ +import hashlib +import datetime +import time +from azext_imagecopy.cli_utils import run_cli_command, prepare_cli_command +from azure.cli.core.application import APPLICATION +from azure.cli.core.util import CLIError +import azure.cli.core.azlogging as azlogging + +logger = azlogging.get_az_logger(__name__) + +PROGRESS_LINE_LENGTH = 40 + +def create_target_image(location, transient_resource_group_name, source_type, source_object_name, \ + source_os_disk_snapshot_name, source_os_disk_snapshot_url, source_os_type, \ + target_resource_group_name, azure_pool_frequency): + + subscription_id = get_subscription_id() + + subscription_hash = hashlib.sha1(subscription_id.encode("UTF-8")).hexdigest() + unique_subscription_string = subscription_hash[:7] + + + # create the target storage account + logger.warn("{0} - Creating target storage account (can be slow sometimes)".format(location)) + target_storage_account_name = location + unique_subscription_string + cmd = prepare_cli_command(['storage', 'account', 'create', \ + '--name', target_storage_account_name, \ + '--resource-group', transient_resource_group_name, \ + '--location', location, \ + '--sku', 'Standard_LRS']) + + json_output = run_cli_command(cmd, return_as_json=True) + target_blob_endpoint = json_output['primaryEndpoints']['blob'] + + + # Setup the target storage account + cmd = prepare_cli_command(['storage', 'account', 'keys', 'list', \ + '--account-name', target_storage_account_name, \ + '--resource-group', transient_resource_group_name]) + + json_output = run_cli_command(cmd, return_as_json=True) + + target_storage_account_key = json_output[0]['value'] + logger.debug(target_storage_account_key) + + expiry_format = "%Y-%m-%dT%H:%MZ" + expiry = datetime.datetime.utcnow() + datetime.timedelta(hours=1) + + cmd = prepare_cli_command(['storage', 'account', 'generate-sas', \ + '--account-name', target_storage_account_name, \ + '--account-key', target_storage_account_key, \ + '--expiry', expiry.strftime(expiry_format), \ + '--permissions', 'aclrpuw', '--resource-types', \ + 'sco', '--services', 'b', '--https-only'], \ + output_as_json=False) + + sas_token = run_cli_command(cmd) + sas_token = sas_token.rstrip("\n\r") #STRANGE + logger.debug("sas token: " + sas_token) + + + # create a container in the target blob storage account + logger.warn("{0} - Creating container in the target storage account".format(location)) + target_container_name = 'snapshots' + cmd = prepare_cli_command(['storage', 'container', 'create', \ + '--name', target_container_name, \ + '--account-name', target_storage_account_name]) + + run_cli_command(cmd) + + + # Copy the snapshot to the target region using the SAS URL + blob_name = source_os_disk_snapshot_name + '.vhd' + logger.warn("{0} - Copying blob to target storage account".format(location)) + cmd = prepare_cli_command(['storage', 'blob', 'copy', 'start', \ + '--source-uri', source_os_disk_snapshot_url, \ + '--destination-blob', blob_name, \ + '--destination-container', target_container_name, \ + '--account-name', target_storage_account_name, \ + '--sas-token', sas_token]) + + run_cli_command(cmd) + + + # Wait for the copy to complete + start_datetime = datetime.datetime.now() + wait_for_blob_copy_operation(blob_name, target_container_name, \ + target_storage_account_name, azure_pool_frequency, location) + msg = "{0} - Copy time: {1}".format(location, datetime.datetime.now()-start_datetime).ljust(PROGRESS_LINE_LENGTH) + logger.warn(msg) + + + # Create the snapshot in the target region from the copied blob + logger.warn("{0} - Creating snapshot in target region from the copied blob".format(location)) + target_blob_path = target_blob_endpoint + target_container_name + '/' + blob_name + target_snapshot_name = source_os_disk_snapshot_name + '-' + location + cmd = prepare_cli_command(['snapshot', 'create', \ + '--resource-group', transient_resource_group_name, \ + '--name', target_snapshot_name, \ + '--location', location, \ + '--source', target_blob_path]) + + json_output = run_cli_command(cmd, return_as_json=True) + target_snapshot_id = json_output['id'] + + # Create the final image + logger.warn("{0} - Creating final image".format(location)) + target_image_name = source_object_name + if source_type != 'image': + target_image_name += '-image' + target_image_name += '-' + location + + cmd = prepare_cli_command(['image', 'create', \ + '--resource-group', target_resource_group_name, \ + '--name', target_image_name, \ + '--location', location, \ + '--source', target_blob_path, \ + '--os-type', source_os_type, \ + '--source', target_snapshot_id]) + + run_cli_command(cmd) + + +def wait_for_blob_copy_operation(blob_name, target_container_name, target_storage_account_name, azure_pool_frequency, location): + progress_controller = APPLICATION.get_progress_controller() + copy_status = "pending" + prev_progress = -1 + while copy_status == "pending": + cmd = prepare_cli_command(['storage', 'blob', 'show', \ + '--name', blob_name, \ + '--container-name', target_container_name, \ + '--account-name', target_storage_account_name]) + + json_output = run_cli_command(cmd, return_as_json=True) + copy_status = json_output["properties"]["copy"]["status"] + copy_progress_1, copy_progress_2 = json_output["properties"]["copy"]["progress"].split("/") + current_progress = round(int(copy_progress_1)/int(copy_progress_2), 1) + + if current_progress != prev_progress: + msg = "{0} - copy progress: {1}%"\ + .format(location, str(current_progress))\ + .ljust(PROGRESS_LINE_LENGTH) #need to justify since messages overide each other + progress_controller.add(message=msg) + + prev_progress = current_progress + + try: + time.sleep(azure_pool_frequency) + except KeyboardInterrupt: + progress_controller.stop() + return + + + if copy_status == 'success': + progress_controller.stop() + else: + logger.error("The copy operation didn't succeed. Last status: %s", copy_status) + raise CLIError('Blob copy failed') + + +def get_subscription_id(): + cmd = prepare_cli_command(['account', 'show']) + json_output = run_cli_command(cmd, return_as_json=True) + subscription_id = json_output['id'] + + return subscription_id diff --git a/src/image-copy/azext_imagecopy/custom.py b/src/image-copy/azext_imagecopy/custom.py new file mode 100644 index 00000000000..e8ea2952c7a --- /dev/null +++ b/src/image-copy/azext_imagecopy/custom.py @@ -0,0 +1,131 @@ +from multiprocessing import Pool + +from azext_imagecopy.cli_utils import run_cli_command, prepare_cli_command +from azext_imagecopy.create_target import create_target_image +import azure.cli.core.azlogging as azlogging + +logger = azlogging.get_az_logger(__name__) + + +def imagecopy(source_resource_group_name, source_object_name, target_location, \ + target_resource_group_name, source_type='image', cleanup='false', parallel_degree=-1): + + # get the os disk id from source vm/image + logger.warn("Getting os disk id of the source vm/image") + cmd = prepare_cli_command([source_type, 'show', \ + '--name', source_object_name, \ + '--resource-group', source_resource_group_name]) + + json_cmd_output = run_cli_command(cmd, return_as_json=True) + + source_os_disk_id = json_cmd_output['storageProfile']['osDisk']['managedDisk']['id'] + source_os_type = json_cmd_output['storageProfile']['osDisk']['osType'] + logger.debug("source_os_disk_id: %s. source_os_type: %s", source_os_disk_id, source_os_type) + + + # create source snapshots + logger.warn("Creating source snapshot") + source_os_disk_snapshot_name = source_object_name + '_os_disk_snapshot' + cmd = prepare_cli_command(['snapshot', 'create', \ + '--name', source_os_disk_snapshot_name, \ + '--resource-group', source_resource_group_name, \ + '--source', source_os_disk_id]) + + run_cli_command(cmd) + + + # Get SAS URL for the snapshotName + logger.warn("Getting sas url for the source snapshot") + cmd = prepare_cli_command(['snapshot', 'grant-access', \ + '--name', source_os_disk_snapshot_name, \ + '--resource-group', source_resource_group_name, \ + '--duration-in-seconds', '3600']) + + json_output = run_cli_command(cmd, return_as_json=True) + + source_os_disk_snapshot_url = json_output['accessSas'] + logger.debug("source os disk snapshot url: %s" , source_os_disk_snapshot_url) + + + # Start processing in the target locations + + transient_resource_group_name = 'image-copy-rg' + create_resource_group(transient_resource_group_name, 'eastus') + + target_locations_count = len(target_location) + logger.warn("Target location count: %s", target_locations_count) + + create_resource_group(target_resource_group_name, target_location[0].strip()) + + if parallel_degree == -1: + pool = Pool(target_locations_count) + else: + pool = Pool(min(parallel_degree, target_locations_count)) + + # try to get a handle on arm's 409s + azure_pool_frequency = 5 + if target_locations_count >= 5: + azure_pool_frequency = 15 + elif target_locations_count >= 3: + azure_pool_frequency = 10 + + tasks = [] + for location in target_location: + location = location.strip() + tasks.append((location, transient_resource_group_name, source_type, \ + source_object_name, source_os_disk_snapshot_name, source_os_disk_snapshot_url, \ + source_os_type, target_resource_group_name, azure_pool_frequency)) + + logger.warn("Starting async process for all locations") + + for task in tasks: + pool.apply_async(create_target_image, task) + + try: + pool.close() + pool.join() + except KeyboardInterrupt: + logger.warn('User cancelled the operation') + if cleanup: + logger.warn('To cleanup temporary resources look for ones tagged with "image-copy-extension". \nYou can use the following command: az resource list --tag created_by=image-copy-extension') + pool.terminate() + return + + + # Cleanup + if cleanup: + logger.warn('Deleting transient resources') + + # Delete resource group + cmd = prepare_cli_command(['group', 'delete', '--no-wait', '--yes', \ + '--name', transient_resource_group_name]) + run_cli_command(cmd) + + # Revoke sas for source snapshot + cmd = prepare_cli_command(['snapshot', 'revoke-access', \ + '--name', source_os_disk_snapshot_name, \ + '--resource-group', source_resource_group_name]) + run_cli_command(cmd) + + # Delete source snapshot + cmd = prepare_cli_command(['snapshot', 'delete', \ + '--name', source_os_disk_snapshot_name, \ + '--resource-group', source_resource_group_name]) + run_cli_command(cmd) + + +def create_resource_group(resource_group_name, location): + # check if target resource group exists + cmd = prepare_cli_command(['group', 'exists', \ + '--name', resource_group_name], output_as_json=False) + + cmd_output = run_cli_command(cmd) + + if 'false' in cmd_output: + # create the target resource group + logger.warn("Creating resource group: %s", resource_group_name) + cmd = prepare_cli_command(['group', 'create', \ + '--name', resource_group_name, \ + '--location', location]) + + run_cli_command(cmd) diff --git a/src/image-copy/setup.cfg b/src/image-copy/setup.cfg new file mode 100644 index 00000000000..3c6e79cf31d --- /dev/null +++ b/src/image-copy/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal=1 diff --git a/src/image-copy/setup.py b/src/image-copy/setup.py new file mode 100644 index 00000000000..131d096e944 --- /dev/null +++ b/src/image-copy/setup.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python + +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +from codecs import open +from setuptools import setup, find_packages + +VERSION = "0.0.3" + +CLASSIFIERS = [ + 'Development Status :: 4 - Beta', + 'Intended Audience :: Developers', + 'Intended Audience :: System Administrators', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'License :: OSI Approved :: MIT License', +] + +DEPENDENCIES = [] + +setup( + name='imagecopyextension', + version=VERSION, + description='An Azure CLI Extension that copies images from region to region.', + long_description='An Azure CLI Extension that copies images from region to region.', + license='MIT', + author='Tamir Kamara', + author_email='tamir.kamara@microsoft.com', + url='https://github.com/Azure/azure-cli-extensions', + classifiers=CLASSIFIERS, + packages=find_packages(), + install_requires=DEPENDENCIES +)