From 6f74be7a83cd1f1e8af051ff2511207b50365bf4 Mon Sep 17 00:00:00 2001 From: Tamir Kamara Date: Thu, 26 Oct 2017 22:13:39 +0100 Subject: [PATCH 01/11] image-copy: first commit --- src/image-copy/azext_imagecopy/__init__.py | 26 +++ .../azext_imagecopy/azext_metadata.json | 3 + src/image-copy/azext_imagecopy/cli_utils.py | 44 +++++ .../azext_imagecopy/create_target.py | 170 ++++++++++++++++++ src/image-copy/azext_imagecopy/custom.py | 135 ++++++++++++++ src/image-copy/setup.cfg | 2 + src/image-copy/setup.py | 42 +++++ 7 files changed, 422 insertions(+) create mode 100644 src/image-copy/azext_imagecopy/__init__.py create mode 100644 src/image-copy/azext_imagecopy/azext_metadata.json create mode 100644 src/image-copy/azext_imagecopy/cli_utils.py create mode 100644 src/image-copy/azext_imagecopy/create_target.py create mode 100644 src/image-copy/azext_imagecopy/custom.py create mode 100644 src/image-copy/setup.cfg create mode 100644 src/image-copy/setup.py diff --git a/src/image-copy/azext_imagecopy/__init__.py b/src/image-copy/azext_imagecopy/__init__.py new file mode 100644 index 00000000000..614db8f6dba --- /dev/null +++ b/src/image-copy/azext_imagecopy/__init__.py @@ -0,0 +1,26 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +from azure.cli.core.help_files import helps +from azure.cli.core.sdk.util import ParametersContext + +helps['image copy'] = """ + type: command + short-summary: Allows to copy an image (or vm) to other regions. Keep in mind that it requires the source disk to be available. +""" + +def load_params(_): + with ParametersContext('image copy') as c: + c.register('source_resource_group_name', '--source-resource-group-name', help='Name of the resource gorup of the source resource') + c.register('source_object_name', '--source-object-name', help='The name of the image or vm resource') + c.register('target_location', '--target-location', help='Comma seperated location list to create the image in') + c.register('source_type', '--source-type', help='image (default) or vm') + c.register('target_resource_group_name', '--target-resource-group-name', help='Name of the resource group to create images in') + c.register('parallel_degree', '--parallel-degree', help='Number of parallel copy operations') + c.register('cleanup', '--cleanup', help='Will delete temporary resources created in process (false by default)') + +def load_commands(): + from azure.cli.core.commands import cli_command + cli_command(__name__, 'image copy', 'azext_imagecopy.custom#imagecopy') diff --git a/src/image-copy/azext_imagecopy/azext_metadata.json b/src/image-copy/azext_imagecopy/azext_metadata.json new file mode 100644 index 00000000000..b6c27f96fe3 --- /dev/null +++ b/src/image-copy/azext_imagecopy/azext_metadata.json @@ -0,0 +1,3 @@ +{ + "azext.minCliVersion": "2.0.12" +} \ No newline at end of file diff --git a/src/image-copy/azext_imagecopy/cli_utils.py b/src/image-copy/azext_imagecopy/cli_utils.py new file mode 100644 index 00000000000..a3388f9c226 --- /dev/null +++ b/src/image-copy/azext_imagecopy/cli_utils.py @@ -0,0 +1,44 @@ +import sys +import json + +from subprocess import check_output, STDOUT, CalledProcessError + +import azure.cli.core.azlogging as azlogging + +logger = azlogging.get_az_logger(__name__) + + +def run_cli_command(cmd, return_as_json=False): + try: + cmd_output = check_output(cmd, stderr=STDOUT, universal_newlines=True) + logger.debug(cmd_output) + + if return_as_json is True: + if cmd_output: + json_output = json.loads(cmd_output) + return json_output + else: + raise Exception("Command returned an unexpected empty string.") + else: + return cmd_output + except CalledProcessError as ex: + print('command failed: ', cmd) + print('output: ', ex.output) + raise ex + except: + print('command: ', cmd) + raise + +def prepare_cli_command(cmd, output_as_json=True): + full_cmd = [sys.executable, '-m', 'azure.cli'] + cmd + + if output_as_json: + full_cmd += ['--output', 'json'] + else: + full_cmd += ['--output', 'tsv'] + + # tag newly created resources + if 'create' in cmd and ('container' not in cmd): + full_cmd += ['--tags', 'created_by=image-copy-extension'] + + return full_cmd diff --git a/src/image-copy/azext_imagecopy/create_target.py b/src/image-copy/azext_imagecopy/create_target.py new file mode 100644 index 00000000000..f3c28a34226 --- /dev/null +++ b/src/image-copy/azext_imagecopy/create_target.py @@ -0,0 +1,170 @@ +import hashlib +import datetime +import time +from azext_imagecopy.cli_utils import run_cli_command, prepare_cli_command +from azure.cli.core.application import APPLICATION +import azure.cli.core.azlogging as azlogging + +logger = azlogging.get_az_logger(__name__) + + +def create_target_image(location, transient_resource_group_name, source_type, source_object_name, \ + source_os_disk_snapshot_name, source_os_disk_snapshot_url, source_os_type, \ + target_resource_group_name, azure_pool_frequency): + + #from azure.cli.core.commands.client_factory import get_subscription_id + subscription_id = get_subscription_id() + + subscription_hash = hashlib.sha1(subscription_id.encode("UTF-8")).hexdigest() + unique_subscription_string = subscription_hash[:7] + + + # create the target storage account + logger.warn("{0} - Creating target storage account (can be slow sometimes)".format(location)) + target_storage_account_name = location + unique_subscription_string + cmd = prepare_cli_command(['storage', 'account', 'create', \ + '--name', target_storage_account_name, \ + '--resource-group', transient_resource_group_name, \ + '--location', location, \ + '--sku', 'Standard_LRS']) + + json_output = run_cli_command(cmd, True) + target_blob_endpoint = json_output['primaryEndpoints']['blob'] + + + # Setup the target storage account + cmd = prepare_cli_command(['storage', 'account', 'keys', 'list', \ + '--account-name', target_storage_account_name, \ + '--resource-group', transient_resource_group_name]) + + json_output = run_cli_command(cmd, True) + + target_storage_account_key = json_output[0]['value'] + logger.debug(target_storage_account_key) + + expiry_format = "%Y-%m-%dT%H:%MZ" + expiry = datetime.datetime.utcnow() + datetime.timedelta(hours=1) + + cmd = prepare_cli_command(['storage', 'account', 'generate-sas', \ + '--account-name', target_storage_account_name, \ + '--account-key', target_storage_account_key, \ + '--expiry', expiry.strftime(expiry_format), \ + '--permissions', 'aclrpuw', '--resource-types', \ + 'sco', '--services', 'b', '--https-only'], \ + False) + + sas_token = run_cli_command(cmd) + sas_token = sas_token.rstrip("\n\r") #STRANGE + logger.debug("sas token: " + sas_token) + + + # create a container in the target blob storage account + logger.warn("{0} - Creating container in the target storage account".format(location)) + target_container_name = 'snapshots' + cmd = prepare_cli_command(['storage', 'container', 'create', \ + '--name', target_container_name, \ + '--account-name', target_storage_account_name]) + + run_cli_command(cmd) + + + # Copy the snapshot to the target region using the SAS URL + blob_name = source_os_disk_snapshot_name + '.vhd' + logger.warn("{0} - Copying blob to target storage account".format(location)) + cmd = prepare_cli_command(['storage', 'blob', 'copy', 'start', \ + '--source-uri', source_os_disk_snapshot_url, \ + '--destination-blob', blob_name, \ + '--destination-container', target_container_name, \ + '--account-name', target_storage_account_name, \ + '--sas-token', sas_token]) + + run_cli_command(cmd) + + + # Wait for the copy to complete + start_datetime = datetime.datetime.now() + wait_for_blob_copy_operation(blob_name, target_container_name, \ + target_storage_account_name, azure_pool_frequency, location) + msg = "{0} - Copy time: {1}".format(location, datetime.datetime.now()-start_datetime).ljust(40) + logger.warn(msg) + + + # Create the snapshot in the target region from the copied blob + logger.warn("{0} - Creating snapshot in target region from the copied blob".format(location)) + target_blob_path = target_blob_endpoint + target_container_name + '/' + blob_name + target_snapshot_name = source_os_disk_snapshot_name + '-' + location + cmd = prepare_cli_command(['snapshot', 'create', \ + '--resource-group', transient_resource_group_name, \ + '--name', target_snapshot_name, \ + '--location', location, \ + '--source', target_blob_path]) + + json_output = run_cli_command(cmd, True) + target_snapshot_id = json_output['id'] + + # Create the final image + logger.warn("{0} - Creating final image".format(location)) + target_image_name = source_object_name + if source_type != 'image': + target_image_name += '-image' + target_image_name += '-' + location + + cmd = prepare_cli_command(['image', 'create', \ + '--resource-group', target_resource_group_name, \ + '--name', target_image_name, \ + '--location', location, \ + '--source', target_blob_path, \ + '--os-type', source_os_type, \ + '--source', target_snapshot_id]) + + run_cli_command(cmd) + + +def wait_for_blob_copy_operation(blob_name, target_container_name, target_storage_account_name, azure_pool_frequency, location): + progress_controller = APPLICATION.get_progress_controller() + copy_status = "pending" + prev_progress = -1 + while copy_status == "pending": + cmd = prepare_cli_command(['storage', 'blob', 'show', \ + '--name', blob_name, \ + '--container-name', target_container_name, \ + '--account-name', target_storage_account_name]) + + json_output = run_cli_command(cmd, True) + copy_status = json_output["properties"]["copy"]["status"] + copy_progress_1, copy_progress_2 = json_output["properties"]["copy"]["progress"].split("/") + current_progress = round(int(copy_progress_1)/int(copy_progress_2), 1) + + if current_progress != prev_progress: + msg = "{0} - copy progress: {1}%".format(location, str(current_progress)).ljust(40) #need to justify since message overide each other + progress_controller.add(message=msg) + + prev_progress = current_progress + + try: + time.sleep(azure_pool_frequency) + except KeyboardInterrupt: + print('xxx - child') + progress_controller.stop() + return + + + if copy_status == 'success': + progress_controller.stop() + else: + logger.error("The copy operation didn't succeed. Last status: " + copy_status) + raise Exception('Blob copy failed') + + +def get_subscription_id(): + logger.warn("Get subscription id") + + cmd = prepare_cli_command(['account', 'show']) + json_output = run_cli_command(cmd, True) + subscription_id = json_output['id'] + + # from azure.cli.core._profile import Profile + # profile = Profile() + # _, subscription_id, _ = profile.get_login_credentials() + + return subscription_id diff --git a/src/image-copy/azext_imagecopy/custom.py b/src/image-copy/azext_imagecopy/custom.py new file mode 100644 index 00000000000..cad6f8e5c9b --- /dev/null +++ b/src/image-copy/azext_imagecopy/custom.py @@ -0,0 +1,135 @@ +from multiprocessing import Pool + +from azext_imagecopy.cli_utils import run_cli_command, prepare_cli_command +from azext_imagecopy.create_target import create_target_image +import azure.cli.core.azlogging as azlogging + +logger = azlogging.get_az_logger(__name__) + + +def imagecopy(source_resource_group_name, source_object_name, target_location, \ + target_resource_group_name, source_type='image', cleanup='false', parallel_degree=-1): + + # get the os disk id from source vm/image + logger.warn("Getting os disk id of the source vm/image") + cmd = prepare_cli_command([source_type, 'show', \ + '--name', source_object_name, \ + '--resource-group', source_resource_group_name]) + + json_cmd_output = run_cli_command(cmd, True) + + source_os_disk_id = json_cmd_output['storageProfile']['osDisk']['managedDisk']['id'] + source_os_type = json_cmd_output['storageProfile']['osDisk']['osType'] + logger.debug("source_os_disk_id: " + source_os_disk_id + " source_os_type: " + source_os_type) + + + # create source snapshots + logger.warn("Creating source snapshot") + source_os_disk_snapshot_name = source_object_name + '_os_disk_snapshot' + cmd = prepare_cli_command(['snapshot', 'create', \ + '--name', source_os_disk_snapshot_name, \ + '--resource-group', source_resource_group_name, \ + '--source', source_os_disk_id]) + + run_cli_command(cmd) + + + # Get SAS URL for the snapshotName + logger.warn("Getting sas url for the source snapshot") + cmd = prepare_cli_command(['snapshot', 'grant-access', \ + '--name', source_os_disk_snapshot_name, \ + '--resource-group', source_resource_group_name, \ + '--duration-in-seconds', '3600']) + + json_output = run_cli_command(cmd, True) + + source_os_disk_snapshot_url = json_output['accessSas'] + logger.debug(source_os_disk_snapshot_url) + + + # Start processing in the target locations + + transient_resource_group_name = 'image-copy-rg' + create_resource_group(transient_resource_group_name, 'eastus') + + target_locations_list = target_location.split(',') + target_locations_count = len(target_locations_list) + logger.warn("Target location count: {0}".format(target_locations_count)) + + create_resource_group(target_resource_group_name, target_locations_list[0].strip()) + + if parallel_degree == -1: + pool = Pool(target_locations_count) + elif parallel_degree == 1: + pool = Pool(1) + else: + pool = Pool(min(parallel_degree, target_locations_count)) + + # try to get a handle on arm 409s + azure_pool_frequency = 5 + if target_locations_count >= 5: + azure_pool_frequency = 15 + elif target_locations_count >= 3: + azure_pool_frequency = 10 + + tasks = [] + for location in target_location.split(','): + location = location.strip() + tasks.append((location, transient_resource_group_name, source_type, \ + source_object_name, source_os_disk_snapshot_name, source_os_disk_snapshot_url, \ + source_os_type, target_resource_group_name, azure_pool_frequency)) + + logger.warn("Starting async process for all locations") + + for task in tasks: + pool.apply_async(create_target_image, task) + + try: + pool.close() + pool.join() + except KeyboardInterrupt: + print('xxx - parent') + logger.warn('User cancelled the operation') + if 'true' in cleanup: + logger.warn('To cleanup temporary resources look for ones tagged with "image-copy-extension"') + pool.terminate() + return + + + # Cleanup + if 'true' in cleanup: + logger.warn('Deleting transient resources') + + # Delete resource group + cmd = prepare_cli_command(['group', 'delete', '--no-wait', '--yes', \ + '--name', transient_resource_group_name]) + run_cli_command(cmd) + + # Revoke sas for source snapshot + cmd = prepare_cli_command(['snapshot', 'revoke-access', \ + '--name', source_os_disk_snapshot_name, \ + '--resource-group', source_resource_group_name]) + run_cli_command(cmd) + + # Delete source snapshot + cmd = prepare_cli_command(['snapshot', 'delete', \ + '--name', source_os_disk_snapshot_name, \ + '--resource-group', source_resource_group_name]) + run_cli_command(cmd) + + +def create_resource_group(resource_group_name, location): + # check if target resource group exists + cmd = prepare_cli_command(['group', 'exists', \ + '--name', resource_group_name], False) + + cmd_output = run_cli_command(cmd) + + if 'false' in cmd_output: + # create the target resource group + logger.warn("Creating resource group") + cmd = prepare_cli_command(['group', 'create', \ + '--name', resource_group_name, \ + '--location', location]) + + run_cli_command(cmd) diff --git a/src/image-copy/setup.cfg b/src/image-copy/setup.cfg new file mode 100644 index 00000000000..3c6e79cf31d --- /dev/null +++ b/src/image-copy/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal=1 diff --git a/src/image-copy/setup.py b/src/image-copy/setup.py new file mode 100644 index 00000000000..058eb4c8bfd --- /dev/null +++ b/src/image-copy/setup.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python + +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +from codecs import open +from setuptools import setup, find_packages + +VERSION = "0.0.2" + +CLASSIFIERS = [ + 'Development Status :: 4 - Beta', + 'Intended Audience :: Developers', + 'Intended Audience :: System Administrators', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'License :: OSI Approved :: MIT License', +] + +DEPENDENCIES = [] + +setup( + name='imagecopyextension', + version=VERSION, + description='An Azure CLI Extension that copies images from region to region.', + long_description='An Azure CLI Extension that copies images from region to region.', + license='MIT', + author='Tamir Kamara', + author_email='tamir.kamara@microsoft.com', + url='https://github.com/ORG/REPO', + classifiers=CLASSIFIERS, + packages=find_packages(), + package_data={'azext_hello': ['azext_metadata.json']}, + install_requires=DEPENDENCIES +) From 298aae43a1a8c20ee940c68fb520b3a5a3bb9889 Mon Sep 17 00:00:00 2001 From: Tamir Kamara Date: Mon, 30 Oct 2017 21:10:10 +0200 Subject: [PATCH 02/11] fixes after @derekbekoe reviewed --- src/image-copy/azext_imagecopy/__init__.py | 15 +++++---- src/image-copy/azext_imagecopy/cli_utils.py | 12 +++---- .../azext_imagecopy/create_target.py | 27 ++++++++-------- src/image-copy/azext_imagecopy/custom.py | 32 ++++++++----------- src/image-copy/setup.py | 6 ++-- 5 files changed, 45 insertions(+), 47 deletions(-) diff --git a/src/image-copy/azext_imagecopy/__init__.py b/src/image-copy/azext_imagecopy/__init__.py index 614db8f6dba..b01594a7080 100644 --- a/src/image-copy/azext_imagecopy/__init__.py +++ b/src/image-copy/azext_imagecopy/__init__.py @@ -8,18 +8,19 @@ helps['image copy'] = """ type: command - short-summary: Allows to copy an image (or vm) to other regions. Keep in mind that it requires the source disk to be available. + short-summary: Allows to copy a managed image (or vm) to other regions. Keep in mind that it requires the source disk to be available. """ def load_params(_): with ParametersContext('image copy') as c: - c.register('source_resource_group_name', '--source-resource-group-name', help='Name of the resource gorup of the source resource') + c.register('source_resource_group_name', '--source-resource-group', help='Name of the resource group of the source resource') c.register('source_object_name', '--source-object-name', help='The name of the image or vm resource') - c.register('target_location', '--target-location', help='Comma seperated location list to create the image in') - c.register('source_type', '--source-type', help='image (default) or vm') - c.register('target_resource_group_name', '--target-resource-group-name', help='Name of the resource group to create images in') - c.register('parallel_degree', '--parallel-degree', help='Number of parallel copy operations') - c.register('cleanup', '--cleanup', help='Will delete temporary resources created in process (false by default)') + c.register('target_location', '--target-location', nargs='+', help='Space separated location list to create the image in (use location short codes like westeurope etc.)') + c.register('source_type', '--source-type', default='image', help='image or vm') + c.register('target_resource_group_name', '--target-resource-group', help='Name of the resource group to create images in') + c.register('parallel_degree', '--parallel-degree', type=int, default=-1, help='Number of parallel copy operations') + c.register('cleanup', '--cleanup', action='store_true', default=False, \ + help='Include this switch to delete temporary resources upon completion') def load_commands(): from azure.cli.core.commands import cli_command diff --git a/src/image-copy/azext_imagecopy/cli_utils.py b/src/image-copy/azext_imagecopy/cli_utils.py index a3388f9c226..9247c170edd 100644 --- a/src/image-copy/azext_imagecopy/cli_utils.py +++ b/src/image-copy/azext_imagecopy/cli_utils.py @@ -2,7 +2,7 @@ import json from subprocess import check_output, STDOUT, CalledProcessError - +from azure.cli.core.util import CLIError import azure.cli.core.azlogging as azlogging logger = azlogging.get_az_logger(__name__) @@ -11,22 +11,22 @@ def run_cli_command(cmd, return_as_json=False): try: cmd_output = check_output(cmd, stderr=STDOUT, universal_newlines=True) - logger.debug(cmd_output) + logger.debug('command: %s ended with output: %s', cmd, cmd_output) if return_as_json is True: if cmd_output: json_output = json.loads(cmd_output) return json_output else: - raise Exception("Command returned an unexpected empty string.") + raise CLIError("Command returned an unexpected empty string.") else: return cmd_output except CalledProcessError as ex: - print('command failed: ', cmd) - print('output: ', ex.output) + logger.error('command failed: %s', cmd) + logger.error('output: %s', ex.output) raise ex except: - print('command: ', cmd) + logger.error('command ended with an error: %s', cmd) raise def prepare_cli_command(cmd, output_as_json=True): diff --git a/src/image-copy/azext_imagecopy/create_target.py b/src/image-copy/azext_imagecopy/create_target.py index f3c28a34226..c65ebb0dcc8 100644 --- a/src/image-copy/azext_imagecopy/create_target.py +++ b/src/image-copy/azext_imagecopy/create_target.py @@ -3,10 +3,12 @@ import time from azext_imagecopy.cli_utils import run_cli_command, prepare_cli_command from azure.cli.core.application import APPLICATION +from azure.cli.core.util import CLIError import azure.cli.core.azlogging as azlogging logger = azlogging.get_az_logger(__name__) +PROGRESS_LINE_LENGTH = 40 def create_target_image(location, transient_resource_group_name, source_type, source_object_name, \ source_os_disk_snapshot_name, source_os_disk_snapshot_url, source_os_type, \ @@ -28,7 +30,7 @@ def create_target_image(location, transient_resource_group_name, source_type, so '--location', location, \ '--sku', 'Standard_LRS']) - json_output = run_cli_command(cmd, True) + json_output = run_cli_command(cmd, return_as_json=True) target_blob_endpoint = json_output['primaryEndpoints']['blob'] @@ -37,7 +39,7 @@ def create_target_image(location, transient_resource_group_name, source_type, so '--account-name', target_storage_account_name, \ '--resource-group', transient_resource_group_name]) - json_output = run_cli_command(cmd, True) + json_output = run_cli_command(cmd, return_as_json=True) target_storage_account_key = json_output[0]['value'] logger.debug(target_storage_account_key) @@ -51,7 +53,7 @@ def create_target_image(location, transient_resource_group_name, source_type, so '--expiry', expiry.strftime(expiry_format), \ '--permissions', 'aclrpuw', '--resource-types', \ 'sco', '--services', 'b', '--https-only'], \ - False) + output_as_json=False) sas_token = run_cli_command(cmd) sas_token = sas_token.rstrip("\n\r") #STRANGE @@ -85,7 +87,7 @@ def create_target_image(location, transient_resource_group_name, source_type, so start_datetime = datetime.datetime.now() wait_for_blob_copy_operation(blob_name, target_container_name, \ target_storage_account_name, azure_pool_frequency, location) - msg = "{0} - Copy time: {1}".format(location, datetime.datetime.now()-start_datetime).ljust(40) + msg = "{0} - Copy time: {1}".format(location, datetime.datetime.now()-start_datetime).ljust(PROGRESS_LINE_LENGTH) logger.warn(msg) @@ -99,7 +101,7 @@ def create_target_image(location, transient_resource_group_name, source_type, so '--location', location, \ '--source', target_blob_path]) - json_output = run_cli_command(cmd, True) + json_output = run_cli_command(cmd, return_as_json=True) target_snapshot_id = json_output['id'] # Create the final image @@ -130,13 +132,15 @@ def wait_for_blob_copy_operation(blob_name, target_container_name, target_storag '--container-name', target_container_name, \ '--account-name', target_storage_account_name]) - json_output = run_cli_command(cmd, True) + json_output = run_cli_command(cmd, return_as_json=True) copy_status = json_output["properties"]["copy"]["status"] copy_progress_1, copy_progress_2 = json_output["properties"]["copy"]["progress"].split("/") current_progress = round(int(copy_progress_1)/int(copy_progress_2), 1) if current_progress != prev_progress: - msg = "{0} - copy progress: {1}%".format(location, str(current_progress)).ljust(40) #need to justify since message overide each other + msg = "{0} - copy progress: {1}%"\ + .format(location, str(current_progress))\ + .ljust(PROGRESS_LINE_LENGTH) #need to justify since messages overide each other progress_controller.add(message=msg) prev_progress = current_progress @@ -144,7 +148,6 @@ def wait_for_blob_copy_operation(blob_name, target_container_name, target_storag try: time.sleep(azure_pool_frequency) except KeyboardInterrupt: - print('xxx - child') progress_controller.stop() return @@ -152,15 +155,13 @@ def wait_for_blob_copy_operation(blob_name, target_container_name, target_storag if copy_status == 'success': progress_controller.stop() else: - logger.error("The copy operation didn't succeed. Last status: " + copy_status) - raise Exception('Blob copy failed') + logger.error("The copy operation didn't succeed. Last status: %s", copy_status) + raise CLIError('Blob copy failed') def get_subscription_id(): - logger.warn("Get subscription id") - cmd = prepare_cli_command(['account', 'show']) - json_output = run_cli_command(cmd, True) + json_output = run_cli_command(cmd, return_as_json=True) subscription_id = json_output['id'] # from azure.cli.core._profile import Profile diff --git a/src/image-copy/azext_imagecopy/custom.py b/src/image-copy/azext_imagecopy/custom.py index cad6f8e5c9b..0a0171889d7 100644 --- a/src/image-copy/azext_imagecopy/custom.py +++ b/src/image-copy/azext_imagecopy/custom.py @@ -16,11 +16,11 @@ def imagecopy(source_resource_group_name, source_object_name, target_location, \ '--name', source_object_name, \ '--resource-group', source_resource_group_name]) - json_cmd_output = run_cli_command(cmd, True) + json_cmd_output = run_cli_command(cmd, return_as_json=True) source_os_disk_id = json_cmd_output['storageProfile']['osDisk']['managedDisk']['id'] source_os_type = json_cmd_output['storageProfile']['osDisk']['osType'] - logger.debug("source_os_disk_id: " + source_os_disk_id + " source_os_type: " + source_os_type) + logger.debug("source_os_disk_id: %s. source_os_type: %s", source_os_disk_id, source_os_type) # create source snapshots @@ -41,10 +41,10 @@ def imagecopy(source_resource_group_name, source_object_name, target_location, \ '--resource-group', source_resource_group_name, \ '--duration-in-seconds', '3600']) - json_output = run_cli_command(cmd, True) + json_output = run_cli_command(cmd, return_as_json=True) source_os_disk_snapshot_url = json_output['accessSas'] - logger.debug(source_os_disk_snapshot_url) + logger.debug("source os disk snapshot url: %s" , source_os_disk_snapshot_url) # Start processing in the target locations @@ -52,20 +52,17 @@ def imagecopy(source_resource_group_name, source_object_name, target_location, \ transient_resource_group_name = 'image-copy-rg' create_resource_group(transient_resource_group_name, 'eastus') - target_locations_list = target_location.split(',') - target_locations_count = len(target_locations_list) - logger.warn("Target location count: {0}".format(target_locations_count)) + target_locations_count = len(target_location) + logger.warn("Target location count: %s", target_locations_count) - create_resource_group(target_resource_group_name, target_locations_list[0].strip()) + create_resource_group(target_resource_group_name, target_location[0].strip()) if parallel_degree == -1: pool = Pool(target_locations_count) - elif parallel_degree == 1: - pool = Pool(1) else: pool = Pool(min(parallel_degree, target_locations_count)) - # try to get a handle on arm 409s + # try to get a handle on arm's 409s azure_pool_frequency = 5 if target_locations_count >= 5: azure_pool_frequency = 15 @@ -73,7 +70,7 @@ def imagecopy(source_resource_group_name, source_object_name, target_location, \ azure_pool_frequency = 10 tasks = [] - for location in target_location.split(','): + for location in target_location: location = location.strip() tasks.append((location, transient_resource_group_name, source_type, \ source_object_name, source_os_disk_snapshot_name, source_os_disk_snapshot_url, \ @@ -88,16 +85,15 @@ def imagecopy(source_resource_group_name, source_object_name, target_location, \ pool.close() pool.join() except KeyboardInterrupt: - print('xxx - parent') logger.warn('User cancelled the operation') - if 'true' in cleanup: - logger.warn('To cleanup temporary resources look for ones tagged with "image-copy-extension"') + if cleanup: + logger.warn('To cleanup temporary resources look for ones tagged with "image-copy-extension". \nYou can use the following command: az resource list --tag created_by=image-copy-extension') pool.terminate() return # Cleanup - if 'true' in cleanup: + if cleanup: logger.warn('Deleting transient resources') # Delete resource group @@ -121,13 +117,13 @@ def imagecopy(source_resource_group_name, source_object_name, target_location, \ def create_resource_group(resource_group_name, location): # check if target resource group exists cmd = prepare_cli_command(['group', 'exists', \ - '--name', resource_group_name], False) + '--name', resource_group_name], output_as_json=False) cmd_output = run_cli_command(cmd) if 'false' in cmd_output: # create the target resource group - logger.warn("Creating resource group") + logger.warn("Creating resource group: %s", ) cmd = prepare_cli_command(['group', 'create', \ '--name', resource_group_name, \ '--location', location]) diff --git a/src/image-copy/setup.py b/src/image-copy/setup.py index 058eb4c8bfd..015fd807b15 100644 --- a/src/image-copy/setup.py +++ b/src/image-copy/setup.py @@ -8,7 +8,7 @@ from codecs import open from setuptools import setup, find_packages -VERSION = "0.0.2" +VERSION = "0.0.3" CLASSIFIERS = [ 'Development Status :: 4 - Beta', @@ -34,9 +34,9 @@ license='MIT', author='Tamir Kamara', author_email='tamir.kamara@microsoft.com', - url='https://github.com/ORG/REPO', + url='https://github.com/Azure/azure-cli-extensions', classifiers=CLASSIFIERS, packages=find_packages(), - package_data={'azext_hello': ['azext_metadata.json']}, + package_data={'azext_imagecopy': ['azext_metadata.json']}, install_requires=DEPENDENCIES ) From 6a529a09693839e957340c697450f0ca9fd87eab Mon Sep 17 00:00:00 2001 From: Tamir Kamara Date: Mon, 30 Oct 2017 21:15:39 +0200 Subject: [PATCH 03/11] remove dead code --- src/image-copy/azext_imagecopy/create_target.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/image-copy/azext_imagecopy/create_target.py b/src/image-copy/azext_imagecopy/create_target.py index c65ebb0dcc8..e81e45fda90 100644 --- a/src/image-copy/azext_imagecopy/create_target.py +++ b/src/image-copy/azext_imagecopy/create_target.py @@ -14,7 +14,6 @@ def create_target_image(location, transient_resource_group_name, source_type, so source_os_disk_snapshot_name, source_os_disk_snapshot_url, source_os_type, \ target_resource_group_name, azure_pool_frequency): - #from azure.cli.core.commands.client_factory import get_subscription_id subscription_id = get_subscription_id() subscription_hash = hashlib.sha1(subscription_id.encode("UTF-8")).hexdigest() @@ -164,8 +163,4 @@ def get_subscription_id(): json_output = run_cli_command(cmd, return_as_json=True) subscription_id = json_output['id'] - # from azure.cli.core._profile import Profile - # profile = Profile() - # _, subscription_id, _ = profile.get_login_credentials() - return subscription_id From 6a551220b64e153df07a6e81c82a041574030889 Mon Sep 17 00:00:00 2001 From: Tamir Kamara Date: Tue, 31 Oct 2017 19:58:54 +0200 Subject: [PATCH 04/11] round of small fixes --- src/image-copy/azext_imagecopy/__init__.py | 2 +- src/image-copy/azext_imagecopy/azext_metadata.json | 1 - src/image-copy/azext_imagecopy/cli_utils.py | 2 +- src/image-copy/azext_imagecopy/custom.py | 2 +- src/image-copy/setup.py | 1 - src/image-copy/test.cmd | 12 ++++++++++++ 6 files changed, 15 insertions(+), 5 deletions(-) create mode 100644 src/image-copy/test.cmd diff --git a/src/image-copy/azext_imagecopy/__init__.py b/src/image-copy/azext_imagecopy/__init__.py index b01594a7080..6b540870c11 100644 --- a/src/image-copy/azext_imagecopy/__init__.py +++ b/src/image-copy/azext_imagecopy/__init__.py @@ -16,7 +16,7 @@ def load_params(_): c.register('source_resource_group_name', '--source-resource-group', help='Name of the resource group of the source resource') c.register('source_object_name', '--source-object-name', help='The name of the image or vm resource') c.register('target_location', '--target-location', nargs='+', help='Space separated location list to create the image in (use location short codes like westeurope etc.)') - c.register('source_type', '--source-type', default='image', help='image or vm') + c.register('source_type', '--source-type', default='image', choices=['image', 'vm'], help='image or vm') c.register('target_resource_group_name', '--target-resource-group', help='Name of the resource group to create images in') c.register('parallel_degree', '--parallel-degree', type=int, default=-1, help='Number of parallel copy operations') c.register('cleanup', '--cleanup', action='store_true', default=False, \ diff --git a/src/image-copy/azext_imagecopy/azext_metadata.json b/src/image-copy/azext_imagecopy/azext_metadata.json index b6c27f96fe3..7a73a41bfdf 100644 --- a/src/image-copy/azext_imagecopy/azext_metadata.json +++ b/src/image-copy/azext_imagecopy/azext_metadata.json @@ -1,3 +1,2 @@ { - "azext.minCliVersion": "2.0.12" } \ No newline at end of file diff --git a/src/image-copy/azext_imagecopy/cli_utils.py b/src/image-copy/azext_imagecopy/cli_utils.py index 9247c170edd..32276cfa29a 100644 --- a/src/image-copy/azext_imagecopy/cli_utils.py +++ b/src/image-copy/azext_imagecopy/cli_utils.py @@ -37,7 +37,7 @@ def prepare_cli_command(cmd, output_as_json=True): else: full_cmd += ['--output', 'tsv'] - # tag newly created resources + # tag newly created resources, containers don't have tags if 'create' in cmd and ('container' not in cmd): full_cmd += ['--tags', 'created_by=image-copy-extension'] diff --git a/src/image-copy/azext_imagecopy/custom.py b/src/image-copy/azext_imagecopy/custom.py index 0a0171889d7..e8ea2952c7a 100644 --- a/src/image-copy/azext_imagecopy/custom.py +++ b/src/image-copy/azext_imagecopy/custom.py @@ -123,7 +123,7 @@ def create_resource_group(resource_group_name, location): if 'false' in cmd_output: # create the target resource group - logger.warn("Creating resource group: %s", ) + logger.warn("Creating resource group: %s", resource_group_name) cmd = prepare_cli_command(['group', 'create', \ '--name', resource_group_name, \ '--location', location]) diff --git a/src/image-copy/setup.py b/src/image-copy/setup.py index 015fd807b15..131d096e944 100644 --- a/src/image-copy/setup.py +++ b/src/image-copy/setup.py @@ -37,6 +37,5 @@ url='https://github.com/Azure/azure-cli-extensions', classifiers=CLASSIFIERS, packages=find_packages(), - package_data={'azext_imagecopy': ['azext_metadata.json']}, install_requires=DEPENDENCIES ) diff --git a/src/image-copy/test.cmd b/src/image-copy/test.cmd new file mode 100644 index 00000000000..9f31dc749d7 --- /dev/null +++ b/src/image-copy/test.cmd @@ -0,0 +1,12 @@ +SET AZURE_EXTENSION_DIR=c:\Users\takamara\.azure\devcliextensions + +C:\Python\Python36-32\python setup.py bdist_wheel +C:\Python\Python36-32\scripts\pip install --upgrade --target C:\Users\takamara\.azure\devcliextensions\imagecopyextension C:\dev\azure-cli-extensions\src\image-copy + +rem az image copy --help + +az image copy --source-resource-group "test-rg" --source-object-name "vm2" --source-type "vm" --target-location uksouth northeurope --target-resource-group "images-repo" + +rem az image copy --source-resource-group "test-img-rg" --source-object-name "vm1-image" --target-location uksouth northeurope --target-resource-group "images-repo" --cleanup + +rem az vm-image-copy --source-resource-group "test-img-rg" --source-object-name "vm1-image" --source-type "image" --target-location "uksouth, northeurope, westus, eastus, australiaeast, eastasia" --target-resource-group "images-repo" --cleanup "true" \ No newline at end of file From b90cd9674b9b5522f3c64ba208df9fd24b380860 Mon Sep 17 00:00:00 2001 From: Tamir Kamara Date: Tue, 31 Oct 2017 20:04:47 +0200 Subject: [PATCH 05/11] remove test file --- src/image-copy/test.cmd | 12 ------------ 1 file changed, 12 deletions(-) delete mode 100644 src/image-copy/test.cmd diff --git a/src/image-copy/test.cmd b/src/image-copy/test.cmd deleted file mode 100644 index 9f31dc749d7..00000000000 --- a/src/image-copy/test.cmd +++ /dev/null @@ -1,12 +0,0 @@ -SET AZURE_EXTENSION_DIR=c:\Users\takamara\.azure\devcliextensions - -C:\Python\Python36-32\python setup.py bdist_wheel -C:\Python\Python36-32\scripts\pip install --upgrade --target C:\Users\takamara\.azure\devcliextensions\imagecopyextension C:\dev\azure-cli-extensions\src\image-copy - -rem az image copy --help - -az image copy --source-resource-group "test-rg" --source-object-name "vm2" --source-type "vm" --target-location uksouth northeurope --target-resource-group "images-repo" - -rem az image copy --source-resource-group "test-img-rg" --source-object-name "vm1-image" --target-location uksouth northeurope --target-resource-group "images-repo" --cleanup - -rem az vm-image-copy --source-resource-group "test-img-rg" --source-object-name "vm1-image" --source-type "image" --target-location "uksouth, northeurope, westus, eastus, australiaeast, eastasia" --target-resource-group "images-repo" --cleanup "true" \ No newline at end of file From 2258d3c0001b13fabec4ab6a79bfe0610e1d9e28 Mon Sep 17 00:00:00 2001 From: Tamir Kamara Date: Fri, 3 Nov 2017 23:20:25 +0200 Subject: [PATCH 06/11] merg commit --- .flake8 | 9 + .github/CODEOWNERS | 3 +- .github/PULL_REQUEST_TEMPLATE.md | 8 + .gitignore | 3 + .travis.yml | 33 ++- CONTRIBUTING.rst | 9 + pylintrc | 48 ++++ scripts/ci/test_integration.py | 231 ++++++++++++++++++ scripts/ci/test_source.sh | 16 ++ scripts/ci/test_static.sh | 12 + scripts/ci/util.py | 13 + scripts/ci/verify_codeowners.py | 42 ++++ scripts/ci/verify_license.py | 44 ++++ src/image-copy/azext_imagecopy/__init__.py | 26 +- src/image-copy/azext_imagecopy/cli_utils.py | 9 + .../azext_imagecopy/create_target.py | 115 ++++----- src/image-copy/azext_imagecopy/custom.py | 75 +++--- src/index.json | 2 +- 18 files changed, 588 insertions(+), 110 deletions(-) create mode 100644 .flake8 create mode 100644 .github/PULL_REQUEST_TEMPLATE.md create mode 100644 CONTRIBUTING.rst create mode 100644 pylintrc create mode 100644 scripts/ci/test_integration.py create mode 100755 scripts/ci/test_source.sh create mode 100755 scripts/ci/test_static.sh create mode 100644 scripts/ci/util.py create mode 100644 scripts/ci/verify_codeowners.py create mode 100644 scripts/ci/verify_license.py diff --git a/.flake8 b/.flake8 new file mode 100644 index 00000000000..283ad44ac5e --- /dev/null +++ b/.flake8 @@ -0,0 +1,9 @@ +[flake8] +max-line-length = 120 +max-complexity = 10 +ignore = + E501, # line too long, it is covered by pylint + E722, # bare except, bad practice, to be removed in the future + F401, # imported but unused, too many violations, to be removed in the future + F811, # redefinition of unused, to be removed in the future + C901 # code flow is too complex, too many violations, to be removed in the future diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index bd090e10614..b349363eb8a 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,4 +1,5 @@ # See for instructions on this file https://help.github.com/articles/about-codeowners/ -index.json @derekbekoe +/src/index.json @derekbekoe +/src/image-copy/ @tamirkamara diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 00000000000..a72068a9928 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,8 @@ +--- + +This checklist is used to make sure that common guidelines for a pull request are followed. + +### General Guidelines + +- [ ] Have you run `./scripts/ci/test_static.sh` locally? (`pip install pylint flake8` required) +- [ ] Have you run `python scripts/ci/test_integration.py -q` locally? diff --git a/.gitignore b/.gitignore index 7bbc71c0920..8b2dc2f0065 100644 --- a/.gitignore +++ b/.gitignore @@ -97,5 +97,8 @@ ENV/ # mkdocs documentation /site +# VS Code +.vscode/settings.json + # mypy .mypy_cache/ diff --git a/.travis.yml b/.travis.yml index 3ccb28a58fd..217c6b29309 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,10 +1,29 @@ dist: trusty sudo: off language: python -python: - - "2.7" - - "3.5" - - "3.6" -install: true -script: - - ls +install: + - pip install pylint flake8 requests -q +jobs: + include: + - stage: verify + env: PURPOSE='SourceStatic' + script: ./scripts/ci/test_static.sh + python: 2.7 + - stage: verify + env: PURPOSE='SourceStatic' + script: ./scripts/ci/test_static.sh + python: 3.6 + - stage: verify + env: PURPOSE='SourceTests' + script: ./scripts/ci/test_source.sh + python: 2.7 + - stage: verify + env: PURPOSE='SourceTests' + script: ./scripts/ci/test_source.sh + python: 3.6 + - stage: verify + env: PURPOSE='IndexVerify' + script: python ./scripts/ci/test_integration.py -v + python: 3.6 + allow_failures: + - env: PURPOSE='SourceTests' diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst new file mode 100644 index 00000000000..8191b7f1909 --- /dev/null +++ b/CONTRIBUTING.rst @@ -0,0 +1,9 @@ +Contribute Code +=================================== + +This project has adopted the `Microsoft Open Source Code of Conduct `__. + +For more information see the `Code of Conduct FAQ `__ or contact `opencode@microsoft.com `__ with any additional questions or comments. + +If you would like to become an active contributor to this project please +follow the instructions provided in `Microsoft Azure Projects Contribution Guidelines `__ diff --git a/pylintrc b/pylintrc new file mode 100644 index 00000000000..054f0f4333b --- /dev/null +++ b/pylintrc @@ -0,0 +1,48 @@ +[MASTER] +reports=no +score=no + +[MESSAGES CONTROL] +# For all codes, run 'pylint --list-msgs' or go to 'https://pylint.readthedocs.io/en/latest/reference_guide/features.html' +# locally-disabled: Warning locally suppressed using disable-msg +# cyclic-import: because of https://github.com/PyCQA/pylint/issues/850 +# too-many-arguments: Due to the nature of the CLI many commands have large arguments set which reflect in large arguments set in corresponding methods. +disable=missing-docstring,locally-disabled,fixme,cyclic-import,too-many-arguments,invalid-name,duplicate-code + +[TYPECHECK] +# For Azure CLI extensions, we ignore import errors for azure.cli as they'll be available in the environment of the CLI +ignored-modules=azure.cli + +[FORMAT] +max-line-length=120 + +[VARIABLES] +# Tells whether we should check for unused import in __init__ files. +init-import=yes + +[DESIGN] +# Maximum number of locals for function / method body +max-locals=25 +# Maximum number of branch for function / method body +max-branches=20 + +[SIMILARITIES] +min-similarity-lines=10 + +[BASIC] +# Naming hints based on PEP 8 (https://www.python.org/dev/peps/pep-0008/#naming-conventions). +# Consider these guidelines and not hard rules. Read PEP 8 for more details. + +# The invalid-name checker must be **enabled** for these hints to be used. +include-naming-hint=yes + +module-name-hint=lowercase (keep short; underscores are discouraged) +const-name-hint=UPPER_CASE_WITH_UNDERSCORES +class-name-hint=CapitalizedWords +class-attribute-name-hint=lower_case_with_underscores +attr-name-hint=lower_case_with_underscores +method-name-hint=lower_case_with_underscores +function-name-hint=lower_case_with_underscores +argument-name-hint=lower_case_with_underscores +variable-name-hint=lower_case_with_underscores +inlinevar-name-hint=lower_case_with_underscores (short is OK) diff --git a/scripts/ci/test_integration.py b/scripts/ci/test_integration.py new file mode 100644 index 00000000000..1f3a2898480 --- /dev/null +++ b/scripts/ci/test_integration.py @@ -0,0 +1,231 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +""" Test the index and the wheels from both the index and from source extensions in repository """ + +from __future__ import print_function + +import os +import json +import tempfile +import unittest +import zipfile +import hashlib +import shutil +import subprocess +from util import get_repo_root +from wheel.install import WHEEL_INFO_RE + +INDEX_PATH = os.path.join(get_repo_root(), 'src', 'index.json') +SRC_PATH = os.path.join(get_repo_root(), 'src') + +# Extensions to skip dep. check. Aim to keep this list empty. +SKIP_DEP_CHECK = ['azure_cli_iot_ext'] + + +def catch_dup_keys(pairs): + seen = {} + for k, v in pairs: + if k in seen: + raise ValueError("duplicate key {}".format(k)) + seen[k] = v + return seen + + +def get_index_data(): + try: + with open(INDEX_PATH) as f: + return json.load(f, object_pairs_hook=catch_dup_keys) + except ValueError as err: + raise AssertionError("Invalid JSON in {}: {}".format(INDEX_PATH, err)) + + +def get_whl_from_url(url, filename, tmp_dir, whl_cache): + if url in whl_cache: + return whl_cache[url] + import requests + r = requests.get(url, stream=True) + assert r.status_code == 200, "Request to {} failed with {}".format(url, r.status_code) + ext_file = os.path.join(tmp_dir, filename) + with open(ext_file, 'wb') as f: + for chunk in r.iter_content(chunk_size=1024): + if chunk: # ignore keep-alive new chunks + f.write(chunk) + whl_cache[url] = ext_file + return ext_file + + +def get_sha256sum(a_file): + sha256 = hashlib.sha256() + with open(a_file, 'rb') as f: + sha256.update(f.read()) + return sha256.hexdigest() + + +def get_extension_modname(ext_dir): + # Modification of https://github.com/Azure/azure-cli/blob/dev/src/azure-cli-core/azure/cli/core/extension.py#L153 + EXTENSIONS_MOD_PREFIX = 'azext_' + pos_mods = [n for n in os.listdir(ext_dir) + if n.startswith(EXTENSIONS_MOD_PREFIX) and os.path.isdir(os.path.join(ext_dir, n))] + if len(pos_mods) != 1: + raise AssertionError("Expected 1 module to load starting with " + "'{}': got {}".format(EXTENSIONS_MOD_PREFIX, pos_mods)) + return pos_mods[0] + + +def get_azext_metadata(ext_dir): + # Modification of https://github.com/Azure/azure-cli/blob/dev/src/azure-cli-core/azure/cli/core/extension.py#L109 + AZEXT_METADATA_FILENAME = 'azext_metadata.json' + azext_metadata = None + ext_modname = get_extension_modname(ext_dir=ext_dir) + azext_metadata_filepath = os.path.join(ext_dir, ext_modname, AZEXT_METADATA_FILENAME) + if os.path.isfile(azext_metadata_filepath): + with open(azext_metadata_filepath) as f: + azext_metadata = json.load(f) + return azext_metadata + + +def get_ext_metadata(ext_dir, ext_file, ext_name): + # Modification of https://github.com/Azure/azure-cli/blob/dev/src/azure-cli-core/azure/cli/core/extension.py#L89 + WHL_METADATA_FILENAME = 'metadata.json' + zip_ref = zipfile.ZipFile(ext_file, 'r') + zip_ref.extractall(ext_dir) + zip_ref.close() + metadata = {} + dist_info_dirs = [f for f in os.listdir(ext_dir) if f.endswith('.dist-info')] + azext_metadata = get_azext_metadata(ext_dir) + if azext_metadata: + metadata.update(azext_metadata) + for dist_info_dirname in dist_info_dirs: + parsed_dist_info_dir = WHEEL_INFO_RE(dist_info_dirname) + if parsed_dist_info_dir and parsed_dist_info_dir.groupdict().get('name') == ext_name: + whl_metadata_filepath = os.path.join(ext_dir, dist_info_dirname, WHL_METADATA_FILENAME) + if os.path.isfile(whl_metadata_filepath): + with open(whl_metadata_filepath) as f: + metadata.update(json.load(f)) + return metadata + + +class TestIndex(unittest.TestCase): + + @classmethod + def setUpClass(cls): + cls.longMessage = True + cls.index = get_index_data() + cls.whl_cache_dir = tempfile.mkdtemp() + cls.whl_cache = {} + + @classmethod + def tearDownClass(cls): + shutil.rmtree(cls.whl_cache_dir) + + def test_format_version(self): + self.assertEqual(self.index['formatVersion'], '1') + + def test_format_extensions_key(self): + self.assertIn('extensions', self.index) + + def test_format_extensions_value(self): + self.assertIsInstance(self.index['extensions'], dict) + + def test_extension_filenames(self): + for ext_name, exts in self.index['extensions'].items(): + for item in exts: + self.assertTrue(item['filename'].endswith('.whl'), + "Filename {} must end with .whl".format(item['filename'])) + self.assertTrue(item['filename'].startswith(ext_name), + "Filename {} must start with {}".format(item['filename'], ext_name)) + parsed_filename = WHEEL_INFO_RE(item['filename']) + p = parsed_filename.groupdict() + self.assertTrue(p.get('name'), "Can't get name for {}".format(item['filename'])) + universal_wheel = p.get('pyver') == 'py2.py3' and p.get('abi') == 'none' and p.get('plat') == 'any' + self.assertTrue(universal_wheel, + "{} of {} not universal (platform independent) wheel. " + "It should end in py2.py3-none-any.whl".format(item['filename'], ext_name)) + + def test_extension_url_filename(self): + for exts in self.index['extensions'].values(): + for item in exts: + self.assertEqual(os.path.basename(item['downloadUrl']), item['filename'], + "Filename must match last segment of downloadUrl") + + def test_filename_duplicates(self): + filenames = [] + for exts in self.index['extensions'].values(): + for item in exts: + filenames.append(item['filename']) + filename_seen = set() + dups = [] + for f in filenames: + if f in filename_seen: + dups.append(f) + filename_seen.add(f) + self.assertFalse(dups, "Duplicate filenames found {}".format(dups)) + + @unittest.skipUnless(os.getenv('CI'), 'Skipped as not running on CI') + def test_checksums(self): + for exts in self.index['extensions'].values(): + for item in exts: + ext_file = get_whl_from_url(item['downloadUrl'], item['filename'], + self.whl_cache_dir, self.whl_cache) + computed_hash = get_sha256sum(ext_file) + self.assertEqual(computed_hash, item['sha256Digest'], + "Computed {} but found {} in index for {}".format(computed_hash, + item['sha256Digest'], + item['filename'])) + + @unittest.skipUnless(os.getenv('CI'), 'Skipped as not running on CI') + def test_metadata(self): + self.maxDiff = None + extensions_dir = tempfile.mkdtemp() + for ext_name, exts in self.index['extensions'].items(): + for item in exts: + ext_dir = tempfile.mkdtemp(dir=extensions_dir) + ext_file = get_whl_from_url(item['downloadUrl'], item['filename'], + self.whl_cache_dir, self.whl_cache) + metadata = get_ext_metadata(ext_dir, ext_file, ext_name) + self.assertDictEqual(metadata, item['metadata'], + "Metadata for {} in index doesn't match the expected of: \n" + "{}".format(item['filename'], json.dumps(metadata, indent=2, sort_keys=True, + separators=(',', ': ')))) + run_requires = metadata.get('run_requires') + if run_requires and ext_name not in SKIP_DEP_CHECK: + deps = run_requires[0]['requires'] + self.assertTrue(all(not dep.startswith('azure-') for dep in deps), + "Dependencies of {} use disallowed extension dependencies. " + "Remove these dependencies: {}".format(item['filename'], deps)) + shutil.rmtree(extensions_dir) + + +class TestSourceWheels(unittest.TestCase): + + def test_source_wheels(self): + # Test we can build all sources into wheels and that metadata from the wheel is valid + from subprocess import PIPE + built_whl_dir = tempfile.mkdtemp() + source_extensions = [os.path.join(SRC_PATH, n) for n in os.listdir(SRC_PATH) + if os.path.isdir(os.path.join(SRC_PATH, n))] + for s in source_extensions: + try: + subprocess.check_call(['python', 'setup.py', 'bdist_wheel', '-q', '-d', built_whl_dir], + cwd=s, stdout=PIPE, stderr=PIPE) + except subprocess.CalledProcessError as err: + self.fail("Unable to build extension {} : {}".format(s, err)) + for filename in os.listdir(built_whl_dir): + ext_file = os.path.join(built_whl_dir, filename) + ext_dir = tempfile.mkdtemp(dir=built_whl_dir) + ext_name = WHEEL_INFO_RE(filename).groupdict().get('name') + metadata = get_ext_metadata(ext_dir, ext_file, ext_name) + run_requires = metadata.get('run_requires') + if run_requires and ext_name not in SKIP_DEP_CHECK: + deps = run_requires[0]['requires'] + self.assertTrue(all(not dep.startswith('azure-') for dep in deps), + "Dependencies of {} use disallowed extension dependencies. " + "Remove these dependencies: {}".format(filename, deps)) + shutil.rmtree(built_whl_dir) + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/ci/test_source.sh b/scripts/ci/test_source.sh new file mode 100755 index 00000000000..da99936ba45 --- /dev/null +++ b/scripts/ci/test_source.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -e + +# Install CLI & CLI testsdk +echo "Installing azure-cli-testsdk and azure-cli..." +pip install "git+https://github.com/Azure/azure-cli@dev#egg=azure-cli-testsdk&subdirectory=src/azure-cli-testsdk" -q +echo "Installed." + +for d in src/*/azext_*/tests; + do echo "Running tests for $d"; + if [ -d $d ]; then + python -m unittest discover -v $d; + else + echo "Skipped $d as not a directory." + fi +done; diff --git a/scripts/ci/test_static.sh b/scripts/ci/test_static.sh new file mode 100755 index 00000000000..8d8b96efc2c --- /dev/null +++ b/scripts/ci/test_static.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -e + +proc_number=`python -c 'import multiprocessing; print(multiprocessing.cpu_count())'` +pylint ./src/*/azext_*/ --rcfile=./pylintrc -j $proc_number +flake8 --statistics --append-config=./.flake8 ./src/*/azext_*/ + +pylint ./scripts/ci/*.py --rcfile=./pylintrc +flake8 --append-config=./.flake8 ./scripts/ci/*.py + +python ./scripts/ci/verify_codeowners.py +python ./scripts/ci/verify_license.py diff --git a/scripts/ci/util.py b/scripts/ci/util.py new file mode 100644 index 00000000000..c873d3bd83f --- /dev/null +++ b/scripts/ci/util.py @@ -0,0 +1,13 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import os + + +def get_repo_root(): + current_dir = os.path.dirname(os.path.abspath(__file__)) + while not os.path.exists(os.path.join(current_dir, 'CONTRIBUTING.rst')): + current_dir = os.path.dirname(current_dir) + return current_dir diff --git a/scripts/ci/verify_codeowners.py b/scripts/ci/verify_codeowners.py new file mode 100644 index 00000000000..da1bf1a1fdb --- /dev/null +++ b/scripts/ci/verify_codeowners.py @@ -0,0 +1,42 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +from __future__ import print_function + +import os +import sys + +from util import get_repo_root + +REPO_ROOT = get_repo_root() +CODEOWNERS = os.path.join(REPO_ROOT, '.github', 'CODEOWNERS') +SRC_DIR = os.path.join(REPO_ROOT, 'src') + + +def get_src_dir_codeowners(): + contents = [] + with open(CODEOWNERS) as f: + contents = [x.strip() for x in f.readlines()] + return dict([x.split(' ') for x in contents if x.startswith('/src/') and x.split(' ')[0].endswith('/')]) + + +def main(): + owners = get_src_dir_codeowners() + dangling_entries = [e for e in owners if not os.path.isdir(os.path.join(REPO_ROOT, e[1:]))] + missing_entries = ['/src/{}/'.format(p) for p in os.listdir(SRC_DIR) + if os.path.isdir(os.path.join(SRC_DIR, p)) and '/src/{}/'.format(p) not in owners] + if dangling_entries or missing_entries: + print('Errors whilst verifying {}!'.format(CODEOWNERS)) + if dangling_entries: + print("Remove the following {} as these directories don't exist.".format(dangling_entries), + file=sys.stderr) + if missing_entries: + print("The following directories are missing codeowners {}.".format(missing_entries), + file=sys.stderr) + sys.exit(1) + + +if __name__ == '__main__': + main() diff --git a/scripts/ci/verify_license.py b/scripts/ci/verify_license.py new file mode 100644 index 00000000000..e7e1d0aca9b --- /dev/null +++ b/scripts/ci/verify_license.py @@ -0,0 +1,44 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +from __future__ import print_function + +import os +import sys + +from util import get_repo_root + +REPO_ROOT = get_repo_root() +SRC_DIR = os.path.join(REPO_ROOT, 'src') + +LICENSE_HEADER = """# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- +""" + + +def main(): + env_path = os.path.join(REPO_ROOT, 'env') + + files_without_header = [] + for current_dir, _, files in os.walk(get_repo_root()): + if current_dir.startswith(env_path): + continue + file_itr = (os.path.join(current_dir, p) for p in files if p.endswith('.py')) + for python_file in file_itr: + with open(python_file, 'r') as f: + file_text = f.read() + if file_text and LICENSE_HEADER not in file_text: + files_without_header.append(os.path.join(current_dir, python_file)) + + if files_without_header: + print("Error: The following files don't have the required license headers: \n{}".format( + '\n'.join(files_without_header)), file=sys.stderr) + sys.exit(1) + + +if __name__ == '__main__': + main() diff --git a/src/image-copy/azext_imagecopy/__init__.py b/src/image-copy/azext_imagecopy/__init__.py index 6b540870c11..79e589c1019 100644 --- a/src/image-copy/azext_imagecopy/__init__.py +++ b/src/image-copy/azext_imagecopy/__init__.py @@ -8,19 +8,29 @@ helps['image copy'] = """ type: command - short-summary: Allows to copy a managed image (or vm) to other regions. Keep in mind that it requires the source disk to be available. + short-summary: Copy a managed image (or vm) to other regions + long-summary: > + Allows to copy a managed image (or vm) to other regions. + Keep in mind that it requires the source disk to be available. """ + def load_params(_): with ParametersContext('image copy') as c: - c.register('source_resource_group_name', '--source-resource-group', help='Name of the resource group of the source resource') - c.register('source_object_name', '--source-object-name', help='The name of the image or vm resource') - c.register('target_location', '--target-location', nargs='+', help='Space separated location list to create the image in (use location short codes like westeurope etc.)') + c.register('source_resource_group_name', '--source-resource-group', + help='Name of the resource group of the source resource') + c.register('source_object_name', '--source-object-name', + help='The name of the image or vm resource') + c.register('target_location', '--target-location', nargs='+', + help='Space separated location list to create the image in (e.g. westeurope etc.)') c.register('source_type', '--source-type', default='image', choices=['image', 'vm'], help='image or vm') - c.register('target_resource_group_name', '--target-resource-group', help='Name of the resource group to create images in') - c.register('parallel_degree', '--parallel-degree', type=int, default=-1, help='Number of parallel copy operations') - c.register('cleanup', '--cleanup', action='store_true', default=False, \ - help='Include this switch to delete temporary resources upon completion') + c.register('target_resource_group_name', '--target-resource-group', + help='Name of the resource group to create images in') + c.register('parallel_degree', '--parallel-degree', type=int, default=-1, + help='Number of parallel copy operations') + c.register('cleanup', '--cleanup', action='store_true', default=False, + help='Include this switch to delete temporary resources upon completion') + def load_commands(): from azure.cli.core.commands import cli_command diff --git a/src/image-copy/azext_imagecopy/cli_utils.py b/src/image-copy/azext_imagecopy/cli_utils.py index 32276cfa29a..c61dc92b7ba 100644 --- a/src/image-copy/azext_imagecopy/cli_utils.py +++ b/src/image-copy/azext_imagecopy/cli_utils.py @@ -1,3 +1,8 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + import sys import json @@ -29,6 +34,10 @@ def run_cli_command(cmd, return_as_json=False): logger.error('command ended with an error: %s', cmd) raise +<<<<<<< HEAD +======= + +>>>>>>> dc94a3bd618f74acdb3167141b26db31e0d06bee def prepare_cli_command(cmd, output_as_json=True): full_cmd = [sys.executable, '-m', 'azure.cli'] + cmd diff --git a/src/image-copy/azext_imagecopy/create_target.py b/src/image-copy/azext_imagecopy/create_target.py index e81e45fda90..40539c2c009 100644 --- a/src/image-copy/azext_imagecopy/create_target.py +++ b/src/image-copy/azext_imagecopy/create_target.py @@ -1,3 +1,8 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + import hashlib import datetime import time @@ -10,33 +15,33 @@ PROGRESS_LINE_LENGTH = 40 -def create_target_image(location, transient_resource_group_name, source_type, source_object_name, \ - source_os_disk_snapshot_name, source_os_disk_snapshot_url, source_os_type, \ - target_resource_group_name, azure_pool_frequency): + +# pylint: disable=too-many-locals +def create_target_image(location, transient_resource_group_name, source_type, source_object_name, + source_os_disk_snapshot_name, source_os_disk_snapshot_url, source_os_type, + target_resource_group_name, azure_pool_frequency): subscription_id = get_subscription_id() subscription_hash = hashlib.sha1(subscription_id.encode("UTF-8")).hexdigest() unique_subscription_string = subscription_hash[:7] - # create the target storage account logger.warn("{0} - Creating target storage account (can be slow sometimes)".format(location)) target_storage_account_name = location + unique_subscription_string - cmd = prepare_cli_command(['storage', 'account', 'create', \ - '--name', target_storage_account_name, \ - '--resource-group', transient_resource_group_name, \ - '--location', location, \ - '--sku', 'Standard_LRS']) + cmd = prepare_cli_command(['storage', 'account', 'create', + '--name', target_storage_account_name, + '--resource-group', transient_resource_group_name, + '--location', location, + '--sku', 'Standard_LRS']) json_output = run_cli_command(cmd, return_as_json=True) target_blob_endpoint = json_output['primaryEndpoints']['blob'] - # Setup the target storage account - cmd = prepare_cli_command(['storage', 'account', 'keys', 'list', \ - '--account-name', target_storage_account_name, \ - '--resource-group', transient_resource_group_name]) + cmd = prepare_cli_command(['storage', 'account', 'keys', 'list', + '--account-name', target_storage_account_name, + '--resource-group', transient_resource_group_name]) json_output = run_cli_command(cmd, return_as_json=True) @@ -46,59 +51,55 @@ def create_target_image(location, transient_resource_group_name, source_type, so expiry_format = "%Y-%m-%dT%H:%MZ" expiry = datetime.datetime.utcnow() + datetime.timedelta(hours=1) - cmd = prepare_cli_command(['storage', 'account', 'generate-sas', \ - '--account-name', target_storage_account_name, \ - '--account-key', target_storage_account_key, \ - '--expiry', expiry.strftime(expiry_format), \ - '--permissions', 'aclrpuw', '--resource-types', \ - 'sco', '--services', 'b', '--https-only'], \ - output_as_json=False) + cmd = prepare_cli_command(['storage', 'account', 'generate-sas', + '--account-name', target_storage_account_name, + '--account-key', target_storage_account_key, + '--expiry', expiry.strftime(expiry_format), + '--permissions', 'aclrpuw', '--resource-types', + 'sco', '--services', 'b', '--https-only'], + output_as_json=False) sas_token = run_cli_command(cmd) - sas_token = sas_token.rstrip("\n\r") #STRANGE + sas_token = sas_token.rstrip("\n\r") # STRANGE logger.debug("sas token: " + sas_token) - # create a container in the target blob storage account logger.warn("{0} - Creating container in the target storage account".format(location)) target_container_name = 'snapshots' - cmd = prepare_cli_command(['storage', 'container', 'create', \ - '--name', target_container_name, \ - '--account-name', target_storage_account_name]) + cmd = prepare_cli_command(['storage', 'container', 'create', + '--name', target_container_name, + '--account-name', target_storage_account_name]) run_cli_command(cmd) - # Copy the snapshot to the target region using the SAS URL blob_name = source_os_disk_snapshot_name + '.vhd' logger.warn("{0} - Copying blob to target storage account".format(location)) - cmd = prepare_cli_command(['storage', 'blob', 'copy', 'start', \ - '--source-uri', source_os_disk_snapshot_url, \ - '--destination-blob', blob_name, \ - '--destination-container', target_container_name, \ - '--account-name', target_storage_account_name, \ - '--sas-token', sas_token]) + cmd = prepare_cli_command(['storage', 'blob', 'copy', 'start', + '--source-uri', source_os_disk_snapshot_url, + '--destination-blob', blob_name, + '--destination-container', target_container_name, + '--account-name', target_storage_account_name, + '--sas-token', sas_token]) run_cli_command(cmd) - # Wait for the copy to complete start_datetime = datetime.datetime.now() - wait_for_blob_copy_operation(blob_name, target_container_name, \ - target_storage_account_name, azure_pool_frequency, location) - msg = "{0} - Copy time: {1}".format(location, datetime.datetime.now()-start_datetime).ljust(PROGRESS_LINE_LENGTH) + wait_for_blob_copy_operation(blob_name, target_container_name, + target_storage_account_name, azure_pool_frequency, location) + msg = "{0} - Copy time: {1}".format(location, datetime.datetime.now() - start_datetime).ljust(PROGRESS_LINE_LENGTH) logger.warn(msg) - # Create the snapshot in the target region from the copied blob logger.warn("{0} - Creating snapshot in target region from the copied blob".format(location)) target_blob_path = target_blob_endpoint + target_container_name + '/' + blob_name target_snapshot_name = source_os_disk_snapshot_name + '-' + location - cmd = prepare_cli_command(['snapshot', 'create', \ - '--resource-group', transient_resource_group_name, \ - '--name', target_snapshot_name, \ - '--location', location, \ - '--source', target_blob_path]) + cmd = prepare_cli_command(['snapshot', 'create', + '--resource-group', transient_resource_group_name, + '--name', target_snapshot_name, + '--location', location, + '--source', target_blob_path]) json_output = run_cli_command(cmd, return_as_json=True) target_snapshot_id = json_output['id'] @@ -110,36 +111,37 @@ def create_target_image(location, transient_resource_group_name, source_type, so target_image_name += '-image' target_image_name += '-' + location - cmd = prepare_cli_command(['image', 'create', \ - '--resource-group', target_resource_group_name, \ - '--name', target_image_name, \ - '--location', location, \ - '--source', target_blob_path, \ - '--os-type', source_os_type, \ - '--source', target_snapshot_id]) + cmd = prepare_cli_command(['image', 'create', + '--resource-group', target_resource_group_name, + '--name', target_image_name, + '--location', location, + '--source', target_blob_path, + '--os-type', source_os_type, + '--source', target_snapshot_id]) run_cli_command(cmd) -def wait_for_blob_copy_operation(blob_name, target_container_name, target_storage_account_name, azure_pool_frequency, location): +def wait_for_blob_copy_operation(blob_name, target_container_name, target_storage_account_name, + azure_pool_frequency, location): progress_controller = APPLICATION.get_progress_controller() copy_status = "pending" prev_progress = -1 while copy_status == "pending": - cmd = prepare_cli_command(['storage', 'blob', 'show', \ - '--name', blob_name, \ - '--container-name', target_container_name, \ - '--account-name', target_storage_account_name]) + cmd = prepare_cli_command(['storage', 'blob', 'show', + '--name', blob_name, + '--container-name', target_container_name, + '--account-name', target_storage_account_name]) json_output = run_cli_command(cmd, return_as_json=True) copy_status = json_output["properties"]["copy"]["status"] copy_progress_1, copy_progress_2 = json_output["properties"]["copy"]["progress"].split("/") - current_progress = round(int(copy_progress_1)/int(copy_progress_2), 1) + current_progress = round(int(copy_progress_1) / int(copy_progress_2), 1) if current_progress != prev_progress: msg = "{0} - copy progress: {1}%"\ .format(location, str(current_progress))\ - .ljust(PROGRESS_LINE_LENGTH) #need to justify since messages overide each other + .ljust(PROGRESS_LINE_LENGTH) # need to justify since messages overide each other progress_controller.add(message=msg) prev_progress = current_progress @@ -150,7 +152,6 @@ def wait_for_blob_copy_operation(blob_name, target_container_name, target_storag progress_controller.stop() return - if copy_status == 'success': progress_controller.stop() else: diff --git a/src/image-copy/azext_imagecopy/custom.py b/src/image-copy/azext_imagecopy/custom.py index e8ea2952c7a..01de885922d 100644 --- a/src/image-copy/azext_imagecopy/custom.py +++ b/src/image-copy/azext_imagecopy/custom.py @@ -1,3 +1,8 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + from multiprocessing import Pool from azext_imagecopy.cli_utils import run_cli_command, prepare_cli_command @@ -7,14 +12,15 @@ logger = azlogging.get_az_logger(__name__) -def imagecopy(source_resource_group_name, source_object_name, target_location, \ - target_resource_group_name, source_type='image', cleanup='false', parallel_degree=-1): +# pylint: disable=too-many-statements +def imagecopy(source_resource_group_name, source_object_name, target_location, + target_resource_group_name, source_type='image', cleanup='false', parallel_degree=-1): # get the os disk id from source vm/image logger.warn("Getting os disk id of the source vm/image") - cmd = prepare_cli_command([source_type, 'show', \ - '--name', source_object_name, \ - '--resource-group', source_resource_group_name]) + cmd = prepare_cli_command([source_type, 'show', + '--name', source_object_name, + '--resource-group', source_resource_group_name]) json_cmd_output = run_cli_command(cmd, return_as_json=True) @@ -22,30 +28,27 @@ def imagecopy(source_resource_group_name, source_object_name, target_location, \ source_os_type = json_cmd_output['storageProfile']['osDisk']['osType'] logger.debug("source_os_disk_id: %s. source_os_type: %s", source_os_disk_id, source_os_type) - # create source snapshots logger.warn("Creating source snapshot") source_os_disk_snapshot_name = source_object_name + '_os_disk_snapshot' - cmd = prepare_cli_command(['snapshot', 'create', \ - '--name', source_os_disk_snapshot_name, \ - '--resource-group', source_resource_group_name, \ - '--source', source_os_disk_id]) + cmd = prepare_cli_command(['snapshot', 'create', + '--name', source_os_disk_snapshot_name, + '--resource-group', source_resource_group_name, + '--source', source_os_disk_id]) run_cli_command(cmd) - # Get SAS URL for the snapshotName logger.warn("Getting sas url for the source snapshot") - cmd = prepare_cli_command(['snapshot', 'grant-access', \ - '--name', source_os_disk_snapshot_name, \ - '--resource-group', source_resource_group_name, \ - '--duration-in-seconds', '3600']) + cmd = prepare_cli_command(['snapshot', 'grant-access', + '--name', source_os_disk_snapshot_name, + '--resource-group', source_resource_group_name, + '--duration-in-seconds', '3600']) json_output = run_cli_command(cmd, return_as_json=True) source_os_disk_snapshot_url = json_output['accessSas'] - logger.debug("source os disk snapshot url: %s" , source_os_disk_snapshot_url) - + logger.debug("source os disk snapshot url: %s", source_os_disk_snapshot_url) # Start processing in the target locations @@ -66,15 +69,15 @@ def imagecopy(source_resource_group_name, source_object_name, target_location, \ azure_pool_frequency = 5 if target_locations_count >= 5: azure_pool_frequency = 15 - elif target_locations_count >= 3: + elif target_locations_count >= 3: azure_pool_frequency = 10 tasks = [] for location in target_location: location = location.strip() - tasks.append((location, transient_resource_group_name, source_type, \ - source_object_name, source_os_disk_snapshot_name, source_os_disk_snapshot_url, \ - source_os_type, target_resource_group_name, azure_pool_frequency)) + tasks.append((location, transient_resource_group_name, source_type, + source_object_name, source_os_disk_snapshot_name, source_os_disk_snapshot_url, + source_os_type, target_resource_group_name, azure_pool_frequency)) logger.warn("Starting async process for all locations") @@ -87,45 +90,45 @@ def imagecopy(source_resource_group_name, source_object_name, target_location, \ except KeyboardInterrupt: logger.warn('User cancelled the operation') if cleanup: - logger.warn('To cleanup temporary resources look for ones tagged with "image-copy-extension". \nYou can use the following command: az resource list --tag created_by=image-copy-extension') + logger.warn('To cleanup temporary resources look for ones tagged with "image-copy-extension". \n' + 'You can use the following command: az resource list --tag created_by=image-copy-extension') pool.terminate() return - # Cleanup if cleanup: logger.warn('Deleting transient resources') # Delete resource group - cmd = prepare_cli_command(['group', 'delete', '--no-wait', '--yes', \ - '--name', transient_resource_group_name]) + cmd = prepare_cli_command(['group', 'delete', '--no-wait', '--yes', + '--name', transient_resource_group_name]) run_cli_command(cmd) # Revoke sas for source snapshot - cmd = prepare_cli_command(['snapshot', 'revoke-access', \ - '--name', source_os_disk_snapshot_name, \ - '--resource-group', source_resource_group_name]) + cmd = prepare_cli_command(['snapshot', 'revoke-access', + '--name', source_os_disk_snapshot_name, + '--resource-group', source_resource_group_name]) run_cli_command(cmd) # Delete source snapshot - cmd = prepare_cli_command(['snapshot', 'delete', \ - '--name', source_os_disk_snapshot_name, \ - '--resource-group', source_resource_group_name]) + cmd = prepare_cli_command(['snapshot', 'delete', + '--name', source_os_disk_snapshot_name, + '--resource-group', source_resource_group_name]) run_cli_command(cmd) def create_resource_group(resource_group_name, location): # check if target resource group exists - cmd = prepare_cli_command(['group', 'exists', \ - '--name', resource_group_name], output_as_json=False) + cmd = prepare_cli_command(['group', 'exists', + '--name', resource_group_name], output_as_json=False) cmd_output = run_cli_command(cmd) if 'false' in cmd_output: # create the target resource group logger.warn("Creating resource group: %s", resource_group_name) - cmd = prepare_cli_command(['group', 'create', \ - '--name', resource_group_name, \ - '--location', location]) + cmd = prepare_cli_command(['group', 'create', + '--name', resource_group_name, + '--location', location]) run_cli_command(cmd) diff --git a/src/index.json b/src/index.json index 6b66ce4fd2f..44432f0ad5a 100644 --- a/src/index.json +++ b/src/index.json @@ -14,7 +14,7 @@ "contacts": [ { "email": "azpycli@microsoft.com", - "name": "Microsoft Corporation.", + "name": "Microsoft Corporation", "role": "author" } ], From c00437144f3d8d2f570bfb104e97173f71e0d8d8 Mon Sep 17 00:00:00 2001 From: Tamir Kamara Date: Sun, 5 Nov 2017 08:36:24 +0200 Subject: [PATCH 07/11] add image-copy to index.json --- src/index.json | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/src/index.json b/src/index.json index 44432f0ad5a..2660fedc24b 100644 --- a/src/index.json +++ b/src/index.json @@ -93,6 +93,52 @@ "version": "0.1.1" } } + ], + "image-copy-extension": [ + { + "filename": "imagecopyextension-0.0.3-py2.py3-none-any.whl", + "sha256Digest": "d6c889c5b2eb154e727f9684df3230461e2402dc6433061bc489bed890687fc7", + "downloadUrl": "https://files.pythonhosted.org/packages/e7/54/a010d59dcb86abe4b3f0e0a958c403490df97c8cd2da431025683ebc8425/imagecopyextension-0.0.3-py2.py3-none-any.whl", + "metadata": { + "azext.minCliVersion": "2.0.12", + "classifiers": [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "Intended Audience :: System Administrators", + "Programming Language :: Python", + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.4", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "License :: OSI Approved :: MIT License" + ], + "extensions": { + "python.details": { + "contacts": [ + { + "email": "tamir.kamara@microsoft.com", + "name": "Tamir Kamara", + "role": "author" + } + ], + "document_names": { + "description": "DESCRIPTION.rst" + }, + "project_urls": { + "Home": "https://github.com/Azure/azure-cli-extensions" + } + } + }, + "generator": "bdist_wheel (0.30.0)", + "license": "MIT", + "metadata_version": "2.0", + "name": "imagecopyextension", + "summary": "An Azure CLI Extension that copies images from region to region.", + "version": "0.0.3" + } + } ] } } \ No newline at end of file From 10ce43486af8428e6d985f8d8343420abd86b807 Mon Sep 17 00:00:00 2001 From: Tamir Kamara Date: Sun, 5 Nov 2017 16:14:20 +0200 Subject: [PATCH 08/11] Add examples to help --- src/image-copy/azext_imagecopy/__init__.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/image-copy/azext_imagecopy/__init__.py b/src/image-copy/azext_imagecopy/__init__.py index 79e589c1019..ebc4791b02a 100644 --- a/src/image-copy/azext_imagecopy/__init__.py +++ b/src/image-copy/azext_imagecopy/__init__.py @@ -12,6 +12,15 @@ long-summary: > Allows to copy a managed image (or vm) to other regions. Keep in mind that it requires the source disk to be available. + examples: + - name: Copy an image to several regions and cleanup at the end. + text: > + az image copy --source-resource-group mySources-rg --source-object-name myImage \\ + --target-location uksouth northeurope --target-resource-group "images-repo-rg" --cleanup + - name: Use an already generalized vm to create images in other regions. + text: > + az image copy --source-resource-group mySources-rg --source-object-name myVm \\ + --source-type vm --target-location uksouth northeurope --target-resource-group "images-repo-rg" """ From 3eb6784659388df4340d67ce8a18eef9f73e5727 Mon Sep 17 00:00:00 2001 From: Tamir Kamara Date: Sun, 5 Nov 2017 16:14:51 +0200 Subject: [PATCH 09/11] fix package names to allow index ci to pass --- src/image-copy/setup.py | 4 ++-- src/index.json | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/image-copy/setup.py b/src/image-copy/setup.py index 131d096e944..1412b91fe0b 100644 --- a/src/image-copy/setup.py +++ b/src/image-copy/setup.py @@ -8,7 +8,7 @@ from codecs import open from setuptools import setup, find_packages -VERSION = "0.0.3" +VERSION = "0.0.4" CLASSIFIERS = [ 'Development Status :: 4 - Beta', @@ -27,7 +27,7 @@ DEPENDENCIES = [] setup( - name='imagecopyextension', + name='image-copy-extension', version=VERSION, description='An Azure CLI Extension that copies images from region to region.', long_description='An Azure CLI Extension that copies images from region to region.', diff --git a/src/index.json b/src/index.json index 2660fedc24b..451529df9e7 100644 --- a/src/index.json +++ b/src/index.json @@ -94,11 +94,11 @@ } } ], - "image-copy-extension": [ + "image_copy_extension": [ { - "filename": "imagecopyextension-0.0.3-py2.py3-none-any.whl", - "sha256Digest": "d6c889c5b2eb154e727f9684df3230461e2402dc6433061bc489bed890687fc7", - "downloadUrl": "https://files.pythonhosted.org/packages/e7/54/a010d59dcb86abe4b3f0e0a958c403490df97c8cd2da431025683ebc8425/imagecopyextension-0.0.3-py2.py3-none-any.whl", + "filename": "image_copy_extension-0.0.4-py2.py3-none-any.whl", + "sha256Digest": "47d0f4293e833bb16bac1abbc63f7946dd87ac8db40378a230ec376d1f128fd3", + "downloadUrl": "https://files.pythonhosted.org/packages/c3/60/723ad5e968d042d9d4564af420626b661e8cde0820b6dc90f8933e6059bf/image_copy_extension-0.0.4-py2.py3-none-any.whl", "metadata": { "azext.minCliVersion": "2.0.12", "classifiers": [ @@ -134,9 +134,9 @@ "generator": "bdist_wheel (0.30.0)", "license": "MIT", "metadata_version": "2.0", - "name": "imagecopyextension", + "name": "image-copy-extension", "summary": "An Azure CLI Extension that copies images from region to region.", - "version": "0.0.3" + "version": "0.0.4" } } ] From 025a81bf59810a2c0827bd626a09dccc86158aff Mon Sep 17 00:00:00 2001 From: Tamir Kamara Date: Sun, 5 Nov 2017 16:24:46 +0200 Subject: [PATCH 10/11] fix whitespace --- src/image-copy/azext_imagecopy/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/image-copy/azext_imagecopy/__init__.py b/src/image-copy/azext_imagecopy/__init__.py index ebc4791b02a..9a145a44ef3 100644 --- a/src/image-copy/azext_imagecopy/__init__.py +++ b/src/image-copy/azext_imagecopy/__init__.py @@ -12,7 +12,7 @@ long-summary: > Allows to copy a managed image (or vm) to other regions. Keep in mind that it requires the source disk to be available. - examples: + examples: - name: Copy an image to several regions and cleanup at the end. text: > az image copy --source-resource-group mySources-rg --source-object-name myImage \\ From 4131e2bd5593b802d1690731640b0cd6457f7cd0 Mon Sep 17 00:00:00 2001 From: Tamir Kamara Date: Mon, 6 Nov 2017 10:08:55 +0200 Subject: [PATCH 11/11] image-copy: remove mincli from index.json --- src/index.json | 1 - 1 file changed, 1 deletion(-) diff --git a/src/index.json b/src/index.json index 451529df9e7..aace38bf951 100644 --- a/src/index.json +++ b/src/index.json @@ -100,7 +100,6 @@ "sha256Digest": "47d0f4293e833bb16bac1abbc63f7946dd87ac8db40378a230ec376d1f128fd3", "downloadUrl": "https://files.pythonhosted.org/packages/c3/60/723ad5e968d042d9d4564af420626b661e8cde0820b6dc90f8933e6059bf/image_copy_extension-0.0.4-py2.py3-none-any.whl", "metadata": { - "azext.minCliVersion": "2.0.12", "classifiers": [ "Development Status :: 4 - Beta", "Intended Audience :: Developers",