diff --git a/appscale/tools/agents/azure_agent.py b/appscale/tools/agents/azure_agent.py index 7bb119a6..3daafc40 100644 --- a/appscale/tools/agents/azure_agent.py +++ b/appscale/tools/agents/azure_agent.py @@ -6,10 +6,11 @@ # General-purpose Python library imports import adal +import concurrent.futures +import logging import math import os.path import re -import threading import time from itertools import count, ifilter @@ -64,6 +65,7 @@ from azure.mgmt.storage.models import Sku as StorageSku from msrestazure.azure_active_directory import ServicePrincipalCredentials +from msrest.exceptions import ClientException from msrestazure.azure_exceptions import CloudError from haikunator import Haikunator @@ -182,23 +184,18 @@ def assert_credentials_are_valid(self, parameters): Args: parameters: A dict, containing all the parameters necessary to authenticate this user with Azure. - Returns: - True, if the credentials were valid. - A list, of resource group names under the subscription. Raises: - AgentConfigurationException: If an error is encountered during - authentication. + AgentConfigurationException: if we are unable to authenticate with Azure. """ credentials = self.open_connection(parameters) subscription_id = str(parameters[self.PARAM_SUBSCRIBER_ID]) try: resource_client = ResourceManagementClient(credentials, subscription_id) - resource_groups = resource_client.resource_groups.list() - rg_names = [] - for rg in resource_groups: - rg_names.append(rg.name) - return True, rg_names - except CloudError as error: + # Try listing resource groups to make sure we can, a CloudError will be + # raised if the credentials are invalid. + resource_client.resource_groups.list() + except ClientException as error: + logging.exception("Error authenticating with provided credentials.") raise AgentConfigurationException("Unable to authenticate using the " "credentials provided. Reason: {}".format(error.message)) @@ -215,6 +212,7 @@ def configure_instance_security(self, parameters): Raises: AgentRuntimeException: If security features could not be successfully configured in the underlying cloud. + AgentConfigurationException: If we are unable to authenticate with Azure. """ is_autoscale = parameters['autoscale_agent'] @@ -247,8 +245,19 @@ def configure_instance_security(self, parameters): self.create_resource_group(parameters, credentials) resource_client = ResourceManagementClient(credentials, subscription_id) - resource_client.providers.register(self.MICROSOFT_COMPUTE_RESOURCE) - resource_client.providers.register(self.MICROSOFT_NETWORK_RESOURCE) + try: + resource_client.providers.register(self.MICROSOFT_COMPUTE_RESOURCE) + resource_client.providers.register(self.MICROSOFT_NETWORK_RESOURCE) + except CloudError as error: + logging.exception("Encountered an error while registering provider.") + raise AgentRuntimeException("Unable to register provider. Reason: {}" + .format(error.message)) + except ClientException as e: + logging.exception("ClientException received while attempting to contact " + "Azure.") + raise AgentConfigurationException("Unable to communicate with Azure " + "while trying to register provider. Please check your cloud " + "configuration. Reason: {}".format(e.message)) def attach_disk(self, parameters, disk_name, instance_id): """ Attaches the persistent disk specified in 'disk_name' to this virtual @@ -362,6 +371,10 @@ def describe_instances(self, parameters, pending=False): public_ips: A list of public IP addresses. private_ips: A list of private IP addresses. instance_ids: A list of unique Azure VM names. + Raises: + AgentRuntimeException: If we are unable to list instances in the + cloud. + AgentConfigurationException: If we are unable to authenticate with Azure. """ credentials = self.open_connection(parameters) subscription_id = str(parameters[self.PARAM_SUBSCRIBER_ID]) @@ -370,45 +383,45 @@ def describe_instances(self, parameters, pending=False): network_client = NetworkManagementClient(credentials, subscription_id, api_version=self.NETWORK_MGMT_API_VERSION) compute_client = ComputeManagementClient(credentials, subscription_id) - public_ips = [] - private_ips = [] - instance_ids = [] - - public_ip_addresses = network_client.public_ip_addresses.list(resource_group) - for public_ip in public_ip_addresses: - public_ips.append(public_ip.ip_address) - - network_interfaces = network_client.network_interfaces.list(resource_group) - for network_interface in network_interfaces: - for ip_config in network_interface.ip_configurations: - private_ips.append(ip_config.private_ip_address) - - virtual_machines = compute_client.virtual_machines.list(resource_group) - for vm in virtual_machines: - instance_ids.append(vm.name) - - vmss_list = compute_client.virtual_machine_scale_sets.list(resource_group) - for vmss in vmss_list: - vm_list = compute_client.virtual_machine_scale_set_vms.list(resource_group, - vmss.name) - for vm in vm_list: + + try: + public_ips = [public_ip.ip_address for public_ip in + network_client.public_ip_addresses.list(resource_group)] + + private_ips = [ip_config.private_ip_address + for network_interface in + network_client.network_interfaces.list(resource_group) + for ip_config in network_interface.ip_configurations] + + instance_ids = [vm.name for vm in + compute_client.virtual_machines.list(resource_group)] + vm_vmss_list = [(vm, vmss.name) + for vmss in compute_client.virtual_machine_scale_sets.list( + resource_group) + for vm in compute_client.virtual_machine_scale_set_vms.list( + resource_group, vmss.name)] + + for vm, vmss_name in vm_vmss_list: network_interface_list = network_client.network_interfaces. \ - list_virtual_machine_scale_set_vm_network_interfaces(resource_group, - vmss.name, - vm.instance_id) - ip_config_private_ip = None - for network_interface in network_interface_list: - for ip_config in network_interface.ip_configurations: - ip_config_private_ip = ip_config.private_ip_address - break - - if ip_config_private_ip: - break - - if ip_config_private_ip: - public_ips.append(ip_config_private_ip) - private_ips.append(ip_config_private_ip) - instance_ids.append(vm.name) + list_virtual_machine_scale_set_vm_network_interfaces( + resource_group, vmss_name, vm.instance_id) + private_ip = next(ip_config.private_ip_address + for network_interface in network_interface_list + for ip_config in network_interface.ip_configurations + if ip_config.private_ip_address) + public_ips.append(private_ip) + private_ips.append(private_ip) + instance_ids.append(vm.name) + except CloudError as e: + logging.exception("CloudError received while trying to describe " + "instances.") + raise AgentRuntimeException(e.message) + except ClientException as e: + logging.exception("ClientException received while attempting to contact " + "Azure.") + raise AgentConfigurationException("Unable to communicate with Azure " + "while trying to describe instances. Please check your cloud " + "configuration. Reason: {}".format(e.message)) return public_ips, private_ips, instance_ids @@ -427,6 +440,8 @@ def run_instances(self, count, parameters, security_configured, public_ip_needed instance_ids: A list of unique Azure VM names. public_ips: A list of public IP addresses. private_ips: A list of private IP addresses. + Raises: + AgentRuntimeException: If an error has occurred talking to Azure. """ credentials = self.open_connection(parameters) subscription_id = str(parameters[self.PARAM_SUBSCRIBER_ID]) @@ -444,18 +459,23 @@ def run_instances(self, count, parameters, security_configured, public_ip_needed if using_disks and not self.MARKETPLACE_IMAGE.match(azure_image_id): raise AgentConfigurationException("Managed Disks require use of a " "publisher image.") - if public_ip_needed or using_disks: - lb_vms_threads = [] - for _ in range(count): - thread = threading.Thread(target=self.setup_virtual_machine_creation, - args=(credentials, network_client, - parameters, subnet)) - thread.start() - lb_vms_threads.append(thread) - - for vm_thread in lb_vms_threads: - vm_thread.join() + lb_vms_exceptions = [] + # We can use a with statement to ensure threads are cleaned up promptly + with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: + lb_vms_futures = [executor.submit(self.setup_virtual_machine_creation, + credentials, network_client, parameters, subnet) + for _ in range(count)] + for future in concurrent.futures.as_completed(lb_vms_futures): + exception = future.exception() + if exception: + lb_vms_exceptions.append(exception) + + for exception in lb_vms_exceptions: + if not isinstance(exception, (CloudError, AgentRuntimeException)): + logging.exception(exception) + if lb_vms_exceptions: + raise AgentRuntimeException(str(lb_vms_exceptions)) else: self.create_or_update_vm_scale_sets(count, parameters, subnet) @@ -499,6 +519,10 @@ def create_virtual_machine(self, credentials, network_client, network_id, parameters: A dict, containing all the parameters necessary to authenticate this user with Azure. vm_network_name: The name of the virtual machine to use. + Raises: + AgentRuntimeException: If a virtual machine could not be successfully + created. + AgentConfigurationException: If we are unable to authenticate with Azure. """ resource_group = parameters[self.PARAM_RESOURCE_GROUP] zone = parameters[self.PARAM_ZONE] @@ -520,7 +544,6 @@ def create_virtual_machine(self, credentials, network_client, network_id, network_profile = NetworkProfile( network_interfaces=[NetworkInterfaceReference(id=network_id)]) - os_type = OperatingSystemTypes.linux azure_image_id = parameters[self.PARAM_IMAGE_ID] @@ -553,25 +576,32 @@ def create_virtual_machine(self, credentials, network_client, network_id, name=vm_network_name, vhd=virtual_hd, image=image_hd) storage_profile = StorageProfile(image_reference=image_ref, os_disk=os_disk) - compute_client.virtual_machines.create_or_update( - resource_group, vm_network_name, VirtualMachine(location=zone, - plan=plan, - os_profile=os_profile, - hardware_profile=hardware_profile, - network_profile=network_profile, - storage_profile=storage_profile)) - - # Sleep until an IP address gets associated with the VM. - while True: - public_ip_address = network_client.public_ip_addresses.get(resource_group, - vm_network_name) - if public_ip_address.ip_address: - AppScaleLogger.log('Azure load balancer VM is available at {}'. - format(public_ip_address.ip_address)) - break - AppScaleLogger.verbose("Waiting {} second(s) for IP address to be " - "available".format(self.SLEEP_TIME), verbose) - time.sleep(self.SLEEP_TIME) + try: + compute_client.virtual_machines.create_or_update( + resource_group, vm_network_name, VirtualMachine(location=zone, + plan=plan, os_profile=os_profile, hardware_profile=hardware_profile, + network_profile=network_profile, storage_profile=storage_profile)) + # Sleep until an IP address gets associated with the VM. + while True: + public_ip_address = network_client.public_ip_addresses.get( + resource_group, vm_network_name) + if public_ip_address.ip_address: + AppScaleLogger.log('Azure load balancer VM is available at {}'. + format(public_ip_address.ip_address)) + break + AppScaleLogger.verbose("Waiting {} second(s) for IP address to be " + "available".format(self.SLEEP_TIME), verbose) + time.sleep(self.SLEEP_TIME) + except CloudError as error: + logging.exception("Azure agent received a CloudError.") + raise AgentRuntimeException("Unable to create virtual machine. " + "Reason: {}".format(error.message)) + except ClientException as e: + logging.exception("ClientException received while attempting to contact " + "Azure.") + raise AgentConfigurationException("Unable to communicate with Azure " + "while trying to create virtual machine. Please check your cloud " + "configuration. Reason: {}".format(e.message)) def create_linux_configuration(self, parameters): """ Creates a Linux Configuration to pass in to the virtual machine @@ -610,6 +640,11 @@ def add_instances_to_existing_ss(self, count, parameters): authenticate this user with Azure. Returns: The number of instances created and added to the existing scale sets. + Raises: + Raises: + AgentRuntimeException: If instances could not successfully be added to + a Scale Set. + AgentConfigurationException: If we are unable to authenticate with Azure. """ credentials = self.open_connection(parameters) subscription_id = str(parameters[self.PARAM_SUBSCRIBER_ID]) @@ -618,26 +653,38 @@ def add_instances_to_existing_ss(self, count, parameters): compute_client = ComputeManagementClient(credentials, subscription_id) num_instances_added = 0 - vmss_list = compute_client.virtual_machine_scale_sets.list(resource_group) - for vmss in vmss_list: - vm_list = compute_client.virtual_machine_scale_set_vms.list( - resource_group, vmss.name) - ss_instance_count = 0 - for _ in vm_list: - ss_instance_count += 1 - - if ss_instance_count >= self.MAX_VMSS_CAPACITY: - continue - - if not vmss.sku.name == instance_type: - continue - - scaleset = compute_client.virtual_machine_scale_sets.get( - resource_group, vmss.name) - ss_upgrade_policy = scaleset.upgrade_policy - ss_location = scaleset.location - ss_profile = scaleset.virtual_machine_profile - ss_overprovision = scaleset.overprovision + try: + scalesets_and_counts = [] + for vmss in compute_client.virtual_machine_scale_sets.list( + resource_group): + ss_instance_count = len(list( + compute_client.virtual_machine_scale_set_vms.list(resource_group, vmss.name))) + + if ss_instance_count >= self.MAX_VMSS_CAPACITY: + continue + + if not vmss.sku.name == instance_type: + continue + vmss = compute_client.virtual_machine_scale_sets.get(resource_group, + vmss.name) + scalesets_and_counts.append((vmss, ss_instance_count)) + except CloudError as error: + logging.exception("Azure agent received a CloudError trying to add " + "to Scale Sets.") + raise AgentRuntimeException("Unable to add to Scale Sets due to: {}" + .format(error.message)) + except ClientException as e: + logging.exception("ClientException received while attempting to contact " + "Azure.") + raise AgentConfigurationException("Unable to communicate with Azure " + "while trying to add to Scale Sets. Please check your cloud " + "configuration. Reason: {}".format(e.message)) + + for vmss, ss_instance_count in scalesets_and_counts: + ss_upgrade_policy = vmss.upgrade_policy + ss_location = vmss.location + ss_profile = vmss.virtual_machine_profile + ss_overprovision = vmss.overprovision new_capacity = min(ss_instance_count + count, self.MAX_VMSS_CAPACITY) sku = ComputeSku(name=parameters[self.PARAM_INSTANCE_TYPE], @@ -647,9 +694,21 @@ def add_instances_to_existing_ss(self, count, parameters): location=ss_location, virtual_machine_profile=ss_profile, overprovision=ss_overprovision) - create_update_response = compute_client.virtual_machine_scale_sets.\ - create_or_update(resource_group, vmss.name, scaleset) - self.wait_for_ss_update(new_capacity, create_update_response, vmss.name) + try: + create_update_response = compute_client.virtual_machine_scale_sets. \ + create_or_update(resource_group, vmss.name, scaleset) + + self.wait_for_ss_update(new_capacity, create_update_response, vmss.name) + except CloudError as error: + logging.exception("Azure agent received a CloudError.") + raise AgentRuntimeException("Unable to create/update ScaleSet. " + "Reason: {}".format(error.message)) + except ClientException as e: + logging.exception("ClientException received while attempting to contact" + " Azure.") + raise AgentConfigurationException("Unable to communicate with Azure " + "while trying to create/update ScaleSet. Please check your cloud " + "configuration. Reason: {}".format(e.message)) newly_added = new_capacity - ss_instance_count num_instances_added += newly_added @@ -700,25 +759,34 @@ def create_or_update_vm_scale_sets(self, count, parameters, subnet): self.MAX_VMSS_CAPACITY))) remaining_vms_count = num_instances_to_add - scalesets_threads = [] - for ss_count in range(scale_set_count): - resource_name = random_resource_name + "-resource-{}".format(ss_count) - scale_set_name = random_resource_name + "-scaleset-{}".format(ss_count) - capacity = self.MAX_VMSS_CAPACITY - if remaining_vms_count < self.MAX_VMSS_CAPACITY: - capacity = remaining_vms_count - AppScaleLogger.verbose('Creating a Scale Set {0} with {1} VM(s)'. - format(scale_set_name, capacity), verbose) - - thread = threading.Thread(target=self.create_scale_set, - args=(capacity, parameters, resource_name, - scale_set_name, subnet)) - thread.start() - scalesets_threads.append(thread) + scalesets_futures = [] + scalesets_exceptions = [] + # We can use a with statement to ensure threads are cleaned up promptly + with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: + for ss_count in range(scale_set_count): + resource_name = random_resource_name + "-resource-{}".format(ss_count) + scale_set_name = random_resource_name + "-scaleset-{}".format( + ss_count) + capacity = self.MAX_VMSS_CAPACITY + if remaining_vms_count < self.MAX_VMSS_CAPACITY: + capacity = remaining_vms_count + AppScaleLogger.verbose('Creating a Scale Set {0} with {1} VM(s)'. + format(scale_set_name, capacity), verbose) + + # Start creating scalesets. + scalesets_futures.append(executor.submit(self.create_scale_set, + capacity, parameters, resource_name, scale_set_name, subnet)) remaining_vms_count = remaining_vms_count - self.MAX_VMSS_CAPACITY + for future in concurrent.futures.as_completed(scalesets_futures): + exception = future.exception() + if exception: + scalesets_exceptions.append(exception) - for ss_thread in scalesets_threads: - ss_thread.join() + for exception in scalesets_exceptions: + if not isinstance(exception, (CloudError, AgentRuntimeException)): + logging.exception(exception) + if scalesets_exceptions: + raise AgentRuntimeException(str(scalesets_exceptions)) # Create a scale set using the count of VMs provided. else: @@ -742,8 +810,8 @@ def create_scale_set(self, count, parameters, resource_name, subnet: A reference to the subnet ID of the virtual network created. Raises: - AgentConfigurationException: If the operation to create a virtual - machine scale set did not succeed. + AgentRuntimeException: If a scale set could not be successfully created. + AgentConfigurationException: If we are unable to authenticate with Azure. """ credentials = self.open_connection(parameters) subscription_id = str(parameters[self.PARAM_SUBSCRIBER_ID]) @@ -804,9 +872,22 @@ def create_scale_set(self, count, parameters, resource_name, sku=sku, upgrade_policy=upgrade_policy, location=zone, plan=plan, virtual_machine_profile=virtual_machine_profile, overprovision=False) - create_update_response = compute_client.virtual_machine_scale_sets.create_or_update( - resource_group, scale_set_name, vm_scale_set) - self.wait_for_ss_update(count, create_update_response, scale_set_name) + try: + create_update_response = \ + compute_client.virtual_machine_scale_sets.create_or_update( + resource_group, scale_set_name, vm_scale_set) + + self.wait_for_ss_update(count, create_update_response, scale_set_name) + except CloudError as error: + logging.exception("Azure agent received a CloudError.") + raise AgentRuntimeException("Unable to create or update Scale Set. " + "Reason: {}".format(error.message)) + except ClientException as e: + logging.exception("ClientException received while attempting to contact " + "Azure.") + raise AgentConfigurationException("Unable to communicate with Azure " + "while trying to create a scale set. Please check your cloud " + "configuration. Reason: {}".format(e.message)) def wait_for_ss_update(self, count, create_update_response, scale_set_name): """ Waits until the scale set has been successfully updated and all the @@ -819,8 +900,9 @@ def wait_for_ss_update(self, count, create_update_response, scale_set_name): scale_set_name: The name of the scale set being updated. Raises: - AgentConfigurationException: If it encounters a problem updating - the virtual machine scale set. + AgentRuntimeException: If there is a problem updating the virtual + machine scale set. + AgentConfigurationException: If we are unable to authenticate with Azure. """ try: create_update_response.wait(timeout=self.MAX_VMSS_WAIT_TIME) @@ -834,8 +916,15 @@ def wait_for_ss_update(self, count, create_update_response, scale_set_name): .format(count, result.provisioning_state)) except CloudError as error: - raise AgentConfigurationException("Unable to create a Scale Set of {0} " - "VM(s): {1}".format(count, error.message)) + logging.exception("CloudError during creation of Scale Set.") + raise AgentRuntimeException("Unable to create a Scale Set of {0} " + "VM(s): {1}".format(count, error.message)) + except ClientException as e: + logging.exception("ClientException received while attempting to " + "contact Azure.") + raise AgentConfigurationException("Unable to communicate with Azure " + "while trying to create Scale Set. Please check your cloud " + "configuration. Reason: {}".format(e.message)) def associate_static_ip(self, instance_id, static_ip): """ Associates the given static IP address with the given instance ID. @@ -851,6 +940,9 @@ def terminate_instances(self, parameters): Args: parameters: A dict, containing all the parameters necessary to authenticate this user with Azure. + Raises: + AgentRuntimeException: If instances could not successfully be terminated. + AgentConfigurationException: If we are unable to authenticate with Azure. """ credentials = self.open_connection(parameters) resource_group = parameters[self.PARAM_RESOURCE_GROUP] @@ -861,78 +953,161 @@ def terminate_instances(self, parameters): format(instances_to_delete), verbose) compute_client = ComputeManagementClient(credentials, subscription_id) - vmss_list = compute_client.virtual_machine_scale_sets.list(resource_group) + try: + vmss_list = list(compute_client.virtual_machine_scale_sets.list( + resource_group)) + except CloudError as e: + logging.exception("CloudError received trying to list Scale Sets.") + raise AgentRuntimeException(e.message) + except ClientException as e: + logging.exception("ClientException received while attempting to contact " + "Azure.") + raise AgentConfigurationException("Unable to communicate with Azure " + "while trying to list Scale Sets. Please check your cloud " + "configuration. Reason: {}".format(e.message)) + downscale = parameters['autoscale_agent'] # On downscaling of instances, we need to delete the specific instance # from the Scale Set. if downscale in ['True', True]: # Delete the scale set virtual machines matching the given instance ids. - vmss_vm_delete_threads = [] - for vmss in vmss_list: - vm_list = compute_client.virtual_machine_scale_set_vms.list( - resource_group, vmss.name) - for vm in vm_list: - if vm.name in instances_to_delete: - instances_to_delete.remove(vm.name) - thread = threading.Thread(target=self.delete_vmss_instance, - args=(compute_client, parameters, - vmss.name, vm.instance_id)) - thread.start() - vmss_vm_delete_threads.append(thread) - - for delete_thread in vmss_vm_delete_threads: - delete_thread.join() + vmss_vm_delete_futures = [] + vmss_vm_delete_exceptions = [] + + try: + # Get the instance_ids for the Scale Set VMs. + vmss_vms_to_delete = [(vm.instance_id, vmss.name) for vmss in vmss_list + for vm in compute_client.virtual_machine_scale_set_vms.list( + resource_group, vmss.name) + if vm.name in instances_to_delete] + except CloudError as e: + logging.exception("CloudError received while trying to terminate " + "instances.") + raise AgentRuntimeException(e.message) + except ClientException as e: + logging.exception("ClientException received while attempting to " + "contact Azure.") + raise AgentConfigurationException("Unable to communicate with Azure " + "while trying to terminate instances. Please check your cloud " + "configuration. Reason: {}".format(e.message)) + + with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: + for vm_instance_id, vmss_name in vmss_vms_to_delete: + vmss_vm_delete_futures.append(executor.submit( + self.delete_vmss_instance, compute_client, parameters, + vmss_name, vm_instance_id)) + + for future in concurrent.futures.as_completed(vmss_vm_delete_futures): + exception = future.exception() + if exception: + vmss_vm_delete_exceptions.append(exception) + + for exception in vmss_vm_delete_exceptions: + if not isinstance(exception, (CloudError, AgentRuntimeException)): + logging.exception(exception) + if vmss_vm_delete_exceptions: + raise AgentRuntimeException(str(vmss_vm_delete_exceptions)) AppScaleLogger.log("Virtual machine(s) have been successfully downscaled.") AppScaleLogger.log("Cleaning up any Scale Sets, if needed ...") - vmss_delete_threads = [] - for vmss in vmss_list: - vm_list = compute_client.virtual_machine_scale_set_vms.list( - resource_group, vmss.name) - if not any(True for _ in vm_list): - thread = threading.Thread( - target=self.delete_virtual_machine_scale_set, args=( - compute_client, parameters, vmss.name)) - thread.start() - vmss_delete_threads.append(thread) - - for delete_thread in vmss_delete_threads: - delete_thread.join() + vmss_delete_futures = [] + vmss_delete_exceptions = [] + + try: + ss_to_delete = [vmss.name for vmss in vmss_list if not + any(compute_client.virtual_machine_scale_set_vms.list(resource_group, vmss.name))] + except CloudError as e: + logging.exception("CloudError received while trying to terminate " + "instances.") + raise AgentRuntimeException(e.message) + except ClientException as e: + logging.exception("ClientException received while attempting to " + "contact Azure.") + raise AgentConfigurationException("Unable to communicate with Azure " + "while trying to terminate instances. Please check your cloud " + "configuration. Reason: {}".format(e.message)) + + with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: + for vmss_name in ss_to_delete: + vmss_delete_futures.append(executor.submit( + self.delete_virtual_machine_scale_set, compute_client, + parameters, vmss_name)) + + for future in concurrent.futures.as_completed(vmss_delete_futures): + exception = future.exception() + if exception: + vmss_delete_exceptions.append(exception) + + for exception in vmss_delete_exceptions: + if not isinstance(exception, (CloudError, AgentRuntimeException)): + logging.exception(exception) + if vmss_delete_exceptions: + raise AgentRuntimeException(str(vmss_delete_exceptions)) return # On appscale down --terminate, we delete all the Scale Sets within the # resource group specified, as it is faster than deleting the individual # instances within each Scale Set. - delete_ss_instances = [] - vmss_delete_threads = [] - for vmss in vmss_list: - vm_list = compute_client.virtual_machine_scale_set_vms.list( - resource_group, vmss.name) - for vm in vm_list: - delete_ss_instances.append(vm.name) - thread = threading.Thread(target=self.delete_virtual_machine_scale_set, - args=(compute_client, parameters, vmss.name)) - thread.start() - vmss_delete_threads.append(thread) - # Delete the load balancer virtual machines matching the given instance ids. - delete_lb_instances = self.diff(instances_to_delete, delete_ss_instances) - lb_delete_threads = [] - for vm_name in delete_lb_instances: - thread = threading.Thread(target=self.delete_virtual_machine, - args=(compute_client, parameters, vm_name)) - thread.start() - lb_delete_threads.append(thread) - - for delete_thread in vmss_delete_threads: - delete_thread.join() + # Get the list of all ScaleSet Instance "Names" before deleting + # ScaleSets. We refer to the Scale Set VM instances by name rather than + # instance id. + try: + delete_ss_instances = [vm.name for vmss in vmss_list + for vm in compute_client.virtual_machine_scale_set_vms.list( + resource_group, vmss.name)] + except CloudError as e: + logging.exception("CloudError received while trying to terminate " + "instances.") + raise AgentRuntimeException(e.message) + except ClientException as e: + logging.exception("ClientException received while attempting to contact " + "Azure.") + raise AgentConfigurationException("Unable to communicate with Azure " + "while trying to terminate instances. Please check your cloud " + "configuration. Reason: {}".format(e.message)) + + # Delete ScaleSets. + vmss_delete_futures = [] + vmss_delete_exceptions = [] + with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: + for vmss in vmss_list: + vmss_delete_futures.append(executor.submit( + self.delete_virtual_machine_scale_set, compute_client, + parameters, vmss.name)) + for future in concurrent.futures.as_completed(vmss_delete_futures): + exception = future.exception() + if exception: + vmss_delete_exceptions.append(exception) + + for exception in vmss_delete_exceptions: + if not isinstance(exception, (CloudError, AgentRuntimeException)): + logging.exception(exception) + if vmss_delete_exceptions: + raise AgentRuntimeException(str(vmss_delete_exceptions)) AppScaleLogger.log("Virtual machine scale set(s) have been successfully " - "deleted.") - - for delete_thread in lb_delete_threads: - delete_thread.join() + "deleted.") + # Delete the load balancer virtual machines matching the given instance ids. + delete_lb_instances = self.diff(instances_to_delete, delete_ss_instances) + lb_delete_futures = [] + lb_delete_exceptions = [] + with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: + for vm_name in delete_lb_instances: + lb_delete_futures.append(executor.submit(self.delete_virtual_machine, + compute_client, parameters, vm_name)) + + for future in concurrent.futures.as_completed(lb_delete_futures): + exception = future.exception() + if exception: + lb_delete_exceptions.append(exception) + + for exception in lb_delete_exceptions: + if not isinstance(exception, (CloudError, AgentRuntimeException)): + logging.exception(exception) + if lb_delete_exceptions: + raise AgentRuntimeException(str(lb_delete_exceptions)) AppScaleLogger.log("Load balancer virtual machine(s) have been " "successfully deleted") @@ -945,22 +1120,31 @@ def delete_virtual_machine_scale_set(self, compute_client, parameters, vmss_name parameters: A dict, containing all the parameters necessary to authenticate this user with Azure. vmss_name: The name of the virtual machine scale set to be deleted. + Raises: + AgentConfigurationException: If we are unable to authenticate with Azure. + AgentRuntimeException: If a scale set could not be successfully deleted. """ resource_group = parameters[self.PARAM_RESOURCE_GROUP] verbose = parameters[self.PARAM_VERBOSE] AppScaleLogger.verbose("Deleting Scale Set {} ...".format(vmss_name), verbose) try: delete_response = compute_client.virtual_machine_scale_sets.delete( - resource_group, vmss_name) - resource_name = 'Virtual Machine Scale Set' + ":" + vmss_name - self.sleep_until_delete_operation_done(delete_response, resource_name, - self.MAX_VM_UPDATE_TIME, verbose) - AppScaleLogger.verbose("Virtual Machine Scale Set {} has been successfully " - "deleted.".format(vmss_name), verbose) + resource_group, vmss_name) except CloudError as error: - raise AgentConfigurationException("There was a problem while deleting the " - "Scale Set {0} due to the error: {1}" - .format(vmss_name, error.message)) + logging.exception("CloudError received trying to clean up Scale Set.") + raise AgentRuntimeException("Unable to clean up Scale Set {}. " + "Reason: {}".format(vmss_name, error.message)) + except ClientException as e: + logging.exception("ClientException received while attempting to contact " + "Azure.") + raise AgentConfigurationException("Unable to communicate with Azure " + "while trying to clean up Scale Set {}. Please check your " + "cloud configuration. Reason: {}".format(vmss_name, e.message)) + resource_name = 'Virtual Machine Scale Set' + ":" + vmss_name + self.sleep_until_delete_operation_done(delete_response, resource_name, + self.MAX_VM_UPDATE_TIME, verbose) + AppScaleLogger.verbose("Virtual Machine Scale Set {} has been successfully " + "deleted.".format(vmss_name), verbose) def delete_vmss_instance(self, compute_client, parameters, vmss_name, instance_id): """ Deletes the specified virtual machine instance from the given Scale Set. @@ -973,16 +1157,32 @@ def delete_vmss_instance(self, compute_client, parameters, vmss_name, instance_i """ resource_group = parameters[self.PARAM_RESOURCE_GROUP] verbose = parameters[self.PARAM_VERBOSE] - AppScaleLogger.verbose("Deleting Virtual Machine Instance {0} from Scale " - "Set {1} ...".format(instance_id, vmss_name), verbose) - result = compute_client.virtual_machine_scale_set_vms.delete(resource_group, - vmss_name, - instance_id) + + vm_info = "Virtual Machine {0} from Scale Set {1}".format(instance_id, + vmss_name) + + AppScaleLogger.verbose("Deleting {0} ...".format(vm_info), verbose) + try: + result = compute_client.virtual_machine_scale_set_vms.delete( + resource_group, vmss_name, instance_id) + except CloudError as error: + logging.exception("CloudError received trying to clean up scale set.") + raise AgentRuntimeException("Unable to clean up VM {} from Scale Set {}. " + "Reason: {}".format(instance_id, vmss_name, + error.message)) + except ClientException as e: + logging.exception("ClientException received while attempting to contact " + "Azure.") + raise AgentConfigurationException("Unable to communicate with Azure " + "while trying to clean up VM {} from Scale Set {}. Please check your " + "cloud configuration. Reason: {}".format(instance_id, vmss_name, + e.message)) resource_name = 'Virtual Machine Instance ' + instance_id self.sleep_until_delete_operation_done(result, resource_name, self.MAX_VM_UPDATE_TIME, verbose) - AppScaleLogger.verbose("Virtual Machine Instance {0} from Scale Set {1} " - "has been successfully deleted".format(instance_id, vmss_name), verbose) + + AppScaleLogger.verbose("{0} from scaleset {1} has been successfully " + "deleted".format(instance_id, vmss_name), verbose) def delete_virtual_machine(self, compute_client, parameters, vm_name): """ Deletes the virtual machine from the resource_group specified. @@ -995,7 +1195,18 @@ def delete_virtual_machine(self, compute_client, parameters, vm_name): resource_group = parameters[self.PARAM_RESOURCE_GROUP] verbose = parameters[self.PARAM_VERBOSE] AppScaleLogger.verbose("Deleting Virtual Machine {} ...".format(vm_name), verbose) - result = compute_client.virtual_machines.delete(resource_group, vm_name) + try: + result = compute_client.virtual_machines.delete(resource_group, vm_name) + except CloudError as error: + logging.exception("CloudError received trying to clean up scale set.") + raise AgentRuntimeException("Unable to clean up Virtual Machine {}. " + "Reason: {}".format(vm_name, error.message)) + except ClientException as e: + logging.exception("ClientException received while attempting to contact " + "Azure.") + raise AgentConfigurationException("Unable to communicate with Azure " + "while trying to clean up Virtual Machine {}. Please check your " + "cloud configuration. Reason: {}".format(vm_name, e.message)) resource_name = 'Virtual Machine' + ':' + vm_name self.sleep_until_delete_operation_done(result, resource_name, self.MAX_VM_UPDATE_TIME, verbose) @@ -1013,6 +1224,8 @@ def sleep_until_delete_operation_done(self, result, resource_name, max_sleep: The maximum number of seconds to sleep for the resources to be deleted. verbose: A boolean indicating whether or not in verbose mode. + Raises: + AgentRuntimeException if we time out waiting for the operation to finish. """ time_start = time.time() while not result.done(): @@ -1021,9 +1234,10 @@ def sleep_until_delete_operation_done(self, result, resource_name, time.sleep(self.SLEEP_TIME) total_sleep_time = time.time() - time_start if total_sleep_time > max_sleep: - AppScaleLogger.log("Waited {0} second(s) for {1} to be deleted. " - "Operation has timed out.".format(total_sleep_time, resource_name)) - break + err_msg = "Waited {0} second(s) for {1} to be deleted. Operation has " \ + "timed out.".format(total_sleep_time, resource_name) + AppScaleLogger.log(err_msg) + raise AgentRuntimeException(err_msg) def does_address_exist(self, parameters): """ Verifies that the specified static IP address has been allocated, and @@ -1070,16 +1284,27 @@ def does_zone_exist(self, parameters): existence. Returns: True if the zone exists, and False otherwise. + Raises: + AgentConfigurationException: If an error is encountered during + checking for the zone due to configuration errors (zone does not + exist or authentication issues). """ credentials = self.open_connection(parameters) subscription_id = str(parameters[self.PARAM_SUBSCRIBER_ID]) zone = parameters[self.PARAM_ZONE] resource_client = ResourceManagementClient(credentials, subscription_id) - resource_providers = resource_client.providers.list() - for provider in resource_providers: - for resource_type in provider.resource_types: - if zone in resource_type.locations: - return True + try: + resource_providers = resource_client.providers.list() + for provider in resource_providers: + for resource_type in provider.resource_types: + if zone in resource_type.locations: + return True + except ClientException as error: + logging.exception("Unable to check if zone exists.") + raise AgentConfigurationException("Unable to check if zone exists. " + "Please check your cloud " + "configuration. Reason: {}".format( + error.message)) return False def cleanup_state(self, parameters): @@ -1088,6 +1313,10 @@ def cleanup_state(self, parameters): Args: parameters: A dict that includes keys indicating the remote state that should be deleted. + Raises: + AgentConfigurationException: If we are unable to authenticate with Azure. + AgentRuntimeException: If an error is encountered trying to clean up + the state in Azure. """ subscription_id = str(parameters[self.PARAM_SUBSCRIBER_ID]) resource_group = parameters[self.PARAM_RESOURCE_GROUP] @@ -1097,36 +1326,69 @@ def cleanup_state(self, parameters): AppScaleLogger.log("Cleaning up the network configuration created for this " "deployment ...") - network_interfaces = network_client.network_interfaces.list(resource_group) - for interface in network_interfaces: - result = network_client.network_interfaces.delete(resource_group, interface.name) - resource_name = 'Network Interface' + ':' + interface.name - self.sleep_until_delete_operation_done(result, resource_name, - self.MAX_SLEEP_TIME, verbose) - AppScaleLogger.verbose("Network Interface {} has been successfully deleted.". - format(interface.name), verbose) + try: + network_interfaces = network_client.network_interfaces.list(resource_group) + for interface in network_interfaces: + result = network_client.network_interfaces.delete(resource_group, interface.name) + resource_name = 'Network Interface' + ':' + interface.name + self.sleep_until_delete_operation_done(result, resource_name, + self.MAX_SLEEP_TIME, verbose) + AppScaleLogger.verbose("Network Interface {} has been successfully deleted.". + format(interface.name), verbose) + except CloudError as error: + logging.exception("CloudError received trying to clean up network interfaces.") + raise AgentRuntimeException("Unable to clean up network interfaces. " + "Reason: {}".format(error.message)) + except ClientException as e: + logging.exception("ClientException received while attempting to contact " + "Azure.") + raise AgentConfigurationException("Unable to communicate with Azure " + "while trying to clean up network interfaces. Please check your " + "cloud configuration. Reason: {}".format(e.message)) AppScaleLogger.log("Network Interface(s) have been successfully deleted.") - public_ip_addresses = network_client.public_ip_addresses.list(resource_group) - for public_ip in public_ip_addresses: - result = network_client.public_ip_addresses.delete(resource_group, public_ip.name) - resource_name = 'Public IP Address' + ':' + public_ip.name - self.sleep_until_delete_operation_done(result, resource_name, - self.MAX_SLEEP_TIME, verbose) - AppScaleLogger.verbose("Public IP Address {} has been successfully deleted.". - format(public_ip.name), verbose) + try: + public_ip_addresses = network_client.public_ip_addresses.list(resource_group) + for public_ip in public_ip_addresses: + result = network_client.public_ip_addresses.delete(resource_group, public_ip.name) + resource_name = 'Public IP Address' + ':' + public_ip.name + self.sleep_until_delete_operation_done(result, resource_name, + self.MAX_SLEEP_TIME, verbose) + AppScaleLogger.verbose("Public IP Address {} has been successfully deleted.". + format(public_ip.name), verbose) + except CloudError as error: + logging.exception("Unable to clean up public ips.") + raise AgentRuntimeException("Unable to clean up public ips. " + "Reason: {}".format(error.message)) + except ClientException as e: + logging.exception("ClientException received while attempting to contact " + "Azure.") + raise AgentConfigurationException("Unable to communicate with Azure " + "while trying to clean up public ips. Please check your cloud " + "configuration. Reason: {}".format(e.message)) AppScaleLogger.log("Public IP Address(s) have been successfully deleted.") - virtual_networks = network_client.virtual_networks.list(resource_group) - for network in virtual_networks: - result = network_client.virtual_networks.delete(resource_group, network.name) - resource_name = 'Virtual Network' + ':' + network.name - self.sleep_until_delete_operation_done(result, resource_name, - self.MAX_SLEEP_TIME, verbose) - AppScaleLogger.verbose("Virtual Network {} has been successfully deleted.". - format(network.name), verbose) + try: + virtual_networks = network_client.virtual_networks.list(resource_group) + for network in virtual_networks: + result = network_client.virtual_networks.delete(resource_group, network.name) + resource_name = 'Virtual Network' + ':' + network.name + self.sleep_until_delete_operation_done(result, resource_name, + self.MAX_SLEEP_TIME, verbose) + AppScaleLogger.verbose("Virtual Network {} has been successfully deleted.". + format(network.name), verbose) + except CloudError as error: + logging.exception("Unable to clean up virtual networks.") + raise AgentRuntimeException("Unable to clean up virtual networks. " + "Reason: {}".format(error.message)) + except ClientException as e: + logging.exception("ClientException received while attempting to " + "contact Azure.") + raise AgentConfigurationException("Unable to communicate with Azure " + "while trying to clean up virtual networks. Please check your cloud " + "configuration. Reason: {}".format(e.message)) AppScaleLogger.log("Virtual Network(s) have been successfully deleted.") @@ -1163,10 +1425,7 @@ def get_params_from_args(self, args): self.PARAM_ZONE : args[self.PARAM_ZONE], 'autoscale_agent': False } - is_valid, rg_names = self.assert_credentials_are_valid(params) - if not is_valid: - raise AgentConfigurationException("Unable to authenticate using the " - "credentials provided.") + self.assert_credentials_are_valid(params) # In case no resource group is passed, pass a default group. if not args[self.PARAM_RESOURCE_GROUP]: @@ -1196,7 +1455,8 @@ def get_marketplace_image_version(self, parameters): "appscale-marketplace:appscale:3:latest" was received at this time would return "appscale-marketplace:appscale:3:3.5.2" Raises: - AgentConfigurationException: If we cannot find the image. + AgentConfigurationException: If the image is invalid or there was an + authentication issue trying to contact Azure. """ credentials = self.open_connection(parameters) subscription_id = str(parameters[self.PARAM_SUBSCRIBER_ID]) @@ -1214,19 +1474,24 @@ def get_marketplace_image_version(self, parameters): compute_client.virtual_machine_images.get(compatible_zone, publisher, offer, sku, version) return azure_image_id - except CloudError as e: + except ClientException as e: raise AgentConfigurationException("Received CloudError trying to " - "request specified image. Please ensure your image is valid in " - "the AppScalefile. Reason: {}".format(e)) + "request specified image. Please " + "ensure your image is valid in " + "the AppScalefile and that your " + "cloud configuration is correct. " + "Reason: {}".format(e.message)) try: top_one = compute_client.virtual_machine_images.list( compatible_zone, publisher, offer, sku, top=1, orderby='name desc') - except CloudError as e: + except ClientException as e: raise AgentConfigurationException("Received CloudError trying to " - "request specified image. Please ensure your image is valid in " - "the AppScalefile. Reason: {}".format(e)) - + "request specified image. Please " + "ensure your image is valid in " + "the AppScalefile and that your cloud " + "configuration is correct. " + "Reason: {}".format(e.message)) if len(top_one) == 0: raise AgentConfigurationException("Can't resolve the vesion of '{}'" .format(azure_image_id)) @@ -1334,8 +1599,13 @@ def open_connection(self, parameters): # Get an Authentication token using ADAL. context = adal.AuthenticationContext(self.AZURE_AUTH_ENDPOINT + tenant_id) - token_response = context.acquire_token_with_client_credentials( - self.AZURE_RESOURCE_URL, app_id, app_secret_key) + try: + token_response = context.acquire_token_with_client_credentials( + self.AZURE_RESOURCE_URL, app_id, app_secret_key) + except adal.adal_error.AdalError as e: + raise AgentConfigurationException( + "Unable to communicate with Azure! Please check your cloud " + "configuration. Reason: {}".format(e.message)) token_response.get('accessToken') # To access Azure resources for an application, we need a Service Principal @@ -1357,6 +1627,10 @@ def create_virtual_network(self, network_client, parameters, network_name, subnet_name: The name to use for the Subnet resource. Returns: A Subnet instance from the Virtual Network created. + Raises: + AgentConfigurationException: If we are unable to authenticate with Azure. + AgentRuntimeException: If an error is encountered trying to create a + virtual network in Azure. """ group_name = parameters[self.PARAM_RESOURCE_GROUP] region = parameters[self.PARAM_ZONE] @@ -1365,10 +1639,22 @@ def create_virtual_network(self, network_client, parameters, network_name, format(network_name), verbose) address_space = AddressSpace(address_prefixes=['10.1.0.0/16']) subnet1 = Subnet(name=subnet_name, address_prefix='10.1.0.0/24') - result = network_client.virtual_networks.create_or_update(group_name, network_name, - VirtualNetwork(location=region, address_space=address_space, - subnets=[subnet1])) - self.sleep_until_update_operation_done(result, network_name, verbose) + try: + result = network_client.virtual_networks.create_or_update( + group_name, network_name, + VirtualNetwork(location=region, address_space=address_space, subnets=[subnet1])) + self.sleep_until_update_operation_done(result, network_name, verbose) + except CloudError as error: + logging.exception("Azure agent received a CloudError.") + raise AgentRuntimeException("Unable to create virtual network. Reason: " + "{}".format(error.message)) + except ClientException as e: + logging.exception("ClientException received while attempting to contact " + "Azure.") + raise AgentConfigurationException("Unable to communicate with Azure " + "while trying to create virtual network. Please check your cloud " + "configuration. Reason: {}".format(e.message)) + subnet = network_client.subnets.get(group_name, network_name, subnet_name) return subnet @@ -1383,6 +1669,10 @@ def create_network_interface(self, network_client, interface_name, ip_name, subnet: The Subnet resource from the Virtual Network created. parameters: A dict, containing all the parameters necessary to authenticate this user with Azure. + Raises: + AgentConfigurationException: If we are unable to authenticate with Azure. + AgentRuntimeException: If an error is encountered trying to create a + network interface in Azure. """ group_name = parameters[self.PARAM_RESOURCE_GROUP] region = parameters[self.PARAM_ZONE] @@ -1392,9 +1682,21 @@ def create_network_interface(self, network_client, interface_name, ip_name, ip_address = PublicIPAddress( location=region, public_ip_allocation_method=IPAllocationMethod.dynamic, idle_timeout_in_minutes=4) - result = network_client.public_ip_addresses.create_or_update( - group_name, ip_name, ip_address) - self.sleep_until_update_operation_done(result, ip_name, verbose) + try: + result = network_client.public_ip_addresses.create_or_update( + group_name, ip_name, ip_address) + self.sleep_until_update_operation_done(result, ip_name, verbose) + except CloudError as error: + logging.exception("Azure agent received a CloudError.") + raise AgentRuntimeException("Unable to create public ip address. " + "Reason: {}".format(error.message)) + except ClientException as e: + logging.exception("ClientException received while attempting to contact " + "Azure.") + raise AgentConfigurationException("Unable to communicate with Azure " + "while trying to create a public ip address. Please check your cloud " + "configuration. Reason: {}".format(e.message)) + public_ip_address = network_client.public_ip_addresses.get(group_name, ip_name) AppScaleLogger.verbose("Creating/Updating the Network Interface '{}'". @@ -1403,10 +1705,21 @@ def create_network_interface(self, network_client, interface_name, ip_name, name=interface_name, private_ip_allocation_method=IPAllocationMethod.dynamic, subnet=subnet, public_ip_address=PublicIPAddress(id=(public_ip_address.id))) - result = network_client.network_interfaces.create_or_update(group_name, - interface_name, NetworkInterface(location=region, - ip_configurations=[network_interface_ip_conf])) - self.sleep_until_update_operation_done(result, interface_name, verbose) + try: + result = network_client.network_interfaces.create_or_update(group_name, + interface_name, NetworkInterface(location=region, + ip_configurations=[network_interface_ip_conf])) + self.sleep_until_update_operation_done(result, interface_name, verbose) + except CloudError as error: + logging.exception("Azure agent received a CloudError.") + raise AgentRuntimeException("Unable to create network interface. " + "Reason: {}".format(error.message)) + except ClientException as e: + logging.exception("ClientException received while attempting to contact " + "Azure.") + raise AgentConfigurationException("Unable to communicate with Azure " + "while trying to create network interface. Please check your cloud " + "configuration. Reason: {}".format(e.message)) def sleep_until_update_operation_done(self, result, resource_name, verbose): """ Sleeps until the create/update operation for the resource is completed @@ -1437,7 +1750,8 @@ def create_resource_group(self, parameters, credentials): credentials: A ServicePrincipalCredentials instance, that can be used to access or create any resources. Raises: - AgentConfigurationException: If there was a problem creating or accessing + AgentConfigurationException: If we are unable to authenticate with Azure. + AgentRuntimeException: If there was a problem creating or accessing a resource group with the given subscription. """ subscription_id = str(parameters[self.PARAM_SUBSCRIBER_ID]) @@ -1449,8 +1763,8 @@ def create_resource_group(self, parameters, credentials): tag_name = parameters[self.PARAM_TAG] storage_client = StorageManagementClient(credentials, subscription_id) - resource_client.providers.register(self.MICROSOFT_STORAGE_RESOURCE) try: + resource_client.providers.register(self.MICROSOFT_STORAGE_RESOURCE) # If the resource group does not already exist, create a new one with the # specified storage account. if not self.does_resource_group_exist(resource_group_name, resource_client): @@ -1477,8 +1791,15 @@ def create_resource_group(self, parameters, credentials): else: self.create_storage_account(parameters, storage_client) except CloudError as error: + logging.exception("Unable to create resource group.") raise AgentConfigurationException("Unable to create a resource group " "using the credentials provided: {}".format(error.message)) + except ClientException as e: + logging.exception("ClientException received while attempting to contact " + "Azure.") + raise AgentConfigurationException("Unable to communicate with Azure " + "while trying to create resource group. Please check your cloud " + "configuration. Reason: {}".format(e.message)) def create_storage_account(self, parameters, storage_client): """ Creates a Storage Account under the Resource Group, if it does not @@ -1507,6 +1828,7 @@ def create_storage_account(self, parameters, storage_client): # wait() insures polling the underlying async operation until it's done. result.wait() except CloudError as error: + logging.exception("Unable to create storage account.") raise AgentConfigurationException("Unable to create a storage account " "using the credentials provided: {}".format(error.message)) @@ -1520,10 +1842,7 @@ def does_resource_group_exist(self, resource_group_name, resource_client): False, otherwise. """ resource_groups = resource_client.resource_groups.list() - resource_group_names = [] - for rg in resource_groups: - resource_group_names.append(rg.name) - - if resource_group_name in resource_group_names: - return True + for resource_group in resource_groups: + if resource_group_name == resource_group.name: + return True return False