From 4af971479836424d0a172e672694bb3fd0c2fc37 Mon Sep 17 00:00:00 2001 From: Chris Coutinho <12901868+cbcoutinho@users.noreply.github.com> Date: Sat, 8 May 2021 00:33:53 +0200 Subject: [PATCH 01/38] Add permission to upload ACL in ExtraArgs (#318) Add permission to upload ACL in ExtraArgs Reviewed-by: https://github.com/apps/ansible-zuul --- changelogs/fragments/318-s3-upload-acl.yml | 2 + plugins/modules/aws_s3.py | 10 ++- .../targets/aws_s3/defaults/main.yml | 3 +- .../integration/targets/aws_s3/tasks/main.yml | 68 +++++++++++++++++++ .../targets/aws_s3/templates/policy.json.j2 | 21 ++++++ 5 files changed, 102 insertions(+), 2 deletions(-) create mode 100644 changelogs/fragments/318-s3-upload-acl.yml create mode 100644 tests/integration/targets/aws_s3/templates/policy.json.j2 diff --git a/changelogs/fragments/318-s3-upload-acl.yml b/changelogs/fragments/318-s3-upload-acl.yml new file mode 100644 index 00000000000..19326ceb315 --- /dev/null +++ b/changelogs/fragments/318-s3-upload-acl.yml @@ -0,0 +1,2 @@ +bugfixes: +- aws_s3 - Fix upload permission when an S3 bucket ACL policy requires a particular canned ACL (https://github.com/ansible-collections/amazon.aws/pull/318) diff --git a/plugins/modules/aws_s3.py b/plugins/modules/aws_s3.py index 5c2dfda181a..50dac4561a4 100644 --- a/plugins/modules/aws_s3.py +++ b/plugins/modules/aws_s3.py @@ -78,7 +78,8 @@ - This option lets the user set the canned permissions on the object/bucket that are created. The permissions that can be set are C(private), C(public-read), C(public-read-write), C(authenticated-read) for a bucket or C(private), C(public-read), C(public-read-write), C(aws-exec-read), C(authenticated-read), C(bucket-owner-read), - C(bucket-owner-full-control) for an object. Multiple permissions can be specified as a list. + C(bucket-owner-full-control) for an object. Multiple permissions can be specified as a list; although only the first one + will be used during the initial upload of the file default: ['private'] type: list elements: str @@ -532,6 +533,13 @@ def upload_s3file(module, s3, bucket, obj, expiry, metadata, encrypt, headers, s else: extra['Metadata'][option] = metadata[option] + if module.params.get('permission'): + permissions = module.params['permission'] + if isinstance(permissions, str): + extra['ACL'] = permissions + elif isinstance(permissions, list): + extra['ACL'] = permissions[0] + if 'ContentType' not in extra: content_type = None if src is not None: diff --git a/tests/integration/targets/aws_s3/defaults/main.yml b/tests/integration/targets/aws_s3/defaults/main.yml index eb7dd2d3712..67d026de087 100644 --- a/tests/integration/targets/aws_s3/defaults/main.yml +++ b/tests/integration/targets/aws_s3/defaults/main.yml @@ -1,3 +1,4 @@ --- # defaults file for s3 -bucket_name: '{{resource_prefix}}' +bucket_name: '{{ resource_prefix }}' +bucket_name_acl: '{{ bucket_name }}-with-acl' diff --git a/tests/integration/targets/aws_s3/tasks/main.yml b/tests/integration/targets/aws_s3/tasks/main.yml index 12283c81612..968116f014c 100644 --- a/tests/integration/targets/aws_s3/tasks/main.yml +++ b/tests/integration/targets/aws_s3/tasks/main.yml @@ -8,6 +8,14 @@ region: "{{ aws_region }}" block: + - name: get ARN of calling user + aws_caller_info: + register: aws_caller_info + + - name: register account id + set_fact: + aws_account: "{{ aws_caller_info.account }}" + - name: Create temporary directory tempfile: state: directory @@ -537,6 +545,47 @@ - result is not changed when: ansible_system == 'Linux' or ansible_distribution == 'MacOSX' + - name: make a bucket with the bucket-owner-full-control ACL + s3_bucket: + name: "{{ bucket_name_acl }}" + state: present + policy: "{{ lookup('template', 'policy.json.j2') }}" + register: bucket_with_policy + + - assert: + that: + - bucket_with_policy is changed + + - name: fail to upload the file to the bucket with an ACL + aws_s3: + bucket: "{{ bucket_name_acl }}" + mode: put + src: "{{ tmpdir.path }}/upload.txt" + object: file-with-permissions.txt + permission: private + ignore_nonexistent_bucket: True + register: upload_private + ignore_errors: True + + # XXX Doesn't fail... + # - assert: + # that: + # - upload_private is failed + + - name: upload the file to the bucket with an ACL + aws_s3: + bucket: "{{ bucket_name_acl }}" + mode: put + src: "{{ tmpdir.path }}/upload.txt" + object: file-with-permissions.txt + permission: bucket-owner-full-control + ignore_nonexistent_bucket: True + register: upload_owner + + - assert: + that: + - upload_owner is changed + - name: create an object from static content aws_s3: bucket: "{{ bucket_name }}" @@ -650,9 +699,22 @@ - delete.txt - delete_encrypt.txt - delete_encrypt_kms.txt + - multipart.txt - put-content.txt - put-template.txt - put-binary.txt + - foo/bar/baz + - foo/bar + - foo + ignore_errors: yes + + - name: remove uploaded files (bucket with ACL) + aws_s3: + bucket: "{{ bucket_name_acl }}" + mode: delobj + object: "{{ item }}" + loop: + - file-with-permissions.txt ignore_errors: yes - name: delete temporary files @@ -672,3 +734,9 @@ bucket: "{{ bucket_name | hash('md5') + '.bucket' }}" mode: delete ignore_errors: yes + + - name: delete the acl bucket + aws_s3: + bucket: "{{ bucket_name_acl }}" + mode: delete + ignore_errors: yes diff --git a/tests/integration/targets/aws_s3/templates/policy.json.j2 b/tests/integration/targets/aws_s3/templates/policy.json.j2 new file mode 100644 index 00000000000..4af2e0713b1 --- /dev/null +++ b/tests/integration/targets/aws_s3/templates/policy.json.j2 @@ -0,0 +1,21 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "Only allow writes to my bucket with bucket owner full control", + "Effect": "Allow", + "Principal": { "AWS":"{{ aws_account }}" }, + "Action": [ + "s3:PutObject" + ], + "Resource": [ + "arn:aws:s3:::{{ bucket_name_acl }}/*" + ], + "Condition": { + "StringEquals": { + "s3:x-amz-acl": "bucket-owner-full-control" + } + } + } + ] +} From 126338aeec6b78920f3832ea64834d9d54844a19 Mon Sep 17 00:00:00 2001 From: jillr Date: Mon, 2 Mar 2020 19:25:18 +0000 Subject: [PATCH 02/38] Initial commit --- plugins/modules/ec2_instance.py | 1803 +++++++++++++++++ plugins/modules/ec2_instance_info.py | 570 ++++++ .../integration/targets/ec2_instance/aliases | 3 + .../targets/ec2_instance/inventory | 17 + .../integration/targets/ec2_instance/main.yml | 43 + .../targets/ec2_instance/meta/main.yml | 4 + .../roles/ec2_instance/defaults/main.yml | 14 + .../files/assume-role-policy.json | 13 + .../roles/ec2_instance/meta/main.yml | 3 + .../ec2_instance/tasks/block_devices.yml | 82 + .../ec2_instance/tasks/checkmode_tests.yml | 172 ++ .../roles/ec2_instance/tasks/cpu_options.yml | 86 + .../ec2_instance/tasks/default_vpc_tests.yml | 57 + .../ec2_instance/tasks/ebs_optimized.yml | 41 + .../roles/ec2_instance/tasks/env_cleanup.yml | 93 + .../roles/ec2_instance/tasks/env_setup.yml | 79 + .../tasks/external_resource_attach.yml | 129 ++ .../roles/ec2_instance/tasks/find_ami.yml | 15 + .../ec2_instance/tasks/iam_instance_role.yml | 127 ++ .../ec2_instance/tasks/instance_no_wait.yml | 68 + .../roles/ec2_instance/tasks/main.yml | 48 + .../tasks/tags_and_vpc_settings.yml | 158 ++ .../tasks/termination_protection.yml | 184 ++ .../roles/ec2_instance/tasks/version_fail.yml | 29 + .../tasks/version_fail_wrapper.yml | 30 + .../integration/targets/ec2_instance/runme.sh | 12 + 26 files changed, 3880 insertions(+) create mode 100644 plugins/modules/ec2_instance.py create mode 100644 plugins/modules/ec2_instance_info.py create mode 100644 tests/integration/targets/ec2_instance/aliases create mode 100644 tests/integration/targets/ec2_instance/inventory create mode 100644 tests/integration/targets/ec2_instance/main.yml create mode 100644 tests/integration/targets/ec2_instance/meta/main.yml create mode 100644 tests/integration/targets/ec2_instance/roles/ec2_instance/defaults/main.yml create mode 100644 tests/integration/targets/ec2_instance/roles/ec2_instance/files/assume-role-policy.json create mode 100644 tests/integration/targets/ec2_instance/roles/ec2_instance/meta/main.yml create mode 100644 tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/block_devices.yml create mode 100644 tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/checkmode_tests.yml create mode 100644 tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/cpu_options.yml create mode 100644 tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/default_vpc_tests.yml create mode 100644 tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/ebs_optimized.yml create mode 100644 tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/env_cleanup.yml create mode 100644 tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/env_setup.yml create mode 100644 tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/external_resource_attach.yml create mode 100644 tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/find_ami.yml create mode 100644 tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/iam_instance_role.yml create mode 100644 tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/instance_no_wait.yml create mode 100644 tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/main.yml create mode 100644 tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/tags_and_vpc_settings.yml create mode 100644 tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/termination_protection.yml create mode 100644 tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/version_fail.yml create mode 100644 tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/version_fail_wrapper.yml create mode 100755 tests/integration/targets/ec2_instance/runme.sh diff --git a/plugins/modules/ec2_instance.py b/plugins/modules/ec2_instance.py new file mode 100644 index 00000000000..ea7f49c5f32 --- /dev/null +++ b/plugins/modules/ec2_instance.py @@ -0,0 +1,1803 @@ +#!/usr/bin/python +# Copyright: Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import absolute_import, division, print_function +__metaclass__ = type + + +ANSIBLE_METADATA = {'metadata_version': '1.1', + 'status': ['preview'], + 'supported_by': 'community'} + +DOCUMENTATION = ''' +--- +module: ec2_instance +short_description: Create & manage EC2 instances +description: + - Create and manage AWS EC2 instances. + - > + Note: This module does not support creating + L(EC2 Spot instances,https://aws.amazon.com/ec2/spot/). The M(ec2) module + can create and manage spot instances. +author: + - Ryan Scott Brown (@ryansb) +requirements: [ "boto3", "botocore" ] +options: + instance_ids: + description: + - If you specify one or more instance IDs, only instances that have the specified IDs are returned. + type: list + state: + description: + - Goal state for the instances. + choices: [present, terminated, running, started, stopped, restarted, rebooted, absent] + default: present + type: str + wait: + description: + - Whether or not to wait for the desired state (use wait_timeout to customize this). + default: true + type: bool + wait_timeout: + description: + - How long to wait (in seconds) for the instance to finish booting/terminating. + default: 600 + type: int + instance_type: + description: + - Instance type to use for the instance, see U(https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-types.html) + Only required when instance is not already present. + default: t2.micro + type: str + user_data: + description: + - Opaque blob of data which is made available to the ec2 instance + type: str + tower_callback: + description: + - Preconfigured user-data to enable an instance to perform a Tower callback (Linux only). + - Mutually exclusive with I(user_data). + - For Windows instances, to enable remote access via Ansible set I(tower_callback.windows) to true, and optionally set an admin password. + - If using 'windows' and 'set_password', callback to Tower will not be performed but the instance will be ready to receive winrm connections from Ansible. + type: dict + suboptions: + tower_address: + description: + - IP address or DNS name of Tower server. Must be accessible via this address from the VPC that this instance will be launched in. + type: str + job_template_id: + description: + - Either the integer ID of the Tower Job Template, or the name (name supported only for Tower 3.2+). + type: str + host_config_key: + description: + - Host configuration secret key generated by the Tower job template. + type: str + tags: + description: + - A hash/dictionary of tags to add to the new instance or to add/remove from an existing one. + type: dict + purge_tags: + description: + - Delete any tags not specified in the task that are on the instance. + This means you have to specify all the desired tags on each task affecting an instance. + default: false + type: bool + image: + description: + - An image to use for the instance. The M(ec2_ami_info) module may be used to retrieve images. + One of I(image) or I(image_id) are required when instance is not already present. + type: dict + suboptions: + id: + description: + - The AMI ID. + type: str + ramdisk: + description: + - Overrides the AMI's default ramdisk ID. + type: str + kernel: + description: + - a string AKI to override the AMI kernel. + image_id: + description: + - I(ami) ID to use for the instance. One of I(image) or I(image_id) are required when instance is not already present. + - This is an alias for I(image.id). + type: str + security_groups: + description: + - A list of security group IDs or names (strings). Mutually exclusive with I(security_group). + type: list + security_group: + description: + - A security group ID or name. Mutually exclusive with I(security_groups). + type: str + name: + description: + - The Name tag for the instance. + type: str + vpc_subnet_id: + description: + - The subnet ID in which to launch the instance (VPC) + If none is provided, ec2_instance will chose the default zone of the default VPC. + aliases: ['subnet_id'] + type: str + network: + description: + - Either a dictionary containing the key 'interfaces' corresponding to a list of network interface IDs or + containing specifications for a single network interface. + - Use the ec2_eni module to create ENIs with special settings. + type: dict + suboptions: + interfaces: + description: + - a list of ENI IDs (strings) or a list of objects containing the key I(id). + type: list + assign_public_ip: + description: + - when true assigns a public IP address to the interface + type: bool + private_ip_address: + description: + - an IPv4 address to assign to the interface + type: str + ipv6_addresses: + description: + - a list of IPv6 addresses to assign to the network interface + type: list + source_dest_check: + description: + - controls whether source/destination checking is enabled on the interface + type: bool + description: + description: + - a description for the network interface + type: str + private_ip_addresses: + description: + - a list of IPv4 addresses to assign to the network interface + type: list + subnet_id: + description: + - the subnet to connect the network interface to + type: str + delete_on_termination: + description: + - Delete the interface when the instance it is attached to is + terminated. + type: bool + device_index: + description: + - The index of the interface to modify + type: int + groups: + description: + - a list of security group IDs to attach to the interface + type: list + volumes: + description: + - A list of block device mappings, by default this will always use the AMI root device so the volumes option is primarily for adding more storage. + - A mapping contains the (optional) keys device_name, virtual_name, ebs.volume_type, ebs.volume_size, ebs.kms_key_id, + ebs.iops, and ebs.delete_on_termination. + - For more information about each parameter, see U(https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_BlockDeviceMapping.html). + type: list + launch_template: + description: + - The EC2 launch template to base instance configuration on. + type: dict + suboptions: + id: + description: + - the ID of the launch template (optional if name is specified). + type: str + name: + description: + - the pretty name of the launch template (optional if id is specified). + type: str + version: + description: + - the specific version of the launch template to use. If unspecified, the template default is chosen. + key_name: + description: + - Name of the SSH access key to assign to the instance - must exist in the region the instance is created. + type: str + availability_zone: + description: + - Specify an availability zone to use the default subnet it. Useful if not specifying the I(vpc_subnet_id) parameter. + - If no subnet, ENI, or availability zone is provided, the default subnet in the default VPC will be used in the first AZ (alphabetically sorted). + type: str + instance_initiated_shutdown_behavior: + description: + - Whether to stop or terminate an instance upon shutdown. + choices: ['stop', 'terminate'] + type: str + tenancy: + description: + - What type of tenancy to allow an instance to use. Default is shared tenancy. Dedicated tenancy will incur additional charges. + choices: ['dedicated', 'default'] + type: str + termination_protection: + description: + - Whether to enable termination protection. + This module will not terminate an instance with termination protection active, it must be turned off first. + type: bool + cpu_credit_specification: + description: + - For T series instances, choose whether to allow increased charges to buy CPU credits if the default pool is depleted. + - Choose I(unlimited) to enable buying additional CPU credits. + choices: ['unlimited', 'standard'] + type: str + cpu_options: + description: + - Reduce the number of vCPU exposed to the instance. + - Those parameters can only be set at instance launch. The two suboptions threads_per_core and core_count are mandatory. + - See U(https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-optimize-cpu.html) for combinations available. + - Requires botocore >= 1.10.16 + type: dict + suboptions: + threads_per_core: + description: + - Select the number of threads per core to enable. Disable or Enable Intel HT. + choices: [1, 2] + required: true + type: int + core_count: + description: + - Set the number of core to enable. + required: true + type: int + detailed_monitoring: + description: + - Whether to allow detailed cloudwatch metrics to be collected, enabling more detailed alerting. + type: bool + ebs_optimized: + description: + - Whether instance is should use optimized EBS volumes, see U(https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSOptimized.html). + type: bool + filters: + description: + - A dict of filters to apply when deciding whether existing instances match and should be altered. Each dict item + consists of a filter key and a filter value. See + U(https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeInstances.html). + for possible filters. Filter names and values are case sensitive. + - By default, instances are filtered for counting by their "Name" tag, base AMI, state (running, by default), and + subnet ID. Any queryable filter can be used. Good candidates are specific tags, SSH keys, or security groups. + type: dict + instance_role: + description: + - The ARN or name of an EC2-enabled instance role to be used. If a name is not provided in arn format + then the ListInstanceProfiles permission must also be granted. + U(https://docs.aws.amazon.com/IAM/latest/APIReference/API_ListInstanceProfiles.html) If no full ARN is provided, + the role with a matching name will be used from the active AWS account. + type: str + placement_group: + description: + - The placement group that needs to be assigned to the instance + type: str + +extends_documentation_fragment: +- ansible.amazon.aws +- ansible.amazon.ec2 + +''' + +EXAMPLES = ''' +# Note: These examples do not set authentication details, see the AWS Guide for details. + +# Terminate every running instance in a region. Use with EXTREME caution. +- ec2_instance: + state: absent + filters: + instance-state-name: running + +# restart a particular instance by its ID +- ec2_instance: + state: restarted + instance_ids: + - i-12345678 + +# start an instance with a public IP address +- ec2_instance: + name: "public-compute-instance" + key_name: "prod-ssh-key" + vpc_subnet_id: subnet-5ca1ab1e + instance_type: c5.large + security_group: default + network: + assign_public_ip: true + image_id: ami-123456 + tags: + Environment: Testing + +# start an instance and Add EBS +- ec2_instance: + name: "public-withebs-instance" + vpc_subnet_id: subnet-5ca1ab1e + instance_type: t2.micro + key_name: "prod-ssh-key" + security_group: default + volumes: + - device_name: /dev/sda1 + ebs: + volume_size: 16 + delete_on_termination: true + +# start an instance with a cpu_options +- ec2_instance: + name: "public-cpuoption-instance" + vpc_subnet_id: subnet-5ca1ab1e + tags: + Environment: Testing + instance_type: c4.large + volumes: + - device_name: /dev/sda1 + ebs: + delete_on_termination: true + cpu_options: + core_count: 1 + threads_per_core: 1 + +# start an instance and have it begin a Tower callback on boot +- ec2_instance: + name: "tower-callback-test" + key_name: "prod-ssh-key" + vpc_subnet_id: subnet-5ca1ab1e + security_group: default + tower_callback: + # IP or hostname of tower server + tower_address: 1.2.3.4 + job_template_id: 876 + host_config_key: '[secret config key goes here]' + network: + assign_public_ip: true + image_id: ami-123456 + cpu_credit_specification: unlimited + tags: + SomeThing: "A value" + +# start an instance with ENI (An existing ENI ID is required) +- ec2_instance: + name: "public-eni-instance" + key_name: "prod-ssh-key" + vpc_subnet_id: subnet-5ca1ab1e + network: + interfaces: + - id: "eni-12345" + tags: + Env: "eni_on" + volumes: + - device_name: /dev/sda1 + ebs: + delete_on_termination: true + instance_type: t2.micro + image_id: ami-123456 + +# add second ENI interface +- ec2_instance: + name: "public-eni-instance" + network: + interfaces: + - id: "eni-12345" + - id: "eni-67890" + image_id: ami-123456 + tags: + Env: "eni_on" + instance_type: t2.micro +''' + +RETURN = ''' +instances: + description: a list of ec2 instances + returned: when wait == true + type: complex + contains: + ami_launch_index: + description: The AMI launch index, which can be used to find this instance in the launch group. + returned: always + type: int + sample: 0 + architecture: + description: The architecture of the image + returned: always + type: str + sample: x86_64 + block_device_mappings: + description: Any block device mapping entries for the instance. + returned: always + type: complex + contains: + device_name: + description: The device name exposed to the instance (for example, /dev/sdh or xvdh). + returned: always + type: str + sample: /dev/sdh + ebs: + description: Parameters used to automatically set up EBS volumes when the instance is launched. + returned: always + type: complex + contains: + attach_time: + description: The time stamp when the attachment initiated. + returned: always + type: str + sample: "2017-03-23T22:51:24+00:00" + delete_on_termination: + description: Indicates whether the volume is deleted on instance termination. + returned: always + type: bool + sample: true + status: + description: The attachment state. + returned: always + type: str + sample: attached + volume_id: + description: The ID of the EBS volume + returned: always + type: str + sample: vol-12345678 + client_token: + description: The idempotency token you provided when you launched the instance, if applicable. + returned: always + type: str + sample: mytoken + ebs_optimized: + description: Indicates whether the instance is optimized for EBS I/O. + returned: always + type: bool + sample: false + hypervisor: + description: The hypervisor type of the instance. + returned: always + type: str + sample: xen + iam_instance_profile: + description: The IAM instance profile associated with the instance, if applicable. + returned: always + type: complex + contains: + arn: + description: The Amazon Resource Name (ARN) of the instance profile. + returned: always + type: str + sample: "arn:aws:iam::000012345678:instance-profile/myprofile" + id: + description: The ID of the instance profile + returned: always + type: str + sample: JFJ397FDG400FG9FD1N + image_id: + description: The ID of the AMI used to launch the instance. + returned: always + type: str + sample: ami-0011223344 + instance_id: + description: The ID of the instance. + returned: always + type: str + sample: i-012345678 + instance_type: + description: The instance type size of the running instance. + returned: always + type: str + sample: t2.micro + key_name: + description: The name of the key pair, if this instance was launched with an associated key pair. + returned: always + type: str + sample: my-key + launch_time: + description: The time the instance was launched. + returned: always + type: str + sample: "2017-03-23T22:51:24+00:00" + monitoring: + description: The monitoring for the instance. + returned: always + type: complex + contains: + state: + description: Indicates whether detailed monitoring is enabled. Otherwise, basic monitoring is enabled. + returned: always + type: str + sample: disabled + network_interfaces: + description: One or more network interfaces for the instance. + returned: always + type: complex + contains: + association: + description: The association information for an Elastic IPv4 associated with the network interface. + returned: always + type: complex + contains: + ip_owner_id: + description: The ID of the owner of the Elastic IP address. + returned: always + type: str + sample: amazon + public_dns_name: + description: The public DNS name. + returned: always + type: str + sample: "" + public_ip: + description: The public IP address or Elastic IP address bound to the network interface. + returned: always + type: str + sample: 1.2.3.4 + attachment: + description: The network interface attachment. + returned: always + type: complex + contains: + attach_time: + description: The time stamp when the attachment initiated. + returned: always + type: str + sample: "2017-03-23T22:51:24+00:00" + attachment_id: + description: The ID of the network interface attachment. + returned: always + type: str + sample: eni-attach-3aff3f + delete_on_termination: + description: Indicates whether the network interface is deleted when the instance is terminated. + returned: always + type: bool + sample: true + device_index: + description: The index of the device on the instance for the network interface attachment. + returned: always + type: int + sample: 0 + status: + description: The attachment state. + returned: always + type: str + sample: attached + description: + description: The description. + returned: always + type: str + sample: My interface + groups: + description: One or more security groups. + returned: always + type: list + elements: dict + contains: + group_id: + description: The ID of the security group. + returned: always + type: str + sample: sg-abcdef12 + group_name: + description: The name of the security group. + returned: always + type: str + sample: mygroup + ipv6_addresses: + description: One or more IPv6 addresses associated with the network interface. + returned: always + type: list + elements: dict + contains: + ipv6_address: + description: The IPv6 address. + returned: always + type: str + sample: "2001:0db8:85a3:0000:0000:8a2e:0370:7334" + mac_address: + description: The MAC address. + returned: always + type: str + sample: "00:11:22:33:44:55" + network_interface_id: + description: The ID of the network interface. + returned: always + type: str + sample: eni-01234567 + owner_id: + description: The AWS account ID of the owner of the network interface. + returned: always + type: str + sample: 01234567890 + private_ip_address: + description: The IPv4 address of the network interface within the subnet. + returned: always + type: str + sample: 10.0.0.1 + private_ip_addresses: + description: The private IPv4 addresses associated with the network interface. + returned: always + type: list + elements: dict + contains: + association: + description: The association information for an Elastic IP address (IPv4) associated with the network interface. + returned: always + type: complex + contains: + ip_owner_id: + description: The ID of the owner of the Elastic IP address. + returned: always + type: str + sample: amazon + public_dns_name: + description: The public DNS name. + returned: always + type: str + sample: "" + public_ip: + description: The public IP address or Elastic IP address bound to the network interface. + returned: always + type: str + sample: 1.2.3.4 + primary: + description: Indicates whether this IPv4 address is the primary private IP address of the network interface. + returned: always + type: bool + sample: true + private_ip_address: + description: The private IPv4 address of the network interface. + returned: always + type: str + sample: 10.0.0.1 + source_dest_check: + description: Indicates whether source/destination checking is enabled. + returned: always + type: bool + sample: true + status: + description: The status of the network interface. + returned: always + type: str + sample: in-use + subnet_id: + description: The ID of the subnet for the network interface. + returned: always + type: str + sample: subnet-0123456 + vpc_id: + description: The ID of the VPC for the network interface. + returned: always + type: str + sample: vpc-0123456 + placement: + description: The location where the instance launched, if applicable. + returned: always + type: complex + contains: + availability_zone: + description: The Availability Zone of the instance. + returned: always + type: str + sample: ap-southeast-2a + group_name: + description: The name of the placement group the instance is in (for cluster compute instances). + returned: always + type: str + sample: "" + tenancy: + description: The tenancy of the instance (if the instance is running in a VPC). + returned: always + type: str + sample: default + private_dns_name: + description: The private DNS name. + returned: always + type: str + sample: ip-10-0-0-1.ap-southeast-2.compute.internal + private_ip_address: + description: The IPv4 address of the network interface within the subnet. + returned: always + type: str + sample: 10.0.0.1 + product_codes: + description: One or more product codes. + returned: always + type: list + elements: dict + contains: + product_code_id: + description: The product code. + returned: always + type: str + sample: aw0evgkw8ef3n2498gndfgasdfsd5cce + product_code_type: + description: The type of product code. + returned: always + type: str + sample: marketplace + public_dns_name: + description: The public DNS name assigned to the instance. + returned: always + type: str + sample: + public_ip_address: + description: The public IPv4 address assigned to the instance + returned: always + type: str + sample: 52.0.0.1 + root_device_name: + description: The device name of the root device + returned: always + type: str + sample: /dev/sda1 + root_device_type: + description: The type of root device used by the AMI. + returned: always + type: str + sample: ebs + security_groups: + description: One or more security groups for the instance. + returned: always + type: list + elements: dict + contains: + group_id: + description: The ID of the security group. + returned: always + type: str + sample: sg-0123456 + group_name: + description: The name of the security group. + returned: always + type: str + sample: my-security-group + network.source_dest_check: + description: Indicates whether source/destination checking is enabled. + returned: always + type: bool + sample: true + state: + description: The current state of the instance. + returned: always + type: complex + contains: + code: + description: The low byte represents the state. + returned: always + type: int + sample: 16 + name: + description: The name of the state. + returned: always + type: str + sample: running + state_transition_reason: + description: The reason for the most recent state transition. + returned: always + type: str + sample: + subnet_id: + description: The ID of the subnet in which the instance is running. + returned: always + type: str + sample: subnet-00abcdef + tags: + description: Any tags assigned to the instance. + returned: always + type: dict + sample: + virtualization_type: + description: The type of virtualization of the AMI. + returned: always + type: str + sample: hvm + vpc_id: + description: The ID of the VPC the instance is in. + returned: always + type: dict + sample: vpc-0011223344 +''' + +import re +import uuid +import string +import textwrap +import time +from collections import namedtuple + +try: + import boto3 + import botocore.exceptions +except ImportError: + pass # caught by AnsibleAWSModule + +from ansible.module_utils.six import text_type, string_types +from ansible.module_utils.six.moves.urllib import parse as urlparse +from ansible.module_utils._text import to_bytes, to_native +import ansible_collections.ansible.amazon.plugins.module_utils.ec2 as ec2_utils +from ansible_collections.ansible.amazon.plugins.module_utils.ec2 import (AWSRetry, + ansible_dict_to_boto3_filter_list, + compare_aws_tags, + boto3_tag_list_to_ansible_dict, + ansible_dict_to_boto3_tag_list, + camel_dict_to_snake_dict) + +from ansible_collections.ansible.amazon.plugins.module_utils.aws.core import AnsibleAWSModule + +module = None + + +def tower_callback_script(tower_conf, windows=False, passwd=None): + script_url = 'https://raw.githubusercontent.com/ansible/ansible/devel/examples/scripts/ConfigureRemotingForAnsible.ps1' + if windows and passwd is not None: + script_tpl = """ + $admin = [adsi]("WinNT://./administrator, user") + $admin.PSBase.Invoke("SetPassword", "{PASS}") + Invoke-Expression ((New-Object System.Net.Webclient).DownloadString('{SCRIPT}')) + + """ + return to_native(textwrap.dedent(script_tpl).format(PASS=passwd, SCRIPT=script_url)) + elif windows and passwd is None: + script_tpl = """ + $admin = [adsi]("WinNT://./administrator, user") + Invoke-Expression ((New-Object System.Net.Webclient).DownloadString('{SCRIPT}')) + + """ + return to_native(textwrap.dedent(script_tpl).format(PASS=passwd, SCRIPT=script_url)) + elif not windows: + for p in ['tower_address', 'job_template_id', 'host_config_key']: + if p not in tower_conf: + module.fail_json(msg="Incomplete tower_callback configuration. tower_callback.{0} not set.".format(p)) + + if isinstance(tower_conf['job_template_id'], string_types): + tower_conf['job_template_id'] = urlparse.quote(tower_conf['job_template_id']) + tpl = string.Template(textwrap.dedent("""#!/bin/bash + set -x + + retry_attempts=10 + attempt=0 + while [[ $attempt -lt $retry_attempts ]] + do + status_code=`curl --max-time 10 -v -k -s -i \ + --data "host_config_key=${host_config_key}" \ + 'https://${tower_address}/api/v2/job_templates/${template_id}/callback/' \ + | head -n 1 \ + | awk '{print $2}'` + if [[ $status_code == 404 ]] + then + status_code=`curl --max-time 10 -v -k -s -i \ + --data "host_config_key=${host_config_key}" \ + 'https://${tower_address}/api/v1/job_templates/${template_id}/callback/' \ + | head -n 1 \ + | awk '{print $2}'` + # fall back to using V1 API for Tower 3.1 and below, since v2 API will always 404 + fi + if [[ $status_code == 201 ]] + then + exit 0 + fi + attempt=$(( attempt + 1 )) + echo "$${status_code} received... retrying in 1 minute. (Attempt $${attempt})" + sleep 60 + done + exit 1 + """)) + return tpl.safe_substitute(tower_address=tower_conf['tower_address'], + template_id=tower_conf['job_template_id'], + host_config_key=tower_conf['host_config_key']) + raise NotImplementedError("Only windows with remote-prep or non-windows with tower job callback supported so far.") + + +@AWSRetry.jittered_backoff() +def manage_tags(match, new_tags, purge_tags, ec2): + changed = False + old_tags = boto3_tag_list_to_ansible_dict(match['Tags']) + tags_to_set, tags_to_delete = compare_aws_tags( + old_tags, new_tags, + purge_tags=purge_tags, + ) + if tags_to_set: + ec2.create_tags( + Resources=[match['InstanceId']], + Tags=ansible_dict_to_boto3_tag_list(tags_to_set)) + changed |= True + if tags_to_delete: + delete_with_current_values = dict((k, old_tags.get(k)) for k in tags_to_delete) + ec2.delete_tags( + Resources=[match['InstanceId']], + Tags=ansible_dict_to_boto3_tag_list(delete_with_current_values)) + changed |= True + return changed + + +def build_volume_spec(params): + volumes = params.get('volumes') or [] + for volume in volumes: + if 'ebs' in volume: + for int_value in ['volume_size', 'iops']: + if int_value in volume['ebs']: + volume['ebs'][int_value] = int(volume['ebs'][int_value]) + return [ec2_utils.snake_dict_to_camel_dict(v, capitalize_first=True) for v in volumes] + + +def add_or_update_instance_profile(instance, desired_profile_name): + instance_profile_setting = instance.get('IamInstanceProfile') + if instance_profile_setting and desired_profile_name: + if desired_profile_name in (instance_profile_setting.get('Name'), instance_profile_setting.get('Arn')): + # great, the profile we asked for is what's there + return False + else: + desired_arn = determine_iam_role(desired_profile_name) + if instance_profile_setting.get('Arn') == desired_arn: + return False + # update association + ec2 = module.client('ec2') + try: + association = ec2.describe_iam_instance_profile_associations(Filters=[{'Name': 'instance-id', 'Values': [instance['InstanceId']]}]) + except botocore.exceptions.ClientError as e: + # check for InvalidAssociationID.NotFound + module.fail_json_aws(e, "Could not find instance profile association") + try: + resp = ec2.replace_iam_instance_profile_association( + AssociationId=association['IamInstanceProfileAssociations'][0]['AssociationId'], + IamInstanceProfile={'Arn': determine_iam_role(desired_profile_name)} + ) + return True + except botocore.exceptions.ClientError as e: + module.fail_json_aws(e, "Could not associate instance profile") + + if not instance_profile_setting and desired_profile_name: + # create association + ec2 = module.client('ec2') + try: + resp = ec2.associate_iam_instance_profile( + IamInstanceProfile={'Arn': determine_iam_role(desired_profile_name)}, + InstanceId=instance['InstanceId'] + ) + return True + except botocore.exceptions.ClientError as e: + module.fail_json_aws(e, "Could not associate new instance profile") + + return False + + +def build_network_spec(params, ec2=None): + """ + Returns list of interfaces [complex] + Interface type: { + 'AssociatePublicIpAddress': True|False, + 'DeleteOnTermination': True|False, + 'Description': 'string', + 'DeviceIndex': 123, + 'Groups': [ + 'string', + ], + 'Ipv6AddressCount': 123, + 'Ipv6Addresses': [ + { + 'Ipv6Address': 'string' + }, + ], + 'NetworkInterfaceId': 'string', + 'PrivateIpAddress': 'string', + 'PrivateIpAddresses': [ + { + 'Primary': True|False, + 'PrivateIpAddress': 'string' + }, + ], + 'SecondaryPrivateIpAddressCount': 123, + 'SubnetId': 'string' + }, + """ + if ec2 is None: + ec2 = module.client('ec2') + + interfaces = [] + network = params.get('network') or {} + if not network.get('interfaces'): + # they only specified one interface + spec = { + 'DeviceIndex': 0, + } + if network.get('assign_public_ip') is not None: + spec['AssociatePublicIpAddress'] = network['assign_public_ip'] + + if params.get('vpc_subnet_id'): + spec['SubnetId'] = params['vpc_subnet_id'] + else: + default_vpc = get_default_vpc(ec2) + if default_vpc is None: + raise module.fail_json( + msg="No default subnet could be found - you must include a VPC subnet ID (vpc_subnet_id parameter) to create an instance") + else: + sub = get_default_subnet(ec2, default_vpc) + spec['SubnetId'] = sub['SubnetId'] + + if network.get('private_ip_address'): + spec['PrivateIpAddress'] = network['private_ip_address'] + + if params.get('security_group') or params.get('security_groups'): + groups = discover_security_groups( + group=params.get('security_group'), + groups=params.get('security_groups'), + subnet_id=spec['SubnetId'], + ec2=ec2 + ) + spec['Groups'] = [g['GroupId'] for g in groups] + if network.get('description') is not None: + spec['Description'] = network['description'] + # TODO more special snowflake network things + + return [spec] + + # handle list of `network.interfaces` options + for idx, interface_params in enumerate(network.get('interfaces', [])): + spec = { + 'DeviceIndex': idx, + } + + if isinstance(interface_params, string_types): + # naive case where user gave + # network_interfaces: [eni-1234, eni-4567, ....] + # put into normal data structure so we don't dupe code + interface_params = {'id': interface_params} + + if interface_params.get('id') is not None: + # if an ID is provided, we don't want to set any other parameters. + spec['NetworkInterfaceId'] = interface_params['id'] + interfaces.append(spec) + continue + + spec['DeleteOnTermination'] = interface_params.get('delete_on_termination', True) + + if interface_params.get('ipv6_addresses'): + spec['Ipv6Addresses'] = [{'Ipv6Address': a} for a in interface_params.get('ipv6_addresses', [])] + + if interface_params.get('private_ip_address'): + spec['PrivateIpAddress'] = interface_params.get('private_ip_address') + + if interface_params.get('description'): + spec['Description'] = interface_params.get('description') + + if interface_params.get('subnet_id', params.get('vpc_subnet_id')): + spec['SubnetId'] = interface_params.get('subnet_id', params.get('vpc_subnet_id')) + elif not spec.get('SubnetId') and not interface_params['id']: + # TODO grab a subnet from default VPC + raise ValueError('Failed to assign subnet to interface {0}'.format(interface_params)) + + interfaces.append(spec) + return interfaces + + +def warn_if_public_ip_assignment_changed(instance): + # This is a non-modifiable attribute. + assign_public_ip = (module.params.get('network') or {}).get('assign_public_ip') + if assign_public_ip is None: + return + + # Check that public ip assignment is the same and warn if not + public_dns_name = instance.get('PublicDnsName') + if (public_dns_name and not assign_public_ip) or (assign_public_ip and not public_dns_name): + module.warn( + "Unable to modify public ip assignment to {0} for instance {1}. " + "Whether or not to assign a public IP is determined during instance creation.".format( + assign_public_ip, instance['InstanceId'])) + + +def warn_if_cpu_options_changed(instance): + # This is a non-modifiable attribute. + cpu_options = module.params.get('cpu_options') + if cpu_options is None: + return + + # Check that the CpuOptions set are the same and warn if not + core_count_curr = instance['CpuOptions'].get('CoreCount') + core_count = cpu_options.get('core_count') + threads_per_core_curr = instance['CpuOptions'].get('ThreadsPerCore') + threads_per_core = cpu_options.get('threads_per_core') + if core_count_curr != core_count: + module.warn( + "Unable to modify core_count from {0} to {1}. " + "Assigning a number of core is determinted during instance creation".format( + core_count_curr, core_count)) + + if threads_per_core_curr != threads_per_core: + module.warn( + "Unable to modify threads_per_core from {0} to {1}. " + "Assigning a number of threads per core is determined during instance creation.".format( + threads_per_core_curr, threads_per_core)) + + +def discover_security_groups(group, groups, parent_vpc_id=None, subnet_id=None, ec2=None): + if ec2 is None: + ec2 = module.client('ec2') + + if subnet_id is not None: + try: + sub = ec2.describe_subnets(SubnetIds=[subnet_id]) + except botocore.exceptions.ClientError as e: + if e.response['Error']['Code'] == 'InvalidGroup.NotFound': + module.fail_json( + "Could not find subnet {0} to associate security groups. Please check the vpc_subnet_id and security_groups parameters.".format( + subnet_id + ) + ) + module.fail_json_aws(e, msg="Error while searching for subnet {0} parent VPC.".format(subnet_id)) + except botocore.exceptions.BotoCoreError as e: + module.fail_json_aws(e, msg="Error while searching for subnet {0} parent VPC.".format(subnet_id)) + parent_vpc_id = sub['Subnets'][0]['VpcId'] + + vpc = { + 'Name': 'vpc-id', + 'Values': [parent_vpc_id] + } + + # because filter lists are AND in the security groups API, + # make two separate requests for groups by ID and by name + id_filters = [vpc] + name_filters = [vpc] + + if group: + name_filters.append( + dict( + Name='group-name', + Values=[group] + ) + ) + if group.startswith('sg-'): + id_filters.append( + dict( + Name='group-id', + Values=[group] + ) + ) + if groups: + name_filters.append( + dict( + Name='group-name', + Values=groups + ) + ) + if [g for g in groups if g.startswith('sg-')]: + id_filters.append( + dict( + Name='group-id', + Values=[g for g in groups if g.startswith('sg-')] + ) + ) + + found_groups = [] + for f_set in (id_filters, name_filters): + if len(f_set) > 1: + found_groups.extend(ec2.get_paginator( + 'describe_security_groups' + ).paginate( + Filters=f_set + ).search('SecurityGroups[]')) + return list(dict((g['GroupId'], g) for g in found_groups).values()) + + +def build_top_level_options(params): + spec = {} + if params.get('image_id'): + spec['ImageId'] = params['image_id'] + elif isinstance(params.get('image'), dict): + image = params.get('image', {}) + spec['ImageId'] = image.get('id') + if 'ramdisk' in image: + spec['RamdiskId'] = image['ramdisk'] + if 'kernel' in image: + spec['KernelId'] = image['kernel'] + if not spec.get('ImageId') and not params.get('launch_template'): + module.fail_json(msg="You must include an image_id or image.id parameter to create an instance, or use a launch_template.") + + if params.get('key_name') is not None: + spec['KeyName'] = params.get('key_name') + if params.get('user_data') is not None: + spec['UserData'] = to_native(params.get('user_data')) + elif params.get('tower_callback') is not None: + spec['UserData'] = tower_callback_script( + tower_conf=params.get('tower_callback'), + windows=params.get('tower_callback').get('windows', False), + passwd=params.get('tower_callback').get('set_password'), + ) + + if params.get('launch_template') is not None: + spec['LaunchTemplate'] = {} + if not params.get('launch_template').get('id') or params.get('launch_template').get('name'): + module.fail_json(msg="Could not create instance with launch template. Either launch_template.name or launch_template.id parameters are required") + + if params.get('launch_template').get('id') is not None: + spec['LaunchTemplate']['LaunchTemplateId'] = params.get('launch_template').get('id') + if params.get('launch_template').get('name') is not None: + spec['LaunchTemplate']['LaunchTemplateName'] = params.get('launch_template').get('name') + if params.get('launch_template').get('version') is not None: + spec['LaunchTemplate']['Version'] = to_native(params.get('launch_template').get('version')) + + if params.get('detailed_monitoring', False): + spec['Monitoring'] = {'Enabled': True} + if params.get('cpu_credit_specification') is not None: + spec['CreditSpecification'] = {'CpuCredits': params.get('cpu_credit_specification')} + if params.get('tenancy') is not None: + spec['Placement'] = {'Tenancy': params.get('tenancy')} + if params.get('placement_group'): + if 'Placement' in spec: + spec['Placement']['GroupName'] = str(params.get('placement_group')) + else: + spec.setdefault('Placement', {'GroupName': str(params.get('placement_group'))}) + if params.get('ebs_optimized') is not None: + spec['EbsOptimized'] = params.get('ebs_optimized') + if params.get('instance_initiated_shutdown_behavior'): + spec['InstanceInitiatedShutdownBehavior'] = params.get('instance_initiated_shutdown_behavior') + if params.get('termination_protection') is not None: + spec['DisableApiTermination'] = params.get('termination_protection') + if params.get('cpu_options') is not None: + spec['CpuOptions'] = {} + spec['CpuOptions']['ThreadsPerCore'] = params.get('cpu_options').get('threads_per_core') + spec['CpuOptions']['CoreCount'] = params.get('cpu_options').get('core_count') + return spec + + +def build_instance_tags(params, propagate_tags_to_volumes=True): + tags = params.get('tags', {}) + if params.get('name') is not None: + if tags is None: + tags = {} + tags['Name'] = params.get('name') + return [ + { + 'ResourceType': 'volume', + 'Tags': ansible_dict_to_boto3_tag_list(tags), + }, + { + 'ResourceType': 'instance', + 'Tags': ansible_dict_to_boto3_tag_list(tags), + }, + ] + + +def build_run_instance_spec(params, ec2=None): + if ec2 is None: + ec2 = module.client('ec2') + + spec = dict( + ClientToken=uuid.uuid4().hex, + MaxCount=1, + MinCount=1, + ) + # network parameters + spec['NetworkInterfaces'] = build_network_spec(params, ec2) + spec['BlockDeviceMappings'] = build_volume_spec(params) + spec.update(**build_top_level_options(params)) + spec['TagSpecifications'] = build_instance_tags(params) + + # IAM profile + if params.get('instance_role'): + spec['IamInstanceProfile'] = dict(Arn=determine_iam_role(params.get('instance_role'))) + + spec['InstanceType'] = params['instance_type'] + return spec + + +def await_instances(ids, state='OK'): + if not module.params.get('wait', True): + # the user asked not to wait for anything + return + + if module.check_mode: + # In check mode, there is no change even if you wait. + return + + state_opts = { + 'OK': 'instance_status_ok', + 'STOPPED': 'instance_stopped', + 'TERMINATED': 'instance_terminated', + 'EXISTS': 'instance_exists', + 'RUNNING': 'instance_running', + } + if state not in state_opts: + module.fail_json(msg="Cannot wait for state {0}, invalid state".format(state)) + waiter = module.client('ec2').get_waiter(state_opts[state]) + try: + waiter.wait( + InstanceIds=ids, + WaiterConfig={ + 'Delay': 15, + 'MaxAttempts': module.params.get('wait_timeout', 600) // 15, + } + ) + except botocore.exceptions.WaiterConfigError as e: + module.fail_json(msg="{0}. Error waiting for instances {1} to reach state {2}".format( + to_native(e), ', '.join(ids), state)) + except botocore.exceptions.WaiterError as e: + module.warn("Instances {0} took too long to reach state {1}. {2}".format( + ', '.join(ids), state, to_native(e))) + + +def diff_instance_and_params(instance, params, ec2=None, skip=None): + """boto3 instance obj, module params""" + if ec2 is None: + ec2 = module.client('ec2') + + if skip is None: + skip = [] + + changes_to_apply = [] + id_ = instance['InstanceId'] + + ParamMapper = namedtuple('ParamMapper', ['param_key', 'instance_key', 'attribute_name', 'add_value']) + + def value_wrapper(v): + return {'Value': v} + + param_mappings = [ + ParamMapper('ebs_optimized', 'EbsOptimized', 'ebsOptimized', value_wrapper), + ParamMapper('termination_protection', 'DisableApiTermination', 'disableApiTermination', value_wrapper), + # user data is an immutable property + # ParamMapper('user_data', 'UserData', 'userData', value_wrapper), + ] + + for mapping in param_mappings: + if params.get(mapping.param_key) is not None and mapping.instance_key not in skip: + value = AWSRetry.jittered_backoff()(ec2.describe_instance_attribute)(Attribute=mapping.attribute_name, InstanceId=id_) + if params.get(mapping.param_key) is not None and value[mapping.instance_key]['Value'] != params.get(mapping.param_key): + arguments = dict( + InstanceId=instance['InstanceId'], + # Attribute=mapping.attribute_name, + ) + arguments[mapping.instance_key] = mapping.add_value(params.get(mapping.param_key)) + changes_to_apply.append(arguments) + + if (params.get('network') or {}).get('source_dest_check') is not None: + # network.source_dest_check is nested, so needs to be treated separately + check = bool(params.get('network').get('source_dest_check')) + if instance['SourceDestCheck'] != check: + changes_to_apply.append(dict( + InstanceId=instance['InstanceId'], + SourceDestCheck={'Value': check}, + )) + + return changes_to_apply + + +def change_network_attachments(instance, params, ec2): + if (params.get('network') or {}).get('interfaces') is not None: + new_ids = [] + for inty in params.get('network').get('interfaces'): + if isinstance(inty, dict) and 'id' in inty: + new_ids.append(inty['id']) + elif isinstance(inty, string_types): + new_ids.append(inty) + # network.interfaces can create the need to attach new interfaces + old_ids = [inty['NetworkInterfaceId'] for inty in instance['NetworkInterfaces']] + to_attach = set(new_ids) - set(old_ids) + for eni_id in to_attach: + ec2.attach_network_interface( + DeviceIndex=new_ids.index(eni_id), + InstanceId=instance['InstanceId'], + NetworkInterfaceId=eni_id, + ) + return bool(len(to_attach)) + return False + + +def find_instances(ec2, ids=None, filters=None): + paginator = ec2.get_paginator('describe_instances') + if ids: + return list(paginator.paginate( + InstanceIds=ids, + ).search('Reservations[].Instances[]')) + elif filters is None: + module.fail_json(msg="No filters provided when they were required") + elif filters is not None: + for key in list(filters.keys()): + if not key.startswith("tag:"): + filters[key.replace("_", "-")] = filters.pop(key) + return list(paginator.paginate( + Filters=ansible_dict_to_boto3_filter_list(filters) + ).search('Reservations[].Instances[]')) + return [] + + +@AWSRetry.jittered_backoff() +def get_default_vpc(ec2): + vpcs = ec2.describe_vpcs(Filters=ansible_dict_to_boto3_filter_list({'isDefault': 'true'})) + if len(vpcs.get('Vpcs', [])): + return vpcs.get('Vpcs')[0] + return None + + +@AWSRetry.jittered_backoff() +def get_default_subnet(ec2, vpc, availability_zone=None): + subnets = ec2.describe_subnets( + Filters=ansible_dict_to_boto3_filter_list({ + 'vpc-id': vpc['VpcId'], + 'state': 'available', + 'default-for-az': 'true', + }) + ) + if len(subnets.get('Subnets', [])): + if availability_zone is not None: + subs_by_az = dict((subnet['AvailabilityZone'], subnet) for subnet in subnets.get('Subnets')) + if availability_zone in subs_by_az: + return subs_by_az[availability_zone] + + # to have a deterministic sorting order, we sort by AZ so we'll always pick the `a` subnet first + # there can only be one default-for-az subnet per AZ, so the AZ key is always unique in this list + by_az = sorted(subnets.get('Subnets'), key=lambda s: s['AvailabilityZone']) + return by_az[0] + return None + + +def ensure_instance_state(state, ec2=None): + if ec2 is None: + module.client('ec2') + if state in ('running', 'started'): + changed, failed, instances, failure_reason = change_instance_state(filters=module.params.get('filters'), desired_state='RUNNING') + + if failed: + module.fail_json( + msg="Unable to start instances: {0}".format(failure_reason), + reboot_success=list(changed), + reboot_failed=failed) + + module.exit_json( + msg='Instances started', + reboot_success=list(changed), + changed=bool(len(changed)), + reboot_failed=[], + instances=[pretty_instance(i) for i in instances], + ) + elif state in ('restarted', 'rebooted'): + changed, failed, instances, failure_reason = change_instance_state( + filters=module.params.get('filters'), + desired_state='STOPPED') + changed, failed, instances, failure_reason = change_instance_state( + filters=module.params.get('filters'), + desired_state='RUNNING') + + if failed: + module.fail_json( + msg="Unable to restart instances: {0}".format(failure_reason), + reboot_success=list(changed), + reboot_failed=failed) + + module.exit_json( + msg='Instances restarted', + reboot_success=list(changed), + changed=bool(len(changed)), + reboot_failed=[], + instances=[pretty_instance(i) for i in instances], + ) + elif state in ('stopped',): + changed, failed, instances, failure_reason = change_instance_state( + filters=module.params.get('filters'), + desired_state='STOPPED') + + if failed: + module.fail_json( + msg="Unable to stop instances: {0}".format(failure_reason), + stop_success=list(changed), + stop_failed=failed) + + module.exit_json( + msg='Instances stopped', + stop_success=list(changed), + changed=bool(len(changed)), + stop_failed=[], + instances=[pretty_instance(i) for i in instances], + ) + elif state in ('absent', 'terminated'): + terminated, terminate_failed, instances, failure_reason = change_instance_state( + filters=module.params.get('filters'), + desired_state='TERMINATED') + + if terminate_failed: + module.fail_json( + msg="Unable to terminate instances: {0}".format(failure_reason), + terminate_success=list(terminated), + terminate_failed=terminate_failed) + module.exit_json( + msg='Instances terminated', + terminate_success=list(terminated), + changed=bool(len(terminated)), + terminate_failed=[], + instances=[pretty_instance(i) for i in instances], + ) + + +@AWSRetry.jittered_backoff() +def change_instance_state(filters, desired_state, ec2=None): + """Takes STOPPED/RUNNING/TERMINATED""" + if ec2 is None: + ec2 = module.client('ec2') + + changed = set() + instances = find_instances(ec2, filters=filters) + to_change = set(i['InstanceId'] for i in instances if i['State']['Name'].upper() != desired_state) + unchanged = set() + failure_reason = "" + + for inst in instances: + try: + if desired_state == 'TERMINATED': + if module.check_mode: + changed.add(inst['InstanceId']) + continue + + # TODO use a client-token to prevent double-sends of these start/stop/terminate commands + # https://docs.aws.amazon.com/AWSEC2/latest/APIReference/Run_Instance_Idempotency.html + resp = ec2.terminate_instances(InstanceIds=[inst['InstanceId']]) + [changed.add(i['InstanceId']) for i in resp['TerminatingInstances']] + if desired_state == 'STOPPED': + if inst['State']['Name'] in ('stopping', 'stopped'): + unchanged.add(inst['InstanceId']) + continue + + if module.check_mode: + changed.add(inst['InstanceId']) + continue + + resp = ec2.stop_instances(InstanceIds=[inst['InstanceId']]) + [changed.add(i['InstanceId']) for i in resp['StoppingInstances']] + if desired_state == 'RUNNING': + if module.check_mode: + changed.add(inst['InstanceId']) + continue + + resp = ec2.start_instances(InstanceIds=[inst['InstanceId']]) + [changed.add(i['InstanceId']) for i in resp['StartingInstances']] + except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: + try: + failure_reason = to_native(e.message) + except AttributeError: + failure_reason = to_native(e) + + if changed: + await_instances(ids=list(changed) + list(unchanged), state=desired_state) + + change_failed = list(to_change - changed) + instances = find_instances(ec2, ids=list(i['InstanceId'] for i in instances)) + return changed, change_failed, instances, failure_reason + + +def pretty_instance(i): + instance = camel_dict_to_snake_dict(i, ignore_list=['Tags']) + instance['tags'] = boto3_tag_list_to_ansible_dict(i['Tags']) + return instance + + +def determine_iam_role(name_or_arn): + if re.match(r'^arn:aws:iam::\d+:instance-profile/[\w+=/,.@-]+$', name_or_arn): + return name_or_arn + iam = module.client('iam', retry_decorator=AWSRetry.jittered_backoff()) + try: + role = iam.get_instance_profile(InstanceProfileName=name_or_arn, aws_retry=True) + return role['InstanceProfile']['Arn'] + except botocore.exceptions.ClientError as e: + if e.response['Error']['Code'] == 'NoSuchEntity': + module.fail_json_aws(e, msg="Could not find instance_role {0}".format(name_or_arn)) + module.fail_json_aws(e, msg="An error occurred while searching for instance_role {0}. Please try supplying the full ARN.".format(name_or_arn)) + + +def handle_existing(existing_matches, changed, ec2, state): + if state in ('running', 'started') and [i for i in existing_matches if i['State']['Name'] != 'running']: + ins_changed, failed, instances, failure_reason = change_instance_state(filters=module.params.get('filters'), desired_state='RUNNING') + if failed: + module.fail_json(msg="Couldn't start instances: {0}. Failure reason: {1}".format(instances, failure_reason)) + module.exit_json( + changed=bool(len(ins_changed)) or changed, + instances=[pretty_instance(i) for i in instances], + instance_ids=[i['InstanceId'] for i in instances], + ) + changes = diff_instance_and_params(existing_matches[0], module.params) + for c in changes: + AWSRetry.jittered_backoff()(ec2.modify_instance_attribute)(**c) + changed |= bool(changes) + changed |= add_or_update_instance_profile(existing_matches[0], module.params.get('instance_role')) + changed |= change_network_attachments(existing_matches[0], module.params, ec2) + altered = find_instances(ec2, ids=[i['InstanceId'] for i in existing_matches]) + module.exit_json( + changed=bool(len(changes)) or changed, + instances=[pretty_instance(i) for i in altered], + instance_ids=[i['InstanceId'] for i in altered], + changes=changes, + ) + + +def ensure_present(existing_matches, changed, ec2, state): + if len(existing_matches): + try: + handle_existing(existing_matches, changed, ec2, state) + except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e: + module.fail_json_aws( + e, msg="Failed to handle existing instances {0}".format(', '.join([i['InstanceId'] for i in existing_matches])), + # instances=[pretty_instance(i) for i in existing_matches], + # instance_ids=[i['InstanceId'] for i in existing_matches], + ) + try: + instance_spec = build_run_instance_spec(module.params) + # If check mode is enabled,suspend 'ensure function'. + if module.check_mode: + module.exit_json( + changed=True, + spec=instance_spec, + ) + instance_response = run_instances(ec2, **instance_spec) + instances = instance_response['Instances'] + instance_ids = [i['InstanceId'] for i in instances] + + for ins in instances: + changes = diff_instance_and_params(ins, module.params, skip=['UserData', 'EbsOptimized']) + for c in changes: + try: + AWSRetry.jittered_backoff()(ec2.modify_instance_attribute)(**c) + except botocore.exceptions.ClientError as e: + module.fail_json_aws(e, msg="Could not apply change {0} to new instance.".format(str(c))) + + if not module.params.get('wait'): + module.exit_json( + changed=True, + instance_ids=instance_ids, + spec=instance_spec, + ) + await_instances(instance_ids) + instances = ec2.get_paginator('describe_instances').paginate( + InstanceIds=instance_ids + ).search('Reservations[].Instances[]') + + module.exit_json( + changed=True, + instances=[pretty_instance(i) for i in instances], + instance_ids=instance_ids, + spec=instance_spec, + ) + except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e: + module.fail_json_aws(e, msg="Failed to create new EC2 instance") + + +@AWSRetry.jittered_backoff() +def run_instances(ec2, **instance_spec): + try: + return ec2.run_instances(**instance_spec) + except botocore.exceptions.ClientError as e: + if e.response['Error']['Code'] == 'InvalidParameterValue' and "Invalid IAM Instance Profile ARN" in e.response['Error']['Message']: + # If the instance profile has just been created, it takes some time to be visible by ec2 + # So we wait 10 second and retry the run_instances + time.sleep(10) + return ec2.run_instances(**instance_spec) + else: + raise e + + +def main(): + global module + argument_spec = dict( + state=dict(default='present', choices=['present', 'started', 'running', 'stopped', 'restarted', 'rebooted', 'terminated', 'absent']), + wait=dict(default=True, type='bool'), + wait_timeout=dict(default=600, type='int'), + # count=dict(default=1, type='int'), + image=dict(type='dict'), + image_id=dict(type='str'), + instance_type=dict(default='t2.micro', type='str'), + user_data=dict(type='str'), + tower_callback=dict(type='dict'), + ebs_optimized=dict(type='bool'), + vpc_subnet_id=dict(type='str', aliases=['subnet_id']), + availability_zone=dict(type='str'), + security_groups=dict(default=[], type='list'), + security_group=dict(type='str'), + instance_role=dict(type='str'), + name=dict(type='str'), + tags=dict(type='dict'), + purge_tags=dict(type='bool', default=False), + filters=dict(type='dict', default=None), + launch_template=dict(type='dict'), + key_name=dict(type='str'), + cpu_credit_specification=dict(type='str', choices=['standard', 'unlimited']), + cpu_options=dict(type='dict', options=dict( + core_count=dict(type='int', required=True), + threads_per_core=dict(type='int', choices=[1, 2], required=True) + )), + tenancy=dict(type='str', choices=['dedicated', 'default']), + placement_group=dict(type='str'), + instance_initiated_shutdown_behavior=dict(type='str', choices=['stop', 'terminate']), + termination_protection=dict(type='bool'), + detailed_monitoring=dict(type='bool'), + instance_ids=dict(default=[], type='list'), + network=dict(default=None, type='dict'), + volumes=dict(default=None, type='list'), + ) + # running/present are synonyms + # as are terminated/absent + module = AnsibleAWSModule( + argument_spec=argument_spec, + mutually_exclusive=[ + ['security_groups', 'security_group'], + ['availability_zone', 'vpc_subnet_id'], + ['tower_callback', 'user_data'], + ['image_id', 'image'], + ], + supports_check_mode=True + ) + + if module.params.get('network'): + if module.params.get('network').get('interfaces'): + if module.params.get('security_group'): + module.fail_json(msg="Parameter network.interfaces can't be used with security_group") + if module.params.get('security_groups'): + module.fail_json(msg="Parameter network.interfaces can't be used with security_groups") + + state = module.params.get('state') + ec2 = module.client('ec2') + if module.params.get('filters') is None: + filters = { + # all states except shutting-down and terminated + 'instance-state-name': ['pending', 'running', 'stopping', 'stopped'] + } + if state == 'stopped': + # only need to change instances that aren't already stopped + filters['instance-state-name'] = ['stopping', 'pending', 'running'] + + if isinstance(module.params.get('instance_ids'), string_types): + filters['instance-id'] = [module.params.get('instance_ids')] + elif isinstance(module.params.get('instance_ids'), list) and len(module.params.get('instance_ids')): + filters['instance-id'] = module.params.get('instance_ids') + else: + if not module.params.get('vpc_subnet_id'): + if module.params.get('network'): + # grab AZ from one of the ENIs + ints = module.params.get('network').get('interfaces') + if ints: + filters['network-interface.network-interface-id'] = [] + for i in ints: + if isinstance(i, dict): + i = i['id'] + filters['network-interface.network-interface-id'].append(i) + else: + sub = get_default_subnet(ec2, get_default_vpc(ec2), availability_zone=module.params.get('availability_zone')) + filters['subnet-id'] = sub['SubnetId'] + else: + filters['subnet-id'] = [module.params.get('vpc_subnet_id')] + + if module.params.get('name'): + filters['tag:Name'] = [module.params.get('name')] + + if module.params.get('image_id'): + filters['image-id'] = [module.params.get('image_id')] + elif (module.params.get('image') or {}).get('id'): + filters['image-id'] = [module.params.get('image', {}).get('id')] + + module.params['filters'] = filters + + if module.params.get('cpu_options') and not module.botocore_at_least('1.10.16'): + module.fail_json(msg="cpu_options is only supported with botocore >= 1.10.16") + + existing_matches = find_instances(ec2, filters=module.params.get('filters')) + changed = False + + if state not in ('terminated', 'absent') and existing_matches: + for match in existing_matches: + warn_if_public_ip_assignment_changed(match) + warn_if_cpu_options_changed(match) + tags = module.params.get('tags') or {} + name = module.params.get('name') + if name: + tags['Name'] = name + changed |= manage_tags(match, tags, module.params.get('purge_tags', False), ec2) + + if state in ('present', 'running', 'started'): + ensure_present(existing_matches=existing_matches, changed=changed, ec2=ec2, state=state) + elif state in ('restarted', 'rebooted', 'stopped', 'absent', 'terminated'): + if existing_matches: + ensure_instance_state(state, ec2) + else: + module.exit_json( + msg='No matching instances found', + changed=False, + instances=[], + ) + else: + module.fail_json(msg="We don't handle the state {0}".format(state)) + + +if __name__ == '__main__': + main() diff --git a/plugins/modules/ec2_instance_info.py b/plugins/modules/ec2_instance_info.py new file mode 100644 index 00000000000..e16a3c2f164 --- /dev/null +++ b/plugins/modules/ec2_instance_info.py @@ -0,0 +1,570 @@ +#!/usr/bin/python +# Copyright: Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import absolute_import, division, print_function +__metaclass__ = type + + +ANSIBLE_METADATA = {'metadata_version': '1.1', + 'status': ['preview'], + 'supported_by': 'community'} + +DOCUMENTATION = ''' +--- +module: ec2_instance_info +short_description: Gather information about ec2 instances in AWS +description: + - Gather information about ec2 instances in AWS + - This module was called C(ec2_instance_facts) before Ansible 2.9. The usage did not change. +author: + - Michael Schuett (@michaeljs1990) + - Rob White (@wimnat) +requirements: [ "boto3", "botocore" ] +options: + instance_ids: + description: + - If you specify one or more instance IDs, only instances that have the specified IDs are returned. + required: false + type: list + filters: + description: + - A dict of filters to apply. Each dict item consists of a filter key and a filter value. See + U(https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeInstances.html) for possible filters. Filter + names and values are case sensitive. + required: false + default: {} + type: dict + +extends_documentation_fragment: +- ansible.amazon.aws +- ansible.amazon.ec2 + +''' + +EXAMPLES = ''' +# Note: These examples do not set authentication details, see the AWS Guide for details. + +# Gather information about all instances +- ec2_instance_info: + +# Gather information about all instances in AZ ap-southeast-2a +- ec2_instance_info: + filters: + availability-zone: ap-southeast-2a + +# Gather information about a particular instance using ID +- ec2_instance_info: + instance_ids: + - i-12345678 + +# Gather information about any instance with a tag key Name and value Example +- ec2_instance_info: + filters: + "tag:Name": Example + +# Gather information about any instance in states "shutting-down", "stopping", "stopped" +- ec2_instance_info: + filters: + instance-state-name: [ "shutting-down", "stopping", "stopped" ] + +''' + +RETURN = ''' +instances: + description: a list of ec2 instances + returned: always + type: complex + contains: + ami_launch_index: + description: The AMI launch index, which can be used to find this instance in the launch group. + returned: always + type: int + sample: 0 + architecture: + description: The architecture of the image + returned: always + type: str + sample: x86_64 + block_device_mappings: + description: Any block device mapping entries for the instance. + returned: always + type: complex + contains: + device_name: + description: The device name exposed to the instance (for example, /dev/sdh or xvdh). + returned: always + type: str + sample: /dev/sdh + ebs: + description: Parameters used to automatically set up EBS volumes when the instance is launched. + returned: always + type: complex + contains: + attach_time: + description: The time stamp when the attachment initiated. + returned: always + type: str + sample: "2017-03-23T22:51:24+00:00" + delete_on_termination: + description: Indicates whether the volume is deleted on instance termination. + returned: always + type: bool + sample: true + status: + description: The attachment state. + returned: always + type: str + sample: attached + volume_id: + description: The ID of the EBS volume + returned: always + type: str + sample: vol-12345678 + cpu_options: + description: The CPU options set for the instance. + returned: always if botocore version >= 1.10.16 + type: complex + contains: + core_count: + description: The number of CPU cores for the instance. + returned: always + type: int + sample: 1 + threads_per_core: + description: The number of threads per CPU core. On supported instance, a value of 1 means Intel Hyper-Threading Technology is disabled. + returned: always + type: int + sample: 1 + client_token: + description: The idempotency token you provided when you launched the instance, if applicable. + returned: always + type: str + sample: mytoken + ebs_optimized: + description: Indicates whether the instance is optimized for EBS I/O. + returned: always + type: bool + sample: false + hypervisor: + description: The hypervisor type of the instance. + returned: always + type: str + sample: xen + iam_instance_profile: + description: The IAM instance profile associated with the instance, if applicable. + returned: always + type: complex + contains: + arn: + description: The Amazon Resource Name (ARN) of the instance profile. + returned: always + type: str + sample: "arn:aws:iam::000012345678:instance-profile/myprofile" + id: + description: The ID of the instance profile + returned: always + type: str + sample: JFJ397FDG400FG9FD1N + image_id: + description: The ID of the AMI used to launch the instance. + returned: always + type: str + sample: ami-0011223344 + instance_id: + description: The ID of the instance. + returned: always + type: str + sample: i-012345678 + instance_type: + description: The instance type size of the running instance. + returned: always + type: str + sample: t2.micro + key_name: + description: The name of the key pair, if this instance was launched with an associated key pair. + returned: always + type: str + sample: my-key + launch_time: + description: The time the instance was launched. + returned: always + type: str + sample: "2017-03-23T22:51:24+00:00" + monitoring: + description: The monitoring for the instance. + returned: always + type: complex + contains: + state: + description: Indicates whether detailed monitoring is enabled. Otherwise, basic monitoring is enabled. + returned: always + type: str + sample: disabled + network_interfaces: + description: One or more network interfaces for the instance. + returned: always + type: complex + contains: + association: + description: The association information for an Elastic IPv4 associated with the network interface. + returned: always + type: complex + contains: + ip_owner_id: + description: The ID of the owner of the Elastic IP address. + returned: always + type: str + sample: amazon + public_dns_name: + description: The public DNS name. + returned: always + type: str + sample: "" + public_ip: + description: The public IP address or Elastic IP address bound to the network interface. + returned: always + type: str + sample: 1.2.3.4 + attachment: + description: The network interface attachment. + returned: always + type: complex + contains: + attach_time: + description: The time stamp when the attachment initiated. + returned: always + type: str + sample: "2017-03-23T22:51:24+00:00" + attachment_id: + description: The ID of the network interface attachment. + returned: always + type: str + sample: eni-attach-3aff3f + delete_on_termination: + description: Indicates whether the network interface is deleted when the instance is terminated. + returned: always + type: bool + sample: true + device_index: + description: The index of the device on the instance for the network interface attachment. + returned: always + type: int + sample: 0 + status: + description: The attachment state. + returned: always + type: str + sample: attached + description: + description: The description. + returned: always + type: str + sample: My interface + groups: + description: One or more security groups. + returned: always + type: list + elements: dict + contains: + group_id: + description: The ID of the security group. + returned: always + type: str + sample: sg-abcdef12 + group_name: + description: The name of the security group. + returned: always + type: str + sample: mygroup + ipv6_addresses: + description: One or more IPv6 addresses associated with the network interface. + returned: always + type: list + elements: dict + contains: + ipv6_address: + description: The IPv6 address. + returned: always + type: str + sample: "2001:0db8:85a3:0000:0000:8a2e:0370:7334" + mac_address: + description: The MAC address. + returned: always + type: str + sample: "00:11:22:33:44:55" + network_interface_id: + description: The ID of the network interface. + returned: always + type: str + sample: eni-01234567 + owner_id: + description: The AWS account ID of the owner of the network interface. + returned: always + type: str + sample: 01234567890 + private_ip_address: + description: The IPv4 address of the network interface within the subnet. + returned: always + type: str + sample: 10.0.0.1 + private_ip_addresses: + description: The private IPv4 addresses associated with the network interface. + returned: always + type: list + elements: dict + contains: + association: + description: The association information for an Elastic IP address (IPv4) associated with the network interface. + returned: always + type: complex + contains: + ip_owner_id: + description: The ID of the owner of the Elastic IP address. + returned: always + type: str + sample: amazon + public_dns_name: + description: The public DNS name. + returned: always + type: str + sample: "" + public_ip: + description: The public IP address or Elastic IP address bound to the network interface. + returned: always + type: str + sample: 1.2.3.4 + primary: + description: Indicates whether this IPv4 address is the primary private IP address of the network interface. + returned: always + type: bool + sample: true + private_ip_address: + description: The private IPv4 address of the network interface. + returned: always + type: str + sample: 10.0.0.1 + source_dest_check: + description: Indicates whether source/destination checking is enabled. + returned: always + type: bool + sample: true + status: + description: The status of the network interface. + returned: always + type: str + sample: in-use + subnet_id: + description: The ID of the subnet for the network interface. + returned: always + type: str + sample: subnet-0123456 + vpc_id: + description: The ID of the VPC for the network interface. + returned: always + type: str + sample: vpc-0123456 + placement: + description: The location where the instance launched, if applicable. + returned: always + type: complex + contains: + availability_zone: + description: The Availability Zone of the instance. + returned: always + type: str + sample: ap-southeast-2a + group_name: + description: The name of the placement group the instance is in (for cluster compute instances). + returned: always + type: str + sample: "" + tenancy: + description: The tenancy of the instance (if the instance is running in a VPC). + returned: always + type: str + sample: default + private_dns_name: + description: The private DNS name. + returned: always + type: str + sample: ip-10-0-0-1.ap-southeast-2.compute.internal + private_ip_address: + description: The IPv4 address of the network interface within the subnet. + returned: always + type: str + sample: 10.0.0.1 + product_codes: + description: One or more product codes. + returned: always + type: list + elements: dict + contains: + product_code_id: + description: The product code. + returned: always + type: str + sample: aw0evgkw8ef3n2498gndfgasdfsd5cce + product_code_type: + description: The type of product code. + returned: always + type: str + sample: marketplace + public_dns_name: + description: The public DNS name assigned to the instance. + returned: always + type: str + sample: + public_ip_address: + description: The public IPv4 address assigned to the instance + returned: always + type: str + sample: 52.0.0.1 + root_device_name: + description: The device name of the root device + returned: always + type: str + sample: /dev/sda1 + root_device_type: + description: The type of root device used by the AMI. + returned: always + type: str + sample: ebs + security_groups: + description: One or more security groups for the instance. + returned: always + type: list + elements: dict + contains: + group_id: + description: The ID of the security group. + returned: always + type: str + sample: sg-0123456 + group_name: + description: The name of the security group. + returned: always + type: str + sample: my-security-group + source_dest_check: + description: Indicates whether source/destination checking is enabled. + returned: always + type: bool + sample: true + state: + description: The current state of the instance. + returned: always + type: complex + contains: + code: + description: The low byte represents the state. + returned: always + type: int + sample: 16 + name: + description: The name of the state. + returned: always + type: str + sample: running + state_transition_reason: + description: The reason for the most recent state transition. + returned: always + type: str + sample: + subnet_id: + description: The ID of the subnet in which the instance is running. + returned: always + type: str + sample: subnet-00abcdef + tags: + description: Any tags assigned to the instance. + returned: always + type: dict + sample: + virtualization_type: + description: The type of virtualization of the AMI. + returned: always + type: str + sample: hvm + vpc_id: + description: The ID of the VPC the instance is in. + returned: always + type: dict + sample: vpc-0011223344 +''' + +import traceback + +try: + import boto3 + from botocore.exceptions import ClientError + HAS_BOTO3 = True +except ImportError: + HAS_BOTO3 = False + +from ansible.module_utils.basic import AnsibleModule +from ansible_collections.ansible.amazon.plugins.module_utils.ec2 import (ansible_dict_to_boto3_filter_list, + boto3_conn, boto3_tag_list_to_ansible_dict, camel_dict_to_snake_dict, + ec2_argument_spec, get_aws_connection_info) + + +def list_ec2_instances(connection, module): + + instance_ids = module.params.get("instance_ids") + filters = ansible_dict_to_boto3_filter_list(module.params.get("filters")) + + try: + reservations_paginator = connection.get_paginator('describe_instances') + reservations = reservations_paginator.paginate(InstanceIds=instance_ids, Filters=filters).build_full_result() + except ClientError as e: + module.fail_json(msg=e.message, exception=traceback.format_exc(), **camel_dict_to_snake_dict(e.response)) + + # Get instances from reservations + instances = [] + for reservation in reservations['Reservations']: + instances = instances + reservation['Instances'] + + # Turn the boto3 result in to ansible_friendly_snaked_names + snaked_instances = [camel_dict_to_snake_dict(instance) for instance in instances] + + # Turn the boto3 result in to ansible friendly tag dictionary + for instance in snaked_instances: + instance['tags'] = boto3_tag_list_to_ansible_dict(instance.get('tags', []), 'key', 'value') + + module.exit_json(instances=snaked_instances) + + +def main(): + + argument_spec = ec2_argument_spec() + argument_spec.update( + dict( + instance_ids=dict(default=[], type='list'), + filters=dict(default={}, type='dict') + ) + ) + + module = AnsibleModule(argument_spec=argument_spec, + mutually_exclusive=[ + ['instance_ids', 'filters'] + ], + supports_check_mode=True + ) + if module._name == 'ec2_instance_facts': + module.deprecate("The 'ec2_instance_facts' module has been renamed to 'ec2_instance_info'", version='2.13') + + if not HAS_BOTO3: + module.fail_json(msg='boto3 required for this module') + + region, ec2_url, aws_connect_params = get_aws_connection_info(module, boto3=True) + + if region: + connection = boto3_conn(module, conn_type='client', resource='ec2', region=region, endpoint=ec2_url, **aws_connect_params) + else: + module.fail_json(msg="region must be specified") + + list_ec2_instances(connection, module) + + +if __name__ == '__main__': + main() diff --git a/tests/integration/targets/ec2_instance/aliases b/tests/integration/targets/ec2_instance/aliases new file mode 100644 index 00000000000..62cb1d2c5b5 --- /dev/null +++ b/tests/integration/targets/ec2_instance/aliases @@ -0,0 +1,3 @@ +cloud/aws +shippable/aws/group3 +ec2_instance_info diff --git a/tests/integration/targets/ec2_instance/inventory b/tests/integration/targets/ec2_instance/inventory new file mode 100644 index 00000000000..44b46ec88f7 --- /dev/null +++ b/tests/integration/targets/ec2_instance/inventory @@ -0,0 +1,17 @@ +[tests] +# Sorted fastest to slowest +version_fail_wrapper +ebs_optimized +block_devices +cpu_options +default_vpc_tests +external_resource_attach +instance_no_wait +iam_instance_role +termination_protection +tags_and_vpc_settings +checkmode_tests + +[all:vars] +ansible_connection=local +ansible_python_interpreter="{{ ansible_playbook_python }}" diff --git a/tests/integration/targets/ec2_instance/main.yml b/tests/integration/targets/ec2_instance/main.yml new file mode 100644 index 00000000000..7695f7bcb92 --- /dev/null +++ b/tests/integration/targets/ec2_instance/main.yml @@ -0,0 +1,43 @@ +--- +# Beware: most of our tests here are run in parallel. +# To add new tests you'll need to add a new host to the inventory and a matching +# '{{ inventory_hostname }}'.yml file in roles/ec2_instance/tasks/ + + +# Prepare the VPC and figure out which AMI to use +- hosts: all + gather_facts: no + tasks: + - module_defaults: + group/aws: + aws_access_key: "{{ aws_access_key }}" + aws_secret_key: "{{ aws_secret_key }}" + security_token: "{{ security_token | default(omit) }}" + region: "{{ aws_region }}" + vars: + # We can't just use "run_once" because the facts don't propagate when + # running an 'include' that was run_once + setup_run_once: yes + block: + - include_role: + name: 'ec2_instance' + tasks_from: find_ami.yml + - include_role: + name: 'ec2_instance' + tasks_from: env_setup.yml + rescue: + - include_role: + name: 'ec2_instance' + tasks_from: env_cleanup.yml + run_once: yes + - fail: + msg: 'Environment preparation failed' + run_once: yes + +# VPC should get cleaned up once all hosts have run +- hosts: all + gather_facts: no + strategy: free + #serial: 10 + roles: + - ec2_instance diff --git a/tests/integration/targets/ec2_instance/meta/main.yml b/tests/integration/targets/ec2_instance/meta/main.yml new file mode 100644 index 00000000000..38b31be0728 --- /dev/null +++ b/tests/integration/targets/ec2_instance/meta/main.yml @@ -0,0 +1,4 @@ +dependencies: + - prepare_tests + - setup_ec2 + - setup_remote_tmp_dir diff --git a/tests/integration/targets/ec2_instance/roles/ec2_instance/defaults/main.yml b/tests/integration/targets/ec2_instance/roles/ec2_instance/defaults/main.yml new file mode 100644 index 00000000000..8e70ab6933c --- /dev/null +++ b/tests/integration/targets/ec2_instance/roles/ec2_instance/defaults/main.yml @@ -0,0 +1,14 @@ +--- +# defaults file for ec2_instance +ec2_instance_owner: 'integration-run-{{ resource_prefix }}' +ec2_instance_type: 't3.micro' +ec2_instance_tag_TestId: '{{ resource_prefix }}-{{ inventory_hostname }}' +ec2_ami_name: 'amzn2-ami-hvm-2.*-x86_64-gp2' + +vpc_name: '{{ resource_prefix }}-vpc' +vpc_seed: '{{ resource_prefix }}' +vpc_cidr: '10.{{ 256 | random(seed=vpc_seed) }}.0.0/16' +subnet_a_cidr: '10.{{ 256 | random(seed=vpc_seed) }}.32.0/24' +subnet_a_startswith: '10.{{ 256 | random(seed=vpc_seed) }}.32.' +subnet_b_cidr: '10.{{ 256 | random(seed=vpc_seed) }}.33.0/24' +subnet_b_startswith: '10.{{ 256 | random(seed=vpc_seed) }}.33.' diff --git a/tests/integration/targets/ec2_instance/roles/ec2_instance/files/assume-role-policy.json b/tests/integration/targets/ec2_instance/roles/ec2_instance/files/assume-role-policy.json new file mode 100644 index 00000000000..72413abdd38 --- /dev/null +++ b/tests/integration/targets/ec2_instance/roles/ec2_instance/files/assume-role-policy.json @@ -0,0 +1,13 @@ +{ + "Version": "2008-10-17", + "Statement": [ + { + "Sid": "", + "Effect": "Allow", + "Principal": { + "Service": "ec2.amazonaws.com" + }, + "Action": "sts:AssumeRole" + } + ] +} diff --git a/tests/integration/targets/ec2_instance/roles/ec2_instance/meta/main.yml b/tests/integration/targets/ec2_instance/roles/ec2_instance/meta/main.yml new file mode 100644 index 00000000000..1f64f1169a9 --- /dev/null +++ b/tests/integration/targets/ec2_instance/roles/ec2_instance/meta/main.yml @@ -0,0 +1,3 @@ +dependencies: + - prepare_tests + - setup_ec2 diff --git a/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/block_devices.yml b/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/block_devices.yml new file mode 100644 index 00000000000..0a8ab63f08b --- /dev/null +++ b/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/block_devices.yml @@ -0,0 +1,82 @@ +- block: + - name: "New instance with an extra block device" + ec2_instance: + state: present + name: "{{ resource_prefix }}-test-ebs-vols" + image_id: "{{ ec2_ami_image }}" + vpc_subnet_id: "{{ testing_subnet_b.subnet.id }}" + volumes: + - device_name: /dev/sdb + ebs: + volume_size: 20 + delete_on_termination: true + volume_type: standard + tags: + TestId: "{{ ec2_instance_tag_TestId }}" + instance_type: "{{ ec2_instance_type }}" + wait: true + register: block_device_instances + + - name: "Gather instance info" + ec2_instance_info: + filters: + "tag:Name": "{{ resource_prefix }}-test-ebs-vols" + register: block_device_instances_info + + - assert: + that: + - block_device_instances is not failed + - block_device_instances is changed + - block_device_instances_info.instances[0].block_device_mappings[0] + - block_device_instances_info.instances[0].block_device_mappings[1] + - block_device_instances_info.instances[0].block_device_mappings[1].device_name == '/dev/sdb' + + - name: "New instance with an extra block device (check mode)" + ec2_instance: + state: present + name: "{{ resource_prefix }}-test-ebs-vols-checkmode" + image_id: "{{ ec2_ami_image }}" + vpc_subnet_id: "{{ testing_subnet_b.subnet.id }}" + volumes: + - device_name: /dev/sdb + ebs: + volume_size: 20 + delete_on_termination: true + volume_type: standard + tags: + TestId: "{{ ec2_instance_tag_TestId }}" + instance_type: "{{ ec2_instance_type }}" + check_mode: yes + + - name: "fact presented ec2 instance" + ec2_instance_info: + filters: + "tag:Name": "{{ resource_prefix }}-test-ebs-vols" + "instance-state-name": "running" + register: presented_instance_fact + + - name: "fact checkmode ec2 instance" + ec2_instance_info: + filters: + "tag:Name": "{{ resource_prefix }}-test-ebs-vols-checkmode" + register: checkmode_instance_fact + + - name: "Confirm whether the check mode is working normally." + assert: + that: + - "{{ presented_instance_fact.instances | length }} > 0" + - "{{ checkmode_instance_fact.instances | length }} == 0" + + - name: "Terminate instances" + ec2_instance: + state: absent + instance_ids: "{{ block_device_instances.instance_ids }}" + + always: + - name: "Terminate block_devices instances" + ec2_instance: + state: absent + filters: + "tag:TestId": "{{ ec2_instance_tag_TestId }}" + wait: yes + ignore_errors: yes diff --git a/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/checkmode_tests.yml b/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/checkmode_tests.yml new file mode 100644 index 00000000000..b161eca636e --- /dev/null +++ b/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/checkmode_tests.yml @@ -0,0 +1,172 @@ +- block: + - name: "Make basic instance" + ec2_instance: + state: present + name: "{{ resource_prefix }}-checkmode-comparison" + image_id: "{{ ec2_ami_image }}" + security_groups: "{{ sg.group_id }}" + instance_type: "{{ ec2_instance_type }}" + vpc_subnet_id: "{{ testing_subnet_a.subnet.id }}" + wait: false + tags: + TestId: "{{ ec2_instance_tag_TestId }}" + register: basic_instance + + - name: "Make basic instance (check mode)" + ec2_instance: + state: present + name: "{{ resource_prefix }}-checkmode-comparison-checkmode" + image_id: "{{ ec2_ami_image }}" + security_groups: "{{ sg.group_id }}" + instance_type: "{{ ec2_instance_type }}" + vpc_subnet_id: "{{ testing_subnet_b.subnet.id }}" + tags: + TestId: "{{ ec2_instance_tag_TestId }}" + check_mode: yes + + - name: "fact presented ec2 instance" + ec2_instance_info: + filters: + "tag:Name": "{{ resource_prefix }}-checkmode-comparison" + register: presented_instance_fact + + - name: "fact checkmode ec2 instance" + ec2_instance_info: + filters: + "tag:Name": "{{ resource_prefix }}-checkmode-comparison-checkmode" + register: checkmode_instance_fact + + - name: "Confirm whether the check mode is working normally." + assert: + that: + - "{{ presented_instance_fact.instances | length }} > 0" + - "{{ checkmode_instance_fact.instances | length }} == 0" + + - name: "Stop instance (check mode)" + ec2_instance: + state: stopped + name: "{{ resource_prefix }}-checkmode-comparison" + vpc_subnet_id: "{{ testing_subnet_a.subnet.id }}" + tags: + TestId: "{{ ec2_instance_tag_TestId }}" + check_mode: yes + + - name: "fact ec2 instance" + ec2_instance_info: + filters: + "tag:Name": "{{ resource_prefix }}-checkmode-comparison" + register: confirm_checkmode_stopinstance_fact + + - name: "Verify that it was not stopped." + assert: + that: + - '"{{ confirm_checkmode_stopinstance_fact.instances[0].state.name }}" != "stopped"' + + - name: "Stop instance." + ec2_instance: + state: stopped + name: "{{ resource_prefix }}-checkmode-comparison" + vpc_subnet_id: "{{ testing_subnet_a.subnet.id }}" + tags: + TestId: "{{ ec2_instance_tag_TestId }}" + register: instance_stop + until: not instance_stop.failed + retries: 10 + + - name: "fact stopped ec2 instance" + ec2_instance_info: + filters: + "tag:Name": "{{ resource_prefix }}-checkmode-comparison" + register: confirm_stopinstance_fact + + - name: "Verify that it was stopped." + assert: + that: + - '"{{ confirm_stopinstance_fact.instances[0].state.name }}" in ["stopped", "stopping"]' + + - name: "Running instance in check mode." + ec2_instance: + state: running + name: "{{ resource_prefix }}-checkmode-comparison" + vpc_subnet_id: "{{ testing_subnet_a.subnet.id }}" + tags: + TestId: "{{ ec2_instance_tag_TestId }}" + check_mode: yes + + - name: "fact ec2 instance" + ec2_instance_info: + filters: + "tag:Name": "{{ resource_prefix }}-checkmode-comparison" + register: confirm_checkmode_runninginstance_fact + + - name: "Verify that it was not running." + assert: + that: + - '"{{ confirm_checkmode_runninginstance_fact.instances[0].state.name }}" != "running"' + + - name: "Running instance." + ec2_instance: + state: running + name: "{{ resource_prefix }}-checkmode-comparison" + vpc_subnet_id: "{{ testing_subnet_a.subnet.id }}" + tags: + TestId: "{{ ec2_instance_tag_TestId }}" + + - name: "fact ec2 instance." + ec2_instance_info: + filters: + "tag:Name": "{{ resource_prefix }}-checkmode-comparison" + register: confirm_runninginstance_fact + + - name: "Verify that it was running." + assert: + that: + - '"{{ confirm_runninginstance_fact.instances[0].state.name }}" == "running"' + + - name: "Terminate instance in check mode." + ec2_instance: + state: absent + name: "{{ resource_prefix }}-checkmode-comparison" + vpc_subnet_id: "{{ testing_subnet_a.subnet.id }}" + tags: + TestId: "{{ ec2_instance_tag_TestId }}" + check_mode: yes + + - name: "fact ec2 instance" + ec2_instance_info: + filters: + "tag:Name": "{{ resource_prefix }}-checkmode-comparison" + register: confirm_checkmode_terminatedinstance_fact + + - name: "Verify that it was not terminated," + assert: + that: + - '"{{ confirm_checkmode_terminatedinstance_fact.instances[0].state.name }}" != "terminated"' + + - name: "Terminate instance." + ec2_instance: + state: absent + name: "{{ resource_prefix }}-checkmode-comparison" + vpc_subnet_id: "{{ testing_subnet_a.subnet.id }}" + tags: + TestId: "{{ ec2_instance_tag_TestId }}" + + - name: "fact ec2 instance" + ec2_instance_info: + filters: + "tag:Name": "{{ resource_prefix }}-checkmode-comparison" + register: confirm_terminatedinstance_fact + + - name: "Verify that it was terminated," + assert: + that: + - '"{{ confirm_terminatedinstance_fact.instances[0].state.name }}" == "terminated"' + + always: + - name: "Terminate checkmode instances" + ec2_instance: + state: absent + filters: + "tag:TestId": "{{ ec2_instance_tag_TestId }}" + wait: yes + ignore_errors: yes diff --git a/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/cpu_options.yml b/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/cpu_options.yml new file mode 100644 index 00000000000..947011f75e1 --- /dev/null +++ b/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/cpu_options.yml @@ -0,0 +1,86 @@ +- block: + - name: "create t3.nano instance with cpu_options" + ec2_instance: + state: present + name: "{{ resource_prefix }}-test-t3nano-1-threads-per-core" + image_id: "{{ ec2_ami_image }}" + tags: + TestId: "{{ ec2_instance_tag_TestId }}" + vpc_subnet_id: "{{ testing_subnet_a.subnet.id }}" + instance_type: t3.nano + cpu_options: + core_count: 1 + threads_per_core: 1 + wait: false + register: instance_creation + + - name: "instance with cpu_options created with the right options" + assert: + that: + - instance_creation is success + - instance_creation is changed + + - name: "modify cpu_options on existing instance (warning displayed)" + ec2_instance: + state: present + name: "{{ resource_prefix }}-test-t3nano-1-threads-per-core" + image_id: "{{ ec2_ami_image }}" + tags: + TestId: "{{ ec2_instance_tag_TestId }}" + vpc_subnet_id: "{{ testing_subnet_a.subnet.id }}" + instance_type: t3.nano + cpu_options: + core_count: 1 + threads_per_core: 2 + wait: false + register: cpu_options_update + ignore_errors: yes + + - name: "fact presented ec2 instance" + ec2_instance_info: + filters: + "tag:Name": "{{ resource_prefix }}-test-t3nano-1-threads-per-core" + register: presented_instance_fact + + - name: "modify cpu_options has no effect on existing instance" + assert: + that: + - cpu_options_update is success + - cpu_options_update is not changed + - "{{ presented_instance_fact.instances | length }} > 0" + - "'{{ presented_instance_fact.instances.0.state.name }}' in ['running','pending']" + - "{{ presented_instance_fact.instances.0.cpu_options.core_count }} == 1" + - "{{ presented_instance_fact.instances.0.cpu_options.threads_per_core }} == 1" + + - name: "create t3.nano instance with cpu_options(check mode)" + ec2_instance: + name: "{{ resource_prefix }}-test-t3nano-1-threads-per-core-checkmode" + image_id: "{{ ec2_ami_image }}" + tags: + TestId: "{{ ec2_instance_tag_TestId }}" + vpc_subnet_id: "{{ testing_subnet_a.subnet.id }}" + instance_type: t3.nano + cpu_options: + core_count: 1 + threads_per_core: 1 + check_mode: yes + + - name: "fact checkmode ec2 instance" + ec2_instance_info: + filters: + "tag:Name": "{{ resource_prefix }}-test-t3nano-1-threads-per-core-checkmode" + register: checkmode_instance_fact + + - name: "Confirm existence of instance id." + assert: + that: + - "{{ checkmode_instance_fact.instances | length }} == 0" + + always: + - name: "Terminate cpu_options instances" + ec2_instance: + state: absent + filters: + "tag:TestId": "{{ ec2_instance_tag_TestId }}" + wait: yes + ignore_errors: yes diff --git a/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/default_vpc_tests.yml b/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/default_vpc_tests.yml new file mode 100644 index 00000000000..a69dfe9f866 --- /dev/null +++ b/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/default_vpc_tests.yml @@ -0,0 +1,57 @@ +- block: + - name: "Make instance in a default subnet of the VPC" + ec2_instance: + state: present + name: "{{ resource_prefix }}-test-default-vpc" + image_id: "{{ ec2_ami_image }}" + tags: + TestId: "{{ ec2_instance_tag_TestId }}" + security_group: "default" + instance_type: "{{ ec2_instance_type }}" + wait: false + register: in_default_vpc + + - name: "Make instance in a default subnet of the VPC(check mode)" + ec2_instance: + state: present + name: "{{ resource_prefix }}-test-default-vpc-checkmode" + image_id: "{{ ec2_ami_image }}" + tags: + TestId: "{{ ec2_instance_tag_TestId }}" + security_group: "default" + instance_type: "{{ ec2_instance_type }}" + check_mode: yes + + - name: "fact presented ec2 instance" + ec2_instance_info: + filters: + "tag:Name": "{{ resource_prefix }}-test-default-vpc" + register: presented_instance_fact + + - name: "fact checkmode ec2 instance" + ec2_instance_info: + filters: + "tag:Name": "{{ resource_prefix }}-test-default-vpc-checkmode" + register: checkmode_instance_fact + + - name: "Confirm whether the check mode is working normally." + assert: + that: + - "{{ presented_instance_fact.instances | length }} > 0" + - "{{ checkmode_instance_fact.instances | length }} == 0" + + - name: "Terminate instances" + ec2_instance: + state: absent + instance_ids: "{{ in_default_vpc.instance_ids }}" + tags: + TestId: "{{ ec2_instance_tag_TestId }}" + + always: + - name: "Terminate vpc_tests instances" + ec2_instance: + state: absent + filters: + "tag:TestId": "{{ ec2_instance_tag_TestId }}" + wait: yes + ignore_errors: yes diff --git a/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/ebs_optimized.yml b/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/ebs_optimized.yml new file mode 100644 index 00000000000..5bfdc086e76 --- /dev/null +++ b/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/ebs_optimized.yml @@ -0,0 +1,41 @@ +- block: + - name: "Make EBS optimized instance in the testing subnet of the test VPC" + ec2_instance: + state: present + name: "{{ resource_prefix }}-test-ebs-optimized-instance-in-vpc" + image_id: "{{ ec2_ami_image }}" + tags: + TestId: "{{ ec2_instance_tag_TestId }}" + security_groups: "{{ sg.group_id }}" + vpc_subnet_id: "{{ testing_subnet_b.subnet.id }}" + ebs_optimized: true + instance_type: t3.nano + wait: false + register: ebs_opt_in_vpc + + - name: "Get ec2 instance info" + ec2_instance_info: + filters: + "tag:Name": "{{ resource_prefix }}-test-ebs-optimized-instance-in-vpc" + register: ebs_opt_instance_info + + - name: "Assert instance is ebs_optimized" + assert: + that: + - "{{ ebs_opt_instance_info.instances.0.ebs_optimized }}" + + - name: "Terminate instances" + ec2_instance: + state: absent + instance_ids: "{{ ebs_opt_in_vpc.instance_ids }}" + tags: + TestId: "{{ ec2_instance_tag_TestId }}" + + always: + - name: "Terminate ebs_optimzed instances" + ec2_instance: + state: absent + filters: + "tag:TestId": "{{ ec2_instance_tag_TestId }}" + wait: yes + ignore_errors: yes diff --git a/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/env_cleanup.yml b/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/env_cleanup.yml new file mode 100644 index 00000000000..1b6c79e0d95 --- /dev/null +++ b/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/env_cleanup.yml @@ -0,0 +1,93 @@ +- name: "remove Instances" + ec2_instance: + state: absent + filters: + vpc-id: "{{ testing_vpc.vpc.id }}" + wait: yes + ignore_errors: yes + retries: 10 + +- name: "remove ENIs" + ec2_eni_info: + filters: + vpc-id: "{{ testing_vpc.vpc.id }}" + register: enis + +- name: "delete all ENIs" + ec2_eni: + state: absent + eni_id: "{{ item.id }}" + until: removed is not failed + with_items: "{{ enis.network_interfaces }}" + ignore_errors: yes + retries: 10 + +- name: "remove the security group" + ec2_group: + state: absent + name: "{{ resource_prefix }}-sg" + description: a security group for ansible tests + vpc_id: "{{ testing_vpc.vpc.id }}" + register: removed + until: removed is not failed + ignore_errors: yes + retries: 10 + +- name: "remove routing rules" + ec2_vpc_route_table: + state: absent + vpc_id: "{{ testing_vpc.vpc.id }}" + tags: + created: "{{ resource_prefix }}-route" + routes: + - dest: 0.0.0.0/0 + gateway_id: "{{ igw.gateway_id }}" + subnets: + - "{{ testing_subnet_a.subnet.id }}" + - "{{ testing_subnet_b.subnet.id }}" + register: removed + until: removed is not failed + ignore_errors: yes + retries: 10 + +- name: "remove internet gateway" + ec2_vpc_igw: + state: absent + vpc_id: "{{ testing_vpc.vpc.id }}" + register: removed + until: removed is not failed + ignore_errors: yes + retries: 10 + +- name: "remove subnet A" + ec2_vpc_subnet: + state: absent + vpc_id: "{{ testing_vpc.vpc.id }}" + cidr: "{{ subnet_a_cidr }}" + register: removed + until: removed is not failed + ignore_errors: yes + retries: 10 + +- name: "remove subnet B" + ec2_vpc_subnet: + state: absent + vpc_id: "{{ testing_vpc.vpc.id }}" + cidr: "{{ subnet_b_cidr }}" + register: removed + until: removed is not failed + ignore_errors: yes + retries: 10 + +- name: "remove the VPC" + ec2_vpc_net: + state: absent + name: "{{ vpc_name }}" + cidr_block: "{{ vpc_cidr }}" + tags: + Name: Ansible Testing VPC + tenancy: default + register: removed + until: removed is not failed + ignore_errors: yes + retries: 10 diff --git a/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/env_setup.yml b/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/env_setup.yml new file mode 100644 index 00000000000..6c76b7bf79f --- /dev/null +++ b/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/env_setup.yml @@ -0,0 +1,79 @@ +- run_once: '{{ setup_run_once | default("no") | bool }}' + block: + - name: "fetch AZ availability" + aws_az_info: + register: az_info + - name: "Assert that we have multiple AZs available to us" + assert: + that: az_info.availability_zones | length >= 2 + + - name: "pick AZs" + set_fact: + subnet_a_az: '{{ az_info.availability_zones[0].zone_name }}' + subnet_b_az: '{{ az_info.availability_zones[1].zone_name }}' + + - name: "Create VPC for use in testing" + ec2_vpc_net: + state: present + name: "{{ vpc_name }}" + cidr_block: "{{ vpc_cidr }}" + tags: + Name: Ansible ec2_instance Testing VPC + tenancy: default + register: testing_vpc + + - name: "Create internet gateway for use in testing" + ec2_vpc_igw: + state: present + vpc_id: "{{ testing_vpc.vpc.id }}" + register: igw + + - name: "Create default subnet in zone A" + ec2_vpc_subnet: + state: present + vpc_id: "{{ testing_vpc.vpc.id }}" + cidr: "{{ subnet_a_cidr }}" + az: "{{ subnet_a_az }}" + resource_tags: + Name: "{{ resource_prefix }}-subnet-a" + register: testing_subnet_a + + - name: "Create secondary subnet in zone B" + ec2_vpc_subnet: + state: present + vpc_id: "{{ testing_vpc.vpc.id }}" + cidr: "{{ subnet_b_cidr }}" + az: "{{ subnet_b_az }}" + resource_tags: + Name: "{{ resource_prefix }}-subnet-b" + register: testing_subnet_b + + - name: "create routing rules" + ec2_vpc_route_table: + state: present + vpc_id: "{{ testing_vpc.vpc.id }}" + tags: + created: "{{ resource_prefix }}-route" + routes: + - dest: 0.0.0.0/0 + gateway_id: "{{ igw.gateway_id }}" + subnets: + - "{{ testing_subnet_a.subnet.id }}" + - "{{ testing_subnet_b.subnet.id }}" + + - name: "create a security group with the vpc" + ec2_group: + state: present + name: "{{ resource_prefix }}-sg" + description: a security group for ansible tests + vpc_id: "{{ testing_vpc.vpc.id }}" + rules: + - proto: tcp + from_port: 22 + to_port: 22 + cidr_ip: 0.0.0.0/0 + - proto: tcp + from_port: 80 + to_port: 80 + cidr_ip: 0.0.0.0/0 + register: sg diff --git a/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/external_resource_attach.yml b/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/external_resource_attach.yml new file mode 100644 index 00000000000..2625977f416 --- /dev/null +++ b/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/external_resource_attach.yml @@ -0,0 +1,129 @@ +- block: + # Make custom ENIs and attach via the `network` parameter + - ec2_eni: + state: present + delete_on_termination: true + subnet_id: "{{ testing_subnet_b.subnet.id }}" + security_groups: + - "{{ sg.group_id }}" + register: eni_a + + - ec2_eni: + state: present + delete_on_termination: true + subnet_id: "{{ testing_subnet_b.subnet.id }}" + security_groups: + - "{{ sg.group_id }}" + register: eni_b + + - ec2_eni: + state: present + delete_on_termination: true + subnet_id: "{{ testing_subnet_b.subnet.id }}" + security_groups: + - "{{ sg.group_id }}" + register: eni_c + + - ec2_key: + name: "{{ resource_prefix }}_test_key" + + - name: "Make instance in the testing subnet created in the test VPC" + ec2_instance: + state: present + name: "{{ resource_prefix }}-test-eni-vpc" + key_name: "{{ resource_prefix }}_test_key" + network: + interfaces: + - id: "{{ eni_a.interface.id }}" + image_id: "{{ ec2_ami_image }}" + availability_zone: '{{ subnet_b_az }}' + tags: + TestId: "{{ ec2_instance_tag_TestId }}" + instance_type: "{{ ec2_instance_type }}" + wait: false + register: in_test_vpc + + - name: "Gather {{ resource_prefix }}-test-eni-vpc info" + ec2_instance_info: + filters: + "tag:Name": '{{ resource_prefix }}-test-eni-vpc' + register: in_test_vpc_instance + + - assert: + that: + - 'in_test_vpc_instance.instances.0.key_name == "{{ resource_prefix }}_test_key"' + - '(in_test_vpc_instance.instances.0.network_interfaces | length) == 1' + + - name: "Add a second interface" + ec2_instance: + state: present + name: "{{ resource_prefix }}-test-eni-vpc" + network: + interfaces: + - id: "{{ eni_a.interface.id }}" + - id: "{{ eni_b.interface.id }}" + image_id: "{{ ec2_ami_image }}" + tags: + TestId: "{{ ec2_instance_tag_TestId }}" + instance_type: "{{ ec2_instance_type }}" + wait: false + register: add_interface + until: add_interface is not failed + ignore_errors: yes + retries: 10 + + - name: "Make instance in the testing subnet created in the test VPC(check mode)" + ec2_instance: + state: present + name: "{{ resource_prefix }}-test-eni-vpc-checkmode" + key_name: "{{ resource_prefix }}_test_key" + network: + interfaces: + - id: "{{ eni_c.interface.id }}" + image_id: "{{ ec2_ami_image }}" + availability_zone: '{{ subnet_b_az }}' + tags: + TestId: "{{ ec2_instance_tag_TestId }}" + instance_type: "{{ ec2_instance_type }}" + check_mode: yes + + - name: "fact presented ec2 instance" + ec2_instance_info: + filters: + "tag:Name": "{{ resource_prefix }}-test-eni-vpc" + register: presented_instance_fact + + - name: "fact checkmode ec2 instance" + ec2_instance_info: + filters: + "tag:Name": "{{ resource_prefix }}-test-eni-vpc-checkmode" + register: checkmode_instance_fact + + - name: "Confirm existence of instance id." + assert: + that: + - "{{ presented_instance_fact.instances | length }} > 0" + - "{{ checkmode_instance_fact.instances | length }} == 0" + + always: + - name: "Terminate external_resource_attach instances" + ec2_instance: + state: absent + filters: + "tag:TestId": "{{ ec2_instance_tag_TestId }}" + wait: yes + ignore_errors: yes + + - ec2_key: + state: absent + name: "{{ resource_prefix }}_test_key" + ignore_errors: yes + + - ec2_eni: + state: absent + eni_id: '{{ item.interface.id }}' + ignore_errors: yes + with_items: + - '{{ eni_a }}' + - '{{ eni_b }}' + - '{{ eni_c }}' diff --git a/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/find_ami.yml b/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/find_ami.yml new file mode 100644 index 00000000000..5c0e61f84c6 --- /dev/null +++ b/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/find_ami.yml @@ -0,0 +1,15 @@ +- run_once: '{{ setup_run_once | default("no") | bool }}' + block: + - name: "Find AMI to use" + run_once: yes + ec2_ami_info: + owners: 'amazon' + filters: + name: '{{ ec2_ami_name }}' + register: ec2_amis + - name: "Set fact with latest AMI" + run_once: yes + vars: + latest_ami: '{{ ec2_amis.images | sort(attribute="creation_date") | last }}' + set_fact: + ec2_ami_image: '{{ latest_ami.image_id }}' diff --git a/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/iam_instance_role.yml b/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/iam_instance_role.yml new file mode 100644 index 00000000000..6e29b74674f --- /dev/null +++ b/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/iam_instance_role.yml @@ -0,0 +1,127 @@ +- block: + - name: "Create IAM role for test" + iam_role: + state: present + name: "ansible-test-sts-{{ resource_prefix }}-test-policy" + assume_role_policy_document: "{{ lookup('file','assume-role-policy.json') }}" + create_instance_profile: yes + managed_policy: + - AmazonEC2ContainerServiceRole + register: iam_role + + - name: "Create second IAM role for test" + iam_role: + state: present + name: "ansible-test-sts-{{ resource_prefix }}-test-policy-2" + assume_role_policy_document: "{{ lookup('file','assume-role-policy.json') }}" + create_instance_profile: yes + managed_policy: + - AmazonEC2ContainerServiceRole + register: iam_role_2 + + - name: "wait 10 seconds for roles to become available" + wait_for: + timeout: 10 + delegate_to: localhost + + - name: "Make instance with an instance_role" + ec2_instance: + state: present + name: "{{ resource_prefix }}-test-instance-role" + image_id: "{{ ec2_ami_image }}" + security_groups: "{{ sg.group_id }}" + instance_type: "{{ ec2_instance_type }}" + instance_role: "ansible-test-sts-{{ resource_prefix }}-test-policy" + vpc_subnet_id: "{{ testing_subnet_a.subnet.id }}" + tags: + TestId: "{{ ec2_instance_tag_TestId }}" + register: instance_with_role + + - assert: + that: + - 'instance_with_role.instances[0].iam_instance_profile.arn == iam_role.arn.replace(":role/", ":instance-profile/")' + + - name: "Make instance with an instance_role(check mode)" + ec2_instance: + state: present + name: "{{ resource_prefix }}-test-instance-role-checkmode" + image_id: "{{ ec2_ami_image }}" + security_groups: "{{ sg.group_id }}" + instance_type: "{{ ec2_instance_type }}" + instance_role: "{{ iam_role.arn.replace(':role/', ':instance-profile/') }}" + vpc_subnet_id: "{{ testing_subnet_a.subnet.id }}" + tags: + TestId: "{{ ec2_instance_tag_TestId }}" + check_mode: yes + + - name: "fact presented ec2 instance" + ec2_instance_info: + filters: + "tag:Name": "{{ resource_prefix }}-test-instance-role" + register: presented_instance_fact + + - name: "fact checkmode ec2 instance" + ec2_instance_info: + filters: + "tag:Name": "{{ resource_prefix }}-test-instance-role-checkmode" + register: checkmode_instance_fact + + - name: "Confirm whether the check mode is working normally." + assert: + that: + - "{{ presented_instance_fact.instances | length }} > 0" + - "{{ checkmode_instance_fact.instances | length }} == 0" + + - name: "Update instance with new instance_role" + ec2_instance: + state: present + name: "{{ resource_prefix }}-test-instance-role" + image_id: "{{ ec2_ami_image }}" + security_groups: "{{ sg.group_id }}" + instance_type: "{{ ec2_instance_type }}" + instance_role: "{{ iam_role_2.arn.replace(':role/', ':instance-profile/') }}" + vpc_subnet_id: "{{ testing_subnet_a.subnet.id }}" + tags: + TestId: "{{ ec2_instance_tag_TestId }}" + register: instance_with_updated_role + + - name: "wait 10 seconds for role update to complete" + wait_for: + timeout: 10 + delegate_to: localhost + + - name: "fact checkmode ec2 instance" + ec2_instance_info: + filters: + "tag:Name": "{{ resource_prefix }}-test-instance-role" + register: updates_instance_info + + - assert: + that: + - 'updates_instance_info.instances[0].iam_instance_profile.arn == iam_role_2.arn.replace(":role/", ":instance-profile/")' + - 'updates_instance_info.instances[0].instance_id == instance_with_role.instances[0].instance_id' + + always: + - name: "Terminate iam_instance_role instances" + ec2_instance: + state: absent + filters: + "tag:TestId": "{{ ec2_instance_tag_TestId }}" + wait: yes + ignore_errors: yes + + - name: "Delete IAM role for test" + iam_role: + state: absent + name: "{{ item }}" + assume_role_policy_document: "{{ lookup('file','assume-role-policy.json') }}" + create_instance_profile: yes + managed_policy: + - AmazonEC2ContainerServiceRole + loop: + - "ansible-test-sts-{{ resource_prefix }}-test-policy" + - "ansible-test-sts-{{ resource_prefix }}-test-policy-2" + register: removed + until: removed is not failed + ignore_errors: yes + retries: 10 diff --git a/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/instance_no_wait.yml b/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/instance_no_wait.yml new file mode 100644 index 00000000000..418d7ef3e82 --- /dev/null +++ b/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/instance_no_wait.yml @@ -0,0 +1,68 @@ +- block: + - name: "New instance and don't wait for it to complete" + ec2_instance: + state: present + name: "{{ resource_prefix }}-test-no-wait" + image_id: "{{ ec2_ami_image }}" + vpc_subnet_id: "{{ testing_subnet_b.subnet.id }}" + tags: + TestId: "{{ ec2_instance_tag_TestId }}" + wait: false + instance_type: "{{ ec2_instance_type }}" + register: in_test_vpc + + - assert: + that: + - in_test_vpc is not failed + - in_test_vpc is changed + - in_test_vpc.instances is not defined + - in_test_vpc.instance_ids is defined + - in_test_vpc.instance_ids | length > 0 + + - name: "New instance and don't wait for it to complete ( check mode )" + ec2_instance: + state: present + name: "{{ resource_prefix }}-test-no-wait-checkmode" + image_id: "{{ ec2_ami_image }}" + vpc_subnet_id: "{{ testing_subnet_b.subnet.id }}" + tags: + TestId: "{{ ec2_instance_tag_TestId }}" + wait: false + instance_type: "{{ ec2_instance_type }}" + check_mode: yes + + - name: "Facts for ec2 test instance" + ec2_instance_info: + filters: + "tag:Name": "{{ resource_prefix }}-test-no-wait" + register: real_instance_fact + until: real_instance_fact.instances | length > 0 + retries: 10 + + - name: "Facts for checkmode ec2 test instance" + ec2_instance_info: + filters: + "tag:Name": "{{ resource_prefix }}-test-no-wait-checkmode" + register: checkmode_instance_fact + + - name: "Confirm whether the check mode is working normally." + assert: + that: + - "{{ real_instance_fact.instances | length }} > 0" + - "{{ checkmode_instance_fact.instances | length }} == 0" + + - name: "Terminate instances" + ec2_instance: + state: absent + instance_ids: "{{ in_test_vpc.instance_ids }}" + tags: + TestId: "{{ ec2_instance_tag_TestId }}" + + always: + - name: "Terminate instance_no_wait instances" + ec2_instance: + state: absent + filters: + "tag:TestId": "{{ ec2_instance_tag_TestId }}" + wait: yes + ignore_errors: yes diff --git a/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/main.yml b/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/main.yml new file mode 100644 index 00000000000..e10aebcefe2 --- /dev/null +++ b/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/main.yml @@ -0,0 +1,48 @@ +--- +# Beware: most of our tests here are run in parallel. +# To add new tests you'll need to add a new host to the inventory and a matching +# '{{ inventory_hostname }}'.yml file in roles/ec2_instance/tasks/ +# +# Please make sure you tag your instances with +# tags: +# "tag:TestId": "{{ ec2_instance_tag_TestId }}" +# And delete them based off that tag at the end of your specific set of tests +# +# ############################################################################### +# +# A Note about ec2 environment variable name preference: +# - EC2_URL -> AWS_URL +# - EC2_ACCESS_KEY -> AWS_ACCESS_KEY_ID -> AWS_ACCESS_KEY +# - EC2_SECRET_KEY -> AWS_SECRET_ACCESS_KEY -> AWX_SECRET_KEY +# - EC2_REGION -> AWS_REGION +# + +- name: "Wrap up all tests and setup AWS credentials" + module_defaults: + group/aws: + aws_access_key: "{{ aws_access_key }}" + aws_secret_key: "{{ aws_secret_key }}" + security_token: "{{ security_token | default(omit) }}" + region: "{{ aws_region }}" + block: + - debug: + msg: "{{ inventory_hostname }} start: {{ lookup('pipe','date') }}" + - include_tasks: '{{ inventory_hostname }}.yml' + - debug: + msg: "{{ inventory_hostname }} finish: {{ lookup('pipe','date') }}" + + always: + - set_fact: + _role_complete: True + - vars: + completed_hosts: '{{ ansible_play_hosts_all | map("extract", hostvars, "_role_complete") | list | select("defined") | list | length }}' + hosts_in_play: '{{ ansible_play_hosts_all | length }}' + debug: + msg: "{{ completed_hosts }} of {{ hosts_in_play }} complete" + - include_tasks: env_cleanup.yml + vars: + completed_hosts: '{{ ansible_play_hosts_all | map("extract", hostvars, "_role_complete") | list | select("defined") | list | length }}' + hosts_in_play: '{{ ansible_play_hosts_all | length }}' + when: + - aws_cleanup + - completed_hosts == hosts_in_play diff --git a/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/tags_and_vpc_settings.yml b/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/tags_and_vpc_settings.yml new file mode 100644 index 00000000000..d38b53f76fb --- /dev/null +++ b/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/tags_and_vpc_settings.yml @@ -0,0 +1,158 @@ +- block: + - name: "Make instance in the testing subnet created in the test VPC" + ec2_instance: + state: present + name: "{{ resource_prefix }}-test-basic-vpc-create" + image_id: "{{ ec2_ami_image }}" + user_data: | + #cloud-config + package_upgrade: true + package_update: true + tags: + TestId: "{{ ec2_instance_tag_TestId }}" + Something: else + security_groups: "{{ sg.group_id }}" + network: + source_dest_check: false + vpc_subnet_id: "{{ testing_subnet_b.subnet.id }}" + instance_type: "{{ ec2_instance_type }}" + wait: false + register: in_test_vpc + + - name: "Make instance in the testing subnet created in the test VPC(check mode)" + ec2_instance: + state: present + name: "{{ resource_prefix }}-test-basic-vpc-create-checkmode" + image_id: "{{ ec2_ami_image }}" + user_data: | + #cloud-config + package_upgrade: true + package_update: true + tags: + TestId: "{{ ec2_instance_tag_TestId }}" + Something: else + security_groups: "{{ sg.group_id }}" + network: + source_dest_check: false + vpc_subnet_id: "{{ testing_subnet_b.subnet.id }}" + instance_type: "{{ ec2_instance_type }}" + check_mode: yes + + - name: "Try to re-make the instance, hopefully this shows changed=False" + ec2_instance: + state: present + name: "{{ resource_prefix }}-test-basic-vpc-create" + image_id: "{{ ec2_ami_image }}" + user_data: | + #cloud-config + package_upgrade: true + package_update: true + tags: + TestId: "{{ ec2_instance_tag_TestId }}" + Something: else + security_groups: "{{ sg.group_id }}" + vpc_subnet_id: "{{ testing_subnet_b.subnet.id }}" + instance_type: "{{ ec2_instance_type }}" + register: remake_in_test_vpc + - name: "Remaking the same instance resulted in no changes" + assert: + that: not remake_in_test_vpc.changed + - name: "check that instance IDs match anyway" + assert: + that: 'remake_in_test_vpc.instance_ids[0] == in_test_vpc.instance_ids[0]' + - name: "check that source_dest_check was set to false" + assert: + that: 'not remake_in_test_vpc.instances[0].source_dest_check' + + - name: "fact presented ec2 instance" + ec2_instance_info: + filters: + "tag:Name": "{{ resource_prefix }}-test-basic-vpc-create" + register: presented_instance_fact + + - name: "fact checkmode ec2 instance" + ec2_instance_info: + filters: + "tag:Name": "{{ resource_prefix }}-test-basic-vpc-create-checkmode" + register: checkmode_instance_fact + + - name: "Confirm whether the check mode is working normally." + assert: + that: + - "{{ presented_instance_fact.instances | length }} > 0" + - "{{ checkmode_instance_fact.instances | length }} == 0" + + - name: "Alter it by adding tags" + ec2_instance: + state: present + name: "{{ resource_prefix }}-test-basic-vpc-create" + image_id: "{{ ec2_ami_image }}" + tags: + TestId: "{{ ec2_instance_tag_TestId }}" + Another: thing + security_groups: "{{ sg.group_id }}" + vpc_subnet_id: "{{ testing_subnet_b.subnet.id }}" + instance_type: "{{ ec2_instance_type }}" + register: add_another_tag + + - ec2_instance_info: + instance_ids: "{{ add_another_tag.instance_ids }}" + register: check_tags + - name: "Remaking the same instance resulted in no changes" + assert: + that: + - check_tags.instances[0].tags.Another == 'thing' + - check_tags.instances[0].tags.Something == 'else' + + - name: "Purge a tag" + ec2_instance: + state: present + name: "{{ resource_prefix }}-test-basic-vpc-create" + image_id: "{{ ec2_ami_image }}" + purge_tags: true + tags: + TestId: "{{ ec2_instance_tag_TestId }}" + Another: thing + security_groups: "{{ sg.group_id }}" + vpc_subnet_id: "{{ testing_subnet_b.subnet.id }}" + instance_type: "{{ ec2_instance_type }}" + + - ec2_instance_info: + instance_ids: "{{ add_another_tag.instance_ids }}" + register: check_tags + + - name: "Remaking the same instance resulted in no changes" + assert: + that: + - "'Something' not in check_tags.instances[0].tags" + + - name: "check that subnet-default public IP rule was followed" + assert: + that: + - check_tags.instances[0].public_dns_name == "" + - check_tags.instances[0].private_ip_address.startswith(subnet_b_startswith) + - check_tags.instances[0].subnet_id == testing_subnet_b.subnet.id + - name: "check that tags were applied" + assert: + that: + - check_tags.instances[0].tags.Name.startswith(resource_prefix) + - "'{{ check_tags.instances[0].state.name }}' in ['pending', 'running']" + + - name: "Terminate instance" + ec2_instance: + state: absent + filters: + "tag:TestId": "{{ ec2_instance_tag_TestId }}" + wait: false + register: result + - assert: + that: result.changed + + always: + - name: "Terminate tags_and_vpc_settings instances" + ec2_instance: + state: absent + filters: + "tag:TestId": "{{ ec2_instance_tag_TestId }}" + wait: yes + ignore_errors: yes diff --git a/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/termination_protection.yml b/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/termination_protection.yml new file mode 100644 index 00000000000..418e3c398dc --- /dev/null +++ b/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/termination_protection.yml @@ -0,0 +1,184 @@ +- block: + + - name: Create instance with termination protection (check mode) + ec2_instance: + name: "{{ resource_prefix }}-termination-protection" + image_id: "{{ ec2_ami_image }}" + tags: + TestId: "{{ resource_prefix }}" + security_groups: "{{ sg.group_id }}" + vpc_subnet_id: "{{ testing_subnet_b.subnet.id }}" + termination_protection: true + instance_type: "{{ ec2_instance_type }}" + state: running + wait: yes + check_mode: yes + register: create_instance_check_mode_results + + - name: Check the returned value for the earlier task + assert: + that: + - "{{ create_instance_check_mode_results.changed }}" + - "{{ create_instance_check_mode_results.spec.DisableApiTermination }}" + + - name: Create instance with termination protection + ec2_instance: + name: "{{ resource_prefix }}-termination-protection" + image_id: "{{ ec2_ami_image }}" + tags: + TestId: "{{ resource_prefix }}" + security_groups: "{{ sg.group_id }}" + vpc_subnet_id: "{{ testing_subnet_b.subnet.id }}" + termination_protection: true + instance_type: "{{ ec2_instance_type }}" + state: running + wait: yes + register: create_instance_results + + - name: Check return values of the create instance task + assert: + that: + - "{{ create_instance_results.instances | length }} > 0" + - "'{{ create_instance_results.instances.0.state.name }}' == 'running'" + - "'{{ create_instance_results.spec.DisableApiTermination }}'" + + - name: Create instance with termination protection (check mode) (idempotent) + ec2_instance: + name: "{{ resource_prefix }}-termination-protection" + image_id: "{{ ec2_ami_image }}" + tags: + TestId: "{{ resource_prefix }}" + security_groups: "{{ sg.group_id }}" + vpc_subnet_id: "{{ testing_subnet_b.subnet.id }}" + termination_protection: true + instance_type: "{{ ec2_instance_type }}" + state: running + wait: yes + check_mode: yes + register: create_instance_check_mode_results + + - name: Check the returned value for the earlier task + assert: + that: + - "{{ not create_instance_check_mode_results.changed }}" + + - name: Create instance with termination protection (idempotent) + ec2_instance: + name: "{{ resource_prefix }}-termination-protection" + image_id: "{{ ec2_ami_image }}" + tags: + TestId: "{{ resource_prefix }}" + security_groups: "{{ sg.group_id }}" + vpc_subnet_id: "{{ testing_subnet_b.subnet.id }}" + termination_protection: true + instance_type: "{{ ec2_instance_type }}" + state: running + wait: yes + register: create_instance_results + + - name: Check return values of the create instance task + assert: + that: + - "{{ not create_instance_results.changed }}" + - "{{ create_instance_results.instances | length }} > 0" + + - name: Try to terminate the instance (expected to fail) + ec2_instance: + filters: + tag:Name: "{{ resource_prefix }}-termination-protection" + state: absent + failed_when: "'Unable to terminate instances' not in terminate_instance_results.msg" + register: terminate_instance_results + + # https://github.com/ansible/ansible/issues/67716 + # Updates to termination protection in check mode has a bug (listed above) + + - name: Set termination protection to false + ec2_instance: + name: "{{ resource_prefix }}-termination-protection" + image_id: "{{ ec2_ami_image }}" + termination_protection: false + instance_type: "{{ ec2_instance_type }}" + vpc_subnet_id: "{{ testing_subnet_b.subnet.id }}" + register: set_termination_protection_results + + - name: Check return value + assert: + that: + - "{{ set_termination_protection_results.changed }}" + - "{{ not set_termination_protection_results.changes[0].DisableApiTermination.Value }}" + + - name: Set termination protection to false (idempotent) + ec2_instance: + name: "{{ resource_prefix }}-termination-protection" + image_id: "{{ ec2_ami_image }}" + termination_protection: false + instance_type: "{{ ec2_instance_type }}" + vpc_subnet_id: "{{ testing_subnet_b.subnet.id }}" + register: set_termination_protection_results + + - name: Check return value + assert: + that: + - "{{ not set_termination_protection_results.changed }}" + + - name: Set termination protection to true + ec2_instance: + name: "{{ resource_prefix }}-termination-protection" + image_id: "{{ ec2_ami_image }}" + termination_protection: true + instance_type: "{{ ec2_instance_type }}" + vpc_subnet_id: "{{ testing_subnet_b.subnet.id }}" + register: set_termination_protection_results + + - name: Check return value + assert: + that: + - "{{ set_termination_protection_results.changed }}" + - "{{ set_termination_protection_results.changes[0].DisableApiTermination.Value }}" + + - name: Set termination protection to true (idempotent) + ec2_instance: + name: "{{ resource_prefix }}-termination-protection" + image_id: "{{ ec2_ami_image }}" + termination_protection: true + instance_type: "{{ ec2_instance_type }}" + vpc_subnet_id: "{{ testing_subnet_b.subnet.id }}" + register: set_termination_protection_results + + - name: Check return value + assert: + that: + - "{{ not set_termination_protection_results.changed }}" + + - name: Set termination protection to false (so we can terminate instance) + ec2_instance: + name: "{{ resource_prefix }}-termination-protection" + image_id: "{{ ec2_ami_image }}" + termination_protection: false + instance_type: "{{ ec2_instance_type }}" + vpc_subnet_id: "{{ testing_subnet_b.subnet.id }}" + register: set_termination_protection_results + + - name: Terminate the instance + ec2_instance: + filters: + tag:TestId: "{{ resource_prefix }}" + state: absent + + always: + + - name: Set termination protection to false (so we can terminate instance) (cleanup) + ec2_instance: + filters: + tag:TestId: "{{ resource_prefix }}" + termination_protection: false + ignore_errors: yes + + - name: Terminate instance + ec2_instance: + filters: + tag:TestId: "{{ resource_prefix }}" + state: absent + wait: false + ignore_errors: yes diff --git a/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/version_fail.yml b/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/version_fail.yml new file mode 100644 index 00000000000..67370ebe37c --- /dev/null +++ b/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/version_fail.yml @@ -0,0 +1,29 @@ +- block: + - name: "create t3.nano with cpu options (fails gracefully)" + ec2_instance: + state: present + name: "ansible-test-{{ resource_prefix | regex_search('([0-9]+)$') }}-ec2" + image_id: "{{ ec2_ami_image }}" + instance_type: "t3.nano" + cpu_options: + core_count: 1 + threads_per_core: 1 + tags: + TestId: "{{ ec2_instance_tag_TestId }}" + register: ec2_instance_cpu_options_creation + ignore_errors: yes + + - name: "check that graceful error message is returned when creation with cpu_options and old botocore" + assert: + that: + - ec2_instance_cpu_options_creation.failed + - 'ec2_instance_cpu_options_creation.msg == "cpu_options is only supported with botocore >= 1.10.16"' + + always: + - name: "Terminate version_fail instances" + ec2_instance: + state: absent + filters: + "tag:TestId": "{{ ec2_instance_tag_TestId }}" + wait: yes + ignore_errors: yes diff --git a/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/version_fail_wrapper.yml b/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/version_fail_wrapper.yml new file mode 100644 index 00000000000..ae5bd785003 --- /dev/null +++ b/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/version_fail_wrapper.yml @@ -0,0 +1,30 @@ +--- +- include_role: + name: 'setup_remote_tmp_dir' + +- set_fact: + virtualenv: "{{ remote_tmp_dir }}/virtualenv" + virtualenv_command: "{{ ansible_python_interpreter }} -m virtualenv" + +- set_fact: + virtualenv_interpreter: "{{ virtualenv }}/bin/python" + +- pip: + name: "virtualenv" + +- pip: + name: + - 'botocore<1.10.16' + - boto3 + - coverage + virtualenv: "{{ virtualenv }}" + virtualenv_command: "{{ virtualenv_command }}" + virtualenv_site_packages: no + +- include_tasks: version_fail.yml + vars: + ansible_python_interpreter: "{{ virtualenv_interpreter }}" + +- file: + state: absent + path: "{{ virtualenv }}" diff --git a/tests/integration/targets/ec2_instance/runme.sh b/tests/integration/targets/ec2_instance/runme.sh new file mode 100755 index 00000000000..aa324772bbe --- /dev/null +++ b/tests/integration/targets/ec2_instance/runme.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# +# Beware: most of our tests here are run in parallel. +# To add new tests you'll need to add a new host to the inventory and a matching +# '{{ inventory_hostname }}'.yml file in roles/ec2_instance/tasks/ + + +set -eux + +export ANSIBLE_ROLES_PATH=../ + +ansible-playbook main.yml -i inventory "$@" From e0d5ec35e63880d12ef8e69f0b8fda7eb22c083e Mon Sep 17 00:00:00 2001 From: jillr Date: Tue, 3 Mar 2020 19:43:21 +0000 Subject: [PATCH 03/38] migration test cleanup --- plugins/modules/ec2_instance.py | 11 ++++++----- plugins/modules/ec2_instance_info.py | 8 ++++++-- .../ec2_instance/roles/ec2_instance/meta/main.yml | 2 ++ .../ec2_instance/roles/ec2_instance/tasks/main.yml | 2 ++ 4 files changed, 16 insertions(+), 7 deletions(-) diff --git a/plugins/modules/ec2_instance.py b/plugins/modules/ec2_instance.py index ea7f49c5f32..ca090c13d7c 100644 --- a/plugins/modules/ec2_instance.py +++ b/plugins/modules/ec2_instance.py @@ -813,11 +813,12 @@ from ansible.module_utils._text import to_bytes, to_native import ansible_collections.ansible.amazon.plugins.module_utils.ec2 as ec2_utils from ansible_collections.ansible.amazon.plugins.module_utils.ec2 import (AWSRetry, - ansible_dict_to_boto3_filter_list, - compare_aws_tags, - boto3_tag_list_to_ansible_dict, - ansible_dict_to_boto3_tag_list, - camel_dict_to_snake_dict) + ansible_dict_to_boto3_filter_list, + compare_aws_tags, + boto3_tag_list_to_ansible_dict, + ansible_dict_to_boto3_tag_list, + camel_dict_to_snake_dict, + ) from ansible_collections.ansible.amazon.plugins.module_utils.aws.core import AnsibleAWSModule diff --git a/plugins/modules/ec2_instance_info.py b/plugins/modules/ec2_instance_info.py index e16a3c2f164..865b7d70d06 100644 --- a/plugins/modules/ec2_instance_info.py +++ b/plugins/modules/ec2_instance_info.py @@ -504,8 +504,12 @@ from ansible.module_utils.basic import AnsibleModule from ansible_collections.ansible.amazon.plugins.module_utils.ec2 import (ansible_dict_to_boto3_filter_list, - boto3_conn, boto3_tag_list_to_ansible_dict, camel_dict_to_snake_dict, - ec2_argument_spec, get_aws_connection_info) + boto3_conn, + boto3_tag_list_to_ansible_dict, + camel_dict_to_snake_dict, + ec2_argument_spec, + get_aws_connection_info, + ) def list_ec2_instances(connection, module): diff --git a/tests/integration/targets/ec2_instance/roles/ec2_instance/meta/main.yml b/tests/integration/targets/ec2_instance/roles/ec2_instance/meta/main.yml index 1f64f1169a9..2545bb5c99a 100644 --- a/tests/integration/targets/ec2_instance/roles/ec2_instance/meta/main.yml +++ b/tests/integration/targets/ec2_instance/roles/ec2_instance/meta/main.yml @@ -1,3 +1,5 @@ dependencies: - prepare_tests - setup_ec2 +collections: + - ansible.amazon diff --git a/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/main.yml b/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/main.yml index e10aebcefe2..d7ae6d0a3bf 100644 --- a/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/main.yml +++ b/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/main.yml @@ -24,6 +24,8 @@ aws_secret_key: "{{ aws_secret_key }}" security_token: "{{ security_token | default(omit) }}" region: "{{ aws_region }}" + collections: + - ansible.amazon block: - debug: msg: "{{ inventory_hostname }} start: {{ lookup('pipe','date') }}" From 154bffa1f06169c3787d0d802ccd5c2719f9d8db Mon Sep 17 00:00:00 2001 From: Jill R <4121322+jillr@users.noreply.github.com> Date: Wed, 25 Mar 2020 15:39:40 -0700 Subject: [PATCH 04/38] Rename collection (#12) * Rename core collection Rename references to ansible.amazon to amazon.aws. * Rename community.amazon to community.aws Fix pep8 line lengths for rewritten amazon.aws imports * Missed a path in shippable.sh * Dependency repos moved --- plugins/modules/ec2_instance.py | 24 +++++++++---------- plugins/modules/ec2_instance_info.py | 18 +++++++------- .../roles/ec2_instance/meta/main.yml | 2 +- .../roles/ec2_instance/tasks/main.yml | 2 +- 4 files changed, 23 insertions(+), 23 deletions(-) diff --git a/plugins/modules/ec2_instance.py b/plugins/modules/ec2_instance.py index ca090c13d7c..0b268a6f05a 100644 --- a/plugins/modules/ec2_instance.py +++ b/plugins/modules/ec2_instance.py @@ -278,8 +278,8 @@ type: str extends_documentation_fragment: -- ansible.amazon.aws -- ansible.amazon.ec2 +- amazon.aws.aws +- amazon.aws.ec2 ''' @@ -811,16 +811,16 @@ from ansible.module_utils.six import text_type, string_types from ansible.module_utils.six.moves.urllib import parse as urlparse from ansible.module_utils._text import to_bytes, to_native -import ansible_collections.ansible.amazon.plugins.module_utils.ec2 as ec2_utils -from ansible_collections.ansible.amazon.plugins.module_utils.ec2 import (AWSRetry, - ansible_dict_to_boto3_filter_list, - compare_aws_tags, - boto3_tag_list_to_ansible_dict, - ansible_dict_to_boto3_tag_list, - camel_dict_to_snake_dict, - ) - -from ansible_collections.ansible.amazon.plugins.module_utils.aws.core import AnsibleAWSModule +import ansible_collections.amazon.aws.plugins.module_utils.ec2 as ec2_utils +from ansible_collections.amazon.aws.plugins.module_utils.ec2 import (AWSRetry, + ansible_dict_to_boto3_filter_list, + compare_aws_tags, + boto3_tag_list_to_ansible_dict, + ansible_dict_to_boto3_tag_list, + camel_dict_to_snake_dict, + ) + +from ansible_collections.amazon.aws.plugins.module_utils.aws.core import AnsibleAWSModule module = None diff --git a/plugins/modules/ec2_instance_info.py b/plugins/modules/ec2_instance_info.py index 865b7d70d06..9bb1ff56e7d 100644 --- a/plugins/modules/ec2_instance_info.py +++ b/plugins/modules/ec2_instance_info.py @@ -37,8 +37,8 @@ type: dict extends_documentation_fragment: -- ansible.amazon.aws -- ansible.amazon.ec2 +- amazon.aws.aws +- amazon.aws.ec2 ''' @@ -503,13 +503,13 @@ HAS_BOTO3 = False from ansible.module_utils.basic import AnsibleModule -from ansible_collections.ansible.amazon.plugins.module_utils.ec2 import (ansible_dict_to_boto3_filter_list, - boto3_conn, - boto3_tag_list_to_ansible_dict, - camel_dict_to_snake_dict, - ec2_argument_spec, - get_aws_connection_info, - ) +from ansible_collections.amazon.aws.plugins.module_utils.ec2 import (ansible_dict_to_boto3_filter_list, + boto3_conn, + boto3_tag_list_to_ansible_dict, + camel_dict_to_snake_dict, + ec2_argument_spec, + get_aws_connection_info, + ) def list_ec2_instances(connection, module): diff --git a/tests/integration/targets/ec2_instance/roles/ec2_instance/meta/main.yml b/tests/integration/targets/ec2_instance/roles/ec2_instance/meta/main.yml index 2545bb5c99a..77589cc2b48 100644 --- a/tests/integration/targets/ec2_instance/roles/ec2_instance/meta/main.yml +++ b/tests/integration/targets/ec2_instance/roles/ec2_instance/meta/main.yml @@ -2,4 +2,4 @@ dependencies: - prepare_tests - setup_ec2 collections: - - ansible.amazon + - amazon.aws diff --git a/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/main.yml b/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/main.yml index d7ae6d0a3bf..188d97d2e9f 100644 --- a/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/main.yml +++ b/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/main.yml @@ -25,7 +25,7 @@ security_token: "{{ security_token | default(omit) }}" region: "{{ aws_region }}" collections: - - ansible.amazon + - amazon.aws block: - debug: msg: "{{ inventory_hostname }} start: {{ lookup('pipe','date') }}" From 7580f5221da6ae0dfd545f9787ecbb86dc611cd1 Mon Sep 17 00:00:00 2001 From: Matt Clay Date: Fri, 27 Mar 2020 14:58:08 -0700 Subject: [PATCH 05/38] Use `coverage<5` in integration tests. (#13) Coverage versions 5 and later are not supported by ansible-test. --- .../roles/ec2_instance/tasks/version_fail_wrapper.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/version_fail_wrapper.yml b/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/version_fail_wrapper.yml index ae5bd785003..4513ae71119 100644 --- a/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/version_fail_wrapper.yml +++ b/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/version_fail_wrapper.yml @@ -16,7 +16,7 @@ name: - 'botocore<1.10.16' - boto3 - - coverage + - coverage<5 virtualenv: "{{ virtualenv }}" virtualenv_command: "{{ virtualenv_command }}" virtualenv_site_packages: no From 35e1adbcb6ab1459b1ecc1d6086049a87da65d3a Mon Sep 17 00:00:00 2001 From: Jill R <4121322+jillr@users.noreply.github.com> Date: Tue, 19 May 2020 16:06:12 -0700 Subject: [PATCH 06/38] Remove METADATA and cleanup galaxy.yml (#70) * Remove ANSIBLE_METADATA entirely, see ansible/ansible/pull/69454. Remove `license` field from galaxy.yml, in favor of `license_file`. --- plugins/modules/ec2_instance.py | 4 ---- plugins/modules/ec2_instance_info.py | 4 ---- 2 files changed, 8 deletions(-) diff --git a/plugins/modules/ec2_instance.py b/plugins/modules/ec2_instance.py index 0b268a6f05a..4238a7c15e7 100644 --- a/plugins/modules/ec2_instance.py +++ b/plugins/modules/ec2_instance.py @@ -6,10 +6,6 @@ __metaclass__ = type -ANSIBLE_METADATA = {'metadata_version': '1.1', - 'status': ['preview'], - 'supported_by': 'community'} - DOCUMENTATION = ''' --- module: ec2_instance diff --git a/plugins/modules/ec2_instance_info.py b/plugins/modules/ec2_instance_info.py index 9bb1ff56e7d..79d056d4ea6 100644 --- a/plugins/modules/ec2_instance_info.py +++ b/plugins/modules/ec2_instance_info.py @@ -6,10 +6,6 @@ __metaclass__ = type -ANSIBLE_METADATA = {'metadata_version': '1.1', - 'status': ['preview'], - 'supported_by': 'community'} - DOCUMENTATION = ''' --- module: ec2_instance_info From dc6ffefaa930ccabc1e8944660c46d152a199adf Mon Sep 17 00:00:00 2001 From: Jill R <4121322+jillr@users.noreply.github.com> Date: Tue, 16 Jun 2020 11:23:52 -0700 Subject: [PATCH 07/38] Collections related fixes for CI (#96) * Update module deprecations Switch version to `removed_at_date` * Don't install amazon.aws from galaxy We've been using galaxy to install amazon.aws in shippable, but that doesn't really work if we aren't publising faster. Get that collection from git so it is most up to date. * We need to declare python test deps now * missed a python dep --- plugins/modules/ec2_instance_info.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/modules/ec2_instance_info.py b/plugins/modules/ec2_instance_info.py index 79d056d4ea6..d2da8b96b6f 100644 --- a/plugins/modules/ec2_instance_info.py +++ b/plugins/modules/ec2_instance_info.py @@ -551,7 +551,7 @@ def main(): supports_check_mode=True ) if module._name == 'ec2_instance_facts': - module.deprecate("The 'ec2_instance_facts' module has been renamed to 'ec2_instance_info'", version='2.13') + module.deprecate("The 'ec2_instance_facts' module has been renamed to 'ec2_instance_info'", date='2021-12-01', collection_name='community.aws') if not HAS_BOTO3: module.fail_json(msg='boto3 required for this module') From 790826257a70ef568ec061b8bb852842afa35ff1 Mon Sep 17 00:00:00 2001 From: Abhijeet Kasurde Date: Wed, 17 Jun 2020 01:24:54 +0530 Subject: [PATCH 08/38] Update Examples with FQCN (#67) Updated module examples with FQCN Signed-off-by: Abhijeet Kasurde --- plugins/modules/ec2_instance.py | 40 ++++++++++++++-------------- plugins/modules/ec2_instance_info.py | 20 +++++++------- 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/plugins/modules/ec2_instance.py b/plugins/modules/ec2_instance.py index 4238a7c15e7..8a682c56e12 100644 --- a/plugins/modules/ec2_instance.py +++ b/plugins/modules/ec2_instance.py @@ -14,7 +14,7 @@ - Create and manage AWS EC2 instances. - > Note: This module does not support creating - L(EC2 Spot instances,https://aws.amazon.com/ec2/spot/). The M(ec2) module + L(EC2 Spot instances,https://aws.amazon.com/ec2/spot/). The M(amazon.aws.ec2) module can create and manage spot instances. author: - Ryan Scott Brown (@ryansb) @@ -82,7 +82,7 @@ type: bool image: description: - - An image to use for the instance. The M(ec2_ami_info) module may be used to retrieve images. + - An image to use for the instance. The M(amazon.aws.ec2_ami_info) module may be used to retrieve images. One of I(image) or I(image_id) are required when instance is not already present. type: dict suboptions: @@ -117,14 +117,14 @@ vpc_subnet_id: description: - The subnet ID in which to launch the instance (VPC) - If none is provided, ec2_instance will chose the default zone of the default VPC. + If none is provided, M(community.aws.ec2_instance) will chose the default zone of the default VPC. aliases: ['subnet_id'] type: str network: description: - Either a dictionary containing the key 'interfaces' corresponding to a list of network interface IDs or containing specifications for a single network interface. - - Use the ec2_eni module to create ENIs with special settings. + - Use the M(amazon.aws.ec2_eni) module to create ENIs with special settings. type: dict suboptions: interfaces: @@ -282,20 +282,20 @@ EXAMPLES = ''' # Note: These examples do not set authentication details, see the AWS Guide for details. -# Terminate every running instance in a region. Use with EXTREME caution. -- ec2_instance: +- name: Terminate every running instance in a region. Use with EXTREME caution. + community.aws.ec2_instance: state: absent filters: instance-state-name: running -# restart a particular instance by its ID -- ec2_instance: +- name: restart a particular instance by its ID + community.aws.ec2_instance: state: restarted instance_ids: - i-12345678 -# start an instance with a public IP address -- ec2_instance: +- name: start an instance with a public IP address + community.aws.ec2_instance: name: "public-compute-instance" key_name: "prod-ssh-key" vpc_subnet_id: subnet-5ca1ab1e @@ -307,8 +307,8 @@ tags: Environment: Testing -# start an instance and Add EBS -- ec2_instance: +- name: start an instance and Add EBS + community.aws.ec2_instance: name: "public-withebs-instance" vpc_subnet_id: subnet-5ca1ab1e instance_type: t2.micro @@ -320,8 +320,8 @@ volume_size: 16 delete_on_termination: true -# start an instance with a cpu_options -- ec2_instance: +- name: start an instance with a cpu_options + community.aws.ec2_instance: name: "public-cpuoption-instance" vpc_subnet_id: subnet-5ca1ab1e tags: @@ -335,8 +335,8 @@ core_count: 1 threads_per_core: 1 -# start an instance and have it begin a Tower callback on boot -- ec2_instance: +- name: start an instance and have it begin a Tower callback on boot + community.aws.ec2_instance: name: "tower-callback-test" key_name: "prod-ssh-key" vpc_subnet_id: subnet-5ca1ab1e @@ -353,8 +353,8 @@ tags: SomeThing: "A value" -# start an instance with ENI (An existing ENI ID is required) -- ec2_instance: +- name: start an instance with ENI (An existing ENI ID is required) + community.aws.ec2_instance: name: "public-eni-instance" key_name: "prod-ssh-key" vpc_subnet_id: subnet-5ca1ab1e @@ -370,8 +370,8 @@ instance_type: t2.micro image_id: ami-123456 -# add second ENI interface -- ec2_instance: +- name: add second ENI interface + community.aws.ec2_instance: name: "public-eni-instance" network: interfaces: diff --git a/plugins/modules/ec2_instance_info.py b/plugins/modules/ec2_instance_info.py index d2da8b96b6f..e94aaa74b21 100644 --- a/plugins/modules/ec2_instance_info.py +++ b/plugins/modules/ec2_instance_info.py @@ -41,26 +41,26 @@ EXAMPLES = ''' # Note: These examples do not set authentication details, see the AWS Guide for details. -# Gather information about all instances -- ec2_instance_info: +- name: Gather information about all instances + community.aws.ec2_instance_info: -# Gather information about all instances in AZ ap-southeast-2a -- ec2_instance_info: +- name: Gather information about all instances in AZ ap-southeast-2a + community.aws.ec2_instance_info: filters: availability-zone: ap-southeast-2a -# Gather information about a particular instance using ID -- ec2_instance_info: +- name: Gather information about a particular instance using ID + community.aws.ec2_instance_info: instance_ids: - i-12345678 -# Gather information about any instance with a tag key Name and value Example -- ec2_instance_info: +- name: Gather information about any instance with a tag key Name and value Example + community.aws.ec2_instance_info: filters: "tag:Name": Example -# Gather information about any instance in states "shutting-down", "stopping", "stopped" -- ec2_instance_info: +- name: Gather information about any instance in states "shutting-down", "stopping", "stopped" + community.aws.ec2_instance_info: filters: instance-state-name: [ "shutting-down", "stopping", "stopped" ] From 3ebea9820e6cce316033ad50bf0a24c42ab0e11f Mon Sep 17 00:00:00 2001 From: flowerysong Date: Tue, 16 Jun 2020 19:30:00 -0400 Subject: [PATCH 09/38] Update module_utils paths to remove aws subdir (#23) Co-authored-by: Ezekiel Hendrickson --- plugins/modules/ec2_instance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/modules/ec2_instance.py b/plugins/modules/ec2_instance.py index 8a682c56e12..912fa7cbe72 100644 --- a/plugins/modules/ec2_instance.py +++ b/plugins/modules/ec2_instance.py @@ -816,7 +816,7 @@ camel_dict_to_snake_dict, ) -from ansible_collections.amazon.aws.plugins.module_utils.aws.core import AnsibleAWSModule +from ansible_collections.amazon.aws.plugins.module_utils.core import AnsibleAWSModule module = None From 2caba195094fc637275f90513d8ae16bb84c2fc2 Mon Sep 17 00:00:00 2001 From: Jill R <4121322+jillr@users.noreply.github.com> Date: Wed, 17 Jun 2020 09:31:32 -0700 Subject: [PATCH 10/38] Update docs (#99) * Update docs Remove .git from repo url so links in readme will generate correctly Add required ansible version Run latest version of add_docs.py Add version_added string to modules * galaxy.yml was missing authors --- plugins/modules/ec2_instance.py | 1 + plugins/modules/ec2_instance_info.py | 1 + 2 files changed, 2 insertions(+) diff --git a/plugins/modules/ec2_instance.py b/plugins/modules/ec2_instance.py index 912fa7cbe72..9382659f71b 100644 --- a/plugins/modules/ec2_instance.py +++ b/plugins/modules/ec2_instance.py @@ -9,6 +9,7 @@ DOCUMENTATION = ''' --- module: ec2_instance +version_added: 1.0.0 short_description: Create & manage EC2 instances description: - Create and manage AWS EC2 instances. diff --git a/plugins/modules/ec2_instance_info.py b/plugins/modules/ec2_instance_info.py index e94aaa74b21..8883be6923d 100644 --- a/plugins/modules/ec2_instance_info.py +++ b/plugins/modules/ec2_instance_info.py @@ -9,6 +9,7 @@ DOCUMENTATION = ''' --- module: ec2_instance_info +version_added: 1.0.0 short_description: Gather information about ec2 instances in AWS description: - Gather information about ec2 instances in AWS From 384aa84b0c613aee0e7ee93e6b7cbb4a6f8413de Mon Sep 17 00:00:00 2001 From: Abhijeet Kasurde Date: Thu, 16 Jul 2020 01:31:41 +0530 Subject: [PATCH 11/38] Docs: sanity fixes (#133) Signed-off-by: Abhijeet Kasurde --- plugins/modules/ec2_instance.py | 11 +++++++---- plugins/modules/ec2_instance_info.py | 9 +++++---- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/plugins/modules/ec2_instance.py b/plugins/modules/ec2_instance.py index 9382659f71b..bbaa092bd5c 100644 --- a/plugins/modules/ec2_instance.py +++ b/plugins/modules/ec2_instance.py @@ -6,7 +6,7 @@ __metaclass__ = type -DOCUMENTATION = ''' +DOCUMENTATION = r''' --- module: ec2_instance version_added: 1.0.0 @@ -25,6 +25,7 @@ description: - If you specify one or more instance IDs, only instances that have the specified IDs are returned. type: list + elements: str state: description: - Goal state for the instances. @@ -107,6 +108,7 @@ description: - A list of security group IDs or names (strings). Mutually exclusive with I(security_group). type: list + elements: str security_group: description: - A security group ID or name. Mutually exclusive with I(security_groups). @@ -180,6 +182,7 @@ ebs.iops, and ebs.delete_on_termination. - For more information about each parameter, see U(https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_BlockDeviceMapping.html). type: list + elements: dict launch_template: description: - The EC2 launch template to base instance configuration on. @@ -1681,7 +1684,7 @@ def main(): ebs_optimized=dict(type='bool'), vpc_subnet_id=dict(type='str', aliases=['subnet_id']), availability_zone=dict(type='str'), - security_groups=dict(default=[], type='list'), + security_groups=dict(default=[], type='list', elements='str'), security_group=dict(type='str'), instance_role=dict(type='str'), name=dict(type='str'), @@ -1700,9 +1703,9 @@ def main(): instance_initiated_shutdown_behavior=dict(type='str', choices=['stop', 'terminate']), termination_protection=dict(type='bool'), detailed_monitoring=dict(type='bool'), - instance_ids=dict(default=[], type='list'), + instance_ids=dict(default=[], type='list', elements='str'), network=dict(default=None, type='dict'), - volumes=dict(default=None, type='list'), + volumes=dict(default=None, type='list', elements='dict'), ) # running/present are synonyms # as are terminated/absent diff --git a/plugins/modules/ec2_instance_info.py b/plugins/modules/ec2_instance_info.py index 8883be6923d..c9820a58f59 100644 --- a/plugins/modules/ec2_instance_info.py +++ b/plugins/modules/ec2_instance_info.py @@ -6,7 +6,7 @@ __metaclass__ = type -DOCUMENTATION = ''' +DOCUMENTATION = r''' --- module: ec2_instance_info version_added: 1.0.0 @@ -24,6 +24,7 @@ - If you specify one or more instance IDs, only instances that have the specified IDs are returned. required: false type: list + elements: str filters: description: - A dict of filters to apply. Each dict item consists of a filter key and a filter value. See @@ -39,7 +40,7 @@ ''' -EXAMPLES = ''' +EXAMPLES = r''' # Note: These examples do not set authentication details, see the AWS Guide for details. - name: Gather information about all instances @@ -67,7 +68,7 @@ ''' -RETURN = ''' +RETURN = r''' instances: description: a list of ec2 instances returned: always @@ -540,7 +541,7 @@ def main(): argument_spec = ec2_argument_spec() argument_spec.update( dict( - instance_ids=dict(default=[], type='list'), + instance_ids=dict(default=[], type='list', elements='str'), filters=dict(default={}, type='dict') ) ) From 5f66e43f69e2f34d7e6445327d930466f39206f2 Mon Sep 17 00:00:00 2001 From: Mark Chappell Date: Wed, 12 Aug 2020 13:06:35 +0200 Subject: [PATCH 12/38] Bulk migration to AnsibleAWSModule (#173) * Update comments to reference AnsibleAWSModule rather than AnsibleModule * Bulk re-order imports and split onto one from import per-line. * Add AnsibleAWSModule imports * Migrate boto 2 based modules to AnsibleAWSModule * Move boto3-only modules over to AnsibleAWSModule * Remove extra ec2_argument_spec calls - not needed now we're using AnsibleAWSModule * Remove most HAS_BOTO3 code, it's handled by AnsibleAWSModule * Handle missing Boto 2 consistently (HAS_BOTO) * Remove AnsibleModule imports * Changelog fragment --- plugins/modules/ec2_instance_info.py | 42 +++++++++++----------------- 1 file changed, 17 insertions(+), 25 deletions(-) diff --git a/plugins/modules/ec2_instance_info.py b/plugins/modules/ec2_instance_info.py index c9820a58f59..707df983c1b 100644 --- a/plugins/modules/ec2_instance_info.py +++ b/plugins/modules/ec2_instance_info.py @@ -496,18 +496,15 @@ try: import boto3 from botocore.exceptions import ClientError - HAS_BOTO3 = True except ImportError: - HAS_BOTO3 = False + pass # Handled by AnsibleAWSModule -from ansible.module_utils.basic import AnsibleModule -from ansible_collections.amazon.aws.plugins.module_utils.ec2 import (ansible_dict_to_boto3_filter_list, - boto3_conn, - boto3_tag_list_to_ansible_dict, - camel_dict_to_snake_dict, - ec2_argument_spec, - get_aws_connection_info, - ) +from ansible_collections.amazon.aws.plugins.module_utils.core import AnsibleAWSModule +from ansible_collections.amazon.aws.plugins.module_utils.ec2 import ansible_dict_to_boto3_filter_list +from ansible_collections.amazon.aws.plugins.module_utils.ec2 import boto3_conn +from ansible_collections.amazon.aws.plugins.module_utils.ec2 import boto3_tag_list_to_ansible_dict +from ansible_collections.amazon.aws.plugins.module_utils.ec2 import camel_dict_to_snake_dict +from ansible_collections.amazon.aws.plugins.module_utils.ec2 import get_aws_connection_info def list_ec2_instances(connection, module): @@ -538,26 +535,21 @@ def list_ec2_instances(connection, module): def main(): - argument_spec = ec2_argument_spec() - argument_spec.update( - dict( - instance_ids=dict(default=[], type='list', elements='str'), - filters=dict(default={}, type='dict') - ) + argument_spec = dict( + instance_ids=dict(default=[], type='list', elements='str'), + filters=dict(default={}, type='dict') ) - module = AnsibleModule(argument_spec=argument_spec, - mutually_exclusive=[ - ['instance_ids', 'filters'] - ], - supports_check_mode=True - ) + module = AnsibleAWSModule( + argument_spec=argument_spec, + mutually_exclusive=[ + ['instance_ids', 'filters'] + ], + supports_check_mode=True, + ) if module._name == 'ec2_instance_facts': module.deprecate("The 'ec2_instance_facts' module has been renamed to 'ec2_instance_info'", date='2021-12-01', collection_name='community.aws') - if not HAS_BOTO3: - module.fail_json(msg='boto3 required for this module') - region, ec2_url, aws_connect_params = get_aws_connection_info(module, boto3=True) if region: From 65674dca8237076d7701d4a66bde3f72afb32700 Mon Sep 17 00:00:00 2001 From: flowerysong Date: Sat, 15 Aug 2020 07:02:41 -0400 Subject: [PATCH 13/38] ec2_instance: Fix spurious error message when we lose a race (#7) It is possible for all instances to stop matching the filters between the initial check for existing instances and the first call to find_instances() in change_instance_state(). If this happened, find_instances() would previously be called a second time with an empty list of instance IDs and no filters, which should not happen and immediately ends module execution with the error "No filters provided when they were required". --- plugins/modules/ec2_instance.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/plugins/modules/ec2_instance.py b/plugins/modules/ec2_instance.py index bbaa092bd5c..595cac73157 100644 --- a/plugins/modules/ec2_instance.py +++ b/plugins/modules/ec2_instance.py @@ -1556,7 +1556,9 @@ def change_instance_state(filters, desired_state, ec2=None): await_instances(ids=list(changed) + list(unchanged), state=desired_state) change_failed = list(to_change - changed) - instances = find_instances(ec2, ids=list(i['InstanceId'] for i in instances)) + + if instances: + instances = find_instances(ec2, ids=list(i['InstanceId'] for i in instances)) return changed, change_failed, instances, failure_reason From 258206f21f71ea5970394acc5f9fd9dc5bc88fa4 Mon Sep 17 00:00:00 2001 From: Vincent Vinet Date: Sat, 15 Aug 2020 09:11:59 -0400 Subject: [PATCH 14/38] =?UTF-8?q?Python=203=20compatibility=20error=20hand?= =?UTF-8?q?ling:=20use=20to=5Fnative(e)=20instead=20of=20str(e)=20or=20e.m?= =?UTF-8?q?e=E2=80=A6=20(#26)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Py3 compat error handling: use to_native(e) instead of str(e) or e.message * PR comment changes, use fail_json_aws and is_boto3_error_code --- plugins/modules/ec2_instance_info.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/modules/ec2_instance_info.py b/plugins/modules/ec2_instance_info.py index 707df983c1b..88a07d05f61 100644 --- a/plugins/modules/ec2_instance_info.py +++ b/plugins/modules/ec2_instance_info.py @@ -516,7 +516,7 @@ def list_ec2_instances(connection, module): reservations_paginator = connection.get_paginator('describe_instances') reservations = reservations_paginator.paginate(InstanceIds=instance_ids, Filters=filters).build_full_result() except ClientError as e: - module.fail_json(msg=e.message, exception=traceback.format_exc(), **camel_dict_to_snake_dict(e.response)) + module.fail_json_aws(e, msg="Failed to list ec2 instances") # Get instances from reservations instances = [] From 4cdfe56c782477e0625279dde272965f4cf8642b Mon Sep 17 00:00:00 2001 From: Josh Date: Sun, 16 Aug 2020 09:41:00 -0400 Subject: [PATCH 15/38] Bugfix/ec2 instance mod sgs (#22) Fixes #54174 * Added SG handling for existing instances + some cleanup * tests(ec2_instance): Tests for SG modifications to existing instances * tests(ec2_instance): Test simultaneous state and SG changes * refactor(ec2_instance): Move security out of for loop * style(ec2_instance): Update fail message to reflect security groups * Add changelog Co-authored-by: Andrea Tartaglia Co-authored-by: Mark Chappell --- plugins/modules/ec2_instance.py | 54 ++++++++++--- .../targets/ec2_instance/inventory | 1 + .../roles/ec2_instance/tasks/env_cleanup.yml | 11 +++ .../roles/ec2_instance/tasks/env_setup.yml | 16 ++++ .../ec2_instance/tasks/security_group.yml | 81 +++++++++++++++++++ 5 files changed, 152 insertions(+), 11 deletions(-) create mode 100644 tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/security_group.yml diff --git a/plugins/modules/ec2_instance.py b/plugins/modules/ec2_instance.py index 595cac73157..ddedd379573 100644 --- a/plugins/modules/ec2_instance.py +++ b/plugins/modules/ec2_instance.py @@ -808,9 +808,9 @@ except ImportError: pass # caught by AnsibleAWSModule -from ansible.module_utils.six import text_type, string_types +from ansible.module_utils.six import string_types from ansible.module_utils.six.moves.urllib import parse as urlparse -from ansible.module_utils._text import to_bytes, to_native +from ansible.module_utils._text import to_native import ansible_collections.amazon.aws.plugins.module_utils.ec2 as ec2_utils from ansible_collections.amazon.aws.plugins.module_utils.ec2 import (AWSRetry, ansible_dict_to_boto3_filter_list, @@ -1337,15 +1337,47 @@ def value_wrapper(v): ] for mapping in param_mappings: - if params.get(mapping.param_key) is not None and mapping.instance_key not in skip: - value = AWSRetry.jittered_backoff()(ec2.describe_instance_attribute)(Attribute=mapping.attribute_name, InstanceId=id_) - if params.get(mapping.param_key) is not None and value[mapping.instance_key]['Value'] != params.get(mapping.param_key): - arguments = dict( - InstanceId=instance['InstanceId'], - # Attribute=mapping.attribute_name, - ) - arguments[mapping.instance_key] = mapping.add_value(params.get(mapping.param_key)) - changes_to_apply.append(arguments) + if params.get(mapping.param_key) is None: + continue + if mapping.instance_key in skip: + continue + + value = AWSRetry.jittered_backoff()(ec2.describe_instance_attribute)(Attribute=mapping.attribute_name, InstanceId=id_) + if value[mapping.instance_key]['Value'] != params.get(mapping.param_key): + arguments = dict( + InstanceId=instance['InstanceId'], + # Attribute=mapping.attribute_name, + ) + arguments[mapping.instance_key] = mapping.add_value(params.get(mapping.param_key)) + changes_to_apply.append(arguments) + + if params.get('security_group') or params.get('security_groups'): + value = AWSRetry.jittered_backoff()(ec2.describe_instance_attribute)(Attribute="groupSet", InstanceId=id_) + # managing security groups + if params.get('vpc_subnet_id'): + subnet_id = params.get('vpc_subnet_id') + else: + default_vpc = get_default_vpc(ec2) + if default_vpc is None: + module.fail_json( + msg="No default subnet could be found - you must include a VPC subnet ID (vpc_subnet_id parameter) to modify security groups.") + else: + sub = get_default_subnet(ec2, default_vpc) + subnet_id = sub['SubnetId'] + + groups = discover_security_groups( + group=params.get('security_group'), + groups=params.get('security_groups'), + subnet_id=subnet_id, + ec2=ec2 + ) + expected_groups = [g['GroupId'] for g in groups] + instance_groups = [g['GroupId'] for g in value['Groups']] + if set(instance_groups) != set(expected_groups): + changes_to_apply.append(dict( + Groups=expected_groups, + InstanceId=instance['InstanceId'] + )) if (params.get('network') or {}).get('source_dest_check') is not None: # network.source_dest_check is nested, so needs to be treated separately diff --git a/tests/integration/targets/ec2_instance/inventory b/tests/integration/targets/ec2_instance/inventory index 44b46ec88f7..09bae76beb1 100644 --- a/tests/integration/targets/ec2_instance/inventory +++ b/tests/integration/targets/ec2_instance/inventory @@ -11,6 +11,7 @@ iam_instance_role termination_protection tags_and_vpc_settings checkmode_tests +security_group [all:vars] ansible_connection=local diff --git a/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/env_cleanup.yml b/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/env_cleanup.yml index 1b6c79e0d95..07c7f72bd8e 100644 --- a/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/env_cleanup.yml +++ b/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/env_cleanup.yml @@ -33,6 +33,17 @@ ignore_errors: yes retries: 10 +- name: "remove the second security group" + ec2_group: + name: "{{ resource_prefix }}-sg-2" + description: a security group for ansible tests + vpc_id: "{{ testing_vpc.vpc.id }}" + state: absent + register: removed + until: removed is not failed + ignore_errors: yes + retries: 10 + - name: "remove routing rules" ec2_vpc_route_table: state: absent diff --git a/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/env_setup.yml b/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/env_setup.yml index 6c76b7bf79f..7c99f807177 100644 --- a/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/env_setup.yml +++ b/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/env_setup.yml @@ -77,3 +77,19 @@ to_port: 80 cidr_ip: 0.0.0.0/0 register: sg + + - name: "create secondary security group with the vpc" + ec2_group: + name: "{{ resource_prefix }}-sg-2" + description: a secondary security group for ansible tests + vpc_id: "{{ testing_vpc.vpc.id }}" + rules: + - proto: tcp + from_port: 22 + to_port: 22 + cidr_ip: 0.0.0.0/0 + - proto: tcp + from_port: 80 + to_port: 80 + cidr_ip: 0.0.0.0/0 + register: sg2 diff --git a/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/security_group.yml b/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/security_group.yml new file mode 100644 index 00000000000..c0e52a5f386 --- /dev/null +++ b/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/security_group.yml @@ -0,0 +1,81 @@ +- block: + - name: "New instance with 2 security groups" + ec2_instance: + name: "{{ resource_prefix }}-test-security-groups" + image_id: "{{ ec2_ami_image }}" + vpc_subnet_id: "{{ testing_subnet_a.subnet.id }}" + tags: + TestId: "{{ resource_prefix }}" + instance_type: t2.micro + wait: false + security_groups: + - "{{ sg.group_id }}" + - "{{ sg2.group_id }}" + register: security_groups_test + + - name: "Recreate same instance with 2 security groups ( Idempotency )" + ec2_instance: + name: "{{ resource_prefix }}-test-security-groups" + image_id: "{{ ec2_ami_image }}" + vpc_subnet_id: "{{ testing_subnet_a.subnet.id }}" + tags: + TestId: "{{ resource_prefix }}" + instance_type: t2.micro + wait: false + security_groups: + - "{{ sg.group_id }}" + - "{{ sg2.group_id }}" + register: security_groups_test_idempotency + + - name: "Gather ec2 facts to check SGs have been added" + ec2_instance_info: + filters: + "tag:Name": "{{ resource_prefix }}-test-security-groups" + "instance-state-name": "running" + register: dual_sg_instance_facts + until: dual_sg_instance_facts.instances | length > 0 + retries: 10 + + - name: "Remove secondary security group from instance" + ec2_instance: + name: "{{ resource_prefix }}-test-security-groups" + image_id: "{{ ec2_ami_image }}" + vpc_subnet_id: "{{ testing_subnet_a.subnet.id }}" + tags: + TestId: "{{ resource_prefix }}" + instance_type: t2.micro + security_groups: + - "{{ sg.group_id }}" + register: remove_secondary_security_group + + - name: "Gather ec2 facts to check seconday SG has been removed" + ec2_instance_info: + filters: + "tag:Name": "{{ resource_prefix }}-test-security-groups" + "instance-state-name": "running" + register: single_sg_instance_facts + until: single_sg_instance_facts.instances | length > 0 + retries: 10 + + - name: "Add secondary security group to instance" + ec2_instance: + name: "{{ resource_prefix }}-test-security-groups" + image_id: "{{ ec2_ami_image }}" + vpc_subnet_id: "{{ testing_subnet_a.subnet.id }}" + tags: + TestId: "{{ resource_prefix }}" + instance_type: t2.micro + security_groups: + - "{{ sg.group_id }}" + - "{{ sg2.group_id }}" + register: add_secondary_security_group + + - assert: + that: + - security_groups_test is not failed + - security_groups_test is changed + - security_groups_test_idempotency is not changed + - remove_secondary_security_group is changed + - single_sg_instance_facts.instances.0.security_groups | length == 1 + - dual_sg_instance_facts.instances.0.security_groups | length == 2 + - add_secondary_security_group is changed From a67dfc844eba58dea6a00d96b71d0190b83bebfc Mon Sep 17 00:00:00 2001 From: Mark Chappell Date: Wed, 26 Aug 2020 11:35:32 +0200 Subject: [PATCH 16/38] Cleanup: Bulk Migration from boto3_conn to module.client() (#188) * Migrate from boto3_conn to module.client * Simplify error handling when creating connections * Simplify Region handling * Remove unused imports * Changelog --- plugins/modules/ec2_instance_info.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/plugins/modules/ec2_instance_info.py b/plugins/modules/ec2_instance_info.py index 88a07d05f61..1c4c1f0df33 100644 --- a/plugins/modules/ec2_instance_info.py +++ b/plugins/modules/ec2_instance_info.py @@ -495,16 +495,15 @@ try: import boto3 + import botocore from botocore.exceptions import ClientError except ImportError: pass # Handled by AnsibleAWSModule from ansible_collections.amazon.aws.plugins.module_utils.core import AnsibleAWSModule from ansible_collections.amazon.aws.plugins.module_utils.ec2 import ansible_dict_to_boto3_filter_list -from ansible_collections.amazon.aws.plugins.module_utils.ec2 import boto3_conn from ansible_collections.amazon.aws.plugins.module_utils.ec2 import boto3_tag_list_to_ansible_dict from ansible_collections.amazon.aws.plugins.module_utils.ec2 import camel_dict_to_snake_dict -from ansible_collections.amazon.aws.plugins.module_utils.ec2 import get_aws_connection_info def list_ec2_instances(connection, module): @@ -550,12 +549,10 @@ def main(): if module._name == 'ec2_instance_facts': module.deprecate("The 'ec2_instance_facts' module has been renamed to 'ec2_instance_info'", date='2021-12-01', collection_name='community.aws') - region, ec2_url, aws_connect_params = get_aws_connection_info(module, boto3=True) - - if region: - connection = boto3_conn(module, conn_type='client', resource='ec2', region=region, endpoint=ec2_url, **aws_connect_params) - else: - module.fail_json(msg="region must be specified") + try: + connection = module.client('ec2') + except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: + module.fail_json_aws(e, msg='Failed to connect to AWS') list_ec2_instances(connection, module) From 63702ff4d7003e061b50791dade02f60c62c2d10 Mon Sep 17 00:00:00 2001 From: Mark Chappell Date: Fri, 28 Aug 2020 02:12:41 +0200 Subject: [PATCH 17/38] ec2_instance - Fix check_mode behaviour with tags (#189) * Add test for changing tags in check_mode * ec2_instance: Fix check_mode behaviour with tags * Add changelog fragment --- plugins/modules/ec2_instance.py | 2 ++ .../ec2_instance/tasks/checkmode_tests.yml | 29 +++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/plugins/modules/ec2_instance.py b/plugins/modules/ec2_instance.py index ddedd379573..e87f64cdf29 100644 --- a/plugins/modules/ec2_instance.py +++ b/plugins/modules/ec2_instance.py @@ -894,6 +894,8 @@ def manage_tags(match, new_tags, purge_tags, ec2): old_tags, new_tags, purge_tags=purge_tags, ) + if module.check_mode: + return bool(tags_to_delete or tags_to_set) if tags_to_set: ec2.create_tags( Resources=[match['InstanceId']], diff --git a/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/checkmode_tests.yml b/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/checkmode_tests.yml index b161eca636e..e13ad44063b 100644 --- a/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/checkmode_tests.yml +++ b/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/checkmode_tests.yml @@ -10,6 +10,7 @@ wait: false tags: TestId: "{{ ec2_instance_tag_TestId }}" + TestTag: "Some Value" register: basic_instance - name: "Make basic instance (check mode)" @@ -22,6 +23,7 @@ vpc_subnet_id: "{{ testing_subnet_b.subnet.id }}" tags: TestId: "{{ ec2_instance_tag_TestId }}" + TestTag: "Some Value" check_mode: yes - name: "fact presented ec2 instance" @@ -49,6 +51,7 @@ vpc_subnet_id: "{{ testing_subnet_a.subnet.id }}" tags: TestId: "{{ ec2_instance_tag_TestId }}" + TestTag: "Some Value" check_mode: yes - name: "fact ec2 instance" @@ -69,6 +72,7 @@ vpc_subnet_id: "{{ testing_subnet_a.subnet.id }}" tags: TestId: "{{ ec2_instance_tag_TestId }}" + TestTag: "Some Value" register: instance_stop until: not instance_stop.failed retries: 10 @@ -91,6 +95,7 @@ vpc_subnet_id: "{{ testing_subnet_a.subnet.id }}" tags: TestId: "{{ ec2_instance_tag_TestId }}" + TestTag: "Some Value" check_mode: yes - name: "fact ec2 instance" @@ -111,6 +116,7 @@ vpc_subnet_id: "{{ testing_subnet_a.subnet.id }}" tags: TestId: "{{ ec2_instance_tag_TestId }}" + TestTag: "Some Value" - name: "fact ec2 instance." ec2_instance_info: @@ -123,6 +129,27 @@ that: - '"{{ confirm_runninginstance_fact.instances[0].state.name }}" == "running"' + - name: "Tag instance." + ec2_instance: + state: running + name: "{{ resource_prefix }}-checkmode-comparison" + vpc_subnet_id: "{{ testing_subnet_a.subnet.id }}" + tags: + TestId: "{{ ec2_instance_tag_TestId }}" + TestTag: "Some Other Value" + check_mode: yes + + - name: "fact ec2 instance." + ec2_instance_info: + filters: + "tag:Name": "{{ resource_prefix }}-checkmode-comparison" + register: confirm_not_tagged + + - name: "Verify that it hasn't been re-tagged." + assert: + that: + - '"{{ confirm_not_tagged.instances[0].tags.TestTag }}" == "Some Value"' + - name: "Terminate instance in check mode." ec2_instance: state: absent @@ -130,6 +157,7 @@ vpc_subnet_id: "{{ testing_subnet_a.subnet.id }}" tags: TestId: "{{ ec2_instance_tag_TestId }}" + TestTag: "Some Value" check_mode: yes - name: "fact ec2 instance" @@ -150,6 +178,7 @@ vpc_subnet_id: "{{ testing_subnet_a.subnet.id }}" tags: TestId: "{{ ec2_instance_tag_TestId }}" + TestTag: "Some Value" - name: "fact ec2 instance" ec2_instance_info: From 4ff8a3622fcfff25055df4ca7a2018dbb9aea333 Mon Sep 17 00:00:00 2001 From: Mark Chappell Date: Fri, 23 Oct 2020 00:25:24 +0200 Subject: [PATCH 18/38] stability: Increase the number of retries on ec2_instance tests (#187) * Split imports into a single line * Increase the max_attempts retries for the ec2_instance tests, we're running a *lot* in parallel which triggers RateLimiting errors --- plugins/modules/ec2_instance.py | 18 +++++++++--------- .../roles/ec2_instance/tasks/main.yml | 5 +++++ 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/plugins/modules/ec2_instance.py b/plugins/modules/ec2_instance.py index e87f64cdf29..aba7ac26b10 100644 --- a/plugins/modules/ec2_instance.py +++ b/plugins/modules/ec2_instance.py @@ -810,17 +810,17 @@ from ansible.module_utils.six import string_types from ansible.module_utils.six.moves.urllib import parse as urlparse +from ansible.module_utils._text import to_bytes from ansible.module_utils._text import to_native -import ansible_collections.amazon.aws.plugins.module_utils.ec2 as ec2_utils -from ansible_collections.amazon.aws.plugins.module_utils.ec2 import (AWSRetry, - ansible_dict_to_boto3_filter_list, - compare_aws_tags, - boto3_tag_list_to_ansible_dict, - ansible_dict_to_boto3_tag_list, - camel_dict_to_snake_dict, - ) from ansible_collections.amazon.aws.plugins.module_utils.core import AnsibleAWSModule +from ansible_collections.amazon.aws.plugins.module_utils.ec2 import AWSRetry +from ansible_collections.amazon.aws.plugins.module_utils.ec2 import ansible_dict_to_boto3_filter_list +from ansible_collections.amazon.aws.plugins.module_utils.ec2 import compare_aws_tags +from ansible_collections.amazon.aws.plugins.module_utils.ec2 import boto3_tag_list_to_ansible_dict +from ansible_collections.amazon.aws.plugins.module_utils.ec2 import ansible_dict_to_boto3_tag_list +from ansible_collections.amazon.aws.plugins.module_utils.ec2 import camel_dict_to_snake_dict +from ansible_collections.amazon.aws.plugins.module_utils.ec2 import snake_dict_to_camel_dict module = None @@ -917,7 +917,7 @@ def build_volume_spec(params): for int_value in ['volume_size', 'iops']: if int_value in volume['ebs']: volume['ebs'][int_value] = int(volume['ebs'][int_value]) - return [ec2_utils.snake_dict_to_camel_dict(v, capitalize_first=True) for v in volumes] + return [snake_dict_to_camel_dict(v, capitalize_first=True) for v in volumes] def add_or_update_instance_profile(instance, desired_profile_name): diff --git a/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/main.yml b/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/main.yml index 188d97d2e9f..dc81199aabe 100644 --- a/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/main.yml +++ b/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/main.yml @@ -24,6 +24,11 @@ aws_secret_key: "{{ aws_secret_key }}" security_token: "{{ security_token | default(omit) }}" region: "{{ aws_region }}" + aws_config: + retries: + # Unfortunately AWSRetry doesn't support paginators and boto3's paginators + # don't support any configuration of the delay between retries. + max_attempts: 20 collections: - amazon.aws block: From 17cf19295e1f43068fc1d302f8c3c7df55a6fa10 Mon Sep 17 00:00:00 2001 From: Sean Cavanaugh Date: Wed, 13 Jan 2021 13:48:15 -0500 Subject: [PATCH 19/38] add uptime parameter for ec2_instance_info module in minutes (#356) * syncing module and tests for uptime with tons of help from Yanis, we now have uptime in there * updating pr with fixes from suggestions adding to https://github.com/ansible-collections/community.aws/pull/356 with comments from @tremble and @duderamos * Create 356_add_minimum_uptime_parameter.yaml adding changelog fragment per @gravesm suggestion * Update 356_add_minimum_uptime_parameter.yaml last comment from @tremble Co-authored-by: Sean Cavanaugh --- plugins/modules/ec2_instance_info.py | 33 +++++++++- .../roles/ec2_instance/tasks/uptime.yml | 66 +++++++++++++++++++ 2 files changed, 96 insertions(+), 3 deletions(-) create mode 100644 tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/uptime.yml diff --git a/plugins/modules/ec2_instance_info.py b/plugins/modules/ec2_instance_info.py index 1c4c1f0df33..e37f2cf9cd1 100644 --- a/plugins/modules/ec2_instance_info.py +++ b/plugins/modules/ec2_instance_info.py @@ -33,6 +33,13 @@ required: false default: {} type: dict + minimum_uptime: + description: + - Minimum running uptime in minutes of instances. For example if I(uptime) is C(60) return all instances that have run more than 60 minutes. + required: false + aliases: ['uptime'] + type: int + extends_documentation_fragment: - amazon.aws.aws @@ -66,6 +73,15 @@ filters: instance-state-name: [ "shutting-down", "stopping", "stopped" ] +- name: Gather information about any instance with Name beginning with RHEL and an uptime of at least 60 minutes + community.aws.ec2_instance_info: + region: "{{ ec2_region }}" + uptime: 60 + filters: + "tag:Name": "RHEL-*" + instance-state-name: [ "running"] + register: ec2_node_info + ''' RETURN = r''' @@ -492,6 +508,8 @@ ''' import traceback +import datetime + try: import boto3 @@ -509,6 +527,7 @@ def list_ec2_instances(connection, module): instance_ids = module.params.get("instance_ids") + uptime = module.params.get('minimum_uptime') filters = ansible_dict_to_boto3_filter_list(module.params.get("filters")) try: @@ -517,10 +536,17 @@ def list_ec2_instances(connection, module): except ClientError as e: module.fail_json_aws(e, msg="Failed to list ec2 instances") - # Get instances from reservations instances = [] - for reservation in reservations['Reservations']: - instances = instances + reservation['Instances'] + + if uptime: + timedelta = int(uptime) if uptime else 0 + oldest_launch_time = datetime.datetime.utcnow() - datetime.timedelta(minutes=timedelta) + # Get instances from reservations + for reservation in reservations['Reservations']: + instances += [instance for instance in reservation['Instances'] if instance['LaunchTime'].replace(tzinfo=None) < oldest_launch_time] + else: + for reservation in reservations['Reservations']: + instances = instances + reservation['Instances'] # Turn the boto3 result in to ansible_friendly_snaked_names snaked_instances = [camel_dict_to_snake_dict(instance) for instance in instances] @@ -535,6 +561,7 @@ def list_ec2_instances(connection, module): def main(): argument_spec = dict( + minimum_uptime=dict(required=False, type='int', default=None, aliases=['uptime']), instance_ids=dict(default=[], type='list', elements='str'), filters=dict(default={}, type='dict') ) diff --git a/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/uptime.yml b/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/uptime.yml new file mode 100644 index 00000000000..6f6c5fe0d49 --- /dev/null +++ b/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/uptime.yml @@ -0,0 +1,66 @@ +--- +- block: + - name: "create t3.nano instance" + ec2_instance: + name: "{{ resource_prefix }}-test-uptime" + region: "{{ ec2_region }}" + image_id: "{{ ec2_ami_image }}" + tags: + TestId: "{{ ec2_instance_tag_TestId }}" + vpc_subnet_id: "{{ testing_subnet_a.subnet.id }}" + instance_type: t3.nano + wait: yes + + - name: "check ec2 instance" + ec2_instance_info: + filters: + "tag:Name": "{{ resource_prefix }}-test-uptime" + instance-state-name: [ "running"] + register: instance_facts + + - name: "Confirm existence of instance id." + assert: + that: + - "{{ instance_facts.instances | length }} == 1" + + - name: "check using uptime 100 hours - should find nothing" + ec2_instance_info: + region: "{{ ec2_region }}" + uptime: 6000 + filters: + instance-state-name: [ "running"] + "tag:Name": "{{ resource_prefix }}-test-uptime" + register: instance_facts + + - name: "Confirm there is no running instance" + assert: + that: + - "{{ instance_facts.instances | length }} == 0" + + - name: Sleep for 61 seconds and continue with play + wait_for: + timeout: 61 + delegate_to: localhost + + - name: "check using uptime 1 minute" + ec2_instance_info: + region: "{{ ec2_region }}" + uptime: 1 + filters: + instance-state-name: [ "running"] + "tag:Name": "{{ resource_prefix }}-test-uptime" + register: instance_facts + + - name: "Confirm there is one running instance" + assert: + that: + - "{{ instance_facts.instances | length }} == 1" + + always: + - name: "Terminate instances" + ec2_instance: + state: absent + filters: + "tag:TestId": "{{ ec2_instance_tag_TestId }}" + wait: yes + ignore_errors: yes From 3f478768efbebfa7eec1ec4603b0c9d73981a1e6 Mon Sep 17 00:00:00 2001 From: Mark Chappell Date: Sat, 16 Jan 2021 10:50:49 +0100 Subject: [PATCH 20/38] Bulk import cleanup (#360) * Split imports and reorder * Import camel_dict_to_snake_dict and snake_dict_to_camel_dict direct from ansible.module_utils.common.dict_transformations * Remove unused imports * Route53 Info was migrated to Boto3 drop the HAS_BOTO check and import * changelog --- plugins/modules/ec2_instance.py | 16 +++++++--------- plugins/modules/ec2_instance_info.py | 6 ++---- 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/plugins/modules/ec2_instance.py b/plugins/modules/ec2_instance.py index aba7ac26b10..a240a350d13 100644 --- a/plugins/modules/ec2_instance.py +++ b/plugins/modules/ec2_instance.py @@ -795,32 +795,30 @@ sample: vpc-0011223344 ''' +from collections import namedtuple import re -import uuid import string import textwrap import time -from collections import namedtuple +import uuid try: - import boto3 import botocore.exceptions except ImportError: pass # caught by AnsibleAWSModule +from ansible.module_utils._text import to_native +from ansible.module_utils.common.dict_transformations import camel_dict_to_snake_dict +from ansible.module_utils.common.dict_transformations import snake_dict_to_camel_dict from ansible.module_utils.six import string_types from ansible.module_utils.six.moves.urllib import parse as urlparse -from ansible.module_utils._text import to_bytes -from ansible.module_utils._text import to_native from ansible_collections.amazon.aws.plugins.module_utils.core import AnsibleAWSModule from ansible_collections.amazon.aws.plugins.module_utils.ec2 import AWSRetry from ansible_collections.amazon.aws.plugins.module_utils.ec2 import ansible_dict_to_boto3_filter_list -from ansible_collections.amazon.aws.plugins.module_utils.ec2 import compare_aws_tags -from ansible_collections.amazon.aws.plugins.module_utils.ec2 import boto3_tag_list_to_ansible_dict from ansible_collections.amazon.aws.plugins.module_utils.ec2 import ansible_dict_to_boto3_tag_list -from ansible_collections.amazon.aws.plugins.module_utils.ec2 import camel_dict_to_snake_dict -from ansible_collections.amazon.aws.plugins.module_utils.ec2 import snake_dict_to_camel_dict +from ansible_collections.amazon.aws.plugins.module_utils.ec2 import boto3_tag_list_to_ansible_dict +from ansible_collections.amazon.aws.plugins.module_utils.ec2 import compare_aws_tags module = None diff --git a/plugins/modules/ec2_instance_info.py b/plugins/modules/ec2_instance_info.py index e37f2cf9cd1..be5f1e68892 100644 --- a/plugins/modules/ec2_instance_info.py +++ b/plugins/modules/ec2_instance_info.py @@ -507,21 +507,19 @@ sample: vpc-0011223344 ''' -import traceback import datetime - try: - import boto3 import botocore from botocore.exceptions import ClientError except ImportError: pass # Handled by AnsibleAWSModule +from ansible.module_utils.common.dict_transformations import camel_dict_to_snake_dict + from ansible_collections.amazon.aws.plugins.module_utils.core import AnsibleAWSModule from ansible_collections.amazon.aws.plugins.module_utils.ec2 import ansible_dict_to_boto3_filter_list from ansible_collections.amazon.aws.plugins.module_utils.ec2 import boto3_tag_list_to_ansible_dict -from ansible_collections.amazon.aws.plugins.module_utils.ec2 import camel_dict_to_snake_dict def list_ec2_instances(connection, module): From 584c3879eb2a47c3e2e977ea3ac6201b81dea407 Mon Sep 17 00:00:00 2001 From: Mark Chappell Date: Wed, 3 Feb 2021 18:11:15 +0100 Subject: [PATCH 21/38] ec2_instance - Apply retry decorators more consistently. (#373) * ec2_instance: build results inside find_instances and add backoff * Add retry decorator to ec2 clients --- plugins/modules/ec2_instance.py | 89 +++++++++++++++++---------------- 1 file changed, 47 insertions(+), 42 deletions(-) diff --git a/plugins/modules/ec2_instance.py b/plugins/modules/ec2_instance.py index a240a350d13..a13b00c680b 100644 --- a/plugins/modules/ec2_instance.py +++ b/plugins/modules/ec2_instance.py @@ -884,7 +884,6 @@ def tower_callback_script(tower_conf, windows=False, passwd=None): raise NotImplementedError("Only windows with remote-prep or non-windows with tower job callback supported so far.") -@AWSRetry.jittered_backoff() def manage_tags(match, new_tags, purge_tags, ec2): changed = False old_tags = boto3_tag_list_to_ansible_dict(match['Tags']) @@ -896,12 +895,14 @@ def manage_tags(match, new_tags, purge_tags, ec2): return bool(tags_to_delete or tags_to_set) if tags_to_set: ec2.create_tags( + aws_retry=True, Resources=[match['InstanceId']], Tags=ansible_dict_to_boto3_tag_list(tags_to_set)) changed |= True if tags_to_delete: delete_with_current_values = dict((k, old_tags.get(k)) for k in tags_to_delete) ec2.delete_tags( + aws_retry=True, Resources=[match['InstanceId']], Tags=ansible_dict_to_boto3_tag_list(delete_with_current_values)) changed |= True @@ -929,14 +930,17 @@ def add_or_update_instance_profile(instance, desired_profile_name): if instance_profile_setting.get('Arn') == desired_arn: return False # update association - ec2 = module.client('ec2') + ec2 = module.client('ec2', retry_decorator=AWSRetry.jittered_backoff()) try: - association = ec2.describe_iam_instance_profile_associations(Filters=[{'Name': 'instance-id', 'Values': [instance['InstanceId']]}]) + association = ec2.describe_iam_instance_profile_associations( + aws_retry=True, + Filters=[{'Name': 'instance-id', 'Values': [instance['InstanceId']]}]) except botocore.exceptions.ClientError as e: # check for InvalidAssociationID.NotFound module.fail_json_aws(e, "Could not find instance profile association") try: resp = ec2.replace_iam_instance_profile_association( + aws_retry=True, AssociationId=association['IamInstanceProfileAssociations'][0]['AssociationId'], IamInstanceProfile={'Arn': determine_iam_role(desired_profile_name)} ) @@ -946,9 +950,10 @@ def add_or_update_instance_profile(instance, desired_profile_name): if not instance_profile_setting and desired_profile_name: # create association - ec2 = module.client('ec2') + ec2 = module.client('ec2', retry_decorator=AWSRetry.jittered_backoff()) try: resp = ec2.associate_iam_instance_profile( + aws_retry=True, IamInstanceProfile={'Arn': determine_iam_role(desired_profile_name)}, InstanceId=instance['InstanceId'] ) @@ -989,7 +994,7 @@ def build_network_spec(params, ec2=None): }, """ if ec2 is None: - ec2 = module.client('ec2') + ec2 = module.client('ec2', retry_decorator=AWSRetry.jittered_backoff()) interfaces = [] network = params.get('network') or {} @@ -1109,11 +1114,11 @@ def warn_if_cpu_options_changed(instance): def discover_security_groups(group, groups, parent_vpc_id=None, subnet_id=None, ec2=None): if ec2 is None: - ec2 = module.client('ec2') + ec2 = module.client('ec2', retry_decorator=AWSRetry.jittered_backoff()) if subnet_id is not None: try: - sub = ec2.describe_subnets(SubnetIds=[subnet_id]) + sub = ec2.describe_subnets(aws_retry=True, SubnetIds=[subnet_id]) except botocore.exceptions.ClientError as e: if e.response['Error']['Code'] == 'InvalidGroup.NotFound': module.fail_json( @@ -1168,14 +1173,17 @@ def discover_security_groups(group, groups, parent_vpc_id=None, subnet_id=None, found_groups = [] for f_set in (id_filters, name_filters): if len(f_set) > 1: - found_groups.extend(ec2.get_paginator( - 'describe_security_groups' - ).paginate( - Filters=f_set - ).search('SecurityGroups[]')) + found_groups.extend(describe_security_groups(ec2, Filters=f_set)) return list(dict((g['GroupId'], g) for g in found_groups).values()) +@AWSRetry.jittered_backoff() +def describe_security_groups(ec2, **params): + paginator = ec2.get_paginator('describe_security_groups') + results = paginator.paginate(**params) + return list(results.search('SecurityGroups[]')) + + def build_top_level_options(params): spec = {} if params.get('image_id'): @@ -1257,7 +1265,7 @@ def build_instance_tags(params, propagate_tags_to_volumes=True): def build_run_instance_spec(params, ec2=None): if ec2 is None: - ec2 = module.client('ec2') + ec2 = module.client('ec2', retry_decorator=AWSRetry.jittered_backoff()) spec = dict( ClientToken=uuid.uuid4().hex, @@ -1296,7 +1304,7 @@ def await_instances(ids, state='OK'): } if state not in state_opts: module.fail_json(msg="Cannot wait for state {0}, invalid state".format(state)) - waiter = module.client('ec2').get_waiter(state_opts[state]) + waiter = module.client('ec2', retry_decorator=AWSRetry.jittered_backoff()).get_waiter(state_opts[state]) try: waiter.wait( InstanceIds=ids, @@ -1316,7 +1324,7 @@ def await_instances(ids, state='OK'): def diff_instance_and_params(instance, params, ec2=None, skip=None): """boto3 instance obj, module params""" if ec2 is None: - ec2 = module.client('ec2') + ec2 = module.client('ec2', retry_decorator=AWSRetry.jittered_backoff()) if skip is None: skip = [] @@ -1342,7 +1350,7 @@ def value_wrapper(v): if mapping.instance_key in skip: continue - value = AWSRetry.jittered_backoff()(ec2.describe_instance_attribute)(Attribute=mapping.attribute_name, InstanceId=id_) + value = ec2.describe_instance_attribute(aws_retry=True, Attribute=mapping.attribute_name, InstanceId=id_) if value[mapping.instance_key]['Value'] != params.get(mapping.param_key): arguments = dict( InstanceId=instance['InstanceId'], @@ -1352,7 +1360,7 @@ def value_wrapper(v): changes_to_apply.append(arguments) if params.get('security_group') or params.get('security_groups'): - value = AWSRetry.jittered_backoff()(ec2.describe_instance_attribute)(Attribute="groupSet", InstanceId=id_) + value = ec2.describe_instance_attribute(aws_retry=True, Attribute="groupSet", InstanceId=id_) # managing security groups if params.get('vpc_subnet_id'): subnet_id = params.get('vpc_subnet_id') @@ -1404,6 +1412,7 @@ def change_network_attachments(instance, params, ec2): to_attach = set(new_ids) - set(old_ids) for eni_id in to_attach: ec2.attach_network_interface( + aws_retry=True, DeviceIndex=new_ids.index(eni_id), InstanceId=instance['InstanceId'], NetworkInterfaceId=eni_id, @@ -1412,35 +1421,35 @@ def change_network_attachments(instance, params, ec2): return False +@AWSRetry.jittered_backoff() def find_instances(ec2, ids=None, filters=None): paginator = ec2.get_paginator('describe_instances') if ids: - return list(paginator.paginate( - InstanceIds=ids, - ).search('Reservations[].Instances[]')) + params = dict(InstanceIds=ids) elif filters is None: module.fail_json(msg="No filters provided when they were required") - elif filters is not None: + else: for key in list(filters.keys()): if not key.startswith("tag:"): filters[key.replace("_", "-")] = filters.pop(key) - return list(paginator.paginate( - Filters=ansible_dict_to_boto3_filter_list(filters) - ).search('Reservations[].Instances[]')) - return [] + params = dict(Filters=ansible_dict_to_boto3_filter_list(filters)) + + results = paginator.paginate(**params).search('Reservations[].Instances[]') + return list(results) -@AWSRetry.jittered_backoff() def get_default_vpc(ec2): - vpcs = ec2.describe_vpcs(Filters=ansible_dict_to_boto3_filter_list({'isDefault': 'true'})) + vpcs = ec2.describe_vpcs( + aws_retry=True, + Filters=ansible_dict_to_boto3_filter_list({'isDefault': 'true'})) if len(vpcs.get('Vpcs', [])): return vpcs.get('Vpcs')[0] return None -@AWSRetry.jittered_backoff() def get_default_subnet(ec2, vpc, availability_zone=None): subnets = ec2.describe_subnets( + aws_retry=True, Filters=ansible_dict_to_boto3_filter_list({ 'vpc-id': vpc['VpcId'], 'state': 'available', @@ -1462,7 +1471,7 @@ def get_default_subnet(ec2, vpc, availability_zone=None): def ensure_instance_state(state, ec2=None): if ec2 is None: - module.client('ec2') + module.client('ec2', retry_decorator=AWSRetry.jittered_backoff()) if state in ('running', 'started'): changed, failed, instances, failure_reason = change_instance_state(filters=module.params.get('filters'), desired_state='RUNNING') @@ -1537,11 +1546,10 @@ def ensure_instance_state(state, ec2=None): ) -@AWSRetry.jittered_backoff() def change_instance_state(filters, desired_state, ec2=None): """Takes STOPPED/RUNNING/TERMINATED""" if ec2 is None: - ec2 = module.client('ec2') + ec2 = module.client('ec2', retry_decorator=AWSRetry.jittered_backoff()) changed = set() instances = find_instances(ec2, filters=filters) @@ -1558,7 +1566,7 @@ def change_instance_state(filters, desired_state, ec2=None): # TODO use a client-token to prevent double-sends of these start/stop/terminate commands # https://docs.aws.amazon.com/AWSEC2/latest/APIReference/Run_Instance_Idempotency.html - resp = ec2.terminate_instances(InstanceIds=[inst['InstanceId']]) + resp = ec2.terminate_instances(aws_retry=True, InstanceIds=[inst['InstanceId']]) [changed.add(i['InstanceId']) for i in resp['TerminatingInstances']] if desired_state == 'STOPPED': if inst['State']['Name'] in ('stopping', 'stopped'): @@ -1569,14 +1577,14 @@ def change_instance_state(filters, desired_state, ec2=None): changed.add(inst['InstanceId']) continue - resp = ec2.stop_instances(InstanceIds=[inst['InstanceId']]) + resp = ec2.stop_instances(aws_retry=True, InstanceIds=[inst['InstanceId']]) [changed.add(i['InstanceId']) for i in resp['StoppingInstances']] if desired_state == 'RUNNING': if module.check_mode: changed.add(inst['InstanceId']) continue - resp = ec2.start_instances(InstanceIds=[inst['InstanceId']]) + resp = ec2.start_instances(aws_retry=True, InstanceIds=[inst['InstanceId']]) [changed.add(i['InstanceId']) for i in resp['StartingInstances']] except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: try: @@ -1625,7 +1633,7 @@ def handle_existing(existing_matches, changed, ec2, state): ) changes = diff_instance_and_params(existing_matches[0], module.params) for c in changes: - AWSRetry.jittered_backoff()(ec2.modify_instance_attribute)(**c) + ec2.modify_instance_attribute(aws_retry=True, **c) changed |= bool(changes) changed |= add_or_update_instance_profile(existing_matches[0], module.params.get('instance_role')) changed |= change_network_attachments(existing_matches[0], module.params, ec2) @@ -1664,7 +1672,7 @@ def ensure_present(existing_matches, changed, ec2, state): changes = diff_instance_and_params(ins, module.params, skip=['UserData', 'EbsOptimized']) for c in changes: try: - AWSRetry.jittered_backoff()(ec2.modify_instance_attribute)(**c) + ec2.modify_instance_attribute(aws_retry=True, **c) except botocore.exceptions.ClientError as e: module.fail_json_aws(e, msg="Could not apply change {0} to new instance.".format(str(c))) @@ -1675,9 +1683,7 @@ def ensure_present(existing_matches, changed, ec2, state): spec=instance_spec, ) await_instances(instance_ids) - instances = ec2.get_paginator('describe_instances').paginate( - InstanceIds=instance_ids - ).search('Reservations[].Instances[]') + instances = find_instances(ec2, ids=instance_ids) module.exit_json( changed=True, @@ -1689,10 +1695,9 @@ def ensure_present(existing_matches, changed, ec2, state): module.fail_json_aws(e, msg="Failed to create new EC2 instance") -@AWSRetry.jittered_backoff() def run_instances(ec2, **instance_spec): try: - return ec2.run_instances(**instance_spec) + return ec2.run_instances(aws_retry=True, **instance_spec) except botocore.exceptions.ClientError as e: if e.response['Error']['Code'] == 'InvalidParameterValue' and "Invalid IAM Instance Profile ARN" in e.response['Error']['Message']: # If the instance profile has just been created, it takes some time to be visible by ec2 @@ -1762,7 +1767,7 @@ def main(): module.fail_json(msg="Parameter network.interfaces can't be used with security_groups") state = module.params.get('state') - ec2 = module.client('ec2') + ec2 = module.client('ec2', retry_decorator=AWSRetry.jittered_backoff()) if module.params.get('filters') is None: filters = { # all states except shutting-down and terminated From 6eeb8b6b01be9a356c6dd63bb73531abc124dc50 Mon Sep 17 00:00:00 2001 From: Mark Chappell Date: Fri, 5 Feb 2021 09:43:09 +0100 Subject: [PATCH 22/38] Cleanup - use is_boto3_error_(message|code) (#268) * Reorder imports * Make use of is_boto3_error_message * Mass-migration over to is_boto3_error_code * Remove unused imports * unused vars in exception * Improve consistency around catching BotoCoreError and ClientError * Remove unused imports * Remove unused 'PolicyError' from iam_policy_info * Avoid catching botocore.exceptions.ClientError when we only want some error codes * Import camel_dict_to_snake_dict/snake_dict_to_camel_dict from ansible.module_utils.common.dict_transformations --- plugins/modules/ec2_instance.py | 37 +++++++++++++++------------------ 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/plugins/modules/ec2_instance.py b/plugins/modules/ec2_instance.py index a13b00c680b..06ebda60341 100644 --- a/plugins/modules/ec2_instance.py +++ b/plugins/modules/ec2_instance.py @@ -814,6 +814,8 @@ from ansible.module_utils.six.moves.urllib import parse as urlparse from ansible_collections.amazon.aws.plugins.module_utils.core import AnsibleAWSModule +from ansible_collections.amazon.aws.plugins.module_utils.core import is_boto3_error_code +from ansible_collections.amazon.aws.plugins.module_utils.core import is_boto3_error_message from ansible_collections.amazon.aws.plugins.module_utils.ec2 import AWSRetry from ansible_collections.amazon.aws.plugins.module_utils.ec2 import ansible_dict_to_boto3_filter_list from ansible_collections.amazon.aws.plugins.module_utils.ec2 import ansible_dict_to_boto3_tag_list @@ -1119,15 +1121,13 @@ def discover_security_groups(group, groups, parent_vpc_id=None, subnet_id=None, if subnet_id is not None: try: sub = ec2.describe_subnets(aws_retry=True, SubnetIds=[subnet_id]) - except botocore.exceptions.ClientError as e: - if e.response['Error']['Code'] == 'InvalidGroup.NotFound': - module.fail_json( - "Could not find subnet {0} to associate security groups. Please check the vpc_subnet_id and security_groups parameters.".format( - subnet_id - ) + except is_boto3_error_code('InvalidGroup.NotFound'): + module.fail_json( + "Could not find subnet {0} to associate security groups. Please check the vpc_subnet_id and security_groups parameters.".format( + subnet_id ) - module.fail_json_aws(e, msg="Error while searching for subnet {0} parent VPC.".format(subnet_id)) - except botocore.exceptions.BotoCoreError as e: + ) + except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: # pylint: disable=duplicate-except module.fail_json_aws(e, msg="Error while searching for subnet {0} parent VPC.".format(subnet_id)) parent_vpc_id = sub['Subnets'][0]['VpcId'] @@ -1615,9 +1615,9 @@ def determine_iam_role(name_or_arn): try: role = iam.get_instance_profile(InstanceProfileName=name_or_arn, aws_retry=True) return role['InstanceProfile']['Arn'] - except botocore.exceptions.ClientError as e: - if e.response['Error']['Code'] == 'NoSuchEntity': - module.fail_json_aws(e, msg="Could not find instance_role {0}".format(name_or_arn)) + except is_boto3_error_code('NoSuchEntity'): + module.fail_json_aws(e, msg="Could not find instance_role {0}".format(name_or_arn)) + except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: # pylint: disable=duplicate-except module.fail_json_aws(e, msg="An error occurred while searching for instance_role {0}. Please try supplying the full ARN.".format(name_or_arn)) @@ -1697,15 +1697,12 @@ def ensure_present(existing_matches, changed, ec2, state): def run_instances(ec2, **instance_spec): try: - return ec2.run_instances(aws_retry=True, **instance_spec) - except botocore.exceptions.ClientError as e: - if e.response['Error']['Code'] == 'InvalidParameterValue' and "Invalid IAM Instance Profile ARN" in e.response['Error']['Message']: - # If the instance profile has just been created, it takes some time to be visible by ec2 - # So we wait 10 second and retry the run_instances - time.sleep(10) - return ec2.run_instances(**instance_spec) - else: - raise e + return ec2.run_instances(**instance_spec) + except is_boto3_error_message('Invalid IAM Instance Profile ARN'): + # If the instance profile has just been created, it takes some time to be visible by ec2 + # So we wait 10 second and retry the run_instances + time.sleep(10) + return ec2.run_instances(**instance_spec) def main(): From 6540fc9a9d801b5f8d8d99f52441a2cad98a2807 Mon Sep 17 00:00:00 2001 From: Mark Chappell Date: Wed, 10 Feb 2021 17:30:55 +0100 Subject: [PATCH 23/38] ec2_instance - Use shared module implementation of get_ec2_security_group_ids_from_names (#214) * ec2_instance - Use shared module implementation of get_ec2_security_group_ids_from_names * changelog --- plugins/modules/ec2_instance.py | 59 ++++----------------------------- 1 file changed, 7 insertions(+), 52 deletions(-) diff --git a/plugins/modules/ec2_instance.py b/plugins/modules/ec2_instance.py index 06ebda60341..380e3527910 100644 --- a/plugins/modules/ec2_instance.py +++ b/plugins/modules/ec2_instance.py @@ -803,7 +803,7 @@ import uuid try: - import botocore.exceptions + import botocore except ImportError: pass # caught by AnsibleAWSModule @@ -821,6 +821,7 @@ from ansible_collections.amazon.aws.plugins.module_utils.ec2 import ansible_dict_to_boto3_tag_list from ansible_collections.amazon.aws.plugins.module_utils.ec2 import boto3_tag_list_to_ansible_dict from ansible_collections.amazon.aws.plugins.module_utils.ec2 import compare_aws_tags +from ansible_collections.amazon.aws.plugins.module_utils.ec2 import get_ec2_security_group_ids_from_names module = None @@ -1029,7 +1030,7 @@ def build_network_spec(params, ec2=None): subnet_id=spec['SubnetId'], ec2=ec2 ) - spec['Groups'] = [g['GroupId'] for g in groups] + spec['Groups'] = groups if network.get('description') is not None: spec['Description'] = network['description'] # TODO more special snowflake network things @@ -1131,57 +1132,11 @@ def discover_security_groups(group, groups, parent_vpc_id=None, subnet_id=None, module.fail_json_aws(e, msg="Error while searching for subnet {0} parent VPC.".format(subnet_id)) parent_vpc_id = sub['Subnets'][0]['VpcId'] - vpc = { - 'Name': 'vpc-id', - 'Values': [parent_vpc_id] - } - - # because filter lists are AND in the security groups API, - # make two separate requests for groups by ID and by name - id_filters = [vpc] - name_filters = [vpc] - if group: - name_filters.append( - dict( - Name='group-name', - Values=[group] - ) - ) - if group.startswith('sg-'): - id_filters.append( - dict( - Name='group-id', - Values=[group] - ) - ) + return get_ec2_security_group_ids_from_names(group, ec2, vpc_id=parent_vpc_id) if groups: - name_filters.append( - dict( - Name='group-name', - Values=groups - ) - ) - if [g for g in groups if g.startswith('sg-')]: - id_filters.append( - dict( - Name='group-id', - Values=[g for g in groups if g.startswith('sg-')] - ) - ) - - found_groups = [] - for f_set in (id_filters, name_filters): - if len(f_set) > 1: - found_groups.extend(describe_security_groups(ec2, Filters=f_set)) - return list(dict((g['GroupId'], g) for g in found_groups).values()) - - -@AWSRetry.jittered_backoff() -def describe_security_groups(ec2, **params): - paginator = ec2.get_paginator('describe_security_groups') - results = paginator.paginate(**params) - return list(results.search('SecurityGroups[]')) + return get_ec2_security_group_ids_from_names(groups, ec2, vpc_id=parent_vpc_id) + return [] def build_top_level_options(params): @@ -1379,7 +1334,7 @@ def value_wrapper(v): subnet_id=subnet_id, ec2=ec2 ) - expected_groups = [g['GroupId'] for g in groups] + expected_groups = groups instance_groups = [g['GroupId'] for g in value['Groups']] if set(instance_groups) != set(expected_groups): changes_to_apply.append(dict( From 220ca6c855954c72e3dd4934f85335e767646bb6 Mon Sep 17 00:00:00 2001 From: Mark Chappell Date: Sun, 21 Feb 2021 15:40:04 +0100 Subject: [PATCH 24/38] Yet more integration test aliases file cleanup (#431) * More aliases cleanup * Mark ec2_classic_lb tests unstable * Add more comments about why tests aren't enabled --- tests/integration/targets/ec2_instance/aliases | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/targets/ec2_instance/aliases b/tests/integration/targets/ec2_instance/aliases index 62cb1d2c5b5..c451297ee90 100644 --- a/tests/integration/targets/ec2_instance/aliases +++ b/tests/integration/targets/ec2_instance/aliases @@ -1,3 +1,4 @@ cloud/aws shippable/aws/group3 + ec2_instance_info From 4eb0e510e8f01ce85b80cad6420fbd0e5aaaab78 Mon Sep 17 00:00:00 2001 From: Markus Bergholz Date: Sun, 14 Mar 2021 12:34:12 +0100 Subject: [PATCH 25/38] fix KeyError: 'Tags' for ec2_instance (#476) * fix key error when ec2 instance has no tags * add changelog fragment --- plugins/modules/ec2_instance.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/modules/ec2_instance.py b/plugins/modules/ec2_instance.py index 380e3527910..18af847aed6 100644 --- a/plugins/modules/ec2_instance.py +++ b/plugins/modules/ec2_instance.py @@ -889,7 +889,7 @@ def tower_callback_script(tower_conf, windows=False, passwd=None): def manage_tags(match, new_tags, purge_tags, ec2): changed = False - old_tags = boto3_tag_list_to_ansible_dict(match['Tags']) + old_tags = boto3_tag_list_to_ansible_dict(match.get('Tags', {})) tags_to_set, tags_to_delete = compare_aws_tags( old_tags, new_tags, purge_tags=purge_tags, @@ -1559,7 +1559,7 @@ def change_instance_state(filters, desired_state, ec2=None): def pretty_instance(i): instance = camel_dict_to_snake_dict(i, ignore_list=['Tags']) - instance['tags'] = boto3_tag_list_to_ansible_dict(i['Tags']) + instance['tags'] = boto3_tag_list_to_ansible_dict(i.get('Tags', {})) return instance From 6609c6daee8879c41b94032c263ec15b4a7508a5 Mon Sep 17 00:00:00 2001 From: Jill R <4121322+jillr@users.noreply.github.com> Date: Sat, 3 Apr 2021 07:12:18 -0700 Subject: [PATCH 26/38] ec2_instance don't change termination protection in check mode (#505) * ec2_instance don't change termination protection in check mode Fixes: ansible/ansible/issues/67716 Extend termination protection tests * Set the path for the aws CLI tool - setting ansible_python_interpreter updates the python search path but not the shell search path * changelog Co-authored-by: Mark Chappell --- plugins/modules/ec2_instance.py | 4 +- .../targets/ec2_instance/inventory | 2 +- .../tasks/termination_protection.yml | 87 +++++++++++++++++-- .../tasks/termination_protection_wrapper.yml | 32 +++++++ 4 files changed, 118 insertions(+), 7 deletions(-) create mode 100644 tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/termination_protection_wrapper.yml diff --git a/plugins/modules/ec2_instance.py b/plugins/modules/ec2_instance.py index 18af847aed6..9f61882491b 100644 --- a/plugins/modules/ec2_instance.py +++ b/plugins/modules/ec2_instance.py @@ -1512,6 +1512,7 @@ def change_instance_state(filters, desired_state, ec2=None): unchanged = set() failure_reason = "" + # TODO: better check_moding in here https://github.com/ansible-collections/community.aws/issues/16 for inst in instances: try: if desired_state == 'TERMINATED': @@ -1588,7 +1589,8 @@ def handle_existing(existing_matches, changed, ec2, state): ) changes = diff_instance_and_params(existing_matches[0], module.params) for c in changes: - ec2.modify_instance_attribute(aws_retry=True, **c) + if not module.check_mode: + ec2.modify_instance_attribute(aws_retry=True, **c) changed |= bool(changes) changed |= add_or_update_instance_profile(existing_matches[0], module.params.get('instance_role')) changed |= change_network_attachments(existing_matches[0], module.params, ec2) diff --git a/tests/integration/targets/ec2_instance/inventory b/tests/integration/targets/ec2_instance/inventory index 09bae76beb1..a49c076d2f2 100644 --- a/tests/integration/targets/ec2_instance/inventory +++ b/tests/integration/targets/ec2_instance/inventory @@ -8,7 +8,7 @@ default_vpc_tests external_resource_attach instance_no_wait iam_instance_role -termination_protection +termination_protection_wrapper tags_and_vpc_settings checkmode_tests security_group diff --git a/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/termination_protection.yml b/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/termination_protection.yml index 418e3c398dc..bcbef1bfd84 100644 --- a/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/termination_protection.yml +++ b/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/termination_protection.yml @@ -1,5 +1,4 @@ - block: - - name: Create instance with termination protection (check mode) ec2_instance: name: "{{ resource_prefix }}-termination-protection" @@ -35,6 +34,9 @@ wait: yes register: create_instance_results + - set_fact: + instance_id: '{{ create_instance_results.instances[0].instance_id }}' + - name: Check return values of the create instance task assert: that: @@ -42,6 +44,24 @@ - "'{{ create_instance_results.instances.0.state.name }}' == 'running'" - "'{{ create_instance_results.spec.DisableApiTermination }}'" + - name: Get info on termination protection + command: '{{ aws_cli }} ec2 describe-instance-attribute --attribute disableApiTermination --instance-id {{ instance_id }}' + environment: + AWS_ACCESS_KEY_ID: "{{ aws_access_key }}" + AWS_SECRET_ACCESS_KEY: "{{ aws_secret_key }}" + AWS_SESSION_TOKEN: "{{ security_token | default('') }}" + AWS_DEFAULT_REGION: "{{ aws_region }}" + register: instance_termination_check + + - name: convert it to an object + set_fact: + instance_termination_status: "{{ instance_termination_check.stdout | from_json }}" + + - name: Assert termination protection status did not change in check_mode + assert: + that: + - instance_termination_status.DisableApiTermination.Value == true + - name: Create instance with termination protection (check mode) (idempotent) ec2_instance: name: "{{ resource_prefix }}-termination-protection" @@ -90,13 +110,46 @@ failed_when: "'Unable to terminate instances' not in terminate_instance_results.msg" register: terminate_instance_results - # https://github.com/ansible/ansible/issues/67716 - # Updates to termination protection in check mode has a bug (listed above) + - name: Set termination protection to false (check_mode) + ec2_instance: + name: "{{ resource_prefix }}-termination-protection" + image_id: "{{ ec2_ami_image }}" + tags: + TestId: "{{ resource_prefix }}" + termination_protection: false + instance_type: "{{ ec2_instance_type }}" + vpc_subnet_id: "{{ testing_subnet_b.subnet.id }}" + check_mode: True + register: set_termination_protectioncheck_mode_results + + - name: Check return value + assert: + that: + - "{{ set_termination_protectioncheck_mode_results.changed }}" + + - name: Get info on termination protection + command: '{{ aws_cli }} ec2 describe-instance-attribute --attribute disableApiTermination --instance-id {{ instance_id }}' + environment: + AWS_ACCESS_KEY_ID: "{{ aws_access_key }}" + AWS_SECRET_ACCESS_KEY: "{{ aws_secret_key }}" + AWS_SESSION_TOKEN: "{{ security_token | default('') }}" + AWS_DEFAULT_REGION: "{{ aws_region }}" + register: instance_termination_check + + - name: convert it to an object + set_fact: + instance_termination_status: "{{ instance_termination_check.stdout | from_json }}" + + - assert: + that: + - instance_termination_status.DisableApiTermination.Value == true - name: Set termination protection to false ec2_instance: name: "{{ resource_prefix }}-termination-protection" image_id: "{{ ec2_ami_image }}" + tags: + TestId: "{{ resource_prefix }}" termination_protection: false instance_type: "{{ ec2_instance_type }}" vpc_subnet_id: "{{ testing_subnet_b.subnet.id }}" @@ -105,13 +158,31 @@ - name: Check return value assert: that: - - "{{ set_termination_protection_results.changed }}" - - "{{ not set_termination_protection_results.changes[0].DisableApiTermination.Value }}" + - set_termination_protection_results.changed + + - name: Get info on termination protection + command: '{{ aws_cli }} ec2 describe-instance-attribute --attribute disableApiTermination --instance-id {{ instance_id }}' + environment: + AWS_ACCESS_KEY_ID: "{{ aws_access_key }}" + AWS_SECRET_ACCESS_KEY: "{{ aws_secret_key }}" + AWS_SESSION_TOKEN: "{{ security_token | default('') }}" + AWS_DEFAULT_REGION: "{{ aws_region }}" + register: instance_termination_check + + - name: convert it to an object + set_fact: + instance_termination_status: "{{ instance_termination_check.stdout | from_json }}" + + - assert: + that: + - instance_termination_status.DisableApiTermination.Value == false - name: Set termination protection to false (idempotent) ec2_instance: name: "{{ resource_prefix }}-termination-protection" image_id: "{{ ec2_ami_image }}" + tags: + TestId: "{{ resource_prefix }}" termination_protection: false instance_type: "{{ ec2_instance_type }}" vpc_subnet_id: "{{ testing_subnet_b.subnet.id }}" @@ -126,6 +197,8 @@ ec2_instance: name: "{{ resource_prefix }}-termination-protection" image_id: "{{ ec2_ami_image }}" + tags: + TestId: "{{ resource_prefix }}" termination_protection: true instance_type: "{{ ec2_instance_type }}" vpc_subnet_id: "{{ testing_subnet_b.subnet.id }}" @@ -141,6 +214,8 @@ ec2_instance: name: "{{ resource_prefix }}-termination-protection" image_id: "{{ ec2_ami_image }}" + tags: + TestId: "{{ resource_prefix }}" termination_protection: true instance_type: "{{ ec2_instance_type }}" vpc_subnet_id: "{{ testing_subnet_b.subnet.id }}" @@ -155,6 +230,8 @@ ec2_instance: name: "{{ resource_prefix }}-termination-protection" image_id: "{{ ec2_ami_image }}" + tags: + TestId: "{{ resource_prefix }}" termination_protection: false instance_type: "{{ ec2_instance_type }}" vpc_subnet_id: "{{ testing_subnet_b.subnet.id }}" diff --git a/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/termination_protection_wrapper.yml b/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/termination_protection_wrapper.yml new file mode 100644 index 00000000000..41a00882bd1 --- /dev/null +++ b/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/termination_protection_wrapper.yml @@ -0,0 +1,32 @@ +--- +- include_role: + name: 'setup_remote_tmp_dir' + +- set_fact: + virtualenv: "{{ remote_tmp_dir }}/virtualenv" + virtualenv_command: "{{ ansible_python_interpreter }} -m virtualenv" + +- set_fact: + virtualenv_interpreter: "{{ virtualenv }}/bin/python" + aws_cli: "{{ virtualenv }}/bin/aws" + +- pip: + name: "virtualenv" + +- pip: + name: + - awscli<=1.18.159 + - botocore<1.19.0,>=1.13.3 + - boto3 >= 1.9.250, <= 1.15.18 + - coverage<5 + virtualenv: "{{ virtualenv }}" + virtualenv_command: "{{ virtualenv_command }}" + virtualenv_site_packages: no + +- include_tasks: termination_protection.yml + vars: + ansible_python_interpreter: "{{ virtualenv_interpreter }}" + +- file: + state: absent + path: "{{ virtualenv }}" From 82d2ad33e7e8544b02d974f65c8b194cdf57edfb Mon Sep 17 00:00:00 2001 From: Mark Chappell Date: Tue, 6 Apr 2021 16:51:10 +0200 Subject: [PATCH 27/38] ec2_instance_info - Add AWS Retries (#521) * Add retries to ec2_instance_info * changelog --- plugins/modules/ec2_instance_info.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/plugins/modules/ec2_instance_info.py b/plugins/modules/ec2_instance_info.py index be5f1e68892..4c743688627 100644 --- a/plugins/modules/ec2_instance_info.py +++ b/plugins/modules/ec2_instance_info.py @@ -511,17 +511,23 @@ try: import botocore - from botocore.exceptions import ClientError except ImportError: pass # Handled by AnsibleAWSModule from ansible.module_utils.common.dict_transformations import camel_dict_to_snake_dict from ansible_collections.amazon.aws.plugins.module_utils.core import AnsibleAWSModule +from ansible_collections.amazon.aws.plugins.module_utils.ec2 import AWSRetry from ansible_collections.amazon.aws.plugins.module_utils.ec2 import ansible_dict_to_boto3_filter_list from ansible_collections.amazon.aws.plugins.module_utils.ec2 import boto3_tag_list_to_ansible_dict +@AWSRetry.jittered_backoff() +def _describe_instances(connection, **params): + paginator = connection.get_paginator('describe_instances') + return paginator.paginate(**params).build_full_result() + + def list_ec2_instances(connection, module): instance_ids = module.params.get("instance_ids") @@ -529,9 +535,8 @@ def list_ec2_instances(connection, module): filters = ansible_dict_to_boto3_filter_list(module.params.get("filters")) try: - reservations_paginator = connection.get_paginator('describe_instances') - reservations = reservations_paginator.paginate(InstanceIds=instance_ids, Filters=filters).build_full_result() - except ClientError as e: + reservations = _describe_instances(connection, InstanceIds=instance_ids, Filters=filters) + except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: module.fail_json_aws(e, msg="Failed to list ec2 instances") instances = [] From ba0410cd1f5f1cd9c81db0c2558ee138f3cdf605 Mon Sep 17 00:00:00 2001 From: Jill R <4121322+jillr@users.noreply.github.com> Date: Thu, 8 Apr 2021 07:54:50 -0700 Subject: [PATCH 28/38] ec2_instance exception handling and client cleanup (#526) * ec2_instance exception handling and client cleanup Catch botocore and client errors on all API calls Pass boto client to functions, rather than creating new clients throughout the code * Add review suggestion to plugins/modules/ec2_instance.py Co-authored-by: Mark Chappell --- plugins/modules/ec2_instance.py | 155 +++++++++++++++------------ plugins/modules/ec2_instance_info.py | 2 +- 2 files changed, 89 insertions(+), 68 deletions(-) diff --git a/plugins/modules/ec2_instance.py b/plugins/modules/ec2_instance.py index 9f61882491b..22db3c88f79 100644 --- a/plugins/modules/ec2_instance.py +++ b/plugins/modules/ec2_instance.py @@ -896,19 +896,22 @@ def manage_tags(match, new_tags, purge_tags, ec2): ) if module.check_mode: return bool(tags_to_delete or tags_to_set) - if tags_to_set: - ec2.create_tags( - aws_retry=True, - Resources=[match['InstanceId']], - Tags=ansible_dict_to_boto3_tag_list(tags_to_set)) - changed |= True - if tags_to_delete: - delete_with_current_values = dict((k, old_tags.get(k)) for k in tags_to_delete) - ec2.delete_tags( - aws_retry=True, - Resources=[match['InstanceId']], - Tags=ansible_dict_to_boto3_tag_list(delete_with_current_values)) - changed |= True + try: + if tags_to_set: + ec2.create_tags( + aws_retry=True, + Resources=[match['InstanceId']], + Tags=ansible_dict_to_boto3_tag_list(tags_to_set)) + changed |= True + if tags_to_delete: + delete_with_current_values = dict((k, old_tags.get(k)) for k in tags_to_delete) + ec2.delete_tags( + aws_retry=True, + Resources=[match['InstanceId']], + Tags=ansible_dict_to_boto3_tag_list(delete_with_current_values)) + changed |= True + except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e: + module.fail_json_aws(e, msg="Could not update tags for instance {0}".format(match['InstanceId'])) return changed @@ -922,7 +925,7 @@ def build_volume_spec(params): return [snake_dict_to_camel_dict(v, capitalize_first=True) for v in volumes] -def add_or_update_instance_profile(instance, desired_profile_name): +def add_or_update_instance_profile(instance, desired_profile_name, ec2): instance_profile_setting = instance.get('IamInstanceProfile') if instance_profile_setting and desired_profile_name: if desired_profile_name in (instance_profile_setting.get('Name'), instance_profile_setting.get('Arn')): @@ -932,13 +935,13 @@ def add_or_update_instance_profile(instance, desired_profile_name): desired_arn = determine_iam_role(desired_profile_name) if instance_profile_setting.get('Arn') == desired_arn: return False + # update association - ec2 = module.client('ec2', retry_decorator=AWSRetry.jittered_backoff()) try: association = ec2.describe_iam_instance_profile_associations( aws_retry=True, Filters=[{'Name': 'instance-id', 'Values': [instance['InstanceId']]}]) - except botocore.exceptions.ClientError as e: + except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e: # check for InvalidAssociationID.NotFound module.fail_json_aws(e, "Could not find instance profile association") try: @@ -953,7 +956,6 @@ def add_or_update_instance_profile(instance, desired_profile_name): if not instance_profile_setting and desired_profile_name: # create association - ec2 = module.client('ec2', retry_decorator=AWSRetry.jittered_backoff()) try: resp = ec2.associate_iam_instance_profile( aws_retry=True, @@ -961,13 +963,13 @@ def add_or_update_instance_profile(instance, desired_profile_name): InstanceId=instance['InstanceId'] ) return True - except botocore.exceptions.ClientError as e: + except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e: module.fail_json_aws(e, "Could not associate new instance profile") return False -def build_network_spec(params, ec2=None): +def build_network_spec(params, ec2): """ Returns list of interfaces [complex] Interface type: { @@ -996,8 +998,6 @@ def build_network_spec(params, ec2=None): 'SubnetId': 'string' }, """ - if ec2 is None: - ec2 = module.client('ec2', retry_decorator=AWSRetry.jittered_backoff()) interfaces = [] network = params.get('network') or {} @@ -1116,8 +1116,6 @@ def warn_if_cpu_options_changed(instance): def discover_security_groups(group, groups, parent_vpc_id=None, subnet_id=None, ec2=None): - if ec2 is None: - ec2 = module.client('ec2', retry_decorator=AWSRetry.jittered_backoff()) if subnet_id is not None: try: @@ -1218,9 +1216,7 @@ def build_instance_tags(params, propagate_tags_to_volumes=True): ] -def build_run_instance_spec(params, ec2=None): - if ec2 is None: - ec2 = module.client('ec2', retry_decorator=AWSRetry.jittered_backoff()) +def build_run_instance_spec(params, ec2): spec = dict( ClientToken=uuid.uuid4().hex, @@ -1276,10 +1272,8 @@ def await_instances(ids, state='OK'): ', '.join(ids), state, to_native(e))) -def diff_instance_and_params(instance, params, ec2=None, skip=None): +def diff_instance_and_params(instance, params, ec2, skip=None): """boto3 instance obj, module params""" - if ec2 is None: - ec2 = module.client('ec2', retry_decorator=AWSRetry.jittered_backoff()) if skip is None: skip = [] @@ -1305,7 +1299,10 @@ def value_wrapper(v): if mapping.instance_key in skip: continue - value = ec2.describe_instance_attribute(aws_retry=True, Attribute=mapping.attribute_name, InstanceId=id_) + try: + value = ec2.describe_instance_attribute(aws_retry=True, Attribute=mapping.attribute_name, InstanceId=id_) + except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e: + module.fail_json_aws(e, msg="Could not describe attribute {0} for instance {1}".format(mapping.attribute_name, id_)) if value[mapping.instance_key]['Value'] != params.get(mapping.param_key): arguments = dict( InstanceId=instance['InstanceId'], @@ -1315,7 +1312,10 @@ def value_wrapper(v): changes_to_apply.append(arguments) if params.get('security_group') or params.get('security_groups'): - value = ec2.describe_instance_attribute(aws_retry=True, Attribute="groupSet", InstanceId=id_) + try: + value = ec2.describe_instance_attribute(aws_retry=True, Attribute="groupSet", InstanceId=id_) + except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e: + module.fail_json_aws(e, msg="Could not describe attribute groupSet for instance {0}".format(id_)) # managing security groups if params.get('vpc_subnet_id'): subnet_id = params.get('vpc_subnet_id') @@ -1366,12 +1366,15 @@ def change_network_attachments(instance, params, ec2): old_ids = [inty['NetworkInterfaceId'] for inty in instance['NetworkInterfaces']] to_attach = set(new_ids) - set(old_ids) for eni_id in to_attach: - ec2.attach_network_interface( - aws_retry=True, - DeviceIndex=new_ids.index(eni_id), - InstanceId=instance['InstanceId'], - NetworkInterfaceId=eni_id, - ) + try: + ec2.attach_network_interface( + aws_retry=True, + DeviceIndex=new_ids.index(eni_id), + InstanceId=instance['InstanceId'], + NetworkInterfaceId=eni_id, + ) + except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e: + module.fail_json_aws(e, msg="Could not attach interface {0} to instance {1}".format(eni_id, instance['InstanceId'])) return bool(len(to_attach)) return False @@ -1389,28 +1392,43 @@ def find_instances(ec2, ids=None, filters=None): filters[key.replace("_", "-")] = filters.pop(key) params = dict(Filters=ansible_dict_to_boto3_filter_list(filters)) - results = paginator.paginate(**params).search('Reservations[].Instances[]') + try: + results = _describe_instances(ec2, **params) + except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e: + module.fail_json_aws(e, msg="Could not describe instances") return list(results) +@AWSRetry.jittered_backoff() +def _describe_instances(ec2, **params): + paginator = ec2.get_paginator('describe_instances') + return paginator.paginate(**params).search('Reservations[].Instances[]') + + def get_default_vpc(ec2): - vpcs = ec2.describe_vpcs( - aws_retry=True, - Filters=ansible_dict_to_boto3_filter_list({'isDefault': 'true'})) + try: + vpcs = ec2.describe_vpcs( + aws_retry=True, + Filters=ansible_dict_to_boto3_filter_list({'isDefault': 'true'})) + except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e: + module.fail_json_aws(e, msg="Could not describe default VPC") if len(vpcs.get('Vpcs', [])): return vpcs.get('Vpcs')[0] return None def get_default_subnet(ec2, vpc, availability_zone=None): - subnets = ec2.describe_subnets( - aws_retry=True, - Filters=ansible_dict_to_boto3_filter_list({ - 'vpc-id': vpc['VpcId'], - 'state': 'available', - 'default-for-az': 'true', - }) - ) + try: + subnets = ec2.describe_subnets( + aws_retry=True, + Filters=ansible_dict_to_boto3_filter_list({ + 'vpc-id': vpc['VpcId'], + 'state': 'available', + 'default-for-az': 'true', + }) + ) + except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e: + module.fail_json_aws(e, msg="Could not describe default subnets for VPC {0}".format(vpc['VpcId'])) if len(subnets.get('Subnets', [])): if availability_zone is not None: subs_by_az = dict((subnet['AvailabilityZone'], subnet) for subnet in subnets.get('Subnets')) @@ -1424,11 +1442,9 @@ def get_default_subnet(ec2, vpc, availability_zone=None): return None -def ensure_instance_state(state, ec2=None): - if ec2 is None: - module.client('ec2', retry_decorator=AWSRetry.jittered_backoff()) +def ensure_instance_state(state, ec2): if state in ('running', 'started'): - changed, failed, instances, failure_reason = change_instance_state(filters=module.params.get('filters'), desired_state='RUNNING') + changed, failed, instances, failure_reason = change_instance_state(filters=module.params.get('filters'), desired_state='RUNNING', ec2=ec2) if failed: module.fail_json( @@ -1446,10 +1462,12 @@ def ensure_instance_state(state, ec2=None): elif state in ('restarted', 'rebooted'): changed, failed, instances, failure_reason = change_instance_state( filters=module.params.get('filters'), - desired_state='STOPPED') + desired_state='STOPPED', + ec2=ec2) changed, failed, instances, failure_reason = change_instance_state( filters=module.params.get('filters'), - desired_state='RUNNING') + desired_state='RUNNING', + ec2=ec2) if failed: module.fail_json( @@ -1467,7 +1485,8 @@ def ensure_instance_state(state, ec2=None): elif state in ('stopped',): changed, failed, instances, failure_reason = change_instance_state( filters=module.params.get('filters'), - desired_state='STOPPED') + desired_state='STOPPED', + ec2=ec2) if failed: module.fail_json( @@ -1485,7 +1504,8 @@ def ensure_instance_state(state, ec2=None): elif state in ('absent', 'terminated'): terminated, terminate_failed, instances, failure_reason = change_instance_state( filters=module.params.get('filters'), - desired_state='TERMINATED') + desired_state='TERMINATED', + ec2=ec2) if terminate_failed: module.fail_json( @@ -1501,10 +1521,8 @@ def ensure_instance_state(state, ec2=None): ) -def change_instance_state(filters, desired_state, ec2=None): +def change_instance_state(filters, desired_state, ec2): """Takes STOPPED/RUNNING/TERMINATED""" - if ec2 is None: - ec2 = module.client('ec2', retry_decorator=AWSRetry.jittered_backoff()) changed = set() instances = find_instances(ec2, filters=filters) @@ -1571,7 +1589,7 @@ def determine_iam_role(name_or_arn): try: role = iam.get_instance_profile(InstanceProfileName=name_or_arn, aws_retry=True) return role['InstanceProfile']['Arn'] - except is_boto3_error_code('NoSuchEntity'): + except is_boto3_error_code('NoSuchEntity') as e: module.fail_json_aws(e, msg="Could not find instance_role {0}".format(name_or_arn)) except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: # pylint: disable=duplicate-except module.fail_json_aws(e, msg="An error occurred while searching for instance_role {0}. Please try supplying the full ARN.".format(name_or_arn)) @@ -1579,7 +1597,7 @@ def determine_iam_role(name_or_arn): def handle_existing(existing_matches, changed, ec2, state): if state in ('running', 'started') and [i for i in existing_matches if i['State']['Name'] != 'running']: - ins_changed, failed, instances, failure_reason = change_instance_state(filters=module.params.get('filters'), desired_state='RUNNING') + ins_changed, failed, instances, failure_reason = change_instance_state(filters=module.params.get('filters'), desired_state='RUNNING', ec2=ec2) if failed: module.fail_json(msg="Couldn't start instances: {0}. Failure reason: {1}".format(instances, failure_reason)) module.exit_json( @@ -1587,12 +1605,15 @@ def handle_existing(existing_matches, changed, ec2, state): instances=[pretty_instance(i) for i in instances], instance_ids=[i['InstanceId'] for i in instances], ) - changes = diff_instance_and_params(existing_matches[0], module.params) + changes = diff_instance_and_params(existing_matches[0], module.params, ec2) for c in changes: if not module.check_mode: - ec2.modify_instance_attribute(aws_retry=True, **c) + try: + ec2.modify_instance_attribute(aws_retry=True, **c) + except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e: + module.fail_json_aws(e, msg="Could not apply change {0} to existing instance.".format(str(c))) changed |= bool(changes) - changed |= add_or_update_instance_profile(existing_matches[0], module.params.get('instance_role')) + changed |= add_or_update_instance_profile(existing_matches[0], module.params.get('instance_role'), ec2) changed |= change_network_attachments(existing_matches[0], module.params, ec2) altered = find_instances(ec2, ids=[i['InstanceId'] for i in existing_matches]) module.exit_json( @@ -1614,7 +1635,7 @@ def ensure_present(existing_matches, changed, ec2, state): # instance_ids=[i['InstanceId'] for i in existing_matches], ) try: - instance_spec = build_run_instance_spec(module.params) + instance_spec = build_run_instance_spec(module.params, ec2) # If check mode is enabled,suspend 'ensure function'. if module.check_mode: module.exit_json( @@ -1626,7 +1647,7 @@ def ensure_present(existing_matches, changed, ec2, state): instance_ids = [i['InstanceId'] for i in instances] for ins in instances: - changes = diff_instance_and_params(ins, module.params, skip=['UserData', 'EbsOptimized']) + changes = diff_instance_and_params(ins, module.params, ec2, skip=['UserData', 'EbsOptimized']) for c in changes: try: ec2.modify_instance_attribute(aws_retry=True, **c) diff --git a/plugins/modules/ec2_instance_info.py b/plugins/modules/ec2_instance_info.py index 4c743688627..dafe60ea4dd 100644 --- a/plugins/modules/ec2_instance_info.py +++ b/plugins/modules/ec2_instance_info.py @@ -512,7 +512,7 @@ try: import botocore except ImportError: - pass # Handled by AnsibleAWSModule + pass # caught by AnsibleAWSModule from ansible.module_utils.common.dict_transformations import camel_dict_to_snake_dict From 65b05d67bb9cfce931e08a5ac66c8b78500f1268 Mon Sep 17 00:00:00 2001 From: Mark Chappell Date: Fri, 9 Apr 2021 14:48:31 +0200 Subject: [PATCH 29/38] ec2_instance - fetch status of instance before attempting to set additional parameters (#533) --- plugins/modules/ec2_instance.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/plugins/modules/ec2_instance.py b/plugins/modules/ec2_instance.py index 22db3c88f79..5138fd7647a 100644 --- a/plugins/modules/ec2_instance.py +++ b/plugins/modules/ec2_instance.py @@ -1647,6 +1647,18 @@ def ensure_present(existing_matches, changed, ec2, state): instance_ids = [i['InstanceId'] for i in instances] for ins in instances: + # Wait for instances to exist (don't check state) + try: + AWSRetry.jittered_backoff( + catch_extra_error_codes=['InvalidInstanceID.NotFound'], + )( + ec2.describe_instance_status + )( + InstanceIds=[ins['InstanceId']], + IncludeAllInstances=True, + ) + except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e: + module.fail_json_aws(e, msg="Failed to fetch status of new EC2 instance") changes = diff_instance_and_params(ins, module.params, ec2, skip=['UserData', 'EbsOptimized']) for c in changes: try: From 9c42b6c3f2cb1c66f54c9cd8fca5ae79f29e0941 Mon Sep 17 00:00:00 2001 From: Jill R <4121322+jillr@users.noreply.github.com> Date: Thu, 13 May 2021 11:16:37 -0700 Subject: [PATCH 30/38] Migrate ec2_instance modules (#354) Migrate ec2 instance modules Reviewed-by: https://github.com/apps/ansible-zuul --- changelogs/fragments/migrate_ec2_instance.yml | 3 +++ meta/runtime.yml | 2 ++ plugins/modules/ec2_instance.py | 18 +++++++++--------- plugins/modules/ec2_instance_facts.py | 1 + plugins/modules/ec2_instance_info.py | 14 +++++++------- tests/integration/targets/ec2_instance/aliases | 2 -- .../roles/ec2_instance/defaults/main.yml | 2 ++ .../ec2_instance/tasks/iam_instance_role.yml | 8 ++++---- .../roles/ec2_instance/tasks/main.yml | 2 +- tests/sanity/ignore-2.9.txt | 1 + 10 files changed, 30 insertions(+), 23 deletions(-) create mode 100644 changelogs/fragments/migrate_ec2_instance.yml create mode 120000 plugins/modules/ec2_instance_facts.py diff --git a/changelogs/fragments/migrate_ec2_instance.yml b/changelogs/fragments/migrate_ec2_instance.yml new file mode 100644 index 00000000000..988822f7d08 --- /dev/null +++ b/changelogs/fragments/migrate_ec2_instance.yml @@ -0,0 +1,3 @@ +major_changes: + - ec2_instance - The module has been migrated from the ``community.aws`` collection. Playbooks using the Fully Qualified Collection Name for this module should be updated to use ``amazon.aws.ec2_instance``. + - ec2_instance_info - The module has been migrated from the ``community.aws`` collection. Playbooks using the Fully Qualified Collection Name for this module should be updated to use ``amazon.aws.ec2_instance_info``. diff --git a/meta/runtime.yml b/meta/runtime.yml index 6b938e41c4c..df3b4370954 100644 --- a/meta/runtime.yml +++ b/meta/runtime.yml @@ -33,6 +33,8 @@ action_groups: - ec2_eni_info - ec2_group - ec2_group_info + - ec2_instance + - ec2_instance_info - ec2_key - ec2_snapshot - ec2_snapshot_info diff --git a/plugins/modules/ec2_instance.py b/plugins/modules/ec2_instance.py index 5138fd7647a..81841761a53 100644 --- a/plugins/modules/ec2_instance.py +++ b/plugins/modules/ec2_instance.py @@ -120,7 +120,7 @@ vpc_subnet_id: description: - The subnet ID in which to launch the instance (VPC) - If none is provided, M(community.aws.ec2_instance) will chose the default zone of the default VPC. + If none is provided, M(amazon.aws.ec2_instance) will chose the default zone of the default VPC. aliases: ['subnet_id'] type: str network: @@ -287,19 +287,19 @@ # Note: These examples do not set authentication details, see the AWS Guide for details. - name: Terminate every running instance in a region. Use with EXTREME caution. - community.aws.ec2_instance: + amazon.aws.ec2_instance: state: absent filters: instance-state-name: running - name: restart a particular instance by its ID - community.aws.ec2_instance: + amazon.aws.ec2_instance: state: restarted instance_ids: - i-12345678 - name: start an instance with a public IP address - community.aws.ec2_instance: + amazon.aws.ec2_instance: name: "public-compute-instance" key_name: "prod-ssh-key" vpc_subnet_id: subnet-5ca1ab1e @@ -312,7 +312,7 @@ Environment: Testing - name: start an instance and Add EBS - community.aws.ec2_instance: + amazon.aws.ec2_instance: name: "public-withebs-instance" vpc_subnet_id: subnet-5ca1ab1e instance_type: t2.micro @@ -325,7 +325,7 @@ delete_on_termination: true - name: start an instance with a cpu_options - community.aws.ec2_instance: + amazon.aws.ec2_instance: name: "public-cpuoption-instance" vpc_subnet_id: subnet-5ca1ab1e tags: @@ -340,7 +340,7 @@ threads_per_core: 1 - name: start an instance and have it begin a Tower callback on boot - community.aws.ec2_instance: + amazon.aws.ec2_instance: name: "tower-callback-test" key_name: "prod-ssh-key" vpc_subnet_id: subnet-5ca1ab1e @@ -358,7 +358,7 @@ SomeThing: "A value" - name: start an instance with ENI (An existing ENI ID is required) - community.aws.ec2_instance: + amazon.aws.ec2_instance: name: "public-eni-instance" key_name: "prod-ssh-key" vpc_subnet_id: subnet-5ca1ab1e @@ -375,7 +375,7 @@ image_id: ami-123456 - name: add second ENI interface - community.aws.ec2_instance: + amazon.aws.ec2_instance: name: "public-eni-instance" network: interfaces: diff --git a/plugins/modules/ec2_instance_facts.py b/plugins/modules/ec2_instance_facts.py new file mode 120000 index 00000000000..7010fdcb95f --- /dev/null +++ b/plugins/modules/ec2_instance_facts.py @@ -0,0 +1 @@ +ec2_instance_info.py \ No newline at end of file diff --git a/plugins/modules/ec2_instance_info.py b/plugins/modules/ec2_instance_info.py index dafe60ea4dd..ff6fca00075 100644 --- a/plugins/modules/ec2_instance_info.py +++ b/plugins/modules/ec2_instance_info.py @@ -51,30 +51,30 @@ # Note: These examples do not set authentication details, see the AWS Guide for details. - name: Gather information about all instances - community.aws.ec2_instance_info: + amazon.aws.ec2_instance_info: - name: Gather information about all instances in AZ ap-southeast-2a - community.aws.ec2_instance_info: + amazon.aws.ec2_instance_info: filters: availability-zone: ap-southeast-2a - name: Gather information about a particular instance using ID - community.aws.ec2_instance_info: + amazon.aws.ec2_instance_info: instance_ids: - i-12345678 - name: Gather information about any instance with a tag key Name and value Example - community.aws.ec2_instance_info: + amazon.aws.ec2_instance_info: filters: "tag:Name": Example - name: Gather information about any instance in states "shutting-down", "stopping", "stopped" - community.aws.ec2_instance_info: + amazon.aws.ec2_instance_info: filters: instance-state-name: [ "shutting-down", "stopping", "stopped" ] - name: Gather information about any instance with Name beginning with RHEL and an uptime of at least 60 minutes - community.aws.ec2_instance_info: + amazon.aws.ec2_instance_info: region: "{{ ec2_region }}" uptime: 60 filters: @@ -577,7 +577,7 @@ def main(): supports_check_mode=True, ) if module._name == 'ec2_instance_facts': - module.deprecate("The 'ec2_instance_facts' module has been renamed to 'ec2_instance_info'", date='2021-12-01', collection_name='community.aws') + module.deprecate("The 'ec2_instance_facts' module has been renamed to 'ec2_instance_info'", date='2021-12-01', collection_name='amazon.aws') try: connection = module.client('ec2') diff --git a/tests/integration/targets/ec2_instance/aliases b/tests/integration/targets/ec2_instance/aliases index c451297ee90..6a794c03bc1 100644 --- a/tests/integration/targets/ec2_instance/aliases +++ b/tests/integration/targets/ec2_instance/aliases @@ -1,4 +1,2 @@ cloud/aws -shippable/aws/group3 - ec2_instance_info diff --git a/tests/integration/targets/ec2_instance/roles/ec2_instance/defaults/main.yml b/tests/integration/targets/ec2_instance/roles/ec2_instance/defaults/main.yml index 8e70ab6933c..5dc70554e02 100644 --- a/tests/integration/targets/ec2_instance/roles/ec2_instance/defaults/main.yml +++ b/tests/integration/targets/ec2_instance/roles/ec2_instance/defaults/main.yml @@ -12,3 +12,5 @@ subnet_a_cidr: '10.{{ 256 | random(seed=vpc_seed) }}.32.0/24' subnet_a_startswith: '10.{{ 256 | random(seed=vpc_seed) }}.32.' subnet_b_cidr: '10.{{ 256 | random(seed=vpc_seed) }}.33.0/24' subnet_b_startswith: '10.{{ 256 | random(seed=vpc_seed) }}.33.' +first_iam_role: "ansible-test-sts-{{ resource_prefix | hash('md5') }}-test-policy" +second_iam_role: "ansible-test-sts-{{ resource_prefix | hash('md5') }}-test-policy-2" diff --git a/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/iam_instance_role.yml b/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/iam_instance_role.yml index 6e29b74674f..f2da199e02b 100644 --- a/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/iam_instance_role.yml +++ b/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/iam_instance_role.yml @@ -2,7 +2,7 @@ - name: "Create IAM role for test" iam_role: state: present - name: "ansible-test-sts-{{ resource_prefix }}-test-policy" + name: '{{ first_iam_role }}' assume_role_policy_document: "{{ lookup('file','assume-role-policy.json') }}" create_instance_profile: yes managed_policy: @@ -12,7 +12,7 @@ - name: "Create second IAM role for test" iam_role: state: present - name: "ansible-test-sts-{{ resource_prefix }}-test-policy-2" + name: '{{ second_iam_role }}' assume_role_policy_document: "{{ lookup('file','assume-role-policy.json') }}" create_instance_profile: yes managed_policy: @@ -119,8 +119,8 @@ managed_policy: - AmazonEC2ContainerServiceRole loop: - - "ansible-test-sts-{{ resource_prefix }}-test-policy" - - "ansible-test-sts-{{ resource_prefix }}-test-policy-2" + - '{{ first_iam_role }}' + - '{{ second_iam_role }}' register: removed until: removed is not failed ignore_errors: yes diff --git a/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/main.yml b/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/main.yml index dc81199aabe..5f06153db1a 100644 --- a/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/main.yml +++ b/tests/integration/targets/ec2_instance/roles/ec2_instance/tasks/main.yml @@ -30,7 +30,7 @@ # don't support any configuration of the delay between retries. max_attempts: 20 collections: - - amazon.aws + - community.aws block: - debug: msg: "{{ inventory_hostname }} start: {{ lookup('pipe','date') }}" diff --git a/tests/sanity/ignore-2.9.txt b/tests/sanity/ignore-2.9.txt index 7815aaa28dd..d9d68e5a3d2 100644 --- a/tests/sanity/ignore-2.9.txt +++ b/tests/sanity/ignore-2.9.txt @@ -16,3 +16,4 @@ plugins/modules/ec2_vpc_subnet_info.py pylint:ansible-deprecated-no-version # W plugins/module_utils/compat/_ipaddress.py no-assert # Vendored library plugins/module_utils/compat/_ipaddress.py no-unicode-literals # Vendored library plugins/module_utils/ec2.py pylint:ansible-deprecated-no-version # We use dates for deprecations, Ansible 2.9 only supports this for compatability +plugins/modules/ec2_instance_info.py pylint:ansible-deprecated-no-version # We use dates for deprecations, Ansible 2.9 only supports this for compatability From 76b04f81401c1349a149b65689eba16aa251f1b0 Mon Sep 17 00:00:00 2001 From: Mark Chappell Date: Wed, 19 May 2021 10:57:22 +0200 Subject: [PATCH 31/38] scrub_none_parameters - set default for descend_into_lists to True (breaking change) (#297) scrub_none_parameters - set default for descend_into_lists to True (breaking change) Reviewed-by: https://github.com/apps/ansible-zuul --- .../fragments/297-scrub_none_parameters-descend-default.yml | 2 ++ plugins/module_utils/core.py | 2 +- tests/unit/module_utils/core/test_scrub_none_parameters.py | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 changelogs/fragments/297-scrub_none_parameters-descend-default.yml diff --git a/changelogs/fragments/297-scrub_none_parameters-descend-default.yml b/changelogs/fragments/297-scrub_none_parameters-descend-default.yml new file mode 100644 index 00000000000..8874e379b04 --- /dev/null +++ b/changelogs/fragments/297-scrub_none_parameters-descend-default.yml @@ -0,0 +1,2 @@ +breaking_changes: +- module_utils/core - updated the ``scrub_none_parameters`` function so that ``descend_into_lists`` is set to ``True`` by default (https://github.com/ansible-collections/amazon.aws/pull/297). diff --git a/plugins/module_utils/core.py b/plugins/module_utils/core.py index 7e72843d62a..44855fdf3af 100644 --- a/plugins/module_utils/core.py +++ b/plugins/module_utils/core.py @@ -360,7 +360,7 @@ def get_boto3_client_method_parameters(client, method_name, required=False): return parameters -def scrub_none_parameters(parameters, descend_into_lists=False): +def scrub_none_parameters(parameters, descend_into_lists=True): """ Iterate over a dictionary removing any keys that have a None value diff --git a/tests/unit/module_utils/core/test_scrub_none_parameters.py b/tests/unit/module_utils/core/test_scrub_none_parameters.py index a1a1b491788..8c1faf42832 100644 --- a/tests/unit/module_utils/core/test_scrub_none_parameters.py +++ b/tests/unit/module_utils/core/test_scrub_none_parameters.py @@ -83,6 +83,6 @@ @pytest.mark.parametrize("input_params, output_params_no_descend, output_params_descend", scrub_none_test_data) def test_scrub_none_parameters(input_params, output_params_no_descend, output_params_descend): - assert scrub_none_parameters(input_params) == output_params_no_descend + assert scrub_none_parameters(input_params) == output_params_descend assert scrub_none_parameters(input_params, descend_into_lists=False) == output_params_no_descend assert scrub_none_parameters(input_params, descend_into_lists=True) == output_params_descend From b6c067268535329efd7933449a694b34cfbbf5a9 Mon Sep 17 00:00:00 2001 From: Mark Chappell Date: Wed, 19 May 2021 19:22:29 +0200 Subject: [PATCH 32/38] Core parameter sanity test cleanup (breaking change) (#290) Core parameter sanity test cleanup (breaking change) Reviewed-by: https://github.com/apps/ansible-zuul --- changelogs/fragments/290-lint-cleanup.yml | 2 ++ plugins/module_utils/core.py | 4 ++-- tests/sanity/ignore-2.11.txt | 1 - tests/sanity/ignore-2.12.txt | 1 - 4 files changed, 4 insertions(+), 4 deletions(-) create mode 100644 changelogs/fragments/290-lint-cleanup.yml diff --git a/changelogs/fragments/290-lint-cleanup.yml b/changelogs/fragments/290-lint-cleanup.yml new file mode 100644 index 00000000000..36ab84b6df3 --- /dev/null +++ b/changelogs/fragments/290-lint-cleanup.yml @@ -0,0 +1,2 @@ +breaking_changes: +- module_utils.core - The boto3 switch has been removed from the region parameter (https://github.com/ansible-collections/amazon.aws/pull/287). diff --git a/plugins/module_utils/core.py b/plugins/module_utils/core.py index 44855fdf3af..35fc24df98b 100644 --- a/plugins/module_utils/core.py +++ b/plugins/module_utils/core.py @@ -190,8 +190,8 @@ def resource(self, service): region=region, endpoint=ec2_url, **aws_connect_kwargs) @property - def region(self, boto3=True): - return get_aws_region(self, boto3) + def region(self): + return get_aws_region(self, True) def fail_json_aws(self, exception, msg=None, **kwargs): """call fail_json with processed exception diff --git a/tests/sanity/ignore-2.11.txt b/tests/sanity/ignore-2.11.txt index 82cef71c1ff..31a4d4c9c6d 100644 --- a/tests/sanity/ignore-2.11.txt +++ b/tests/sanity/ignore-2.11.txt @@ -2,4 +2,3 @@ plugins/modules/ec2_tag.py validate-modules:parameter-state-invalid-choice # De plugins/modules/ec2_vol.py validate-modules:parameter-state-invalid-choice # Deprecated choice that can't be removed until 2022 plugins/module_utils/compat/_ipaddress.py no-assert # Vendored library plugins/module_utils/compat/_ipaddress.py no-unicode-literals # Vendored library -plugins/module_utils/core.py pylint:property-with-parameters # Breaking change required to fix - https://github.com/ansible-collections/amazon.aws/pull/290 diff --git a/tests/sanity/ignore-2.12.txt b/tests/sanity/ignore-2.12.txt index 82cef71c1ff..31a4d4c9c6d 100644 --- a/tests/sanity/ignore-2.12.txt +++ b/tests/sanity/ignore-2.12.txt @@ -2,4 +2,3 @@ plugins/modules/ec2_tag.py validate-modules:parameter-state-invalid-choice # De plugins/modules/ec2_vol.py validate-modules:parameter-state-invalid-choice # Deprecated choice that can't be removed until 2022 plugins/module_utils/compat/_ipaddress.py no-assert # Vendored library plugins/module_utils/compat/_ipaddress.py no-unicode-literals # Vendored library -plugins/module_utils/core.py pylint:property-with-parameters # Breaking change required to fix - https://github.com/ansible-collections/amazon.aws/pull/290 From 33c7196f984cfc02ca6d609cc0237102643d85a4 Mon Sep 17 00:00:00 2001 From: Mark Chappell Date: Wed, 19 May 2021 19:25:13 +0200 Subject: [PATCH 33/38] Update the default module requirements from python-2.6/boto to python-3.6/botocore-1.16 (breaking change) (#298) Update the default module requirements from python-2.6/boto to python-3.6/botocore-1.16 (breaking change) Reviewed-by: https://github.com/apps/ansible-zuul --- changelogs/fragments/298-python3.6.yml | 2 ++ plugins/doc_fragments/aws.py | 11 ++++++----- plugins/lookup/aws_account_attribute.py | 3 ++- plugins/lookup/aws_secret.py | 3 ++- plugins/lookup/aws_ssm.py | 3 ++- plugins/modules/aws_az_info.py | 2 -- plugins/modules/aws_caller_info.py | 1 - plugins/modules/aws_s3.py | 3 --- plugins/modules/cloudformation.py | 4 +--- plugins/modules/cloudformation_info.py | 3 --- plugins/modules/ec2.py | 3 +++ plugins/modules/ec2_ami_info.py | 1 - plugins/modules/ec2_elb_lb.py | 4 ++++ plugins/modules/ec2_eni_info.py | 1 - plugins/modules/ec2_group.py | 1 - plugins/modules/ec2_group_info.py | 1 - plugins/modules/ec2_key.py | 1 - plugins/modules/ec2_snapshot_info.py | 1 - plugins/modules/ec2_tag.py | 1 - plugins/modules/ec2_tag_info.py | 1 - plugins/modules/ec2_vol_info.py | 1 - plugins/modules/ec2_vpc_dhcp_option.py | 3 --- plugins/modules/ec2_vpc_dhcp_option_info.py | 1 - plugins/modules/ec2_vpc_net.py | 3 --- plugins/modules/ec2_vpc_net_info.py | 3 --- plugins/modules/ec2_vpc_subnet.py | 1 - plugins/modules/ec2_vpc_subnet_info.py | 3 --- plugins/modules/s3_bucket.py | 1 - requirements.txt | 4 ++-- tests/unit/requirements.txt | 2 +- 30 files changed, 25 insertions(+), 47 deletions(-) create mode 100644 changelogs/fragments/298-python3.6.yml diff --git a/changelogs/fragments/298-python3.6.yml b/changelogs/fragments/298-python3.6.yml new file mode 100644 index 00000000000..600414b3412 --- /dev/null +++ b/changelogs/fragments/298-python3.6.yml @@ -0,0 +1,2 @@ +major_changes: +- amazon.aws collection - Due to the AWS SDKs announcing the end of support for Python less than 3.6 (https://boto3.amazonaws.com/v1/documentation/api/1.17.64/guide/migrationpy3.html) this collection now requires Python 3.6+ (https://github.com/ansible-collections/amazon.aws/pull/298). diff --git a/plugins/doc_fragments/aws.py b/plugins/doc_fragments/aws.py index 9eec9a8b3bd..ce7a6eab2fd 100644 --- a/plugins/doc_fragments/aws.py +++ b/plugins/doc_fragments/aws.py @@ -53,17 +53,17 @@ class ModuleDocFragment(object): aws_ca_bundle: description: - "The location of a CA Bundle to use when validating SSL certificates." - - "Only used for boto3 based modules." + - "Not used by boto 2 based modules." - "Note: The CA Bundle is read 'module' side and may need to be explicitly copied from the controller if not run locally." type: path validate_certs: description: - - When set to "no", SSL certificates will not be validated for boto versions >= 2.6.0. + - When set to "no", SSL certificates will not be validated for + communication with the AWS APIs. type: bool default: yes profile: description: - - Uses a boto profile. Only works with boto >= 2.24.0. - Using I(profile) will override I(aws_access_key), I(aws_secret_key) and I(security_token) and support for passing them at the same time as I(profile) has been deprecated. - I(aws_access_key), I(aws_secret_key) and I(security_token) will be made mutually exclusive with I(profile) after 2022-06-01. @@ -76,8 +76,9 @@ class ModuleDocFragment(object): - Only the 'user_agent' key is used for boto modules. See U(http://boto.cloudhackers.com/en/latest/boto_config_tut.html#boto) for more boto configuration. type: dict requirements: - - python >= 2.6 - - boto + - python >= 3.6 + - boto3 >= 1.13.0 + - botocore >= 1.16.0 notes: - If parameters are not set within the module, the following environment variables can be used in decreasing order of precedence diff --git a/plugins/lookup/aws_account_attribute.py b/plugins/lookup/aws_account_attribute.py index e1ba8f23ddf..9b79aa26861 100644 --- a/plugins/lookup/aws_account_attribute.py +++ b/plugins/lookup/aws_account_attribute.py @@ -8,8 +8,9 @@ author: - Sloane Hertel requirements: + - python >= 3.6 - boto3 - - botocore + - botocore >= 1.16.0 extends_documentation_fragment: - amazon.aws.aws_credentials - amazon.aws.aws_region diff --git a/plugins/lookup/aws_secret.py b/plugins/lookup/aws_secret.py index ef7a8f9a909..5e8b2602c00 100644 --- a/plugins/lookup/aws_secret.py +++ b/plugins/lookup/aws_secret.py @@ -9,8 +9,9 @@ author: - Aaron Smith requirements: + - python >= 3.6 - boto3 - - botocore>=1.10.0 + - botocore >= 1.16.0 extends_documentation_fragment: - amazon.aws.aws_credentials - amazon.aws.aws_region diff --git a/plugins/lookup/aws_ssm.py b/plugins/lookup/aws_ssm.py index c9de00d4389..0a4646c4478 100644 --- a/plugins/lookup/aws_ssm.py +++ b/plugins/lookup/aws_ssm.py @@ -14,8 +14,9 @@ - Marat Bakeev - Michael De La Rue requirements: + - python >= 3.6 - boto3 - - botocore + - botocore >= 1.16.0 short_description: Get the value for a SSM parameter or all parameters under a path. description: - Get the value for an Amazon Simple Systems Manager parameter or a hierarchy of parameters. diff --git a/plugins/modules/aws_az_info.py b/plugins/modules/aws_az_info.py index 42f1232345c..1aef86f5cea 100644 --- a/plugins/modules/aws_az_info.py +++ b/plugins/modules/aws_az_info.py @@ -29,8 +29,6 @@ extends_documentation_fragment: - amazon.aws.aws - amazon.aws.ec2 - -requirements: [botocore, boto3] ''' EXAMPLES = ''' diff --git a/plugins/modules/aws_caller_info.py b/plugins/modules/aws_caller_info.py index 91880fdba1e..a66e7c6b9c7 100644 --- a/plugins/modules/aws_caller_info.py +++ b/plugins/modules/aws_caller_info.py @@ -20,7 +20,6 @@ - Ed Costello (@orthanc) - Stijn Dubrul (@sdubrul) -requirements: [ 'botocore', 'boto3' ] extends_documentation_fragment: - amazon.aws.aws - amazon.aws.ec2 diff --git a/plugins/modules/aws_s3.py b/plugins/modules/aws_s3.py index 50dac4561a4..f7628ae9ad1 100644 --- a/plugins/modules/aws_s3.py +++ b/plugins/modules/aws_s3.py @@ -14,7 +14,6 @@ description: - This module allows the user to manage S3 buckets and the objects within them. Includes support for creating and deleting both objects and buckets, retrieving objects as files or strings and generating download links. - This module has a dependency on boto3 and botocore. options: bucket: description: @@ -118,7 +117,6 @@ dualstack: description: - Enables Amazon S3 Dual-Stack Endpoints, allowing S3 communications using both IPv4 and IPv6. - - Requires at least botocore version 1.4.45. type: bool default: false rgw: @@ -157,7 +155,6 @@ description: - KMS key id to use when encrypting objects using I(encrypting=aws:kms). Ignored if I(encryption) is not C(aws:kms). type: str -requirements: [ "boto3", "botocore" ] author: - "Lester Wade (@lwade)" - "Sloane Hertel (@s-hertel)" diff --git a/plugins/modules/cloudformation.py b/plugins/modules/cloudformation.py index 76fb55f19b7..fd42caeac52 100644 --- a/plugins/modules/cloudformation.py +++ b/plugins/modules/cloudformation.py @@ -128,7 +128,7 @@ type: str termination_protection: description: - - Enable or disable termination protection on the stack. Only works with botocore >= 1.7.18. + - Enable or disable termination protection on the stack. type: bool template_body: description: @@ -174,8 +174,6 @@ extends_documentation_fragment: - amazon.aws.aws - amazon.aws.ec2 - -requirements: [ boto3, botocore>=1.5.45 ] ''' EXAMPLES = ''' diff --git a/plugins/modules/cloudformation_info.py b/plugins/modules/cloudformation_info.py index 0c34e8b1d18..492fb23bc8d 100644 --- a/plugins/modules/cloudformation_info.py +++ b/plugins/modules/cloudformation_info.py @@ -15,9 +15,6 @@ - Gets information about an AWS CloudFormation stack. - This module was called C(amazon.aws.cloudformation_facts) before Ansible 2.9, returning C(ansible_facts). Note that the M(amazon.aws.cloudformation_info) module no longer returns C(ansible_facts)! -requirements: - - boto3 >= 1.0.0 - - python >= 2.6 author: - Justin Menga (@jmenga) - Kevin Coming (@waffie1) diff --git a/plugins/modules/ec2.py b/plugins/modules/ec2.py index 990a7e69be5..bb76b3109b1 100644 --- a/plugins/modules/ec2.py +++ b/plugins/modules/ec2.py @@ -257,6 +257,9 @@ extends_documentation_fragment: - amazon.aws.aws - amazon.aws.ec2 +requirements: +- python >= 2.6 +- boto ''' diff --git a/plugins/modules/ec2_ami_info.py b/plugins/modules/ec2_ami_info.py index 11c1bb6e687..3dd8f71e69c 100644 --- a/plugins/modules/ec2_ami_info.py +++ b/plugins/modules/ec2_ami_info.py @@ -16,7 +16,6 @@ - This module was called C(amazon.aws.ec2_ami_facts) before Ansible 2.9. The usage did not change. author: - Prasad Katti (@prasadkatti) -requirements: [ boto3 ] options: image_ids: description: One or more image IDs. diff --git a/plugins/modules/ec2_elb_lb.py b/plugins/modules/ec2_elb_lb.py index 9a005b29faf..1c3c8d34fbb 100644 --- a/plugins/modules/ec2_elb_lb.py +++ b/plugins/modules/ec2_elb_lb.py @@ -132,6 +132,10 @@ - amazon.aws.aws - amazon.aws.ec2 +requirements: +- python >= 2.6 +- boto + ''' EXAMPLES = """ diff --git a/plugins/modules/ec2_eni_info.py b/plugins/modules/ec2_eni_info.py index 17a5fff38ea..9ed67bcabfa 100644 --- a/plugins/modules/ec2_eni_info.py +++ b/plugins/modules/ec2_eni_info.py @@ -15,7 +15,6 @@ - Gather information about ec2 ENI interfaces in AWS. - This module was called C(ec2_eni_facts) before Ansible 2.9. The usage did not change. author: "Rob White (@wimnat)" -requirements: [ boto3 ] options: eni_id: description: diff --git a/plugins/modules/ec2_group.py b/plugins/modules/ec2_group.py index 8bbb112a313..7683ecb1a83 100644 --- a/plugins/modules/ec2_group.py +++ b/plugins/modules/ec2_group.py @@ -12,7 +12,6 @@ module: ec2_group version_added: 1.0.0 author: "Andrew de Quincey (@adq)" -requirements: [ boto3 ] short_description: maintain an ec2 VPC security group. description: - Maintains ec2 security groups. diff --git a/plugins/modules/ec2_group_info.py b/plugins/modules/ec2_group_info.py index 228b82d9923..63d9e7ecfca 100644 --- a/plugins/modules/ec2_group_info.py +++ b/plugins/modules/ec2_group_info.py @@ -14,7 +14,6 @@ description: - Gather information about ec2 security groups in AWS. - This module was called C(amazon.aws.ec2_group_facts) before Ansible 2.9. The usage did not change. -requirements: [ boto3 ] author: - Henrique Rodrigues (@Sodki) options: diff --git a/plugins/modules/ec2_key.py b/plugins/modules/ec2_key.py index 1a861c11635..e9e8660b7cf 100644 --- a/plugins/modules/ec2_key.py +++ b/plugins/modules/ec2_key.py @@ -52,7 +52,6 @@ - amazon.aws.aws - amazon.aws.ec2 -requirements: [ boto3 ] author: - "Vincent Viallet (@zbal)" - "Prasad Katti (@prasadkatti)" diff --git a/plugins/modules/ec2_snapshot_info.py b/plugins/modules/ec2_snapshot_info.py index de949553411..087d7c1ca2c 100644 --- a/plugins/modules/ec2_snapshot_info.py +++ b/plugins/modules/ec2_snapshot_info.py @@ -14,7 +14,6 @@ description: - Gather information about ec2 volume snapshots in AWS. - This module was called C(ec2_snapshot_facts) before Ansible 2.9. The usage did not change. -requirements: [ boto3 ] author: - "Rob White (@wimnat)" - Aubin Bikouo (@abikouo) diff --git a/plugins/modules/ec2_tag.py b/plugins/modules/ec2_tag.py index f04d1102ff6..b74c21573e8 100644 --- a/plugins/modules/ec2_tag.py +++ b/plugins/modules/ec2_tag.py @@ -15,7 +15,6 @@ - Creates, modifies and removes tags for any EC2 resource. - Resources are referenced by their resource id (for example, an instance being i-XXXXXXX, a VPC being vpc-XXXXXXX). - This module is designed to be used with complex args (tags), see the examples. -requirements: [ "boto3", "botocore" ] options: resource: description: diff --git a/plugins/modules/ec2_tag_info.py b/plugins/modules/ec2_tag_info.py index cf326fd20a5..e29a2952125 100644 --- a/plugins/modules/ec2_tag_info.py +++ b/plugins/modules/ec2_tag_info.py @@ -15,7 +15,6 @@ - Lists tags for any EC2 resource. - Resources are referenced by their resource id (e.g. an instance being i-XXXXXXX, a vpc being vpc-XXXXXX). - Resource tags can be managed using the M(amazon.aws.ec2_tag) module. -requirements: [ "boto3", "botocore" ] options: resource: description: diff --git a/plugins/modules/ec2_vol_info.py b/plugins/modules/ec2_vol_info.py index fb6a658790b..ba20d45ee4f 100644 --- a/plugins/modules/ec2_vol_info.py +++ b/plugins/modules/ec2_vol_info.py @@ -14,7 +14,6 @@ description: - Gather information about ec2 volumes in AWS. - This module was called C(ec2_vol_facts) before Ansible 2.9. The usage did not change. -requirements: [ boto3 ] author: "Rob White (@wimnat)" options: filters: diff --git a/plugins/modules/ec2_vpc_dhcp_option.py b/plugins/modules/ec2_vpc_dhcp_option.py index d2c02efb284..ac3e4a16bf9 100644 --- a/plugins/modules/ec2_vpc_dhcp_option.py +++ b/plugins/modules/ec2_vpc_dhcp_option.py @@ -104,9 +104,6 @@ extends_documentation_fragment: - amazon.aws.aws - amazon.aws.ec2 - -requirements: - - boto ''' RETURN = """ diff --git a/plugins/modules/ec2_vpc_dhcp_option_info.py b/plugins/modules/ec2_vpc_dhcp_option_info.py index a33dff30bd8..d3be0bd53f2 100644 --- a/plugins/modules/ec2_vpc_dhcp_option_info.py +++ b/plugins/modules/ec2_vpc_dhcp_option_info.py @@ -14,7 +14,6 @@ description: - Gather information about dhcp options sets in AWS. - This module was called C(ec2_vpc_dhcp_option_facts) before Ansible 2.9. The usage did not change. -requirements: [ boto3 ] author: "Nick Aslanidis (@naslanidis)" options: filters: diff --git a/plugins/modules/ec2_vpc_net.py b/plugins/modules/ec2_vpc_net.py index 9e76e4ba60c..555f51389a9 100644 --- a/plugins/modules/ec2_vpc_net.py +++ b/plugins/modules/ec2_vpc_net.py @@ -78,9 +78,6 @@ duplicate VPCs created. type: bool default: false -requirements: - - boto3 - - botocore extends_documentation_fragment: - amazon.aws.aws - amazon.aws.ec2 diff --git a/plugins/modules/ec2_vpc_net_info.py b/plugins/modules/ec2_vpc_net_info.py index 62a9b1eecf3..a28df3e0e17 100644 --- a/plugins/modules/ec2_vpc_net_info.py +++ b/plugins/modules/ec2_vpc_net_info.py @@ -15,9 +15,6 @@ - Gather information about ec2 VPCs in AWS - This module was called C(ec2_vpc_net_facts) before Ansible 2.9. The usage did not change. author: "Rob White (@wimnat)" -requirements: - - boto3 - - botocore options: vpc_ids: description: diff --git a/plugins/modules/ec2_vpc_subnet.py b/plugins/modules/ec2_vpc_subnet.py index 2fe34f6ff35..5ac0c6eebfa 100644 --- a/plugins/modules/ec2_vpc_subnet.py +++ b/plugins/modules/ec2_vpc_subnet.py @@ -16,7 +16,6 @@ author: - Robert Estelle (@erydo) - Brad Davidson (@brandond) -requirements: [ boto3 ] options: az: description: diff --git a/plugins/modules/ec2_vpc_subnet_info.py b/plugins/modules/ec2_vpc_subnet_info.py index 316d532e8ff..e1a85fc858f 100644 --- a/plugins/modules/ec2_vpc_subnet_info.py +++ b/plugins/modules/ec2_vpc_subnet_info.py @@ -15,9 +15,6 @@ - Gather information about ec2 VPC subnets in AWS - This module was called C(ec2_vpc_subnet_facts) before Ansible 2.9. The usage did not change. author: "Rob White (@wimnat)" -requirements: - - boto3 - - botocore options: subnet_ids: description: diff --git a/plugins/modules/s3_bucket.py b/plugins/modules/s3_bucket.py index 35206c0b59c..82950fbe6ba 100644 --- a/plugins/modules/s3_bucket.py +++ b/plugins/modules/s3_bucket.py @@ -24,7 +24,6 @@ short_description: Manage S3 buckets in AWS, DigitalOcean, Ceph, Walrus, FakeS3 and StorageGRID description: - Manage S3 buckets in AWS, DigitalOcean, Ceph, Walrus, FakeS3 and StorageGRID. -requirements: [ boto3 ] author: - Rob White (@wimnat) - Aubin Bikouo (@abikouo) diff --git a/requirements.txt b/requirements.txt index 5c4c76b86f0..0d58b96112d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ boto>=2.49.0 -botocore>=1.12.249 -boto3>=1.9.249 +botocore>=1.16.0 +boto3>=1.13.0 diff --git a/tests/unit/requirements.txt b/tests/unit/requirements.txt index 917ee278d67..063eab0c1ed 100644 --- a/tests/unit/requirements.txt +++ b/tests/unit/requirements.txt @@ -1,2 +1,2 @@ -boto3 +boto3>=1.13.0 placebo From 465428356d5a6abe596ec6e865f0a151c7cb9690 Mon Sep 17 00:00:00 2001 From: Alina Buzachis Date: Thu, 20 May 2021 18:56:35 +0200 Subject: [PATCH 34/38] ec2: Update documentation (#364) ec2: Update documentation Reviewed-by: https://github.com/apps/ansible-zuul --- plugins/modules/ec2.py | 291 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 291 insertions(+) diff --git a/plugins/modules/ec2.py b/plugins/modules/ec2.py index bb76b3109b1..ab0b7ea0ab0 100644 --- a/plugins/modules/ec2.py +++ b/plugins/modules/ec2.py @@ -580,6 +580,297 @@ ''' +RETURN = r''' +changed: + description: If the EC2 instance has changed. + type: bool + returned: always + sample: true +instances: + description: The instances. + type: list + returned: always + contains: + ami_launch_index: + description: The AMI launch index, which can be used to find this instance in the launch group. + type: int + returned: always + sample: 0 + architecture: + description: The architecture of the image. + type: str + returned: always + sample: "x86_64" + block_device_mapping: + description: Any block device mapping entries for the instance. + type: dict + returned: always + sample: { + "/dev/xvda": { + "delete_on_termination": true, + "status": "attached", + "volume_id": "vol-06d364586f5550b62" + } + } + dns_name: + description: The public DNS name assigned to the instance. + type: str + returned: always + sample: "ec2-203-0-113-1.z-2.compute-1.amazonaws.com" + ebs_optimized: + description: Indicates whether the instance is optimized for Amazon EBS I/O. + type: bool + returned: always + sample: false + groups: + description: One or more security groups. + type: dict + returned: always + sample: { + "sg-0c6562ab3d435619f": "ansible-test--88312190_setup" + } + hypervisor: + description: The hypervisor type of the instance. + type: str + returned: always + sample: "xen" + image_id: + description: The ID of the AMI used to launch the instance. + type: str + returned: always + sample: "ami-0d5eff06f840b45e9" + instance_id: + description: The ID of the instance. + type: str + returned: always + sample: "i-0250719204c428be1" + instance_type: + description: The instance type. + type: str + returned: always + sample: "t2.micro" + kernel: + description: The kernel associated with this instance, if applicable. + type: str + returned: always + sample: "" + key_name: + description: The name of the key pair, if this instance was launched with an associated key pair. + type: str + returned: always + sample: "ansible-test-88312190_setup" + launch_time: + description: The time the instance was launched. + type: str + returned: always + sample: "2021-05-09T19:30:26.000Z" + placement: + description: The location where the instance launched, if applicable. + type: dict + returned: always + sample: { + "availability_zone": "us-east-1a", + "group_name": "", + "tenancy": "default" + } + private_dns_name: + description: The private DNS hostname name assigned to the instance. + type: str + returned: always + sample: "ip-10-176-1-249.ec2.internal" + private_ip: + description: The private IPv4 address assigned to the instance. + type: str + returned: always + sample: "10.176.1.249" + public_dns_name: + description: The public DNS name assigned to the instance. + type: str + returned: always + sample: "ec2-203-0-113-1.z-2.compute-1.amazonaws.com" + public_ip: + description: The public IPv4 address, or the Carrier IP address assigned to the instance, if applicable. + type: str + returned: always + sample: "203.0.113.1" + ramdisk: + description: The RAM disk associated with this instance, if applicable. + type: str + returned: always + sample: "" + root_device_name: + description: The device name of the root device volume. + type: str + returned: always + sample: "/dev/xvda" + root_device_type: + description: The root device type used by the AMI. + type: str + returned: always + sample: "ebs" + state: + description: The current state of the instance. + type: dict + returned: always + sample: { + "code": 80, + "name": "stopped" + } + tags: + description: Any tags assigned to the instance. + type: dict + returned: always + sample: { + "ResourcePrefix": "ansible-test-88312190-integration_tests" + } + tenancy: + description: The tenancy of the instance (if the instance is running in a VPC). + type: str + returned: always + sample: "default" + virtualization_type: + description: The virtualization type of the instance. + type: str + returned: always + sample: "hvm" + monitoring: + description: The monitoring for the instance. + type: dict + returned: always + sample: { + "state": "disabled" + } + capacity_reservation_specification: + description: Information about the Capacity Reservation targeting option. + type: dict + returned: always + sample: { + "capacity_reservation_preference": "open" + } + client_token: + description: The idempotency token you provided when you launched the instance, if applicable. + type: str + returned: always + sample: "" + cpu_options: + description: The CPU options for the instance. + type: dict + returned: always + sample: { + "core_count": 1, + "threads_per_core": 1 + } + ena_support: + description: Specifies whether enhanced networking with ENA is enabled. + type: bool + returned: always + sample: true + enclave_options: + description: Indicates whether the instance is enabled for AWS Nitro Enclaves. + type: dict + returned: always + sample: { + "enabled": false + } + hibernation_options: + description: Indicates whether the instance is enabled for hibernation. + type: dict + returned: always + sample: { + "configured": false + } + network_interfaces: + description: The network interfaces for the instance. + type: list + returned: always + sample: [ + { + "attachment": { + "attach_time": "2021-05-09T19:30:57+00:00", + "attachment_id": "eni-attach-07341f2560be6c8fc", + "delete_on_termination": true, + "device_index": 0, + "network_card_index": 0, + "status": "attached" + }, + "description": "", + "groups": [ + { + "group_id": "sg-0c6562ab3d435619f", + "group_name": "ansible-test-88312190_setup" + } + ], + "interface_type": "interface", + "ipv6_addresses": [], + "mac_address": "0e:0e:36:60:67:cf", + "network_interface_id": "eni-061dee20eba3b445a", + "owner_id": "721066863947", + "private_dns_name": "ip-10-176-1-178.ec2.internal", + "private_ip_address": "10.176.1.178", + "private_ip_addresses": [ + { + "primary": true, + "private_dns_name": "ip-10-176-1-178.ec2.internal", + "private_ip_address": "10.176.1.178" + } + ], + "source_dest_check": true, + "status": "in-use", + "subnet_id": "subnet-069d3e2eab081955d", + "vpc_id": "vpc-0b6879b6ca2e9be2b" + } + ] + vpc_id: + description: The ID of the VPC in which the instance is running. + type: str + returned: always + sample: "vpc-0b6879b6ca2e9be2b" + subnet_id: + description: The ID of the subnet in which the instance is running. + type: str + returned: always + sample: "subnet-069d3e2eab081955d" + state_transition_reason: + description: The reason for the most recent state transition. This might be an empty string. + type: str + returned: always + sample: "User initiated (2021-05-09 19:31:28 GMT)" + state_reason: + description: The reason for the most recent state transition. + type: dict + returned: always + sample: { + "code": "Client.UserInitiatedShutdown", + "message": "Client.UserInitiatedShutdown: User initiated shutdown" + } + security_groups: + description: The security groups for the instance. + type: list + returned: always + sample: [ + { + "group_id": "sg-0c6562ab3d435619f", + "group_name": "ansible-test-alinas-mbp-88312190_setup" + } + ] + source_dest_check: + description: Indicates whether source/destination checking is enabled. + type: bool + returned: always + sample: true + metadata: + description: The metadata options for the instance. + type: dict + returned: always + sample: { + "http_endpoint": "enabled", + "http_put_response_hop_limit": 1, + "http_tokens": "optional", + "state": "applied" + } +''' + + import time import datetime from ast import literal_eval From c3906a2e363657695ab7dee2b5c027ce85588edc Mon Sep 17 00:00:00 2001 From: abikouo <79859644+abikouo@users.noreply.github.com> Date: Thu, 20 May 2021 19:16:11 +0200 Subject: [PATCH 35/38] drop support of community.general for integration testing (#361) drop support of community.general for integration testing Reviewed-by: https://github.com/apps/ansible-zuul --- ....general-support-for-integration.tests.yml | 2 + .../targets/ec2_eni/tasks/test_deletion.yaml | 8 +- .../tasks/test_eni_basic_creation.yaml | 20 +-- .../ec2_eni/tasks/test_ipaddress_assign.yaml | 18 +- .../test_modifying_delete_on_termination.yaml | 4 +- .../targets/ec2_snapshot/tasks/main.yml | 4 +- .../ec2_vpc_dhcp_option/tasks/main.yml | 25 ++- .../targets/ec2_vpc_net/tasks/main.yml | 164 +++++++++--------- .../playbooks/populate_cache.yml | 2 - .../inventory_with_constructed.yml.j2 | 2 +- .../templates/inventory_with_constructed.j2 | 2 +- tests/requirements.yml | 1 - 12 files changed, 122 insertions(+), 130 deletions(-) create mode 100644 changelogs/fragments/361-drop-community.general-support-for-integration.tests.yml diff --git a/changelogs/fragments/361-drop-community.general-support-for-integration.tests.yml b/changelogs/fragments/361-drop-community.general-support-for-integration.tests.yml new file mode 100644 index 00000000000..be7557ed5cf --- /dev/null +++ b/changelogs/fragments/361-drop-community.general-support-for-integration.tests.yml @@ -0,0 +1,2 @@ +minor_changes: +- integration tests - remove dependency with collection ``community.general`` (https://github.com/ansible-collections/amazon.aws/pull/361). diff --git a/tests/integration/targets/ec2_eni/tasks/test_deletion.yaml b/tests/integration/targets/ec2_eni/tasks/test_deletion.yaml index aecb625eb32..b6dffc3481e 100644 --- a/tests/integration/targets/ec2_eni/tasks/test_deletion.yaml +++ b/tests/integration/targets/ec2_eni/tasks/test_deletion.yaml @@ -16,7 +16,7 @@ - result.changed - result.interface is undefined - '"network_interfaces" in eni_info' - - eni_id_1 not in ( eni_info | community.general.json_query("network_interfaces[].id") | list ) + - eni_id_1 not in ( eni_info.network_interfaces | selectattr('id') | map(attribute='id') | list ) - name: test removing the network interface by ID is idempotent ec2_eni: @@ -53,7 +53,7 @@ - result.changed - result.interface is undefined - '"network_interfaces" in eni_info' - - eni_id_2 not in ( eni_info | community.general.json_query("network_interfaces[].id") | list ) + - eni_id_2 not in ( eni_info.network_interfaces | selectattr('id') | map(attribute='id') | list ) - name: test removing the network interface by name is idempotent ec2_eni: @@ -88,5 +88,5 @@ assert: that: - '"network_interfaces" in eni_info' - - eni_id_1 not in ( eni_info | community.general.json_query("network_interfaces[].id") | list ) - - eni_id_2 not in ( eni_info | community.general.json_query("network_interfaces[].id") | list ) + - eni_id_1 not in ( eni_info.network_interfaces | selectattr('id') | map(attribute='id') | list ) + - eni_id_2 not in ( eni_info.network_interfaces | selectattr('id') | map(attribute='id') | list ) diff --git a/tests/integration/targets/ec2_eni/tasks/test_eni_basic_creation.yaml b/tests/integration/targets/ec2_eni/tasks/test_eni_basic_creation.yaml index b18af2dc9b3..b6f61c9365e 100644 --- a/tests/integration/targets/ec2_eni/tasks/test_eni_basic_creation.yaml +++ b/tests/integration/targets/ec2_eni/tasks/test_eni_basic_creation.yaml @@ -61,7 +61,7 @@ - _interface_0.private_ip_address == ip_1 - '"private_ip_addresses" in _interface_0' - _interface_0.private_ip_addresses | length == 1 - - ip_1 in ( eni_info | community.general.json_query("network_interfaces[].private_ip_addresses[].private_ip_address") | list ) + - ip_1 in ( eni_info.network_interfaces | map(attribute='private_ip_addresses') | flatten | map(attribute='private_ip_address') | list ) - '"requester_id" in _interface_0' - _interface_0.requester_id is string - '"requester_managed" in _interface_0' @@ -156,7 +156,7 @@ - _interface_0.private_ip_address == ip_5 - '"private_ip_addresses" in _interface_0' - _interface_0.private_ip_addresses | length == 1 - - ip_5 in ( eni_info | community.general.json_query("network_interfaces[].private_ip_addresses[].private_ip_address") | list ) + - ip_5 in ( eni_info.network_interfaces | map(attribute='private_ip_addresses') | flatten | map(attribute='private_ip_address') | list ) - '"requester_id" in _interface_0' - _interface_0.requester_id is string - '"requester_managed" in _interface_0' @@ -181,8 +181,8 @@ that: - '"network_interfaces" in eni_info' - eni_info.network_interfaces | length >= 2 - - eni_id_1 in ( eni_info | community.general.json_query("network_interfaces[].id") | list ) - - eni_id_2 in ( eni_info | community.general.json_query("network_interfaces[].id") | list ) + - eni_id_1 in ( eni_info.network_interfaces | selectattr('id') | map(attribute='id') | list ) + - eni_id_2 in ( eni_info.network_interfaces | selectattr('id') | map(attribute='id') | list ) # ============================================================ # Run some VPC filter based tests of ec2_eni_info @@ -199,8 +199,8 @@ that: - '"network_interfaces" in eni_info' - eni_info.network_interfaces | length == 2 - - eni_id_1 in ( eni_info | community.general.json_query("network_interfaces[].id") | list ) - - eni_id_2 in ( eni_info | community.general.json_query("network_interfaces[].id") | list ) + - eni_id_1 in ( eni_info.network_interfaces | selectattr('id') | map(attribute='id') | list ) + - eni_id_2 in ( eni_info.network_interfaces | selectattr('id') | map(attribute='id') | list ) - name: Fetch ENI info with VPC filters - VPC ec2_eni_info: @@ -213,7 +213,7 @@ that: - '"network_interfaces" in eni_info' - eni_info.network_interfaces | length == 4 - - eni_id_1 in ( eni_info | community.general.json_query("network_interfaces[].id") | list ) - - eni_id_2 in ( eni_info | community.general.json_query("network_interfaces[].id") | list ) - - ec2_ips[0] in ( eni_info | community.general.json_query("network_interfaces[].private_ip_addresses[].private_ip_address") | list ) - - ec2_ips[1] in ( eni_info | community.general.json_query("network_interfaces[].private_ip_addresses[].private_ip_address") | list ) + - eni_id_1 in ( eni_info.network_interfaces | selectattr('id') | map(attribute='id') | list ) + - eni_id_2 in ( eni_info.network_interfaces | selectattr('id') | map(attribute='id') | list ) + - ec2_ips[0] in ( eni_info.network_interfaces | map(attribute='private_ip_addresses') | flatten | map(attribute='private_ip_address') | list ) + - ec2_ips[1] in ( eni_info.network_interfaces | map(attribute='private_ip_addresses') | flatten | map(attribute='private_ip_address') | list ) diff --git a/tests/integration/targets/ec2_eni/tasks/test_ipaddress_assign.yaml b/tests/integration/targets/ec2_eni/tasks/test_ipaddress_assign.yaml index a0a3696e9b5..1e67227cb4f 100644 --- a/tests/integration/targets/ec2_eni/tasks/test_ipaddress_assign.yaml +++ b/tests/integration/targets/ec2_eni/tasks/test_ipaddress_assign.yaml @@ -18,7 +18,7 @@ - result.interface.id == eni_id_1 - result.interface.private_ip_addresses | length == 3 - _interface_0.private_ip_addresses | length == 3 - - ip_1 in ( eni_info | community.general.json_query("network_interfaces[].private_ip_addresses[].private_ip_address") | list ) + - ip_1 in ( eni_info.network_interfaces | map(attribute='private_ip_addresses') | flatten | map(attribute='private_ip_address') | list ) vars: _interface_0: '{{ eni_info.network_interfaces[0] }}' @@ -40,7 +40,7 @@ - result.interface.id == eni_id_1 - result.interface.private_ip_addresses | length == 3 - _interface_0.private_ip_addresses | length == 3 - - ip_1 in ( eni_info | community.general.json_query("network_interfaces[].private_ip_addresses[].private_ip_address") | list ) + - ip_1 in ( eni_info.network_interfaces | map(attribute='private_ip_addresses') | flatten | map(attribute='private_ip_address') | list ) vars: _interface_0: '{{ eni_info.network_interfaces[0] }}' @@ -64,7 +64,7 @@ - result.interface.id == eni_id_1 - result.interface.private_ip_addresses | length == 3 - _interface_0.private_ip_addresses | length == 3 - - ip_1 in ( eni_info | community.general.json_query("network_interfaces[].private_ip_addresses[].private_ip_address") | list ) + - ip_1 in ( eni_info.network_interfaces | map(attribute='private_ip_addresses') | flatten | map(attribute='private_ip_address') | list ) vars: _interface_0: '{{ eni_info.network_interfaces[0] }}' @@ -106,7 +106,7 @@ - new_secondary_ip in _private_ips vars: _interface_0: '{{ eni_info.network_interfaces[0] }}' - _private_ips: '{{ eni_info | community.general.json_query("network_interfaces[].private_ip_addresses[].private_ip_address") | list }}' + _private_ips: "{{ eni_info.network_interfaces | map(attribute='private_ip_addresses') | flatten | map(attribute='private_ip_address') | list }}" # ============================================================ - name: remove secondary address @@ -128,7 +128,7 @@ - result.interface.id == eni_id_1 - result.interface.private_ip_addresses | length == 1 - _interface_0.private_ip_addresses | length == 1 - - ip_1 in ( eni_info | community.general.json_query("network_interfaces[].private_ip_addresses[].private_ip_address") | list ) + - ip_1 in ( eni_info.network_interfaces | map(attribute='private_ip_addresses') | flatten | map(attribute='private_ip_address') | list ) vars: _interface_0: '{{ eni_info.network_interfaces[0] }}' @@ -152,7 +152,7 @@ - result.interface.private_ip_addresses | length == 1 - result.interface.private_ip_addresses | length == 1 - _interface_0.private_ip_addresses | length == 1 - - ip_1 in ( eni_info | community.general.json_query("network_interfaces[].private_ip_addresses[].private_ip_address") | list ) + - ip_1 in ( eni_info.network_interfaces | map(attribute='private_ip_addresses') | flatten | map(attribute='private_ip_address') | list ) vars: _interface_0: '{{ eni_info.network_interfaces[0] }}' @@ -177,8 +177,8 @@ - result.interface.id == eni_id_2 - result.interface.private_ip_addresses | length == 2 - _interface_0.private_ip_addresses | length == 2 - - ip_5 in ( eni_info | community.general.json_query("network_interfaces[].private_ip_addresses[].private_ip_address") | list ) - - ip_4 in ( eni_info | community.general.json_query("network_interfaces[].private_ip_addresses[].private_ip_address") | list ) + - ip_5 in ( eni_info.network_interfaces | map(attribute='private_ip_addresses') | flatten | map(attribute='private_ip_address') | list ) + - ip_4 in ( eni_info.network_interfaces | map(attribute='private_ip_addresses') | flatten | map(attribute='private_ip_address') | list ) vars: _interface_0: '{{ eni_info.network_interfaces[0] }}' @@ -262,6 +262,6 @@ that: - result.changed - _interface_0.private_ip_addresses | length == 1 - - ip_1 in ( eni_info | community.general.json_query("network_interfaces[].private_ip_addresses[].private_ip_address") | list ) + - ip_1 in ( eni_info.network_interfaces | map(attribute='private_ip_addresses') | flatten | map(attribute='private_ip_address') | list ) vars: _interface_0: '{{ eni_info.network_interfaces[0] }}' diff --git a/tests/integration/targets/ec2_eni/tasks/test_modifying_delete_on_termination.yaml b/tests/integration/targets/ec2_eni/tasks/test_modifying_delete_on_termination.yaml index 8e8bd0596d4..54240b4d2a2 100644 --- a/tests/integration/targets/ec2_eni/tasks/test_modifying_delete_on_termination.yaml +++ b/tests/integration/targets/ec2_eni/tasks/test_modifying_delete_on_termination.yaml @@ -149,8 +149,8 @@ - not result.changed - '"network_interfaces" in eni_info' - eni_info.network_interfaces | length >= 1 - - eni_id_1 not in ( eni_info | community.general.json_query("network_interfaces[].id") | list ) - - eni_id_2 in ( eni_info | community.general.json_query("network_interfaces[].id") | list ) + - eni_id_1 not in ( eni_info.network_interfaces | selectattr('id') | map(attribute='id') | list ) + - eni_id_2 in ( eni_info.network_interfaces | selectattr('id') | map(attribute='id') | list ) # ============================================================ diff --git a/tests/integration/targets/ec2_snapshot/tasks/main.yml b/tests/integration/targets/ec2_snapshot/tasks/main.yml index fd00216bc92..94a4c9ab20d 100644 --- a/tests/integration/targets/ec2_snapshot/tasks/main.yml +++ b/tests/integration/targets/ec2_snapshot/tasks/main.yml @@ -165,7 +165,7 @@ that: - result is changed - info_result.snapshots| length == 2 - - '"{{ result.snapshot_id }}" in "{{ info_result| community.general.json_query("snapshots[].snapshot_id") }}"' + - result.snapshot_id in ( info_result.snapshots | map(attribute='snapshot_id') | list ) # JR: Check mode not supported # - name: Take snapshot with a tag (check mode) @@ -296,7 +296,7 @@ - assert: that: - info_result.snapshots| length == 7 - - '"{{ tagged_snapshot_id }}" not in "{{ info_result| community.general.json_query("snapshots[].snapshot_id") }}"' + - tagged_snapshot_id not in ( info_result.snapshots | map(attribute='snapshot_id') | list ) - name: Delete snapshots ec2_snapshot: diff --git a/tests/integration/targets/ec2_vpc_dhcp_option/tasks/main.yml b/tests/integration/targets/ec2_vpc_dhcp_option/tasks/main.yml index ee780170387..5441e4f7f9b 100644 --- a/tests/integration/targets/ec2_vpc_dhcp_option/tasks/main.yml +++ b/tests/integration/targets/ec2_vpc_dhcp_option/tasks/main.yml @@ -14,9 +14,6 @@ security_token: "{{ security_token | default('') }}" region: "{{ aws_region }}" - collections: - - community.general - block: # DHCP option set can be attached to multiple VPCs, we don't want to use any that @@ -26,7 +23,7 @@ register: result - set_fact: - preexisting_option_sets: "{{ result | community.general.json_query('dhcp_options[*].dhcp_options_id') | list }}" + preexisting_option_sets: "{{ result.dhcp_options | map(attribute='dhcp_options_id') | list }}" - name: create a VPC with a default DHCP option set to test inheritance and delete_old ec2_vpc_net: @@ -183,8 +180,8 @@ - dhcp_options.new_options['domain-name'] == ['{{ aws_domain_name }}'] - dhcp_options.new_options['domain-name-servers'] == ['AmazonProvidedDNS'] # We return the list of dicts that boto gives us, in addition to the user-friendly config dict - - dhcp_options_config['ntp-servers'] | community.general.json_query('[*].value') | list | sort == ['10.0.0.2', '10.0.1.2'] - - dhcp_options_config['netbios-name-servers'] | community.general.json_query('[*].value') | list | sort == ['10.0.0.1', '10.0.1.1'] + - dhcp_options_config['ntp-servers'] | map(attribute='value') | list | sort == ['10.0.0.2', '10.0.1.2'] + - dhcp_options_config['netbios-name-servers'] | map(attribute='value') | list | sort == ['10.0.0.1', '10.0.1.1'] - dhcp_options_config['netbios-node-type'][0]['value'] == '2' - dhcp_options_config['domain-name'][0]['value'] == '{{ aws_domain_name }}' - dhcp_options_config['domain-name-servers'][0]['value'] == 'AmazonProvidedDNS' @@ -206,8 +203,8 @@ - new_config.keys() | list | sort == ['domain-name', 'domain-name-servers', 'netbios-name-servers', 'netbios-node-type', 'ntp-servers'] - new_config['domain-name'][0]['value'] == '{{ aws_domain_name }}' - new_config['domain-name-servers'][0]['value'] == 'AmazonProvidedDNS' - - new_config['ntp-servers'] | community.general.json_query('[*].value') | list | sort == ['10.0.0.2', '10.0.1.2'] - - new_config['netbios-name-servers'] | community.general.json_query('[*].value') | list | sort == ['10.0.0.1', '10.0.1.1'] + - new_config['ntp-servers'] | map(attribute='value') | list | sort == ['10.0.0.2', '10.0.1.2'] + - new_config['netbios-name-servers'] | map(attribute='value') | list | sort == ['10.0.0.1', '10.0.1.1'] - new_config['netbios-node-type'][0]['value'] == '2' # We return the list of dicts that boto gives us, in addition to the user-friendly config dict - new_dhcp_options.dhcp_config[0]['ntp-servers'] | sort == ['10.0.0.2', '10.0.1.2'] @@ -305,8 +302,8 @@ - assert: that: - new_config.keys() | list | sort == ['netbios-name-servers', 'netbios-node-type', 'ntp-servers'] - - new_config['ntp-servers'] | community.general.json_query('[*].value') | list | sort == ['10.0.0.2', '10.0.1.2'] - - new_config['netbios-name-servers'] | community.general.json_query('[*].value') | list | sort == ['10.0.0.1', '10.0.1.1'] + - new_config['ntp-servers'] | map(attribute='value') | list | sort == ['10.0.0.2', '10.0.1.2'] + - new_config['netbios-name-servers'] | map(attribute='value') | list | sort == ['10.0.0.1', '10.0.1.1'] - new_config['netbios-node-type'][0]['value'] == '2' - name: disassociate the new DHCP option set so it can be deleted @@ -384,8 +381,8 @@ that: - new_config.keys() | list | sort == ['domain-name', 'domain-name-servers', 'netbios-name-servers', 'netbios-node-type', 'ntp-servers'] - new_config['domain-name'][0]['value'] == '{{ aws_domain_name }}' - - new_config['ntp-servers'] | community.general.json_query('[*].value') | list | sort == ['10.0.0.2', '10.0.1.2'] - - new_config['netbios-name-servers'] | community.general.json_query('[*].value') | list | sort == ['10.0.0.1', '10.0.1.1'] + - new_config['ntp-servers'] | map(attribute='value') | list | sort == ['10.0.0.2', '10.0.1.2'] + - new_config['netbios-name-servers'] | map(attribute='value') | list | sort == ['10.0.0.1', '10.0.1.1'] - new_config['netbios-node-type'][0]['value'] == '1' - name: verify the original set was deleted @@ -533,8 +530,8 @@ - dhcp_options.new_options['netbios-name-servers'] | sort == ['10.0.0.1', '10.0.1.1'] - original_dhcp_options_id != dhcp_options.dhcp_options_id # We return the list of dicts that boto gives us, in addition to the user-friendly config dict - - dhcp_options_config['ntp-servers'] | community.general.json_query('[*].value') | list | sort == ['10.0.0.2', '10.0.1.2'] - - dhcp_options_config['netbios-name-servers'] | community.general.json_query('[*].value') | list | sort == ['10.0.0.1', '10.0.1.1'] + - dhcp_options_config['ntp-servers'] | map(attribute='value') | list | sort == ['10.0.0.2', '10.0.1.2'] + - dhcp_options_config['netbios-name-servers'] | map(attribute='value') | list | sort == ['10.0.0.1', '10.0.1.1'] - dhcp_options.dhcp_options.tags.keys() | length == 2 - dhcp_options.dhcp_options.tags['CreatedBy'] == 'ansible-test' - dhcp_options.dhcp_options.tags['Collection'] == 'amazon.aws' diff --git a/tests/integration/targets/ec2_vpc_net/tasks/main.yml b/tests/integration/targets/ec2_vpc_net/tasks/main.yml index 728667ac32e..19fcd65ae65 100644 --- a/tests/integration/targets/ec2_vpc_net/tasks/main.yml +++ b/tests/integration/targets/ec2_vpc_net/tasks/main.yml @@ -174,7 +174,7 @@ - name: Test that our new VPC shows up in the results assert: that: - - vpc_1 in ( vpc_info | community.general.json_query("vpcs[].vpc_id") | list ) + - vpc_1 in ( vpc_info.vpcs | map(attribute="vpc_id") | list ) - name: VPC info (Simple tag filter) ec2_vpc_net_info: @@ -789,17 +789,17 @@ # - result.vpc.id == vpc_1 # - vpc_info.vpcs | length == 1 # - vpc_info.vpcs[0].cidr_block == vpc_cidr - # - vpc_cidr in (result.vpc | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) - # - vpc_cidr_a not in (result.vpc | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) - # - vpc_cidr_b not in (result.vpc | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) + # - vpc_cidr in (result.vpc.cidr_block_association_set | map(attribute="cidr_block") | list) + # - vpc_cidr_a not in (result.vpc.cidr_block_association_set | map(attribute="cidr_block") | list) + # - vpc_cidr_b not in (result.vpc.cidr_block_association_set | map(attribute="cidr_block") | list) # - vpc_info.vpcs[0].cidr_block_association_set | length == 1 # - vpc_info.vpcs[0].cidr_block_association_set[0].association_id.startswith("vpc-cidr-assoc-") # - vpc_info.vpcs[0].cidr_block_association_set[1].association_id.startswith("vpc-cidr-assoc-") # - vpc_info.vpcs[0].cidr_block_association_set[0].cidr_block_state.state in ["associated", "associating"] # - vpc_info.vpcs[0].cidr_block_association_set[1].cidr_block_state.state in ["associated", "associating"] - # - vpc_cidr in (vpc_info.vpcs[0] | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) - # - vpc_cidr_a not in (vpc_info.vpcs[0] | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) - # - vpc_cidr_b not in (result.vpc | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) + # - vpc_cidr in (vpc_info.vpcs[0].cidr_block_association_set | map(attribute="cidr_block") | list) + # - vpc_cidr_a not in (vpc_info.vpcs[0].cidr_block_association_set | map(attribute="cidr_block") | list) + # - vpc_cidr_b not in (result.vpc.cidr_block_association_set | map(attribute="cidr_block") | list) - name: modify CIDR ec2_vpc_net: @@ -828,17 +828,17 @@ - result.vpc.cidr_block_association_set[1].association_id.startswith("vpc-cidr-assoc-") - result.vpc.cidr_block_association_set[0].cidr_block_state.state in ["associated", "associating"] - result.vpc.cidr_block_association_set[1].cidr_block_state.state in ["associated", "associating"] - - vpc_cidr in (result.vpc | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) - - vpc_cidr_a in (result.vpc | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) - - vpc_cidr_b not in (result.vpc | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) + - vpc_cidr in (result.vpc.cidr_block_association_set | map(attribute="cidr_block") | list) + - vpc_cidr_a in (result.vpc.cidr_block_association_set | map(attribute="cidr_block") | list) + - vpc_cidr_b not in (result.vpc.cidr_block_association_set | map(attribute="cidr_block") | list) - vpc_info.vpcs[0].cidr_block_association_set | length == 2 - vpc_info.vpcs[0].cidr_block_association_set[0].association_id.startswith("vpc-cidr-assoc-") - vpc_info.vpcs[0].cidr_block_association_set[1].association_id.startswith("vpc-cidr-assoc-") - vpc_info.vpcs[0].cidr_block_association_set[0].cidr_block_state.state in ["associated", "associating"] - vpc_info.vpcs[0].cidr_block_association_set[1].cidr_block_state.state in ["associated", "associating"] - - vpc_cidr in (vpc_info.vpcs[0] | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) - - vpc_cidr_a in (vpc_info.vpcs[0] | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) - - vpc_cidr_b not in (vpc_info.vpcs[0] | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) + - vpc_cidr in (vpc_info.vpcs[0].cidr_block_association_set | map(attribute="cidr_block") | list) + - vpc_cidr_a in (vpc_info.vpcs[0].cidr_block_association_set | map(attribute="cidr_block") | list) + - vpc_cidr_b not in (vpc_info.vpcs[0].cidr_block_association_set | map(attribute="cidr_block") | list) - name: modify CIDR (no change) ec2_vpc_net: @@ -867,17 +867,17 @@ - result.vpc.cidr_block_association_set[1].association_id.startswith("vpc-cidr-assoc-") - result.vpc.cidr_block_association_set[0].cidr_block_state.state in ["associated", "associating"] - result.vpc.cidr_block_association_set[1].cidr_block_state.state in ["associated", "associating"] - - vpc_cidr in (result.vpc | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) - - vpc_cidr_a in (result.vpc | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) - - vpc_cidr_b not in (result.vpc | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) + - vpc_cidr in (result.vpc.cidr_block_association_set | map(attribute="cidr_block") | list) + - vpc_cidr_a in (result.vpc.cidr_block_association_set | map(attribute="cidr_block") | list) + - vpc_cidr_b not in (result.vpc.cidr_block_association_set | map(attribute="cidr_block") | list) - vpc_info.vpcs[0].cidr_block_association_set | length == 2 - vpc_info.vpcs[0].cidr_block_association_set[0].association_id.startswith("vpc-cidr-assoc-") - vpc_info.vpcs[0].cidr_block_association_set[1].association_id.startswith("vpc-cidr-assoc-") - vpc_info.vpcs[0].cidr_block_association_set[0].cidr_block_state.state in ["associated", "associating"] - vpc_info.vpcs[0].cidr_block_association_set[1].cidr_block_state.state in ["associated", "associating"] - - vpc_cidr in (vpc_info.vpcs[0] | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) - - vpc_cidr_a in (vpc_info.vpcs[0] | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) - - vpc_cidr_b not in (vpc_info.vpcs[0] | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) + - vpc_cidr in (vpc_info.vpcs[0].cidr_block_association_set | map(attribute="cidr_block") | list) + - vpc_cidr_a in (vpc_info.vpcs[0].cidr_block_association_set | map(attribute="cidr_block") | list) + - vpc_cidr_b not in (vpc_info.vpcs[0].cidr_block_association_set | map(attribute="cidr_block") | list) # #62678 #- name: modify CIDR - no purge (check mode) @@ -901,17 +901,17 @@ # - result is changed # - vpc_info.vpcs | length == 1 # - vpc_info.vpcs[0].cidr_block == vpc_cidr - # - vpc_cidr in (result.vpc | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) - # - vpc_cidr_a in (result.vpc | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) - # - vpc_cidr_b not in (result.vpc | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) + # - vpc_cidr in (result.vpc.cidr_block_association_set | map(attribute="cidr_block") | list) + # - vpc_cidr_a in (result.vpc.cidr_block_association_set | map(attribute="cidr_block") | list) + # - vpc_cidr_b not in (result.vpc.cidr_block_association_set | map(attribute="cidr_block") | list) # - vpc_info.vpcs[0].cidr_block_association_set | length == 2 # - vpc_info.vpcs[0].cidr_block_association_set[0].association_id.startswith("vpc-cidr-assoc-") # - vpc_info.vpcs[0].cidr_block_association_set[1].association_id.startswith("vpc-cidr-assoc-") # - vpc_info.vpcs[0].cidr_block_association_set[0].cidr_block_state.state in ["associated", "associating"] # - vpc_info.vpcs[0].cidr_block_association_set[1].cidr_block_state.state in ["associated", "associating"] - # - vpc_cidr in (vpc_info.vpcs[0] | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) - # - vpc_cidr_a in (vpc_info.vpcs[0] | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) - # - vpc_cidr_b not in (vpc_info.vpcs[0] | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) + # - vpc_cidr in (vpc_info.vpcs[0].cidr_block_association_set | map(attribute="cidr_block") | list) + # - vpc_cidr_a in (vpc_info.vpcs[0].cidr_block_association_set | map(attribute="cidr_block") | list) + # - vpc_cidr_b not in (vpc_info.vpcs[0].cidr_block_association_set | map(attribute="cidr_block") | list) - name: modify CIDR - no purge ec2_vpc_net: @@ -942,9 +942,9 @@ - result.vpc.cidr_block_association_set[0].cidr_block_state.state in ["associated", "associating"] - result.vpc.cidr_block_association_set[1].cidr_block_state.state in ["associated", "associating"] - result.vpc.cidr_block_association_set[2].cidr_block_state.state in ["associated", "associating"] - - vpc_cidr in (result.vpc | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) - - vpc_cidr_a in (result.vpc | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) - - vpc_cidr_b in (result.vpc | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) + - vpc_cidr in (result.vpc.cidr_block_association_set | map(attribute="cidr_block") | list) + - vpc_cidr_a in (result.vpc.cidr_block_association_set | map(attribute="cidr_block") | list) + - vpc_cidr_b in (result.vpc.cidr_block_association_set | map(attribute="cidr_block") | list) - vpc_info.vpcs[0].cidr_block_association_set | length == 3 - vpc_info.vpcs[0].cidr_block_association_set[0].association_id.startswith("vpc-cidr-assoc-") - vpc_info.vpcs[0].cidr_block_association_set[1].association_id.startswith("vpc-cidr-assoc-") @@ -952,9 +952,9 @@ - vpc_info.vpcs[0].cidr_block_association_set[0].cidr_block_state.state in ["associated", "associating"] - vpc_info.vpcs[0].cidr_block_association_set[1].cidr_block_state.state in ["associated", "associating"] - vpc_info.vpcs[0].cidr_block_association_set[2].cidr_block_state.state in ["associated", "associating"] - - vpc_cidr in (vpc_info.vpcs[0] | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) - - vpc_cidr_a in (vpc_info.vpcs[0] | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) - - vpc_cidr_b in (vpc_info.vpcs[0] | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) + - vpc_cidr in (vpc_info.vpcs[0].cidr_block_association_set | map(attribute="cidr_block") | list) + - vpc_cidr_a in (vpc_info.vpcs[0].cidr_block_association_set | map(attribute="cidr_block") | list) + - vpc_cidr_b in (vpc_info.vpcs[0].cidr_block_association_set | map(attribute="cidr_block") | list) - name: modify CIDR - no purge (no change) ec2_vpc_net: @@ -984,9 +984,9 @@ - result.vpc.cidr_block_association_set[0].cidr_block_state.state in ["associated", "associating"] - result.vpc.cidr_block_association_set[1].cidr_block_state.state in ["associated", "associating"] - result.vpc.cidr_block_association_set[2].cidr_block_state.state in ["associated", "associating"] - - vpc_cidr in (result.vpc | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) - - vpc_cidr_a in (result.vpc | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) - - vpc_cidr_b in (result.vpc | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) + - vpc_cidr in (result.vpc.cidr_block_association_set | map(attribute="cidr_block") | list) + - vpc_cidr_a in (result.vpc.cidr_block_association_set | map(attribute="cidr_block") | list) + - vpc_cidr_b in (result.vpc.cidr_block_association_set | map(attribute="cidr_block") | list) - vpc_info.vpcs[0].cidr_block_association_set | length == 3 - vpc_info.vpcs[0].cidr_block_association_set[0].association_id.startswith("vpc-cidr-assoc-") - vpc_info.vpcs[0].cidr_block_association_set[1].association_id.startswith("vpc-cidr-assoc-") @@ -994,9 +994,9 @@ - vpc_info.vpcs[0].cidr_block_association_set[0].cidr_block_state.state in ["associated", "associating"] - vpc_info.vpcs[0].cidr_block_association_set[1].cidr_block_state.state in ["associated", "associating"] - vpc_info.vpcs[0].cidr_block_association_set[2].cidr_block_state.state in ["associated", "associating"] - - vpc_cidr in (vpc_info.vpcs[0] | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) - - vpc_cidr_a in (vpc_info.vpcs[0] | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) - - vpc_cidr_b in (vpc_info.vpcs[0] | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) + - vpc_cidr in (vpc_info.vpcs[0].cidr_block_association_set | map(attribute="cidr_block") | list) + - vpc_cidr_a in (vpc_info.vpcs[0].cidr_block_association_set | map(attribute="cidr_block") | list) + - vpc_cidr_b in (vpc_info.vpcs[0].cidr_block_association_set | map(attribute="cidr_block") | list) - name: modify CIDR - no purge (no change - list all - check mode) ec2_vpc_net: @@ -1027,9 +1027,9 @@ - result.vpc.cidr_block_association_set[0].cidr_block_state.state in ["associated", "associating"] - result.vpc.cidr_block_association_set[1].cidr_block_state.state in ["associated", "associating"] - result.vpc.cidr_block_association_set[2].cidr_block_state.state in ["associated", "associating"] - - vpc_cidr in (result.vpc | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) - - vpc_cidr_a in (result.vpc | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) - - vpc_cidr_b in (result.vpc | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) + - vpc_cidr in (result.vpc.cidr_block_association_set | map(attribute="cidr_block") | list) + - vpc_cidr_a in (result.vpc.cidr_block_association_set | map(attribute="cidr_block") | list) + - vpc_cidr_b in (result.vpc.cidr_block_association_set | map(attribute="cidr_block") | list) - vpc_info.vpcs[0].cidr_block_association_set | length == 3 - vpc_info.vpcs[0].cidr_block_association_set[0].association_id.startswith("vpc-cidr-assoc-") - vpc_info.vpcs[0].cidr_block_association_set[1].association_id.startswith("vpc-cidr-assoc-") @@ -1037,9 +1037,9 @@ - vpc_info.vpcs[0].cidr_block_association_set[0].cidr_block_state.state in ["associated", "associating"] - vpc_info.vpcs[0].cidr_block_association_set[1].cidr_block_state.state in ["associated", "associating"] - vpc_info.vpcs[0].cidr_block_association_set[2].cidr_block_state.state in ["associated", "associating"] - - vpc_cidr in (vpc_info.vpcs[0] | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) - - vpc_cidr_a in (vpc_info.vpcs[0] | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) - - vpc_cidr_b in (vpc_info.vpcs[0] | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) + - vpc_cidr in (vpc_info.vpcs[0].cidr_block_association_set | map(attribute="cidr_block") | list) + - vpc_cidr_a in (vpc_info.vpcs[0].cidr_block_association_set | map(attribute="cidr_block") | list) + - vpc_cidr_b in (vpc_info.vpcs[0].cidr_block_association_set | map(attribute="cidr_block") | list) - name: modify CIDR - no purge (no change - list all) ec2_vpc_net: @@ -1070,9 +1070,9 @@ - result.vpc.cidr_block_association_set[0].cidr_block_state.state in ["associated", "associating"] - result.vpc.cidr_block_association_set[1].cidr_block_state.state in ["associated", "associating"] - result.vpc.cidr_block_association_set[2].cidr_block_state.state in ["associated", "associating"] - - vpc_cidr in (result.vpc | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) - - vpc_cidr_a in (result.vpc | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) - - vpc_cidr_b in (result.vpc | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) + - vpc_cidr in (result.vpc.cidr_block_association_set | map(attribute="cidr_block") | list) + - vpc_cidr_a in (result.vpc.cidr_block_association_set | map(attribute="cidr_block") | list) + - vpc_cidr_b in (result.vpc.cidr_block_association_set | map(attribute="cidr_block") | list) - vpc_info.vpcs[0].cidr_block_association_set | length == 3 - vpc_info.vpcs[0].cidr_block_association_set[0].association_id.startswith("vpc-cidr-assoc-") - vpc_info.vpcs[0].cidr_block_association_set[1].association_id.startswith("vpc-cidr-assoc-") @@ -1080,9 +1080,9 @@ - vpc_info.vpcs[0].cidr_block_association_set[0].cidr_block_state.state in ["associated", "associating"] - vpc_info.vpcs[0].cidr_block_association_set[1].cidr_block_state.state in ["associated", "associating"] - vpc_info.vpcs[0].cidr_block_association_set[2].cidr_block_state.state in ["associated", "associating"] - - vpc_cidr in (vpc_info.vpcs[0] | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) - - vpc_cidr_a in (vpc_info.vpcs[0] | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) - - vpc_cidr_b in (vpc_info.vpcs[0] | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) + - vpc_cidr in (vpc_info.vpcs[0].cidr_block_association_set | map(attribute="cidr_block") | list) + - vpc_cidr_a in (vpc_info.vpcs[0].cidr_block_association_set | map(attribute="cidr_block") | list) + - vpc_cidr_b in (vpc_info.vpcs[0].cidr_block_association_set | map(attribute="cidr_block") | list) - name: modify CIDR - no purge (no change - different order - check mode) ec2_vpc_net: @@ -1113,9 +1113,9 @@ - result.vpc.cidr_block_association_set[0].cidr_block_state.state in ["associated", "associating"] - result.vpc.cidr_block_association_set[1].cidr_block_state.state in ["associated", "associating"] - result.vpc.cidr_block_association_set[2].cidr_block_state.state in ["associated", "associating"] - - vpc_cidr in (result.vpc | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) - - vpc_cidr_a in (result.vpc | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) - - vpc_cidr_b in (result.vpc | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) + - vpc_cidr in (result.vpc.cidr_block_association_set | map(attribute="cidr_block") | list) + - vpc_cidr_a in (result.vpc.cidr_block_association_set | map(attribute="cidr_block") | list) + - vpc_cidr_b in (result.vpc.cidr_block_association_set | map(attribute="cidr_block") | list) - vpc_info.vpcs[0].cidr_block_association_set | length == 3 - vpc_info.vpcs[0].cidr_block_association_set[0].association_id.startswith("vpc-cidr-assoc-") - vpc_info.vpcs[0].cidr_block_association_set[1].association_id.startswith("vpc-cidr-assoc-") @@ -1123,9 +1123,9 @@ - vpc_info.vpcs[0].cidr_block_association_set[0].cidr_block_state.state in ["associated", "associating"] - vpc_info.vpcs[0].cidr_block_association_set[1].cidr_block_state.state in ["associated", "associating"] - vpc_info.vpcs[0].cidr_block_association_set[2].cidr_block_state.state in ["associated", "associating"] - - vpc_cidr in (vpc_info.vpcs[0] | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) - - vpc_cidr_a in (vpc_info.vpcs[0] | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) - - vpc_cidr_b in (vpc_info.vpcs[0] | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) + - vpc_cidr in (vpc_info.vpcs[0].cidr_block_association_set | map(attribute="cidr_block") | list) + - vpc_cidr_a in (vpc_info.vpcs[0].cidr_block_association_set | map(attribute="cidr_block") | list) + - vpc_cidr_b in (vpc_info.vpcs[0].cidr_block_association_set | map(attribute="cidr_block") | list) - name: modify CIDR - no purge (no change - different order) ec2_vpc_net: @@ -1156,9 +1156,9 @@ - result.vpc.cidr_block_association_set[0].cidr_block_state.state in ["associated", "associating"] - result.vpc.cidr_block_association_set[1].cidr_block_state.state in ["associated", "associating"] - result.vpc.cidr_block_association_set[2].cidr_block_state.state in ["associated", "associating"] - - vpc_cidr in (result.vpc | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) - - vpc_cidr_a in (result.vpc | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) - - vpc_cidr_b in (result.vpc | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) + - vpc_cidr in (result.vpc.cidr_block_association_set | map(attribute="cidr_block") | list) + - vpc_cidr_a in (result.vpc.cidr_block_association_set | map(attribute="cidr_block") | list) + - vpc_cidr_b in (result.vpc.cidr_block_association_set | map(attribute="cidr_block") | list) - vpc_info.vpcs[0].cidr_block_association_set | length == 3 - vpc_info.vpcs[0].cidr_block_association_set[0].association_id.startswith("vpc-cidr-assoc-") - vpc_info.vpcs[0].cidr_block_association_set[1].association_id.startswith("vpc-cidr-assoc-") @@ -1166,9 +1166,9 @@ - vpc_info.vpcs[0].cidr_block_association_set[0].cidr_block_state.state in ["associated", "associating"] - vpc_info.vpcs[0].cidr_block_association_set[1].cidr_block_state.state in ["associated", "associating"] - vpc_info.vpcs[0].cidr_block_association_set[2].cidr_block_state.state in ["associated", "associating"] - - vpc_cidr in (vpc_info.vpcs[0] | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) - - vpc_cidr_a in (vpc_info.vpcs[0] | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) - - vpc_cidr_b in (vpc_info.vpcs[0] | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) + - vpc_cidr in (vpc_info.vpcs[0].cidr_block_association_set | map(attribute="cidr_block") | list) + - vpc_cidr_a in (vpc_info.vpcs[0].cidr_block_association_set | map(attribute="cidr_block") | list) + - vpc_cidr_b in (vpc_info.vpcs[0].cidr_block_association_set | map(attribute="cidr_block") | list) # #62678 #- name: modify CIDR - purge (check mode) @@ -1200,9 +1200,9 @@ # - vpc_info.vpcs[0].cidr_block_association_set[0].cidr_block_state.state in ["associated", "associating"] # - vpc_info.vpcs[0].cidr_block_association_set[1].cidr_block_state.state in ["associated", "associating"] # - vpc_info.vpcs[0].cidr_block_association_set[2].cidr_block_state.state in ["associated", "associating"] - # - vpc_cidr in (vpc_info.vpcs[0] | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) - # - vpc_cidr_a in (vpc_info.vpcs[0] | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) - # - vpc_cidr_b in (vpc_info.vpcs[0] | community.general.json_query("cidr_block_association_set[*].cidr_block") | list) + # - vpc_cidr in (vpc_info.vpcs[0].cidr_block_association_set | map(attribute="cidr_block") | list) + # - vpc_cidr_a in (vpc_info.vpcs[0].cidr_block_association_set | map(attribute="cidr_block") | list) + # - vpc_cidr_b in (vpc_info.vpcs[0].cidr_block_association_set | map(attribute="cidr_block") | list) - name: modify CIDR - purge ec2_vpc_net: @@ -1219,8 +1219,6 @@ register: vpc_info - name: assert the CIDRs changed - vars: - cidr_query: 'cidr_block_association_set[?cidr_block_state.state == `associated`].cidr_block' assert: that: - result is successful @@ -1229,14 +1227,14 @@ - vpc_info.vpcs | length == 1 - result.vpc.cidr_block == vpc_cidr - vpc_info.vpcs[0].cidr_block == vpc_cidr - - result.vpc | community.general.json_query(cidr_query) | list | length == 2 - - vpc_cidr in (result.vpc | community.general.json_query(cidr_query) | list) - - vpc_cidr_a not in (result.vpc | community.general.json_query(cidr_query) | list) - - vpc_cidr_b in (result.vpc | community.general.json_query(cidr_query) | list) - - vpc_info.vpcs[0] | community.general.json_query(cidr_query) | list | length == 2 - - vpc_cidr in (vpc_info.vpcs[0] | community.general.json_query(cidr_query) | list) - - vpc_cidr_a not in (vpc_info.vpcs[0] | community.general.json_query(cidr_query) | list) - - vpc_cidr_b in (vpc_info.vpcs[0] | community.general.json_query(cidr_query) | list) + - result.vpc.cidr_block_association_set | selectattr('cidr_block_state.state', 'equalto', 'associated') | map(attribute='cidr_block') | list | length == 2 + - vpc_cidr in (result.vpc.cidr_block_association_set | selectattr('cidr_block_state.state', 'equalto', 'associated') | map(attribute='cidr_block') | list) + - vpc_cidr_a not in (result.vpc.cidr_block_association_set | selectattr('cidr_block_state.state', 'equalto', 'associated') | map(attribute='cidr_block')) + - vpc_cidr_b in (result.vpc.cidr_block_association_set | selectattr('cidr_block_state.state', 'equalto', 'associated') | map(attribute='cidr_block')) + - vpc_info.vpcs[0].cidr_block_association_set | selectattr('cidr_block_state.state', 'equalto', 'associated') | map(attribute='cidr_block') | list | length == 2 + - vpc_cidr in (vpc_info.vpcs[0].cidr_block_association_set | selectattr('cidr_block_state.state', 'equalto', 'associated') | map(attribute='cidr_block') | list) + - vpc_cidr_a not in (vpc_info.vpcs[0].cidr_block_association_set | selectattr('cidr_block_state.state', 'equalto', 'associated') | map(attribute='cidr_block') | list) + - vpc_cidr_b in (vpc_info.vpcs[0].cidr_block_association_set | selectattr('cidr_block_state.state', 'equalto', 'associated') | map(attribute='cidr_block') | list) - name: modify CIDR - purge (no change) ec2_vpc_net: @@ -1253,8 +1251,6 @@ register: vpc_info - name: assert the CIDRs didn't change - vars: - cidr_query: 'cidr_block_association_set[?cidr_block_state.state == `associated`].cidr_block' assert: that: - result is successful @@ -1263,14 +1259,14 @@ - vpc_info.vpcs | length == 1 - result.vpc.cidr_block == vpc_cidr - vpc_info.vpcs[0].cidr_block == vpc_cidr - - result.vpc | community.general.json_query(cidr_query) | list | length == 2 - - vpc_cidr in (result.vpc | community.general.json_query(cidr_query) | list) - - vpc_cidr_a not in (result.vpc | community.general.json_query(cidr_query) | list) - - vpc_cidr_b in (result.vpc | community.general.json_query(cidr_query) | list) - - vpc_info.vpcs[0] | community.general.json_query(cidr_query) | list | length == 2 - - vpc_cidr in (vpc_info.vpcs[0] | community.general.json_query(cidr_query) | list) - - vpc_cidr_a not in (vpc_info.vpcs[0] | community.general.json_query(cidr_query) | list) - - vpc_cidr_b in (vpc_info.vpcs[0] | community.general.json_query(cidr_query) | list) + - result.vpc.cidr_block_association_set | selectattr('cidr_block_state.state', 'equalto', 'associated') | map(attribute='cidr_block') | list | length == 2 + - vpc_cidr in (result.vpc.cidr_block_association_set | selectattr('cidr_block_state.state', 'equalto', 'associated') | map(attribute='cidr_block') | list) + - vpc_cidr_a not in (result.vpc.cidr_block_association_set | selectattr('cidr_block_state.state', 'equalto', 'associated') | map(attribute='cidr_block') | list) + - vpc_cidr_b in (result.vpc.cidr_block_association_set | selectattr('cidr_block_state.state', 'equalto', 'associated') | map(attribute='cidr_block') | list) + - vpc_info.vpcs[0].cidr_block_association_set | selectattr('cidr_block_state.state', 'equalto', 'associated') | map(attribute='cidr_block') | list | length == 2 + - vpc_cidr in (vpc_info.vpcs[0].cidr_block_association_set | selectattr('cidr_block_state.state', 'equalto', 'associated') | map(attribute='cidr_block') | list) + - vpc_cidr_a not in (vpc_info.vpcs[0].cidr_block_association_set | selectattr('cidr_block_state.state', 'equalto', 'associated') | map(attribute='cidr_block') | list) + - vpc_cidr_b in (vpc_info.vpcs[0].cidr_block_association_set | selectattr('cidr_block_state.state', 'equalto', 'associated') | map(attribute='cidr_block') | list) # ============================================================ diff --git a/tests/integration/targets/inventory_aws_ec2/playbooks/populate_cache.yml b/tests/integration/targets/inventory_aws_ec2/playbooks/populate_cache.yml index 64e8da4c749..1308fab93a7 100644 --- a/tests/integration/targets/inventory_aws_ec2/playbooks/populate_cache.yml +++ b/tests/integration/targets/inventory_aws_ec2/playbooks/populate_cache.yml @@ -3,8 +3,6 @@ connection: local gather_facts: no environment: "{{ ansible_test.environment }}" - collections: - - community.general tasks: - module_defaults: diff --git a/tests/integration/targets/inventory_aws_ec2/templates/inventory_with_constructed.yml.j2 b/tests/integration/targets/inventory_aws_ec2/templates/inventory_with_constructed.yml.j2 index c0ebcbfcb47..a33f03e21c7 100644 --- a/tests/integration/targets/inventory_aws_ec2/templates/inventory_with_constructed.yml.j2 +++ b/tests/integration/targets/inventory_aws_ec2/templates/inventory_with_constructed.yml.j2 @@ -10,7 +10,7 @@ filters: tag:Name: - '{{ resource_prefix }}' keyed_groups: -- key: 'security_groups|community.general.json_query("[].group_id")' +- key: 'security_groups|map(attribute="group_id")' prefix: security_groups - key: tags prefix: tag diff --git a/tests/integration/targets/inventory_aws_rds/templates/inventory_with_constructed.j2 b/tests/integration/targets/inventory_aws_rds/templates/inventory_with_constructed.j2 index a0636a971bd..c5603ef874a 100644 --- a/tests/integration/targets/inventory_aws_rds/templates/inventory_with_constructed.j2 +++ b/tests/integration/targets/inventory_aws_rds/templates/inventory_with_constructed.j2 @@ -7,7 +7,7 @@ aws_security_token: '{{ security_token }}' regions: - '{{ aws_region }}' keyed_groups: - - key: 'db_parameter_groups|community.general.json_query("[].db_parameter_group_name")' + - key: 'db_parameter_groups|map(attribute="db_parameter_group_name")' prefix: rds_parameter_group - key: tags prefix: tag diff --git a/tests/requirements.yml b/tests/requirements.yml index 63120f9b2af..77938b9e2dc 100644 --- a/tests/requirements.yml +++ b/tests/requirements.yml @@ -1,5 +1,4 @@ integration_tests_dependencies: - ansible.windows -- community.general - ansible.netcommon # ipv6 filter unit_tests_dependencies: [] From 0164b48e96e3785d7c4783f438ced544e45d800b Mon Sep 17 00:00:00 2001 From: Mark Chappell Date: Thu, 20 May 2021 20:37:56 +0200 Subject: [PATCH 36/38] Cloudformation: use is_boto3_error_message (#355) Cloudformation: use is_boto3_error_message Reviewed-by: https://github.com/apps/ansible-zuul --- plugins/modules/cloudformation.py | 62 +++++++++---------- .../plugins/modules/test_cloudformation.py | 34 +++++++--- 2 files changed, 52 insertions(+), 44 deletions(-) diff --git a/plugins/modules/cloudformation.py b/plugins/modules/cloudformation.py index fd42caeac52..e4eb99e52e1 100644 --- a/plugins/modules/cloudformation.py +++ b/plugins/modules/cloudformation.py @@ -342,6 +342,7 @@ from ansible.module_utils._text import to_native from ..module_utils.core import AnsibleAWSModule +from ..module_utils.core import is_boto3_error_message from ..module_utils.ec2 import AWSRetry from ..module_utils.ec2 import ansible_dict_to_boto3_tag_list from ..module_utils.ec2 import boto_exception @@ -364,12 +365,11 @@ def get_stack_events(cfn, stack_name, events_limit, token_filter=None): )) else: events = list(pg.search("StackEvents[*]")) - except (botocore.exceptions.ValidationError, botocore.exceptions.ClientError) as err: + except is_boto3_error_message('does not exist'): + ret['log'].append('Stack does not exist.') + return ret + except (botocore.exceptions.ValidationError, botocore.exceptions.ClientError) as err: # pylint: disable=duplicate-except error_msg = boto_exception(err) - if 'does not exist' in error_msg: - # missing stack, don't bail. - ret['log'].append('Stack does not exist.') - return ret ret['log'].append('Unknown error: ' + str(error_msg)) return ret @@ -406,7 +406,7 @@ def create_stack(module, stack_params, cfn, events_limit): try: response = cfn.create_stack(**stack_params) # Use stack ID to follow stack state in case of on_create_failure = DELETE - result = stack_operation(cfn, response['StackId'], 'CREATE', events_limit, stack_params.get('ClientRequestToken', None)) + result = stack_operation(module, cfn, response['StackId'], 'CREATE', events_limit, stack_params.get('ClientRequestToken', None)) except Exception as err: module.fail_json_aws(err, msg="Failed to create stack {0}".format(stack_params.get('StackName'))) if not result: @@ -459,17 +459,15 @@ def create_changeset(module, stack_params, cfn, events_limit): break # Lets not hog the cpu/spam the AWS API time.sleep(1) - result = stack_operation(cfn, stack_params['StackName'], 'CREATE_CHANGESET', events_limit) + result = stack_operation(module, cfn, stack_params['StackName'], 'CREATE_CHANGESET', events_limit) result['change_set_id'] = cs['Id'] result['warnings'] = ['Created changeset named %s for stack %s' % (changeset_name, stack_params['StackName']), 'You can execute it using: aws cloudformation execute-change-set --change-set-name %s' % cs['Id'], 'NOTE that dependencies on this stack might fail due to pending changes!'] + except is_boto3_error_message('No updates are to be performed.'): + result = dict(changed=False, output='Stack is already up-to-date.') except Exception as err: - error_msg = boto_exception(err) - if 'No updates are to be performed.' in error_msg: - result = dict(changed=False, output='Stack is already up-to-date.') - else: - module.fail_json_aws(err, msg='Failed to create change set') + module.fail_json_aws(err, msg='Failed to create change set') if not result: module.fail_json(msg="empty result") @@ -488,13 +486,11 @@ def update_stack(module, stack_params, cfn, events_limit): # don't need to be updated. try: cfn.update_stack(**stack_params) - result = stack_operation(cfn, stack_params['StackName'], 'UPDATE', events_limit, stack_params.get('ClientRequestToken', None)) + result = stack_operation(module, cfn, stack_params['StackName'], 'UPDATE', events_limit, stack_params.get('ClientRequestToken', None)) + except is_boto3_error_message('No updates are to be performed.'): + result = dict(changed=False, output='Stack is already up-to-date.') except Exception as err: - error_msg = boto_exception(err) - if 'No updates are to be performed.' in error_msg: - result = dict(changed=False, output='Stack is already up-to-date.') - else: - module.fail_json_aws(err, msg="Failed to update stack {0}".format(stack_params.get('StackName'))) + module.fail_json_aws(err, msg="Failed to update stack {0}".format(stack_params.get('StackName'))) if not result: module.fail_json(msg="empty result") return result @@ -504,7 +500,7 @@ def update_termination_protection(module, cfn, stack_name, desired_termination_p '''updates termination protection of a stack''' if not boto_supports_termination_protection(cfn): module.fail_json(msg="termination_protection parameter requires botocore >= 1.7.18") - stack = get_stack_facts(cfn, stack_name) + stack = get_stack_facts(module, cfn, stack_name) if stack: if stack['EnableTerminationProtection'] is not desired_termination_protection_state: try: @@ -520,12 +516,12 @@ def boto_supports_termination_protection(cfn): return hasattr(cfn, "update_termination_protection") -def stack_operation(cfn, stack_name, operation, events_limit, op_token=None): +def stack_operation(module, cfn, stack_name, operation, events_limit, op_token=None): '''gets the status of a stack while it is created/updated/deleted''' existed = [] while True: try: - stack = get_stack_facts(cfn, stack_name) + stack = get_stack_facts(module, cfn, stack_name, raise_errors=True) existed.append('yes') except Exception: # If the stack previously existed, and now can't be found then it's @@ -611,18 +607,16 @@ def check_mode_changeset(module, stack_params, cfn): module.fail_json_aws(err) -def get_stack_facts(cfn, stack_name): +def get_stack_facts(module, cfn, stack_name, raise_errors=False): try: stack_response = cfn.describe_stacks(StackName=stack_name) stack_info = stack_response['Stacks'][0] - except (botocore.exceptions.ValidationError, botocore.exceptions.ClientError) as err: - error_msg = boto_exception(err) - if 'does not exist' in error_msg: - # missing stack, don't bail. - return None - - # other error, bail. - raise err + except is_boto3_error_message('does not exist'): + return None + except (botocore.exceptions.ValidationError, botocore.exceptions.ClientError) as err: # pylint: disable=duplicate-except + if raise_errors: + raise err + module.fail_json_aws(err, msg="Failed to describe stack") if stack_response and stack_response.get('Stacks', None): stacks = stack_response['Stacks'] @@ -753,7 +747,7 @@ def main(): if boto_supports_termination_protection(cfn): cfn.update_termination_protection = backoff_wrapper(cfn.update_termination_protection) - stack_info = get_stack_facts(cfn, stack_params['StackName']) + stack_info = get_stack_facts(module, cfn, stack_params['StackName']) if module.check_mode: if state == 'absent' and stack_info: @@ -778,7 +772,7 @@ def main(): # format the stack output - stack = get_stack_facts(cfn, stack_params['StackName']) + stack = get_stack_facts(module, cfn, stack_params['StackName']) if stack is not None: if result.get('stack_outputs') is None: # always define stack_outputs, but it may be empty @@ -804,7 +798,7 @@ def main(): # so must describe the stack first try: - stack = get_stack_facts(cfn, stack_params['StackName']) + stack = get_stack_facts(module, cfn, stack_params['StackName']) if not stack: result = {'changed': False, 'output': 'Stack not found.'} else: @@ -812,7 +806,7 @@ def main(): cfn.delete_stack(StackName=stack_params['StackName']) else: cfn.delete_stack(StackName=stack_params['StackName'], RoleARN=stack_params['RoleARN']) - result = stack_operation(cfn, stack_params['StackName'], 'DELETE', module.params.get('events_limit'), + result = stack_operation(module, cfn, stack_params['StackName'], 'DELETE', module.params.get('events_limit'), stack_params.get('ClientRequestToken', None)) except Exception as err: module.fail_json_aws(err) diff --git a/tests/unit/plugins/modules/test_cloudformation.py b/tests/unit/plugins/modules/test_cloudformation.py index 6ee1fcf95d5..3b0e7c9fb5e 100644 --- a/tests/unit/plugins/modules/test_cloudformation.py +++ b/tests/unit/plugins/modules/test_cloudformation.py @@ -12,6 +12,9 @@ # Magic... from ansible_collections.amazon.aws.tests.unit.utils.amazon_placebo_fixtures import maybe_sleep, placeboify # pylint: disable=unused-import +import ansible_collections.amazon.aws.plugins.module_utils.core as aws_core +import ansible_collections.amazon.aws.plugins.module_utils.ec2 as aws_ec2 + from ansible_collections.amazon.aws.plugins.module_utils.ec2 import boto_exception from ansible_collections.amazon.aws.plugins.modules import cloudformation as cfn_module @@ -78,8 +81,15 @@ def exit_json(self, *args, **kwargs): raise Exception('EXIT') -def test_invalid_template_json(placeboify): +def _create_wrapped_client(placeboify): connection = placeboify.client('cloudformation') + retry_decorator = aws_ec2.AWSRetry.jittered_backoff() + wrapped_conn = aws_core._RetryingBotoClientWrapper(connection, retry_decorator) + return wrapped_conn + + +def test_invalid_template_json(placeboify): + connection = _create_wrapped_client(placeboify) params = { 'StackName': 'ansible-test-wrong-json', 'TemplateBody': bad_json_tpl, @@ -94,7 +104,7 @@ def test_invalid_template_json(placeboify): def test_client_request_token_s3_stack(maybe_sleep, placeboify): - connection = placeboify.client('cloudformation') + connection = _create_wrapped_client(placeboify) params = { 'StackName': 'ansible-test-client-request-token-yaml', 'TemplateBody': basic_yaml_tpl, @@ -111,7 +121,7 @@ def test_client_request_token_s3_stack(maybe_sleep, placeboify): def test_basic_s3_stack(maybe_sleep, placeboify): - connection = placeboify.client('cloudformation') + connection = _create_wrapped_client(placeboify) params = { 'StackName': 'ansible-test-basic-yaml', 'TemplateBody': basic_yaml_tpl @@ -127,15 +137,19 @@ def test_basic_s3_stack(maybe_sleep, placeboify): def test_delete_nonexistent_stack(maybe_sleep, placeboify): - connection = placeboify.client('cloudformation') - result = cfn_module.stack_operation(connection, 'ansible-test-nonexist', 'DELETE', default_events_limit) + connection = _create_wrapped_client(placeboify) + # module is only used if we threw an unexpected error + module = None + result = cfn_module.stack_operation(module, connection, 'ansible-test-nonexist', 'DELETE', default_events_limit) assert result['changed'] assert 'Stack does not exist.' in result['log'] def test_get_nonexistent_stack(placeboify): - connection = placeboify.client('cloudformation') - assert cfn_module.get_stack_facts(connection, 'ansible-test-nonexist') is None + connection = _create_wrapped_client(placeboify) + # module is only used if we threw an unexpected error + module = None + assert cfn_module.get_stack_facts(module, connection, 'ansible-test-nonexist') is None def test_missing_template_body(): @@ -159,7 +173,7 @@ def test_on_create_failure_delete(maybe_sleep, placeboify): on_create_failure='DELETE', disable_rollback=False, ) - connection = placeboify.client('cloudformation') + connection = _create_wrapped_client(placeboify) params = { 'StackName': 'ansible-test-on-create-failure-delete', 'TemplateBody': failing_yaml_tpl @@ -178,7 +192,7 @@ def test_on_create_failure_rollback(maybe_sleep, placeboify): on_create_failure='ROLLBACK', disable_rollback=False, ) - connection = placeboify.client('cloudformation') + connection = _create_wrapped_client(placeboify) params = { 'StackName': 'ansible-test-on-create-failure-rollback', 'TemplateBody': failing_yaml_tpl @@ -198,7 +212,7 @@ def test_on_create_failure_do_nothing(maybe_sleep, placeboify): on_create_failure='DO_NOTHING', disable_rollback=False, ) - connection = placeboify.client('cloudformation') + connection = _create_wrapped_client(placeboify) params = { 'StackName': 'ansible-test-on-create-failure-do-nothing', 'TemplateBody': failing_yaml_tpl From cf39a04cd75512109995abe496f58260766af63a Mon Sep 17 00:00:00 2001 From: Mark Chappell Date: Fri, 21 May 2021 11:40:23 +0200 Subject: [PATCH 37/38] cloudformation - use standard retry decorator pattern (#358) cloudformation - use standard retry decorator pattern Reviewed-by: https://github.com/apps/ansible-zuul --- plugins/modules/cloudformation.py | 48 ++++++++++++++----------------- 1 file changed, 21 insertions(+), 27 deletions(-) diff --git a/plugins/modules/cloudformation.py b/plugins/modules/cloudformation.py index e4eb99e52e1..2d3e0453f44 100644 --- a/plugins/modules/cloudformation.py +++ b/plugins/modules/cloudformation.py @@ -347,6 +347,10 @@ from ..module_utils.ec2 import ansible_dict_to_boto3_tag_list from ..module_utils.ec2 import boto_exception +# Set a default, mostly for our integration tests. This will be overridden in +# the main() loop to match the parameters we're passed +retry_decorator = AWSRetry.jittered_backoff() + def get_stack_events(cfn, stack_name, events_limit, token_filter=None): '''This event data was never correct, it worked as a side effect. So the v2.3 format is different.''' @@ -360,7 +364,7 @@ def get_stack_events(cfn, stack_name, events_limit, token_filter=None): PaginationConfig={'MaxItems': events_limit} ) if token_filter is not None: - events = list(pg.search( + events = list(retry_decorator(pg.search)( "StackEvents[?ClientRequestToken == '{0}']".format(token_filter) )) else: @@ -404,7 +408,7 @@ def create_stack(module, stack_params, cfn, events_limit): module.fail_json(msg="termination_protection parameter requires botocore >= 1.7.18") try: - response = cfn.create_stack(**stack_params) + response = cfn.create_stack(aws_retry=True, **stack_params) # Use stack ID to follow stack state in case of on_create_failure = DELETE result = stack_operation(module, cfn, response['StackId'], 'CREATE', events_limit, stack_params.get('ClientRequestToken', None)) except Exception as err: @@ -415,7 +419,7 @@ def create_stack(module, stack_params, cfn, events_limit): def list_changesets(cfn, stack_name): - res = cfn.list_change_sets(StackName=stack_name) + res = cfn.list_change_sets(aws_retry=True, StackName=stack_name) return [cs['ChangeSetName'] for cs in res['Summaries']] @@ -438,18 +442,18 @@ def create_changeset(module, stack_params, cfn, events_limit): warning = 'WARNING: %d pending changeset(s) exist(s) for this stack!' % len(pending_changesets) result = dict(changed=False, output='ChangeSet %s already exists.' % changeset_name, warnings=[warning]) else: - cs = cfn.create_change_set(**stack_params) + cs = cfn.create_change_set(aws_retry=True, **stack_params) # Make sure we don't enter an infinite loop time_end = time.time() + 600 while time.time() < time_end: try: - newcs = cfn.describe_change_set(ChangeSetName=cs['Id']) + newcs = cfn.describe_change_set(aws_retry=True, ChangeSetName=cs['Id']) except botocore.exceptions.BotoCoreError as err: module.fail_json_aws(err) if newcs['Status'] == 'CREATE_PENDING' or newcs['Status'] == 'CREATE_IN_PROGRESS': time.sleep(1) elif newcs['Status'] == 'FAILED' and "The submitted information didn't contain changes" in newcs['StatusReason']: - cfn.delete_change_set(ChangeSetName=cs['Id']) + cfn.delete_change_set(aws_retry=True, ChangeSetName=cs['Id']) result = dict(changed=False, output='The created Change Set did not contain any changes to this stack and was deleted.') # a failed change set does not trigger any stack events so we just want to @@ -485,7 +489,7 @@ def update_stack(module, stack_params, cfn, events_limit): # AWS will tell us if the stack template and parameters are the same and # don't need to be updated. try: - cfn.update_stack(**stack_params) + cfn.update_stack(aws_retry=True, **stack_params) result = stack_operation(module, cfn, stack_params['StackName'], 'UPDATE', events_limit, stack_params.get('ClientRequestToken', None)) except is_boto3_error_message('No updates are to be performed.'): result = dict(changed=False, output='Stack is already up-to-date.') @@ -505,6 +509,7 @@ def update_termination_protection(module, cfn, stack_name, desired_termination_p if stack['EnableTerminationProtection'] is not desired_termination_protection_state: try: cfn.update_termination_protection( + aws_retry=True, EnableTerminationProtection=desired_termination_protection_state, StackName=stack_name) except botocore.exceptions.ClientError as e: @@ -585,9 +590,9 @@ def check_mode_changeset(module, stack_params, cfn): stack_params.pop('ClientRequestToken', None) try: - change_set = cfn.create_change_set(**stack_params) + change_set = cfn.create_change_set(aws_retry=True, **stack_params) for i in range(60): # total time 5 min - description = cfn.describe_change_set(ChangeSetName=change_set['Id']) + description = cfn.describe_change_set(aws_retry=True, ChangeSetName=change_set['Id']) if description['Status'] in ('CREATE_COMPLETE', 'FAILED'): break time.sleep(5) @@ -595,7 +600,7 @@ def check_mode_changeset(module, stack_params, cfn): # if the changeset doesn't finish in 5 mins, this `else` will trigger and fail module.fail_json(msg="Failed to create change set %s" % stack_params['ChangeSetName']) - cfn.delete_change_set(ChangeSetName=change_set['Id']) + cfn.delete_change_set(aws_retry=True, ChangeSetName=change_set['Id']) reason = description.get('StatusReason') @@ -609,7 +614,7 @@ def check_mode_changeset(module, stack_params, cfn): def get_stack_facts(module, cfn, stack_name, raise_errors=False): try: - stack_response = cfn.describe_stacks(StackName=stack_name) + stack_response = cfn.describe_stacks(aws_retry=True, StackName=stack_name) stack_info = stack_response['Stacks'][0] except is_boto3_error_message('does not exist'): return None @@ -727,25 +732,14 @@ def main(): result = {} - cfn = module.client('cloudformation') - # Wrap the cloudformation client methods that this module uses with # automatic backoff / retry for throttling error codes - backoff_wrapper = AWSRetry.jittered_backoff( + retry_decorator = AWSRetry.jittered_backoff( retries=module.params.get('backoff_retries'), delay=module.params.get('backoff_delay'), max_delay=module.params.get('backoff_max_delay') ) - cfn.describe_stack_events = backoff_wrapper(cfn.describe_stack_events) - cfn.create_stack = backoff_wrapper(cfn.create_stack) - cfn.list_change_sets = backoff_wrapper(cfn.list_change_sets) - cfn.create_change_set = backoff_wrapper(cfn.create_change_set) - cfn.update_stack = backoff_wrapper(cfn.update_stack) - cfn.describe_stacks = backoff_wrapper(cfn.describe_stacks) - cfn.list_stack_resources = backoff_wrapper(cfn.list_stack_resources) - cfn.delete_stack = backoff_wrapper(cfn.delete_stack) - if boto_supports_termination_protection(cfn): - cfn.update_termination_protection = backoff_wrapper(cfn.update_termination_protection) + cfn = module.client('cloudformation', retry_decorator=retry_decorator) stack_info = get_stack_facts(module, cfn, stack_params['StackName']) @@ -780,7 +774,7 @@ def main(): for output in stack.get('Outputs', []): result['stack_outputs'][output['OutputKey']] = output['OutputValue'] stack_resources = [] - reslist = cfn.list_stack_resources(StackName=stack_params['StackName']) + reslist = cfn.list_stack_resources(aws_retry=True, StackName=stack_params['StackName']) for res in reslist.get('StackResourceSummaries', []): stack_resources.append({ "logical_resource_id": res['LogicalResourceId'], @@ -803,9 +797,9 @@ def main(): result = {'changed': False, 'output': 'Stack not found.'} else: if stack_params.get('RoleARN') is None: - cfn.delete_stack(StackName=stack_params['StackName']) + cfn.delete_stack(aws_retry=True, StackName=stack_params['StackName']) else: - cfn.delete_stack(StackName=stack_params['StackName'], RoleARN=stack_params['RoleARN']) + cfn.delete_stack(aws_retry=True, StackName=stack_params['StackName'], RoleARN=stack_params['RoleARN']) result = stack_operation(module, cfn, stack_params['StackName'], 'DELETE', module.params.get('events_limit'), stack_params.get('ClientRequestToken', None)) except Exception as err: From dff146be9cb89b0e4857a721dd02103d084cb0fc Mon Sep 17 00:00:00 2001 From: Jill R <4121322+jillr@users.noreply.github.com> Date: Fri, 21 May 2021 09:51:55 -0700 Subject: [PATCH 38/38] Update README with new python and botocore requirements (#369) Update README with new python and botocore requirements Reviewed-by: https://github.com/apps/ansible-zuul --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 4b281c37f66..676d987142b 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,11 @@ PEP440 is the schema used to describe the versions of Ansible. ## Python version compatibility -This collection depends on the AWS SDK for Python (Boto3 and Botocore). As AWS has [ceased supporting Python 2.6](https://aws.amazon.com/blogs/developer/deprecation-of-python-2-6-and-python-3-3-in-botocore-boto3-and-the-aws-cli/), this collection requires Python 2.7 or greater. +As the AWS SDK for Python (Boto3 and Botocore) has [ceased supporting Python 2.7](https://aws.amazon.com/blogs/developer/announcing-end-of-support-for-python-2-7-in-aws-sdk-for-python-and-aws-cli-v1/), this collection requires Python 3.6 or greater. + +Starting with the 2.0.0 releases of amazon.aws and community.aws, it is generally the collection's policy to support the versions of `botocore` and `boto3` that were released 12 months prior to the most recent major collection release, following semantic versioning (for example, 2.0.0, 3.0.0). + +Version 2.0.0 of this collection supports `boto3 >= 1.13.0` and `botocore >= 1.16.0` ## Included content