tests: Randomise VPC/subnet CIDRs and fix elb_target timing issues

.github/workflows/integration.yml

@@ -63,7 +63,7 @@ jobs:
 
       - name: list changes for pull request
         id: compute
-        uses: ansible/cloud-content-ci-automation/.github/actions/ansible_test_splitter@0b891487c4170ef856f02313c66cb1c776814c42
+        uses: ansible/cloud-content-ci-automation/.github/actions/ansible_test_splitter@6659b608aab58bf1d20641b805eb9cb4fd2b771f
         with:
           collections_to_test: "${{ env.source_dir }}"
           total_jobs: 24
@@ -110,7 +110,7 @@ jobs:
 
       - name: Read job targets
         id: read-targets
-        uses: ansible/cloud-content-ci-automation/.github/actions/ansible_read_targets@0b891487c4170ef856f02313c66cb1c776814c42
+        uses: ansible/cloud-content-ci-automation/.github/actions/ansible_read_targets@6659b608aab58bf1d20641b805eb9cb4fd2b771f
         with:
           job_id: "${{ matrix.job-id }}"
           all_targets: "${{ env.all_targets }}"

changelogs/fragments/2457-randomise-test-cidrs.yml

@@ -0,0 +1,9 @@
+trivial:
+  - integration tests - add setup_ec2_instance_env and setup_ec2_vpc infrastructure roles from amazon.aws for comprehensive VPC setup/cleanup with ENI handling (https://github.com/ansible-collections/community.aws/pull/2457).
+  - integration tests - fix elb_target Lambda state polling and ENI cleanup timing to prevent test failures (https://github.com/ansible-collections/community.aws/pull/2457).
+  - integration tests - fix glue_job to use supported Glue versions (4.0, 5.0) instead of deprecated versions (0.9, 2.0) (https://github.com/ansible-collections/community.aws/pull/2457).
+  - integration tests - migrate all VPC and subnet CIDR definitions from task files into defaults files for consistency (https://github.com/ansible-collections/community.aws/pull/2457).
+  - integration tests - randomise VPC and subnet CIDR allocations across 24 integration tests using pattern ``10.{{ 256 | random(seed=resource_prefix) }}.Y.0/Z`` to prevent collisions when running tests in parallel (https://github.com/ansible-collections/community.aws/pull/2457).
+  - integration tests - distribute third octets from 252 down in decrements of 4 (252, 248, 244, ..., 160) mirroring amazon.aws pattern from opposite end of range (https://github.com/ansible-collections/community.aws/pull/2457).
+  - integration tests - reduce VPC size from /16 to /22 for more appropriate test environment resource usage (https://github.com/ansible-collections/community.aws/pull/2457).
+  - integration tests - update ansible_test_splitter to match amazon.aws version with fix for correctly ignoring infrastructure roles in test target detection (https://github.com/ansible-collections/community.aws/pull/2457).

tests/integration/targets/autoscaling_complete_lifecycle_action/defaults/main.yml

@@ -7,13 +7,13 @@ asg_name: '{{ tiny_prefix }}-{{ resource_sufix }}'
 
 vpc_name: '{{ tiny_prefix }}-{{ resource_sufix }}'
 vpc_seed: '{{ resource_prefix }}'
-vpc_cidr: '10.{{ 256 | random(seed=vpc_seed) }}.0.0/16'
+vpc_cidr: '10.{{ 256 | random(seed=vpc_seed) }}.252.0/22'
 
 rtb_name: '{{ tiny_prefix }}-{{ resource_sufix }}'
 sg_name: '{{ tiny_prefix }}-{{ resource_sufix }}'
 lc_name: '{{ tiny_prefix }}-{{ resource_sufix }}'
 
 subnet_az: '{{ ec2_availability_zone_names[0] }}'
-subnet_cidr: '10.{{ 256 | random(seed=vpc_seed) }}.157.0/24'
-subnet_startswith: '10.{{ 256 | random(seed=vpc_seed) }}.157.'
+subnet_cidr: "{{ vpc_cidr | ansible.utils.ipsubnet(24, 0) }}"
+subnet_startswith: "{{ (subnet_cidr.split('.')[:3] + ['']) | join('.') }}"
 subnet_name: '{{ tiny_prefix }}-{{ resource_sufix }}'

tests/integration/targets/autoscaling_launch_config/defaults/main.yml

@@ -4,3 +4,8 @@ ec2_instance_name: '{{ resource_prefix }}-node'
 ec2_instance_owner: 'integration-run-{{ resource_prefix }}'
 ec2_instance_type: t2.micro
 alarm_prefix: "ansible-test"
+
+# VPC CIDR allocation - randomised to avoid conflicts
+vpc_cidr: 10.{{ 256 | random(seed=resource_prefix) }}.248.0/23
+subnet_a_cidr: "{{ vpc_cidr | ansible.utils.ipsubnet(24, 0) }}"
+subnet_b_cidr: "{{ vpc_cidr | ansible.utils.ipsubnet(24, 1) }}"

tests/integration/targets/autoscaling_launch_config/tasks/env_cleanup.yml

@@ -64,7 +64,7 @@
   ec2_vpc_subnet:
     state: absent
     vpc_id: "{{ testing_vpc.vpc.id }}"
-    cidr: 10.22.32.0/24
+    cidr: "{{ subnet_a_cidr }}"
   register: removed
   until: removed is not failed
   ignore_errors: yes
@@ -74,7 +74,7 @@
   ec2_vpc_subnet:
     state: absent
     vpc_id: "{{ testing_vpc.vpc.id }}"
-    cidr: 10.22.33.0/24
+    cidr: "{{ subnet_b_cidr }}"
   register: removed
   until: removed is not failed
   ignore_errors: yes
@@ -83,7 +83,7 @@
 - name: remove the VPC
   ec2_vpc_net:
     name: "{{ resource_prefix }}-vpc"
-    cidr_block: 10.22.32.0/23
+    cidr_block: "{{ vpc_cidr }}"
     state: absent
     tags:
       Name: Ansible Testing VPC

tests/integration/targets/autoscaling_launch_config/tasks/env_setup.yml

@@ -1,7 +1,7 @@
 - name: Create VPC for use in testing
   ec2_vpc_net:
     name: "{{ resource_prefix }}-vpc"
-    cidr_block: 10.22.32.0/23
+    cidr_block: "{{ vpc_cidr }}"
     tags:
       Name: Ansible ec2_lc Testing VPC
     tenancy: default
@@ -19,7 +19,7 @@
   ec2_vpc_subnet:
     state: present
     vpc_id: "{{ testing_vpc.vpc.id }}"
-    cidr: 10.22.32.0/24
+    cidr: "{{ subnet_a_cidr }}"
     az: "{{ aws_region }}a"
     resource_tags:
       Name: "{{ resource_prefix }}-subnet-a"
@@ -29,7 +29,7 @@
   ec2_vpc_subnet:
     state: present
     vpc_id: "{{ testing_vpc.vpc.id }}"
-    cidr: 10.22.33.0/24
+    cidr: "{{ subnet_b_cidr }}"
     az: "{{ aws_region }}b"
     resource_tags:
       Name: "{{ resource_prefix }}-subnet-b"

tests/integration/targets/autoscaling_lifecycle_hook/roles/ec2_asg_lifecycle_hook/defaults/main.yml

@@ -2,3 +2,7 @@
 # defaults file for ec2_asg_lifecycle_hook
 # Amazon Linux 2 AMI 2019.06.12 (HVM), GP2 Volume Type
 load_balancer_name: "{{ tiny_prefix }}-lb"
+
+# VPC CIDR allocation - randomised to avoid conflicts
+vpc_cidr: 10.{{ 256 | random(seed=resource_prefix) }}.244.0/24
+subnet_cidr: "{{ vpc_cidr | ansible.utils.ipsubnet(24, 0) }}"

tests/integration/targets/autoscaling_lifecycle_hook/roles/ec2_asg_lifecycle_hook/tasks/env_cleanup.yml

@@ -107,7 +107,7 @@
   amazon.aws.ec2_vpc_subnet:
     state: absent
     vpc_id: "{{ testing_vpc.vpc.id }}"
-    cidr: 10.55.77.0/24
+    cidr: "{{ subnet_cidr }}"
   register: removed
   until: removed is not failed
   ignore_errors: true
@@ -116,7 +116,7 @@
 - name: Remove the VPC
   amazon.aws.ec2_vpc_net:
     name: "{{ resource_prefix }}-vpc"
-    cidr_block: 10.55.77.0/24
+    cidr_block: "{{ vpc_cidr }}"
     state: absent
   register: removed
   until: removed is not failed

tests/integration/targets/autoscaling_lifecycle_hook/roles/ec2_asg_lifecycle_hook/tasks/env_setup.yml

@@ -8,7 +8,7 @@
     - name: Create VPC for use in testing
       amazon.aws.ec2_vpc_net:
         name: "{{ resource_prefix }}-vpc"
-        cidr_block: 10.55.77.0/24
+        cidr_block: "{{ vpc_cidr }}"
         tenancy: default
       register: testing_vpc
 
@@ -22,7 +22,7 @@
       amazon.aws.ec2_vpc_subnet:
         state: present
         vpc_id: "{{ testing_vpc.vpc.id }}"
-        cidr: 10.55.77.0/24
+        cidr: "{{ subnet_cidr }}"
         az: "{{ aws_region }}a"
         resource_tags:
           Name: "{{ resource_prefix }}-subnet"

tests/integration/targets/autoscaling_scheduled_action/defaults/main.yml

@@ -1 +1,5 @@
 ---
+
+# VPC CIDR allocation - randomised to avoid conflicts
+vpc_cidr: 10.{{ 256 | random(seed=resource_prefix) }}.240.0/24
+subnet_cidr: "{{ vpc_cidr | ansible.utils.ipsubnet(24, 0) }}"

tests/integration/targets/autoscaling_scheduled_action/tasks/main.yml

@@ -14,15 +14,15 @@
     - name: Create VPC for use in testing
       ec2_vpc_net:
         name: "{{ resource_prefix }}-vpc"
-        cidr_block: 10.55.77.0/24
+        cidr_block: "{{ vpc_cidr }}"
         tenancy: default
       register: testing_vpc
 
     - name: Create subnet for use in testing
       ec2_vpc_subnet:
         state: present
         vpc_id: "{{ testing_vpc.vpc.id }}"
-        cidr: 10.55.77.0/24
+        cidr: "{{ vpc_cidr }}"
         az: "{{ aws_region }}a"
         resource_tags:
           Name: "{{ resource_prefix }}-subnet"
@@ -318,7 +318,7 @@
       ec2_vpc_subnet:
         state: absent
         vpc_id: "{{ testing_vpc.vpc.id }}"
-        cidr: 10.55.77.0/24
+        cidr: "{{ vpc_cidr }}"
       register: removed
       until: removed is not failed
       ignore_errors: yes
@@ -327,7 +327,7 @@
     - name: Remove the VPC
       ec2_vpc_net:
         name: "{{ resource_prefix }}-vpc"
-        cidr_block: 10.55.77.0/24
+        cidr_block: "{{ vpc_cidr }}"
         state: absent
       register: removed
       until: removed is not failed

tests/integration/targets/dms_replication_subnet_group/defaults/main.yml

@@ -1,2 +1,7 @@
 dms_sg_identifier: "{{ resource_prefix }}-dms"
 dms_role_role_name: dms-vpc-role
+
+# VPC CIDR allocation - randomised to avoid conflicts
+vpc_cidr: 10.{{ 256 | random(seed=resource_prefix) }}.236.0/23
+subnet_a_cidr: "{{ vpc_cidr | ansible.utils.ipsubnet(28, 0) }}"
+subnet_b_cidr: "{{ vpc_cidr | ansible.utils.ipsubnet(28, 1) }}"

tests/integration/targets/dms_replication_subnet_group/tasks/main.yml

@@ -24,7 +24,7 @@
     - name: Create VPC for use in testing
       ec2_vpc_net:
         name: "{{ resource_prefix }}-vpc"
-        cidr_block: 10.22.32.0/23
+        cidr_block: "{{ vpc_cidr }}"
         tags:
           Name: Ansible ec2_instance Testing VPC
         tenancy: default
@@ -47,15 +47,15 @@
       ec2_vpc_subnet:
         state: present
         vpc_id:  "{{ testing_vpc.vpc.id }}"
-        cidr: 10.22.32.16/28
+        cidr: "{{ subnet_a_cidr }}"
         az: "{{ az_one }}"
       register: subnet1
 
     - name: create subnet2
       ec2_vpc_subnet:
         state: present
         vpc_id:  "{{ testing_vpc.vpc.id }}"
-        cidr: 10.22.32.32/28
+        cidr: "{{ subnet_b_cidr }}"
         az: "{{ az_two }}"
       register: subnet2
 
@@ -142,20 +142,20 @@
       ec2_vpc_subnet:
         state: absent
         vpc_id:  "{{ testing_vpc.vpc.id }}"
-        cidr: 10.22.32.16/28
+        cidr: "{{ subnet_a_cidr }}"
         az: "{{ az_one }}"
 
     - name: delete subnet2
       ec2_vpc_subnet:
         state: absent
         vpc_id:  "{{ testing_vpc.vpc.id }}"
-        cidr: 10.22.32.32/28
+        cidr: "{{ subnet_b_cidr }}"
         az: "{{ az_two }}"
 
     - name: delete VPC for use in testing
       ec2_vpc_net:
         name: "{{ resource_prefix }}-vpc"
-        cidr_block: 10.22.32.0/23
+        cidr_block: "{{ vpc_cidr }}"
         tags:
           Name: Ansible ec2_instance Testing VPC
         tenancy: default

tests/integration/targets/ecs_cluster/defaults/main.yml

@@ -1,4 +1,10 @@
 ecs_cluster_name: "{{ resource_prefix }}"
+
+# VPC CIDR allocation - randomised to avoid conflicts
+vpc_cidr: 10.{{ 256 | random(seed=resource_prefix) }}.232.0/22
+subnet_a_cidr: "{{ vpc_cidr | ansible.utils.ipsubnet(24, 0) }}"
+subnet_b_cidr: "{{ vpc_cidr | ansible.utils.ipsubnet(24, 1) }}"
+
 user_data: |
   #!/bin/bash
   echo ECS_CLUSTER={{ ecs_cluster_name }} >> /etc/ecs/ecs.config

tests/integration/targets/ecs_cluster/tasks/01_create_requirements.yml

@@ -36,7 +36,7 @@
 
 - name: create a VPC to work in
   ec2_vpc_net:
-    cidr_block: 10.0.0.0/16
+    cidr_block: "{{ vpc_cidr }}"
     state: present
     name: '{{ resource_prefix }}_ecs_cluster'
     resource_tags:
@@ -61,9 +61,9 @@
   register: setup_subnet
   with_items:
     - zone: a
-      cidr: 10.0.1.0/24
+      cidr: "{{ subnet_a_cidr }}"
     - zone: b
-      cidr: 10.0.2.0/24
+      cidr: "{{ subnet_b_cidr }}"
 
 - name: create an internet gateway so that ECS agents can talk to ECS
   ec2_vpc_igw:

tests/integration/targets/ecs_cluster/tasks/99_terminate_everything.yml

@@ -364,14 +364,14 @@
     state: absent
   with_items:
     - zone: a
-      cidr: 10.0.1.0/24
+      cidr: "{{ subnet_a_cidr }}"
     - zone: b
-      cidr: 10.0.2.0/24
+      cidr: "{{ subnet_b_cidr }}"
   ignore_errors: true
 
 - name: remove setup VPC
   ec2_vpc_net:
-    cidr_block: 10.0.0.0/16
+    cidr_block: "{{ vpc_cidr }}"
     state: absent
     name: '{{ resource_prefix }}_ecs_cluster'
   ignore_errors: true

tests/integration/targets/efs/defaults/main.yml

@@ -1 +1,5 @@
-efs_name: "ansible-test-{{ tiny_prefix }}"
+efs_name: "ansible-test-{{ tiny_prefix }}"
+# VPC CIDR allocation - randomised to avoid conflicts
+vpc_cidr: 10.{{ 256 | random(seed=resource_prefix) }}.228.0/23
+subnet_a_cidr: "{{ vpc_cidr | ansible.utils.ipsubnet(24, 0) }}"
+subnet_b_cidr: "{{ vpc_cidr | ansible.utils.ipsubnet(24, 1) }}"

tests/integration/targets/efs/tasks/main.yml

@@ -14,7 +14,7 @@
     - name: Create VPC for testing
       ec2_vpc_net:
         name: "{{ efs_name }}-vpc"
-        cidr_block: 10.22.32.0/23
+        cidr_block: "{{ vpc_cidr }}"
         tags:
           Name: Ansible ec2_instance Testing VPC
         tenancy: default
@@ -24,7 +24,7 @@
       ec2_vpc_subnet:
         state: present
         vpc_id: "{{ testing_vpc.vpc.id }}"
-        cidr: 10.22.32.0/24
+        cidr: "{{ subnet_a_cidr }}"
         az: "{{ aws_region }}a"
         resource_tags:
           Name: "{{ efs_name }}-subnet-a"
@@ -34,7 +34,7 @@
       ec2_vpc_subnet:
         state: present
         vpc_id: "{{ testing_vpc.vpc.id }}"
-        cidr: 10.22.33.0/24
+        cidr: "{{ subnet_b_cidr }}"
         az: "{{ aws_region }}b"
         resource_tags:
           Name: "{{ efs_name }}-subnet-b"
@@ -589,7 +589,7 @@
       ec2_vpc_subnet:
         state: absent
         vpc_id: "{{ testing_vpc.vpc.id }}"
-        cidr: 10.22.32.0/24
+        cidr: "{{ subnet_a_cidr }}"
         az: "{{ aws_region }}a"
         resource_tags:
           Name: "{{ efs_name }}-subnet-a"
@@ -602,7 +602,7 @@
       ec2_vpc_subnet:
         state: absent
         vpc_id: "{{ testing_vpc.vpc.id }}"
-        cidr: 10.22.33.0/24
+        cidr: "{{ subnet_b_cidr }}"
         az: "{{ aws_region }}b"
         resource_tags:
           Name: "{{ efs_name }}-subnet-b"
@@ -614,7 +614,7 @@
     - name: remove the VPC
       ec2_vpc_net:
         name: "{{ efs_name }}-vpc"
-        cidr_block: 10.22.32.0/23
+        cidr_block: "{{ vpc_cidr }}"
         state: absent
       register: removed
       until: removed is not failed

tests/integration/targets/eks_cluster/defaults/main.yml

@@ -1,12 +1,16 @@
 eks_cluster_name: "{{ resource_prefix }}"
 eks_cluster_short_name: "{{ lookup('password', '/dev/null chars=ascii_lowercase length=5') }}"
+
+# VPC CIDR allocation - randomised to avoid conflicts
+vpc_cidr: 10.{{ 256 | random(seed=resource_prefix) }}.224.0/22
+
 eks_subnets:
   - zone: a
-    cidr: 10.0.1.0/24
+    cidr: "{{ vpc_cidr | ansible.utils.ipsubnet(24, 0) }}"
   - zone: b
-    cidr: 10.0.2.0/24
+    cidr: "{{ vpc_cidr | ansible.utils.ipsubnet(24, 1) }}"
   - zone: c
-    cidr: 10.0.3.0/24
+    cidr: "{{ vpc_cidr | ansible.utils.ipsubnet(24, 2) }}"
 
 eks_security_groups:
   - name: "{{ eks_cluster_name }}-control-plane-sg"