Add comments to the disabled/unsupported integration test aliase files (

#411) Add some additional comments so we know *why* the various tests aren't running. Looks like most of them just need policy updates
ansible-collections · Feb 13, 2021 · 0d24559 · 0d24559
1 parent 2534412
commit 0d24559
Show file tree

Hide file tree

Showing 27 changed files with 114 additions and 29 deletions.
diff --git a/tests/integration/targets/aws_config/aliases b/tests/integration/targets/aws_config/aliases
@@ -1,6 +1,10 @@
-cloud/aws
+# reason: missing-policy
+# We don't have CI or 'unsupported' policy for AWS config
 disabled
+
+cloud/aws
 shippable/aws/group2
+
 aws_config_aggregation_authorization
 aws_config_aggregator
 aws_config_delivery_channel

diff --git a/tests/integration/targets/aws_eks_cluster/aliases b/tests/integration/targets/aws_eks_cluster/aliases
@@ -1,2 +1,4 @@
-cloud/aws
+# reason: unknown
 unsupported
+
+cloud/aws
diff --git a/tests/integration/targets/aws_secret/aliases b/tests/integration/targets/aws_secret/aliases
@@ -1,2 +1,4 @@
-cloud/aws
+# reason: missing-policy
 unsupported
+
+cloud/aws
diff --git a/tests/integration/targets/aws_waf_web_acl/aliases b/tests/integration/targets/aws_waf_web_acl/aliases
@@ -1,6 +1,10 @@
+# reason: broken
+# ansible/ansible#38258
+unsupported
+
 cloud/aws
+
 aws_waf_info
 aws_waf_web_acl
 aws_waf_web_match
 aws_waf_web_rule
-unsupported
diff --git a/tests/integration/targets/cloudformation_stack_set/aliases b/tests/integration/targets/cloudformation_stack_set/aliases
@@ -1,2 +1,5 @@
+# reason: broken
+# Tests rely on additional non-standard secondary_aws_ credentials
+disabled
+
 cloud/aws
-unsupported
diff --git a/tests/integration/targets/cloudfront_distribution/aliases b/tests/integration/targets/cloudfront_distribution/aliases
@@ -1,2 +1,4 @@
+# reason: broken
+disabled
+
 cloud/aws
-unsupported
diff --git a/tests/integration/targets/cloudtrail/aliases b/tests/integration/targets/cloudtrail/aliases
@@ -1,2 +1,4 @@
-cloud/aws
+# reason: missing-policy
 unsupported
+
+cloud/aws
diff --git a/tests/integration/targets/connection_aws_ssm/aliases b/tests/integration/targets/connection_aws_ssm/aliases
@@ -1,7 +1,11 @@
+# reason: unstable
+# reason: slow
+disabled
+
 cloud/aws
-destructive
 shippable/aws/group4
+
+destructive
 non_local
 needs/root
 needs/target/connection
-disabled
diff --git a/tests/integration/targets/dms_endpoint/aliases b/tests/integration/targets/dms_endpoint/aliases
@@ -1,2 +1,4 @@
-cloud/aws
+# reason: missing-policy
 unsupported
+
+cloud/aws
diff --git a/tests/integration/targets/dms_replication_subnet_group/aliases b/tests/integration/targets/dms_replication_subnet_group/aliases
@@ -1,2 +1,4 @@
-cloud/aws
+# reason: missing-policy
 unsupported
+
+cloud/aws
diff --git a/tests/integration/targets/ec2_asg/aliases b/tests/integration/targets/ec2_asg/aliases
@@ -1,2 +1,6 @@
-cloud/aws
+# reason: slow
+# reason: broken
+# Tests take around 30 minutes
 unsupported
+
+cloud/aws
diff --git a/tests/integration/targets/ecs_cluster/aliases b/tests/integration/targets/ecs_cluster/aliases
@@ -1,6 +1,10 @@
+# reason: slow
+# Tests take around 15 minutes to run
+unsupported
+
 cloud/aws
+
 ecs_service_info
 ecs_task
 ecs_taskdefinition
 ecs_taskdefinition_info
-unsupported
diff --git a/tests/integration/targets/ecs_tag/aliases b/tests/integration/targets/ecs_tag/aliases
@@ -1,3 +1,6 @@
+# reason: missing-policy
+unsupported
+
 cloud/aws
+
 ecs_tag
-unsupported
diff --git a/tests/integration/targets/efs/aliases b/tests/integration/targets/efs/aliases
@@ -1,3 +1,7 @@
-cloud/aws
+# reason: missing-policy
+# ansible/ansible#36520 - attempts were made to get it running in CI but were unsuccessful
 unsupported
+
+cloud/aws
+
 efs_info
diff --git a/tests/integration/targets/elb_network_lb/aliases b/tests/integration/targets/elb_network_lb/aliases
@@ -1,2 +1,4 @@
-cloud/aws
+# reason:
 unsupported
+
+cloud/aws
diff --git a/tests/integration/targets/elb_target_info/aliases b/tests/integration/targets/elb_target_info/aliases
@@ -1,2 +1,4 @@
-cloud/aws
+# reason: missing-policy
 unsupported
+
+cloud/aws
diff --git a/tests/integration/targets/iam_group/aliases b/tests/integration/targets/iam_group/aliases
@@ -1,2 +1,7 @@
+# reason: missing-policy
+# It should be possible to test iam_groups by limiting which policies can be
+# attached to the groups as well as which users can be added to the groups.
+# Careful review is needed prior to adding this to the main CI.
 unsupported
+
 cloud/aws
diff --git a/tests/integration/targets/iam_password_policy/aliases b/tests/integration/targets/iam_password_policy/aliases
@@ -1,2 +1,6 @@
-cloud/aws
+# reason: missing-policy
+# IAM Password Policies configure account-wide settings, this makes then
+# difficult to safely test
 unsupported
+
+cloud/aws
diff --git a/tests/integration/targets/iam_policy/aliases b/tests/integration/targets/iam_policy/aliases
@@ -1,3 +1,8 @@
-iam_policy_info
-cloud/aws
+# reason: missing-policy
+# It's not possible to control what permissions are granted to a policy.
+# This makes securely testing iam_policy very difficult
 unsupported
+
+cloud/aws
+
+iam_policy_info
diff --git a/tests/integration/targets/iam_role/aliases b/tests/integration/targets/iam_role/aliases
@@ -1,3 +1,9 @@
-iam_role_info
+# reason: missing-policy
+# It should be possible to test iam_role by limiting which policies can be
+# attached to the roles.
+# Careful review is needed prior to adding this to the main CI.
 unsupported
+
 cloud/aws
+
+iam_role_info
diff --git a/tests/integration/targets/iam_saml_federation/aliases b/tests/integration/targets/iam_saml_federation/aliases
@@ -1,2 +1,4 @@
-cloud/aws
+# reason: missing-policy
 unsupported
+
+cloud/aws
diff --git a/tests/integration/targets/iam_user/aliases b/tests/integration/targets/iam_user/aliases
@@ -1,3 +1,9 @@
+# reason: missing-policy
+# It should be possible to test iam_user by limiting which policies can be
+# attached to the users.
+# Careful review is needed prior to adding this to the main CI.
+unsupported
+
 cloud/aws
+
 iam_user_info
-unsupported
diff --git a/tests/integration/targets/rds_instance/aliases b/tests/integration/targets/rds_instance/aliases
@@ -1,2 +1,4 @@
-cloud/aws
+# reason: missing-policy
 unsupported
+
+cloud/aws
diff --git a/tests/integration/targets/s3_lifecycle/aliases b/tests/integration/targets/s3_lifecycle/aliases
@@ -1,3 +1,6 @@
+# reason: broken
+# ansible/ansible#59311
+disabled
+
 cloud/aws
 shippable/aws/group1
-disabled
diff --git a/tests/integration/targets/s3_logging/aliases b/tests/integration/targets/s3_logging/aliases
@@ -1,4 +1,6 @@
+# reason: unstable
+# When running tests a failure rate of around 20% was observed
+unsupported
+
 cloud/aws
 #shippable/aws/group1
-# when running tests we saw an ~20% failure rate
-unsupported
diff --git a/tests/integration/targets/sns/aliases b/tests/integration/targets/sns/aliases
@@ -1,2 +1,4 @@
-cloud/aws
+# reason: missing-policy
 unsupported
+
+cloud/aws
diff --git a/tests/integration/targets/sns_topic/aliases b/tests/integration/targets/sns_topic/aliases
@@ -1,2 +1,4 @@
-cloud/aws
+# reason: missing-policy
 unsupported
+
+cloud/aws