From f1d338d8412ac3aedbab9956ed82a4d28cf34ea4 Mon Sep 17 00:00:00 2001
From: Jill R <4121322+jillr@users.noreply.github.com>
Date: Wed, 24 Mar 2021 13:35:52 -0700
Subject: [PATCH] Stabilize and improve ec2_vpc_endpoint modules (#473)

* Stabilize and improve ec2_vpc_endpoint modules

- Add tagging support
- Make idempotent
- Better exception handling
- Better check_mode support
- Use module_utils for common functions
- Enable retries on common AWS failures

* Make endpoint deletion idempotent in check_mode

* Sanity fixes

* Address review feedback
---
 .../473-ec2_vpc_endpoint_stabilization.yml    |   4 +
 plugins/modules/ec2_vpc_endpoint.py           | 190 ++++++--
 plugins/modules/ec2_vpc_endpoint_info.py      |  22 +-
 .../targets/ec2_vpc_endpoint/tasks/main.yml   | 450 +++++++++++++++++-
 4 files changed, 578 insertions(+), 88 deletions(-)
 create mode 100644 changelogs/fragments/473-ec2_vpc_endpoint_stabilization.yml

diff --git a/changelogs/fragments/473-ec2_vpc_endpoint_stabilization.yml b/changelogs/fragments/473-ec2_vpc_endpoint_stabilization.yml
new file mode 100644
index 00000000000..995517f9903
--- /dev/null
+++ b/changelogs/fragments/473-ec2_vpc_endpoint_stabilization.yml
@@ -0,0 +1,4 @@
+minor_changes:
+- ec2_vpc_endpoint - The module now supports tagging endpoints. (https://github.com/ansible-collections/community.aws/pull/473)
+- ec2_vpc_endpoint - Add retries on common AWS failures. (https://github.com/ansible-collections/community.aws/pull/473)
+- ec2_vpc_endpoint - The module will now lookup existing endpoints and try to match on the provided parameters before creating a new endpoint for better idempotency.  (https://github.com/ansible-collections/community.aws/pull/473)
diff --git a/plugins/modules/ec2_vpc_endpoint.py b/plugins/modules/ec2_vpc_endpoint.py
index 2bfe89008e5..2aa4441fac7 100644
--- a/plugins/modules/ec2_vpc_endpoint.py
+++ b/plugins/modules/ec2_vpc_endpoint.py
@@ -66,6 +66,19 @@
     default: present
     choices: [ "present", "absent" ]
     type: str
+  tags:
+    description:
+      - A dict of tags to apply to the internet gateway.
+      - To remove all tags set I(tags={}) and I(purge_tags=true).
+    type: dict
+    version_added: 1.5.0
+  purge_tags:
+    description:
+      - Delete any tags not specified in the task that are on the instance.
+        This means you have to specify all the desired tags on each task affecting an instance.
+    default: false
+    type: bool
+    version_added: 1.5.0
   wait:
     description:
       - When specified, will wait for either available status for state present.
@@ -200,58 +213,112 @@
 from ansible.module_utils.common.dict_transformations import camel_dict_to_snake_dict
 
 from ansible_collections.amazon.aws.plugins.module_utils.core import AnsibleAWSModule
+from ansible_collections.amazon.aws.plugins.module_utils.core import normalize_boto3_result
+from ansible_collections.amazon.aws.plugins.module_utils.ec2 import AWSRetry
+from ansible_collections.amazon.aws.plugins.module_utils.ec2 import ansible_dict_to_boto3_filter_list
+from ansible_collections.amazon.aws.plugins.module_utils.ec2 import ansible_dict_to_boto3_tag_list
+from ansible_collections.amazon.aws.plugins.module_utils.ec2 import boto3_tag_list_to_ansible_dict
+from ansible_collections.amazon.aws.plugins.module_utils.ec2 import compare_aws_tags
+
 from ansible_collections.amazon.aws.plugins.module_utils.core import is_boto3_error_code
+from ansible_collections.amazon.aws.plugins.module_utils.waiters import get_waiter
 
 
-def date_handler(obj):
-    return obj.isoformat() if hasattr(obj, 'isoformat') else obj
+def get_endpoints(client, module, endpoint_id=None):
+    params = dict()
+    if endpoint_id:
+        params['VpcEndpointIds'] = [endpoint_id]
+    else:
+        filters = list()
+        if module.params.get('service'):
+            filters.append({'Name': 'service-name', 'Values': [module.params.get('service')]})
+        if module.params.get('vpc_id'):
+            filters.append({'Name': 'vpc-id', 'Values': [module.params.get('vpc_id')]})
+        params['Filters'] = filters
+    try:
+        result = client.describe_vpc_endpoints(aws_retry=True, **params)
+    except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e:
+        module.fail_json_aws(e, msg="Failed to get endpoints")
 
+    # normalize iso datetime fields in result
+    normalized_result = normalize_boto3_result(result)
+    return normalized_result
 
-def wait_for_status(client, module, resource_id, status):
-    polling_increment_secs = 15
-    max_retries = (module.params.get('wait_timeout') // polling_increment_secs)
-    status_achieved = False
 
-    for x in range(0, max_retries):
-        try:
-            resource = get_endpoints(client, module, resource_id)['VpcEndpoints'][0]
-            if resource['State'] == status:
-                status_achieved = True
-                break
-            else:
-                time.sleep(polling_increment_secs)
-        except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
-            module.fail_json_aws(e, msg='Failure while waiting for status')
+def match_endpoints(route_table_ids, service_name, vpc_id, endpoint):
+    found = False
+    sorted_route_table_ids = []
 
-    return status_achieved, resource
+    if route_table_ids:
+        sorted_route_table_ids = sorted(route_table_ids)
 
+    if endpoint['VpcId'] == vpc_id and endpoint['ServiceName'] == service_name:
+        sorted_endpoint_rt_ids = sorted(endpoint['RouteTableIds'])
+        if sorted_endpoint_rt_ids == sorted_route_table_ids:
 
-def get_endpoints(client, module, resource_id=None):
-    params = dict()
-    if resource_id:
-        params['VpcEndpointIds'] = [resource_id]
+            found = True
+    return found
 
-    result = json.loads(json.dumps(client.describe_vpc_endpoints(**params), default=date_handler))
-    return result
+
+def ensure_tags(client, module, vpc_endpoint_id):
+    changed = False
+    tags = module.params['tags']
+    purge_tags = module.params['purge_tags']
+
+    filters = ansible_dict_to_boto3_filter_list({'resource-id': vpc_endpoint_id})
+    try:
+        current_tags = client.describe_tags(aws_retry=True, Filters=filters)
+    except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e:
+        module.fail_json_aws(e, msg="Failed to describe tags for VPC Endpoint: {0}".format(vpc_endpoint_id))
+
+    tags_to_set, tags_to_unset = compare_aws_tags(boto3_tag_list_to_ansible_dict(current_tags.get('Tags')), tags, purge_tags=purge_tags)
+    if purge_tags and not tags:
+        tags_to_unset = current_tags
+
+    if tags_to_unset:
+        changed = True
+        if not module.check_mode:
+            try:
+                client.delete_tags(aws_retry=True, Resources=[vpc_endpoint_id], Tags=[dict(Key=tagkey) for tagkey in tags_to_unset])
+            except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e:
+                module.fail_json_aws(e, msg="Unable to delete tags {0}".format(tags_to_unset))
+
+    if tags_to_set:
+        changed = True
+        if not module.check_mode:
+            try:
+                client.create_tags(aws_retry=True, Resources=[vpc_endpoint_id], Tags=ansible_dict_to_boto3_tag_list(tags_to_set))
+            except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e:
+                module.fail_json_aws(e, msg="Unable to add tags {0}".format(tags_to_set))
+    return changed
 
 
 def setup_creation(client, module):
-    vpc_id = module.params.get('vpc_id')
+    endpoint_id = module.params.get('vpc_endpoint_id')
+    route_table_ids = module.params.get('route_table_ids')
     service_name = module.params.get('service')
+    vpc_id = module.params.get('vpc_id')
+    changed = False
 
-    if module.params.get('route_table_ids'):
-        route_table_ids = module.params.get('route_table_ids')
-        existing_endpoints = get_endpoints(client, module)
-        for endpoint in existing_endpoints['VpcEndpoints']:
-            if endpoint['VpcId'] == vpc_id and endpoint['ServiceName'] == service_name:
-                sorted_endpoint_rt_ids = sorted(endpoint['RouteTableIds'])
-                sorted_route_table_ids = sorted(route_table_ids)
-                if sorted_endpoint_rt_ids == sorted_route_table_ids:
-                    return False, camel_dict_to_snake_dict(endpoint)
+    if not endpoint_id:
+        # Try to use the module parameters to match any existing endpoints
+        all_endpoints = get_endpoints(client, module, endpoint_id)
+        if len(all_endpoints['VpcEndpoints']) > 0:
+            for endpoint in all_endpoints['VpcEndpoints']:
+                if match_endpoints(route_table_ids, service_name, vpc_id, endpoint):
+                    endpoint_id = endpoint['VpcEndpointId']
+                    break
+
+    if endpoint_id:
+        # If we have an endpoint now, just ensure tags and exit
+        if module.params.get('tags'):
+            changed = ensure_tags(client, module, endpoint_id)
+        normalized_result = get_endpoints(client, module, endpoint_id=endpoint_id)['VpcEndpoints'][0]
+        return changed, camel_dict_to_snake_dict(normalized_result, ignore_list=['Tags'])
 
     changed, result = create_vpc_endpoint(client, module)
 
-    return changed, json.loads(json.dumps(result, default=date_handler))
+    return changed, camel_dict_to_snake_dict(result, ignore_list=['Tags'])
 
 
 def create_vpc_endpoint(client, module):
@@ -261,7 +328,11 @@ def create_vpc_endpoint(client, module):
     params['VpcId'] = module.params.get('vpc_id')
     params['VpcEndpointType'] = module.params.get('vpc_endpoint_type')
     params['ServiceName'] = module.params.get('service')
-    params['DryRun'] = module.check_mode
+
+    if module.check_mode:
+        changed = True
+        result = 'Would have created VPC Endpoint if not in check mode'
+        module.exit_json(changed=changed, result=result)
 
     if module.params.get('route_table_ids'):
         params['RouteTableIds'] = module.params.get('route_table_ids')
@@ -292,16 +363,18 @@ def create_vpc_endpoint(client, module):
 
     try:
         changed = True
-        result = camel_dict_to_snake_dict(client.create_vpc_endpoint(**params)['VpcEndpoint'])
+        result = client.create_vpc_endpoint(aws_retry=True, **params)['VpcEndpoint']
         if token_provided and (request_time > result['creation_timestamp'].replace(tzinfo=None)):
             changed = False
         elif module.params.get('wait') and not module.check_mode:
-            status_achieved, result = wait_for_status(client, module, result['vpc_endpoint_id'], 'available')
-            if not status_achieved:
-                module.fail_json(msg='Error waiting for vpc endpoint to become available - please check the AWS console')
-    except is_boto3_error_code('DryRunOperation'):
-        changed = True
-        result = 'Would have created VPC Endpoint if not in check mode'
+            try:
+                waiter = get_waiter(client, 'vpc_endpoint_exists')
+                waiter.wait(VpcEndpointIds=[result['VpcEndpointId']], WaiterConfig=dict(Delay=15, MaxAttempts=module.params.get('wait_timeout') // 15))
+            except botocore.exceptions.WaiterError as e:
+                module.fail_json_aws(msg='Error waiting for vpc endpoint to become available - please check the AWS console')
+            except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:  # pylint: disable=duplicate-except
+                module.fail_json_aws(e, msg='Failure while waiting for status')
+
     except is_boto3_error_code('IdempotentParameterMismatch'):  # pylint: disable=duplicate-except
         module.fail_json(msg="IdempotentParameterMismatch - updates of endpoints are not allowed by the API")
     except is_boto3_error_code('RouteAlreadyExists'):  # pylint: disable=duplicate-except
@@ -309,19 +382,38 @@ def create_vpc_endpoint(client, module):
     except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:  # pylint: disable=duplicate-except
         module.fail_json_aws(e, msg="Failed to create VPC.")
 
-    return changed, result
+    if module.params.get('tags'):
+        ensure_tags(client, module, result['VpcEndpointId'])
+
+    # describe and normalize iso datetime fields in result after adding tags
+    normalized_result = get_endpoints(client, module, endpoint_id=result['VpcEndpointId'])['VpcEndpoints'][0]
+    return changed, normalized_result
 
 
 def setup_removal(client, module):
     params = dict()
     changed = False
-    params['DryRun'] = module.check_mode
+
+    if module.check_mode:
+        try:
+            exists = client.describe_vpc_endpoints(aws_retry=True, VpcEndpointIds=[module.params.get('vpc_endpoint_id')])
+            if exists:
+                result = {'msg': 'Would have deleted VPC Endpoint if not in check mode'}
+                changed = True
+        except is_boto3_error_code('InvalidVpcEndpointId.NotFound'):
+            result = {'msg': 'Endpoint does not exist, nothing to delete.'}
+            changed = False
+        except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e:  # pylint: disable=duplicate-except
+            module.fail_json_aws(e, msg="Failed to get endpoints")
+
+        return changed, result
+
     if isinstance(module.params.get('vpc_endpoint_id'), string_types):
         params['VpcEndpointIds'] = [module.params.get('vpc_endpoint_id')]
     else:
         params['VpcEndpointIds'] = module.params.get('vpc_endpoint_id')
     try:
-        result = client.delete_vpc_endpoints(**params)['Unsuccessful']
+        result = client.delete_vpc_endpoints(aws_retry=True, **params)['Unsuccessful']
         if len(result) < len(params['VpcEndpointIds']):
             changed = True
         # For some reason delete_vpc_endpoints doesn't throw exceptions it
@@ -332,9 +424,7 @@ def setup_removal(client, module):
                 raise botocore.exceptions.ClientError(r, 'delete_vpc_endpoints')
             except is_boto3_error_code('InvalidVpcEndpoint.NotFound'):
                 continue
-    except is_boto3_error_code('DryRunOperation'):
-        changed = True
-        result = 'Would have deleted VPC Endpoint if not in check mode'
+
     except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:  # pylint: disable=duplicate-except
         module.fail_json_aws(e, "Failed to delete VPC endpoint")
     return changed, result
@@ -353,6 +443,8 @@ def main():
         route_table_ids=dict(type='list', elements='str'),
         vpc_endpoint_id=dict(),
         client_token=dict(no_log=False),
+        tags=dict(type='dict'),
+        purge_tags=dict(type='bool', default=False),
     )
     module = AnsibleAWSModule(
         argument_spec=argument_spec,
@@ -373,7 +465,7 @@ def main():
                          date='2022-12-01', collection_name='community.aws')
 
     try:
-        ec2 = module.client('ec2')
+        ec2 = module.client('ec2', retry_decorator=AWSRetry.jittered_backoff())
     except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
         module.fail_json_aws(e, msg='Failed to connect to AWS')
 
diff --git a/plugins/modules/ec2_vpc_endpoint_info.py b/plugins/modules/ec2_vpc_endpoint_info.py
index 7706a00c915..425e0c63ec7 100644
--- a/plugins/modules/ec2_vpc_endpoint_info.py
+++ b/plugins/modules/ec2_vpc_endpoint_info.py
@@ -120,14 +120,12 @@
 from ansible.module_utils.common.dict_transformations import camel_dict_to_snake_dict
 
 from ansible_collections.amazon.aws.plugins.module_utils.core import AnsibleAWSModule
+from ansible_collections.amazon.aws.plugins.module_utils.core import is_boto3_error_code
+from ansible_collections.amazon.aws.plugins.module_utils.core import normalize_boto3_result
 from ansible_collections.amazon.aws.plugins.module_utils.ec2 import AWSRetry
 from ansible_collections.amazon.aws.plugins.module_utils.ec2 import ansible_dict_to_boto3_filter_list
 
 
-def date_handler(obj):
-    return obj.isoformat() if hasattr(obj, 'isoformat') else obj
-
-
 @AWSRetry.exponential_backoff()
 def get_supported_services(client, module):
     results = list()
@@ -149,16 +147,14 @@ def get_endpoints(client, module):
     params['Filters'] = ansible_dict_to_boto3_filter_list(module.params.get('filters'))
     if module.params.get('vpc_endpoint_ids'):
         params['VpcEndpointIds'] = module.params.get('vpc_endpoint_ids')
-    while True:
-        response = client.describe_vpc_endpoints(**params)
-        results.extend(response['VpcEndpoints'])
-        if 'NextToken' in response:
-            params['NextToken'] = response['NextToken']
-        else:
-            break
     try:
-        results = json.loads(json.dumps(results, default=date_handler))
-    except Exception as e:
+        paginator = client.get_paginator('describe_vpc_endpoints')
+        results = paginator.paginate(**params).build_full_result()['VpcEndpoints']
+
+        results = normalize_boto3_result(results)
+    except is_boto3_error_code('InvalidVpcEndpointId.NotFound'):
+        module.exit_json(msg='VpcEndpoint {0} does not exist'.format(module.params.get('vpc_endpoint_ids')), vpc_endpoints=[])
+    except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e:  # pylint: disable=duplicate-except
         module.fail_json_aws(e, msg="Failed to get endpoints")
     return dict(vpc_endpoints=[camel_dict_to_snake_dict(result) for result in results])
 
diff --git a/tests/integration/targets/ec2_vpc_endpoint/tasks/main.yml b/tests/integration/targets/ec2_vpc_endpoint/tasks/main.yml
index a8fdbec889d..daa2a103624 100644
--- a/tests/integration/targets/ec2_vpc_endpoint/tasks/main.yml
+++ b/tests/integration/targets/ec2_vpc_endpoint/tasks/main.yml
@@ -137,6 +137,7 @@
       state: present
       vpc_id: '{{ vpc_id }}'
       service: '{{ endpoint_service_a }}'
+      wait: true
     register: create_endpoint
   - name: Check standard return values
     assert:
@@ -183,6 +184,7 @@
         - '"creation_timestamp" in first_endpoint'
         - '"policy_document" in first_endpoint'
         - '"route_table_ids" in first_endpoint'
+        - first_endpoint.route_table_ids | length == 0
         - '"service_name" in first_endpoint'
         - '"state" in first_endpoint'
         - '"vpc_endpoint_id" in first_endpoint'
@@ -215,6 +217,7 @@
         - '"creation_timestamp" in first_endpoint'
         - '"policy_document" in first_endpoint'
         - '"route_table_ids" in first_endpoint'
+        - first_endpoint.route_table_ids | length == 0
         - '"service_name" in first_endpoint'
         - first_endpoint.service_name == endpoint_service_a
         - '"state" in first_endpoint'
@@ -276,25 +279,408 @@
     vars:
       first_endpoint: '{{ endpoint_info.vpc_endpoints[0] }}'
 
-# ec2_vpc_endpoint is not idempotent without explicitly passing the endpoint ID
-#  - name: Create minimal endpoint - idempotency (check mode)
+
+  # matches on parameters without explicitly passing the endpoint ID
+  - name: Create minimal endpoint - idempotency (check mode)
+    ec2_vpc_endpoint:
+      state: present
+      vpc_id: '{{ vpc_id }}'
+      service: '{{ endpoint_service_a }}'
+    register: create_endpoint_idem_check
+    check_mode: True
+  - assert:
+      that:
+        - create_endpoint_idem_check is not changed
+
+  - name: Create minimal endpoint - idempotency
+    ec2_vpc_endpoint:
+      state: present
+      vpc_id: '{{ vpc_id }}'
+      service: '{{ endpoint_service_a }}'
+    register: create_endpoint_idem
+  - assert:
+      that:
+        - create_endpoint_idem is not changed
+
+  - name: Delete minimal endpoint by ID (check_mode)
+    ec2_vpc_endpoint:
+      state: absent
+      vpc_endpoint_id: "{{ endpoint_id }}"
+    check_mode: true
+    register: endpoint_delete_check
+  - assert:
+      that:
+        - endpoint_delete_check is changed
+
+
+  - name: Delete minimal endpoint by ID
+    ec2_vpc_endpoint:
+      state: absent
+      vpc_endpoint_id: "{{ endpoint_id }}"
+    register: endpoint_delete_check
+  - assert:
+      that:
+        - endpoint_delete_check is changed
+
+  - name: Delete minimal endpoint by ID - idempotency (check_mode)
+    ec2_vpc_endpoint:
+      state: absent
+      vpc_endpoint_id: "{{ endpoint_id }}"
+    check_mode: true
+    register: endpoint_delete_check
+  - assert:
+      that:
+        - endpoint_delete_check is not changed
+
+  - name: Delete minimal endpoint by ID - idempotency
+    ec2_vpc_endpoint:
+      state: absent
+      vpc_endpoint_id: "{{ endpoint_id }}"
+    register: endpoint_delete_check
+  - assert:
+      that:
+        - endpoint_delete_check is not changed
+
+  - name: Fetch Endpoints by ID (expect failed)
+    ec2_vpc_endpoint_info:
+      query: endpoints
+      vpc_endpoint_ids: "{{ endpoint_id }}"
+    ignore_errors: True
+    register: endpoint_info
+  - name: Assert endpoint does not exist
+    assert:
+      that:
+        - endpoint_info is successful
+        - '"does not exist" in endpoint_info.msg'
+        - endpoint_info.vpc_endpoints | length == 0
+
+  # Attempt to create an endpoint with a route table
+  - name: Create an endpoint with route table (check mode)
+    ec2_vpc_endpoint:
+      state: present
+      vpc_id: '{{ vpc_id }}'
+      service: '{{ endpoint_service_a }}'
+      route_table_ids:
+        - '{{ rtb_empty_id }}'
+    register: create_endpoint_check
+    check_mode: True
+  - name: Assert changed
+    assert:
+      that:
+        - create_endpoint_check is changed
+
+  - name: Create an endpoint with route table
+    ec2_vpc_endpoint:
+      state: present
+      vpc_id: '{{ vpc_id }}'
+      service: '{{ endpoint_service_a }}'
+      route_table_ids:
+        - '{{ rtb_empty_id }}'
+      wait: true
+    register: create_rtb_endpoint
+  - name: Check standard return values
+    assert:
+      that:
+        - create_rtb_endpoint is changed
+        - '"result" in create_rtb_endpoint'
+        - '"creation_timestamp" in create_rtb_endpoint.result'
+        - '"dns_entries" in create_rtb_endpoint.result'
+        - '"groups" in create_rtb_endpoint.result'
+        - '"network_interface_ids" in create_rtb_endpoint.result'
+        - '"owner_id" in create_rtb_endpoint.result'
+        - '"policy_document" in create_rtb_endpoint.result'
+        - '"private_dns_enabled" in create_rtb_endpoint.result'
+        - '"route_table_ids" in create_rtb_endpoint.result'
+        - create_rtb_endpoint.result.route_table_ids | length == 1
+        - create_rtb_endpoint.result.route_table_ids[0] == '{{ rtb_empty_id }}'
+        - create_rtb_endpoint.result.private_dns_enabled == False
+        - '"requester_managed" in create_rtb_endpoint.result'
+        - create_rtb_endpoint.result.requester_managed == False
+        - '"service_name" in create_rtb_endpoint.result'
+        - create_rtb_endpoint.result.service_name == endpoint_service_a
+        - '"state" in create_endpoint.result'
+        - create_rtb_endpoint.result.state == "available"
+        - '"vpc_endpoint_id" in create_rtb_endpoint.result'
+        - create_rtb_endpoint.result.vpc_endpoint_id.startswith("vpce-")
+        - '"vpc_endpoint_type" in create_rtb_endpoint.result'
+        - create_rtb_endpoint.result.vpc_endpoint_type == "Gateway"
+        - '"vpc_id" in create_rtb_endpoint.result'
+        - create_rtb_endpoint.result.vpc_id == vpc_id
+
+  - name: Save Endpoint info in a fact
+    set_fact:
+      rtb_endpoint_id: '{{ create_rtb_endpoint.result.vpc_endpoint_id }}'
+
+  - name: Create an endpoint with route table - idempotency (check mode)
+    ec2_vpc_endpoint:
+      state: present
+      vpc_id: '{{ vpc_id }}'
+      service: '{{ endpoint_service_a }}'
+      route_table_ids:
+        - '{{ rtb_empty_id }}'
+    register: create_endpoint_check
+    check_mode: True
+  - name: Assert changed
+    assert:
+      that:
+        - create_endpoint_check is not changed
+
+  - name: Create an endpoint with route table - idempotency
+    ec2_vpc_endpoint:
+      state: present
+      vpc_id: '{{ vpc_id }}'
+      service: '{{ endpoint_service_a }}'
+      route_table_ids:
+        - '{{ rtb_empty_id }}'
+    register: create_endpoint_check
+    check_mode: True
+  - name: Assert changed
+    assert:
+      that:
+        - create_endpoint_check is not changed
+
+# # Endpoint modifications are not yet supported by the module
+#  # A Change the route table for the endpoint
+#  - name: Change the route table for the endpoint (check_mode)
 #    ec2_vpc_endpoint:
 #      state: present
 #      vpc_id: '{{ vpc_id }}'
+#      vpc_endpoint_id: "{{ rtb_endpoint_id }}"
 #      service: '{{ endpoint_service_a }}'
-#    register: create_endpoint_idem_check
+#      route_table_ids:
+#        - '{{ rtb_igw_id }}'
 #    check_mode: True
+#    register: check_two_rtbs_endpoint
+#
+#  - name: Assert second route table would be added
+#    assert:
+#      that:
+#        - check_two_rtbs_endpoint.changed
+#
+#  - name: Change the route table for the endpoint
+#    ec2_vpc_endpoint:
+#      state: present
+#      vpc_id: '{{ vpc_id }}'
+#      vpc_endpoint_id: "{{ rtb_endpoint_id }}"
+#      service: '{{ endpoint_service_a }}'
+#      route_table_ids:
+#        - '{{ rtb_igw_id }}'
+#    register: two_rtbs_endpoint
 #
-#  - name: Create minimal endpoint - idempotency
+#  - name: Assert second route table would be added
+#    assert:
+#      that:
+#        - check_two_rtbs_endpoint.changed
+#        - two_rtbs_endpoint.result.route_table_ids | length == 1
+#        - two_rtbs_endpoint.result.route_table_ids[0] == '{{ rtb_igw_id }}'
+#
+#  - name: Change the route table for the endpoint - idempotency (check_mode)
 #    ec2_vpc_endpoint:
 #      state: present
 #      vpc_id: '{{ vpc_id }}'
+#      vpc_endpoint_id: "{{ rtb_endpoint_id }}"
 #      service: '{{ endpoint_service_a }}'
-#    register: create_endpoint_idem
+#      route_table_ids:
+#        - '{{ rtb_igw_id }}'
+#    check_mode: True
+#    register: check_two_rtbs_endpoint
+#
+#  - name: Assert route table would not change
+#    assert:
+#      that:
+#        - not check_two_rtbs_endpoint.changed
+#
+#  - name: Change the route table for the endpoint - idempotency
+#    ec2_vpc_endpoint:
+#      state: present
+#      vpc_id: '{{ vpc_id }}'
+#      vpc_endpoint_id: "{{ rtb_endpoint_id }}"
+#      service: '{{ endpoint_service_a }}'
+#      route_table_ids:
+#        - '{{ rtb_igw_id }}'
+#    register: two_rtbs_endpoint
+#
+#  - name: Assert route table would not change
+#    assert:
+#      that:
+#        - not check_two_rtbs_endpoint.changed
+
+  - name: Tag the endpoint (check_mode)
+    ec2_vpc_endpoint:
+      state: present
+      vpc_id: '{{ vpc_id }}'
+      vpc_endpoint_id: "{{ rtb_endpoint_id }}"
+      service: '{{ endpoint_service_a }}'
+      route_table_ids:
+        - '{{ rtb_empty_id }}'
+      tags:
+        camelCase: "helloWorld"
+        PascalCase: "HelloWorld"
+        snake_case: "hello_world"
+        "Title Case": "Hello World"
+        "lowercase spaced": "hello world"
+    check_mode: true
+    register: check_tag_vpc_endpoint
+
+  - name: Assert tags would have changed
+    assert:
+      that:
+        - check_tag_vpc_endpoint.changed
+
+  - name: Tag the endpoint
+    ec2_vpc_endpoint:
+      state: present
+      vpc_id: '{{ vpc_id }}'
+      vpc_endpoint_id: "{{ rtb_endpoint_id }}"
+      service: '{{ endpoint_service_a }}'
+      route_table_ids:
+        - '{{ rtb_igw_id }}'
+      tags:
+        camelCase: "helloWorld"
+        PascalCase: "HelloWorld"
+        snake_case: "hello_world"
+        "Title Case": "Hello World"
+        "lowercase spaced": "hello world"
+    register: tag_vpc_endpoint
+
+  - name: Assert tags are successful
+    assert:
+      that:
+        - tag_vpc_endpoint.changed
+        - tag_vpc_endpoint.result.tags | length == 5
+        - endpoint_tags["camelCase"] == "helloWorld"
+        - endpoint_tags["PascalCase"] == "HelloWorld"
+        - endpoint_tags["snake_case"] == "hello_world"
+        - endpoint_tags["Title Case"] == "Hello World"
+        - endpoint_tags["lowercase spaced"] == "hello world"
+    vars:
+      endpoint_tags: "{{ tag_vpc_endpoint.result.tags | items2dict(key_name='Key', value_name='Value') }}"
+
+  - name: Query by tag
+    ec2_vpc_endpoint_info:
+      query: endpoints
+      filters:
+        "tag:camelCase":
+        - "helloWorld"
+    register: tag_result
+
+  - name: Assert tag lookup found endpoint
+    assert:
+      that:
+        - tag_result is successful
+        - '"vpc_endpoints" in tag_result'
+        - first_endpoint.vpc_endpoint_id == rtb_endpoint_id
+    vars:
+      first_endpoint: '{{ tag_result.vpc_endpoints[0] }}'
+
+  - name: Tag the endpoint - idempotency (check_mode)
+    ec2_vpc_endpoint:
+      state: present
+      vpc_id: '{{ vpc_id }}'
+      vpc_endpoint_id: "{{ rtb_endpoint_id }}"
+      service: '{{ endpoint_service_a }}'
+      route_table_ids:
+        - '{{ rtb_igw_id }}'
+      tags:
+        camelCase: "helloWorld"
+        PascalCase: "HelloWorld"
+        snake_case: "hello_world"
+        "Title Case": "Hello World"
+        "lowercase spaced": "hello world"
+    register: tag_vpc_endpoint_again
+
+  - name: Assert tags would not change
+    assert:
+      that:
+        - not tag_vpc_endpoint_again.changed
+
+  - name: Tag the endpoint - idempotency
+    ec2_vpc_endpoint:
+      state: present
+      vpc_id: '{{ vpc_id }}'
+      vpc_endpoint_id: "{{ rtb_endpoint_id }}"
+      service: '{{ endpoint_service_a }}'
+      route_table_ids:
+        - '{{ rtb_igw_id }}'
+      tags:
+        camelCase: "helloWorld"
+        PascalCase: "HelloWorld"
+        snake_case: "hello_world"
+        "Title Case": "Hello World"
+        "lowercase spaced": "hello world"
+    register: tag_vpc_endpoint_again
+
+  - name: Assert tags would not change
+    assert:
+      that:
+        - not tag_vpc_endpoint_again.changed
+
+  - name: Add a tag (check_mode)
+    ec2_vpc_endpoint:
+      state: present
+      vpc_id: '{{ vpc_id }}'
+      vpc_endpoint_id: "{{ rtb_endpoint_id }}"
+      service: '{{ endpoint_service_a }}'
+      route_table_ids:
+        - '{{ rtb_igw_id }}'
+      tags:
+        new_tag: "ANewTag"
+    check_mode: true
+    register: check_tag_vpc_endpoint
+
+  - name: Assert tags would have changed
+    assert:
+      that:
+        - check_tag_vpc_endpoint.changed
+
+  - name: Add a tag (purge_tags=False)
+    ec2_vpc_endpoint:
+      state: present
+      vpc_id: '{{ vpc_id }}'
+      vpc_endpoint_id: "{{ rtb_endpoint_id }}"
+      service: '{{ endpoint_service_a }}'
+      route_table_ids:
+        - '{{ rtb_igw_id }}'
+      tags:
+        new_tag: "ANewTag"
+    register: add_tag_vpc_endpoint
+
+  - name: Assert tags changed
+    assert:
+      that:
+        - add_tag_vpc_endpoint.changed
+        - add_tag_vpc_endpoint.result.tags | length == 6
+        - endpoint_tags["camelCase"] == "helloWorld"
+        - endpoint_tags["PascalCase"] == "HelloWorld"
+        - endpoint_tags["snake_case"] == "hello_world"
+        - endpoint_tags["Title Case"] == "Hello World"
+        - endpoint_tags["lowercase spaced"] == "hello world"
+        - endpoint_tags["new_tag"] == "ANewTag"
+    vars:
+      endpoint_tags: "{{ add_tag_vpc_endpoint.result.tags | items2dict(key_name='Key', value_name='Value') }}"
+
+  - name: Add a tag (purge_tags=True)
+    ec2_vpc_endpoint:
+      state: present
+      vpc_id: '{{ vpc_id }}'
+      vpc_endpoint_id: "{{ rtb_endpoint_id }}"
+      service: '{{ endpoint_service_a }}'
+      route_table_ids:
+        - '{{ rtb_igw_id }}'
+      tags:
+        another_new_tag: "AnotherNewTag"
+      purge_tags: True
+    register: purge_tag_vpc_endpoint
+
+  - name: Assert tags changed
+    assert:
+      that:
+        - purge_tag_vpc_endpoint.changed
+        - purge_tag_vpc_endpoint.result.tags | length == 1
+        - endpoint_tags["another_new_tag"] == "AnotherNewTag"
+    vars:
+      endpoint_tags: "{{ purge_tag_vpc_endpoint.result.tags | items2dict(key_name='Key', value_name='Value') }}"
 
-  # Deletion
-  # Delete the routes first - you can't delete an endpoint with a route
-  # attached.
   - name: Delete minimal route table (no routes)
     ec2_vpc_route_table:
       state: absent
@@ -314,35 +700,24 @@
       that:
         - rtb_delete is changed
 
-  - name: Delete endpoint by ID (check_mode)
+  - name: Delete route table endpoint by ID
     ec2_vpc_endpoint:
       state: absent
-      vpc_endpoint_id: "{{ endpoint_id }}"
-    check_mode: true
+      vpc_endpoint_id: "{{ rtb_endpoint_id }}"
     register: endpoint_delete_check
   - assert:
       that:
         - endpoint_delete_check is changed
 
-  - name: Delete endpoint by ID
+  - name: Delete minimal endpoint by ID - idempotency (check_mode)
     ec2_vpc_endpoint:
       state: absent
-      vpc_endpoint_id: "{{ endpoint_id }}"
+      vpc_endpoint_id: "{{ rtb_endpoint_id }}"
+    check_mode: true
     register: endpoint_delete_check
   - assert:
       that:
-        - endpoint_delete_check is changed
-
-# XXX Bug: uses AWS's check mode which only checks permissions
-#  - name: Delete endpoint by ID - idempotency (check_mode)
-#    ec2_vpc_endpoint:
-#      state: absent
-#      vpc_endpoint_id: "{{ endpoint_id }}"
-#    check_mode: true
-#    register: endpoint_delete_check
-#  - assert:
-#      that:
-#        - endpoint_delete_check is not changed
+        - endpoint_delete_check is not changed
 
   - name: Delete endpoint by ID - idempotency
     ec2_vpc_endpoint:
@@ -377,6 +752,8 @@
   # ============================================================
   # BEGIN POST-TEST CLEANUP
   always:
+  # Delete the routes first - you can't delete an endpoint with a route
+  # attached.
   - name: Delete minimal route table (no routes)
     ec2_vpc_route_table:
       state: absent
@@ -397,10 +774,31 @@
       vpc_endpoint_id: "{{ create_endpoint.result.vpc_endpoint_id }}"
     ignore_errors: True
 
+  - name: Delete endpoint
+    ec2_vpc_endpoint:
+      state: absent
+      vpc_endpoint_id: "{{ create_rtb_endpoint.result.vpc_endpoint_id }}"
+    ignore_errors: True
+
+  - name: Query any remain endpoints we created (idempotency work is ongoing)  # FIXME
+    ec2_vpc_endpoint_info:
+      query: endpoints
+      filters:
+        vpc-id:
+          - '{{ vpc_id }}'
+    register: test_endpoints
+
+  - name: Delete all endpoints
+    ec2_vpc_endpoint:
+      state: absent
+      vpc_endpoint_id: '{{ item.vpc_endpoint_id }}'
+    with_items: '{{ test_endpoints.vpc_endpoints }}'
+    ignore_errors: True
+
   - name: Remove IGW
     ec2_vpc_igw:
       state: absent
-      vpc_id: "{{ vpc_creation.vpc.id }}"
+      vpc_id: "{{ vpc_id }}"
     register: igw_deletion
     retries: 10
     delay: 5