Add support for subscriptions attributes parameter - currently only f…

…or RawMessageDelievery (SQS) (#640) (#1302)

[PR #640/932263d4 backport][stable-4] Add support for SQS RawMessageDelievery attribute in subscriptions - issue #193

This is a backport of PR #640 as merged into main (932263d).
SUMMARY

Added support to configure RawMessageDelievery option when configuring sqs endpoints.
It use boto3 set_subscription_attributes() to configure changes.
It currently supports only this option, but should be easily extended in the future.
Attributes are expected in the form as in the boto3 docs:
https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sns.html#SNS.Client.set_subscription_attributes
Fixes #193

ISSUE TYPE


Feature Pull Request

COMPONENT NAME

updates sns_topic.py to support new functionality

example:

- sns_topic:
    name: '{{ sns_topic_topic_name }}'
    display_name: My new topic name
    subscriptions:
    - endpoint: "{{ sqs_arn }}"
      protocol: sqs
      attributes:
        RawMessageDelivery: true

Reviewed-by: Mark Chappell <None>

changelogs/fragments/640-sns_topic-sub_attr.yml

@@ -0,0 +1,2 @@
+minor_changes:
+  - sns_topic - Added ``attributes`` parameter to ``subscriptions`` items with support for RawMessageDelievery (SQS)

plugins/modules/sns_topic.py

@@ -133,6 +133,10 @@
       protocol:
         description: Protocol of subscription.
         required: true
+      attributes:
+        description: Attributes of subscription. Only supports RawMessageDelievery for SQS endpoints.
+        default: {}
+        version_added: "4.1.0"
     type: list
     elements: dict
     default: []
@@ -358,6 +362,8 @@ def __init__(self,
         self.subscriptions_existing = []
         self.subscriptions_deleted = []
         self.subscriptions_added = []
+        self.subscriptions_attributes_set = []
+        self.desired_subscription_attributes = dict()
         self.purge_subscriptions = purge_subscriptions
         self.check_mode = check_mode
         self.topic_created = False
@@ -455,6 +461,45 @@ def _set_topic_subs(self):
                     self.module.fail_json_aws(e, msg="Couldn't subscribe to topic %s" % self.topic_arn)
         return changed
 
+    def _init_desired_subscription_attributes(self):
+        for sub in self.subscriptions:
+            sub_key = (sub['protocol'], canonicalize_endpoint(sub['protocol'], sub['endpoint']))
+            tmp_dict = sub.get('attributes', {})
+            # aws sdk expects values to be strings
+            # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sns.html#SNS.Client.set_subscription_attributes
+            for k, v in tmp_dict.items():
+                tmp_dict[k] = str(v)
+
+            self.desired_subscription_attributes[sub_key] = tmp_dict
+
+    def _set_topic_subs_attributes(self):
+        changed = False
+        for sub in list_topic_subscriptions(self.connection, self.module, self.topic_arn):
+            sub_key = (sub['Protocol'], sub['Endpoint'])
+            sub_arn = sub['SubscriptionArn']
+            if sub_key not in self.desired_subscription_attributes:
+                # subscription isn't defined in desired, skipping
+                continue
+
+            try:
+                sub_current_attributes = self.connection.get_subscription_attributes(SubscriptionArn=sub_arn)['Attributes']
+            except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
+                self.module.fail_json_aws(e, "Couldn't get subscription attributes for subscription %s" % sub_arn)
+
+            raw_message = self.desired_subscription_attributes[sub_key].get('RawMessageDelivery')
+            if raw_message is not None and 'RawMessageDelivery' in sub_current_attributes:
+                if sub_current_attributes['RawMessageDelivery'].lower() != raw_message.lower():
+                    changed = True
+                    if not self.check_mode:
+                        try:
+                            self.connection.set_subscription_attributes(SubscriptionArn=sub_arn,
+                                                                        AttributeName='RawMessageDelivery',
+                                                                        AttributeValue=raw_message)
+                        except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
+                            self.module.fail_json_aws(e, "Couldn't set RawMessageDelivery subscription attribute")
+
+        return changed
+
     def _delete_subscriptions(self):
         # NOTE: subscriptions in 'PendingConfirmation' timeout in 3 days
         #       https://forums.aws.amazon.com/thread.jspa?threadID=85993
@@ -496,6 +541,13 @@ def ensure_ok(self):
         elif self.display_name or self.policy or self.delivery_policy:
             self.module.fail_json(msg="Cannot set display name, policy or delivery policy for SNS topics not owned by this account")
         changed |= self._set_topic_subs()
+
+        self._init_desired_subscription_attributes()
+        if self.topic_arn in list_topics(self.connection, self.module):
+            changed |= self._set_topic_subs_attributes()
+        elif any(self.desired_subscription_attributes.values()):
+            self.module.fail_json(msg="Cannot set subscription attributes for SNS topics not owned by this account")
+
         return changed
 
     def ensure_gone(self):

tests/integration/targets/sns_topic/defaults/main.yml

@@ -1,12 +1,18 @@
 # we hash the resource_prefix to get a shorter, unique string
 sns_topic_topic_name: "ansible-test-{{ tiny_prefix }}-topic"
+sns_sqs_subscription_attributes: {}
 sns_topic_subscriptions:
   - endpoint: "{{ sns_topic_subscriber_arn }}"
     protocol: "lambda"
+  - endpoint: "{{ sns_topic_subscriber_sqs_arn }}"
+    protocol: sqs
+    attributes: "{{ sns_sqs_subscription_attributes }}"
 sns_topic_third_party_topic_arn: "arn:aws:sns:us-east-1:806199016981:AmazonIpSpaceChanged"
 sns_topic_third_party_region: "{{ sns_topic_third_party_topic_arn.split(':')[3] }}"
 
 # additional test resource namings
 sns_topic_lambda_function: "sns_topic_lambda"
 sns_topic_lambda_name: "ansible-test-{{ tiny_prefix }}-{{ sns_topic_lambda_function }}"
 sns_topic_lambda_role: "ansible-test-{{ tiny_prefix }}-sns-lambda"
+
+sns_topic_sqs_name: "ansible-test-{{ tiny_prefix }}-sns"

tests/integration/targets/sns_topic/tasks/main.yml

@@ -251,6 +251,14 @@
       - delivery_policy.http.defaultHealthyRetryPolicy.maxDelayTarget == 40
       - delivery_policy.http.defaultHealthyRetryPolicy.numRetries == 6
 
+  - name: create SQS queue for subscribing
+    sqs_queue:
+      name: '{{ sns_topic_sqs_name }}'
+    register: sqs_result
+
+  - set_fact:
+      sns_topic_subscriber_sqs_arn: '{{ sqs_result.queue_arn }}'
+
   - name: create temp dir
     tempfile:
       state: directory
@@ -287,7 +295,37 @@
     assert:
       that:
       - sns_topic_subscribe.changed
-      - sns_topic_subscribe.sns_topic.subscriptions|length == 1
+      - sns_topic_subscribe.sns_topic.subscriptions|length == 2
+
+  - name: enable raw message delivery for sqs subscription (attributes)
+    set_fact:
+      sns_sqs_subscription_attributes:
+        RawMessageDelivery: true
+
+  - name: update topic subscriptions - raw message enabled
+    sns_topic:
+      name: '{{ sns_topic_topic_name }}'
+      display_name: My new topic name
+      purge_subscriptions: false
+      subscriptions: '{{ sns_topic_subscriptions }}'
+    register: sns_topic_subscribe_update_raw_on
+
+  - name: assert sqs subscription was updated
+    assert:
+      that:
+        - sns_topic_subscribe_update_raw_on.changed
+
+  - name: rerun topic subscriptions with raw message enabled - expect no changes
+    sns_topic:
+      name: '{{ sns_topic_topic_name }}'
+      display_name: My new topic name
+      purge_subscriptions: false
+      subscriptions: '{{ sns_topic_subscriptions }}'
+    register: rerun_sns_topic_subscribe_update_raw_on
+  - name: assert no changes after rerun
+    assert:
+      that:
+        - not rerun_sns_topic_subscribe_update_raw_on.changed
 
   - name: run again with purge_subscriptions set to false
     sns_topic:
@@ -300,7 +338,7 @@
     assert:
       that:
       - not sns_topic_no_purge.changed
-      - sns_topic_no_purge.sns_topic.subscriptions|length == 1
+      - sns_topic_no_purge.sns_topic.subscriptions|length == 2
 
   - name: run again with purge_subscriptions set to true
     sns_topic:
@@ -319,6 +357,10 @@
       name: '{{ sns_topic_topic_name }}'
       state: absent
 
+  - name: remove subscription attributes before dealing with third party topic
+    set_fact:
+      sns_sqs_subscription_attributes: {}
+
   - name: no-op with third party topic (effectively get existing subscriptions)
     sns_topic:
       name: '{{ sns_topic_third_party_topic_arn }}'
@@ -336,7 +378,7 @@
     assert:
       that:
       - third_party_topic_subscribe is changed
-      - (third_party_topic_subscribe.sns_topic.subscriptions|length) - (third_party_topic.sns_topic.subscriptions|length) == 1
+      - (third_party_topic_subscribe.sns_topic.subscriptions|length) - (third_party_topic.sns_topic.subscriptions|length) == 2
 
   - name: attempt to change name of third party topic
     sns_topic:
@@ -412,6 +454,12 @@
       state: absent
     ignore_errors: true
 
+  - name: remove SQS queue
+    sqs_queue:
+      name: '{{ sns_topic_sqs_name }}'
+      state: absent
+    ignore_errors: true
+
   - name: remove tempdir
     file:
       path: '{{ tempdir.path }}'