Cloud Event Abstraction (#13542)

* initial commit

* oops

* some changes

* lint

* test fix

* sas key

* some more cahgnes

* test fix

* 2.7 compat

* comments

* extensions

* changes

* Update sdk/eventgrid/azure-eventgrid/azure/eventgrid/_models.py

* remove test

Author: Rakshith Bhyravabhotla

sdk/eventgrid/azure-eventgrid/azure/eventgrid/_consumer.py

@@ -38,7 +38,7 @@ def decode_cloud_event(self, cloud_event, **kwargs):
         encode = kwargs.pop('encoding', 'utf-8')
         try:
             cloud_event = CloudEvent._from_json(cloud_event, encode)
-            deserialized_event = CloudEvent.deserialize(cloud_event)
+            deserialized_event = CloudEvent._from_generated(cloud_event)
             CloudEvent._deserialize_data(deserialized_event, deserialized_event.type)
             return deserialized_event 
         except Exception as err:

sdk/eventgrid/azure-eventgrid/azure/eventgrid/_models.py

@@ -50,7 +50,7 @@ def _from_json(event, encode):
         return event
 
 
-class CloudEvent(InternalCloudEvent, EventMixin):   #pylint:disable=too-many-instance-attributes
+class CloudEvent(EventMixin):   #pylint:disable=too-many-instance-attributes
     """Properties of an event published to an Event Grid topic using the CloudEvent 1.0 Schema.
 
     All required parameters must be populated in order to send to Azure.
@@ -75,35 +75,60 @@ class CloudEvent(InternalCloudEvent, EventMixin):   #pylint:disable=too-many-ins
      unique for each distinct event.
     :type id: Optional[str]
     """
-
-    _validation = {
-        'source': {'required': True},
-        'type': {'required': True},
-    }
-
-    _attribute_map = {
-        'additional_properties': {'key': '', 'type': '{object}'},
-        'id': {'key': 'id', 'type': 'str'},
-        'source': {'key': 'source', 'type': 'str'},
-        'data': {'key': 'data', 'type': 'object'},
-        'data_base64': {'key': 'data_base64', 'type': 'bytearray'},
-        'type': {'key': 'type', 'type': 'str'},
-        'time': {'key': 'time', 'type': 'iso-8601'},
-        'specversion': {'key': 'specversion', 'type': 'str'},
-        'dataschema': {'key': 'dataschema', 'type': 'str'},
-        'datacontenttype': {'key': 'datacontenttype', 'type': 'str'},
-        'subject': {'key': 'subject', 'type': 'str'},
-    }
-
     def __init__(self, source, type, **kwargs):
         # type: (str, str, Any) -> None
-        kwargs.setdefault('id', uuid.uuid4())
-        kwargs.setdefault("source", source)
-        kwargs.setdefault("type", type)
-        kwargs.setdefault("time", dt.datetime.now(UTC()).isoformat())
-        kwargs.setdefault("specversion", "1.0")
-
-        super(CloudEvent, self).__init__(**kwargs)
+        self.source = source
+        self.type = type
+        self.specversion = kwargs.pop("specversion", "1.0")
+        self.id = kwargs.pop("id", str(uuid.uuid4()))
+        self.time = kwargs.pop("time", dt.datetime.now(UTC()).isoformat())
+        self.data = kwargs.pop("data", None)
+        self.datacontenttype = kwargs.pop("datacontenttype", None)
+        self.dataschema = kwargs.pop("dataschema", None)
+        self.subject = kwargs.pop("subject", None)
+        self.extensions = {}
+        self.extensions.update({k:v for k, v in kwargs.pop('extensions', {}).items()})
+
+    @classmethod
+    def _from_generated(cls, cloud_event, **kwargs):
+        generated = InternalCloudEvent.deserialize(cloud_event)
+        if generated.additional_properties:
+            extensions = {k:v for k, v in generated.additional_properties.items()}
+            kwargs.setdefault('extensions', extensions)
+        return cls(
+            id=generated.id,
+            source=generated.source,
+            type=generated.type,
+            specversion=generated.specversion,
+            data=generated.data or generated.data_base64,
+            time=generated.time,
+            dataschema=generated.dataschema,
+            datacontenttype=generated.datacontenttype,
+            subject=generated.subject,
+            **kwargs
+        )
+
+    def _to_generated(self, **kwargs):
+        if isinstance(self.data, six.binary_type):
+            data_base64 = self.data
+            data = None
+        else:
+            data_base64 = None
+            data = self.data
+        return InternalCloudEvent(
+            id=self.id,
+            source=self.source,
+            type=self.type,
+            specversion=self.specversion,
+            data=data,
+            data_base64=data_base64,
+            time=self.time,
+            dataschema=self.dataschema,
+            datacontenttype=self.datacontenttype,
+            subject=self.subject,
+            additional_properties=self.extensions,
+            **kwargs
+        )
 
 
 class EventGridEvent(InternalEventGridEvent, EventMixin):

sdk/eventgrid/azure-eventgrid/azure/eventgrid/_publisher_client.py

@@ -62,6 +62,10 @@ def send(self, events, **kwargs):
             events = [events]
 
         if all(isinstance(e, CloudEvent) for e in events) or all(_is_cloud_event(e) for e in events):
+            try:
+                events = [e._to_generated(**kwargs) for e in events]
+            except AttributeError:
+                pass # means it's a dictionary
             kwargs.setdefault("content_type", "application/cloudevents-batch+json; charset=utf-8")
             self._client.publish_cloud_event_events(self._topic_hostname, events, **kwargs)
         elif all(isinstance(e, EventGridEvent) for e in events) or all(isinstance(e, dict) for e in events):

sdk/eventgrid/azure-eventgrid/azure/eventgrid/aio/_publisher_client_async.py

@@ -63,6 +63,10 @@ async def send(self, events, **kwargs):
             events = [events]
 
         if all(isinstance(e, CloudEvent) for e in events) or all(_is_cloud_event(e) for e in events):
+            try:
+                events = [e._to_generated(**kwargs) for e in events]
+            except AttributeError:
+                pass # means it's a dictionary
             kwargs.setdefault("content_type", "application/cloudevents-batch+json; charset=utf-8")
             await self._client.publish_cloud_event_events(self._topic_hostname, events, **kwargs)
         elif all(isinstance(e, EventGridEvent) for e in events) or all(isinstance(e, dict) for e in events):

sdk/eventgrid/azure-eventgrid/tests/recordings/test_eg_publisher_client.test_send_cloud_event_data_as_list.yaml

@@ -1,7 +1,7 @@
 interactions:
 - request:
-    body: '[{"id": "3dc4b913-4bc2-41f8-be9b-bf1f67069806", "source": "http://samplesource.dev",
-      "data": "cloudevent", "type": "Sample.Cloud.Event", "time": "2020-08-19T03:36:41.947462Z",
+    body: '[{"id": "4e7a9df5-8e28-42c8-adca-4412c93e92d5", "source": "http://samplesource.dev",
+      "data": "cloudevent", "type": "Sample.Cloud.Event", "time": "2020-09-03T21:37:13.810619Z",
       "specversion": "1.0"}]'
     headers:
       Accept:
@@ -16,8 +16,6 @@ interactions:
       - application/cloudevents-batch+json; charset=utf-8
       User-Agent:
       - azsdk-python-eventgridpublisherclient/unknown Python/3.7.3 (Windows-10-10.0.18362-SP0)
-      aeg-sas-key:
-      - dHUaOOg5xRj+D7iH/AC92GyHweLx9ugrDuMDg4e5Xvw=
     method: POST
     uri: https://cloudeventgridtestegtopic.westus-1.eventgrid.azure.net/api/events?api-version=2018-01-01
   response:
@@ -29,7 +27,7 @@ interactions:
       content-length:
       - '0'
       date:
-      - Wed, 19 Aug 2020 03:36:43 GMT
+      - Thu, 03 Sep 2020 21:37:12 GMT
       server:
       - Microsoft-HTTPAPI/2.0
       strict-transport-security:

sdk/eventgrid/azure-eventgrid/tests/recordings/test_eg_publisher_client.test_send_cloud_event_data_bytes.yaml

@@ -0,0 +1,38 @@
+interactions:
+- request:
+    body: '[{"id": "a3f05c78-9a8d-44d8-a1a3-131b8da61192", "source": "http://samplesource.dev",
+      "data_base64": "Y2xvdWRldmVudA==", "type": "Sample.Cloud.Event", "time": "2020-09-04T17:06:31.601865Z",
+      "specversion": "1.0"}]'
+    headers:
+      Accept:
+      - '*/*'
+      Accept-Encoding:
+      - gzip, deflate
+      Connection:
+      - keep-alive
+      Content-Length:
+      - '211'
+      Content-Type:
+      - application/cloudevents-batch+json; charset=utf-8
+      User-Agent:
+      - azsdk-python-eventgridpublisherclient/unknown Python/3.7.3 (Windows-10-10.0.18362-SP0)
+    method: POST
+    uri: https://cloudeventgridtestegtopic.westus-1.eventgrid.azure.net/api/events?api-version=2018-01-01
+  response:
+    body:
+      string: ''
+    headers:
+      api-supported-versions:
+      - '2018-01-01'
+      content-length:
+      - '0'
+      date:
+      - Fri, 04 Sep 2020 17:06:29 GMT
+      server:
+      - Microsoft-HTTPAPI/2.0
+      strict-transport-security:
+      - max-age=31536000; includeSubDomains
+    status:
+      code: 200
+      message: OK
+version: 1

sdk/eventgrid/azure-eventgrid/tests/recordings/test_eg_publisher_client.test_send_cloud_event_data_dict.yaml

@@ -1,7 +1,7 @@
 interactions:
 - request:
-    body: '[{"id": "51c18497-2a25-45f1-b9ba-fdaf08c00263", "source": "http://samplesource.dev",
-      "data": {"sample": "cloudevent"}, "type": "Sample.Cloud.Event", "time": "2020-08-19T03:36:42.304479Z",
+    body: '[{"id": "51c747e0-7e8f-467c-80e4-920fdf1d95d2", "source": "http://samplesource.dev",
+      "data": {"sample": "cloudevent"}, "type": "Sample.Cloud.Event", "time": "2020-09-03T21:37:14.383108Z",
       "specversion": "1.0"}]'
     headers:
       Accept:
@@ -16,8 +16,6 @@ interactions:
       - application/cloudevents-batch+json; charset=utf-8
       User-Agent:
       - azsdk-python-eventgridpublisherclient/unknown Python/3.7.3 (Windows-10-10.0.18362-SP0)
-      aeg-sas-key:
-      - dHUaOOg5xRj+D7iH/AC92GyHweLx9ugrDuMDg4e5Xvw=
     method: POST
     uri: https://cloudeventgridtestegtopic.westus-1.eventgrid.azure.net/api/events?api-version=2018-01-01
   response:
@@ -29,7 +27,7 @@ interactions:
       content-length:
       - '0'
       date:
-      - Wed, 19 Aug 2020 03:36:43 GMT
+      - Thu, 03 Sep 2020 21:37:12 GMT
       server:
       - Microsoft-HTTPAPI/2.0
       strict-transport-security:

sdk/eventgrid/azure-eventgrid/tests/recordings/test_eg_publisher_client.test_send_cloud_event_data_str.yaml

@@ -1,7 +1,7 @@
 interactions:
 - request:
-    body: '[{"id": "6a315e93-a59c-4eca-b2f2-6bf3b8f27984", "source": "http://samplesource.dev",
-      "data": "cloudevent", "type": "Sample.Cloud.Event", "time": "2020-08-19T03:36:42.629467Z",
+    body: '[{"id": "eef0ee57-9833-44fd-9f52-d5f20b74267d", "source": "http://samplesource.dev",
+      "data": "cloudevent", "type": "Sample.Cloud.Event", "time": "2020-09-03T21:37:14.803665Z",
       "specversion": "1.0"}]'
     headers:
       Accept:
@@ -16,8 +16,6 @@ interactions:
       - application/cloudevents-batch+json; charset=utf-8
       User-Agent:
       - azsdk-python-eventgridpublisherclient/unknown Python/3.7.3 (Windows-10-10.0.18362-SP0)
-      aeg-sas-key:
-      - dHUaOOg5xRj+D7iH/AC92GyHweLx9ugrDuMDg4e5Xvw=
     method: POST
     uri: https://cloudeventgridtestegtopic.westus-1.eventgrid.azure.net/api/events?api-version=2018-01-01
   response:
@@ -29,7 +27,7 @@ interactions:
       content-length:
       - '0'
       date:
-      - Wed, 19 Aug 2020 03:36:43 GMT
+      - Thu, 03 Sep 2020 21:37:13 GMT
       server:
       - Microsoft-HTTPAPI/2.0
       strict-transport-security:

sdk/eventgrid/azure-eventgrid/tests/recordings/test_eg_publisher_client.test_send_cloud_event_data_with_extensions.yaml

@@ -0,0 +1,38 @@
+interactions:
+- request:
+    body: '[{"reason_code": 204, "extension": "extension", "id": "9da8e4ce-df2c-4480-b809-c7132fb3ee74",
+      "source": "http://samplesource.dev", "data": "cloudevent", "type": "Sample.Cloud.Event",
+      "time": "2020-09-03T21:37:15.165652Z", "specversion": "1.0"}]'
+    headers:
+      Accept:
+      - '*/*'
+      Accept-Encoding:
+      - gzip, deflate
+      Connection:
+      - keep-alive
+      Content-Length:
+      - '244'
+      Content-Type:
+      - application/cloudevents-batch+json; charset=utf-8
+      User-Agent:
+      - azsdk-python-eventgridpublisherclient/unknown Python/3.7.3 (Windows-10-10.0.18362-SP0)
+    method: POST
+    uri: https://cloudeventgridtestegtopic.westus-1.eventgrid.azure.net/api/events?api-version=2018-01-01
+  response:
+    body:
+      string: ''
+    headers:
+      api-supported-versions:
+      - '2018-01-01'
+      content-length:
+      - '0'
+      date:
+      - Thu, 03 Sep 2020 21:37:13 GMT
+      server:
+      - Microsoft-HTTPAPI/2.0
+      strict-transport-security:
+      - max-age=31536000; includeSubDomains
+    status:
+      code: 200
+      message: OK
+version: 1

sdk/eventgrid/azure-eventgrid/tests/recordings/test_eg_publisher_client.test_send_cloud_event_dict.yaml

@@ -15,8 +15,6 @@ interactions:
       - application/cloudevents-batch+json; charset=utf-8
       User-Agent:
       - azsdk-python-eventgridpublisherclient/unknown Python/3.7.3 (Windows-10-10.0.18362-SP0)
-      aeg-sas-key:
-      - dHUaOOg5xRj+D7iH/AC92GyHweLx9ugrDuMDg4e5Xvw=
     method: POST
     uri: https://cloudeventgridtestegtopic.westus-1.eventgrid.azure.net/api/events?api-version=2018-01-01
   response:
@@ -28,7 +26,7 @@ interactions:
       content-length:
       - '0'
       date:
-      - Wed, 19 Aug 2020 03:36:44 GMT
+      - Thu, 03 Sep 2020 21:37:13 GMT
       server:
       - Microsoft-HTTPAPI/2.0
       strict-transport-security: