Skip to content
Merged
Show file tree
Hide file tree
Changes from 9 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion sdk/eventgrid/azure-eventgrid/azure/eventgrid/_consumer.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@

_LOGGER = logging.getLogger(__name__)

from ._generated.models import CloudEvent as InternalCloudEvent
from ._models import CloudEvent, EventGridEvent

class EventGridConsumer(object):
Expand All @@ -38,7 +39,9 @@ def decode_cloud_event(self, cloud_event, **kwargs):
encode = kwargs.pop('encoding', 'utf-8')
try:
cloud_event = CloudEvent._from_json(cloud_event, encode)
deserialized_event = CloudEvent.deserialize(cloud_event)
deserialized_event = CloudEvent._from_generated(
InternalCloudEvent.deserialize(cloud_event)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's a weird feeling for me that the caller has to know both about the CloudEvent and the InternalCloudEvent. Would it not offer better separation of concerns/isolation to have the InternalCloudEvent.deserialize within the _from_generated so the caller never sees anything but our "nice and pretty" object contract?

)
CloudEvent._deserialize_data(deserialized_event, deserialized_event.type)
return deserialized_event
except Exception as err:
Expand Down
89 changes: 61 additions & 28 deletions sdk/eventgrid/azure-eventgrid/azure/eventgrid/_models.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
from msrest.serialization import UTC
import datetime as dt
import uuid
import base64
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not needed anymore

import json
import six
from ._generated import models
Expand Down Expand Up @@ -50,7 +51,7 @@ def _from_json(event, encode):
return event


class CloudEvent(InternalCloudEvent, EventMixin): #pylint:disable=too-many-instance-attributes
class CloudEvent(EventMixin): #pylint:disable=too-many-instance-attributes
"""Properties of an event published to an Event Grid topic using the CloudEvent 1.0 Schema.

All required parameters must be populated in order to send to Azure.
Expand All @@ -75,35 +76,67 @@ class CloudEvent(InternalCloudEvent, EventMixin): #pylint:disable=too-many-ins
unique for each distinct event.
:type id: Optional[str]
"""

_validation = {
'source': {'required': True},
'type': {'required': True},
}

_attribute_map = {
'additional_properties': {'key': '', 'type': '{object}'},
'id': {'key': 'id', 'type': 'str'},
'source': {'key': 'source', 'type': 'str'},
'data': {'key': 'data', 'type': 'object'},
'data_base64': {'key': 'data_base64', 'type': 'bytearray'},
'type': {'key': 'type', 'type': 'str'},
'time': {'key': 'time', 'type': 'iso-8601'},
'specversion': {'key': 'specversion', 'type': 'str'},
'dataschema': {'key': 'dataschema', 'type': 'str'},
'datacontenttype': {'key': 'datacontenttype', 'type': 'str'},
'subject': {'key': 'subject', 'type': 'str'},
}

def __init__(self, source, type, **kwargs):
# type: (str, str, Any) -> None
kwargs.setdefault('id', uuid.uuid4())
kwargs.setdefault("source", source)
kwargs.setdefault("type", type)
kwargs.setdefault("time", dt.datetime.now(UTC()).isoformat())
kwargs.setdefault("specversion", "1.0")

super(CloudEvent, self).__init__(**kwargs)
self.source = source
self.type = type
self.specversion = kwargs.get("specversion", "1.0")
self.id = kwargs.get("id", str(uuid.uuid4()))
self.time = kwargs.get("time", dt.datetime.now(UTC()).isoformat())
self.data = kwargs.get("data", None)
self.datacontenttype = kwargs.get("datacontenttype", None)
self.dataschema = kwargs.get("dataschema", None)
self.subject = kwargs.get("subject", None)
self._extensions = {}

spec_attr = [
'source', 'type', 'specversion', 'id', 'time',
'data', 'datacontenttype', 'dataschema', 'subject']
for attr in kwargs:
if attr == 'extensions':
self._extensions.update({k:v for k, v in kwargs.get('extensions').items()})
elif attr not in spec_attr:
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If you kwargs.pop instead you don't need spec_attr, and can just pop 'extensions' as well then update the remaining kwargs straight off.

self._extensions.update({attr: kwargs.get(attr)})

@classmethod
def _from_generated(cls, generated, **kwargs):
if generated.additional_properties:
extensions = {k:v for k, v in generated.additional_properties.items()}
kwargs.setdefault('extensions', extensions)
return cls(
id=generated.id,
source=generated.source,
type=generated.type,
specversion=generated.specversion,
data=generated.data or generated.data_base64,
time=generated.time,
dataschema=generated.dataschema,
datacontenttype=generated.datacontenttype,
subject=generated.subject,
**kwargs
)

def _to_generated(self, **kwargs):
if isinstance(self.data, six.binary_type):
data_base64 = self.data
data = None
else:
data_base64 = None
data = self.data
return InternalCloudEvent(
id=self.id,
source=self.source,
type=self.type,
specversion=self.specversion,
data=data,
data_base64=data_base64,
time=self.time,
dataschema=self.dataschema,
datacontenttype=self.datacontenttype,
subject=self.subject,
additional_properties=self._extensions,
**kwargs
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Extensions should be passed part of additional_properties

In [3]: c=CloudEvent(id=42, source="source", type="type", specversion="1.0", additional_properties={"foo":True})

In [4]: c.serialize()
Out[4]:
{'foo': True,
 'id': '42',
 'source': 'source',
 'type': 'type',
 'specversion': '1.0'}

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Apologies if I misremembered our call the other day but did we reach a conclusion about not using kwargs for extensions? My fear being that we've just lost symmetry if someone **'s a serialized cloudevent back into itself.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i took the liberty to slightly change the behavior to this

CloudEvent(id=42, source="source", type="type", specversion="1.0", ext_as_kw = "hello",  extensions={"foo":True})

will serialize to

{'foo': True,
'ext_as_kw ': 'hello',
 'id': '42',
 'source': 'source',
 'type': 'type',
 'specversion': '1.0'}

In other words, extensions can be accepted as both kwargs and an explicit extensions dictionary

Rationale is

  1. This behavior is quite similar to the CloudEvent sdk's model which i think would benefit users jumping from there.
  2. additional_properties as such can be sent by kwargs when possible

I can disallow kwargs if there is a strong opinion against this

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The pitfall here is if someone has an additional property called additional_properties, we get "undefined behavior". I seem to recall a mention in the call that we weren't TOO worried about that/could document it/reserved keywords and all that, but it's the one thing that irks me.

This may be a good target for getting some broader user input on, having them hit this pitfall and see if it's dig-out-able or understandable to their eyes.

Copy link
Member

@lmazuel lmazuel Sep 3, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There should be one-- and preferably only one --obvious way to do it.

:)

What if you do CloudEvent(foo=True, extensions={"foo":False}) for instance? I think we are opening a pandora's box of things we're unsure the day before the code complete. I would suggest we keep extensions only but make this a priority question in user study.

I'm not against it and it has appeal, but right now I want to be sure we have the base minimal behavior, and improve from it first.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

changed it to accepting only extensions - thank you both for the inputs

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Also @KieranBrantnerMagee , I'm catching up your comment, additional_properties is just for autorest model, I was explaining how to tell autorest model to serialize attributes that were not on the Swagger. It was not a comment about the external API of public CloudEvent for customer.

)


class EventGridEvent(InternalEventGridEvent, EventMixin):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,10 @@ def send(self, events, **kwargs):
events = [events]

if all(isinstance(e, CloudEvent) for e in events) or all(_is_cloud_event(e) for e in events):
try:
events = [e._to_generated(**kwargs) for e in events]
except AttributeError:
pass # means it's a dictionary
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

how?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@rakshith91 still, how could this happen?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

sorry - missed it
dictionary will not have a _to_generated - so we do nothing and assume it's in the right format

kwargs.setdefault("content_type", "application/cloudevents-batch+json; charset=utf-8")
self._client.publish_cloud_event_events(self._topic_hostname, events, **kwargs)
elif all(isinstance(e, EventGridEvent) for e in events) or all(isinstance(e, dict) for e in events):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,10 @@ async def send(self, events, **kwargs):
events = [events]

if all(isinstance(e, CloudEvent) for e in events) or all(_is_cloud_event(e) for e in events):
try:
events = [e._to_generated(**kwargs) for e in events]
except AttributeError:
pass # means it's a dictionary
kwargs.setdefault("content_type", "application/cloudevents-batch+json; charset=utf-8")
await self._client.publish_cloud_event_events(self._topic_hostname, events, **kwargs)
elif all(isinstance(e, EventGridEvent) for e in events) or all(isinstance(e, dict) for e in events):
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
interactions:
- request:
body: '[{"id": "3dc4b913-4bc2-41f8-be9b-bf1f67069806", "source": "http://samplesource.dev",
"data": "cloudevent", "type": "Sample.Cloud.Event", "time": "2020-08-19T03:36:41.947462Z",
body: '[{"id": "4e7a9df5-8e28-42c8-adca-4412c93e92d5", "source": "http://samplesource.dev",
"data": "cloudevent", "type": "Sample.Cloud.Event", "time": "2020-09-03T21:37:13.810619Z",
"specversion": "1.0"}]'
headers:
Accept:
Expand All @@ -16,8 +16,6 @@ interactions:
- application/cloudevents-batch+json; charset=utf-8
User-Agent:
- azsdk-python-eventgridpublisherclient/unknown Python/3.7.3 (Windows-10-10.0.18362-SP0)
aeg-sas-key:
- dHUaOOg5xRj+D7iH/AC92GyHweLx9ugrDuMDg4e5Xvw=
method: POST
uri: https://cloudeventgridtestegtopic.westus-1.eventgrid.azure.net/api/events?api-version=2018-01-01
response:
Expand All @@ -29,7 +27,7 @@ interactions:
content-length:
- '0'
date:
- Wed, 19 Aug 2020 03:36:43 GMT
- Thu, 03 Sep 2020 21:37:12 GMT
server:
- Microsoft-HTTPAPI/2.0
strict-transport-security:
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
interactions:
- request:
body: '[{"id": "51c18497-2a25-45f1-b9ba-fdaf08c00263", "source": "http://samplesource.dev",
"data": {"sample": "cloudevent"}, "type": "Sample.Cloud.Event", "time": "2020-08-19T03:36:42.304479Z",
body: '[{"id": "51c747e0-7e8f-467c-80e4-920fdf1d95d2", "source": "http://samplesource.dev",
"data": {"sample": "cloudevent"}, "type": "Sample.Cloud.Event", "time": "2020-09-03T21:37:14.383108Z",
"specversion": "1.0"}]'
headers:
Accept:
Expand All @@ -16,8 +16,6 @@ interactions:
- application/cloudevents-batch+json; charset=utf-8
User-Agent:
- azsdk-python-eventgridpublisherclient/unknown Python/3.7.3 (Windows-10-10.0.18362-SP0)
aeg-sas-key:
- dHUaOOg5xRj+D7iH/AC92GyHweLx9ugrDuMDg4e5Xvw=
method: POST
uri: https://cloudeventgridtestegtopic.westus-1.eventgrid.azure.net/api/events?api-version=2018-01-01
response:
Expand All @@ -29,7 +27,7 @@ interactions:
content-length:
- '0'
date:
- Wed, 19 Aug 2020 03:36:43 GMT
- Thu, 03 Sep 2020 21:37:12 GMT
server:
- Microsoft-HTTPAPI/2.0
strict-transport-security:
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
interactions:
- request:
body: '[{"id": "6a315e93-a59c-4eca-b2f2-6bf3b8f27984", "source": "http://samplesource.dev",
"data": "cloudevent", "type": "Sample.Cloud.Event", "time": "2020-08-19T03:36:42.629467Z",
body: '[{"id": "eef0ee57-9833-44fd-9f52-d5f20b74267d", "source": "http://samplesource.dev",
"data": "cloudevent", "type": "Sample.Cloud.Event", "time": "2020-09-03T21:37:14.803665Z",
"specversion": "1.0"}]'
headers:
Accept:
Expand All @@ -16,8 +16,6 @@ interactions:
- application/cloudevents-batch+json; charset=utf-8
User-Agent:
- azsdk-python-eventgridpublisherclient/unknown Python/3.7.3 (Windows-10-10.0.18362-SP0)
aeg-sas-key:
- dHUaOOg5xRj+D7iH/AC92GyHweLx9ugrDuMDg4e5Xvw=
method: POST
uri: https://cloudeventgridtestegtopic.westus-1.eventgrid.azure.net/api/events?api-version=2018-01-01
response:
Expand All @@ -29,7 +27,7 @@ interactions:
content-length:
- '0'
date:
- Wed, 19 Aug 2020 03:36:43 GMT
- Thu, 03 Sep 2020 21:37:13 GMT
server:
- Microsoft-HTTPAPI/2.0
strict-transport-security:
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
interactions:
- request:
body: '[{"reason_code": 204, "extension": "extension", "id": "9da8e4ce-df2c-4480-b809-c7132fb3ee74",
"source": "http://samplesource.dev", "data": "cloudevent", "type": "Sample.Cloud.Event",
"time": "2020-09-03T21:37:15.165652Z", "specversion": "1.0"}]'
headers:
Accept:
- '*/*'
Accept-Encoding:
- gzip, deflate
Connection:
- keep-alive
Content-Length:
- '244'
Content-Type:
- application/cloudevents-batch+json; charset=utf-8
User-Agent:
- azsdk-python-eventgridpublisherclient/unknown Python/3.7.3 (Windows-10-10.0.18362-SP0)
method: POST
uri: https://cloudeventgridtestegtopic.westus-1.eventgrid.azure.net/api/events?api-version=2018-01-01
response:
body:
string: ''
headers:
api-supported-versions:
- '2018-01-01'
content-length:
- '0'
date:
- Thu, 03 Sep 2020 21:37:13 GMT
server:
- Microsoft-HTTPAPI/2.0
strict-transport-security:
- max-age=31536000; includeSubDomains
status:
code: 200
message: OK
version: 1
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,6 @@ interactions:
- application/cloudevents-batch+json; charset=utf-8
User-Agent:
- azsdk-python-eventgridpublisherclient/unknown Python/3.7.3 (Windows-10-10.0.18362-SP0)
aeg-sas-key:
- dHUaOOg5xRj+D7iH/AC92GyHweLx9ugrDuMDg4e5Xvw=
method: POST
uri: https://cloudeventgridtestegtopic.westus-1.eventgrid.azure.net/api/events?api-version=2018-01-01
response:
Expand All @@ -28,7 +26,7 @@ interactions:
content-length:
- '0'
date:
- Wed, 19 Aug 2020 03:36:44 GMT
- Thu, 03 Sep 2020 21:37:13 GMT
server:
- Microsoft-HTTPAPI/2.0
strict-transport-security:
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
interactions:
- request:
body: '[{"customSubject": "sample", "customEventType": "sample.event", "customDataVersion":
"2.0", "customId": "1234", "customEventTime": "2020-08-19T03:36:56.936961+00:00",
"2.0", "customId": "1234", "customEventTime": "2020-09-03T21:37:32.440554+00:00",
"customData": "sample data"}]'
headers:
Accept:
Expand All @@ -16,8 +16,6 @@ interactions:
- application/json
User-Agent:
- azsdk-python-eventgridpublisherclient/unknown Python/3.7.3 (Windows-10-10.0.18362-SP0)
aeg-sas-key:
- uPQPJHQHsAhBxWOWtRXslz3sXf7TJ5lcqLZ6SC4QzJ4=
method: POST
uri: https://customeventgridtestegtopic.westus-1.eventgrid.azure.net/api/events?api-version=2018-01-01
response:
Expand All @@ -29,7 +27,7 @@ interactions:
content-length:
- '0'
date:
- Wed, 19 Aug 2020 03:36:57 GMT
- Thu, 03 Sep 2020 21:37:31 GMT
server:
- Microsoft-HTTPAPI/2.0
strict-transport-security:
Expand Down
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
interactions:
- request:
body: '[{"customSubject": "sample", "customEventType": "sample.event", "customDataVersion":
"2.0", "customId": "1234", "customEventTime": "2020-08-19T03:36:57.422734+00:00",
"2.0", "customId": "1234", "customEventTime": "2020-09-03T21:37:32.857620+00:00",
"customData": "sample data"}, {"customSubject": "sample2", "customEventType":
"sample.event", "customDataVersion": "2.0", "customId": "12345", "customEventTime":
"2020-08-19T03:36:57.422734+00:00", "customData": "sample data 2"}]'
"2020-09-03T21:37:32.857620+00:00", "customData": "sample data 2"}]'
headers:
Accept:
- '*/*'
Expand All @@ -18,8 +18,6 @@ interactions:
- application/json
User-Agent:
- azsdk-python-eventgridpublisherclient/unknown Python/3.7.3 (Windows-10-10.0.18362-SP0)
aeg-sas-key:
- uPQPJHQHsAhBxWOWtRXslz3sXf7TJ5lcqLZ6SC4QzJ4=
method: POST
uri: https://customeventgridtestegtopic.westus-1.eventgrid.azure.net/api/events?api-version=2018-01-01
response:
Expand All @@ -28,12 +26,10 @@ interactions:
headers:
api-supported-versions:
- '2018-01-01'
connection:
- close
content-length:
- '0'
date:
- Wed, 19 Aug 2020 03:36:58 GMT
- Thu, 03 Sep 2020 21:37:31 GMT
server:
- Microsoft-HTTPAPI/2.0
strict-transport-security:
Expand Down
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
interactions:
- request:
body: '[{"id": "576f3e27-5433-47b3-8b4e-e31ea6a1bdde", "subject": "sample", "data":
"eventgridevent", "eventType": "Sample.EventGrid.Event", "eventTime": "2020-08-19T03:37:11.07064Z",
"dataVersion": "2.0"}, {"id": "6c359c8d-b1fe-4458-96c5-eaffcd864573", "subject":
body: '[{"id": "a3ad167e-302d-4d3d-b5ee-d9e989aa277a", "subject": "sample", "data":
"eventgridevent", "eventType": "Sample.EventGrid.Event", "eventTime": "2020-09-03T21:37:46.777197Z",
"dataVersion": "2.0"}, {"id": "e36112d1-2413-462b-ac5a-8046bd7add59", "subject":
"sample2", "data": "eventgridevent2", "eventType": "Sample.EventGrid.Event",
"eventTime": "2020-08-19T03:37:11.071641Z", "dataVersion": "2.0"}]'
"eventTime": "2020-09-03T21:37:46.778197Z", "dataVersion": "2.0"}]'
headers:
Accept:
- '*/*'
Expand All @@ -13,13 +13,11 @@ interactions:
Connection:
- keep-alive
Content-Length:
- '401'
- '402'
Content-Type:
- application/json; charset=utf-8
User-Agent:
- azsdk-python-eventgridpublisherclient/unknown Python/3.7.3 (Windows-10-10.0.18362-SP0)
aeg-sas-key:
- dS38eoc9IPjofR5npZ1QXrsb3Gz/Kdt99ZdK9SJ+99w=
method: POST
uri: https://eventgridtestegtopic.westus-1.eventgrid.azure.net/api/events?api-version=2018-01-01
response:
Expand All @@ -31,7 +29,7 @@ interactions:
content-length:
- '0'
date:
- Wed, 19 Aug 2020 03:37:12 GMT
- Thu, 03 Sep 2020 21:37:45 GMT
server:
- Microsoft-HTTPAPI/2.0
strict-transport-security:
Expand Down
Loading