ref(analytics): Analytics Refactor + Types

mypy.ini

@@ -1,6 +1,7 @@
 [mypy]
 python_version = 3.8
-files = src/sentry/api/bases/external_actor.py,
+files = src/sentry/analytics/,
+        src/sentry/api/bases/external_actor.py,
         src/sentry/api/bases/organization_events.py,
         src/sentry/api/endpoints/external_team.py,
         src/sentry/api/endpoints/external_team_details.py,

src/sentry/analytics/__init__.py

@@ -1,24 +1,31 @@
-from django.conf import settings
-
 from sentry import options
 from sentry.utils.services import LazyServiceWrapper
 
-from .base import Analytics  # NOQA
-from .event import *  # NOQA
+from .attribute import Attribute
+from .base import Analytics
+from .event import Event
 from .event_manager import default_manager
+from .map import Map
+from .utils import get_backend_path
 
-
-def get_backend_path(backend):
-    try:
-        backend = settings.SENTRY_ANALYTICS_ALIASES[backend]
-    except KeyError:
-        pass
-    return backend
-
+__all__ = (
+    "Analytics",
+    "Attribute",
+    "Event",
+    "Map",
+    "record",
+    "record_event",
+    "setup",
+)
 
 backend = LazyServiceWrapper(
-    Analytics, get_backend_path(options.get("analytics.backend")), options.get("analytics.options")
+    backend_base=Analytics,
+    backend_path=get_backend_path(options.get("analytics.backend")),
+    options=options.get("analytics.options"),
 )
-backend.expose(locals())
 
+record = backend.record
+record_event = backend.record_event
 register = default_manager.register
+setup = backend.setup
+validate = backend.validate

src/sentry/analytics/attribute.py

@@ -0,0 +1,14 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+
+@dataclass
+class Attribute:
+    name: str
+    type: type = str
+    required: bool = True
+
+    def extract(self, value: str | None) -> Any | None:
+        return None if value is None else self.type(value)

src/sentry/analytics/base.py

@@ -1,22 +1,24 @@
+from __future__ import annotations
+
 __all__ = ("Analytics",)
 
+import abc
+from typing import Any
 
 from sentry.analytics.event import Event
 from sentry.utils.services import Service
 
 from .event_manager import default_manager
 
 
-class Analytics(Service):
+class Analytics(Service, abc.ABC):  # type: ignore
     __all__ = ("record", "validate")
 
     event_manager = default_manager
 
-    def record(self, event_or_event_type, instance=None, **kwargs):
-        """
-        >>> record(Event())
-        >>> record('organization.created', organization)
-        """
+    def record(
+        self, event_or_event_type: str | Event | Any, instance: Any | None = None, **kwargs: Any
+    ) -> None:
         if isinstance(event_or_event_type, str):
             event = self.event_manager.get(event_or_event_type).from_instance(instance, **kwargs)
         elif isinstance(event_or_event_type, Event):
@@ -25,11 +27,11 @@ def record(self, event_or_event_type, instance=None, **kwargs):
             return
         self.record_event(event)
 
-    def record_event(self, event):
-        """
-        >>> record_event(Event())
-        """
+    def record_event(self, event: Event) -> None:
+        pass
 
-    def setup(self):
+    def setup(self) -> None:
         # Load default event types
         import sentry.analytics.events  # NOQA
+
+        pass

src/sentry/analytics/event.py

@@ -1,99 +1,50 @@
-from sentry.utils.compat import map
-
-__all__ = ("Attribute", "Event", "Map")
+from __future__ import annotations
 
+import abc
+import datetime as dt
 from base64 import b64encode
 from collections.abc import Mapping
+from typing import Any, Sequence
 from uuid import uuid1
 
 from django.utils import timezone
 
+from sentry.analytics.attribute import Attribute
+from sentry.analytics.utils import get_data
 from sentry.utils.dates import to_timestamp
 
 
-class Attribute:
-    def __init__(self, name, type=str, required=True):
-        self.name = name
-        self.type = type
-        self.required = required
-
-    def extract(self, value):
-        if value is None:
-            return value
-        return self.type(value)
-
-
-class Map(Attribute):
-    def __init__(self, name, attributes=None, required=True):
-        self.name = name
-        self.required = required
-        self.attributes = attributes
-
-    def extract(self, value):
-        """
-        If passed a non dictionary we assume we can pull attributes from it.
-
-        This will hard error in some situations if you're passing a bad type
-        (like an int).
-        """
-        if value is None:
-            return value
-
-        if not isinstance(value, Mapping):
-            new_value = {}
-            for attr in self.attributes:
-                new_value[attr.name] = attr.extract(getattr(value, attr.name, None))
-            items = new_value
-        else:
-            # ensure we don't mutate the original
-            # we don't need to deepcopy as if it recurses into another Map it
-            # will once again copy itself
-            items = value.copy()
-
-        data = {}
-        for attr in self.attributes:
-            nv = items.pop(attr.name, None)
-            if attr.required and nv is None:
-                raise ValueError(f"{attr.name} is required (cannot be None)")
-
-            data[attr.name] = attr.extract(nv)
-
-        if items:
-            raise ValueError("Unknown attributes: {}".format(", ".join(map(str, items.keys()))))
-
-        return data
-
-
-class Event:
+class Event(abc.ABC):
     __slots__ = ["uuid", "data", "datetime"]
 
+    # This MUST be overridden by child classes.
     type = None
 
-    attributes = ()
+    # These should be overridden by child classes.
+    attributes: Sequence[Attribute] = ()
 
-    def __init__(self, type=None, datetime=None, **items):
+    def __init__(
+        self, type: Any | None = None, datetime: dt.datetime | None = None, **items: Any
+    ) -> None:
         self.uuid = uuid1()
-
         self.datetime = datetime or timezone.now()
-        if type is not None:
-            self.type = type
+        self.type = self._get_type(type)
+        self.data = get_data(self.attributes, items)
+
+    def _get_type(self, _type: Any | None = None) -> Any:
+        """
+        The Event's `type` can either be passed in as a parameter or set as a
+        property on a child class.
+        """
+        if _type is not None:
+            return _type
 
         if self.type is None:
             raise ValueError("Event is missing type")
 
-        data = {}
-        for attr in self.attributes:
-            nv = items.pop(attr.name, None)
-            if attr.required and nv is None:
-                raise ValueError(f"{attr.name} is required (cannot be None)")
-            data[attr.name] = attr.extract(nv)
-
-        if items:
-            raise ValueError("Unknown attributes: {}".format(", ".join(items.keys())))
-
-        self.data = data
+        return self.type
 
-    def serialize(self):
+    def serialize(self) -> Mapping[str, Any]:
         return {
             "uuid": b64encode(self.uuid.bytes),
             "timestamp": to_timestamp(self.datetime),
@@ -102,8 +53,10 @@ def serialize(self):
         }
 
     @classmethod
-    def from_instance(cls, instance, **kwargs):
-        values = {}
-        for attr in cls.attributes:
-            values[attr.name] = kwargs.get(attr.name, getattr(instance, attr.name, None))
-        return cls(**values)
+    def from_instance(cls, instance: Any, **kwargs: Any) -> Event:
+        return cls(
+            **{
+                attr.name: kwargs.get(attr.name, getattr(instance, attr.name, None))
+                for attr in cls.attributes
+            }
+        )

src/sentry/analytics/event_manager.py

@@ -1,20 +1,22 @@
 __all__ = ("default_manager", "EventManager")
 
+from typing import Any, MutableMapping, Type
+
 from sentry.analytics.event import Event
 
 
 class EventManager:
-    def __init__(self):
-        self._event_types = {}
+    def __init__(self) -> None:
+        self._event_types: MutableMapping[Any, Type[Event]] = {}
 
-    def register(self, event_cls: Event) -> None:
+    def register(self, event_cls: Type[Event]) -> None:
         event_type = event_cls.type
         if event_type in self._event_types:
             assert self._event_types[event_type] == event_cls
         else:
             self._event_types[event_type] = event_cls
 
-    def get(self, type: str) -> Event:
+    def get(self, type: str) -> Type[Event]:
         return self._event_types[type]
 
 

src/sentry/analytics/events/__init__.py

@@ -1,3 +1,3 @@
 from sentry.utils.imports import import_submodules
 
-import_submodules(globals(), __name__, __path__)
+import_submodules(globals(), __name__, __path__)  # type: ignore

src/sentry/analytics/events/inapp_request.py

@@ -1,18 +1,20 @@
+import abc
+
 from sentry import analytics
 
 
-class InAppRequestSentEvent(analytics.Event):
-    attributes = (
+class InAppRequestSentEvent(analytics.Event, abc.ABC):
+    attributes = [
         analytics.Attribute("organization_id"),
         analytics.Attribute("user_id", required=False),
         analytics.Attribute("target_user_id"),
         analytics.Attribute("providers"),
         analytics.Attribute("subtype", required=False),
-    )
+    ]
 
 
-class InviteOrJoinRequest(InAppRequestSentEvent):
-    attributes = InAppRequestSentEvent.attributes + (analytics.Attribute("invited_member_id"),)
+class InviteOrJoinRequest(InAppRequestSentEvent, abc.ABC):
+    attributes = InAppRequestSentEvent.attributes + [analytics.Attribute("invited_member_id")]
 
 
 class InviteRequestSentEvent(InviteOrJoinRequest):

src/sentry/analytics/map.py

@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+from collections.abc import Mapping
+from typing import Any, Sequence
+
+from . import Attribute
+from .utils import get_data
+
+
+class Map(Attribute):
+    def __init__(
+        self, name: str, attributes: Sequence[Attribute] | None = None, required: bool = True
+    ) -> None:
+        super().__init__(name, str, required)
+        self.attributes = attributes or ()
+
+    def extract(self, value: dict[str, Any] | Any | None) -> Mapping[str, Any] | None:
+        """
+        If passed a non dictionary we assume we can pull attributes from it.
+
+        This will hard error in some situations if you're passing a bad type
+        (like an int).
+        """
+        if value is None:
+            return value
+
+        if isinstance(value, dict):
+            # Ensure we don't mutate the original. We do not need to deepcopy;
+            # if it recurses into another Map it will once again copy itself.
+            # TODO(mgaeta): If the dictionary has a list or some other complex
+            #  object in it, that object will not be copied.
+            items = value.copy()
+        else:
+            new_value = {}
+            for attr in self.attributes:
+                new_value[attr.name] = attr.extract(getattr(value, attr.name, None))
+            items = new_value
+
+        return get_data(self.attributes, items)

src/sentry/analytics/pubsub.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 __all__ = ("PubSubAnalytics",)
 
 import logging
@@ -7,20 +9,20 @@
 
 from sentry.utils.json import dumps
 
-from .base import Analytics
+from . import Analytics, Event
 
 logger = logging.getLogger(__name__)
 
 
 class PubSubAnalytics(Analytics):
     def __init__(
         self,
-        project,
-        topic,
-        batch_max_bytes=1024 * 1024 * 5,
-        batch_max_latency=0.05,
-        batch_max_messages=1000,
-    ):
+        project: str,
+        topic: str,
+        batch_max_bytes: int = 1024 * 1024 * 5,
+        batch_max_latency: float = 0.05,
+        batch_max_messages: int = 1000,
+    ) -> None:
         settings = pubsub_v1.types.BatchSettings(
             max_bytes=batch_max_bytes,
             max_latency=batch_max_latency,
@@ -34,6 +36,6 @@ def __init__(
         else:
             self.topic = self.publisher.topic_path(project, topic)
 
-    def record_event(self, event):
+    def record_event(self, event: Event) -> None:
         if self.publisher is not None:
             self.publisher.publish(self.topic, data=dumps(event.serialize()).encode("utf-8"))