Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(ddm): Add new metrics/query endpoint base code #64785

Merged
merged 5 commits into from
Feb 8, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions src/sentry/api/bases/organization.py
Original file line number Diff line number Diff line change
Expand Up @@ -201,6 +201,15 @@ class OrgAuthTokenPermission(OrganizationPermission):
}


class OrganizationMetricsPermission(OrganizationPermission):
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We needed different permissions since by default we don't have org:read on POST.

scope_map = {
"GET": ["org:read", "org:write", "org:admin"],
"POST": ["org:read", "org:write", "org:admin"],
"PUT": ["org:write", "org:admin"],
"DELETE": ["org:admin"],
}


class ControlSiloOrganizationEndpoint(Endpoint):
"""
A base class for endpoints that use an organization scoping but lives in the control silo
Expand Down
108 changes: 107 additions & 1 deletion src/sentry/api/endpoints/organization_metrics.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,14 @@
from sentry.api.api_owners import ApiOwner
from sentry.api.api_publish_status import ApiPublishStatus
from sentry.api.base import region_silo_endpoint
from sentry.api.bases.organization import OrganizationEndpoint
from sentry.api.bases.organization import OrganizationEndpoint, OrganizationMetricsPermission
from sentry.api.exceptions import ResourceDoesNotExist
from sentry.api.paginator import GenericOffsetPaginator
from sentry.api.utils import get_date_range_from_params
from sentry.exceptions import InvalidParams
from sentry.sentry_metrics.querying.data import run_metrics_query
from sentry.sentry_metrics.querying.data_v2 import run_metrics_queries_plan
from sentry.sentry_metrics.querying.data_v2.api import FormulaOrder, MetricsQueriesPlan
from sentry.sentry_metrics.querying.errors import (
InvalidMetricsQueryError,
LatestReleaseNotFoundError,
Expand Down Expand Up @@ -304,3 +306,107 @@
prev=Cursor(0, max(0, offset - limit), True, offset > 0),
next=Cursor(0, max(0, offset + limit), False, has_more),
)


@region_silo_endpoint
class OrganizationMetricsQueryEndpoint(OrganizationEndpoint):
publish_status = {
"POST": ApiPublishStatus.EXPERIMENTAL,
}
owner = ApiOwner.TELEMETRY_EXPERIENCE
permission_classes = (OrganizationMetricsPermission,)

"""
Queries one or more metrics over a time range.
"""

# still 40 req/s but allows for bursts of 200 up to req/s for dashboard loading
default_rate_limit = RateLimit(200, 5)

rate_limits = {
"POST": {
RateLimitCategory.IP: default_rate_limit,
RateLimitCategory.USER: default_rate_limit,
RateLimitCategory.ORGANIZATION: default_rate_limit,
},
}

# Number of groups returned by default for each query.
default_limit = 20

def _validate_order(self, order: str | None) -> FormulaOrder | None:
if order is None:
return None

formula_order = FormulaOrder.from_string(order)
if formula_order is None:
order_choices = [v.value for v in FormulaOrder]
raise InvalidMetricsQueryError(
f"The provided `order` is not a valid, only {order_choices} are supported"
)

return formula_order

def _validate_limit(self, limit: str | None) -> int:
if not limit:
return self.default_limit

try:
return int(limit)
except ValueError:
raise InvalidMetricsQueryError(
"The provided `limit` is not valid, an integer is required"
)

def _interval_from_request(self, request: Request) -> int:
"""
Extracts the interval of the query from the request payload.
"""
interval = parse_stats_period(request.data.get("interval", "1h"))
return int(3600 if interval is None else interval.total_seconds())

def _metrics_queries_plan_from_request(self, request: Request) -> MetricsQueriesPlan:
"""
Extracts the metrics queries plan from the request payload.
"""
metrics_queries_plan = MetricsQueriesPlan()

queries = request.data.get("queries") or []
for query in queries:
metrics_queries_plan.declare_query(name=query["name"], mql=query["mql"])

formulas = request.data.get("formulas") or []
for formula in formulas:
metrics_queries_plan.apply_formula(
mql=formula["mql"],
order=self._validate_order(formula.get("order")),
limit=self._validate_limit(formula.get("limit")),
)

return metrics_queries_plan

def post(self, request: Request, organization) -> Response:
try:
start, end = get_date_range_from_params(request.GET)
interval = self._interval_from_request(request)
metrics_queries_plan = self._metrics_queries_plan_from_request(request)

results = run_metrics_queries_plan(
metrics_queries_plan=metrics_queries_plan,
start=start,
end=end,
interval=interval,
organization=organization,
# TODO: figure out how to make these methods work with HTTP body.
projects=self.get_projects(request, organization),
environments=self.get_environments(request, organization),
referrer=Referrer.API_DDM_METRICS_QUERY.value,
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Created a different referrer to separately track in snuba the usage of the new endpoint.

)
except InvalidMetricsQueryError as e:
return Response(status=400, data={"detail": str(e)})

Check warning

Code scanning / CodeQL

Information exposure through an exception Medium

Stack trace information
flows to this location and may be exposed to an external user.
except LatestReleaseNotFoundError as e:
return Response(status=404, data={"detail": str(e)})

Check warning

Code scanning / CodeQL

Information exposure through an exception Medium

Stack trace information
flows to this location and may be exposed to an external user.
except MetricsQueryExecutionError as e:
return Response(status=500, data={"detail": str(e)})

Check warning

Code scanning / CodeQL

Information exposure through an exception Medium

Stack trace information
flows to this location and may be exposed to an external user.

return Response(status=200, data=results)
6 changes: 6 additions & 0 deletions src/sentry/api/urls.py
Original file line number Diff line number Diff line change
Expand Up @@ -385,6 +385,7 @@
OrganizationMetricDetailsEndpoint,
OrganizationMetricsDataEndpoint,
OrganizationMetricsDetailsEndpoint,
OrganizationMetricsQueryEndpoint,
OrganizationMetricsTagDetailsEndpoint,
OrganizationMetricsTagsEndpoint,
)
Expand Down Expand Up @@ -1981,6 +1982,11 @@ def create_group_urls(name_prefix: str) -> list[URLPattern | URLResolver]:
OrganizationMetricsDataEndpoint.as_view(),
name="sentry-api-0-organization-metrics-data",
),
re_path(
r"^(?P<organization_slug>[^/]+)/metrics/query/$",
OrganizationMetricsQueryEndpoint.as_view(),
name="sentry-api-0-organization-metrics-query",
),
re_path(
r"^(?P<organization_slug>[^/]+)/metrics/tags/$",
OrganizationMetricsTagsEndpoint.as_view(),
Expand Down
3 changes: 3 additions & 0 deletions src/sentry/sentry_metrics/querying/data_v2/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
from .api import run_metrics_queries_plan

__all__ = ["run_metrics_queries_plan"]
61 changes: 61 additions & 0 deletions src/sentry/sentry_metrics/querying/data_v2/api.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
from collections.abc import Sequence
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Copied the entire code in data to make the new changes completely independent of the old ones. Should enable us to keep the old endpoint up and running without problems.

from dataclasses import dataclass
from datetime import datetime
from enum import Enum
from typing import Union

from sentry.models.environment import Environment
from sentry.models.organization import Organization
from sentry.models.project import Project


# TODO: lift out types in `types.py` once endpoint is finished.
class FormulaOrder(Enum):
ASC = "asc"
DESC = "desc"

@classmethod
# Used `Union` because `|` conflicts with the parser.
def from_string(cls, value: str) -> Union["FormulaOrder", None]:
for v in cls:
if v.value == value:
return v

return None


@dataclass(frozen=True)
class FormulaDefinition:
mql: str
order: FormulaOrder | None
limit: int | None


class MetricsQueriesPlan:
def __init__(self):
self._queries: dict[str, str] = {}
self._formulas: list[FormulaDefinition] = []

def declare_query(self, name: str, mql: str) -> "MetricsQueriesPlan":
self._queries[name] = mql
return self

def apply_formula(
self, mql: str, order: FormulaOrder | None = None, limit: int | None = None
) -> "MetricsQueriesPlan":
self._formulas.append(FormulaDefinition(mql=mql, order=order, limit=limit))
return self


def run_metrics_queries_plan(
metrics_queries_plan: MetricsQueriesPlan,
start: datetime,
end: datetime,
interval: int,
organization: Organization,
projects: Sequence[Project],
environments: Sequence[Environment],
referrer: str,
):
# TODO: implement new querying logic.
return None
Loading
Loading