Increase the default throttle/warning threshold for allocation polici…

…es (#6189) Allocation policies are our mechanism for doing traffic management for Snuba queries. Currently, we see a lot of warnings (Warning: Query from referrer ... is throttled) on Sentry Issues. This is because ~17% of queries to Snuba are throttled by capacity management/allocation policies. This is an overkill since we actually have a lot of Clickhouse capacity. This PR increases the default throttle/warning threshold to be higher than half of the rejection threshold. I will also increase the thresholds in Snuba Admin. Co-authored-by: Rachel Chen <[email protected]>
getsentry · Aug 9, 2024 · 52cd1ed · 52cd1ed
1 parent 5e70b76
commit 52cd1ed
Show file tree

Hide file tree

Showing 4 changed files with 14 additions and 12 deletions.
diff --git a/snuba/query/allocation_policies/bytes_scanned_rejecting_policy.py b/snuba/query/allocation_policies/bytes_scanned_rejecting_policy.py
@@ -45,7 +45,7 @@
 PETABYTE = 10**12
 DEFAULT_BYTES_SCANNED_LIMIT = int(1.28 * PETABYTE)
 DEFAULT_TIMEOUT_PENALIZATION = DEFAULT_BYTES_SCANNED_LIMIT // 40
-DEFAULT_BYTES_THROTTLE_DIVIDER = 2
+DEFAULT_BYTES_THROTTLE_DIVIDER = 1.5
 DEFAULT_THREADS_THROTTLE_DIVIDER = 2
 QUOTA_UNIT = "bytes"
 SUGGESTION = "The feature, organization/project is scanning too many bytes, this usually means they are abusing that API"
@@ -102,14 +102,14 @@ def _additional_config_definitions(self) -> list[AllocationPolicyConfig]:
             AllocationPolicyConfig(
                 "bytes_throttle_divider",
                 "Divide the scan limit by this number gives the throttling threshold",
-                int,
+                float,
                 DEFAULT_BYTES_THROTTLE_DIVIDER,
             ),
             AllocationPolicyConfig(
                 "threads_throttle_divider",
                 "max threads divided by this number is the number of threads we use to execute queries for a throttled (project_id|organization_id, referrer)",
                 int,
-                DEFAULT_BYTES_THROTTLE_DIVIDER,
+                DEFAULT_THREADS_THROTTLE_DIVIDER,
             ),
         ]
 
@@ -204,7 +204,7 @@ def _get_quota_allowance(
             customer_tenant_key, customer_tenant_value, referrer
         )
         throttle_threshold = max(
-            1, scan_limit // self.get_config_value("bytes_throttle_divider")
+            1, int(scan_limit // self.get_config_value("bytes_throttle_divider"))
         )
         timestamp, granted_quotas = _RATE_LIMITER.check_within_quotas(
             [

diff --git a/snuba/query/allocation_policies/per_referrer.py b/snuba/query/allocation_policies/per_referrer.py
@@ -23,7 +23,7 @@
 _DEFAULT_CONCURRENT_REQUEST_PER_REFERRER = 100
 _REFERRER_CONCURRENT_OVERRIDE = -1
 _REFERRER_MAX_THREADS_OVERRIDE = -1
-_REQUESTS_THROTTLE_DIVIDER = 2
+_REQUESTS_THROTTLE_DIVIDER = 1.5
 _THREADS_THROTTLE_DIVIDER = 2
 
 QUOTA_UNIT = "concurrent_queries"
@@ -77,7 +77,7 @@ def _additional_config_definitions(self) -> list[AllocationPolicyConfig]:
             AllocationPolicyConfig(
                 name="requests_throttle_divider",
                 description="default_concurrent_request_per_referrer divided by this value will be the threshold at which we will decrease the number of threads (THROTTLED_THREADS) used to execute queries",
-                value_type=int,
+                value_type=float,
                 default=_REQUESTS_THROTTLE_DIVIDER,
             ),
             AllocationPolicyConfig(
@@ -129,8 +129,10 @@ def _get_quota_allowance(
         num_threads = self._get_max_threads(referrer)
         requests_throttle_threshold = max(
             1,
-            self.get_config_value("default_concurrent_request_per_referrer")
-            // self.get_config_value("requests_throttle_divider"),
+            int(
+                self.get_config_value("default_concurrent_request_per_referrer")
+                // self.get_config_value("requests_throttle_divider")
+            ),
         )
 
         is_throttled = False

diff --git a/tests/query/allocation_policies/test_bytes_scanned_rejecting_policy.py b/tests/query/allocation_policies/test_bytes_scanned_rejecting_policy.py
@@ -112,7 +112,7 @@ def test_throttles(
     policy: BytesScannedRejectingPolicy,
 ) -> None:
     _configure_policy(policy)
-    policy.set_config_value("bytes_throttle_divider", 2)
+    policy.set_config_value("bytes_throttle_divider", 1.5)
     policy.set_config_value("threads_throttle_divider", 2)
     tenant_ids: dict[str, int | str] = {
         "organization_id": 123,

diff --git a/tests/web/test_db_query.py b/tests/web/test_db_query.py
@@ -308,7 +308,7 @@ def test_db_query_success() -> None:
                 "quota_used": 1560000000000,
                 "quota_unit": "bytes",
                 "suggestion": "The feature, organization/project is scanning too many bytes, this usually means they are abusing that API",
-                "throttle_threshold": 1280000000000,
+                "throttle_threshold": 1706666666666,
             },
         },
         "details": {
@@ -322,7 +322,7 @@ def test_db_query_success() -> None:
                     "storage_key": "StorageKey.ERRORS_RO",
                 },
                 "is_throttled": False,
-                "throttle_threshold": 50,
+                "throttle_threshold": 66,
                 "rejection_threshold": 100,
                 "quota_used": 1,
                 "quota_unit": "concurrent_queries",
@@ -351,7 +351,7 @@ def test_db_query_success() -> None:
                     "storage_key": "StorageKey.ERRORS_RO",
                 },
                 "is_throttled": True,
-                "throttle_threshold": 1280000000000,
+                "throttle_threshold": 1706666666666,
                 "rejection_threshold": 2560000000000,
                 "quota_used": 1560000000000,
                 "quota_unit": "bytes",