|
| 1 | +import inspect |
| 2 | +import sys |
| 3 | + |
| 4 | +import sentry_sdk |
| 5 | +from sentry_sdk.consts import OP, SPANSTATUS |
| 6 | +from sentry_sdk.integrations import DidNotEnable, Integration |
| 7 | +from sentry_sdk.tracing import TRANSACTION_SOURCE_TASK |
| 8 | +from sentry_sdk.utils import ( |
| 9 | + event_from_exception, |
| 10 | + logger, |
| 11 | + package_version, |
| 12 | + qualname_from_function, |
| 13 | + reraise, |
| 14 | +) |
| 15 | + |
| 16 | +try: |
| 17 | + import ray # type: ignore[import-not-found] |
| 18 | +except ImportError: |
| 19 | + raise DidNotEnable("Ray not installed.") |
| 20 | +import functools |
| 21 | + |
| 22 | +from typing import TYPE_CHECKING |
| 23 | + |
| 24 | +if TYPE_CHECKING: |
| 25 | + from collections.abc import Callable |
| 26 | + from typing import Any, Optional |
| 27 | + from sentry_sdk.utils import ExcInfo |
| 28 | + |
| 29 | + |
| 30 | +def _check_sentry_initialized(): |
| 31 | + # type: () -> None |
| 32 | + if sentry_sdk.get_client().is_active(): |
| 33 | + return |
| 34 | + |
| 35 | + logger.debug( |
| 36 | + "[Tracing] Sentry not initialized in ray cluster worker, performance data will be discarded." |
| 37 | + ) |
| 38 | + |
| 39 | + |
| 40 | +def _patch_ray_remote(): |
| 41 | + # type: () -> None |
| 42 | + old_remote = ray.remote |
| 43 | + |
| 44 | + @functools.wraps(old_remote) |
| 45 | + def new_remote(f, *args, **kwargs): |
| 46 | + # type: (Callable[..., Any], *Any, **Any) -> Callable[..., Any] |
| 47 | + if inspect.isclass(f): |
| 48 | + # Ray Actors |
| 49 | + # (https://docs.ray.io/en/latest/ray-core/actors.html) |
| 50 | + # are not supported |
| 51 | + # (Only Ray Tasks are supported) |
| 52 | + return old_remote(f, *args, *kwargs) |
| 53 | + |
| 54 | + def _f(*f_args, _tracing=None, **f_kwargs): |
| 55 | + # type: (Any, Optional[dict[str, Any]], Any) -> Any |
| 56 | + """ |
| 57 | + Ray Worker |
| 58 | + """ |
| 59 | + _check_sentry_initialized() |
| 60 | + |
| 61 | + transaction = sentry_sdk.continue_trace( |
| 62 | + _tracing or {}, |
| 63 | + op=OP.QUEUE_TASK_RAY, |
| 64 | + name=qualname_from_function(f), |
| 65 | + origin=RayIntegration.origin, |
| 66 | + source=TRANSACTION_SOURCE_TASK, |
| 67 | + ) |
| 68 | + |
| 69 | + with sentry_sdk.start_transaction(transaction) as transaction: |
| 70 | + try: |
| 71 | + result = f(*f_args, **f_kwargs) |
| 72 | + transaction.set_status(SPANSTATUS.OK) |
| 73 | + except Exception: |
| 74 | + transaction.set_status(SPANSTATUS.INTERNAL_ERROR) |
| 75 | + exc_info = sys.exc_info() |
| 76 | + _capture_exception(exc_info) |
| 77 | + reraise(*exc_info) |
| 78 | + |
| 79 | + return result |
| 80 | + |
| 81 | + rv = old_remote(_f, *args, *kwargs) |
| 82 | + old_remote_method = rv.remote |
| 83 | + |
| 84 | + def _remote_method_with_header_propagation(*args, **kwargs): |
| 85 | + # type: (*Any, **Any) -> Any |
| 86 | + """ |
| 87 | + Ray Client |
| 88 | + """ |
| 89 | + with sentry_sdk.start_span( |
| 90 | + op=OP.QUEUE_SUBMIT_RAY, |
| 91 | + description=qualname_from_function(f), |
| 92 | + origin=RayIntegration.origin, |
| 93 | + ) as span: |
| 94 | + tracing = { |
| 95 | + k: v |
| 96 | + for k, v in sentry_sdk.get_current_scope().iter_trace_propagation_headers() |
| 97 | + } |
| 98 | + try: |
| 99 | + result = old_remote_method(*args, **kwargs, _tracing=tracing) |
| 100 | + span.set_status(SPANSTATUS.OK) |
| 101 | + except Exception: |
| 102 | + span.set_status(SPANSTATUS.INTERNAL_ERROR) |
| 103 | + exc_info = sys.exc_info() |
| 104 | + _capture_exception(exc_info) |
| 105 | + reraise(*exc_info) |
| 106 | + |
| 107 | + return result |
| 108 | + |
| 109 | + rv.remote = _remote_method_with_header_propagation |
| 110 | + |
| 111 | + return rv |
| 112 | + |
| 113 | + ray.remote = new_remote |
| 114 | + |
| 115 | + |
| 116 | +def _capture_exception(exc_info, **kwargs): |
| 117 | + # type: (ExcInfo, **Any) -> None |
| 118 | + client = sentry_sdk.get_client() |
| 119 | + |
| 120 | + event, hint = event_from_exception( |
| 121 | + exc_info, |
| 122 | + client_options=client.options, |
| 123 | + mechanism={ |
| 124 | + "handled": False, |
| 125 | + "type": RayIntegration.identifier, |
| 126 | + }, |
| 127 | + ) |
| 128 | + sentry_sdk.capture_event(event, hint=hint) |
| 129 | + |
| 130 | + |
| 131 | +class RayIntegration(Integration): |
| 132 | + identifier = "ray" |
| 133 | + origin = f"auto.queue.{identifier}" |
| 134 | + |
| 135 | + @staticmethod |
| 136 | + def setup_once(): |
| 137 | + # type: () -> None |
| 138 | + version = package_version("ray") |
| 139 | + |
| 140 | + if version is None: |
| 141 | + raise DidNotEnable("Unparsable ray version: {}".format(version)) |
| 142 | + |
| 143 | + if version < (2, 7, 0): |
| 144 | + raise DidNotEnable("Ray 2.7.0 or newer required") |
| 145 | + |
| 146 | + _patch_ray_remote() |
0 commit comments