diff --git a/.github/component_owners.yml b/.github/component_owners.yml index c5479fe87a..afad47902a 100644 --- a/.github/component_owners.yml +++ b/.github/component_owners.yml @@ -41,5 +41,8 @@ components: instrumentation/opentelemetry-instrumentation-tornado: - shalevr + instrumentation/opentelemetry-instrumentation-urllib: + - shalevr + instrumentation/opentelemetry-instrumentation-urllib3: - shalevr diff --git a/CHANGELOG.md b/CHANGELOG.md index f60d4041f2..068a067dd1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Add metric instrumentation for urllib + ([#1553](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1553)) - `opentelemetry/sdk/extension/aws` Implement [`aws.ecs.*`](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/resource/semantic_conventions/cloud_provider/aws/ecs.md) and [`aws.logs.*`](https://opentelemetry.io/docs/reference/specification/resource/semantic_conventions/cloud_provider/aws/logs/) resource attributes in the `AwsEcsResourceDetector` detector when the ECS Metadata v4 is available ([#1212](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1212)) diff --git a/instrumentation/README.md b/instrumentation/README.md index 98ebb4c728..a269b09397 100644 --- a/instrumentation/README.md +++ b/instrumentation/README.md @@ -40,6 +40,6 @@ | [opentelemetry-instrumentation-system-metrics](./opentelemetry-instrumentation-system-metrics) | psutil >= 5 | No | [opentelemetry-instrumentation-tornado](./opentelemetry-instrumentation-tornado) | tornado >= 5.1.1 | Yes | [opentelemetry-instrumentation-tortoiseorm](./opentelemetry-instrumentation-tortoiseorm) | tortoise-orm >= 0.17.0 | No -| [opentelemetry-instrumentation-urllib](./opentelemetry-instrumentation-urllib) | urllib | No +| [opentelemetry-instrumentation-urllib](./opentelemetry-instrumentation-urllib) | urllib | Yes | [opentelemetry-instrumentation-urllib3](./opentelemetry-instrumentation-urllib3) | urllib3 >= 1.0.0, < 2.0.0 | Yes | [opentelemetry-instrumentation-wsgi](./opentelemetry-instrumentation-wsgi) | wsgi | Yes \ No newline at end of file diff --git a/instrumentation/opentelemetry-instrumentation-urllib/src/opentelemetry/instrumentation/urllib/__init__.py b/instrumentation/opentelemetry-instrumentation-urllib/src/opentelemetry/instrumentation/urllib/__init__.py index b73a1cacc9..6a80a4a723 100644 --- a/instrumentation/opentelemetry-instrumentation-urllib/src/opentelemetry/instrumentation/urllib/__init__.py +++ b/instrumentation/opentelemetry-instrumentation-urllib/src/opentelemetry/instrumentation/urllib/__init__.py @@ -63,10 +63,9 @@ def response_hook(span, request_obj, response) import functools import types import typing - -# from urllib import response from http import client -from typing import Collection +from timeit import default_timer +from typing import Collection, Dict from urllib.request import ( # pylint: disable=no-name-in-module,import-error OpenerDirector, Request, @@ -83,7 +82,9 @@ def response_hook(span, request_obj, response) _SUPPRESS_INSTRUMENTATION_KEY, http_status_to_status_code, ) +from opentelemetry.metrics import Histogram, get_meter from opentelemetry.propagate import inject +from opentelemetry.semconv.metrics import MetricInstruments from opentelemetry.semconv.trace import SpanAttributes from opentelemetry.trace import Span, SpanKind, get_tracer from opentelemetry.trace.status import Status @@ -114,8 +115,15 @@ def _instrument(self, **kwargs): """ tracer_provider = kwargs.get("tracer_provider") tracer = get_tracer(__name__, __version__, tracer_provider) + + meter_provider = kwargs.get("meter_provider") + meter = get_meter(__name__, __version__, meter_provider) + + histograms = _create_client_histograms(meter) + _instrument( tracer, + histograms, request_hook=kwargs.get("request_hook"), response_hook=kwargs.get("response_hook"), ) @@ -132,6 +140,7 @@ def uninstrument_opener( def _instrument( tracer, + histograms: Dict[str, Histogram], request_hook: _RequestHookT = None, response_hook: _ResponseHookT = None, ): @@ -192,11 +201,13 @@ def _instrumented_open_call( context.set_value(_SUPPRESS_HTTP_INSTRUMENTATION_KEY, True) ) try: + start_time = default_timer() result = call_wrapped() # *** PROCEED except Exception as exc: # pylint: disable=W0703 exception = exc result = getattr(exc, "file", None) finally: + elapsed_time = round((default_timer() - start_time) * 1000) context.detach(token) if result is not None: @@ -214,6 +225,10 @@ def _instrumented_open_call( SpanAttributes.HTTP_FLAVOR ] = f"{ver_[:1]}.{ver_[:-1]}" + _record_histograms( + histograms, labels, request, result, elapsed_time + ) + if callable(response_hook): response_hook(span, request, result) @@ -248,3 +263,45 @@ def _uninstrument_from(instr_root, restore_as_bound_func=False): if restore_as_bound_func: original = types.MethodType(original, instr_root) setattr(instr_root, instr_func_name, original) + + +def _create_client_histograms(meter) -> Dict[str, Histogram]: + histograms = { + MetricInstruments.HTTP_CLIENT_DURATION: meter.create_histogram( + name=MetricInstruments.HTTP_CLIENT_DURATION, + unit="ms", + description="measures the duration outbound HTTP requests", + ), + MetricInstruments.HTTP_CLIENT_REQUEST_SIZE: meter.create_histogram( + name=MetricInstruments.HTTP_CLIENT_REQUEST_SIZE, + unit="By", + description="measures the size of HTTP request messages (compressed)", + ), + MetricInstruments.HTTP_CLIENT_RESPONSE_SIZE: meter.create_histogram( + name=MetricInstruments.HTTP_CLIENT_RESPONSE_SIZE, + unit="By", + description="measures the size of HTTP response messages (compressed)", + ), + } + + return histograms + + +def _record_histograms( + histograms, metric_attributes, request, response, elapsed_time +): + histograms[MetricInstruments.HTTP_CLIENT_DURATION].record( + elapsed_time, attributes=metric_attributes + ) + + data = getattr(request, "data", None) + request_size = 0 if data is None else len(data) + histograms[MetricInstruments.HTTP_CLIENT_REQUEST_SIZE].record( + request_size, attributes=metric_attributes + ) + + if response is not None: + response_size = int(response.headers.get("Content-Length", 0)) + histograms[MetricInstruments.HTTP_CLIENT_RESPONSE_SIZE].record( + response_size, attributes=metric_attributes + ) diff --git a/instrumentation/opentelemetry-instrumentation-urllib/src/opentelemetry/instrumentation/urllib/package.py b/instrumentation/opentelemetry-instrumentation-urllib/src/opentelemetry/instrumentation/urllib/package.py index 7a66a17a93..942f175da1 100644 --- a/instrumentation/opentelemetry-instrumentation-urllib/src/opentelemetry/instrumentation/urllib/package.py +++ b/instrumentation/opentelemetry-instrumentation-urllib/src/opentelemetry/instrumentation/urllib/package.py @@ -14,3 +14,5 @@ _instruments = tuple() + +_supports_metrics = True diff --git a/instrumentation/opentelemetry-instrumentation-urllib/tests/test_metrics_instrumentation.py b/instrumentation/opentelemetry-instrumentation-urllib/tests/test_metrics_instrumentation.py new file mode 100644 index 0000000000..a87e3f97b9 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-urllib/tests/test_metrics_instrumentation.py @@ -0,0 +1,246 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from timeit import default_timer +from typing import Optional, Union +from urllib import request +from urllib.parse import urlencode + +import httpretty + +from opentelemetry.instrumentation.urllib import ( # pylint: disable=no-name-in-module,import-error + URLLibInstrumentor, +) +from opentelemetry.sdk.metrics._internal.point import Metric +from opentelemetry.sdk.metrics.export import ( + HistogramDataPoint, + NumberDataPoint, +) +from opentelemetry.semconv.metrics import MetricInstruments +from opentelemetry.test.test_base import TestBase + + +class TestRequestsIntegration(TestBase): + URL = "http://httpbin.org/status/200" + URL_POST = "http://httpbin.org/post" + + def setUp(self): + super().setUp() + URLLibInstrumentor().instrument() + httpretty.enable() + httpretty.register_uri(httpretty.GET, self.URL, body=b"Hello!") + httpretty.register_uri( + httpretty.POST, self.URL_POST, body=b"Hello World!" + ) + + def tearDown(self): + super().tearDown() + URLLibInstrumentor().uninstrument() + httpretty.disable() + + def get_sorted_metrics(self): + resource_metrics = ( + self.memory_metrics_reader.get_metrics_data().resource_metrics + ) + + all_metrics = [] + for metrics in resource_metrics: + for scope_metrics in metrics.scope_metrics: + all_metrics.extend(scope_metrics.metrics) + + return self.sorted_metrics(all_metrics) + + @staticmethod + def sorted_metrics(metrics): + """ + Sorts metrics by metric name. + """ + return sorted( + metrics, + key=lambda m: m.name, + ) + + def assert_metric_expected( + self, + metric: Metric, + expected_value: Union[int, float], + expected_attributes: dict, + est_delta: Optional[float] = None, + ): + data_point = next(iter(metric.data.data_points)) + + if isinstance(data_point, HistogramDataPoint): + self.assertEqual( + data_point.count, + 1, + ) + if est_delta is None: + self.assertEqual( + data_point.sum, + expected_value, + ) + else: + self.assertAlmostEqual( + data_point.sum, + expected_value, + delta=est_delta, + ) + elif isinstance(data_point, NumberDataPoint): + self.assertEqual( + data_point.value, + expected_value, + ) + + self.assertDictEqual( + expected_attributes, + dict(data_point.attributes), + ) + + def test_basic_metric(self): + start_time = default_timer() + with request.urlopen(self.URL) as result: + client_duration_estimated = (default_timer() - start_time) * 1000 + + metrics = self.get_sorted_metrics() + self.assertEqual(len(metrics), 3) + + ( + client_duration, + client_request_size, + client_response_size, + ) = metrics[:3] + + self.assertEqual( + client_duration.name, MetricInstruments.HTTP_CLIENT_DURATION + ) + self.assert_metric_expected( + client_duration, + client_duration_estimated, + { + "http.status_code": str(result.code), + "http.method": "GET", + "http.url": str(result.url), + "http.flavor": "1.1", + }, + est_delta=200, + ) + + # net.peer.name + + self.assertEqual( + client_request_size.name, + MetricInstruments.HTTP_CLIENT_REQUEST_SIZE, + ) + self.assert_metric_expected( + client_request_size, + 0, + { + "http.status_code": str(result.code), + "http.method": "GET", + "http.url": str(result.url), + "http.flavor": "1.1", + }, + ) + + self.assertEqual( + client_response_size.name, + MetricInstruments.HTTP_CLIENT_RESPONSE_SIZE, + ) + self.assert_metric_expected( + client_response_size, + result.length, + { + "http.status_code": str(result.code), + "http.method": "GET", + "http.url": str(result.url), + "http.flavor": "1.1", + }, + ) + + def test_basic_metric_request_not_empty(self): + data = {"header1": "value1", "header2": "value2"} + data_encoded = urlencode(data).encode() + + start_time = default_timer() + with request.urlopen(self.URL_POST, data=data_encoded) as result: + client_duration_estimated = (default_timer() - start_time) * 1000 + + metrics = self.get_sorted_metrics() + self.assertEqual(len(metrics), 3) + + ( + client_duration, + client_request_size, + client_response_size, + ) = metrics[:3] + + self.assertEqual( + client_duration.name, MetricInstruments.HTTP_CLIENT_DURATION + ) + self.assert_metric_expected( + client_duration, + client_duration_estimated, + { + "http.status_code": str(result.code), + "http.method": "POST", + "http.url": str(result.url), + "http.flavor": "1.1", + }, + est_delta=200, + ) + + self.assertEqual( + client_request_size.name, + MetricInstruments.HTTP_CLIENT_REQUEST_SIZE, + ) + self.assert_metric_expected( + client_request_size, + len(data_encoded), + { + "http.status_code": str(result.code), + "http.method": "POST", + "http.url": str(result.url), + "http.flavor": "1.1", + }, + ) + + self.assertEqual( + client_response_size.name, + MetricInstruments.HTTP_CLIENT_RESPONSE_SIZE, + ) + self.assert_metric_expected( + client_response_size, + result.length, + { + "http.status_code": str(result.code), + "http.method": "POST", + "http.url": str(result.url), + "http.flavor": "1.1", + }, + ) + + def test_metric_uninstrument(self): + with request.urlopen(self.URL): + metrics = self.get_sorted_metrics() + self.assertEqual(len(metrics), 3) + + URLLibInstrumentor().uninstrument() + with request.urlopen(self.URL): + metrics = self.get_sorted_metrics() + self.assertEqual(len(metrics), 3) + + for metric in metrics: + for point in list(metric.data.data_points): + self.assertEqual(point.count, 1)