diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/CHANGELOG.md b/sdk/monitor/azure-monitor-opentelemetry-exporter/CHANGELOG.md index 47f3e4addf2f..f8c6ff137630 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/CHANGELOG.md +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/CHANGELOG.md @@ -16,6 +16,8 @@ ([#26023](https://github.com/Azure/azure-sdk-for-python/pull/26023)) - Implement statsbeat shutdown ([#26077](https://github.com/Azure/azure-sdk-for-python/pull/26077)) +- Add ApplicationInsightsSampler + ([#26224](https://github.com/Azure/azure-sdk-for-python/pull/26224)) ### Breaking Changes diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/README.md b/sdk/monitor/azure-monitor-opentelemetry-exporter/README.md index 4c405d9cc798..bf271331971f 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/README.md +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/README.md @@ -36,8 +36,8 @@ NOTE: The logging signal for the `AzureMonitorLogExporter` is currently in an EX ```Python from azure.monitor.opentelemetry.exporter import AzureMonitorLogExporter -exporter = AzureMonitorLogExporter.from_connection_string( - conn_str = os.environ["APPLICATIONINSIGHTS_CONNECTION_STRING"] +exporter = AzureMonitorLogExporter( + connection_string=os.environ["APPLICATIONINSIGHTS_CONNECTION_STRING"] ) ``` @@ -45,18 +45,17 @@ exporter = AzureMonitorLogExporter.from_connection_string( ```Python from azure.monitor.opentelemetry.exporter import AzureMonitorMetricExporter -exporter = AzureMonitorMetricExporter.from_connection_string( - conn_str = os.environ["APPLICATIONINSIGHTS_CONNECTION_STRING"] +exporter = AzureMonitorMetricExporter( + connection_string=os.environ["APPLICATIONINSIGHTS_CONNECTION_STRING"] ) - ``` #### Tracing ```Python from azure.monitor.opentelemetry.exporter import AzureMonitorTraceExporter -exporter = AzureMonitorTraceExporter.from_connection_string( - conn_str = os.environ["APPLICATIONINSIGHTS_CONNECTION_STRING"] +exporter = AzureMonitorTraceExporter( + connection_string=os.environ["APPLICATIONINSIGHTS_CONNECTION_STRING"] ) ``` @@ -127,11 +126,15 @@ Some of the key concepts for the Azure monitor exporter include: * [Sampling][sampler_ref]: Sampling is a mechanism to control the noise and overhead introduced by OpenTelemetry by reducing the number of samples of traces collected and sent to the backend. +* ApplicationInsightsSampler: Application Insights specific sampler used for consistent sampling across Application Insights SDKs and OpenTelemetry-based SDKs sending data to Application Insights. This sampler MUST be used whenever `AzureMonitorTraceExporter` is used. + For more information about these resources, see [What is Azure Monitor?][product_docs]. ## Examples -### Logging +### Logging (experimental) + +NOTE: The logging signal for the `AzureMonitorLogExporter` is currently in an EXPERIMENTAL state. Possible breaking changes may ensue in the future. The following sections provide several code snippets covering some of the most common tasks, including: @@ -158,8 +161,8 @@ from azure.monitor.opentelemetry.exporter import AzureMonitorLogExporter log_emitter_provider = LogEmitterProvider() set_log_emitter_provider(log_emitter_provider) -exporter = AzureMonitorLogExporter.from_connection_string( - os.environ["APPLICATIONINSIGHTS_CONNECTION_STRING"] +exporter = AzureMonitorLogExporter( + connection_string=os.environ["APPLICATIONINSIGHTS_CONNECTION_STRING"] ) log_emitter_provider.add_log_processor(BatchLogProcessor(exporter)) @@ -196,8 +199,8 @@ tracer = trace.get_tracer(__name__) log_emitter_provider = LogEmitterProvider() set_log_emitter_provider(log_emitter_provider) -exporter = AzureMonitorLogExporter.from_connection_string( - os.environ["APPLICATIONINSIGHTS_CONNECTION_STRING"] +exporter = AzureMonitorLogExporter( + connection_string=os.environ["APPLICATIONINSIGHTS_CONNECTION_STRING"] ) log_emitter_provider.add_log_processor(BatchLogProcessor(exporter)) @@ -233,8 +236,8 @@ from azure.monitor.opentelemetry.exporter import AzureMonitorLogExporter log_emitter_provider = LogEmitterProvider() set_log_emitter_provider(log_emitter_provider) -exporter = AzureMonitorLogExporter.from_connection_string( - os.environ["APPLICATIONINSIGHTS_CONNECTION_STRING"] +exporter = AzureMonitorLogExporter( + connection_string=os.environ["APPLICATIONINSIGHTS_CONNECTION_STRING"] ) log_emitter_provider.add_log_processor(BatchLogProcessor(exporter)) @@ -272,8 +275,8 @@ from opentelemetry.sdk._logs.export import BatchLogProcessor from azure.monitor.opentelemetry.exporter import AzureMonitorLogExporter set_log_emitter_provider(LogEmitterProvider()) -exporter = AzureMonitorLogExporter.from_connection_string( - os.environ["APPLICATIONINSIGHTS_CONNECTION_STRING"] +exporter = AzureMonitorLogExporter( + connection_string=os.environ["APPLICATIONINSIGHTS_CONNECTION_STRING"] ) get_log_emitter_provider().add_log_processor(BatchLogProcessor(exporter)) @@ -324,8 +327,8 @@ from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader from azure.monitor.opentelemetry.exporter import AzureMonitorMetricExporter -exporter = AzureMonitorMetricExporter.from_connection_string( - os.environ["APPLICATIONINSIGHTS_CONNECTION_STRING"] +exporter = AzureMonitorMetricExporter( + connection_string=os.environ["APPLICATIONINSIGHTS_CONNECTION_STRING"] ) reader = PeriodicExportingMetricReader(exporter, export_interval_millis=5000) metrics.set_meter_provider(MeterProvider(metric_readers=[reader])) @@ -381,6 +384,7 @@ The following sections provide several code snippets covering some of the most c * [Exporting a custom span](#export-hello-world-trace) * [Using an instrumentation to track a library](#instrumentation-with-requests-library) +* [Enabling sampling to limit the amount of telemetry sent](#enabling-sampling) #### Export Hello World Trace @@ -391,12 +395,12 @@ from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import BatchSpanProcessor from azure.monitor.opentelemetry.exporter import AzureMonitorTraceExporter -exporter = AzureMonitorTraceExporter.from_connection_string( - connection_string = os.environ["APPLICATIONINSIGHTS_CONNECTION_STRING "] -) - trace.set_tracer_provider(TracerProvider()) tracer = trace.get_tracer(__name__) +# This is the exporter that sends data to Application Insights +exporter = AzureMonitorTraceExporter( + connection_string=os.environ["APPLICATIONINSIGHTS_CONNECTION_STRING"] +) span_processor = BatchSpanProcessor(exporter) trace.get_tracer_provider().add_span_processor(span_processor) @@ -419,27 +423,52 @@ from opentelemetry import trace from opentelemetry.instrumentation.requests import RequestsInstrumentor from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import BatchSpanProcessor - from azure.monitor.opentelemetry.exporter import AzureMonitorTraceExporter -trace.set_tracer_provider(TracerProvider()) -tracer = trace.get_tracer(__name__) - # This line causes your calls made with the requests library to be tracked. RequestsInstrumentor().instrument() -span_processor = BatchSpanProcessor( - AzureMonitorTraceExporter.from_connection_string( - os.environ["APPLICATIONINSIGHTS_CONNECTION_STRING "] - ) + +trace.set_tracer_provider(TracerProvider()) +tracer = trace.get_tracer(__name__) +exporter = AzureMonitorTraceExporter( + connection_string=os.environ["APPLICATIONINSIGHTS_CONNECTION_STRING"] ) +span_processor = BatchSpanProcessor(exporter) trace.get_tracer_provider().add_span_processor(span_processor) -RequestsInstrumentor().instrument() - # This request will be traced response = requests.get(url="https://azure.microsoft.com/") ``` +#### Enabling sampling + +```Python +import os +from opentelemetry import trace +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor +from azure.monitor.opentelemetry.exporter import ( + ApplicationInsightsSampler, + AzureMonitorTraceExporter, +) + +# Sampler expects a sample rate of between 0 and 1 inclusive +# A rate of 0.75 means approximately 75% of your telemetry will be sent +sampler = ApplicationInsightsSampler(0.75) +trace.set_tracer_provider(TracerProvider(sampler=sampler)) +tracer = trace.get_tracer(__name__) +exporter = AzureMonitorTraceExporter( + connection_string=os.environ["APPLICATIONINSIGHTS_CONNECTION_STRING"] +) +span_processor = BatchSpanProcessor(exporter) +trace.get_tracer_provider().add_span_processor(span_processor) + +for i in range(100): + # Approximately 25% of these spans should be sampled out + with tracer.start_as_current_span("hello"): + print("Hello, World!") +``` + ## Troubleshooting The exporter raises exceptions defined in [Azure Core](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/core/azure-core/README.md#azure-core-library-exceptions). diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/__init__.py b/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/__init__.py index f556e141fc69..feaa0fec8584 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/__init__.py +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/__init__.py @@ -7,9 +7,11 @@ from azure.monitor.opentelemetry.exporter.export.logs._exporter import AzureMonitorLogExporter from azure.monitor.opentelemetry.exporter.export.metrics._exporter import AzureMonitorMetricExporter from azure.monitor.opentelemetry.exporter.export.trace._exporter import AzureMonitorTraceExporter +from azure.monitor.opentelemetry.exporter.export.trace._sampling import ApplicationInsightsSampler from ._version import VERSION __all__ = [ + "ApplicationInsightsSampler", "AzureMonitorMetricExporter", "AzureMonitorLogExporter", "AzureMonitorTraceExporter", diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/export/trace/_exporter.py b/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/export/trace/_exporter.py index b8385728944f..2adb91cd49fd 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/export/trace/_exporter.py +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/export/trace/_exporter.py @@ -421,7 +421,7 @@ def _convert_span_to_envelope(span: ReadableSpan) -> TelemetryItem: # Max key length is 150, value is 8192 if not key or len(key) > 150 or val is None: continue - data.properties[key] = val[:8192] + data.properties[key] = str(val)[:8192] if span.links: # Max length for value is 8192 # Since links are a fixed length (80) in json, max number of links would be 102 diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/export/trace/_sampling.py b/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/export/trace/_sampling.py new file mode 100644 index 000000000000..d7aabc37b2d4 --- /dev/null +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/export/trace/_sampling.py @@ -0,0 +1,87 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +from typing import Optional, Sequence + +from fixedint import Int32 +# pylint:disable=W0611 +from opentelemetry.context import Context +from opentelemetry.trace import Link, SpanKind, format_trace_id +from opentelemetry.sdk.trace.sampling import ( + Decision, + Sampler, + SamplingResult, + _get_parent_trace_state, +) +from opentelemetry.trace.span import TraceState +from opentelemetry.util.types import Attributes + + +_HASH = 5381 +_INTEGER_MAX = Int32.maxval +_INTEGER_MIN = Int32.minval + + +# Sampler is responsible for the following: +# Implements same trace id hashing algorithm so that traces are sampled the same across multiple nodes (via AI SDKS) +# Adds item count to span attribute if span is sampled (needed for ingestion service) +# Inherits from the Sampler interface as defined by OpenTelemetry +# https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/sdk.md#sampler +class ApplicationInsightsSampler(Sampler): + """Sampler that implements the same probability sampling algorithm as the ApplicationInsights SDKs.""" + + # sampling_ratio takes values in the range [0,1] + def __init__(self, sampling_ratio: float = 1.0): + self._ratio = sampling_ratio + self._sample_rate = round(sampling_ratio * 100) + + # pylint:disable=C0301 + # See https://github.com/microsoft/Telemetry-Collection-Spec/blob/main/OpenTelemetry/trace/ApplicationInsightsSampler.md + def should_sample( + self, + parent_context: Optional[Context], + trace_id: int, + name: str, + kind: SpanKind = None, + attributes: Attributes = None, + links: Sequence["Link"] = None, + trace_state: "TraceState" = None, + ) -> "SamplingResult": + if self._sample_rate == 0: + decision = Decision.DROP + elif self._sample_rate == 100.0: + decision = Decision.RECORD_AND_SAMPLE + else: + # Determine if should sample from ratio and traceId + sample_score = self._get_DJB2_sample_score(format_trace_id(trace_id).lower()) + if sample_score < self._ratio: + decision = Decision.RECORD_AND_SAMPLE + else: + decision = Decision.DROP + # Add sample rate as span attribute + if attributes is None: + attributes = {} + attributes["sampleRate"] = self._sample_rate + return SamplingResult( + decision, + attributes, + _get_parent_trace_state(parent_context), + ) + + # pylint:disable=R0201 + def _get_DJB2_sample_score(self, trace_id_hex: str) -> int: + # This algorithm uses 32bit integers + hash_value = Int32(_HASH) + for char in trace_id_hex: + hash_value = ((hash_value << 5) + hash_value) + ord(char) + + if hash_value == _INTEGER_MIN: + hash_value = int(_INTEGER_MAX) + else: + hash_value = abs(hash_value) + + # divide by _INTEGER_MAX for value between 0 and 1 for sampling score + return float(hash_value) / _INTEGER_MAX + + + def get_description(self) -> str: + return "ApplicationInsightsSampler{}".format(self._ratio) diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/dev_requirements.txt b/sdk/monitor/azure-monitor-opentelemetry-exporter/dev_requirements.txt index 9938821516f1..39622ae3c9da 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/dev_requirements.txt +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/dev_requirements.txt @@ -2,4 +2,4 @@ -e ../../../tools/azure-sdk-tools ../../core/azure-core -e ../../identity/azure-identity -aiohttp>=3.0; python_version >= '3.5' +aiohttp>=3.0; python_version >= '3.7' diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/setup.py b/sdk/monitor/azure-monitor-opentelemetry-exporter/setup.py index 637ebec8eb24..a22400d0ca64 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/setup.py +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/setup.py @@ -81,6 +81,7 @@ python_requires=">=3.7", install_requires=[ "azure-core<2.0.0,>=1.23.0", + "fixedint==0.1.6", "msrest>=0.6.10", "opentelemetry-api<2.0.0,>=1.12.0", "opentelemetry-sdk<2.0.0,>=1.12.0", diff --git a/shared_requirements.txt b/shared_requirements.txt index c55e517be4f9..5513ec1bc086 100644 --- a/shared_requirements.txt +++ b/shared_requirements.txt @@ -108,6 +108,7 @@ azure-storage-blob~=1.3 azure-storage-file~=1.3 azure-storage-queue~=1.3 cryptography>=2.1.4 +fixedint==0.1.6 futures mock typing @@ -221,6 +222,7 @@ opentelemetry-sdk<2.0.0,>=1.5.0,!=1.10a0 #override azure-ai-metricsadvisor azure-core<2.0.0,>=1.23.0 #override azure-ai-translation-document azure-core<2.0.0,>=1.14.0 #override azure-monitor-opentelemetry-exporter azure-core<2.0.0,>=1.23.0 +#override azure-monitor-opentelemetry-exporter fixedint==0.1.6 #override azure-monitor-opentelemetry-exporter msrest>=0.6.10 #override azure-monitor-opentelemetry-exporter opentelemetry-api<2.0.0,>=1.12.0 #override azure-monitor-opentelemetry-exporter opentelemetry-sdk<2.0.0,>=1.12.0