Implement ApplicationInsightsSampler

sdk/monitor/azure-monitor-opentelemetry-exporter/CHANGELOG.md

@@ -16,6 +16,8 @@
     ([#26023](https://github.com/Azure/azure-sdk-for-python/pull/26023))
 - Implement statsbeat shutdown
     ([#26077](https://github.com/Azure/azure-sdk-for-python/pull/26077))
+- Add ApplicationInsightsSampler
+    ([#26224](https://github.com/Azure/azure-sdk-for-python/pull/26224))
 
 ### Breaking Changes
 

sdk/monitor/azure-monitor-opentelemetry-exporter/README.md

@@ -36,27 +36,26 @@ NOTE: The logging signal for the `AzureMonitorLogExporter` is currently in an EX
 
 ```Python
 from azure.monitor.opentelemetry.exporter import AzureMonitorLogExporter
-exporter = AzureMonitorLogExporter.from_connection_string(
-    conn_str = os.environ["APPLICATIONINSIGHTS_CONNECTION_STRING"]
+exporter = AzureMonitorLogExporter(
+    connection_string=os.environ["APPLICATIONINSIGHTS_CONNECTION_STRING"]
 )
 ```
 
 #### Metrics
 
 ```Python
 from azure.monitor.opentelemetry.exporter import AzureMonitorMetricExporter
-exporter = AzureMonitorMetricExporter.from_connection_string(
-    conn_str = os.environ["APPLICATIONINSIGHTS_CONNECTION_STRING"]
+exporter = AzureMonitorMetricExporter(
+    connection_string=os.environ["APPLICATIONINSIGHTS_CONNECTION_STRING"]
 )
-
 ```
 
 #### Tracing
 
 ```Python
 from azure.monitor.opentelemetry.exporter import AzureMonitorTraceExporter
-exporter = AzureMonitorTraceExporter.from_connection_string(
-    conn_str = os.environ["APPLICATIONINSIGHTS_CONNECTION_STRING"]
+exporter = AzureMonitorTraceExporter(
+    connection_string=os.environ["APPLICATIONINSIGHTS_CONNECTION_STRING"]
 )
 ```
 
@@ -127,11 +126,15 @@ Some of the key concepts for the Azure monitor exporter include:
 
 * [Sampling][sampler_ref]: Sampling is a mechanism to control the noise and overhead introduced by OpenTelemetry by reducing the number of samples of traces collected and sent to the backend.
 
+* ApplicationInsightsSampler: Application Insights specific sampler used for consistent sampling across Application Insights SDKs and OpenTelemetry-based SDKs sending data to Application Insights. This sampler MUST be used whenever `AzureMonitorTraceExporter` is used.
+
 For more information about these resources, see [What is Azure Monitor?][product_docs].
 
 ## Examples
 
-### Logging
+### Logging (experimental)
+
+NOTE: The logging signal for the `AzureMonitorLogExporter` is currently in an EXPERIMENTAL state. Possible breaking changes may ensue in the future.
 
 The following sections provide several code snippets covering some of the most common tasks, including:
 
@@ -158,8 +161,8 @@ from azure.monitor.opentelemetry.exporter import AzureMonitorLogExporter
 log_emitter_provider = LogEmitterProvider()
 set_log_emitter_provider(log_emitter_provider)
 
-exporter = AzureMonitorLogExporter.from_connection_string(
-    os.environ["APPLICATIONINSIGHTS_CONNECTION_STRING"]
+exporter = AzureMonitorLogExporter(
+    connection_string=os.environ["APPLICATIONINSIGHTS_CONNECTION_STRING"]
 )
 
 log_emitter_provider.add_log_processor(BatchLogProcessor(exporter))
@@ -196,8 +199,8 @@ tracer = trace.get_tracer(__name__)
 log_emitter_provider = LogEmitterProvider()
 set_log_emitter_provider(log_emitter_provider)
 
-exporter = AzureMonitorLogExporter.from_connection_string(
-    os.environ["APPLICATIONINSIGHTS_CONNECTION_STRING"]
+exporter = AzureMonitorLogExporter(
+    connection_string=os.environ["APPLICATIONINSIGHTS_CONNECTION_STRING"]
 )
 
 log_emitter_provider.add_log_processor(BatchLogProcessor(exporter))
@@ -233,8 +236,8 @@ from azure.monitor.opentelemetry.exporter import AzureMonitorLogExporter
 log_emitter_provider = LogEmitterProvider()
 set_log_emitter_provider(log_emitter_provider)
 
-exporter = AzureMonitorLogExporter.from_connection_string(
-    os.environ["APPLICATIONINSIGHTS_CONNECTION_STRING"]
+exporter = AzureMonitorLogExporter(
+    connection_string=os.environ["APPLICATIONINSIGHTS_CONNECTION_STRING"]
 )
 
 log_emitter_provider.add_log_processor(BatchLogProcessor(exporter))
@@ -272,8 +275,8 @@ from opentelemetry.sdk._logs.export import BatchLogProcessor
 from azure.monitor.opentelemetry.exporter import AzureMonitorLogExporter
 
 set_log_emitter_provider(LogEmitterProvider())
-exporter = AzureMonitorLogExporter.from_connection_string(
-    os.environ["APPLICATIONINSIGHTS_CONNECTION_STRING"]
+exporter = AzureMonitorLogExporter(
+    connection_string=os.environ["APPLICATIONINSIGHTS_CONNECTION_STRING"]
 )
 get_log_emitter_provider().add_log_processor(BatchLogProcessor(exporter))
 
@@ -324,8 +327,8 @@ from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
 
 from azure.monitor.opentelemetry.exporter import AzureMonitorMetricExporter
 
-exporter = AzureMonitorMetricExporter.from_connection_string(
-    os.environ["APPLICATIONINSIGHTS_CONNECTION_STRING"]
+exporter = AzureMonitorMetricExporter(
+    connection_string=os.environ["APPLICATIONINSIGHTS_CONNECTION_STRING"]
 )
 reader = PeriodicExportingMetricReader(exporter, export_interval_millis=5000)
 metrics.set_meter_provider(MeterProvider(metric_readers=[reader]))
@@ -381,6 +384,7 @@ The following sections provide several code snippets covering some of the most c
 
 * [Exporting a custom span](#export-hello-world-trace)
 * [Using an instrumentation to track a library](#instrumentation-with-requests-library)
+* [Enabling sampling to limit the amount of telemetry sent](#enabling-sampling)
 
 #### Export Hello World Trace
 
@@ -391,12 +395,12 @@ from opentelemetry.sdk.trace import TracerProvider
 from opentelemetry.sdk.trace.export import BatchSpanProcessor
 from azure.monitor.opentelemetry.exporter import AzureMonitorTraceExporter
 
-exporter = AzureMonitorTraceExporter.from_connection_string(
-    connection_string = os.environ["APPLICATIONINSIGHTS_CONNECTION_STRING "]
-)
-
 trace.set_tracer_provider(TracerProvider())
 tracer = trace.get_tracer(__name__)
+# This is the exporter that sends data to Application Insights
+exporter = AzureMonitorTraceExporter(
+    connection_string=os.environ["APPLICATIONINSIGHTS_CONNECTION_STRING"]
+)
 span_processor = BatchSpanProcessor(exporter)
 trace.get_tracer_provider().add_span_processor(span_processor)
 
@@ -419,27 +423,52 @@ from opentelemetry import trace
 from opentelemetry.instrumentation.requests import RequestsInstrumentor
 from opentelemetry.sdk.trace import TracerProvider
 from opentelemetry.sdk.trace.export import BatchSpanProcessor
-
 from azure.monitor.opentelemetry.exporter import AzureMonitorTraceExporter
 
-trace.set_tracer_provider(TracerProvider())
-tracer = trace.get_tracer(__name__)
-
 # This line causes your calls made with the requests library to be tracked.
 RequestsInstrumentor().instrument()
-span_processor = BatchSpanProcessor(
-    AzureMonitorTraceExporter.from_connection_string(
-        os.environ["APPLICATIONINSIGHTS_CONNECTION_STRING "]
-    )
+
+trace.set_tracer_provider(TracerProvider())
+tracer = trace.get_tracer(__name__)
+exporter = AzureMonitorTraceExporter(
+    connection_string=os.environ["APPLICATIONINSIGHTS_CONNECTION_STRING"]
 )
+span_processor = BatchSpanProcessor(exporter)
 trace.get_tracer_provider().add_span_processor(span_processor)
 
-RequestsInstrumentor().instrument()
-
 # This request will be traced
 response = requests.get(url="https://azure.microsoft.com/")
 ```
 
+#### Enabling sampling
+
+```Python
+import os
+from opentelemetry import trace
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor
+from azure.monitor.opentelemetry.exporter import (
+    ApplicationInsightsSampler,
+    AzureMonitorTraceExporter,
+)
+
+# Sampler expects a sample rate of between 0 and 1 inclusive
+# A rate of 0.75 means approximately 75% of your telemetry will be sent
+sampler = ApplicationInsightsSampler(0.75)
+trace.set_tracer_provider(TracerProvider(sampler=sampler))
+tracer = trace.get_tracer(__name__)
+exporter = AzureMonitorTraceExporter(
+    connection_string=os.environ["APPLICATIONINSIGHTS_CONNECTION_STRING"]
+)
+span_processor = BatchSpanProcessor(exporter)
+trace.get_tracer_provider().add_span_processor(span_processor)
+
+for i in range(100):
+    # Approximately 25% of these spans should be sampled out
+    with tracer.start_as_current_span("hello"):
+        print("Hello, World!")
+```
+
 ## Troubleshooting
 
 The exporter raises exceptions defined in [Azure Core](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/core/azure-core/README.md#azure-core-library-exceptions).

sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/__init__.py

@@ -7,9 +7,11 @@
 from azure.monitor.opentelemetry.exporter.export.logs._exporter import AzureMonitorLogExporter
 from azure.monitor.opentelemetry.exporter.export.metrics._exporter import AzureMonitorMetricExporter
 from azure.monitor.opentelemetry.exporter.export.trace._exporter import AzureMonitorTraceExporter
+from azure.monitor.opentelemetry.exporter.export.trace._sampling import ApplicationInsightsSampler
 from ._version import VERSION
 
 __all__ = [
+    "ApplicationInsightsSampler",
     "AzureMonitorMetricExporter",
     "AzureMonitorLogExporter",
     "AzureMonitorTraceExporter",

sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/export/trace/_exporter.py

@@ -421,7 +421,7 @@ def _convert_span_to_envelope(span: ReadableSpan) -> TelemetryItem:
         # Max key length is 150, value is 8192
         if not key or len(key) > 150 or val is None:
             continue
-        data.properties[key] = val[:8192]
+        data.properties[key] = str(val)[:8192]
     if span.links:
         # Max length for value is 8192
         # Since links are a fixed length (80) in json, max number of links would be 102

sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/export/trace/_sampling.py

@@ -0,0 +1,87 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+from typing import Optional, Sequence
+
+from fixedint import Int32
+# pylint:disable=W0611
+from opentelemetry.context import Context
+from opentelemetry.trace import Link, SpanKind, format_trace_id
+from opentelemetry.sdk.trace.sampling import (
+    Decision,
+    Sampler,
+    SamplingResult,
+    _get_parent_trace_state,
+)
+from opentelemetry.trace.span import TraceState
+from opentelemetry.util.types import Attributes
+
+
+_HASH = 5381
+_INTEGER_MAX = Int32.maxval
+_INTEGER_MIN = Int32.minval
+
+
+# Sampler is responsible for the following:
+# Implements same trace id hashing algorithm so that traces are sampled the same across multiple nodes (via AI SDKS)
+# Adds item count to span attribute if span is sampled (needed for ingestion service)
+# Inherits from the Sampler interface as defined by OpenTelemetry
+# https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/sdk.md#sampler
+class ApplicationInsightsSampler(Sampler):
+    """Sampler that implements the same probability sampling algorithm as the ApplicationInsights SDKs."""
+
+    # sampling_ratio takes values in the range [0,1]
+    def __init__(self, sampling_ratio: float = 1.0):
+        self._ratio = sampling_ratio
+        self._sample_rate = round(sampling_ratio * 100)
+
+    # pylint:disable=C0301
+    # See https://github.com/microsoft/Telemetry-Collection-Spec/blob/main/OpenTelemetry/trace/ApplicationInsightsSampler.md
+    def should_sample(
+        self,
+        parent_context: Optional[Context],
+        trace_id: int,
+        name: str,
+        kind: SpanKind = None,
+        attributes: Attributes = None,
+        links: Sequence["Link"] = None,
+        trace_state: "TraceState" = None,
+    ) -> "SamplingResult":
+        if self._sample_rate == 0:
+            decision = Decision.DROP
+        elif self._sample_rate == 100.0:
+            decision = Decision.RECORD_AND_SAMPLE
+        else:
+            # Determine if should sample from ratio and traceId
+            sample_score = self._get_DJB2_sample_score(format_trace_id(trace_id).lower())
+            if sample_score < self._ratio:
+                decision = Decision.RECORD_AND_SAMPLE
+            else:
+                decision = Decision.DROP
+        # Add sample rate as span attribute
+        if attributes is None:
+            attributes = {}
+        attributes["sampleRate"] = self._sample_rate
+        return SamplingResult(
+            decision,
+            attributes,
+            _get_parent_trace_state(parent_context),
+        )
+
+    # pylint:disable=R0201
+    def _get_DJB2_sample_score(self, trace_id_hex: str) -> int:
+        # This algorithm uses 32bit integers
+        hash_value = Int32(_HASH)
+        for char in trace_id_hex:
+            hash_value = ((hash_value << 5) + hash_value) + ord(char)
+
+        if hash_value == _INTEGER_MIN:
+            hash_value = int(_INTEGER_MAX)
+        else:
+            hash_value = abs(hash_value)
+
+        # divide by _INTEGER_MAX for value between 0 and 1 for sampling score
+        return float(hash_value) / _INTEGER_MAX
+
+
+    def get_description(self) -> str:
+        return "ApplicationInsightsSampler{}".format(self._ratio)

sdk/monitor/azure-monitor-opentelemetry-exporter/dev_requirements.txt

@@ -2,4 +2,4 @@
 -e ../../../tools/azure-sdk-tools
 ../../core/azure-core
 -e ../../identity/azure-identity
-aiohttp>=3.0; python_version >= '3.5'
+aiohttp>=3.0; python_version >= '3.7'

sdk/monitor/azure-monitor-opentelemetry-exporter/setup.py

@@ -81,6 +81,7 @@
     python_requires=">=3.7",
     install_requires=[
         "azure-core<2.0.0,>=1.23.0",
+        "fixedint==0.1.6",
         "msrest>=0.6.10",
         "opentelemetry-api<2.0.0,>=1.12.0",
         "opentelemetry-sdk<2.0.0,>=1.12.0",

shared_requirements.txt

@@ -108,6 +108,7 @@ azure-storage-blob~=1.3
 azure-storage-file~=1.3
 azure-storage-queue~=1.3
 cryptography>=2.1.4
+fixedint==0.1.6
 futures
 mock
 typing
@@ -221,6 +222,7 @@ opentelemetry-sdk<2.0.0,>=1.5.0,!=1.10a0
 #override azure-ai-metricsadvisor azure-core<2.0.0,>=1.23.0
 #override azure-ai-translation-document azure-core<2.0.0,>=1.14.0
 #override azure-monitor-opentelemetry-exporter azure-core<2.0.0,>=1.23.0
+#override azure-monitor-opentelemetry-exporter fixedint==0.1.6
 #override azure-monitor-opentelemetry-exporter msrest>=0.6.10
 #override azure-monitor-opentelemetry-exporter opentelemetry-api<2.0.0,>=1.12.0
 #override azure-monitor-opentelemetry-exporter opentelemetry-sdk<2.0.0,>=1.12.0