From e0cf99ae2dc1768234f488248b34809102a047ab Mon Sep 17 00:00:00 2001 From: Vishwesh Bankwar Date: Tue, 19 Sep 2023 11:08:33 -0700 Subject: [PATCH 1/3] Fix offline storage during network errors --- .../src/Internals/AzureMonitorTransmitter.cs | 2 +- .../src/Internals/TransmissionStateManager.cs | 2 +- .../Internals/TransmitFromStorageHandler.cs | 2 +- .../OfflineStorageTests.cs | 50 +++++++++++++++---- 4 files changed, 44 insertions(+), 12 deletions(-) diff --git a/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/src/Internals/AzureMonitorTransmitter.cs b/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/src/Internals/AzureMonitorTransmitter.cs index 4871be2fc1f1..0e7a0fc62a47 100644 --- a/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/src/Internals/AzureMonitorTransmitter.cs +++ b/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/src/Internals/AzureMonitorTransmitter.cs @@ -178,7 +178,7 @@ await _applicationInsightsRestClient.InternalTrackAsync(telemetryItems, cancella if (result == ExportResult.Failure && _fileBlobProvider != null) { - _transmissionStateManager.EnableBackOff(httpMessage.Response); + _transmissionStateManager.EnableBackOff(httpMessage.HasResponse ? httpMessage.Response : null); result = HttpPipelineHelper.HandleFailures(httpMessage, _fileBlobProvider, _connectionVars, origin, _isAadEnabled); } else diff --git a/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/src/Internals/TransmissionStateManager.cs b/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/src/Internals/TransmissionStateManager.cs index ebc167185adb..91be184d57a3 100644 --- a/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/src/Internals/TransmissionStateManager.cs +++ b/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/src/Internals/TransmissionStateManager.cs @@ -96,7 +96,7 @@ internal void ResetTransmission(object? source, System.Timers.ElapsedEventArgs e CloseTransmission(); } - internal void EnableBackOff(Response response) + internal void EnableBackOff(Response? response) { if (Interlocked.Exchange(ref _syncBackOffIntervalCalculation, 1) == 0) { diff --git a/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/src/Internals/TransmitFromStorageHandler.cs b/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/src/Internals/TransmitFromStorageHandler.cs index 43587e481f73..e8ab78c8a5c0 100644 --- a/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/src/Internals/TransmitFromStorageHandler.cs +++ b/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/src/Internals/TransmitFromStorageHandler.cs @@ -69,7 +69,7 @@ internal void TransmitFromStorage(object? sender, ElapsedEventArgs? e) } else { - _transmissionStateManager.EnableBackOff(httpMessage.Response); + _transmissionStateManager.EnableBackOff(httpMessage.HasResponse ? httpMessage.Response : null); HttpPipelineHelper.HandleFailures(httpMessage, blob, _blobProvider, _connectionVars, _isAadEnabled); break; } diff --git a/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/tests/Azure.Monitor.OpenTelemetry.Exporter.Tests/OfflineStorageTests.cs b/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/tests/Azure.Monitor.OpenTelemetry.Exporter.Tests/OfflineStorageTests.cs index 4b9e21cc8293..3780a3f4d917 100644 --- a/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/tests/Azure.Monitor.OpenTelemetry.Exporter.Tests/OfflineStorageTests.cs +++ b/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/tests/Azure.Monitor.OpenTelemetry.Exporter.Tests/OfflineStorageTests.cs @@ -98,6 +98,23 @@ public void FailureResponseCode429() Assert.Single(transmitter._fileBlobProvider.GetBlobs()); } + [Fact] + public void NetworkFailure() + { + using var activity = CreateActivity("TestActivity"); + var telemetryItem = CreateTelemetryItem(activity); + List telemetryItems = new List(); + telemetryItems.Add(telemetryItem); + + // Transmit + using var transmitter = GetTransmitter(null); + transmitter.TrackAsync(telemetryItems, TelemetryItemOrigin.UnitTest, false, CancellationToken.None).EnsureCompleted(); + + //Assert + Assert.NotNull(transmitter._fileBlobProvider); + Assert.Single(transmitter._fileBlobProvider.GetBlobs()); + } + [Fact] public void FailureResponseCode206() { @@ -260,17 +277,32 @@ public void TelemetryIsStoredOfflineWhenTransmissionStateIsSetToOpen() transmitter.Dispose(); } - private static AzureMonitorTransmitter GetTransmitter(params MockResponse[] mockResponse) + private static AzureMonitorTransmitter GetTransmitter(params MockResponse[]? mockResponse) { - MockTransport mockTransport = new MockTransport(mockResponse); - AzureMonitorExporterOptions options = new AzureMonitorExporterOptions + AzureMonitorTransmitter transmitter; + AzureMonitorExporterOptions options; + if (mockResponse == null) { - ConnectionString = $"InstrumentationKey={testIkey};IngestionEndpoint={testEndpoint}", - StorageDirectory = "C:\\test", - Transport = mockTransport, - EnableStatsbeat = false, // disabled in tests. - }; - AzureMonitorTransmitter transmitter = new AzureMonitorTransmitter(options, new MockPlatform()); + options = new AzureMonitorExporterOptions + { + ConnectionString = $"InstrumentationKey={testIkey};IngestionEndpoint={testEndpoint}", + StorageDirectory = "C:\\test", + EnableStatsbeat = false, // disabled in tests. + }; + } + else + { + MockTransport mockTransport = new MockTransport(mockResponse); + options = new AzureMonitorExporterOptions + { + ConnectionString = $"InstrumentationKey={testIkey};IngestionEndpoint={testEndpoint}", + StorageDirectory = "C:\\test", + Transport = mockTransport, + EnableStatsbeat = false, // disabled in tests. + }; + } + + transmitter = new AzureMonitorTransmitter(options, new MockPlatform()); // Overwrite storage with mock transmitter._fileBlobProvider = new MockFileProvider(); From af71cd079133be647a3bba5d50fcca295dee5fcc Mon Sep 17 00:00:00 2001 From: Vishwesh Bankwar Date: Tue, 19 Sep 2023 11:44:41 -0700 Subject: [PATCH 2/3] fix net462 --- .../src/Customizations/ApplicationInsightsRestClient.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/src/Customizations/ApplicationInsightsRestClient.cs b/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/src/Customizations/ApplicationInsightsRestClient.cs index cb76b4a30bb0..d20d439bf9ae 100644 --- a/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/src/Customizations/ApplicationInsightsRestClient.cs +++ b/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/src/Customizations/ApplicationInsightsRestClient.cs @@ -42,7 +42,7 @@ internal async Task InternalTrackAsync(IEnumerable b catch (Exception ex) { AzureMonitorExporterEventSource.Log.FailedToTransmit(ex); - if (ex.InnerException?.Source != "System.Net.Http") + if (ex.InnerException?.Source != "System.Net.Http" && ex.InnerException?.Source != "System") { message?.Dispose(); throw; @@ -70,7 +70,7 @@ internal async Task InternalTrackAsync(ReadOnlyMemory body, C catch (Exception ex) { AzureMonitorExporterEventSource.Log.FailedToTransmit(ex); - if (ex.InnerException?.Source != "System.Net.Http") + if (ex.InnerException?.Source != "System.Net.Http" && ex.InnerException?.Source != "System") { message?.Dispose(); throw; From c436dc156fb72c7f2e4c89ea361f74f7db0961cb Mon Sep 17 00:00:00 2001 From: Vishwesh Bankwar Date: Tue, 19 Sep 2023 11:56:54 -0700 Subject: [PATCH 3/3] changelog --- sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/CHANGELOG.md b/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/CHANGELOG.md index d16f451db429..dafad9560cf4 100644 --- a/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/CHANGELOG.md +++ b/sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/CHANGELOG.md @@ -8,6 +8,10 @@ ### Bugs Fixed +* Fixed an issue during network failures which prevented the exporter to store +the telemetry offline for retrying at a later time. +([#38832](https://github.com/Azure/azure-sdk-for-net/pull/38832)) + ### Other Changes * Update OpenTelemetry dependencies