diff --git a/src/DotNetWorker.ApplicationInsights/DotNetWorker.ApplicationInsights.csproj b/src/DotNetWorker.ApplicationInsights/DotNetWorker.ApplicationInsights.csproj index 39df83d84..19c727c74 100644 --- a/src/DotNetWorker.ApplicationInsights/DotNetWorker.ApplicationInsights.csproj +++ b/src/DotNetWorker.ApplicationInsights/DotNetWorker.ApplicationInsights.csproj @@ -6,9 +6,10 @@ Microsoft.Azure.Functions.Worker.ApplicationInsights Microsoft.Azure.Functions.Worker.ApplicationInsights 1 - 0 + 1 0 README.md + -preview1 $(BeforePack);GetReleaseNotes diff --git a/src/DotNetWorker.ApplicationInsights/FunctionsApplicationInsightsExtensions.cs b/src/DotNetWorker.ApplicationInsights/FunctionsApplicationInsightsExtensions.cs index a80496cb3..66c9e710b 100644 --- a/src/DotNetWorker.ApplicationInsights/FunctionsApplicationInsightsExtensions.cs +++ b/src/DotNetWorker.ApplicationInsights/FunctionsApplicationInsightsExtensions.cs @@ -26,6 +26,11 @@ public static IServiceCollection ConfigureFunctionsApplicationInsights(this ISer throw new ArgumentNullException(nameof(services)); } + services.AddSingleton, AppServiceOptionsInitializer>(); + services.AddSingleton(); + services.AddSingleton>(p => p.GetRequiredService()); + services.AddSingleton(p => p.GetRequiredService()); + services.AddSingleton(); services.TryAddEnumerable(ServiceDescriptor.Singleton()); diff --git a/src/DotNetWorker.ApplicationInsights/Initializers/AppServiceEnvironmentVariableMonitor.cs b/src/DotNetWorker.ApplicationInsights/Initializers/AppServiceEnvironmentVariableMonitor.cs new file mode 100644 index 000000000..419ba40fe --- /dev/null +++ b/src/DotNetWorker.ApplicationInsights/Initializers/AppServiceEnvironmentVariableMonitor.cs @@ -0,0 +1,75 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Primitives; + +namespace Microsoft.Azure.Functions.Worker.ApplicationInsights.Initializers; + +internal class AppServiceEnvironmentVariableMonitor : BackgroundService, IOptionsChangeTokenSource +{ + private readonly TimeSpan _refreshInterval; + + private IChangeToken _changeToken; + private CancellationTokenSource _cancellationTokenSource = new(); + + private readonly Dictionary _monitoredVariableCache = new(StringComparer.OrdinalIgnoreCase); + + public AppServiceEnvironmentVariableMonitor() : this(TimeSpan.FromSeconds(5)) + { + } + + public AppServiceEnvironmentVariableMonitor(TimeSpan refreshInterval) + { + _refreshInterval = refreshInterval; + _changeToken = new CancellationChangeToken(_cancellationTokenSource.Token); + } + + public string Name => string.Empty; + + public IChangeToken GetChangeToken() => _changeToken; + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + while (!stoppingToken.IsCancellationRequested) + { + bool changeDetected = false; + + foreach (string envVar in AppServiceOptionsInitializer.EnvironmentVariablesToMonitor) + { + string? currentVal = Environment.GetEnvironmentVariable(envVar); + _monitoredVariableCache.TryGetValue(envVar, out string? cachedVal); + + if (!string.Equals(currentVal, cachedVal, StringComparison.Ordinal)) + { + changeDetected = true; + _monitoredVariableCache[envVar] = currentVal; + } + } + + if (changeDetected) + { + var oldTokenSource = Interlocked.Exchange(ref _cancellationTokenSource, new CancellationTokenSource()); + Interlocked.Exchange(ref _changeToken, new CancellationChangeToken(_cancellationTokenSource.Token)); + + if (!oldTokenSource.IsCancellationRequested) + { + oldTokenSource.Cancel(); + oldTokenSource.Dispose(); + } + } + + try + { + await Task.Delay(_refreshInterval, stoppingToken); + } + catch (OperationCanceledException) + { + // happens during normal shutdown + break; + } + } + } +} diff --git a/src/DotNetWorker.ApplicationInsights/Initializers/AppServiceOptions.cs b/src/DotNetWorker.ApplicationInsights/Initializers/AppServiceOptions.cs new file mode 100644 index 000000000..a1998f43e --- /dev/null +++ b/src/DotNetWorker.ApplicationInsights/Initializers/AppServiceOptions.cs @@ -0,0 +1,11 @@ +namespace Microsoft.Azure.Functions.Worker.ApplicationInsights.Initializers; + +internal class AppServiceOptions +{ + public string? AzureWebsiteName { get; set; } + + public string? AzureWebsiteSlotName { get; set; } + + public string? AzureWebsiteCloudRoleName { get; set; } +} + diff --git a/src/DotNetWorker.ApplicationInsights/Initializers/AppServiceOptionsInitializer.cs b/src/DotNetWorker.ApplicationInsights/Initializers/AppServiceOptionsInitializer.cs new file mode 100644 index 000000000..005408b43 --- /dev/null +++ b/src/DotNetWorker.ApplicationInsights/Initializers/AppServiceOptionsInitializer.cs @@ -0,0 +1,37 @@ +using System; +using Microsoft.Azure.Functions.Worker.ApplicationInsights.Initializers; +using Microsoft.Extensions.Options; + +internal class AppServiceOptionsInitializer : IConfigureOptions +{ + internal const string AzureWebsiteName = "WEBSITE_SITE_NAME"; + internal const string AzureWebsiteSlotName = "WEBSITE_SLOT_NAME"; + internal const string AzureWebsiteCloudRoleName = "WEBSITE_CLOUD_ROLENAME"; + internal const string DefaultProductionSlotName = "production"; + + internal static string[] EnvironmentVariablesToMonitor = new[] { AzureWebsiteName, AzureWebsiteSlotName, AzureWebsiteCloudRoleName }; + + public void Configure(AppServiceOptions options) + { + options.AzureWebsiteName = Environment.GetEnvironmentVariable(AzureWebsiteName); + options.AzureWebsiteCloudRoleName = Environment.GetEnvironmentVariable(AzureWebsiteCloudRoleName); + + // Compute the slot name by appending non-production slot to the site name (i.e. mysite-staging) + string slotName = Environment.GetEnvironmentVariable(AzureWebsiteSlotName); + options.AzureWebsiteSlotName = GetAzureWebsiteUniqueSlotName(options.AzureWebsiteName, slotName); + } + + /// + /// Gets a value that uniquely identifies the site and slot. + /// + private static string? GetAzureWebsiteUniqueSlotName(string? websiteName, string? slotName) + { + if (!string.IsNullOrEmpty(slotName) && + !string.Equals(slotName, DefaultProductionSlotName, StringComparison.OrdinalIgnoreCase)) + { + websiteName += $"-{slotName}"; + } + + return websiteName?.ToLowerInvariant(); + } +} diff --git a/src/DotNetWorker.ApplicationInsights/Initializers/FunctionsRoleEnvironmentTelmetryInitializer.cs b/src/DotNetWorker.ApplicationInsights/Initializers/FunctionsRoleEnvironmentTelmetryInitializer.cs index 91d7f02f2..90a5e7b7d 100644 --- a/src/DotNetWorker.ApplicationInsights/Initializers/FunctionsRoleEnvironmentTelmetryInitializer.cs +++ b/src/DotNetWorker.ApplicationInsights/Initializers/FunctionsRoleEnvironmentTelmetryInitializer.cs @@ -6,6 +6,7 @@ using Microsoft.ApplicationInsights.Channel; using Microsoft.ApplicationInsights.Extensibility; using Microsoft.ApplicationInsights.Extensibility.Implementation; +using Microsoft.Extensions.Options; namespace Microsoft.Azure.Functions.Worker.ApplicationInsights.Initializers { @@ -17,13 +18,15 @@ namespace Microsoft.Azure.Functions.Worker.ApplicationInsights.Initializers /// internal class FunctionsRoleEnvironmentTelemetryInitializer : ITelemetryInitializer { - internal const string AzureWebsiteName = "WEBSITE_SITE_NAME"; - internal const string AzureWebsiteSlotName = "WEBSITE_SLOT_NAME"; - internal const string AzureWebsiteCloudRoleName = "WEBSITE_CLOUD_ROLENAME"; - private const string DefaultProductionSlotName = "production"; - private const string WebAppSuffix = ".azurewebsites.net"; + internal const string WebAppSuffix = ".azurewebsites.net"; private readonly ConcurrentDictionary _siteNodeNames = new(StringComparer.OrdinalIgnoreCase); + private readonly IOptionsMonitor _appServiceOptions; + + public FunctionsRoleEnvironmentTelemetryInitializer(IOptionsMonitor appServiceOptions) + { + _appServiceOptions = appServiceOptions; + } /// /// Initializes device context. @@ -36,49 +39,27 @@ public void Initialize(ITelemetry telemetry) return; } - var siteSlotName = new Lazy(() => - { - // We cannot cache these values as the environment variables can change on the fly. - return GetAzureWebsiteUniqueSlotName(); - }); - - var websiteCloudRoleName = Environment.GetEnvironmentVariable(AzureWebsiteCloudRoleName); + var options = _appServiceOptions.CurrentValue; - if (!string.IsNullOrEmpty(websiteCloudRoleName)) + if (!string.IsNullOrEmpty(options.AzureWebsiteCloudRoleName)) { - telemetry.Context.Cloud.RoleName = websiteCloudRoleName; + telemetry.Context.Cloud.RoleName = options.AzureWebsiteCloudRoleName; } else { - telemetry.Context.Cloud.RoleName = siteSlotName.Value; + telemetry.Context.Cloud.RoleName = options.AzureWebsiteSlotName; } var internalContext = telemetry.Context.GetInternalContext(); - if (!string.IsNullOrEmpty(siteSlotName.Value)) + if (!string.IsNullOrEmpty(options.AzureWebsiteSlotName)) { - internalContext.NodeName = _siteNodeNames.GetOrAdd(siteSlotName.Value!, p => + internalContext.NodeName = _siteNodeNames.GetOrAdd(options.AzureWebsiteSlotName!, p => { // maintain previous behavior of node having the full url return p += WebAppSuffix; }); } } - - /// - /// Gets a value that uniquely identifies the site and slot. - /// - private static string? GetAzureWebsiteUniqueSlotName() - { - var name = Environment.GetEnvironmentVariable(AzureWebsiteName); - var slotName = Environment.GetEnvironmentVariable(AzureWebsiteSlotName); - - if (!string.IsNullOrEmpty(slotName) && - !string.Equals(slotName, DefaultProductionSlotName, StringComparison.OrdinalIgnoreCase)) - { - name += $"-{slotName}"; - } - - return name?.ToLowerInvariant(); - } } } + diff --git a/src/DotNetWorker.ApplicationInsights/release_notes.md b/src/DotNetWorker.ApplicationInsights/release_notes.md index 760bc777b..c5932c681 100644 --- a/src/DotNetWorker.ApplicationInsights/release_notes.md +++ b/src/DotNetWorker.ApplicationInsights/release_notes.md @@ -1,3 +1,3 @@ ## What's Changed -- GA release (no functional changes) \ No newline at end of file +- Moving hot-path environment variable checks to a background task (#1996) \ No newline at end of file diff --git a/test/DotNetWorkerTests/ApplicationInsights/EndToEndTests.cs b/test/DotNetWorkerTests/ApplicationInsights/EndToEndTests.cs index b9714011c..b5eb82701 100644 --- a/test/DotNetWorkerTests/ApplicationInsights/EndToEndTests.cs +++ b/test/DotNetWorkerTests/ApplicationInsights/EndToEndTests.cs @@ -3,9 +3,14 @@ using System.Collections.Immutable; using System.Diagnostics; using System.Linq; +using System.Threading; using System.Threading.Tasks; +using Microsoft.ApplicationInsights; using Microsoft.ApplicationInsights.Channel; using Microsoft.ApplicationInsights.DataContracts; +using Microsoft.ApplicationInsights.Extensibility.Implementation; +using Microsoft.Azure.Functions.Tests; +using Microsoft.Azure.Functions.Worker.ApplicationInsights.Initializers; using Microsoft.Azure.Functions.Worker.Context.Features; using Microsoft.Azure.Functions.Worker.Diagnostics; using Microsoft.Azure.Functions.Worker.Tests.Features; @@ -17,57 +22,70 @@ namespace Microsoft.Azure.Functions.Worker.Tests.ApplicationInsights; -public class EndToEndTests : IDisposable +public class EndToEndTests { + private const string RoleName = "RoleName"; + private readonly TestTelemetryChannel _channel; - private readonly IHost _host; - private readonly IFunctionsApplication _application; - private readonly IInvocationFeaturesFactory _invocationFeaturesFactory; + private IFunctionsApplication _application; + private IInvocationFeaturesFactory _invocationFeaturesFactory; private readonly AppInsightsFunctionDefinition _funcDefinition = new(); public EndToEndTests() { _channel = new TestTelemetryChannel(); + } - _host = new HostBuilder() - .ConfigureServices(services => - { - var functionsBuilder = services.AddFunctionsWorkerCore(); - functionsBuilder.UseDefaultWorkerMiddleware(); - - services.AddApplicationInsightsTelemetryWorkerService(options => - { + private IHost InitializeHost() + { + var host = new HostBuilder() + .ConfigureServices(services => + { + var functionsBuilder = services.AddFunctionsWorkerCore(); + functionsBuilder.UseDefaultWorkerMiddleware(); + + services.AddApplicationInsightsTelemetryWorkerService(options => + { #pragma warning disable CS0618 // Obsolete member. Test case, this is fine to use. - options.InstrumentationKey = "abc"; + options.InstrumentationKey = "abc"; #pragma warning restore CS0618 // Obsolete member. Test case, this is fine to use. - // keep things more deterministic for tests - options.EnableAdaptiveSampling = false; - options.EnableDependencyTrackingTelemetryModule = false; - }); + // keep things more deterministic for tests + options.EnableAdaptiveSampling = false; + options.EnableDependencyTrackingTelemetryModule = false; + options.EnablePerformanceCounterCollectionModule = false; + options.EnableEventCounterCollectionModule = false; + options.EnableHeartbeat = false; + }); - services.ConfigureFunctionsApplicationInsights(); - services.AddDefaultInputConvertersToWorkerOptions(); + services.ConfigureFunctionsApplicationInsights(); - // Register our own in-memory channel - services.AddSingleton(_channel); - services.AddSingleton(_ => new Mock().Object); - }) - .Build(); + // override this so tests don't have to wait + services.AddSingleton(p => new AppServiceEnvironmentVariableMonitor(TimeSpan.FromMilliseconds(50))); - _application = _host.Services.GetService(); - _invocationFeaturesFactory = _host.Services.GetService(); + services.AddDefaultInputConvertersToWorkerOptions(); + + // Register our own in-memory channel + services.AddSingleton(_channel); + services.AddSingleton(_ => new Mock().Object); + }) + .Build(); + + _application = host.Services.GetService(); + _invocationFeaturesFactory = host.Services.GetService(); _application.LoadFunction(_funcDefinition); + + return host; } - private FunctionContext CreateContext() + private FunctionContext CreateContext(IHost host) { var invocation = new TestFunctionInvocation(functionId: _funcDefinition.Id); var features = _invocationFeaturesFactory.Create(); features.Set(invocation); - var inputConversionProvider = _host.Services.GetRequiredService(); + var inputConversionProvider = host.Services.GetRequiredService(); inputConversionProvider.TryCreate(typeof(DefaultInputConversionFeature), out var inputConversion); features.Set(new TestFunctionBindingsFeature()); features.Set(inputConversion); @@ -78,7 +96,10 @@ private FunctionContext CreateContext() [Fact] public async Task Logger_SendsTraceAndDependencyTelemetry() { - var context = CreateContext(); + using var _ = SetupDefaultEnvironmentVariables(); + using var host = InitializeHost(); + + var context = CreateContext(host); await _application.InvokeFunctionAsync(context); @@ -94,7 +115,10 @@ public async Task Logger_SendsTraceAndDependencyTelemetry() [Fact] public async Task Logger_Exception_SendsTraceAndExceptionAndDependencyTelemetry() { - var context = CreateContext(); + using var _ = SetupDefaultEnvironmentVariables(); + using var host = InitializeHost(); + + var context = CreateContext(host); context.Items["_throw"] = true; await Assert.ThrowsAsync(() => _application.InvokeFunctionAsync(context)); @@ -109,6 +133,40 @@ public async Task Logger_Exception_SendsTraceAndExceptionAndDependencyTelemetry( t => ValidateTraceTelemetry((TraceTelemetry)t, context, activity)); } + [Fact] + public async Task Telemetry_Updates_After_Swap() + { + // Env vars can change during a swap. These should automatically update. + using var _ = SetupDefaultEnvironmentVariables(); + using var host = InitializeHost(); + + // these tests don't use a running host; explicitly start this hosted service + var monitor = host.Services.GetRequiredService(); + await monitor.StartAsync(CancellationToken.None); + + var telemetryClient = host.Services.GetService(); + telemetryClient.TrackTrace("before swap"); + + using var swapped = new TestScopedEnvironmentVariable(new Dictionary + { + { AppServiceOptionsInitializer.AzureWebsiteName, "SwappedRoleName" }, + { AppServiceOptionsInitializer.AzureWebsiteSlotName, "staging" } + }); + + // ensure the monitor has refreshed env var cache + await Task.Delay(100); + + telemetryClient.TrackTrace("after swap"); + + IEnumerable telemetries = await WaitForTelemetries(expectedCount: 2); + Assert.Equal(2, telemetries.Count()); + var beforeSwap = telemetries.Last(); + var afterSwap = telemetries.First(); + + ValidateCommonTelemetry(beforeSwap); + ValidateCommonTelemetry(afterSwap, "SwappedRoleName-staging"); + } + private async Task> WaitForTelemetries(int expectedCount) { IEnumerable telemetries = null; @@ -133,6 +191,8 @@ private static void ValidateDependencyTelemetry(DependencyTelemetry dependency, Assert.Equal(context.InvocationId, dependency.Properties[TraceConstants.AttributeFaasExecution]); Assert.Contains(TraceConstants.AttributeSchemaUrl, dependency.Properties.Keys); + + ValidateCommonTelemetry(dependency); } private static void ValidateTraceTelemetry(TraceTelemetry trace, FunctionContext context, Activity activity) @@ -145,6 +205,8 @@ private static void ValidateTraceTelemetry(TraceTelemetry trace, FunctionContext Assert.Equal(context.InvocationId, trace.Properties[FunctionInvocationScope.FunctionInvocationIdKey]); Assert.Equal(activity.RootId, trace.Context.Operation.Id); + + ValidateCommonTelemetry(trace); } private static void ValidateExceptionTelemetry(ExceptionTelemetry exception, FunctionContext context, Activity activity) @@ -157,10 +219,28 @@ private static void ValidateExceptionTelemetry(ExceptionTelemetry exception, Fun Assert.Contains("boom!", edi.Message); Assert.Equal(activity.RootId, exception.Context.Operation.Id); + + ValidateCommonTelemetry(exception); + } + + private static void ValidateCommonTelemetry(ITelemetry telemetry, string expectedSiteName = RoleName) + { + // tests will set this when swapping out env vars + var internalContext = telemetry.Context.GetInternalContext(); + + expectedSiteName = expectedSiteName.ToLowerInvariant(); + + Assert.Equal(expectedSiteName, telemetry.Context.Cloud.RoleName); + Assert.Equal($"{expectedSiteName}{FunctionsRoleEnvironmentTelemetryInitializer.WebAppSuffix}", internalContext.NodeName); } - public void Dispose() + + private static IDisposable SetupDefaultEnvironmentVariables() { - _host?.Dispose(); + return new TestScopedEnvironmentVariable(new Dictionary + { + { AppServiceOptionsInitializer.AzureWebsiteName, RoleName }, + { AppServiceOptionsInitializer.AzureWebsiteSlotName, AppServiceOptionsInitializer.DefaultProductionSlotName } + }); } internal class AppInsightsFunctionDefinition : FunctionDefinition diff --git a/test/DotNetWorkerTests/DotNetWorkerTests.csproj b/test/DotNetWorkerTests/DotNetWorkerTests.csproj index d488afe14..75956fc5b 100644 --- a/test/DotNetWorkerTests/DotNetWorkerTests.csproj +++ b/test/DotNetWorkerTests/DotNetWorkerTests.csproj @@ -9,6 +9,7 @@ preview ..\..\key.snk disable + true