Add logging sampling

src/Libraries/Microsoft.AspNetCore.Diagnostics.Middleware/Microsoft.AspNetCore.Diagnostics.Middleware.csproj

@@ -17,7 +17,6 @@
     <InjectSharedPools>false</InjectSharedPools>
     <InjectSharedBufferWriterPool>true</InjectSharedBufferWriterPool>
     <InjectSharedNumericExtensions>false</InjectSharedNumericExtensions>
-    <InjectSharedDiagnosticIds>true</InjectSharedDiagnosticIds>
     <InjectExperimentalAttributeOnLegacy>true</InjectExperimentalAttributeOnLegacy>
   </PropertyGroup>
 

src/Libraries/Microsoft.Extensions.Http.Diagnostics/Microsoft.Extensions.Http.Diagnostics.csproj

@@ -14,8 +14,6 @@
     <UseMetricsReportsGenerator>true</UseMetricsReportsGenerator>
     <InjectGetOrAddOnLegacy>false</InjectGetOrAddOnLegacy>
     <InjectTaskWaitAsyncOnLegacy>true</InjectTaskWaitAsyncOnLegacy>
-    <InjectSharedDiagnosticIds>true</InjectSharedDiagnosticIds>
-    <InjectExperimentalAttributeOnLegacy>true</InjectExperimentalAttributeOnLegacy>
     <InjectTrimAttributesOnLegacy>false</InjectTrimAttributesOnLegacy>
     <InjectSharedDebugger>true</InjectSharedDebugger>
     <InjectSharedDataValidation>false</InjectSharedDataValidation>

src/Libraries/Microsoft.Extensions.Telemetry.Abstractions/Sampling/ILoggerSamplerFilterRule.cs

@@ -0,0 +1,36 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Diagnostics.CodeAnalysis;
+using Microsoft.Extensions.Logging;
+using Microsoft.Shared.DiagnosticIds;
+
+namespace Microsoft.Extensions.Diagnostics.Sampling;
+
+/// <summary>
+/// Represents a rule used for filtering log messages for purposes of log sampling and buffering.
+/// </summary>
+[Experimental(diagnosticId: DiagnosticIds.Experiments.Telemetry, UrlFormat = DiagnosticIds.UrlFormat)]
+public interface ILoggerSamplerFilterRule
+{
+    /// <summary>
+    /// Gets the logger category this rule applies to.
+    /// </summary>
+    public string? Category { get; }
+
+    /// <summary>
+    /// Gets the maximum <see cref="LogLevel"/> of messages.
+    /// </summary>
+    public LogLevel? LogLevel { get; }
+
+    /// <summary>
+    /// Gets the maximum <see cref="LogLevel"/> of messages where this rule applies to.
+    /// </summary>
+    public int? EventId { get; }
+
+    /// <summary>
+    /// Gets the filter delegate that would be applied to messages that passed the <see cref="LogLevel"/>.
+    /// </summary>
+    public Func<string?, LogLevel?, int?, bool>? Filter { get; }
+}

src/Libraries/Microsoft.Extensions.Telemetry.Abstractions/Sampling/LoggerSampler.cs

@@ -0,0 +1,23 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Diagnostics.CodeAnalysis;
+using Microsoft.Shared.DiagnosticIds;
+
+namespace Microsoft.Extensions.Logging;
+
+/// <summary>
+/// Controls the number of samples of log records collected and sent to the backend.
+/// </summary>
+#pragma warning disable S1694 // An abstract class should have both abstract and concrete methods
+[Experimental(diagnosticId: DiagnosticIds.Experiments.Telemetry, UrlFormat = DiagnosticIds.UrlFormat)]
+public abstract class LoggerSampler
+#pragma warning restore S1694 // An abstract class should have both abstract and concrete methods
+{
+    /// <summary>
+    /// Makes a sampling decision based on the provided <paramref name="parameters"/>.
+    /// </summary>
+    /// <param name="parameters">The parameters used to make the sampling decision.</param>
+    /// <returns><see langword="true" /> if the log record should be sampled; otherwise, <see langword="false" />.</returns>
+    public abstract bool ShouldSample(SamplingParameters parameters);
+}

src/Libraries/Microsoft.Extensions.Telemetry.Abstractions/Sampling/SamplingParameters.cs

@@ -0,0 +1,90 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Diagnostics.CodeAnalysis;
+using Microsoft.Shared.DiagnosticIds;
+
+namespace Microsoft.Extensions.Logging;
+
+/// <summary>
+/// Contains the parameters helping make sampling decisions for logs.
+/// </summary>
+[Experimental(diagnosticId: DiagnosticIds.Experiments.Telemetry, UrlFormat = DiagnosticIds.UrlFormat)]
+public readonly struct SamplingParameters : IEquatable<SamplingParameters>
+{
+    /// <summary>
+    /// Initializes a new instance of the <see cref="SamplingParameters"/> struct.
+    /// </summary>
+    /// <param name="logLevel"><see cref="Microsoft.Extensions.Logging.LogLevel"/> of the log record.</param>
+    /// <param name="category">Category of the log record.</param>
+    /// <param name="eventId"><see cref="Microsoft.Extensions.Logging.EventId"/> of the log record.</param>
+    public SamplingParameters(LogLevel logLevel, string category, EventId eventId)
+    {
+        LogLevel = logLevel;
+        Category = category;
+        EventId = eventId;
+    }
+
+    /// <summary>
+    /// Gets the log category.
+    /// </summary>
+    public string Category { get; }
+
+    /// <summary>
+    /// Gets the event ID.
+    /// </summary>
+    public EventId EventId { get; }
+
+    /// <summary>
+    /// Gets the log level.
+    /// </summary>
+    public LogLevel LogLevel { get; }
+
+    /// <inheritdoc/>
+    public override bool Equals(object? obj) =>
+        obj is SamplingParameters samplingParameters && Equals(samplingParameters);
+
+    /// <inheritdoc/>
+    public bool Equals(SamplingParameters other)
+    {
+        return other.EventId.Equals(EventId)
+            && string.Equals(other.Category, Category, StringComparison.Ordinal)
+            && other.LogLevel.Equals(LogLevel);
+    }
+
+    /// <inheritdoc/>
+    public override int GetHashCode() =>
+
+        HashCode.Combine(
+            LogLevel.GetHashCode(),
+#if NETFRAMEWORK
+            Category.GetHashCode(),
+#else
+            Category.GetHashCode(StringComparison.Ordinal),
+#endif
+            EventId.GetHashCode());
+
+    /// <summary>
+    /// Checks if two specified <see cref="SamplingParameters"/> instances have the same value.
+    /// They are equal if their respective properties have the same values.
+    /// </summary>
+    /// <param name="left">The first <see cref="SamplingParameters"/>.</param>
+    /// <param name="right">The second <see cref="SamplingParameters"/>.</param>
+    /// <returns><see langword="true" /> if the objects are equal.</returns>
+    public static bool operator ==(SamplingParameters left, SamplingParameters right)
+    {
+        return left.Equals(right);
+    }
+
+    /// <summary>
+    /// Checks if two specified <see cref="SamplingParameters"/> instances have different values.
+    /// </summary>
+    /// <param name="left">The first <see cref="SamplingParameters"/>.</param>
+    /// <param name="right">The second <see cref="SamplingParameters"/>.</param>
+    /// <returns><see langword="true" /> if the objects are not equal.</returns>
+    public static bool operator !=(SamplingParameters left, SamplingParameters right)
+    {
+        return !left.Equals(right);
+    }
+}

src/Libraries/Microsoft.Extensions.Telemetry/Logging/ExtendedLogger.cs

@@ -4,7 +4,6 @@
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
-using Microsoft.Extensions.Logging;
 using Microsoft.Shared.Pools;
 
 namespace Microsoft.Extensions.Logging;
@@ -15,7 +14,7 @@ namespace Microsoft.Extensions.Logging;
 //       redactor code calls recursively back into the logger. Don't do that.
 //
 // NOTE: Unlike the original logger in dotnet/runtime, this logger eats exceptions thrown from invoked loggers, enrichers,
-//       and redactors, rather than forwarding the exceptions to the caller. The fact an exception occured is recorded in
+//       and redactors, rather than forwarding the exceptions to the caller. The fact an exception occurred is recorded in
 //       the event log instead. The idea is that failures in the telemetry stack should not lead to failures in the
 //       application. It's better to keep running with missing telemetry rather than crashing the process completely.
 
@@ -39,6 +38,12 @@ public ExtendedLogger(ExtendedLoggerFactory factory, LoggerInformation[] loggers
 
     public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func<TState, Exception?, string> formatter)
     {
+        if (MessageLoggers.Length == 0 || !_factory.Config.Sampler.ShouldSample(new SamplingParameters(logLevel, MessageLoggers[0].Category, eventId)))
+        {
+            // the record was not selected for being sampled in, so we drop it.
+            return;
+        }
+
         if (typeof(TState) == typeof(LoggerMessageState))
         {
             var msgState = (LoggerMessageState?)(object?)state;
@@ -49,7 +54,7 @@ public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Except
             }
         }
 
-        LegacyPath<TState>(logLevel, eventId, state, exception, formatter);
+        LegacyPath(logLevel, eventId, state, exception, formatter);
     }
 
     public IDisposable? BeginScope<TState>(TState state)

src/Libraries/Microsoft.Extensions.Telemetry/Logging/ExtendedLoggerFactory.cs

@@ -8,6 +8,7 @@
 using Microsoft.Extensions.Compliance.Classification;
 using Microsoft.Extensions.Compliance.Redaction;
 using Microsoft.Extensions.Diagnostics.Enrichment;
+using Microsoft.Extensions.Diagnostics.Sampling;
 using Microsoft.Extensions.Options;
 using Microsoft.Shared.Diagnostics;
 
@@ -23,6 +24,7 @@ internal sealed class ExtendedLoggerFactory : ILoggerFactory
     private readonly IDisposable? _enrichmentOptionsChangeTokenRegistration;
     private readonly IDisposable? _redactionOptionsChangeTokenRegistration;
     private readonly Action<IEnrichmentTagCollector>[] _enrichers;
+    private readonly LoggerSampler _sampler;
     private readonly KeyValuePair<string, object?>[] _staticTags;
     private readonly Func<DataClassificationSet, Redactor> _redactorProvider;
     private volatile bool _disposed;
@@ -35,6 +37,7 @@ public ExtendedLoggerFactory(
         IEnumerable<ILogEnricher> enrichers,
         IEnumerable<IStaticLogEnricher> staticEnrichers,
         IOptionsMonitor<LoggerFilterOptions> filterOptions,
+        LoggerSampler? sampler = null,
         IOptions<LoggerFactoryOptions>? factoryOptions = null,
         IExternalScopeProvider? scopeProvider = null,
         IOptionsMonitor<LoggerEnrichmentOptions>? enrichmentOptions = null,
@@ -43,6 +46,7 @@ public ExtendedLoggerFactory(
 #pragma warning restore S107 // Methods should not have too many parameters
     {
         _scopeProvider = scopeProvider;
+        _sampler = sampler ?? new AlwaysOnSampler();
 
         _factoryOptions = factoryOptions == null || factoryOptions.Value == null ? new LoggerFactoryOptions() : factoryOptions.Value;
 
@@ -284,6 +288,7 @@ private LoggerConfig ComputeConfig(LoggerEnrichmentOptions? enrichmentOptions, L
 
         return new(_staticTags,
                 _enrichers,
+                _sampler,
                 enrichmentOptions.CaptureStackTraces,
                 enrichmentOptions.UseFileInfoForStackTraces,
                 enrichmentOptions.IncludeExceptionMessage,

src/Libraries/Microsoft.Extensions.Telemetry/Logging/Import/LoggerInformation.cs

@@ -17,7 +17,7 @@ namespace Microsoft.Extensions.Logging
 {
     internal readonly struct MessageLogger
     {
-        public MessageLogger(ILogger logger, string? category, string? providerTypeFullName, LogLevel? minLevel, Func<string?, string?, LogLevel, bool>? filter)
+        public MessageLogger(ILogger logger, string category, string? providerTypeFullName, LogLevel? minLevel, Func<string?, string?, LogLevel, bool>? filter)
         {
             Logger = logger;
             Category = category;
@@ -36,7 +36,7 @@ public MessageLogger(ILogger logger, string? category, string? providerTypeFullN
 
         public ILogger Logger { get; }
 
-        public string? Category { get; }
+        public string Category { get; }
 
         private string? ProviderTypeFullName { get; }
 

src/Libraries/Microsoft.Extensions.Telemetry/Logging/LoggerConfig.cs

@@ -15,6 +15,7 @@ internal sealed class LoggerConfig
     public LoggerConfig(
         KeyValuePair<string, object?>[] staticTags,
         Action<IEnrichmentTagCollector>[] enrichers,
+        LoggerSampler sampler,
         bool captureStackTraces,
         bool useFileInfoForStackTraces,
         bool includeExceptionMessage,
@@ -25,6 +26,7 @@ public LoggerConfig(
 #pragma warning restore S107 // Methods should not have too many parameters
         StaticTags = staticTags;
         Enrichers = enrichers;
+        Sampler = sampler;
         CaptureStackTraces = captureStackTraces;
         UseFileInfoForStackTraces = useFileInfoForStackTraces;
         MaxStackTraceLength = maxStackTraceLength;
@@ -35,6 +37,7 @@ public LoggerConfig(
 
     public KeyValuePair<string, object?>[] StaticTags { get; }
     public Action<IEnrichmentTagCollector>[] Enrichers { get; }
+    public LoggerSampler Sampler { get; }
     public bool CaptureStackTraces { get; }
     public bool UseFileInfoForStackTraces { get; }
     public bool IncludeExceptionMessage { get; }

src/Libraries/Microsoft.Extensions.Telemetry/Microsoft.Extensions.Telemetry.csproj

@@ -7,6 +7,8 @@
 
   <PropertyGroup>
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
+    <InjectExperimentalAttributeOnLegacy>true</InjectExperimentalAttributeOnLegacy>
+    <InjectSharedDiagnosticIds>true</InjectSharedDiagnosticIds>
     <InjectSharedText>true</InjectSharedText>
     <InjectSharedDataValidation>true</InjectSharedDataValidation>
     <InjectSharedNumericExtensions>true</InjectSharedNumericExtensions>

src/Libraries/Microsoft.Extensions.Telemetry/Sampling/AlwaysOnSampler.cs

@@ -0,0 +1,11 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using Microsoft.Extensions.Logging;
+
+namespace Microsoft.Extensions.Diagnostics.Sampling;
+
+internal sealed class AlwaysOnSampler : LoggerSampler
+{
+    public override bool ShouldSample(SamplingParameters _) => true;
+}

src/Libraries/Microsoft.Extensions.Telemetry/Sampling/FuncBasedSampler.cs

@@ -0,0 +1,20 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using Microsoft.Extensions.Logging;
+using Microsoft.Shared.Diagnostics;
+
+namespace Microsoft.Extensions.Diagnostics.Sampling;
+
+internal sealed class FuncBasedSampler : LoggerSampler
+{
+    private readonly Func<SamplingParameters, bool> _samplingDecisionFunc;
+
+    public FuncBasedSampler(Func<SamplingParameters, bool> samplingDecisionFunc)
+    {
+        _samplingDecisionFunc = Throw.IfNull(samplingDecisionFunc);
+    }
+
+    public override bool ShouldSample(SamplingParameters parameters) => _samplingDecisionFunc(parameters);
+}

src/Libraries/Microsoft.Extensions.Telemetry/Sampling/RatioBasedSampler.cs

@@ -0,0 +1,60 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+
+namespace Microsoft.Extensions.Diagnostics.Sampling;
+
+#pragma warning disable CA5394 // Do not use insecure randomness
+/// <summary>
+/// Samples logs according to the specified probability.
+/// </summary>
+internal sealed class RatioBasedSampler : LoggerSampler
+{
+#if !NET6_0_OR_GREATER
+    private static readonly System.Threading.ThreadLocal<Random> _randomInstance = new(() => new Random());
+#endif
+
+    private readonly IOptionsMonitor<RatioBasedSamplerOptions> _options;
+
+    /// <summary>
+    /// Initializes a new instance of the <see cref="RatioBasedSampler"/> class.
+    /// </summary>
+    public RatioBasedSampler(IOptionsMonitor<RatioBasedSamplerOptions> options)
+    {
+        _options = options;
+    }
+
+    /// <inheritdoc/>
+    public override bool ShouldSample(SamplingParameters parameters)
+    {
+        if (!TryApply(parameters, out var probability))
+        {
+            return true;
+        }
+
+#if NET6_0_OR_GREATER
+        return Random.Shared.Next(int.MaxValue) < int.MaxValue * probability;
+#else
+        return _randomInstance.Value!.Next(int.MaxValue) < int.MaxValue * probability;
+#endif
+    }
+
+    private bool TryApply(SamplingParameters parameters, out double probability)
+    {
+        probability = 0.0;
+
+        // TO DO: check if we can optimize this. It is a hot path and
+        // we should be able to minimize number of rule selections on every log record.
+        SamplerRuleSelector.Select(_options.CurrentValue.Rules, parameters.Category, parameters.LogLevel, parameters.EventId, out RatioBasedSamplerFilterRule? rule);
+        if (rule is not null)
+        {
+            probability = rule.Probability;
+            return true;
+        }
+
+        return false;
+    }
+}