From 1280d1b6170702e75d92fe5c4073f6f0e5058146 Mon Sep 17 00:00:00 2001 From: R9 Fundamentals Date: Wed, 5 Jun 2024 07:10:32 +0200 Subject: [PATCH 01/35] initial proposal --- LoggerWithSampling/Log.cs | 16 ++++ LoggerWithSampling/LoggerWithSampling.csproj | 13 +++ LoggerWithSampling/Program.cs | 92 +++++++++++++++++++ .../Logging/ExtendedLogger.cs | 18 ++++ .../Logging/ExtendedLoggerFactory.cs | 7 +- .../Logging/LoggerConfig.cs | 4 + .../Logging/Sampling/BufferingTool.cs | 33 +++++++ .../Logging/Sampling/ControlAction.cs | 30 ++++++ .../Logging/Sampling/ILoggingSampler.cs | 17 ++++ .../Sampling/ILoggingSamplingBuilder.cs | 17 ++++ .../Logging/Sampling/LogBuffer.cs | 32 +++++++ .../Logging/Sampling/LogRecordPattern.cs | 39 ++++++++ .../Logging/Sampling/LogSamplingOptions.cs | 50 ++++++++++ .../Sampling/LoggingSamplingBuilder.cs | 16 ++++ .../Sampling/LoggingSamplingExtensions.cs | 59 ++++++++++++ .../Logging/Sampling/Matcher.cs | 83 +++++++++++++++++ .../Logging/Sampling/SimpleSampler.cs | 48 ++++++++++ 17 files changed, 573 insertions(+), 1 deletion(-) create mode 100644 LoggerWithSampling/Log.cs create mode 100644 LoggerWithSampling/LoggerWithSampling.csproj create mode 100644 LoggerWithSampling/Program.cs create mode 100644 src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/BufferingTool.cs create mode 100644 src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/ControlAction.cs create mode 100644 src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/ILoggingSampler.cs create mode 100644 src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/ILoggingSamplingBuilder.cs create mode 100644 src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/LogBuffer.cs create mode 100644 src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/LogRecordPattern.cs create mode 100644 src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/LogSamplingOptions.cs create mode 100644 src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/LoggingSamplingBuilder.cs create mode 100644 src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/LoggingSamplingExtensions.cs create mode 100644 src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/Matcher.cs create mode 100644 src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/SimpleSampler.cs diff --git a/LoggerWithSampling/Log.cs b/LoggerWithSampling/Log.cs new file mode 100644 index 00000000000..88110f9bd20 --- /dev/null +++ b/LoggerWithSampling/Log.cs @@ -0,0 +1,16 @@ +// Copyright (c) Microsoft Corporation. All Rights Reserved. + +using System; +using Microsoft.Extensions.Logging; + +namespace ConsoleLogger +{ + internal static partial class Log + { + [LoggerMessage(1, LogLevel.Information, "Learn more about \"{topic}\" at \"{docLocation}\"")] + public static partial void LearnMoreAt(this ILogger logger, string topic, string docLocation); + + [LoggerMessage(2, LogLevel.Error, "An exception of type ArgumentNullException was thrown")] + public static partial void LogArgumentNullException(this ILogger logger, Exception exception); + } +} diff --git a/LoggerWithSampling/LoggerWithSampling.csproj b/LoggerWithSampling/LoggerWithSampling.csproj new file mode 100644 index 00000000000..295b7feb155 --- /dev/null +++ b/LoggerWithSampling/LoggerWithSampling.csproj @@ -0,0 +1,13 @@ + + + + Sample for R9 Console logger support. + Exe + net8.0 + + + + + + + \ No newline at end of file diff --git a/LoggerWithSampling/Program.cs b/LoggerWithSampling/Program.cs new file mode 100644 index 00000000000..2f456e58139 --- /dev/null +++ b/LoggerWithSampling/Program.cs @@ -0,0 +1,92 @@ +// Copyright (c) Microsoft Corporation. All Rights Reserved. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Security.Principal; +using Microsoft.Extensions.Diagnostics.Logging.Sampling; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace ConsoleLogger +{ + internal static class Program + { + private static void Main() + { + LoggerDemo("Structured Logging", formatMessage: false); + LoggerDemo("Unstructured Logging", formatMessage: true); + } + + private static void LoggerDemo(string loggerCategory, bool formatMessage) + { + Console.WriteLine($"*** {loggerCategory} demo ***"); + + using ILoggerFactory loggerFactory = CreateLoggerFactory(formatMessage); + ILogger logger = loggerFactory.CreateLogger(loggerCategory); + + logger.LearnMoreAt("R9 logger", "aka.ms/r9"); + + using (logger.BeginScope("Exception demo")) + { + try + { + AssertNotNull(null); + } + catch (ArgumentNullException e) + { + logger.LogArgumentNullException(e); + } + } + + Console.WriteLine(); + } + + private static ILoggerFactory CreateLoggerFactory(bool formatMessage) + { + return LoggerFactory.Create(builder => + { + _ = builder.EnableSampling(samplingBuilder => samplingBuilder + + // add the built-in simple sampler: + .SetSimpleSampler(o => + { + o.Matchers = new List + { + new Matcher( + new LogRecordPattern + { + LogLevel = LogLevel.Information, + Category = "Microsoft.Extensions.Hosting", + }, + (pattern) => Random.Shared.NextDouble() < 0.01), + new Matcher( + new LogRecordPattern + { + LogLevel = LogLevel.Error, + }, + (tool, pattern) => tool.Buffer("MyBuffer")), + }; + o.Buffers = new HashSet + { + new LogBuffer + { + Name = "MyBuffer", + SuspendAfterFlushDuration = TimeSpan.FromSeconds(10), + BufferingDuration = TimeSpan.FromSeconds(10), + BufferSize = 1_000_000, + }, + }; + })); + }); + } + + private static void AssertNotNull(object? obj) + { + if (obj == null) + { + throw new ArgumentNullException(nameof(obj)); + } + } + } +} diff --git a/src/Libraries/Microsoft.Extensions.Telemetry/Logging/ExtendedLogger.cs b/src/Libraries/Microsoft.Extensions.Telemetry/Logging/ExtendedLogger.cs index 693637781c8..849e2b20d0d 100644 --- a/src/Libraries/Microsoft.Extensions.Telemetry/Logging/ExtendedLogger.cs +++ b/src/Libraries/Microsoft.Extensions.Telemetry/Logging/ExtendedLogger.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using System.Diagnostics; +using Microsoft.Extensions.Diagnostics.Logging.Sampling; using Microsoft.Extensions.Logging; using Microsoft.Shared.Pools; @@ -39,6 +40,23 @@ public ExtendedLogger(ExtendedLoggerFactory factory, LoggerInformation[] loggers public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) { + var pattern = new LogRecordPattern + { + Category = // get category, + Tags = state, + EventId = eventId, + LogLevel = logLevel, + //and Timestamp is filled in automatically from DateTime.UtcNow + }; + + foreach (var sampler in _factory.Config.Samplers) + { + if (sampler.Sample(pattern)) + { + return; + } + } + if (typeof(TState) == typeof(LoggerMessageState)) { var msgState = (LoggerMessageState?)(object?)state; diff --git a/src/Libraries/Microsoft.Extensions.Telemetry/Logging/ExtendedLoggerFactory.cs b/src/Libraries/Microsoft.Extensions.Telemetry/Logging/ExtendedLoggerFactory.cs index 9a98b8446f4..c17cbbbc216 100644 --- a/src/Libraries/Microsoft.Extensions.Telemetry/Logging/ExtendedLoggerFactory.cs +++ b/src/Libraries/Microsoft.Extensions.Telemetry/Logging/ExtendedLoggerFactory.cs @@ -8,6 +8,7 @@ using Microsoft.Extensions.Compliance.Classification; using Microsoft.Extensions.Compliance.Redaction; using Microsoft.Extensions.Diagnostics.Enrichment; +using Microsoft.Extensions.Diagnostics.Logging.Sampling; using Microsoft.Extensions.Options; using Microsoft.Shared.Diagnostics; @@ -23,6 +24,7 @@ internal sealed class ExtendedLoggerFactory : ILoggerFactory private readonly IDisposable? _enrichmentOptionsChangeTokenRegistration; private readonly IDisposable? _redactionOptionsChangeTokenRegistration; private readonly Action[] _enrichers; + private readonly ILoggingSampler[] _samplers; private readonly KeyValuePair[] _staticTags; private readonly Func _redactorProvider; private volatile bool _disposed; @@ -33,6 +35,7 @@ internal sealed class ExtendedLoggerFactory : ILoggerFactory public ExtendedLoggerFactory( IEnumerable providers, IEnumerable enrichers, + IEnumerable samplers, IEnumerable staticEnrichers, IOptionsMonitor filterOptions, IOptions? factoryOptions = null, @@ -43,6 +46,7 @@ public ExtendedLoggerFactory( #pragma warning restore S107 // Methods should not have too many parameters { _scopeProvider = scopeProvider; + _samplers = samplers.ToArray(); _factoryOptions = factoryOptions == null || factoryOptions.Value == null ? new LoggerFactoryOptions() : factoryOptions.Value; @@ -282,8 +286,9 @@ private LoggerConfig ComputeConfig(LoggerEnrichmentOptions? enrichmentOptions, L }; } - return new(_staticTags, + return new LoggerConfig(_staticTags, _enrichers, + _samplers, enrichmentOptions.CaptureStackTraces, enrichmentOptions.UseFileInfoForStackTraces, enrichmentOptions.IncludeExceptionMessage, diff --git a/src/Libraries/Microsoft.Extensions.Telemetry/Logging/LoggerConfig.cs b/src/Libraries/Microsoft.Extensions.Telemetry/Logging/LoggerConfig.cs index 34716ca4e38..4f4743f449e 100644 --- a/src/Libraries/Microsoft.Extensions.Telemetry/Logging/LoggerConfig.cs +++ b/src/Libraries/Microsoft.Extensions.Telemetry/Logging/LoggerConfig.cs @@ -6,6 +6,7 @@ using Microsoft.Extensions.Compliance.Classification; using Microsoft.Extensions.Compliance.Redaction; using Microsoft.Extensions.Diagnostics.Enrichment; +using Microsoft.Extensions.Diagnostics.Logging.Sampling; namespace Microsoft.Extensions.Logging; @@ -15,6 +16,7 @@ internal sealed class LoggerConfig public LoggerConfig( KeyValuePair[] staticTags, Action[] enrichers, + ILoggingSampler[] samplers, bool captureStackTraces, bool useFileInfoForStackTraces, bool includeExceptionMessage, @@ -25,6 +27,7 @@ public LoggerConfig( #pragma warning restore S107 // Methods should not have too many parameters StaticTags = staticTags; Enrichers = enrichers; + Samplers = samplers; CaptureStackTraces = captureStackTraces; UseFileInfoForStackTraces = useFileInfoForStackTraces; MaxStackTraceLength = maxStackTraceLength; @@ -35,6 +38,7 @@ public LoggerConfig( public KeyValuePair[] StaticTags { get; } public Action[] Enrichers { get; } + public ILoggingSampler[] Samplers { get; } public bool CaptureStackTraces { get; } public bool UseFileInfoForStackTraces { get; } public bool IncludeExceptionMessage { get; } diff --git a/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/BufferingTool.cs b/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/BufferingTool.cs new file mode 100644 index 00000000000..a3ffd3a23ae --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/BufferingTool.cs @@ -0,0 +1,33 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using Microsoft.Extensions.Options; + +namespace Microsoft.Extensions.Diagnostics.Logging.Sampling; + +public class BufferingTool +{ + internal Dictionary Buffers { get; set; } + + public BufferingTool(IOptions samplingOptions) + { + var matchers = samplingOptions.Value.Matchers; + + // TODO: create actual buffer from matchers instead of the pseudocode below: + Buffers = new Dictionary(); + } + + public void Buffer(string bufferName) + { + Buffers[bufferName].Buffer(); + } + + public void Flush(string bufferName) + { + Buffers[bufferName].Flush(); + } +} diff --git a/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/ControlAction.cs b/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/ControlAction.cs new file mode 100644 index 00000000000..7bb078c1959 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/ControlAction.cs @@ -0,0 +1,30 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.Extensions.Diagnostics.Logging.Sampling; + +/// +/// Enumerates actions one of which can be executed on a matching log record. +/// +public enum ControlAction +{ + /// + /// Filter log records globally. + /// + GlobalFilter, + + /// + /// Buffer log records globally. + /// + GlobalBuffer, + + /// + /// Filter log records withing an HTTP request flow. + /// + RequestFilter, + + /// + /// Buffer log records for the duration of an HTTP requst flow. + /// + RequestBuffer +} diff --git a/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/ILoggingSampler.cs b/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/ILoggingSampler.cs new file mode 100644 index 00000000000..380f36362a9 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/ILoggingSampler.cs @@ -0,0 +1,17 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.Extensions.Diagnostics.Logging.Sampling; + +/// +/// Represents a component that samples log records. +/// +public interface ILoggingSampler +{ + /// + /// Sample a log record if it matches the . + /// + /// A log record pattern to match against. + /// True, if the log record was sampled. False otherwise. + public bool Sample(LogRecordPattern logRecordPattern); +} diff --git a/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/ILoggingSamplingBuilder.cs b/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/ILoggingSamplingBuilder.cs new file mode 100644 index 00000000000..525ee3cc624 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/ILoggingSamplingBuilder.cs @@ -0,0 +1,17 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.DependencyInjection; + +namespace Microsoft.Extensions.Diagnostics.Logging.Sampling; + +/// +/// An interface for configuring logging sampling. +/// +public interface ILoggingSamplingBuilder +{ + /// + /// Gets the where logging sampling services are configured. + /// + public IServiceCollection Services { get; } +} diff --git a/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/LogBuffer.cs b/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/LogBuffer.cs new file mode 100644 index 00000000000..ae9d11fada7 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/LogBuffer.cs @@ -0,0 +1,32 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; + +namespace Microsoft.Extensions.Diagnostics.Logging.Sampling; + +/// +/// Represents a circular log buffer configuration. +/// +public class LogBuffer +{ + /// + /// Gets or sets log buffer name. + /// + public string Name { get; set; } + + /// + /// Gets or sets duration to suspend buffering after the flush operation occurred. + /// + public TimeSpan? SuspendAfterFlushDuration { get; set; } + + /// + /// Gets or sets a circular buffer duration. + /// + public TimeSpan? BufferingDuration { get; set; } + + /// + /// Gets or sets buffer size. + /// + public long? BufferSize { get; set; } +} diff --git a/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/LogRecordPattern.cs b/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/LogRecordPattern.cs new file mode 100644 index 00000000000..0142a9beeda --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/LogRecordPattern.cs @@ -0,0 +1,39 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using Microsoft.Extensions.Logging; + +namespace Microsoft.Extensions.Diagnostics.Logging.Sampling; + +/// +/// A pattern to match log records against. +/// +public class LogRecordPattern +{ + /// + /// Gets or sets log record category. + /// + public string? Category { get; set; } + + /// + /// Gets or sets log record event ID. + /// + public EventId? EventId { get; set; } + + /// + /// Gets or sets log record log level. + /// + public LogLevel? LogLevel { get; set; } + + /// + /// Gets or sets log record state tags. + /// + public KeyValuePair[]? Tags { get; set; } + + /// + /// Gets or sets log record timestamp. + /// + public DateTime Timestamp { get; set; } = DateTime.UtcNow; +} diff --git a/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/LogSamplingOptions.cs b/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/LogSamplingOptions.cs new file mode 100644 index 00000000000..f91d9958d18 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/LogSamplingOptions.cs @@ -0,0 +1,50 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using Microsoft.Extensions.Logging; + +namespace Microsoft.Extensions.Diagnostics.Logging.Sampling; + +/// +/// Options to configure log sampling. +/// +public class LogSamplingOptions +{ + /// + /// Gets or sets a list of log pattern matchers. + /// + public List Matchers { get; set; } = new List + { + new Matcher( + new LogRecordPattern + { + LogLevel = LogLevel.Information, + Category = "Microsoft.Extensions.Hosting", + }, + // drop 99% of records: + (pattern) => Random.Shared.NextDouble() < 0.01), + new Matcher( + new LogRecordPattern + { + LogLevel = LogLevel.Error, + }, + // buffer records: + (tool, pattern) => tool.Buffer("bufferName")), + }; + + /// + /// Gets or sets a list of log buffers. + /// + public ISet Buffers { get; set; } = new HashSet + { + new LogBuffer + { + Name = "bufferName", + SuspendAfterFlushDuration = TimeSpan.FromSeconds(10), + BufferingDuration = TimeSpan.FromSeconds(10), + BufferSize = 1_000_000, + }, + }; +} diff --git a/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/LoggingSamplingBuilder.cs b/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/LoggingSamplingBuilder.cs new file mode 100644 index 00000000000..eae10f4ba1d --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/LoggingSamplingBuilder.cs @@ -0,0 +1,16 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.DependencyInjection; + +namespace Microsoft.Extensions.Diagnostics.Logging.Sampling; + +internal sealed class LoggingSamplingBuilder : ILoggingSamplingBuilder +{ + public LoggingSamplingBuilder(IServiceCollection services) + { + Services = services; + } + + public IServiceCollection Services { get; } +} diff --git a/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/LoggingSamplingExtensions.cs b/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/LoggingSamplingExtensions.cs new file mode 100644 index 00000000000..4a1c8cf249d --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/LoggingSamplingExtensions.cs @@ -0,0 +1,59 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +namespace Microsoft.Extensions.Diagnostics.Logging.Sampling; + +/// +/// Lets you register logging samplers in a dependency injection container. +/// +public static class LoggingSamplingExtensions +{ + /// + /// Enable logging sampling. + /// + /// An instance of to enable sampling in. + /// A delegate to fine-tune the sampling. + /// The value of . + public static ILoggingBuilder EnableSampling(this ILoggingBuilder builder, Action configure) + { + configure(new LoggingSamplingBuilder(builder.Services)); + + return builder; + } + + /// + /// Set the built-in simple sampler. + /// + /// An instance of to set the simple sampler in. + /// A delegate to fine-tune the sampling. + /// The value of . + public static ILoggingSamplingBuilder SetSimpleSampler(this ILoggingSamplingBuilder builder, Action configure) + { + builder.Services + .Configure(configure) + .TryAddActivatedSingleton(); + + return builder; + } + + /// + /// Set a logging sampler. + /// + /// A sampler type + /// An instance of to set the logging sampler in. + /// A delegate to fine-tune the sampling. + /// The value of . + public static ILoggingSamplingBuilder SetSampler(this ILoggingSamplingBuilder builder, Action configure) + where T : class, ILoggingSampler + { + builder.Services + .Configure(configure) + .TryAddActivatedSingleton(); + + return builder; + } +} diff --git a/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/Matcher.cs b/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/Matcher.cs new file mode 100644 index 00000000000..418d8937508 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/Matcher.cs @@ -0,0 +1,83 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; + +namespace Microsoft.Extensions.Diagnostics.Logging.Sampling; + +/// +/// A log pattern matcher. +/// +public class Matcher +{ + private readonly LogRecordPattern _pattern; + + /// + /// Gets a filtering delegate. + /// + public Func? Filter { get; } + + /// + /// Gets a buffering delegate. + /// + public Action? Buffer { get; } + + /// + /// Gets a control action to perform in case there is a match. + /// + public ControlAction ControlAction { get; } + + /// + /// Initializes a new instance of the class. + /// + /// A log record pattern to match. + /// A filtering delegate. + public Matcher(LogRecordPattern pattern, Func filter) + { + ControlAction = ControlAction.GlobalFilter; + _pattern = pattern; + Filter = filter; + } + + /// + /// Initializes a new instance of the class. + /// + /// A log record pattern to match. + /// A buffering delegate. + public Matcher(LogRecordPattern pattern, Action buffer) + { + ControlAction = ControlAction.GlobalBuffer; + _pattern = pattern; + Buffer = buffer; + } + + /// + /// Matches the log record pattern against the supplied . + /// + /// A log record pattern to match against. + /// True if there is a match. False otherwise. + public bool Match(LogRecordPattern pattern) + { + if (_pattern.Category != null && _pattern.Category != pattern.Category) + { + return false; + } + + if (_pattern.EventId != null && _pattern.EventId != pattern.EventId) + { + return false; + } + + if (_pattern.LogLevel != null && _pattern.LogLevel != pattern.LogLevel) + { + return false; + } + + if (_pattern.Tags != null && _pattern.Tags != pattern.Tags) + { + return false; + } + + return true; + } +} diff --git a/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/SimpleSampler.cs b/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/SimpleSampler.cs new file mode 100644 index 00000000000..f67d3a60c2f --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/SimpleSampler.cs @@ -0,0 +1,48 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using Microsoft.Extensions.Options; + +namespace Microsoft.Extensions.Diagnostics.Logging.Sampling; + +internal class SimpleSampler : ILoggingSampler +{ + private readonly List _matchers; + private readonly BufferingTool _bufferingTool; + + public SimpleSampler(IOptions options, BufferingTool bufferingTool) + { + _matchers = options.Value.Matchers; + _bufferingTool = bufferingTool; + } + + public bool Sample(LogRecordPattern logRecordPattern) + { + foreach (var matcher in _matchers) + { + if (matcher.Match(logRecordPattern)) + { + switch (matcher.ControlAction) + { + case ControlAction.GlobalFilter: + if (!matcher.Filter(logRecordPattern)) + { + return true; + } + + break; + case ControlAction.GlobalBuffer: + matcher.Buffer(_bufferingTool, logRecordPattern); + return true; + case ControlAction.RequestFilter: + break; + case ControlAction.RequestBuffer: + break; + } + } + } + + return false; + } +} From f8b502cc7a8b39daac5d9a554c7251c922a28373 Mon Sep 17 00:00:00 2001 From: R9 Fundamentals Date: Wed, 3 Jul 2024 15:39:44 +0200 Subject: [PATCH 02/35] update --- LoggerWithSampling/Program.cs | 45 +++++----- .../Logging/Buffering/BufferingMatcher.cs | 67 ++++++++++++++ .../Logging/Buffering/ILogBuffer.cs | 25 ++++++ .../Logging/Buffering/LogBuffer.cs | 41 +++++++++ .../LogBufferConfig.cs} | 12 +-- .../Buffering/LogBufferingExtensions.cs | 51 +++++++++++ .../Logging/Buffering/LogBufferingOptions.cs | 17 ++++ .../Logging/ExtendedLogger.cs | 2 +- .../Logging/ExtendedLoggerFactory.cs | 4 +- .../Logging/LoggerConfig.cs | 4 +- .../Sampling/BufferingFilterOptions.cs | 29 ++++++ .../Logging/Sampling/BufferingTool.cs | 33 ------- .../{ILoggingSampler.cs => ILogSampler.cs} | 2 +- ...plingBuilder.cs => ILogSamplingBuilder.cs} | 6 +- .../Logging/Sampling/IMatcher.cs | 12 +++ ...mplingBuilder.cs => LogSamplingBuilder.cs} | 8 +- .../Logging/Sampling/LogSamplingExtensions.cs | 90 +++++++++++++++++++ .../Logging/Sampling/LogSamplingOptions.cs | 50 ----------- .../Sampling/LoggingSamplingExtensions.cs | 59 ------------ .../Logging/Sampling/SamplingFilterOptions.cs | 30 +++++++ .../{Matcher.cs => SamplingMatcher.cs} | 23 +---- .../Logging/Sampling/SimpleBufferingFilter.cs | 40 +++++++++ ...mpleSampler.cs => SimpleSamplingFilter.cs} | 13 +-- 23 files changed, 454 insertions(+), 209 deletions(-) create mode 100644 src/Libraries/Microsoft.Extensions.Telemetry/Logging/Buffering/BufferingMatcher.cs create mode 100644 src/Libraries/Microsoft.Extensions.Telemetry/Logging/Buffering/ILogBuffer.cs create mode 100644 src/Libraries/Microsoft.Extensions.Telemetry/Logging/Buffering/LogBuffer.cs rename src/Libraries/Microsoft.Extensions.Telemetry/Logging/{Sampling/LogBuffer.cs => Buffering/LogBufferConfig.cs} (69%) create mode 100644 src/Libraries/Microsoft.Extensions.Telemetry/Logging/Buffering/LogBufferingExtensions.cs create mode 100644 src/Libraries/Microsoft.Extensions.Telemetry/Logging/Buffering/LogBufferingOptions.cs create mode 100644 src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/BufferingFilterOptions.cs delete mode 100644 src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/BufferingTool.cs rename src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/{ILoggingSampler.cs => ILogSampler.cs} (95%) rename src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/{ILoggingSamplingBuilder.cs => ILogSamplingBuilder.cs} (66%) create mode 100644 src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/IMatcher.cs rename src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/{LoggingSamplingBuilder.cs => LogSamplingBuilder.cs} (57%) create mode 100644 src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/LogSamplingExtensions.cs delete mode 100644 src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/LogSamplingOptions.cs delete mode 100644 src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/LoggingSamplingExtensions.cs create mode 100644 src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/SamplingFilterOptions.cs rename src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/{Matcher.cs => SamplingMatcher.cs} (69%) create mode 100644 src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/SimpleBufferingFilter.cs rename src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/{SimpleSampler.cs => SimpleSamplingFilter.cs} (65%) diff --git a/LoggerWithSampling/Program.cs b/LoggerWithSampling/Program.cs index 2f456e58139..1b1c9a62e2b 100644 --- a/LoggerWithSampling/Program.cs +++ b/LoggerWithSampling/Program.cs @@ -4,6 +4,7 @@ using System.Collections.Generic; using System.Linq; using System.Security.Principal; +using Microsoft.Extensions.Diagnostics.Logging.Buffering; using Microsoft.Extensions.Diagnostics.Logging.Sampling; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -44,39 +45,41 @@ private static void LoggerDemo(string loggerCategory, bool formatMessage) private static ILoggerFactory CreateLoggerFactory(bool formatMessage) { + const string myGlobalBufferName = "My buffer"; return LoggerFactory.Create(builder => { - _ = builder.EnableSampling(samplingBuilder => samplingBuilder - - // add the built-in simple sampler: - .SetSimpleSampler(o => + builder + .EnableBuffering(opts => + { + opts.Configs.Add(new LogBufferConfig + { + Name = myGlobalBufferName, + SuspendAfterFlushDuration = TimeSpan.FromSeconds(10), + Duration = TimeSpan.FromSeconds(10), + Size = 1_000_000, + }); + }) + .EnableSampling(samplingBuilder => samplingBuilder + .EnableSimpleSamplingFilter(opts => { - o.Matchers = new List - { - new Matcher( + opts.Matchers.Add( + new SamplingMatcher( new LogRecordPattern { LogLevel = LogLevel.Information, Category = "Microsoft.Extensions.Hosting", }, - (pattern) => Random.Shared.NextDouble() < 0.01), - new Matcher( + (pattern) => Random.Shared.NextDouble() < 0.01)); + }) + .EnableSimpleBufferingFilter(opts => + { + opts.Matchers.Add( + new BufferingMatcher( new LogRecordPattern { LogLevel = LogLevel.Error, }, - (tool, pattern) => tool.Buffer("MyBuffer")), - }; - o.Buffers = new HashSet - { - new LogBuffer - { - Name = "MyBuffer", - SuspendAfterFlushDuration = TimeSpan.FromSeconds(10), - BufferingDuration = TimeSpan.FromSeconds(10), - BufferSize = 1_000_000, - }, - }; + (tool, pattern) => tool.Buffer(myGlobalBufferName, pattern))); })); }); } diff --git a/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Buffering/BufferingMatcher.cs b/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Buffering/BufferingMatcher.cs new file mode 100644 index 00000000000..601e2275ae0 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Buffering/BufferingMatcher.cs @@ -0,0 +1,67 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using Microsoft.Extensions.Diagnostics.Logging.Sampling; + +namespace Microsoft.Extensions.Diagnostics.Logging.Buffering; + +/// +/// A log pattern matcher. +/// +public class BufferingMatcher : IMatcher +{ + private readonly LogRecordPattern _pattern; + + /// + /// Gets a buffering delegate. + /// + public Action? Buffer { get; } + + /// + /// Gets a control action to perform in case there is a match. + /// + public ControlAction ControlAction { get; } + + /// + /// Initializes a new instance of the class. + /// + /// A log record pattern to match. + /// A buffering delegate. + public BufferingMatcher(LogRecordPattern pattern, Action buffer) + { + ControlAction = ControlAction.GlobalBuffer; + _pattern = pattern; + Buffer = buffer; + } + + /// + /// Matches the log record pattern against the supplied . + /// + /// A log record pattern to match against. + /// True if there is a match. False otherwise. + public bool Match(LogRecordPattern pattern) + { + if (_pattern.Category != null && _pattern.Category != pattern.Category) + { + return false; + } + + if (_pattern.EventId != null && _pattern.EventId != pattern.EventId) + { + return false; + } + + if (_pattern.LogLevel != null && _pattern.LogLevel != pattern.LogLevel) + { + return false; + } + + if (_pattern.Tags != null && _pattern.Tags != pattern.Tags) + { + return false; + } + + return true; + } +} diff --git a/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Buffering/ILogBuffer.cs b/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Buffering/ILogBuffer.cs new file mode 100644 index 00000000000..a5cc704a083 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Buffering/ILogBuffer.cs @@ -0,0 +1,25 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.Diagnostics.Logging.Sampling; + +namespace Microsoft.Extensions.Diagnostics.Logging.Buffering; + +/// +/// Represent a log buffering interface. +/// +public interface ILogBuffer +{ + /// + /// Buffer the incoming to the . + /// + /// The buffer name to buffer to. + /// The log record to buffer. + void Buffer(string bufferName, LogRecordPattern logRecord); // TODO: consider using the actual LogRecord instead of LogRecordPattern? + + /// + /// Flush the buffer . + /// + /// The buffer name to flush. + void Flush(string bufferName); +} diff --git a/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Buffering/LogBuffer.cs b/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Buffering/LogBuffer.cs new file mode 100644 index 00000000000..a34e2714574 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Buffering/LogBuffer.cs @@ -0,0 +1,41 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using Microsoft.Extensions.Diagnostics.Logging.Sampling; +using Microsoft.Extensions.Options; + +namespace Microsoft.Extensions.Diagnostics.Logging.Buffering; + +/// +/// A wrapper for log buffers. +/// +public class LogBuffer : ILogBuffer +{ + internal Dictionary Buffers { get; set; } + + /// + /// Initializes a new instance of the class. + /// + /// Configuration options for the type. + public LogBuffer(IOptions options) + { + // TODO: create actual buffer from options instead of the pseudocode below: + Buffers = new Dictionary(); + } + + /// + public void Buffer(string bufferName, LogRecordPattern logRecord) + { + Buffers[bufferName].Buffer(logRecord); + } + + /// + public void Flush(string bufferName) + { + Buffers[bufferName].Flush(); + } +} diff --git a/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/LogBuffer.cs b/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Buffering/LogBufferConfig.cs similarity index 69% rename from src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/LogBuffer.cs rename to src/Libraries/Microsoft.Extensions.Telemetry/Logging/Buffering/LogBufferConfig.cs index ae9d11fada7..14f2d654fbc 100644 --- a/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/LogBuffer.cs +++ b/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Buffering/LogBufferConfig.cs @@ -2,18 +2,20 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; +using System.ComponentModel.DataAnnotations; -namespace Microsoft.Extensions.Diagnostics.Logging.Sampling; +namespace Microsoft.Extensions.Diagnostics.Logging.Buffering; /// /// Represents a circular log buffer configuration. /// -public class LogBuffer +public class LogBufferConfig { /// /// Gets or sets log buffer name. /// - public string Name { get; set; } + [Required] + public string Name { get; set; } = string.Empty; /// /// Gets or sets duration to suspend buffering after the flush operation occurred. @@ -23,10 +25,10 @@ public class LogBuffer /// /// Gets or sets a circular buffer duration. /// - public TimeSpan? BufferingDuration { get; set; } + public TimeSpan? Duration { get; set; } /// /// Gets or sets buffer size. /// - public long? BufferSize { get; set; } + public long? Size { get; set; } } diff --git a/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Buffering/LogBufferingExtensions.cs b/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Buffering/LogBufferingExtensions.cs new file mode 100644 index 00000000000..287b513137d --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Buffering/LogBufferingExtensions.cs @@ -0,0 +1,51 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.Diagnostics.Logging.Buffering; + +/// +/// Lets you register log buffers in a dependency injection container. +/// +public static class LogBufferingExtensions +{ + /// + /// Add log buffering. + /// + /// An instance of to enable buffering in. + /// A delegate to fine-tune the buffering. + /// The value of . + public static ILoggingBuilder EnableBuffering(this ILoggingBuilder builder, Action configure) + { + _ = Throw.IfNull(builder); + _ = Throw.IfNull(configure); + + builder.Services + .Configure(configure) + .TryAddActivatedSingleton(); + + return builder; + } + + /// + /// Add log buffering. + /// + /// An instance of to enable buffering in. + /// Configuration section that contains . + /// The value of . + public static ILoggingBuilder EnableBuffering(this ILoggingBuilder builder, IConfigurationSection section) + { + _ = Throw.IfNull(builder); + _ = Throw.IfNull(section); + + _ = builder.Services.AddOptions().Bind(section); + builder.Services.TryAddActivatedSingleton(); + + return builder; + } +} diff --git a/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Buffering/LogBufferingOptions.cs b/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Buffering/LogBufferingOptions.cs new file mode 100644 index 00000000000..bada221510a --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Buffering/LogBufferingOptions.cs @@ -0,0 +1,17 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; + +namespace Microsoft.Extensions.Diagnostics.Logging.Buffering; + +/// +/// Options to configure log buffering. +/// +public class LogBufferingOptions +{ + /// + /// Gets or sets a list of log buffers. + /// + public ISet Configs { get; set; } = new HashSet(); +} diff --git a/src/Libraries/Microsoft.Extensions.Telemetry/Logging/ExtendedLogger.cs b/src/Libraries/Microsoft.Extensions.Telemetry/Logging/ExtendedLogger.cs index 849e2b20d0d..9d9fbaef7cd 100644 --- a/src/Libraries/Microsoft.Extensions.Telemetry/Logging/ExtendedLogger.cs +++ b/src/Libraries/Microsoft.Extensions.Telemetry/Logging/ExtendedLogger.cs @@ -42,7 +42,7 @@ public void Log(LogLevel logLevel, EventId eventId, TState state, Except { var pattern = new LogRecordPattern { - Category = // get category, + Category = //TODO: get category somehow? Tags = state, EventId = eventId, LogLevel = logLevel, diff --git a/src/Libraries/Microsoft.Extensions.Telemetry/Logging/ExtendedLoggerFactory.cs b/src/Libraries/Microsoft.Extensions.Telemetry/Logging/ExtendedLoggerFactory.cs index c17cbbbc216..28458bce932 100644 --- a/src/Libraries/Microsoft.Extensions.Telemetry/Logging/ExtendedLoggerFactory.cs +++ b/src/Libraries/Microsoft.Extensions.Telemetry/Logging/ExtendedLoggerFactory.cs @@ -24,7 +24,7 @@ internal sealed class ExtendedLoggerFactory : ILoggerFactory private readonly IDisposable? _enrichmentOptionsChangeTokenRegistration; private readonly IDisposable? _redactionOptionsChangeTokenRegistration; private readonly Action[] _enrichers; - private readonly ILoggingSampler[] _samplers; + private readonly ILogSampler[] _samplers; private readonly KeyValuePair[] _staticTags; private readonly Func _redactorProvider; private volatile bool _disposed; @@ -35,7 +35,7 @@ internal sealed class ExtendedLoggerFactory : ILoggerFactory public ExtendedLoggerFactory( IEnumerable providers, IEnumerable enrichers, - IEnumerable samplers, + IEnumerable samplers, IEnumerable staticEnrichers, IOptionsMonitor filterOptions, IOptions? factoryOptions = null, diff --git a/src/Libraries/Microsoft.Extensions.Telemetry/Logging/LoggerConfig.cs b/src/Libraries/Microsoft.Extensions.Telemetry/Logging/LoggerConfig.cs index 4f4743f449e..9be33a0d4b7 100644 --- a/src/Libraries/Microsoft.Extensions.Telemetry/Logging/LoggerConfig.cs +++ b/src/Libraries/Microsoft.Extensions.Telemetry/Logging/LoggerConfig.cs @@ -16,7 +16,7 @@ internal sealed class LoggerConfig public LoggerConfig( KeyValuePair[] staticTags, Action[] enrichers, - ILoggingSampler[] samplers, + ILogSampler[] samplers, bool captureStackTraces, bool useFileInfoForStackTraces, bool includeExceptionMessage, @@ -38,7 +38,7 @@ public LoggerConfig( public KeyValuePair[] StaticTags { get; } public Action[] Enrichers { get; } - public ILoggingSampler[] Samplers { get; } + public ILogSampler[] Samplers { get; } public bool CaptureStackTraces { get; } public bool UseFileInfoForStackTraces { get; } public bool IncludeExceptionMessage { get; } diff --git a/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/BufferingFilterOptions.cs b/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/BufferingFilterOptions.cs new file mode 100644 index 00000000000..d0704551d3b --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/BufferingFilterOptions.cs @@ -0,0 +1,29 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using Microsoft.Extensions.Diagnostics.Logging.Buffering; +using Microsoft.Extensions.Logging; + +namespace Microsoft.Extensions.Diagnostics.Logging.Sampling; + +/// +/// Options to configure log sampling. +/// +public class BufferingFilterOptions +{ + /// + /// Gets or sets a list of log pattern matchers. + /// + public IList Matchers { get; set; } = new List + { + new BufferingMatcher( + new LogRecordPattern + { + LogLevel = LogLevel.Error, + }, + + // buffer records: + (tool, pattern) => tool.Buffer("bufferName", pattern)), + }; +} diff --git a/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/BufferingTool.cs b/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/BufferingTool.cs deleted file mode 100644 index a3ffd3a23ae..00000000000 --- a/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/BufferingTool.cs +++ /dev/null @@ -1,33 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Collections.Generic; -using Microsoft.Extensions.Options; - -namespace Microsoft.Extensions.Diagnostics.Logging.Sampling; - -public class BufferingTool -{ - internal Dictionary Buffers { get; set; } - - public BufferingTool(IOptions samplingOptions) - { - var matchers = samplingOptions.Value.Matchers; - - // TODO: create actual buffer from matchers instead of the pseudocode below: - Buffers = new Dictionary(); - } - - public void Buffer(string bufferName) - { - Buffers[bufferName].Buffer(); - } - - public void Flush(string bufferName) - { - Buffers[bufferName].Flush(); - } -} diff --git a/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/ILoggingSampler.cs b/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/ILogSampler.cs similarity index 95% rename from src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/ILoggingSampler.cs rename to src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/ILogSampler.cs index 380f36362a9..3c553321861 100644 --- a/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/ILoggingSampler.cs +++ b/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/ILogSampler.cs @@ -6,7 +6,7 @@ namespace Microsoft.Extensions.Diagnostics.Logging.Sampling; /// /// Represents a component that samples log records. /// -public interface ILoggingSampler +public interface ILogSampler { /// /// Sample a log record if it matches the . diff --git a/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/ILoggingSamplingBuilder.cs b/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/ILogSamplingBuilder.cs similarity index 66% rename from src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/ILoggingSamplingBuilder.cs rename to src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/ILogSamplingBuilder.cs index 525ee3cc624..4e25d0551df 100644 --- a/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/ILoggingSamplingBuilder.cs +++ b/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/ILogSamplingBuilder.cs @@ -6,12 +6,12 @@ namespace Microsoft.Extensions.Diagnostics.Logging.Sampling; /// -/// An interface for configuring logging sampling. +/// An interface for configuring log sampling. /// -public interface ILoggingSamplingBuilder +public interface ILogSamplingBuilder { /// - /// Gets the where logging sampling services are configured. + /// Gets the where log sampling services are configured. /// public IServiceCollection Services { get; } } diff --git a/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/IMatcher.cs b/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/IMatcher.cs new file mode 100644 index 00000000000..a70289bee44 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/IMatcher.cs @@ -0,0 +1,12 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.Extensions.Diagnostics.Logging.Sampling; + +/// +/// Interface for matchin log records. +/// +public interface IMatcher +{ + bool Match(LogRecordPattern pattern); +} diff --git a/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/LoggingSamplingBuilder.cs b/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/LogSamplingBuilder.cs similarity index 57% rename from src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/LoggingSamplingBuilder.cs rename to src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/LogSamplingBuilder.cs index eae10f4ba1d..200a06bed88 100644 --- a/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/LoggingSamplingBuilder.cs +++ b/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/LogSamplingBuilder.cs @@ -1,13 +1,17 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. + +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + using Microsoft.Extensions.DependencyInjection; namespace Microsoft.Extensions.Diagnostics.Logging.Sampling; -internal sealed class LoggingSamplingBuilder : ILoggingSamplingBuilder +internal sealed class LogSamplingBuilder : ILogSamplingBuilder { - public LoggingSamplingBuilder(IServiceCollection services) + public LogSamplingBuilder(IServiceCollection services) { Services = services; } diff --git a/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/LogSamplingExtensions.cs b/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/LogSamplingExtensions.cs new file mode 100644 index 00000000000..e3f8ea3eb60 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/LogSamplingExtensions.cs @@ -0,0 +1,90 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Diagnostics.CodeAnalysis; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.Diagnostics.Logging.Sampling; + +/// +/// Lets you register log samplers in a dependency injection container. +/// +public static class LogSamplingExtensions +{ + /// + /// Enable log sampling. + /// + /// An instance of to enable sampling in. + /// A delegate to fine-tune the sampling. + /// The value of . + public static ILoggingBuilder EnableSampling(this ILoggingBuilder builder, Action configure) + { + _ = Throw.IfNull(builder); + _ = Throw.IfNull(configure); + + configure(new LogSamplingBuilder(builder.Services)); + + return builder; + } + + /// + /// Add the built-in simple sampling filter. + /// + /// An instance of to set the simple sampling in. + /// A delegate to fine-tune the sampling. + /// The value of . + public static ILogSamplingBuilder EnableSimpleSamplingFilter(this ILogSamplingBuilder builder, Action configure) + { + _ = Throw.IfNull(builder); + _ = Throw.IfNull(configure); + + builder.Services + .Configure(configure) + .TryAddActivatedSingleton(); + + return builder; + } + + /// + /// Add the built-in simple buffering filter. + /// + /// An instance of to set the simple buffering filter in. + /// A delegate to fine-tune the sampling. + /// The value of . + public static ILogSamplingBuilder EnableSimpleBufferingFilter(this ILogSamplingBuilder builder, Action configure) + { + _ = Throw.IfNull(builder); + _ = Throw.IfNull(configure); + + builder.Services + .Configure(configure) + .TryAddActivatedSingleton(); + + return builder; + } + + /// + /// Add a log sampler. + /// + /// A sampler type. + /// An instance of to set the log sampler in. + /// A delegate to fine-tune the sampling. + /// The value of . + public static ILogSamplingBuilder EnableSamplingFilter<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] T>( + this ILogSamplingBuilder builder, + Action configure) + where T : class, ILogSampler + { + _ = Throw.IfNull(builder); + _ = Throw.IfNull(configure); + + builder.Services + .Configure(configure) + .TryAddActivatedSingleton(); + + return builder; + } +} diff --git a/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/LogSamplingOptions.cs b/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/LogSamplingOptions.cs deleted file mode 100644 index f91d9958d18..00000000000 --- a/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/LogSamplingOptions.cs +++ /dev/null @@ -1,50 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; -using System.Collections.Generic; -using Microsoft.Extensions.Logging; - -namespace Microsoft.Extensions.Diagnostics.Logging.Sampling; - -/// -/// Options to configure log sampling. -/// -public class LogSamplingOptions -{ - /// - /// Gets or sets a list of log pattern matchers. - /// - public List Matchers { get; set; } = new List - { - new Matcher( - new LogRecordPattern - { - LogLevel = LogLevel.Information, - Category = "Microsoft.Extensions.Hosting", - }, - // drop 99% of records: - (pattern) => Random.Shared.NextDouble() < 0.01), - new Matcher( - new LogRecordPattern - { - LogLevel = LogLevel.Error, - }, - // buffer records: - (tool, pattern) => tool.Buffer("bufferName")), - }; - - /// - /// Gets or sets a list of log buffers. - /// - public ISet Buffers { get; set; } = new HashSet - { - new LogBuffer - { - Name = "bufferName", - SuspendAfterFlushDuration = TimeSpan.FromSeconds(10), - BufferingDuration = TimeSpan.FromSeconds(10), - BufferSize = 1_000_000, - }, - }; -} diff --git a/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/LoggingSamplingExtensions.cs b/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/LoggingSamplingExtensions.cs deleted file mode 100644 index 4a1c8cf249d..00000000000 --- a/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/LoggingSamplingExtensions.cs +++ /dev/null @@ -1,59 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; - -namespace Microsoft.Extensions.Diagnostics.Logging.Sampling; - -/// -/// Lets you register logging samplers in a dependency injection container. -/// -public static class LoggingSamplingExtensions -{ - /// - /// Enable logging sampling. - /// - /// An instance of to enable sampling in. - /// A delegate to fine-tune the sampling. - /// The value of . - public static ILoggingBuilder EnableSampling(this ILoggingBuilder builder, Action configure) - { - configure(new LoggingSamplingBuilder(builder.Services)); - - return builder; - } - - /// - /// Set the built-in simple sampler. - /// - /// An instance of to set the simple sampler in. - /// A delegate to fine-tune the sampling. - /// The value of . - public static ILoggingSamplingBuilder SetSimpleSampler(this ILoggingSamplingBuilder builder, Action configure) - { - builder.Services - .Configure(configure) - .TryAddActivatedSingleton(); - - return builder; - } - - /// - /// Set a logging sampler. - /// - /// A sampler type - /// An instance of to set the logging sampler in. - /// A delegate to fine-tune the sampling. - /// The value of . - public static ILoggingSamplingBuilder SetSampler(this ILoggingSamplingBuilder builder, Action configure) - where T : class, ILoggingSampler - { - builder.Services - .Configure(configure) - .TryAddActivatedSingleton(); - - return builder; - } -} diff --git a/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/SamplingFilterOptions.cs b/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/SamplingFilterOptions.cs new file mode 100644 index 00000000000..45fd1f12124 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/SamplingFilterOptions.cs @@ -0,0 +1,30 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using Microsoft.Extensions.Logging; + +namespace Microsoft.Extensions.Diagnostics.Logging.Sampling; + +/// +/// Options to configure log sampling. +/// +public class SamplingFilterOptions +{ + /// + /// Gets or sets a list of log pattern matchers. + /// + public IList Matchers { get; set; } = new List + { + new SamplingMatcher( + new LogRecordPattern + { + LogLevel = LogLevel.Information, + Category = "Microsoft.Extensions.Hosting", + }, + + // drop 99% of records: + (pattern) => Random.Shared.NextDouble() < 0.01), + }; +} diff --git a/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/Matcher.cs b/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/SamplingMatcher.cs similarity index 69% rename from src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/Matcher.cs rename to src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/SamplingMatcher.cs index 418d8937508..075694032d7 100644 --- a/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/Matcher.cs +++ b/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/SamplingMatcher.cs @@ -8,7 +8,7 @@ namespace Microsoft.Extensions.Diagnostics.Logging.Sampling; /// /// A log pattern matcher. /// -public class Matcher +public class SamplingMatcher : IMatcher { private readonly LogRecordPattern _pattern; @@ -17,40 +17,23 @@ public class Matcher /// public Func? Filter { get; } - /// - /// Gets a buffering delegate. - /// - public Action? Buffer { get; } - /// /// Gets a control action to perform in case there is a match. /// public ControlAction ControlAction { get; } /// - /// Initializes a new instance of the class. + /// Initializes a new instance of the class. /// /// A log record pattern to match. /// A filtering delegate. - public Matcher(LogRecordPattern pattern, Func filter) + public SamplingMatcher(LogRecordPattern pattern, Func filter) { ControlAction = ControlAction.GlobalFilter; _pattern = pattern; Filter = filter; } - /// - /// Initializes a new instance of the class. - /// - /// A log record pattern to match. - /// A buffering delegate. - public Matcher(LogRecordPattern pattern, Action buffer) - { - ControlAction = ControlAction.GlobalBuffer; - _pattern = pattern; - Buffer = buffer; - } - /// /// Matches the log record pattern against the supplied . /// diff --git a/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/SimpleBufferingFilter.cs b/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/SimpleBufferingFilter.cs new file mode 100644 index 00000000000..13423ff8cc9 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/SimpleBufferingFilter.cs @@ -0,0 +1,40 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using Microsoft.Extensions.Diagnostics.Logging.Buffering; +using Microsoft.Extensions.Options; + +namespace Microsoft.Extensions.Diagnostics.Logging.Sampling; + +internal class SimpleBufferingFilter : ILogSampler +{ + private readonly IList _matchers; + private readonly LogBuffer _bufferingTool; + + public SimpleBufferingFilter(IOptions options, LogBuffer bufferingTool) + { + _matchers = options.Value.Matchers; + _bufferingTool = bufferingTool; + } + + public bool Sample(LogRecordPattern logRecordPattern) + { + foreach (var matcher in _matchers) + { + if (matcher.Match(logRecordPattern)) + { + switch (matcher.ControlAction) + { + case ControlAction.GlobalBuffer: + matcher!.Buffer(_bufferingTool, logRecordPattern); + return true; + case ControlAction.RequestBuffer: + break; + } + } + } + + return false; + } +} diff --git a/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/SimpleSampler.cs b/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/SimpleSamplingFilter.cs similarity index 65% rename from src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/SimpleSampler.cs rename to src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/SimpleSamplingFilter.cs index f67d3a60c2f..b3306e8a7dc 100644 --- a/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/SimpleSampler.cs +++ b/src/Libraries/Microsoft.Extensions.Telemetry/Logging/Sampling/SimpleSamplingFilter.cs @@ -6,15 +6,13 @@ namespace Microsoft.Extensions.Diagnostics.Logging.Sampling; -internal class SimpleSampler : ILoggingSampler +internal class SimpleSamplingFilter : ILogSampler { - private readonly List _matchers; - private readonly BufferingTool _bufferingTool; + private readonly IList _matchers; - public SimpleSampler(IOptions options, BufferingTool bufferingTool) + public SimpleSamplingFilter(IOptions options) { _matchers = options.Value.Matchers; - _bufferingTool = bufferingTool; } public bool Sample(LogRecordPattern logRecordPattern) @@ -32,13 +30,8 @@ public bool Sample(LogRecordPattern logRecordPattern) } break; - case ControlAction.GlobalBuffer: - matcher.Buffer(_bufferingTool, logRecordPattern); - return true; case ControlAction.RequestFilter: break; - case ControlAction.RequestBuffer: - break; } } } From ed7e0db42aa7152923e6ffa14c83294b63aaba4e Mon Sep 17 00:00:00 2001 From: evgenyfedorov2 <25526458+evgenyfedorov2@users.noreply.github.com> Date: Thu, 12 Sep 2024 12:37:37 +0200 Subject: [PATCH 03/35] rebase --- global.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/global.json b/global.json index efae9047328..2ed29d5af51 100644 --- a/global.json +++ b/global.json @@ -1,9 +1,9 @@ { "sdk": { - "version": "8.0.200" + "version": "8.0.400" }, "tools": { - "dotnet": "8.0.200", + "dotnet": "8.0.400", "runtimes": { "dotnet/x64": [ "6.0.22" From e15139dfcdcbfa7218e3c383103be9017c7597f0 Mon Sep 17 00:00:00 2001 From: evgenyfedorov2 <25526458+evgenyfedorov2@users.noreply.github.com> Date: Mon, 30 Sep 2024 12:29:30 +0200 Subject: [PATCH 04/35] use .net 9 --- Directory.Build.props | 4 +- eng/Versions.props | 81 ++++++++++++++++++------------------- eng/packages/General.props | 1 + eng/packages/TestOnly.props | 5 +++ global.json | 8 ++-- 5 files changed, 51 insertions(+), 48 deletions(-) diff --git a/Directory.Build.props b/Directory.Build.props index cea28e22ade..3f7e93f627b 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -5,7 +5,7 @@ net - 8 + 9 0 $(TargetFrameworkMajorVersion).$(TargetFrameworkMinorVersion) @@ -13,7 +13,7 @@ $(TargetFrameworkName)$(TargetFrameworkVersion) $(LatestTargetFramework) - $(SupportedNetCoreTargetFrameworks);net6.0 + $(SupportedNetCoreTargetFrameworks);net8.0 net6.0 diff --git a/eng/Versions.props b/eng/Versions.props index f6013bcea1b..34d2d50d744 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -28,49 +28,46 @@ --> - 8.0.1 - 8.0.0 - 8.0.0 - 8.0.0 - 8.0.1 - 8.0.0 - 8.0.0 - 8.0.0 - 8.0.1 - 8.0.0 - 8.0.0 - 8.0.0 - 8.0.0 - 8.0.0 - 8.0.1 - 8.0.0 - 8.0.0 - 8.0.0 - 8.0.0 - 8.0.0 - 8.0.2 - 8.0.0 - 8.0.0 - 8.0.0 - 8.0.0 - 8.0.0 - 8.0.0 - 8.0.0 - 8.0.0 - 8.0.0 - 8.0.0 - 8.0.3 - 8.0.0 + 9.0.0-rc.1.24431.7 + 9.0.0-rc.1.24431.7 + 9.0.0-rc.1.24431.7 + 9.0.0-rc.1.24431.7 + 9.0.0-rc.1.24431.7 + 9.0.0-rc.1.24431.7 + 9.0.0-rc.1.24431.7 + 9.0.0-rc.1.24431.7 + 9.0.0-rc.1.24431.7 + 9.0.0-rc.1.24431.7 + 9.0.0-rc.1.24431.7 + 9.0.0-rc.1.24431.7 + 9.0.0-rc.1.24431.7 + 9.0.0-rc.1.24431.7 + 9.0.0-rc.1.24431.7 + 9.0.0-rc.1.24431.7 + 9.0.0-rc.1.24431.7 + 9.0.0-rc.1.24431.7 + 9.0.0-rc.1.24431.7 + 9.0.0-rc.1.24431.7 + 9.0.0-rc.1.24431.7 + 9.0.0-rc.1.24431.7 + 9.0.0-rc.1.24431.7 + 9.0.0-rc.1.24431.7 + 9.0.0-rc.1.24431.7 + 9.0.0-rc.1.24431.7 + 9.0.0-rc.1.24431.7 + 9.0.0-rc.1.24431.7 + 9.0.0-rc.1.24431.7 + 9.0.0-rc.1.24431.7 + 9.0.0-rc.1.24431.7 - 8.0.3 - 8.0.3 - 8.0.3 - 8.0.3 - 8.0.3 - 8.0.3 - 8.0.3 - 8.0.3 - 8.0.3 + 9.0.0-rc.1.24452.1 + 9.0.0-rc.1.24452.1 + 9.0.0-rc.1.24452.1 + 9.0.0-rc.1.24452.1 + 9.0.0-rc.1.24452.1 + 9.0.0-rc.1.24452.1 + 9.0.0-rc.1.24452.1 + 9.0.0-rc.1.24452.1