From dd0ed592922c573fc695d9e04de9713941f5cd15 Mon Sep 17 00:00:00 2001 From: Liam McLennan Date: Fri, 19 Jul 2024 15:54:19 +1000 Subject: [PATCH 1/6] Redundant using --- src/Serilog.Sinks.Seq/Sinks/Seq/ConstrainedBufferedFormatter.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Serilog.Sinks.Seq/Sinks/Seq/ConstrainedBufferedFormatter.cs b/src/Serilog.Sinks.Seq/Sinks/Seq/ConstrainedBufferedFormatter.cs index 3a5f288..bf83853 100644 --- a/src/Serilog.Sinks.Seq/Sinks/Seq/ConstrainedBufferedFormatter.cs +++ b/src/Serilog.Sinks.Seq/Sinks/Seq/ConstrainedBufferedFormatter.cs @@ -13,6 +13,7 @@ // limitations under the License. using System; +using System.Globalization; using System.IO; using System.Text; using Serilog.Debugging; From 1cd967ef87c42e453222e4ea3c0cbb92bf911f5f Mon Sep 17 00:00:00 2001 From: Liam McLennan Date: Fri, 19 Jul 2024 16:07:13 +1000 Subject: [PATCH 2/6] Support custom format provider --- .../SeqLoggerConfigurationExtensions.cs | 15 +- .../CleanMessageTemplateFormatter.cs | 146 ++++++++++++++++++ .../Sinks/Seq/Formatting/Padding.cs | 54 +++++++ .../Sinks/Seq/SeqCompactJsonFormatter.cs | 26 +++- .../SeqCompactJsonFormatterTests.cs | 39 ++++- 5 files changed, 269 insertions(+), 11 deletions(-) create mode 100644 src/Serilog.Sinks.Seq/Sinks/Seq/Formatting/CleanMessageTemplateFormatter.cs create mode 100644 src/Serilog.Sinks.Seq/Sinks/Seq/Formatting/Padding.cs diff --git a/src/Serilog.Sinks.Seq/SeqLoggerConfigurationExtensions.cs b/src/Serilog.Sinks.Seq/SeqLoggerConfigurationExtensions.cs index 414d536..444094b 100644 --- a/src/Serilog.Sinks.Seq/SeqLoggerConfigurationExtensions.cs +++ b/src/Serilog.Sinks.Seq/SeqLoggerConfigurationExtensions.cs @@ -13,6 +13,7 @@ // limitations under the License. using System; +using System.Globalization; using Serilog.Configuration; using Serilog.Core; using Serilog.Events; @@ -34,7 +35,7 @@ public static class SeqLoggerConfigurationExtensions const int DefaultBatchPostingLimit = 1000; static readonly TimeSpan DefaultPeriod = TimeSpan.FromSeconds(2); const int DefaultQueueSizeLimit = 100000; - static ITextFormatter CreateDefaultFormatter() => new SeqCompactJsonFormatter(); + static ITextFormatter CreateDefaultFormatter(IFormatProvider? formatProvider) => new SeqCompactJsonFormatter(formatProvider); /// /// Write log events to a Seq server. @@ -67,6 +68,7 @@ public static class SeqLoggerConfigurationExtensions /// durable log shipping. /// An that will be used to format (newline-delimited CLEF/JSON) /// payloads. Experimental. + /// An that will be used to render log event tokens. Does not apply if `payloadFormatter` is provided. /// Logger configuration, allowing configuration to continue. /// A required parameter is null. public static LoggerConfiguration Seq( @@ -83,7 +85,8 @@ public static LoggerConfiguration Seq( HttpMessageHandler? messageHandler = null, long? retainedInvalidPayloadsLimitBytes = null, int queueSizeLimit = DefaultQueueSizeLimit, - ITextFormatter? payloadFormatter = null) + ITextFormatter? payloadFormatter = null, + IFormatProvider? formatProvider = null) { if (loggerSinkConfiguration == null) throw new ArgumentNullException(nameof(loggerSinkConfiguration)); if (serverUrl == null) throw new ArgumentNullException(nameof(serverUrl)); @@ -94,7 +97,7 @@ public static LoggerConfiguration Seq( var defaultedPeriod = period ?? DefaultPeriod; var controlledSwitch = new ControlledLevelSwitch(controlLevelSwitch); - var formatter = payloadFormatter ?? CreateDefaultFormatter(); + var formatter = payloadFormatter ?? CreateDefaultFormatter(formatProvider); var ingestionApi = new SeqIngestionApiClient(serverUrl, apiKey, messageHandler); if (bufferBaseFilename == null) @@ -145,6 +148,7 @@ public static LoggerConfiguration Seq( /// Used to construct the HttpClient that will send the log messages to Seq. /// An that will be used to format (newline-delimited CLEF/JSON) /// payloads. Experimental. + /// An that will be used to render log event tokens. /// Logger configuration, allowing configuration to continue. /// A required parameter is null. public static LoggerConfiguration Seq( @@ -153,13 +157,14 @@ public static LoggerConfiguration Seq( LogEventLevel restrictedToMinimumLevel = LevelAlias.Minimum, string? apiKey = null, HttpMessageHandler? messageHandler = null, - ITextFormatter? payloadFormatter = null) + ITextFormatter? payloadFormatter = null, + IFormatProvider? formatProvider = null) { if (loggerAuditSinkConfiguration == null) throw new ArgumentNullException(nameof(loggerAuditSinkConfiguration)); if (serverUrl == null) throw new ArgumentNullException(nameof(serverUrl)); var ingestionApi = new SeqIngestionApiClient(serverUrl, apiKey, messageHandler); - var sink = new SeqAuditSink(ingestionApi, payloadFormatter ?? CreateDefaultFormatter()); + var sink = new SeqAuditSink(ingestionApi, payloadFormatter ?? CreateDefaultFormatter(formatProvider ?? CultureInfo.InvariantCulture)); return loggerAuditSinkConfiguration.Sink(sink, restrictedToMinimumLevel); } } \ No newline at end of file diff --git a/src/Serilog.Sinks.Seq/Sinks/Seq/Formatting/CleanMessageTemplateFormatter.cs b/src/Serilog.Sinks.Seq/Sinks/Seq/Formatting/CleanMessageTemplateFormatter.cs new file mode 100644 index 0000000..d082b35 --- /dev/null +++ b/src/Serilog.Sinks.Seq/Sinks/Seq/Formatting/CleanMessageTemplateFormatter.cs @@ -0,0 +1,146 @@ +// Copyright © Serilog Contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.IO; +using Serilog.Events; +using Serilog.Formatting.Json; +using Serilog.Parsing; + +namespace Serilog.Sinks.Seq.Formatting; + +/// +/// Matches the `:lj` clean formatting style now employed by Serilog.Expressions, Serilog.Sinks.Console, and elsewhere. +/// In this mode, strings embedded in message templates are unquoted, and structured data is rendered as JSON. +/// +/// This implementation is derived from the Serilog.Expressions one, sans theming support, and avoiding the +/// extra dependency. In time there should be core Serilog support for this. +static class CleanMessageTemplateFormatter +{ + static readonly JsonValueFormatter SharedJsonValueFormatter = new("$type"); + + public static string Format(MessageTemplate messageTemplate, IReadOnlyDictionary properties, IFormatProvider? formatProvider) + { + var output = new StringWriter(); + + foreach (var token in messageTemplate.Tokens) + { + switch (token) + { + case TextToken tt: + { + output.Write(tt.Text); + break; + } + case PropertyToken pt: + { + RenderPropertyToken(properties, pt, output, formatProvider); + break; + } + default: + { + output.Write(token); + break; + } + } + } + + return output.ToString(); + } + + static void RenderPropertyToken(IReadOnlyDictionary properties, PropertyToken pt, TextWriter output, IFormatProvider? formatProvider) + { + if (!properties.TryGetValue(pt.PropertyName, out var value)) + { + output.Write(pt.ToString()); + return; + } + + if (pt.Alignment is null) + { + RenderPropertyValueUnaligned(value, output, pt.Format, formatProvider); + return; + } + + var buffer = new StringWriter(); + + RenderPropertyValueUnaligned(value, buffer, pt.Format, formatProvider); + + var result = buffer.ToString(); + + if (result.Length >= pt.Alignment.Value.Width) + output.Write(result); + else + Padding.Apply(output, result, pt.Alignment.Value); + } + + static void RenderPropertyValueUnaligned(LogEventPropertyValue propertyValue, TextWriter output, string? format, IFormatProvider? formatProvider) + { + if (propertyValue is not ScalarValue scalar) + { + SharedJsonValueFormatter.Format(propertyValue, output); + return; + } + + var value = scalar.Value; + + if (value == null) + { + output.Write("null"); + return; + } + + if (value is string str) + { + output.Write(str); + return; + } + + if (value is ValueType) + { + if (value is int or uint or long or ulong or decimal or byte or sbyte or short or ushort) + { + output.Write(((IFormattable)value).ToString(format, formatProvider)); + return; + } + + if (value is double d) + { + output.Write(d.ToString(format, formatProvider)); + return; + } + + if (value is float f) + { + output.Write(f.ToString(format, formatProvider)); + return; + } + + if (value is bool b) + { + output.Write(b); + return; + } + } + + if (value is IFormattable formattable) + { + output.Write(formattable.ToString(format, formatProvider)); + return; + } + + output.Write(value); + } +} \ No newline at end of file diff --git a/src/Serilog.Sinks.Seq/Sinks/Seq/Formatting/Padding.cs b/src/Serilog.Sinks.Seq/Sinks/Seq/Formatting/Padding.cs new file mode 100644 index 0000000..c0db3b1 --- /dev/null +++ b/src/Serilog.Sinks.Seq/Sinks/Seq/Formatting/Padding.cs @@ -0,0 +1,54 @@ +// Copyright © Serilog Contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System.IO; +using System.Linq; +using Serilog.Parsing; + +namespace Serilog.Sinks.Seq.Formatting +{ + static class Padding + { + static readonly char[] PaddingChars = Enumerable.Repeat(' ', 80).ToArray(); + + /// + /// Writes the provided value to the output, applying direction-based padding when is provided. + /// + public static void Apply(TextWriter output, string value, Alignment alignment) + { + if (value.Length >= alignment.Width) + { + output.Write(value); + return; + } + + var pad = alignment.Width - value.Length; + + if (alignment.Direction == AlignmentDirection.Left) + output.Write(value); + + if (pad <= PaddingChars.Length) + { + output.Write(PaddingChars, 0, pad); + } + else + { + output.Write(new string(' ', pad)); + } + + if (alignment.Direction == AlignmentDirection.Right) + output.Write(value); + } + } +} \ No newline at end of file diff --git a/src/Serilog.Sinks.Seq/Sinks/Seq/SeqCompactJsonFormatter.cs b/src/Serilog.Sinks.Seq/Sinks/Seq/SeqCompactJsonFormatter.cs index 9bdfaca..86a8470 100644 --- a/src/Serilog.Sinks.Seq/Sinks/Seq/SeqCompactJsonFormatter.cs +++ b/src/Serilog.Sinks.Seq/Sinks/Seq/SeqCompactJsonFormatter.cs @@ -22,6 +22,7 @@ using Serilog.Formatting.Json; using Serilog.Parsing; using Serilog.Sinks.Seq.Conventions; +using Serilog.Sinks.Seq.Formatting; // ReSharper disable MemberCanBePrivate.Global // ReSharper disable PossibleMultipleEnumeration @@ -41,7 +42,14 @@ public class SeqCompactJsonFormatter: ITextFormatter new PreserveDottedPropertyNames(); readonly JsonValueFormatter _valueFormatter = new("$type"); + readonly IFormatProvider _formatProvider; + /// An that will be used to render log event tokens. + public SeqCompactJsonFormatter(IFormatProvider? formatProvider = null) + { + _formatProvider = formatProvider ?? CultureInfo.InvariantCulture; + } + /// /// Format the log event into the output. Subsequent events will be newline-delimited. /// @@ -49,7 +57,7 @@ public class SeqCompactJsonFormatter: ITextFormatter /// The output. public void Format(LogEvent logEvent, TextWriter output) { - FormatEvent(logEvent, output, _valueFormatter); + FormatEvent(logEvent, output, _valueFormatter, _formatProvider); output.WriteLine(); } @@ -59,7 +67,8 @@ public void Format(LogEvent logEvent, TextWriter output) /// The event to format. /// The output. /// A value formatter for s on the event. - public static void FormatEvent(LogEvent logEvent, TextWriter output, JsonValueFormatter valueFormatter) + /// An that will be used to render log event tokens. + public static void FormatEvent(LogEvent logEvent, TextWriter output, JsonValueFormatter valueFormatter, IFormatProvider formatProvider) { if (logEvent == null) throw new ArgumentNullException(nameof(logEvent)); if (output == null) throw new ArgumentNullException(nameof(output)); @@ -67,9 +76,20 @@ public static void FormatEvent(LogEvent logEvent, TextWriter output, JsonValueFo output.Write("{\"@t\":\""); output.Write(logEvent.Timestamp.UtcDateTime.ToString("O")); + output.Write("\",\"@mt\":"); JsonValueFormatter.WriteQuotedJsonString(logEvent.MessageTemplate.Text, output); + if (!formatProvider.Equals(CultureInfo.InvariantCulture)) + { + // `@m` is normally created during ingestion, however, it must be sent from the client + // to honour non-default IFormatProviders + output.Write(",\"@m\":"); + JsonValueFormatter.WriteQuotedJsonString( + CleanMessageTemplateFormatter.Format(logEvent.MessageTemplate, logEvent.Properties, formatProvider), + output); + } + var tokensWithFormat = logEvent.MessageTemplate.Tokens .OfType() .Where(pt => pt.Format != null); @@ -85,7 +105,7 @@ public static void FormatEvent(LogEvent logEvent, TextWriter output, JsonValueFo output.Write(delim); delim = ","; var space = new StringWriter(); - r.Render(logEvent.Properties, space, CultureInfo.InvariantCulture); + r.Render(logEvent.Properties, space, formatProvider); JsonValueFormatter.WriteQuotedJsonString(space.ToString(), output); } output.Write(']'); diff --git a/test/Serilog.Sinks.Seq.Tests/SeqCompactJsonFormatterTests.cs b/test/Serilog.Sinks.Seq.Tests/SeqCompactJsonFormatterTests.cs index 885ca68..71a5c2c 100644 --- a/test/Serilog.Sinks.Seq.Tests/SeqCompactJsonFormatterTests.cs +++ b/test/Serilog.Sinks.Seq.Tests/SeqCompactJsonFormatterTests.cs @@ -3,6 +3,7 @@ using System; using System.Diagnostics; +using System.Globalization; using System.IO; using System.Linq; using Newtonsoft.Json; @@ -16,13 +17,13 @@ namespace Serilog.Sinks.Seq.Tests; public class SeqCompactJsonFormatterTests { - JObject AssertValidJson(Action act) + JObject AssertValidJson(Action act, IFormatProvider? formatProvider = null, Action? assert = null) { var sw = new StringWriter(); var logger = new LoggerConfiguration() .Destructure.AsScalar() .Destructure.AsScalar() - .WriteTo.TextWriter(new SeqCompactJsonFormatter(), sw) + .WriteTo.TextWriter(new SeqCompactJsonFormatter(formatProvider ?? CultureInfo.InvariantCulture), sw) .CreateLogger(); act(logger); logger.Dispose(); @@ -33,8 +34,40 @@ JObject AssertValidJson(Action act) DateParseHandling = DateParseHandling.None, CheckAdditionalContent = true, }; + + var evt = JsonConvert.DeserializeObject(json, settings)!; + (assert ?? (_ => { }))(evt); + return evt; + } + + [Theory] + [InlineData("fr-FR", "31\u202f415,927 19/07/2024 10:00:59 12\u202f345,67 €", "31\u202f415,927", "12\u202f345,67 €")] + [InlineData("en-US", "31,415.927 7/19/2024 10:00:59 AM $12,345.67", "31,415.927", "$12,345.67")] + public void PropertiesFormatCorrectlyForTheFormatProvider( + string cultureName, + string expectedMessage, + string renderedNumber, + string renderedCurrency) + { + var number = Math.PI * 10000; + var date = new DateTime(2024, 7, 19, 10, 00, 59); + var currency = 12345.67M; - return JsonConvert.DeserializeObject(json, settings)!; + AssertValidJson(log => log.Information("{a:n} {b} {c:C}", number, date, currency), new CultureInfo(cultureName), evt => + { + Assert.Equal(expectedMessage, evt["@m"]!.ToString()); + Assert.Equal(renderedNumber, evt["@r"]![0]!.ToString()); + Assert.Equal(renderedCurrency, evt["@r"]![1]!.ToString()); + }); + } + + [Fact] + public void MessageNotRenderedForDefaultFormatProvider() + { + AssertValidJson(log => log.Information("{a}", 1.234), null, evt => + { + Assert.Null(evt["@m"]); + }); } [Fact] From 2ecf4d0e4a2b17370105667031cbdf9959444d4e Mon Sep 17 00:00:00 2001 From: Liam McLennan Date: Fri, 19 Jul 2024 16:13:38 +1000 Subject: [PATCH 3/6] Comment --- src/Serilog.Sinks.Seq/SeqLoggerConfigurationExtensions.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Serilog.Sinks.Seq/SeqLoggerConfigurationExtensions.cs b/src/Serilog.Sinks.Seq/SeqLoggerConfigurationExtensions.cs index 444094b..789b39a 100644 --- a/src/Serilog.Sinks.Seq/SeqLoggerConfigurationExtensions.cs +++ b/src/Serilog.Sinks.Seq/SeqLoggerConfigurationExtensions.cs @@ -68,7 +68,8 @@ public static class SeqLoggerConfigurationExtensions /// durable log shipping. /// An that will be used to format (newline-delimited CLEF/JSON) /// payloads. Experimental. - /// An that will be used to render log event tokens. Does not apply if `payloadFormatter` is provided. + /// An that will be used to render log event tokens. Does not apply if `payloadFormatter` is provided. + /// If `formatProvider` is provided then event messages will be rendered and included in the payload. /// Logger configuration, allowing configuration to continue. /// A required parameter is null. public static LoggerConfiguration Seq( From 79ad671ad7c8a21778ec49ee55c24045f69b2137 Mon Sep 17 00:00:00 2001 From: Liam McLennan Date: Fri, 19 Jul 2024 16:14:46 +1000 Subject: [PATCH 4/6] Improved comment --- src/Serilog.Sinks.Seq/SeqLoggerConfigurationExtensions.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Serilog.Sinks.Seq/SeqLoggerConfigurationExtensions.cs b/src/Serilog.Sinks.Seq/SeqLoggerConfigurationExtensions.cs index 789b39a..fd2a948 100644 --- a/src/Serilog.Sinks.Seq/SeqLoggerConfigurationExtensions.cs +++ b/src/Serilog.Sinks.Seq/SeqLoggerConfigurationExtensions.cs @@ -68,8 +68,8 @@ public static class SeqLoggerConfigurationExtensions /// durable log shipping. /// An that will be used to format (newline-delimited CLEF/JSON) /// payloads. Experimental. - /// An that will be used to render log event tokens. Does not apply if `payloadFormatter` is provided. - /// If `formatProvider` is provided then event messages will be rendered and included in the payload. + /// An that will be used to render log event tokens. Does not apply if is provided. + /// If is provided then event messages will be rendered and included in the payload. /// Logger configuration, allowing configuration to continue. /// A required parameter is null. public static LoggerConfiguration Seq( From 4f43fa0567521961351ffff21200b04bc255e3b9 Mon Sep 17 00:00:00 2001 From: Liam McLennan Date: Fri, 19 Jul 2024 16:15:44 +1000 Subject: [PATCH 5/6] Removed redundant code --- src/Serilog.Sinks.Seq/SeqLoggerConfigurationExtensions.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Serilog.Sinks.Seq/SeqLoggerConfigurationExtensions.cs b/src/Serilog.Sinks.Seq/SeqLoggerConfigurationExtensions.cs index fd2a948..3ca552a 100644 --- a/src/Serilog.Sinks.Seq/SeqLoggerConfigurationExtensions.cs +++ b/src/Serilog.Sinks.Seq/SeqLoggerConfigurationExtensions.cs @@ -165,7 +165,7 @@ public static LoggerConfiguration Seq( if (serverUrl == null) throw new ArgumentNullException(nameof(serverUrl)); var ingestionApi = new SeqIngestionApiClient(serverUrl, apiKey, messageHandler); - var sink = new SeqAuditSink(ingestionApi, payloadFormatter ?? CreateDefaultFormatter(formatProvider ?? CultureInfo.InvariantCulture)); + var sink = new SeqAuditSink(ingestionApi, payloadFormatter ?? CreateDefaultFormatter(formatProvider)); return loggerAuditSinkConfiguration.Sink(sink, restrictedToMinimumLevel); } } \ No newline at end of file From c2fe74d4d6957982313a4a812a67770ab1d83262 Mon Sep 17 00:00:00 2001 From: Liam McLennan Date: Fri, 19 Jul 2024 16:17:10 +1000 Subject: [PATCH 6/6] Removed unnecessary using --- src/Serilog.Sinks.Seq/Sinks/Seq/ConstrainedBufferedFormatter.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Serilog.Sinks.Seq/Sinks/Seq/ConstrainedBufferedFormatter.cs b/src/Serilog.Sinks.Seq/Sinks/Seq/ConstrainedBufferedFormatter.cs index bf83853..3a5f288 100644 --- a/src/Serilog.Sinks.Seq/Sinks/Seq/ConstrainedBufferedFormatter.cs +++ b/src/Serilog.Sinks.Seq/Sinks/Seq/ConstrainedBufferedFormatter.cs @@ -13,7 +13,6 @@ // limitations under the License. using System; -using System.Globalization; using System.IO; using System.Text; using Serilog.Debugging;