diff --git a/src/OpenTelemetry.Exporter.Prometheus.AspNetCore/CHANGELOG.md b/src/OpenTelemetry.Exporter.Prometheus.AspNetCore/CHANGELOG.md index 909eaa87133..70f03d659bd 100644 --- a/src/OpenTelemetry.Exporter.Prometheus.AspNetCore/CHANGELOG.md +++ b/src/OpenTelemetry.Exporter.Prometheus.AspNetCore/CHANGELOG.md @@ -23,6 +23,10 @@ Notes](../../RELEASENOTES.md). disable scope labels in Prometheus metrics. Defaults to `true`. ([#7436](https://github.com/open-telemetry/opentelemetry-dotnet/pull/7436)) +* Added support for the `dots` and `values` Prometheus UTF-8 name escaping + schemes when negotiated via the `Accept` header. + ([#7439](https://github.com/open-telemetry/opentelemetry-dotnet/pull/7439)) + ## 1.16.0-beta.1 Released 2026-Jun-10 diff --git a/src/OpenTelemetry.Exporter.Prometheus.AspNetCore/PrometheusExporterMiddleware.cs b/src/OpenTelemetry.Exporter.Prometheus.AspNetCore/PrometheusExporterMiddleware.cs index 94c1570e826..008e6febdad 100644 --- a/src/OpenTelemetry.Exporter.Prometheus.AspNetCore/PrometheusExporterMiddleware.cs +++ b/src/OpenTelemetry.Exporter.Prometheus.AspNetCore/PrometheusExporterMiddleware.cs @@ -273,6 +273,7 @@ private static bool TryParse( { // Use the oldest version if no version preference was specified version = isOpenMetrics ? PrometheusProtocol.OpenMetricsV0 : PrometheusProtocol.PrometheusV0; + escaping = null; } else if (version.Major is not > 0) { diff --git a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/CHANGELOG.md b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/CHANGELOG.md index c519eb0ebb7..ddcaa27df69 100644 --- a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/CHANGELOG.md +++ b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/CHANGELOG.md @@ -26,6 +26,10 @@ Notes](../../RELEASENOTES.md). disable scope labels in Prometheus metrics. Defaults to `true`. ([#7436](https://github.com/open-telemetry/opentelemetry-dotnet/pull/7436)) +* Added support for the `dots` and `values` Prometheus UTF-8 name escaping + schemes when negotiated via the `Accept` header. + ([#7439](https://github.com/open-telemetry/opentelemetry-dotnet/pull/7439)) + ## 1.16.0-beta.1 Released 2026-Jun-10 diff --git a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/PrometheusHeadersParser.cs b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/PrometheusHeadersParser.cs index 619818b15e5..95b79390ffb 100644 --- a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/PrometheusHeadersParser.cs +++ b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/PrometheusHeadersParser.cs @@ -25,13 +25,7 @@ internal static PrometheusProtocol Negotiate(string? contentType) const int SupportedProtocols = 4; var preferences = new List<(PrometheusProtocol Protocol, double Quality)>(SupportedProtocols); - var supportedEscapingSchemes = PrometheusProtocol.SupportedEscapingSchemes; - -#if NET8_0_OR_GREATER - SupportedVersions supportedVersions; -#else SupportedVersions supportedVersions; -#endif while (value.Length > 0) { @@ -120,6 +114,7 @@ internal static PrometheusProtocol Negotiate(string? contentType) { // Use the oldest version if no version preference was specified version = isOpenMetrics ? PrometheusProtocol.OpenMetricsV0 : PrometheusProtocol.PrometheusV0; + escaping = null; } else if (version.Major is not > 0) { diff --git a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/Shared/EscapingScheme.cs b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/Shared/EscapingScheme.cs new file mode 100644 index 00000000000..37691cc207d --- /dev/null +++ b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/Shared/EscapingScheme.cs @@ -0,0 +1,30 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +namespace OpenTelemetry.Exporter.Prometheus; + +/// +/// The Prometheus UTF-8 name escaping scheme used when rendering metric and label names. +/// See https://prometheus.io/docs/instrumenting/escaping_schemes/. +/// +internal enum EscapingScheme +{ + /// + /// Discouraged characters are replaced with the _ character. This is the + /// OpenTelemetry default and also covers the legacy (pre-1.0.0) text formats which + /// have no negotiated escaping scheme. + /// + Underscores = 0, + + /// + /// Dots are replaced with _dot_ and underscores are doubled, allowing the + /// original name to be recovered by a Prometheus client. + /// + Dots, + + /// + /// Names that are not valid legacy names are prefixed with U__ and discouraged + /// characters are encoded as their hexadecimal Unicode code point. + /// + Values, +} diff --git a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/Shared/PrometheusCollectionManager.cs b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/Shared/PrometheusCollectionManager.cs index 92da6f6eceb..143469b157c 100644 --- a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/Shared/PrometheusCollectionManager.cs +++ b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/Shared/PrometheusCollectionManager.cs @@ -42,9 +42,9 @@ public PrometheusCollectionManager(PrometheusExporter exporter) internal Func GetElapsedTime { get; set; } #if NET - public ValueTask EnterCollect(PrometheusProtocol protocol) + public ValueTask EnterCollect(in PrometheusProtocol protocol) #else - public Task EnterCollect(PrometheusProtocol protocol) + public Task EnterCollect(in PrometheusProtocol protocol) #endif { CollectionResponse? cachedResponse = null; @@ -213,7 +213,7 @@ public Task EnterCollect(PrometheusProtocol protocol) } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void ExitCollect(PrometheusProtocol protocol) + public void ExitCollect(in PrometheusProtocol protocol) => this.GetProtocolState(protocol).DecrementReaderCount(); #if NET @@ -259,15 +259,15 @@ private void ExitGlobalLock() => Interlocked.Exchange(ref this.globalLockState, 0); [MethodImpl(MethodImplOptions.AggressiveInlining)] - private void IncrementReaderCount(PrometheusProtocol protocol) + private void IncrementReaderCount(in PrometheusProtocol protocol) => this.GetProtocolState(protocol).IncrementReaderCount(); [MethodImpl(MethodImplOptions.AggressiveInlining)] - private bool HasActiveReaders(PrometheusProtocol protocol) + private bool HasActiveReaders(in PrometheusProtocol protocol) => this.protocolStates.TryGetValue(protocol, out var state) && state.HasActiveReaders(); [MethodImpl(MethodImplOptions.AggressiveInlining)] - private bool WaitForReadersToComplete(PrometheusProtocol protocol) + private bool WaitForReadersToComplete(in PrometheusProtocol protocol) { var state = this.GetProtocolState(protocol); var didSpin = false; @@ -347,7 +347,7 @@ private ExportResult OnCollect(in Batch metrics) : ExportResult.Failure; } - private bool TryWriteResponse(PrometheusProtocol protocol, PrometheusProtocolState state, in Batch metrics) + private bool TryWriteResponse(in PrometheusProtocol protocol, PrometheusProtocolState state, in Batch metrics) { try { @@ -455,7 +455,7 @@ private CollectionResult CreateCollectionResult(CollectionContext collectionCont return new CollectionResult(responses); } - private bool TryGetCachedResponse(PrometheusProtocol protocol, out CollectionResponse response) + private bool TryGetCachedResponse(in PrometheusProtocol protocol, out CollectionResponse response) { if (this.protocolStates.TryGetValue(protocol, out var state) && state.GeneratedAt is { } generatedAt && @@ -472,7 +472,7 @@ state.GeneratedAtElapsed is { } generatedAtElapsed && } [MethodImpl(MethodImplOptions.AggressiveInlining)] - private PrometheusProtocolState GetProtocolState(PrometheusProtocol protocol) + private PrometheusProtocolState GetProtocolState(in PrometheusProtocol protocol) => this.protocolStates.GetOrAdd(protocol, static _ => new()); private int WriteTargetInfo(TextFormatSerializer serializer, PrometheusProtocolState state) @@ -637,7 +637,7 @@ public CollectionResult(IReadOnlyDictionary tcs = new(TaskCreationOptions.RunContinuationsAsynchronously); private bool frozen; - public CollectionContext(PrometheusProtocol protocol) + public CollectionContext(in PrometheusProtocol protocol) { this.protocols.Add(protocol); } @@ -755,7 +755,7 @@ public PrometheusProtocol[] FreezeProtocols() public void SetResult(CollectionResult result) => this.tcs.SetResult(result); - public bool TryRegisterProtocol(PrometheusProtocol protocol, bool hasActiveReaders) + public bool TryRegisterProtocol(in PrometheusProtocol protocol, bool hasActiveReaders) { lock (this.gate) { diff --git a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/Shared/PrometheusEscaping.cs b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/Shared/PrometheusEscaping.cs new file mode 100644 index 00000000000..21509cd1fd0 --- /dev/null +++ b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/Shared/PrometheusEscaping.cs @@ -0,0 +1,255 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +using System.Buffers; +using System.Diagnostics; +using System.Text; + +namespace OpenTelemetry.Exporter.Prometheus; + +/// +/// Implements the Prometheus name escaping schemes. +/// +/// +/// The transformations implement the schemes documented at +/// https://prometheus.io/docs/instrumenting/escaping_schemes/. +/// +internal static class PrometheusEscaping +{ + // Upper bound on the number of bytes a single UTF-16 character can expand to. "_dot_" is five + // bytes for the dots scheme and a value-encoded basic-multilingual-plane character ("_FFFD_" + // or "_xxxx_") is at most six bytes; a surrogate pair expands to at most eight bytes across + // two characters, so six per character remains a safe bound. + private const int MaxBytesPerCharacter = 6; + private const string ValuesPrefix = "U__"; + + public static EscapingScheme FromString(string? escaping) => escaping switch + { + PrometheusProtocol.DotsEscaping => EscapingScheme.Dots, + PrometheusProtocol.ValuesEscaping => EscapingScheme.Values, + _ => EscapingScheme.Underscores, + }; + + /// + /// Escapes a fully-constructed metric or label name (including any unit and type-specific + /// suffixes) according to the supplied scheme. The name MUST be escaped as a single unit + /// so that the structural underscores introduced when building the name are doubled and + /// can be reversed by a client. + /// + /// The name to escape. + /// The escaping scheme to apply. + /// + /// when escaping a metric name (where a colon is a valid legacy + /// character); when escaping a label name (where a colon is not). + /// + /// The escaped name. Always a valid legacy (ASCII) name for the dots and values schemes. + public static string EscapeName(string name, EscapingScheme scheme, bool isMetricName = true) + { + if (string.IsNullOrEmpty(name) || scheme == EscapingScheme.Underscores) + { + return name; + } + + if (scheme == EscapingScheme.Values && CanWriteValuesNameUnchanged(name, isMetricName)) + { + return name; + } + + var rented = ArrayPool.Shared.Rent((scheme == EscapingScheme.Values ? 3 : 0) + (name.Length * MaxBytesPerCharacter)); + + try + { + var length = EscapeName(rented, 0, name, scheme, isMetricName); +#if NET + return Encoding.ASCII.GetString(rented.AsSpan(0, length)); +#else + return Encoding.ASCII.GetString(rented, 0, length); +#endif + } + finally + { + ArrayPool.Shared.Return(rented); + } + } + + /// + /// Escapes a fully-constructed metric or label name directly into + /// according to the supplied scheme, avoiding an intermediate string allocation. The scheme + /// MUST be or ; the + /// underscores scheme is handled using the pre-computed names. + /// + /// The buffer to write to. + /// The current position in the buffer. + /// The name to escape. + /// The escaping scheme to apply. + /// + /// when escaping a metric name (where a colon is a valid legacy + /// character); when escaping a label name (where a colon is not). + /// + /// The new cursor position after writing the (ASCII) escaped name. + public static int EscapeName(byte[] buffer, int cursor, string name, EscapingScheme scheme, bool isMetricName = true) + { + Debug.Assert(scheme is EscapingScheme.Dots or EscapingScheme.Values, $"Unexpected escaping scheme: {scheme}"); + + if (string.IsNullOrEmpty(name)) + { + return cursor; + } + + if (scheme == EscapingScheme.Values && CanWriteValuesNameUnchanged(name, isMetricName)) + { + return WriteAscii(buffer, cursor, name); + } + + if (scheme == EscapingScheme.Values) + { + cursor = WriteAscii(buffer, cursor, ValuesPrefix); + } + + var escapeValuesPrefix = scheme == EscapingScheme.Values && name.StartsWith(ValuesPrefix, StringComparison.Ordinal); + var index = 0; + + while (index < name.Length) + { + var codePoint = GetCodePoint(name, index, out var charsConsumed, out var isValidRune); + + if (codePoint == '_') + { + cursor = WriteAscii(buffer, cursor, "__"); + } + else if (scheme == EscapingScheme.Dots && codePoint == '.') + { + cursor = WriteAscii(buffer, cursor, "_dot_"); + } + else if (escapeValuesPrefix && index == 0) + { + cursor = WriteHexCodePoint(buffer, cursor, codePoint); + } + else if (IsValidLegacyRune(codePoint, index == 0, isMetricName)) + { + buffer[cursor++] = unchecked((byte)codePoint); + } + else if (scheme == EscapingScheme.Dots) + { + buffer[cursor++] = unchecked((byte)'_'); + } + else if (!isValidRune) + { + cursor = WriteAscii(buffer, cursor, "_FFFD_"); + } + else + { + cursor = WriteHexCodePoint(buffer, cursor, codePoint); + } + + index += charsConsumed; + } + + return cursor; + } + + private static bool CanWriteValuesNameUnchanged(string name, bool isMetricName) + { + // A pre-existing "U__" prefix must itself be encoded; otherwise a name such as + // "U__foo_2e_bar" collides with the values encoding of "foo.bar". + return IsValidLegacyName(name, isMetricName) && !name.StartsWith(ValuesPrefix, StringComparison.Ordinal); + } + + private static bool IsValidLegacyName(string name, bool isMetricName) + { + var index = 0; + + while (index < name.Length) + { + var codePoint = GetCodePoint(name, index, out var charsConsumed, out _); + + if (!IsValidLegacyRune(codePoint, index == 0, isMetricName)) + { + return false; + } + + index += charsConsumed; + } + + return true; + } + + // A colon is a valid legacy character in a metric name but not in a label name, so it is only + // treated as a valid rune when escaping a metric name. See the legacy name grammars at + // https://prometheus.io/docs/concepts/data_model/ and the OpenMetrics label-name grammar. + private static bool IsValidLegacyRune(int codePoint, bool isFirst, bool isMetricName) => + codePoint is (>= 'a' and <= 'z') or (>= 'A' and <= 'Z') or '_' || + (isMetricName && codePoint == ':') || + (!isFirst && codePoint is >= '0' and <= '9'); + + private static int GetCodePoint(string value, int index, out int charsConsumed, out bool isValidRune) + { +#if NET + var status = Rune.DecodeFromUtf16(value.AsSpan(index), out var rune, out charsConsumed); + + isValidRune = status == OperationStatus.Done; + + return rune.Value; +#else + const int UnicodeReplacementCharacter = 0xFFFD; + + var character = value[index]; + + if (char.IsHighSurrogate(character) && + index + 1 < value.Length && + char.IsLowSurrogate(value[index + 1])) + { + charsConsumed = 2; + isValidRune = true; + return char.ConvertToUtf32(character, value[index + 1]); + } + + if (char.IsSurrogate(character)) + { + // An unpaired surrogate is not a valid Unicode scalar value. + charsConsumed = 1; + isValidRune = false; + return UnicodeReplacementCharacter; + } + + charsConsumed = 1; + isValidRune = true; + + return character; +#endif + } + + private static int WriteAscii(byte[] buffer, int cursor, string value) + { + for (var i = 0; i < value.Length; i++) + { + buffer[cursor++] = unchecked((byte)value[i]); + } + + return cursor; + } + + private static int WriteHexCodePoint(byte[] buffer, int cursor, int codePoint) + { + // Writes "__" with no leading zeros, matching the reference implementation's + // use of strconv.FormatInt(value, 16). Code points are at most 0x10FFFF (six hex digits). + buffer[cursor++] = unchecked((byte)'_'); + + var started = false; + + for (var shift = 20; shift >= 0; shift -= 4) + { + var nibble = (codePoint >> shift) & 0xF; + + if (nibble != 0 || started || shift == 0) + { + buffer[cursor++] = unchecked((byte)(nibble < 10 ? '0' + nibble : 'a' + (nibble - 10))); + started = true; + } + } + + buffer[cursor++] = unchecked((byte)'_'); + + return cursor; + } +} diff --git a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/Shared/PrometheusMetric.cs b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/Shared/PrometheusMetric.cs index 57a22e875a9..f812e0d4015 100644 --- a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/Shared/PrometheusMetric.cs +++ b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/Shared/PrometheusMetric.cs @@ -9,107 +9,39 @@ namespace OpenTelemetry.Exporter.Prometheus; internal sealed class PrometheusMetric { + private readonly string rawName; + private readonly bool disableTotalNameSuffixForCounters; + private readonly NameSet underscoreNames; + private NameSet? dotsNames; + private NameSet? valuesNames; + public PrometheusMetric(string name, string unit, PrometheusType type, bool disableTotalNameSuffixForCounters) { - // The metric name is - // required to match the regex: `[a-zA-Z_:]([a-zA-Z0-9_:])*`. Invalid characters - // in the metric name MUST be replaced with the `_` character. Multiple - // consecutive `_` characters MUST be replaced with a single `_` character. - // https://github.com/open-telemetry/opentelemetry-specification/blob/b2f923fb1650dde1f061507908b834035506a796/specification/compatibility/prometheus_and_openmetrics.md#L230-L233 - var sanitizedName = SanitizeMetricName(name); - var openMetricsName = type == PrometheusType.Counter - ? RemoveOpenMetricsCounterNameSuffix(name) - : name; - - string? sanitizedUnit = null; - if (!string.IsNullOrEmpty(unit)) - { - sanitizedUnit = GetUnit(unit); - - // The resulting unit SHOULD be added to the metric as - // [OpenMetrics UNIT metadata](https://github.com/prometheus/OpenMetrics/blob/v1.0.0/specification/OpenMetrics.md#metricfamily) - // and as a suffix to the metric name. The unit suffix comes before any type-specific suffixes. - // https://github.com/open-telemetry/opentelemetry-specification/blob/3dfb383fe583e3b74a2365c5a1d90256b273ee76/specification/compatibility/prometheus_and_openmetrics.md#metric-metadata-1 - // Each name is checked independently: openMetricsName has the _total counter suffix stripped - // (by RemoveOpenMetricsCounterNameSuffix above), so it may already end with the unit even - // when sanitizedName (which still carries _total) does not. For counter sanitizedName, also - // check for the unit immediately before _total (e.g. "db_bytes_total" with unit "bytes"). - if (!sanitizedName.EndsWith(sanitizedUnit, StringComparison.Ordinal) && - (type != PrometheusType.Counter || !sanitizedName.EndsWith($"{sanitizedUnit}_total", StringComparison.Ordinal))) - { - sanitizedName += $"_{sanitizedUnit}"; - } - - if (!openMetricsName.EndsWith(sanitizedUnit, StringComparison.Ordinal)) - { - openMetricsName += $"_{sanitizedUnit}"; - } - } - - openMetricsName = EscapeOpenMetricsName(openMetricsName); - - // If the metric name for monotonic Sum metric points does not end in a suffix of `_total` a suffix of `_total` MUST be added by default, otherwise the name MUST remain unchanged. - // Exporters SHOULD provide a configuration option to disable the addition of `_total` suffixes. - // https://github.com/open-telemetry/opentelemetry-specification/blob/b2f923fb1650dde1f061507908b834035506a796/specification/compatibility/prometheus_and_openmetrics.md#L286 - // Note that we no longer append '_ratio' for units that are '1', see: https://github.com/open-telemetry/opentelemetry-specification/issues/4058 - if (type == PrometheusType.Counter && !sanitizedName.EndsWith("_total", StringComparison.Ordinal) && !disableTotalNameSuffixForCounters) - { - sanitizedName += "_total"; - } - - // For counters requested using OpenMetrics format, the MetricFamily name MUST be suffixed with '_total', regardless of the setting to disable the 'total' suffix. - // https://github.com/prometheus/OpenMetrics/blob/v1.0.0/specification/OpenMetrics.md#counter-1 - if (type == PrometheusType.Counter && !openMetricsName.EndsWith("_total", StringComparison.Ordinal)) - { - openMetricsName += "_total"; - } - - // In OpenMetrics format, the UNIT, TYPE and HELP metadata must be suffixed with the unit (handled above), and not the '_total' suffix, as in the case for counters. - // https://github.com/prometheus/OpenMetrics/blob/v1.0.0/specification/OpenMetrics.md#unit - var openMetricsMetadataName = type == PrometheusType.Counter - ? RemoveOpenMetricsCounterNameSuffix(openMetricsName) - : openMetricsName; - - this.Name = sanitizedName; - this.OpenMetricsName = openMetricsName; - this.OpenMetricsMetadataName = openMetricsMetadataName; - this.Unit = sanitizedUnit; + this.rawName = name; + this.disableTotalNameSuffixForCounters = disableTotalNameSuffixForCounters; this.Type = type; - this.NameBytes = ConvertToAsciiBytes(sanitizedName); - this.OpenMetricsNameBytes = ConvertToAsciiBytes(openMetricsName); - this.OpenMetricsMetadataNameBytes = ConvertToAsciiBytes(openMetricsMetadataName); - this.UnitBytes = sanitizedUnit == null ? null : ConvertToAsciiBytes(sanitizedUnit); - static byte[] ConvertToAsciiBytes(string value) - { - // Metric names and units are sanitized before conversion, so every character here is ASCII - var bytes = new byte[value.Length]; - - for (var i = 0; i < value.Length; i++) - { - bytes[i] = unchecked((byte)value[i]); - } + var sanitizedUnit = string.IsNullOrEmpty(unit) ? null : GetUnit(unit); + this.Unit = sanitizedUnit; + this.UnitBytes = sanitizedUnit == null ? null : ConvertToBytes(sanitizedUnit); - return bytes; - } + // The underscores names are always required (they are also used by the legacy text + // formats, which have no negotiated escaping) so they are computed eagerly. Other + // escapings' name sets are computed lazily on first use, so the overhead of an + // escaping scheme is only paid when that scheme is actually scraped from the server. + this.underscoreNames = BuildUnderscoreNames(name, sanitizedUnit, type, disableTotalNameSuffixForCounters); } - public string Name { get; } + public string Name => this.underscoreNames.Name; - public string OpenMetricsName { get; } + public string OpenMetricsName => this.underscoreNames.OpenMetricsName; - public string OpenMetricsMetadataName { get; } + public string OpenMetricsMetadataName => this.underscoreNames.OpenMetricsMetadataName; public string? Unit { get; } public PrometheusType Type { get; } - internal byte[] NameBytes { get; } - - internal byte[] OpenMetricsNameBytes { get; } - - internal byte[] OpenMetricsMetadataNameBytes { get; } - internal byte[]? UnitBytes { get; } public static PrometheusMetric Create(Metric metric, bool disableTotalNameSuffixForCounters) @@ -295,6 +227,146 @@ UpDownCounter becomes gauge }; } + /// + /// Returns the metric name set for the requested escaping scheme, computing (and caching) the + /// names lazily on first use. + /// + /// The escaping scheme to use. + /// The metric name set for the requested escaping scheme. + internal NameSet GetNameSet(EscapingScheme escaping) => escaping switch + { + EscapingScheme.Dots => this.dotsNames ??= BuildEscapedNames(this.rawName, this.Unit, this.Type, this.disableTotalNameSuffixForCounters, EscapingScheme.Dots), + EscapingScheme.Values => this.valuesNames ??= BuildEscapedNames(this.rawName, this.Unit, this.Type, this.disableTotalNameSuffixForCounters, EscapingScheme.Values), + _ => this.underscoreNames, + }; + + private static byte[] ConvertToBytes(string value) + { + // Metric names and units are sanitized before conversion, so every character here is ASCII + var bytes = new byte[value.Length]; + + for (var i = 0; i < value.Length; i++) + { + bytes[i] = unchecked((byte)value[i]); + } + + return bytes; + } + + private static NameSet BuildUnderscoreNames(string name, string? sanitizedUnit, PrometheusType type, bool disableTotalNameSuffixForCounters) + { + // The metric name is + // required to match the regex: `[a-zA-Z_:]([a-zA-Z0-9_:])*`. Invalid characters + // in the metric name MUST be replaced with the `_` character. Multiple + // consecutive `_` characters MUST be replaced with a single `_` character. + // https://github.com/open-telemetry/opentelemetry-specification/blob/b2f923fb1650dde1f061507908b834035506a796/specification/compatibility/prometheus_and_openmetrics.md#L230-L233 + var sanitizedName = SanitizeMetricName(name); + var openMetricsName = type == PrometheusType.Counter + ? RemoveOpenMetricsCounterNameSuffix(name) + : name; + + if (sanitizedUnit != null) + { + // The resulting unit SHOULD be added to the metric as + // [OpenMetrics UNIT metadata](https://github.com/prometheus/OpenMetrics/blob/v1.0.0/specification/OpenMetrics.md#metricfamily) + // and as a suffix to the metric name. The unit suffix comes before any type-specific suffixes. + // https://github.com/open-telemetry/opentelemetry-specification/blob/3dfb383fe583e3b74a2365c5a1d90256b273ee76/specification/compatibility/prometheus_and_openmetrics.md#metric-metadata-1 + // Each name is checked independently: openMetricsName has the _total counter suffix stripped + // (by RemoveOpenMetricsCounterNameSuffix above), so it may already end with the unit even + // when sanitizedName (which still carries _total) does not. For counter sanitizedName, also + // check for the unit immediately before _total (e.g. "db_bytes_total" with unit "bytes"). + if (!sanitizedName.EndsWith(sanitizedUnit, StringComparison.Ordinal) && + (type != PrometheusType.Counter || !sanitizedName.EndsWith($"{sanitizedUnit}_total", StringComparison.Ordinal))) + { + sanitizedName += $"_{sanitizedUnit}"; + } + + if (!openMetricsName.EndsWith(sanitizedUnit, StringComparison.Ordinal)) + { + openMetricsName += $"_{sanitizedUnit}"; + } + } + + openMetricsName = EscapeOpenMetricsName(openMetricsName); + + // If the metric name for monotonic Sum metric points does not end in a suffix of `_total` a suffix of `_total` MUST be added by default, otherwise the name MUST remain unchanged. + // Exporters SHOULD provide a configuration option to disable the addition of `_total` suffixes. + // https://github.com/open-telemetry/opentelemetry-specification/blob/b2f923fb1650dde1f061507908b834035506a796/specification/compatibility/prometheus_and_openmetrics.md#L286 + // Note that we no longer append '_ratio' for units that are '1', see: https://github.com/open-telemetry/opentelemetry-specification/issues/4058 + if (type == PrometheusType.Counter && !sanitizedName.EndsWith("_total", StringComparison.Ordinal) && !disableTotalNameSuffixForCounters) + { + sanitizedName += "_total"; + } + + // For counters requested using OpenMetrics format, the MetricFamily name MUST be suffixed with '_total', regardless of the setting to disable the 'total' suffix. + // https://github.com/prometheus/OpenMetrics/blob/v1.0.0/specification/OpenMetrics.md#counter-1 + if (type == PrometheusType.Counter && !openMetricsName.EndsWith("_total", StringComparison.Ordinal)) + { + openMetricsName += "_total"; + } + + // In OpenMetrics format, the UNIT, TYPE and HELP metadata must be suffixed with the unit (handled above), and not the '_total' suffix, as in the case for counters. + // https://github.com/prometheus/OpenMetrics/blob/v1.0.0/specification/OpenMetrics.md#unit + var openMetricsMetadataName = type == PrometheusType.Counter + ? RemoveOpenMetricsCounterNameSuffix(openMetricsName) + : openMetricsName; + + return new(sanitizedName, openMetricsName, openMetricsMetadataName); + } + + private static NameSet BuildEscapedNames( + string name, + string? sanitizedUnit, + PrometheusType type, + bool disableTotalNameSuffixForCounters, + EscapingScheme escaping) + { + // The escaping scheme is applied to the metric family name (the original name plus any + // unit suffix). The type-specific '_total' suffix is a structural suffix that Prometheus + // strips to find the family (like the '_bucket'/'_sum'/'_count' suffixes added during + // serialization), so it is appended literally to the escaped family name rather than + // escaped with it. + var nameFamily = name; + var openMetricsFamily = type == PrometheusType.Counter + ? RemoveOpenMetricsCounterNameSuffix(name) + : name; + + if (sanitizedUnit != null) + { + // Each name is checked independently: openMetricsFamily has the _total counter suffix stripped + // (by RemoveOpenMetricsCounterNameSuffix above), so it may already end with the unit even + // when nameFamily (which still carries _total) does not. For counter nameFamily, also + // check for the unit immediately before _total (e.g. "db_bytes_total" with unit "bytes"). + if (!nameFamily.EndsWith(sanitizedUnit, StringComparison.Ordinal) && + (type != PrometheusType.Counter || !nameFamily.EndsWith($"{sanitizedUnit}_total", StringComparison.Ordinal))) + { + nameFamily += $"_{sanitizedUnit}"; + } + + if (!openMetricsFamily.EndsWith(sanitizedUnit, StringComparison.Ordinal)) + { + openMetricsFamily += $"_{sanitizedUnit}"; + } + } + + var escapedName = PrometheusEscaping.EscapeName(nameFamily, escaping); + + // The metadata (UNIT/TYPE/HELP) name is the escaped OpenMetrics family without the + // '_total' suffix, as in the case for counters. + var openMetricsMetadataName = PrometheusEscaping.EscapeName(openMetricsFamily, escaping); + + if (type == PrometheusType.Counter && !nameFamily.EndsWith("_total", StringComparison.Ordinal) && !disableTotalNameSuffixForCounters) + { + escapedName += "_total"; + } + + var openMetricsName = type == PrometheusType.Counter && !openMetricsFamily.EndsWith("_total", StringComparison.Ordinal) + ? openMetricsMetadataName + "_total" + : openMetricsMetadataName; + + return new(escapedName, openMetricsName, openMetricsMetadataName); + } + private static string RemoveOpenMetricsCounterNameSuffix(string metricName) => metricName.EndsWith("_total", StringComparison.Ordinal) ? metricName.Substring(0, metricName.Length - 6) : metricName; @@ -404,4 +476,32 @@ private static bool TryProcessRateUnits(string updatedUnit, [NotNullWhen(true)] "y" => "year", _ => perUnit.ToString(), }; + + /// + /// Represents a set of pre-computed metric family names (and their ASCII byte representations) for a single escaping scheme. + /// + internal sealed class NameSet + { + public NameSet(string name, string openMetricsName, string openMetricsMetadataName) + { + this.Name = name; + this.OpenMetricsName = openMetricsName; + this.OpenMetricsMetadataName = openMetricsMetadataName; + this.NameBytes = ConvertToBytes(name); + this.OpenMetricsNameBytes = ConvertToBytes(openMetricsName); + this.OpenMetricsMetadataNameBytes = ConvertToBytes(openMetricsMetadataName); + } + + public string Name { get; } + + public string OpenMetricsName { get; } + + public string OpenMetricsMetadataName { get; } + + public byte[] NameBytes { get; } + + public byte[] OpenMetricsNameBytes { get; } + + public byte[] OpenMetricsMetadataNameBytes { get; } + } } diff --git a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/Shared/PrometheusProtocol.cs b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/Shared/PrometheusProtocol.cs index 8dc2252aedb..a57ee151434 100644 --- a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/Shared/PrometheusProtocol.cs +++ b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/Shared/PrometheusProtocol.cs @@ -18,7 +18,9 @@ namespace OpenTelemetry.Exporter.Prometheus; internal readonly struct PrometheusProtocol : IEquatable { public const string AllowUtf8Escaping = "allow-utf-8"; + public const string DotsEscaping = "dots"; public const string UnderscoresEscaping = "underscores"; + public const string ValuesEscaping = "values"; public const string OpenMetricsMediaType = "application/openmetrics-text"; public const string PrometheusTextMediaType = "text/plain"; @@ -30,11 +32,13 @@ namespace OpenTelemetry.Exporter.Prometheus; public static readonly PrometheusProtocol Fallback = new(PrometheusTextMediaType, null, PrometheusV0, false); - // TODO Support other escaping schemes, including at least "allow-utf-8". + // TODO Support the "allow-utf-8" escaping scheme. // See https://github.com/open-telemetry/opentelemetry-dotnet/issues/7246. internal static readonly SupportedEscapingSchemes SupportedEscapingSchemes = [ + DotsEscaping, UnderscoresEscaping, + ValuesEscaping, ]; internal static readonly SupportedVersions SupportedOpenMetricsVersions = @@ -53,6 +57,7 @@ public PrometheusProtocol(string mediaType, string? escaping, Version version, b { this.MediaType = mediaType; this.Escaping = escaping; + this.EscapingScheme = PrometheusEscaping.FromString(escaping); this.IsOpenMetrics = isOpenMetrics; this.Version = version; } @@ -61,11 +66,13 @@ public PrometheusProtocol(string mediaType, string? escaping, Version version, b public readonly string? Escaping { get; } + public readonly EscapingScheme EscapingScheme { get; } + public readonly bool IsOpenMetrics { get; } public readonly Version Version { get; } - public static string GetContentType(PrometheusProtocol protocol) + public static string GetContentType(in PrometheusProtocol protocol) { var builder = new StringBuilder() .Append(protocol.MediaType) @@ -73,7 +80,7 @@ public static string GetContentType(PrometheusProtocol protocol) .Append(protocol.Version.ToString(3)) .Append("; charset=utf-8"); - if (protocol.Escaping is not null) + if (protocol.Escaping is { Length: > 0 }) { builder.Append("; escaping=") .Append(protocol.Escaping); @@ -84,6 +91,7 @@ public static string GetContentType(PrometheusProtocol protocol) public bool Equals(PrometheusProtocol other) => this.IsOpenMetrics == other.IsOpenMetrics && + this.EscapingScheme == other.EscapingScheme && this.MediaType == other.MediaType && this.Escaping == other.Escaping && this.Version == other.Version; @@ -94,10 +102,11 @@ public override bool Equals([NotNullWhen(true)] object? obj) public override int GetHashCode() { #if NET - return HashCode.Combine(this.MediaType, this.Escaping, this.IsOpenMetrics, this.Version); + return HashCode.Combine(this.MediaType, this.EscapingScheme, this.Escaping, this.IsOpenMetrics, this.Version); #else var hashCode = this.MediaType.GetHashCode(); + hashCode = (hashCode * 397) ^ (int)this.EscapingScheme; hashCode = (hashCode * 397) ^ (this.Escaping?.GetHashCode() ?? 0); hashCode = (hashCode * 397) ^ this.IsOpenMetrics.GetHashCode(); hashCode = (hashCode * 397) ^ this.Version.GetHashCode(); diff --git a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/Shared/Serialization/OpenMetricsSerializer.cs b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/Shared/Serialization/OpenMetricsSerializer.cs index ed9adb03cca..44f364460d8 100644 --- a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/Shared/Serialization/OpenMetricsSerializer.cs +++ b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/Shared/Serialization/OpenMetricsSerializer.cs @@ -17,7 +17,7 @@ internal abstract class OpenMetricsSerializer : TextFormatSerializer protected override string TargetInfoTypeValue => "info"; public override string GetMetadataName(PrometheusMetric metric) - => metric.OpenMetricsMetadataName; + => metric.GetNameSet(this.Escaping).OpenMetricsMetadataName; internal static bool ShouldPreferExemplar(DateTimeOffset currentTimestamp, DateTimeOffset candidateTimestamp) => currentTimestamp <= candidateTimestamp; @@ -41,10 +41,10 @@ internal static bool IsHistogramBucketExemplarMatch( } protected override ReadOnlySpan GetMetricNameBytes(PrometheusMetric metric) - => metric.OpenMetricsNameBytes; + => metric.GetNameSet(this.Escaping).OpenMetricsNameBytes; protected override ReadOnlySpan GetMetricMetadataNameBytes(PrometheusMetric metric) - => metric.OpenMetricsMetadataNameBytes; + => metric.GetNameSet(this.Escaping).OpenMetricsMetadataNameBytes; protected override int WriteExplicitBound(byte[] buffer, int cursor, double explicitBound) => WriteCanonicalLabelValue(buffer, cursor, explicitBound); @@ -62,7 +62,7 @@ protected override int WriteCounterExemplar( if (prometheusMetric.Type == PrometheusType.Counter && TryGetLatestExemplar(metricPoint, out var exemplar)) { - cursor = WriteExemplar(buffer, cursor, in exemplar, isLongValue); + cursor = this.WriteExemplar(buffer, cursor, in exemplar, isLongValue); } return cursor; @@ -88,7 +88,7 @@ protected override int WriteHistogramBucketExemplar(byte[] buffer, int cursor, i { if (TryGetLatestHistogramBucketExemplar(metricPoint, lowerBoundExclusive, upperBoundInclusive, out var exemplar)) { - cursor = WriteExemplar(buffer, cursor, in exemplar, isLongValue: false); + cursor = this.WriteExemplar(buffer, cursor, in exemplar, isLongValue: false); } return cursor; @@ -164,10 +164,8 @@ private int WriteCreatedMetric( in TextFormatSerializerOptions options, IReadOnlyCollection? reservedOutputKeys = null) { - cursor = this.WriteMetricMetadataName(buffer, cursor, prometheusMetric); - - cursor = WriteAsciiStringNoEscape(buffer, cursor, "_created"); - cursor = WriteTags(buffer, cursor, metric, metricPoint.Tags, options, reservedOutputKeys: reservedOutputKeys); + cursor = this.WriteMetricNameWithSuffix(buffer, cursor, prometheusMetric, "_created"); + cursor = this.WriteTags(buffer, cursor, metric, metricPoint.Tags, options, reservedOutputKeys: reservedOutputKeys); buffer[cursor++] = unchecked((byte)' '); diff --git a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/Shared/Serialization/PrometheusTextSerializer.cs b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/Shared/Serialization/PrometheusTextSerializer.cs index 0c6dba78ae3..3e7133b8f7d 100644 --- a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/Shared/Serialization/PrometheusTextSerializer.cs +++ b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/Shared/Serialization/PrometheusTextSerializer.cs @@ -21,13 +21,13 @@ internal abstract class PrometheusTextSerializer : TextFormatSerializer protected override string TargetInfoTypeValue => "gauge"; public override string GetMetadataName(PrometheusMetric metric) - => metric.Name; + => metric.GetNameSet(this.Escaping).Name; protected override ReadOnlySpan GetMetricNameBytes(PrometheusMetric metric) - => metric.NameBytes; + => metric.GetNameSet(this.Escaping).NameBytes; protected override ReadOnlySpan GetMetricMetadataNameBytes(PrometheusMetric metric) - => metric.NameBytes; + => metric.GetNameSet(this.Escaping).NameBytes; protected override int WriteExplicitBound(byte[] buffer, int cursor, double explicitBound) => WriteDouble(buffer, cursor, explicitBound); diff --git a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/Shared/Serialization/TextFormatSerializer.cs b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/Shared/Serialization/TextFormatSerializer.cs index 9808974f5a4..3080567995a 100644 --- a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/Shared/Serialization/TextFormatSerializer.cs +++ b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/Shared/Serialization/TextFormatSerializer.cs @@ -86,6 +86,8 @@ internal abstract class TextFormatSerializer 1e10d, ]; + private string[]? reservedExemplarOutputKeys; + public static OpenMetricsV0Serializer OpenMetricsV0 => field ??= new(); public static OpenMetricsV1Serializer OpenMetricsV1 => field ??= new(); @@ -94,6 +96,11 @@ internal abstract class TextFormatSerializer public static PrometheusTextV1Serializer PrometheusV1 => field ??= new(); + /// + /// Gets the name escaping scheme to use for serialization. + /// + public EscapingScheme Escaping { get; private init; } = EscapingScheme.Underscores; + /// /// Gets the type metadata value written for metrics that have no dedicated Prometheus type. /// @@ -109,21 +116,26 @@ internal abstract class TextFormatSerializer /// protected abstract string TargetInfoTypeValue { get; } - public static TextFormatSerializer GetSerializer(PrometheusProtocol protocol) => protocol switch + public static TextFormatSerializer GetSerializer(in PrometheusProtocol protocol) { - { IsOpenMetrics: true } => protocol.Version.Major switch - { - 0 => OpenMetricsV0, - 1 => OpenMetricsV1, - _ => throw new NotSupportedException($"Unsupported OpenMetrics version: {protocol.Version}."), - }, - { IsOpenMetrics: false } => protocol.Version.Major switch - { - 0 => PrometheusV0, - 1 => PrometheusV1, - _ => throw new NotSupportedException($"Unsupported Prometheus version: {protocol.Version}."), - }, - }; + var escaping = protocol.EscapingScheme; + + return protocol switch + { + { IsOpenMetrics: true } => protocol.Version.Major switch + { + 0 => OpenMetricsV0, + 1 => escaping == EscapingScheme.Underscores ? OpenMetricsV1 : new OpenMetricsV1Serializer() { Escaping = escaping }, + _ => throw new NotSupportedException($"Unsupported OpenMetrics version: {protocol.Version}."), + }, + { IsOpenMetrics: false } => protocol.Version.Major switch + { + 0 => PrometheusV0, + 1 => escaping == EscapingScheme.Underscores ? PrometheusV1 : new PrometheusTextV1Serializer() { Escaping = escaping }, + _ => throw new NotSupportedException($"Unsupported Prometheus version: {protocol.Version}."), + }, + }; + } [MethodImpl(MethodImplOptions.AggressiveInlining)] public static int WriteEof(byte[] buffer, int cursor) @@ -169,7 +181,7 @@ public int WriteMetric( { // Counter and Gauge cursor = this.WriteMetricName(buffer, cursor, prometheusMetric); - cursor = WriteTags(buffer, cursor, metric, metricPoint.Tags, options); + cursor = this.WriteTags(buffer, cursor, metric, metricPoint.Tags, options); buffer[cursor++] = unchecked((byte)' '); @@ -198,7 +210,7 @@ public int WriteMetric( foreach (ref readonly var metricPoint in metric.GetMetricPoints()) { var tags = metricPoint.Tags; - var serializedTags = SerializeTags(metric, tags, options, ReservedHistogramLabelNames); + var serializedTags = this.SerializeTags(metric, tags, options, ReservedHistogramLabelNames); var hasNegativeBucketBounds = false; var previousBound = double.NegativeInfinity; @@ -209,8 +221,8 @@ public int WriteMetric( totalCount += histogramMeasurement.BucketCount; - cursor = this.WriteMetricName(buffer, cursor, prometheusMetric); - cursor = WriteAsciiStringNoEscape(buffer, cursor, "_bucket{"); + cursor = this.WriteMetricNameWithSuffix(buffer, cursor, prometheusMetric, "_bucket"); + buffer[cursor++] = unchecked((byte)'{'); cursor = WriteSerializedTagValues(buffer, cursor, serializedTags, appendTrailingComma: true); cursor = WriteAsciiStringNoEscape(buffer, cursor, "le=\""); @@ -239,8 +251,7 @@ public int WriteMetric( // OpenMetrics histograms with negative bucket thresholds MUST NOT expose // _sum and therefore MUST NOT expose _count. // See https://prometheus.io/docs/specs/om/open_metrics_spec/#histogram-1 - cursor = this.WriteMetricName(buffer, cursor, prometheusMetric); - cursor = WriteAsciiStringNoEscape(buffer, cursor, "_sum"); + cursor = this.WriteMetricNameWithSuffix(buffer, cursor, prometheusMetric, "_sum"); cursor = WriteSerializedTags(buffer, cursor, serializedTags); buffer[cursor++] = unchecked((byte)' '); @@ -250,8 +261,7 @@ public int WriteMetric( buffer[cursor++] = AsciiLineFeed; // Histogram count - cursor = this.WriteMetricName(buffer, cursor, prometheusMetric); - cursor = WriteAsciiStringNoEscape(buffer, cursor, "_count"); + cursor = this.WriteMetricNameWithSuffix(buffer, cursor, prometheusMetric, "_count"); cursor = WriteSerializedTags(buffer, cursor, serializedTags); buffer[cursor++] = unchecked((byte)' '); @@ -300,7 +310,7 @@ public int WriteTargetInfo(byte[] buffer, int cursor, Resource resource) do { var attribute = attributes.Current; - AddLabel(attribute.Key, attribute.Value, ref labels); + this.AddLabel(attribute.Key, attribute.Value, ref labels); } while (attributes.MoveNext()); @@ -526,7 +536,53 @@ internal static int WriteLabel(byte[] buffer, int cursor, string labelKey, objec } [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static int WriteExemplar(byte[] buffer, int cursor, in Exemplar exemplar, bool isLongValue) + internal static int WriteUnixTimeSeconds(byte[] buffer, int cursor, DateTimeOffset value) => +#if NET + WriteDouble(buffer, cursor, (value.UtcDateTime.Ticks - DateTimeOffset.UnixEpoch.Ticks) / (double)TimeSpan.TicksPerSecond); +#else + WriteDouble(buffer, cursor, (value.UtcDateTime.Ticks - UnixEpochTicks) / (double)TimeSpan.TicksPerSecond); +#endif + + internal static int WriteSerializedTagValues( + byte[] buffer, + int cursor, + ReadOnlySpan serializedTags, + bool appendTrailingComma = false) + { + if (!serializedTags.IsEmpty) + { + if (serializedTags.Length > buffer.Length - cursor) + { + throw new ArgumentException("Destination buffer too small.", nameof(buffer)); + } + + serializedTags.CopyTo(buffer.AsSpan(cursor)); + cursor += serializedTags.Length; + + if (appendTrailingComma) + { + buffer[cursor++] = unchecked((byte)','); + } + } + + return cursor; + } + + internal static int WriteSerializedTags( + byte[] buffer, + int cursor, + ReadOnlySpan serializedTags, + bool appendTrailingComma = false) + { + buffer[cursor++] = unchecked((byte)'{'); + cursor = WriteSerializedTagValues(buffer, cursor, serializedTags, appendTrailingComma); + + buffer[cursor++] = unchecked((byte)'}'); + return cursor; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal int WriteExemplar(byte[] buffer, int cursor, in Exemplar exemplar, bool isLongValue) { buffer[cursor++] = unchecked((byte)' '); buffer[cursor++] = unchecked((byte)'#'); @@ -535,17 +591,19 @@ internal static int WriteExemplar(byte[] buffer, int cursor, in Exemplar exempla if (exemplar.TraceId != default) { - AddLabel("trace_id", exemplar.TraceId.ToHexString(), ref labels); + this.AddLabel("trace_id", exemplar.TraceId.ToHexString(), ref labels); } if (exemplar.SpanId != default) { - AddLabel("span_id", exemplar.SpanId.ToHexString(), ref labels); + this.AddLabel("span_id", exemplar.SpanId.ToHexString(), ref labels); } + var reservedOutputKeys = this.GetReservedExemplarOutputKeys(); + foreach (var tag in exemplar.FilteredTags) { - AddLabel(tag.Key, tag.Value, ref labels, ReservedExemplarLabelNames); + this.AddLabel(tag.Key, tag.Value, ref labels, reservedOutputKeys); } cursor = WriteLabels( @@ -571,7 +629,7 @@ internal static int WriteExemplar(byte[] buffer, int cursor, in Exemplar exempla } [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static int WriteTags( + internal int WriteTags( byte[] buffer, int cursor, Metric metric, @@ -615,26 +673,28 @@ internal static int WriteTags( { foreach (var scopeLabel in CreateScopeLabelData(metric)) { - AddLabel(scopeLabel.OriginalKey, scopeLabel.OutputKey, scopeLabel.Value, ref labels, reservedOutputKeys); + AddLabel(scopeLabel.OriginalKey, this.GetScopeOutputKey(scopeLabel.OutputKey), scopeLabel.Value, ref labels, reservedOutputKeys); } } foreach (var tag in tags) { - AddLabel(tag.Key, tag.Value, ref labels, reservedOutputKeys); + this.AddLabel(tag.Key, tag.Value, ref labels, reservedOutputKeys); } return WriteLabels(buffer, cursor, labels, writeEnclosingBraces); void WriteScopeLabels() { - // Scope labels are de-duplicated by output key in CreateScopeLabels, so unlike - // point tags they can never collide with an already-written label. They only need - // to be written (which also records their output keys so point tags can detect a - // collision with a scope label). + // Scope labels (otel_scope_*) are OpenTelemetry naming conventions that are already + // in their target Prometheus form, so they are not re-escaped by the negotiated + // scheme. They are de-duplicated by output key in CreateScopeLabels, so unlike point + // tags they can never collide with an already-written label. They only need to be + // written (which also records their output keys so point tags can detect a collision + // with a scope label). foreach (var scopeLabel in CreateScopeLabels(metric)) { - _ = TryWriteLabel(scopeLabel.Key, scopeLabel.Value); + _ = TryWriteLabel(scopeLabel.Key, scopeLabel.Value, isScopeLabel: true); } } @@ -651,9 +711,13 @@ bool TryWritePointTags() return true; } - bool TryWriteLabel(string key, object? value) + bool TryWriteLabel(string key, object? value, bool isScopeLabel = false) { - var outputKey = GetSanitizedLabelKey(key); + // Scope labels arrive in their underscore-normalized (otel_scope_*) form and are then + // escaped using the negotiated scheme, exactly as point tags are. The resulting output + // key is always a legacy-valid ASCII name, so it is both used for de-duplication and + // written verbatim. + var outputKey = isScopeLabel ? this.GetScopeOutputKey(key) : this.GetOutputLabelKey(key); if (reservedOutputKeys?.Contains(outputKey) == true) { @@ -673,22 +737,15 @@ bool TryWriteLabel(string key, object? value) buffer[cursor++] = unchecked((byte)','); } - cursor = WriteLabel(buffer, cursor, key, value); + cursor = WriteAsciiStringNoEscape(buffer, cursor, outputKey); + cursor = WriteSanitizedLabel(buffer, cursor, value); wroteLabel = true; return true; } } - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static int WriteUnixTimeSeconds(byte[] buffer, int cursor, DateTimeOffset value) => -#if NET - WriteDouble(buffer, cursor, (value.UtcDateTime.Ticks - DateTimeOffset.UnixEpoch.Ticks) / (double)TimeSpan.TicksPerSecond); -#else - WriteDouble(buffer, cursor, (value.UtcDateTime.Ticks - UnixEpochTicks) / (double)TimeSpan.TicksPerSecond); -#endif - - internal static byte[] SerializeTags( + internal byte[] SerializeTags( Metric metric, ReadOnlyTagCollection tags, in TextFormatSerializerOptions options, @@ -700,7 +757,7 @@ internal static byte[] SerializeTags( { try { - var cursor = WriteTags( + var cursor = this.WriteTags( buffer, 0, metric, @@ -723,44 +780,6 @@ internal static byte[] SerializeTags( } } - internal static int WriteSerializedTagValues( - byte[] buffer, - int cursor, - ReadOnlySpan serializedTags, - bool appendTrailingComma = false) - { - if (!serializedTags.IsEmpty) - { - if (serializedTags.Length > buffer.Length - cursor) - { - throw new ArgumentException("Destination buffer too small.", nameof(buffer)); - } - - serializedTags.CopyTo(buffer.AsSpan(cursor)); - cursor += serializedTags.Length; - - if (appendTrailingComma) - { - buffer[cursor++] = unchecked((byte)','); - } - } - - return cursor; - } - - internal static int WriteSerializedTags( - byte[] buffer, - int cursor, - ReadOnlySpan serializedTags, - bool appendTrailingComma = false) - { - buffer[cursor++] = unchecked((byte)'{'); - cursor = WriteSerializedTagValues(buffer, cursor, serializedTags, appendTrailingComma); - - buffer[cursor++] = unchecked((byte)'}'); - return cursor; - } - [MethodImpl(MethodImplOptions.AggressiveInlining)] internal int WriteMetricName(byte[] buffer, int cursor, PrometheusMetric metric) => WriteUtf8NoEscape(buffer, cursor, this.GetMetricNameBytes(metric)); @@ -769,6 +788,26 @@ internal int WriteMetricName(byte[] buffer, int cursor, PrometheusMetric metric) internal int WriteMetricMetadataName(byte[] buffer, int cursor, PrometheusMetric metric) => WriteUtf8NoEscape(buffer, cursor, this.GetMetricMetadataNameBytes(metric)); + /// + /// Writes the (already escaped) metric metadata/family name followed by a structural suffix + /// (e.g. "_bucket", "_sum", "_count", "_created"). The suffix is appended literally for all + /// escaping schemes so Prometheus can recognize and strip it when determining the family name. + /// + /// The buffer to write to. + /// The current position in the buffer. + /// The metric to write. + /// The suffix to append to the metric name. + /// The new cursor position in the buffer. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal int WriteMetricNameWithSuffix(byte[] buffer, int cursor, PrometheusMetric metric, string suffix) + { + // The '_total'/'_bucket'/'_sum'/'_count'/'_created' suffixes are structural suffixes that + // Prometheus strips to find the metric family, so they are appended literally to the + // (already escaped) family name regardless of the escaping scheme. + cursor = this.WriteMetricMetadataName(buffer, cursor, metric); + return WriteAsciiStringNoEscape(buffer, cursor, suffix); + } + [MethodImpl(MethodImplOptions.AggressiveInlining)] internal int WriteHelpMetadata(byte[] buffer, int cursor, PrometheusMetric metric, string metricDescription) { @@ -1269,13 +1308,6 @@ private static int WriteSanitizedLabel(byte[] buffer, int cursor, object? labelV return cursor; } - private static string GetSanitizedLabelKey(string value) - { - var builder = new StringBuilder(value.Length + 1); - AppendSanitizedLabelKey(builder, value); - return builder.ToString(); - } - private static void AppendSanitizedLabelKey(StringBuilder builder, string value) { if (string.IsNullOrEmpty(value)) @@ -1312,9 +1344,6 @@ private static void AppendSanitizedLabelKey(StringBuilder builder, string value) } } - private static void AddLabel(string originalKey, object? value, ref List? labels, IReadOnlyCollection? reservedOutputKeys = null) - => AddLabel(originalKey, GetSanitizedLabelKey(originalKey), value, ref labels, reservedOutputKeys); - private static void AddLabel(string originalKey, string outputKey, object? value, ref List? labels, IReadOnlyCollection? reservedOutputKeys = null) { if (reservedOutputKeys?.Contains(outputKey) == true) @@ -1384,7 +1413,10 @@ private static int WriteLabels( labelSetCharacters += labelCharacters; } - cursor = WriteLabel(buffer, cursor, key, value); + // The grouped key is already the final (escaped) output key, so it is written + // verbatim rather than being escaped again. + cursor = WriteAsciiStringNoEscape(buffer, cursor, key); + cursor = WriteSanitizedLabel(buffer, cursor, value); buffer[cursor++] = unchecked((byte)','); wroteLabel = true; } @@ -1633,6 +1665,53 @@ private static bool TryGetPowerOfTenExponent(double absoluteValue, out int expon return true; } + private string[] GetReservedExemplarOutputKeys() + { + // The built-in trace_id/span_id exemplar labels are added under their escaped output keys, + // so the reserved set used to drop colliding filtered tags must hold those same escaped + // keys. Otherwise, for example, the dots scheme escapes the built-in "trace_id" to + // "trace__id" while the reserved set still held "trace_id"; a filtered "trace_id" tag + // (also escaped to "trace__id") would not be dropped and its value would be concatenated + // onto the real trace ID. The built-in trace/span IDs MUST take precedence in a collision. + if (this.Escaping == EscapingScheme.Underscores) + { + return ReservedExemplarLabelNames; + } + + return this.reservedExemplarOutputKeys ??= + [ + this.GetOutputLabelKey("trace_id"), + this.GetOutputLabelKey("span_id"), + ]; + } + + // Scope labels (otel_scope_*) are produced in their underscore-normalized Prometheus form. + // Under the dots and values schemes that form must still be escaped so a client decoding + // the negotiated scheme reverses it correctly; for example "otel_scope_dot_name" would + // otherwise be incorrectly decoded under the dots scheme into "otel_scope.name", losing + // the required "otel_scope_" prefix. The underscores scheme leaves the key unchanged. + private string GetScopeOutputKey(string outputKey) => + this.Escaping == EscapingScheme.Underscores + ? outputKey + : PrometheusEscaping.EscapeName(outputKey, this.Escaping, isMetricName: false); + + private string GetOutputLabelKey(string value) + { + // The dots and values schemes produce a reversible, legacy-valid ASCII label name. The + // underscores scheme replaces discouraged characters and collapses consecutive ones. + if (this.Escaping != EscapingScheme.Underscores) + { + return string.IsNullOrEmpty(value) ? "_" : PrometheusEscaping.EscapeName(value, this.Escaping, isMetricName: false); + } + + var builder = new StringBuilder(value.Length + 1); + AppendSanitizedLabelKey(builder, value); + return builder.ToString(); + } + + private void AddLabel(string originalKey, object? value, ref List? labels, IReadOnlyCollection? reservedOutputKeys = null) + => AddLabel(originalKey, this.GetOutputLabelKey(originalKey), value, ref labels, reservedOutputKeys); + private readonly struct LabelData(string originalKey, string outputKey, string value) { public readonly string OriginalKey { get; } = originalKey; diff --git a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/OpenTelemetry.Exporter.Prometheus.HttpListener.csproj b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/OpenTelemetry.Exporter.Prometheus.HttpListener.csproj index 8638203e007..b58410cec2c 100644 --- a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/OpenTelemetry.Exporter.Prometheus.HttpListener.csproj +++ b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/OpenTelemetry.Exporter.Prometheus.HttpListener.csproj @@ -24,6 +24,7 @@ + diff --git a/test/OpenTelemetry.Exporter.Prometheus.AspNetCore.Tests/PrometheusIntegrationTests.cs b/test/OpenTelemetry.Exporter.Prometheus.AspNetCore.Tests/PrometheusIntegrationTests.cs index 5fa82932616..d810ba3c9ad 100644 --- a/test/OpenTelemetry.Exporter.Prometheus.AspNetCore.Tests/PrometheusIntegrationTests.cs +++ b/test/OpenTelemetry.Exporter.Prometheus.AspNetCore.Tests/PrometheusIntegrationTests.cs @@ -373,6 +373,8 @@ static async Task WaitForServiceDiscoveryAsync( [InlineData("text/plain")] [InlineData("text/plain;version=0.0.4")] [InlineData("text/plain;version=1.0.0")] + [InlineData("text/plain;version=1.0.0;escaping=dots")] + [InlineData("text/plain;version=1.0.0;escaping=values")] [InlineData("application/openmetrics-text", Skip = "https://github.com/prometheus/prometheus/issues/8932")] [InlineData("application/openmetrics-text;version=0.0.4")] [InlineData("application/openmetrics-text;version=1.0.0", Skip = "https://github.com/prometheus/prometheus/issues/8932")] diff --git a/test/OpenTelemetry.Exporter.Prometheus.HttpListener.FuzzTests/PrometheusSerializerFuzzTests.cs b/test/OpenTelemetry.Exporter.Prometheus.HttpListener.FuzzTests/PrometheusSerializerFuzzTests.cs index 1a21a5367f5..1dd0c29ae04 100644 --- a/test/OpenTelemetry.Exporter.Prometheus.HttpListener.FuzzTests/PrometheusSerializerFuzzTests.cs +++ b/test/OpenTelemetry.Exporter.Prometheus.HttpListener.FuzzTests/PrometheusSerializerFuzzTests.cs @@ -49,6 +49,23 @@ public Property WriteDoubleMatchesReferenceImplementation() => Prop.ForAll( Generators.DoubleArbitrary(), static (value) => SerializeDouble(value).SequenceEqual(ReferenceWriteDouble(value))); + [Property(MaxTest = MaxTests)] + public Property EscapeNameWithDotsMatchesReferenceImplementation() => Prop.ForAll( + Generators.PrometheusStringArbitrary(), + static (value) => PrometheusEscaping.EscapeName(value, EscapingScheme.Dots) == ReferenceEscapeName(value, EscapingScheme.Dots)); + + [Property(MaxTest = MaxTests)] + public Property EscapeNameWithValuesMatchesReferenceImplementation() => Prop.ForAll( + Generators.PrometheusStringArbitrary(), + static (value) => PrometheusEscaping.EscapeName(value, EscapingScheme.Values) == ReferenceEscapeName(value, EscapingScheme.Values)); + + [Property(MaxTest = MaxTests)] + public Property EscapeNameToBufferMatchesStringOverload() => Prop.ForAll( + Generators.PrometheusStringArbitrary(), + static (value) => + SerializeEscapeName(value, EscapingScheme.Dots).SequenceEqual(Encoding.ASCII.GetBytes(PrometheusEscaping.EscapeName(value, EscapingScheme.Dots))) && + SerializeEscapeName(value, EscapingScheme.Values).SequenceEqual(Encoding.ASCII.GetBytes(PrometheusEscaping.EscapeName(value, EscapingScheme.Values)))); + private static byte[] Serialize(string value, Func writer) { var buffer = new byte[(value.Length * 8) + 16]; @@ -63,6 +80,13 @@ private static byte[] SerializeOpenMetricsLabelKey(string value) return buffer.AsSpan(0, cursor).ToArray(); } + private static byte[] SerializeEscapeName(string value, EscapingScheme scheme) + { + var buffer = new byte[(value.Length * 8) + 16]; + var cursor = PrometheusEscaping.EscapeName(buffer, 0, value, scheme); + return buffer.AsSpan(0, cursor).ToArray(); + } + private static byte[] SerializeLong(long value) { var buffer = new byte[64]; @@ -146,6 +170,120 @@ var doubleValue when double.IsNaN(doubleValue) => Encoding.UTF8.GetBytes("NaN"), _ => Encoding.UTF8.GetBytes(value.ToString("G17", CultureInfo.InvariantCulture)), }; + // An independent re-implementation of the dots and values escaping schemes used to validate + // PrometheusEscaping.EscapeName. It deliberately uses different mechanisms (manual surrogate + // decoding, a StringBuilder, and int.ToString("x")) than the production code. + private static string ReferenceEscapeName(string value, EscapingScheme scheme) + { + if (string.IsNullOrEmpty(value)) + { + return value; + } + + if (scheme == EscapingScheme.Values && + ReferenceIsValidLegacyName(value) && + !value.StartsWith("U__", StringComparison.Ordinal)) + { + return value; + } + + var text = new StringBuilder(value.Length + 8); + + if (scheme == EscapingScheme.Values) + { + text.Append("U__"); + } + + var escapeValuesPrefix = scheme == EscapingScheme.Values && value.StartsWith("U__", StringComparison.Ordinal); + var index = 0; + + while (index < value.Length) + { + var codePoint = ReferenceGetCodePoint(value, index, out var charsConsumed, out var isValidRune); + + if (codePoint == '_') + { + text.Append("__"); + } + else if (scheme == EscapingScheme.Dots && codePoint == '.') + { + text.Append("_dot_"); + } + else if (escapeValuesPrefix && index == 0) + { + text.Append('_').Append(codePoint.ToString("x", CultureInfo.InvariantCulture)).Append('_'); + } + else if (ReferenceIsValidLegacyRune(codePoint, index == 0)) + { + text.Append((char)codePoint); + } + else if (scheme == EscapingScheme.Dots) + { + text.Append('_'); + } + else if (!isValidRune) + { + text.Append("_FFFD_"); + } + else + { + text.Append('_').Append(codePoint.ToString("x", CultureInfo.InvariantCulture)).Append('_'); + } + + index += charsConsumed; + } + + return text.ToString(); + } + + private static bool ReferenceIsValidLegacyName(string value) + { + var index = 0; + + while (index < value.Length) + { + var codePoint = ReferenceGetCodePoint(value, index, out var charsConsumed, out _); + + if (!ReferenceIsValidLegacyRune(codePoint, index == 0)) + { + return false; + } + + index += charsConsumed; + } + + return true; + } + + private static bool ReferenceIsValidLegacyRune(int codePoint, bool isFirst) => + codePoint is (>= 'a' and <= 'z') or (>= 'A' and <= 'Z') or '_' or ':' || + (!isFirst && codePoint is >= '0' and <= '9'); + + private static int ReferenceGetCodePoint(string value, int index, out int charsConsumed, out bool isValidRune) + { + var character = value[index]; + + if (char.IsHighSurrogate(character) && + index + 1 < value.Length && + char.IsLowSurrogate(value[index + 1])) + { + charsConsumed = 2; + isValidRune = true; + return char.ConvertToUtf32(character, value[index + 1]); + } + + if (char.IsSurrogate(character)) + { + charsConsumed = 1; + isValidRune = false; + return 0xFFFD; + } + + charsConsumed = 1; + isValidRune = true; + return character; + } + private static byte[] ReferenceWriteEscapedString(string value, bool escapeQuotationMarks) { var text = new StringBuilder(value.Length); diff --git a/test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/PrometheusAcceptHeaders.cs b/test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/PrometheusAcceptHeaders.cs index 800693ef610..a3962cdfb90 100644 --- a/test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/PrometheusAcceptHeaders.cs +++ b/test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/PrometheusAcceptHeaders.cs @@ -22,6 +22,8 @@ public static class PrometheusAcceptHeaders "text/plain; q=0.9, application/openmetrics-text; version=1.0.0; q=0.1", "TEXT/PLAIN; q=0.9, Application/OpenMetrics-Text; version=1.0.0; q=0.1", "*/*;q=0.8,text/plain; charset=utf-8; version=0.0.4", + "text/plain; escaping=dots", + "text/plain; escaping=values", ]; foreach (var accept in prometheusV0) @@ -46,6 +48,8 @@ public static class PrometheusAcceptHeaders "application/openmetrics-text; version=0.0.1", "application/openmetrics-text; version=0.0.1; charset=utf-8", "application/openmetrics-text; version=\"0.0.1\"", + "application/openmetrics-text; escaping=dots", + "application/openmetrics-text; escaping=values", ]; foreach (var accept in openMetricsV0) @@ -77,6 +81,14 @@ public static class PrometheusAcceptHeaders testCases.Add(accept, "application/openmetrics-text", true, "1.0.0", "underscores"); } + // The `dots` and `values` escaping schemes are only negotiated for the v1 formats + testCases.Add("application/openmetrics-text; version=1.0.0; escaping=dots", "application/openmetrics-text", true, "1.0.0", "dots"); + testCases.Add("application/openmetrics-text; version=1.0.0; charset=utf-8; escaping=dots", "application/openmetrics-text", true, "1.0.0", "dots"); + testCases.Add("application/openmetrics-text; version=1.0.0; escaping=values", "application/openmetrics-text", true, "1.0.0", "values"); + testCases.Add("application/openmetrics-text; version=\"1.0.0\"; escaping=\"values\"", "application/openmetrics-text", true, "1.0.0", "values"); + testCases.Add("text/plain; version=1.0.0; escaping=dots", "text/plain", false, "1.0.0", "dots"); + testCases.Add("text/plain; version=1.0.0; escaping=values", "text/plain", false, "1.0.0", "values"); + return testCases; } @@ -89,9 +101,7 @@ public static TheoryData Invalid() => "application/openmetrics-text; version=0.0.5", "application/openmetrics-text; version=foo", "application/openmetrics-text; version=1.0.0; q=0", - "application/openmetrics-text; version=1.0.0; escaping=dots", "application/openmetrics-text; version=1.0.0; escaping=foo", - "application/openmetrics-text; version=1.0.0; escaping=values", "application/openmetrics-text; version=2.0.0", "text/plain; q=0, application/openmetrics-text; version=1.0.0; q=0", ]; diff --git a/test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/PrometheusEscapingTests.cs b/test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/PrometheusEscapingTests.cs new file mode 100644 index 00000000000..042b70faf6e --- /dev/null +++ b/test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/PrometheusEscapingTests.cs @@ -0,0 +1,115 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +namespace OpenTelemetry.Exporter.Prometheus.Tests; + +public static class PrometheusEscapingTests +{ + [Theory] + [InlineData("metric.name.with.dots", "metric_dot_name_dot_with_dot_dots")] + [InlineData("mysystem.prod.west.cpu.load", "mysystem_dot_prod_dot_west_dot_cpu_dot_load")] + [InlineData("mysystem.prod.west.cpu.load_total", "mysystem_dot_prod_dot_west_dot_cpu_dot_load__total")] + [InlineData("http.status:sum", "http_dot_status:sum")] + [InlineData("no:escaping_required", "no:escaping__required")] + [InlineData("a_b", "a__b")] + [InlineData("a b", "a_b")] + [InlineData("label with \U0001F631", "label_with__")] + [InlineData("\u82B1\u706B", "__")] + [InlineData("", "")] + public static void EscapeName_Dots(string name, string expected) + => Assert.Equal(expected, PrometheusEscaping.EscapeName(name, EscapingScheme.Dots)); + + [Theory] + [InlineData("metric.name", "U__metric_2e_name")] + [InlineData("mysystem.prod.west.cpu.load", "U__mysystem_2e_prod_2e_west_2e_cpu_2e_load")] + [InlineData("mysystem.prod.west.cpu.load_total", "U__mysystem_2e_prod_2e_west_2e_cpu_2e_load__total")] + [InlineData("http.status:sum", "U__http_2e_status:sum")] + [InlineData("no:escaping_required", "no:escaping_required")] + [InlineData("foo.bar", "U__foo_2e_bar")] + [InlineData("U__foo_2e_bar", "U___55_____foo__2e__bar")] + [InlineData("label with \u0100", "U__label_20_with_20__100_")] + [InlineData("", "")] + public static void EscapeName_Values(string name, string expected) + => Assert.Equal(expected, PrometheusEscaping.EscapeName(name, EscapingScheme.Values)); + + [Fact] + public static void EscapeName_Values_EncodesSpacesAndAstralCharacters() + { + // Input "label with \U0001F631" -> spaces become _20_ and the astral character its code point. + string name = "label with \U0001F631"; + + Assert.Equal( + "U__label_20_with_20__1f631_", + PrometheusEscaping.EscapeName(name, EscapingScheme.Values)); + } + + [Fact] + public static void EscapeName_Values_EncodesNonAsciiCharacters() + { + // Input is the two characters U+82B1 and U+706B. + string name = "\u82B1\u706B"; + + Assert.Equal( + "U___82b1__706b_", + PrometheusEscaping.EscapeName(name, EscapingScheme.Values)); + } + + [Fact] + public static void EscapeName_Values_EncodesUnpairedSurrogateAsReplacementCharacter() + { + // A lone high surrogate is not a valid Unicode scalar value and is encoded as + // the literal _FFFD_ replacement marker used by the reference implementation. + string name = "\uD800"; + + Assert.Equal( + "U___FFFD_", + PrometheusEscaping.EscapeName(name, EscapingScheme.Values)); + } + + [Theory] + [InlineData("metric.name", "metric.name")] + [InlineData("a_b", "a_b")] + [InlineData("", "")] + public static void EscapeName_Underscores_IsHandledElsewhere(string name, string expected) + => Assert.Equal(expected, PrometheusEscaping.EscapeName(name, EscapingScheme.Underscores)); + + [Theory] + [InlineData("http:method", "http_method")] + [InlineData("http.method:value", "http_dot_method_value")] + public static void EscapeName_Dots_LabelNameEncodesColon(string name, string expected) + => Assert.Equal(expected, PrometheusEscaping.EscapeName(name, EscapingScheme.Dots, isMetricName: false)); + + [Theory] + [InlineData("http:method", "U__http_3a_method")] + [InlineData("http.method:value", "U__http_2e_method_3a_value")] + public static void EscapeName_Values_LabelNameEncodesColon(string name, string expected) + => Assert.Equal(expected, PrometheusEscaping.EscapeName(name, EscapingScheme.Values, isMetricName: false)); + + [Theory] + [InlineData(EscapingScheme.Dots, "http:method", "http:method")] + [InlineData(EscapingScheme.Values, "http:method", "http:method")] + internal static void EscapeName_MetricNameKeepsColon(EscapingScheme scheme, string name, string expected) + => Assert.Equal(expected, PrometheusEscaping.EscapeName(name, scheme, isMetricName: true)); + + [Theory] + [InlineData(EscapingScheme.Dots)] + [InlineData(EscapingScheme.Values)] + internal static void EscapeName_ToBuffer_WithEmptyName_LeavesCursorUnchanged(EscapingScheme scheme) + { + var buffer = new byte[16]; + + var cursor = PrometheusEscaping.EscapeName(buffer, 5, string.Empty, scheme); + + Assert.Equal(5, cursor); + } + + [Theory] + [InlineData(null, EscapingScheme.Underscores)] + [InlineData("underscores", EscapingScheme.Underscores)] + [InlineData("dots", EscapingScheme.Dots)] + [InlineData("values", EscapingScheme.Values)] + [InlineData("allow-utf-8", EscapingScheme.Underscores)] + [InlineData("anything-else", EscapingScheme.Underscores)] + internal static void FromString_MapsEscapingScheme(string? escaping, EscapingScheme expected) + => Assert.Equal(expected, PrometheusEscaping.FromString(escaping)); +} diff --git a/test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/PrometheusIntegrationTests.cs b/test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/PrometheusIntegrationTests.cs index c0b620ac771..da926980a71 100644 --- a/test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/PrometheusIntegrationTests.cs +++ b/test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/PrometheusIntegrationTests.cs @@ -17,6 +17,8 @@ public class PrometheusIntegrationTests(PromToolFixture fixture, ITestOutputHelp [InlineData("text/plain")] [InlineData("text/plain;version=0.0.4")] [InlineData("text/plain;version=1.0.0")] + [InlineData("text/plain;version=1.0.0;escaping=dots")] + [InlineData("text/plain;version=1.0.0;escaping=values")] [InlineData("application/openmetrics-text", Skip = "https://github.com/prometheus/prometheus/issues/8932")] [InlineData("application/openmetrics-text;version=0.0.4")] [InlineData("application/openmetrics-text;version=1.0.0", Skip = "https://github.com/prometheus/prometheus/issues/8932")] diff --git a/test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/PrometheusMetricTests.cs b/test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/PrometheusMetricTests.cs index 6f161fe3bf6..5cd82d59dd8 100644 --- a/test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/PrometheusMetricTests.cs +++ b/test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/PrometheusMetricTests.cs @@ -457,6 +457,41 @@ public void Name_UnitWithSpace_Sanitized() public void Name_UnitWithSpecialChars_Sanitized() => AssertName("metric", "req!", PrometheusType.Gauge, false, "metric_req"); + [Fact] + public void GetNames_Dots_EscapesFamilyNameAndAppendsTotalSuffixLiterally() + { + var metric = new PrometheusMetric("http.server.duration", "s", PrometheusType.Counter, false); + var names = metric.GetNameSet(EscapingScheme.Dots); + + // The family name (including the unit) is escaped, so the '.' characters become '_dot_' and + // the structural underscore before the unit is doubled. The '_total' suffix is a structural + // suffix appended literally to the escaped family name. + Assert.Equal("http_dot_server_dot_duration__seconds_total", names.Name); + Assert.Equal("http_dot_server_dot_duration__seconds_total", names.OpenMetricsName); + Assert.Equal("http_dot_server_dot_duration__seconds", names.OpenMetricsMetadataName); + } + + [Fact] + public void GetNames_Values_EscapesFamilyNameAndAppendsTotalSuffixLiterally() + { + var metric = new PrometheusMetric("http.server.duration", "s", PrometheusType.Counter, false); + var names = metric.GetNameSet(EscapingScheme.Values); + + Assert.Equal("U__http_2e_server_2e_duration__seconds_total", names.Name); + Assert.Equal("U__http_2e_server_2e_duration__seconds_total", names.OpenMetricsName); + Assert.Equal("U__http_2e_server_2e_duration__seconds", names.OpenMetricsMetadataName); + } + + [Fact] + public void GetNames_Underscores_MatchesDefaultProperties() + { + var metric = new PrometheusMetric("http.server.duration", "s", PrometheusType.Counter, false); + var names = metric.GetNameSet(EscapingScheme.Underscores); + + Assert.Equal("http_server_duration_seconds_total", names.Name); + Assert.Equal(metric.Name, names.Name); + } + [Theory] [InlineData(PrometheusType.Counter)] [InlineData(PrometheusType.Gauge)] diff --git a/test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/PrometheusSerializerTests.cs b/test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/PrometheusSerializerTests.cs index 01ea0f55072..430520487fd 100644 --- a/test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/PrometheusSerializerTests.cs +++ b/test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/PrometheusSerializerTests.cs @@ -1754,6 +1754,115 @@ public void ExemplarLabelWithSurrogatePairCountsUnicodeCodePoints() Assert.Contains("rkt:\U0001F680", output, StringComparison.Ordinal); } + [Theory] + [InlineData(PrometheusProtocol.DotsEscaping)] + [InlineData(PrometheusProtocol.UnderscoresEscaping)] + [InlineData(PrometheusProtocol.ValuesEscaping)] + public async Task WriteMetric_EscapesMetricAndLabelNamesUsingNegotiatedScheme(string escaping) + { + var buffer = new byte[85000]; + var metrics = new List(); + + using var meter = CreateMeter(); + var counter = meter.CreateCounter("http.server.requests"); + + using var provider = Sdk.CreateMeterProviderBuilder() + .AddMeter(meter.Name) + .AddInMemoryExporter(metrics) + .Build(); + + counter.Add(1, new KeyValuePair("http.method", "GET")); + + provider.ForceFlush(); + + var protocol = new PrometheusProtocol( + PrometheusProtocol.OpenMetricsMediaType, + escaping, + PrometheusProtocol.OpenMetricsV1, + isOpenMetrics: true); + + var serializer = TextFormatSerializer.GetSerializer(protocol); + + var cursor = serializer.WriteMetric( + buffer, + 0, + metrics[0], + PrometheusMetric.Create(metrics[0], false), + writeType: true, + writeUnit: true, + writeHelp: true, + unitOverride: null, + helpOverride: null); + + var output = Encoding.UTF8.GetString(buffer, 0, cursor); + + await Verify(output, "txt", VerifySettings).UseParameters(escaping); + } + + [Fact] + public async Task WriteMetricEscapesScopeLabelKeysUsingNegotiatedDotsScheme() + { + var buffer = new byte[85000]; + var metrics = new List(); + + using var meter = new Meter( + nameof(this.WriteMetricEscapesScopeLabelKeysUsingNegotiatedDotsScheme), + "1.0.0", + [new("dot_name", "value")], + scope: null); + + using var provider = Sdk.CreateMeterProviderBuilder() + .AddMeter(meter.Name) + .AddInMemoryExporter(metrics) + .Build(); + + meter.CreateCounter("test_counter").Add(1); + + provider.ForceFlush(); + + var output = WriteMetricWithDotsEscaping(buffer, metrics[0]); + + await Verify(output, "txt", VerifySettings); + } + + [Fact] + public async Task WriteExemplarEscapesReservedTraceIdKeyToPreventCollisionWithFilteredTag() + { + var buffer = new byte[85000]; + var metrics = new List(); + + using var meter = CreateMeter(); + var counter = meter.CreateCounter("test_counter"); + + using var provider = Sdk.CreateMeterProviderBuilder() + .AddMeter(meter.Name) + .SetExemplarFilter(ExemplarFilterType.AlwaysOn) + .AddView( + counter.Name, + new MetricStreamConfiguration + { + TagKeys = ["keep"], + ExemplarReservoirFactory = () => new SimpleFixedSizeExemplarReservoir(3), + }) + .AddInMemoryExporter(metrics) + .Build(); + + using var activity = new Activity("test"); + activity.Start(); + + // The filtered tag is literally named "trace_id"; under the dots scheme both it and the + // built-in trace_id escape to the same output key "trace__id". The built-in trace ID MUST + // take precedence so its value is emitted intact rather than having the filtered tag's + // value concatenated onto it. + counter.Add(2, new("keep", "value"), new("trace_id", "drop-me")); + + provider.ForceFlush(); + + var output = WriteMetricWithDotsEscaping(buffer, metrics[0]); + + await Verify(output, "txt", VerifySettings); + } + #if NET [Fact] public async Task WriteHistogramMetricSerializesStaticTagsWithoutPreSerializedTags() @@ -1840,6 +1949,30 @@ private static int WriteMetric(byte[] buffer, int cursor, Metric metric, bool us options); } + private static string WriteMetricWithDotsEscaping(byte[] buffer, Metric metric) + { + var protocol = new PrometheusProtocol( + PrometheusProtocol.OpenMetricsMediaType, + PrometheusProtocol.DotsEscaping, + PrometheusProtocol.OpenMetricsV1, + isOpenMetrics: true); + + var serializer = TextFormatSerializer.GetSerializer(protocol); + + var cursor = serializer.WriteMetric( + buffer, + 0, + metric, + PrometheusMetric.Create(metric, false), + writeType: true, + writeUnit: true, + writeHelp: true, + unitOverride: null, + helpOverride: null); + + return Encoding.UTF8.GetString(buffer, 0, cursor); + } + private static void WaitForNextExemplarTimestamp() { var timestamp = DateTimeOffset.UtcNow; @@ -1902,7 +2035,7 @@ private static VerifySettings CreateVerifySettings() [GeneratedRegex("(?m)^(.+?\\s#\\s\\{[^}]*\\}\\s+\\S+\\s+)\\S+$")] private static partial Regex ExemplarTimestamp(); - [GeneratedRegex("(?m)(trace_id|span_id)=\"[^\"]*\"")] + [GeneratedRegex("(?m)(trace_{1,2}id|span_{1,2}id)=\"[^\"]*\"")] private static partial Regex SpanOrTraceIds(); [GeneratedRegex("telemetry_sdk_version=\"[^\"]*\"")] @@ -1912,7 +2045,7 @@ private static VerifySettings CreateVerifySettings() private static Regex ExemplarTimestamp() => new("(?m)^(.+?\\s#\\s\\{[^}]*\\}\\s+\\S+\\s+)\\S+$", RegexOptions.Compiled); - private static Regex SpanOrTraceIds() => new("(?m)(trace_id|span_id)=\"[^\"]*\"", RegexOptions.Compiled); + private static Regex SpanOrTraceIds() => new("(?m)(trace_{1,2}id|span_{1,2}id)=\"[^\"]*\"", RegexOptions.Compiled); private static Regex SdkVersion() => new("telemetry_sdk_version=\"[^\"]*\"", RegexOptions.Compiled); #endif diff --git a/test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/snapshots/PrometheusSerializer.WriteExemplarEscapesReservedTraceIdKeyToPreventCollisionWithFilteredTag.verified.txt b/test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/snapshots/PrometheusSerializer.WriteExemplarEscapesReservedTraceIdKeyToPreventCollisionWithFilteredTag.verified.txt new file mode 100644 index 00000000000..f0099af1c9a --- /dev/null +++ b/test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/snapshots/PrometheusSerializer.WriteExemplarEscapesReservedTraceIdKeyToPreventCollisionWithFilteredTag.verified.txt @@ -0,0 +1,3 @@ +# TYPE test__counter counter +test__counter_total{otel__scope__name="WriteExemplarEscapesReservedTraceIdKeyToPreventCollisionWithFilteredTag",keep="value"} 2 # {trace__id="",span__id=""} 2 +test__counter_created{otel__scope__name="WriteExemplarEscapesReservedTraceIdKeyToPreventCollisionWithFilteredTag",keep="value"} diff --git a/test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/snapshots/PrometheusSerializer.WriteMetricEscapesScopeLabelKeysUsingNegotiatedDotsScheme.verified.txt b/test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/snapshots/PrometheusSerializer.WriteMetricEscapesScopeLabelKeysUsingNegotiatedDotsScheme.verified.txt new file mode 100644 index 00000000000..34e3efda0da --- /dev/null +++ b/test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/snapshots/PrometheusSerializer.WriteMetricEscapesScopeLabelKeysUsingNegotiatedDotsScheme.verified.txt @@ -0,0 +1,3 @@ +# TYPE test__counter counter +test__counter_total{otel__scope__name="WriteMetricEscapesScopeLabelKeysUsingNegotiatedDotsScheme",otel__scope__version="1.0.0",otel__scope__dot__name="value"} 1 +test__counter_created{otel__scope__name="WriteMetricEscapesScopeLabelKeysUsingNegotiatedDotsScheme",otel__scope__version="1.0.0",otel__scope__dot__name="value"} diff --git a/test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/snapshots/PrometheusSerializer.WriteMetric_EscapesMetricAndLabelNamesUsingNegotiatedScheme_escaping=dots.verified.txt b/test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/snapshots/PrometheusSerializer.WriteMetric_EscapesMetricAndLabelNamesUsingNegotiatedScheme_escaping=dots.verified.txt new file mode 100644 index 00000000000..0daa858f7fa --- /dev/null +++ b/test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/snapshots/PrometheusSerializer.WriteMetric_EscapesMetricAndLabelNamesUsingNegotiatedScheme_escaping=dots.verified.txt @@ -0,0 +1,3 @@ +# TYPE http_dot_server_dot_requests counter +http_dot_server_dot_requests_total{otel__scope__name="WriteMetric_EscapesMetricAndLabelNamesUsingNegotiatedScheme",http_dot_method="GET"} 1 +http_dot_server_dot_requests_created{otel__scope__name="WriteMetric_EscapesMetricAndLabelNamesUsingNegotiatedScheme",http_dot_method="GET"} diff --git a/test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/snapshots/PrometheusSerializer.WriteMetric_EscapesMetricAndLabelNamesUsingNegotiatedScheme_escaping=underscores.verified.txt b/test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/snapshots/PrometheusSerializer.WriteMetric_EscapesMetricAndLabelNamesUsingNegotiatedScheme_escaping=underscores.verified.txt new file mode 100644 index 00000000000..aada42f5d0a --- /dev/null +++ b/test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/snapshots/PrometheusSerializer.WriteMetric_EscapesMetricAndLabelNamesUsingNegotiatedScheme_escaping=underscores.verified.txt @@ -0,0 +1,3 @@ +# TYPE http_server_requests counter +http_server_requests_total{otel_scope_name="WriteMetric_EscapesMetricAndLabelNamesUsingNegotiatedScheme",http_method="GET"} 1 +http_server_requests_created{otel_scope_name="WriteMetric_EscapesMetricAndLabelNamesUsingNegotiatedScheme",http_method="GET"} diff --git a/test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/snapshots/PrometheusSerializer.WriteMetric_EscapesMetricAndLabelNamesUsingNegotiatedScheme_escaping=values.verified.txt b/test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/snapshots/PrometheusSerializer.WriteMetric_EscapesMetricAndLabelNamesUsingNegotiatedScheme_escaping=values.verified.txt new file mode 100644 index 00000000000..03cca3e57be --- /dev/null +++ b/test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/snapshots/PrometheusSerializer.WriteMetric_EscapesMetricAndLabelNamesUsingNegotiatedScheme_escaping=values.verified.txt @@ -0,0 +1,3 @@ +# TYPE U__http_2e_server_2e_requests counter +U__http_2e_server_2e_requests_total{otel_scope_name="WriteMetric_EscapesMetricAndLabelNamesUsingNegotiatedScheme",U__http_2e_method="GET"} 1 +U__http_2e_server_2e_requests_created{otel_scope_name="WriteMetric_EscapesMetricAndLabelNamesUsingNegotiatedScheme",U__http_2e_method="GET"}