Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Create a better OperationName for Activity/OpenTelemetry #4700

Merged
merged 45 commits into from
Nov 14, 2023
Merged
Show file tree
Hide file tree
Changes from 37 commits
Commits
Show all changes
45 commits
Select commit Hold shift + click to select a range
22e4577
Add basic ActivityOperationNameMapper and tests
bouwkast Oct 4, 2023
ea8bce5
Update tests to match parametric tests
bouwkast Oct 5, 2023
f602d19
Swap to assert actual span.OperationName
bouwkast Oct 5, 2023
6fde39b
Add code namespace one
bouwkast Oct 5, 2023
987e430
Actually call the mapper
bouwkast Oct 5, 2023
9a4403b
Get all tests passing for operation name mapping
bouwkast Oct 5, 2023
efcbcd4
Slight refactor of tag names
bouwkast Oct 6, 2023
6c23896
Update OpenTelemetrySdk snapshots
bouwkast Oct 6, 2023
b302970
Small refactor to make it easier to read
bouwkast Oct 6, 2023
d3879bc
Update tests to reflect latest spec
bouwkast Oct 31, 2023
f664370
Add special tag mapping tests for OpenTelemetry
bouwkast Oct 31, 2023
f594cfd
Update tests
bouwkast Nov 1, 2023
52c1add
Update basic operation name OpenTelemetry mapping
bouwkast Nov 1, 2023
34a9f9f
Copy over parametric tests
bouwkast Nov 2, 2023
75024f5
Add tests to Activity for operation name remapping
bouwkast Nov 3, 2023
0d88efc
Update Activity tests for new operation name
bouwkast Nov 3, 2023
af30af6
Another update to Activity snapshots
bouwkast Nov 6, 2023
ffd5977
Create OpenTelemetryTags to help map OperationName
bouwkast Nov 6, 2023
15be838
Do not override "operation.name" tag when legacy enabled
bouwkast Nov 7, 2023
eab7cea
Update OpenTelemetrySdkTests for operation name
bouwkast Nov 7, 2023
e0547a8
Update MongoDB tests
bouwkast Nov 8, 2023
a72ec82
Update TelemetryTests snapshots
bouwkast Nov 8, 2023
9943339
Update GrpcLegacy snapshots
bouwkast Nov 8, 2023
a3f7795
Update WCF snapshots
bouwkast Nov 8, 2023
6afd506
Add missing Wcf snapshots
bouwkast Nov 8, 2023
2aa1b75
Update missing OpenTelemetrySdk snapshots
bouwkast Nov 8, 2023
4dc13fd
Update legacy name env variable comment
bouwkast Nov 9, 2023
492e3b2
Change ActivityStarted to require OpenTelemetryTags type
bouwkast Nov 9, 2023
bb3ae28
Remove [Tag] on SpanKind in AzureServiceBusTags
bouwkast Nov 9, 2023
26d3608
Remove "recieve" from metadata check
bouwkast Nov 9, 2023
212d8e4
Update AzureServiceBus metadata for new operation name
bouwkast Nov 9, 2023
d771ba3
Update AzureServiceBus snapshots for new operation name
bouwkast Nov 9, 2023
3f456f3
Remove trailing comma in launchSettings
bouwkast Nov 9, 2023
4080ba2
Rename ActivityOperationNameMapper to OperationNameMapper
bouwkast Nov 9, 2023
e1235d4
Remove TODO
bouwkast Nov 9, 2023
e2d7d42
Add a note when setting "span.kind" in OtlpHelpers
bouwkast Nov 9, 2023
3eb6abb
Clean up some comments
bouwkast Nov 9, 2023
dfedbfe
Swap to .ToLowerInvariant()
bouwkast Nov 10, 2023
f26a152
Attempt to assert the priority/order of operation name mapping
bouwkast Nov 10, 2023
310c394
Assert that OperationName will be lowercase when mapped from OpenTele…
bouwkast Nov 10, 2023
d66ba49
Move Analytics to OpenTelemetryTags from AzureServiceBusTags
bouwkast Nov 13, 2023
1c6c384
Set AnalyticsSampleRate for AzureServiceBus Activities
bouwkast Nov 13, 2023
12521fd
OperationName will always fallback to a span.kind
bouwkast Nov 13, 2023
394ec4f
Revert "Set AnalyticsSampleRate for AzureServiceBus Activities"
bouwkast Nov 13, 2023
143dd3d
Revert "Move Analytics to OpenTelemetryTags from AzureServiceBusTags"
bouwkast Nov 13, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,18 @@ internal class ActivityHandlerCommon
internal static readonly ConcurrentDictionary<string, ActivityMapping> ActivityMappingById = new();
private static readonly IntegrationId IntegrationId = IntegrationId.OpenTelemetry;

public static void ActivityStarted<T>(string sourceName, T activity, ITags? tags, out ActivityMapping activityMapping)
/// <summary>
/// Handles when a new Activity is started to map it to a new <see cref="Span"/>/<see cref="Scope"/>.
/// </summary>
/// <param name="sourceName">The name of the Activity source</param>
/// <param name="activity">The Activity object</param>
/// <param name="tags">
/// The tags that will be associated with the <see cref="Span"/>.
/// <see cref="OpenTelemetryTags"/> is used for mapping the operation name.
/// </param>
/// <param name="activityMapping">The mapping of Activity to its <see cref="Scope"/>.</param>
/// <typeparam name="T">The <see cref="IActivity"/>.</typeparam>
public static void ActivityStarted<T>(string sourceName, T activity, OpenTelemetryTags? tags, out ActivityMapping activityMapping)
where T : IActivity
{
Tracer.Instance.TracerManager.Telemetry.IntegrationRunning(IntegrationId);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
#nullable enable

using Datadog.Trace.Activity.DuckTypes;
using Datadog.Trace.Tagging;

namespace Datadog.Trace.Activity.Handlers
{
Expand All @@ -21,7 +22,7 @@ public bool ShouldListenTo(string sourceName, string? version)

public void ActivityStarted<T>(string sourceName, T activity)
where T : IActivity
=> ActivityHandlerCommon.ActivityStarted(sourceName, activity, tags: null, out _);
=> ActivityHandlerCommon.ActivityStarted(sourceName, activity, tags: new OpenTelemetryTags(), out _);
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you double check that all callers of ActivityHandlerCommon.ActivityStarted are updated? I think that the Azure Service Bus handler may need to be updated to carry some of the new tags

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So what I opted to do was update ActivityHandlerCommon.ActivityStarted to take OpenTelemetryTags instead of an ITags.
Then I made the AzureServiceBusTags inherit OpenTelemetryTags and that seemed to get the mapping working correctly for the operation name.

I'm not super sure whether or not that'll cause issues, I did have it complain about a duplicate span.kind tag, so I removed the [Tag] attribute on the AzureServiceBusTags.SpanKind property and just override OpenTelemetryTags.SpanKind, which seemed to do the trick.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not super sure whether or not that'll cause issues,

In terms of using inheritance, that should be absolutely fine, we do it with various tags 🙂

The only downside to the approach I can see is the increased memory footprint of the big fat OpenTelemetryTags instances 😄


public void ActivityStopped<T>(string sourceName, T activity)
where T : IActivity
Expand Down
156 changes: 156 additions & 0 deletions tracer/src/Datadog.Trace/Activity/OperationNameMapper.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,156 @@
// <copyright file="OperationNameMapper.cs" company="Datadog">
// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
// </copyright>

#nullable enable
using System;
using Datadog.Trace.Tagging;

namespace Datadog.Trace.Activity
{
/// <summary>
/// Helper class to map <see cref="SpanKinds"/> and various tags on an <c>Activity</c> or OpenTelemetry Span to a <see cref="Span.OperationName"/>.
/// </summary>
internal static class OperationNameMapper
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Really nice and declarative 👍

{
internal static void MapToOperationName(Span span)
{
if (!string.IsNullOrEmpty(span.OperationName))
{
return; // if OperationName has a value it means there was an "operation.name" tag
}

string operationName;

if (span.Tags is not OpenTelemetryTags tags)
{
// while span.Tags can be something other than OpenTelemetryTags the
// ActivityHandlerCommon.StartActivity requires its "Tags" type to be
// OpenTelemetryTags to ensure that for all Activity objects that we
// process will be able to be mapped correctly.
return;
}

if (tags.IsHttpServer())
{
operationName = "http.server.request";
}
else if (tags.IsHttpClient())
{
operationName = "http.client.request";
}
else if (tags.IsDatabase())
{
operationName = $"{tags.DbSystem}.query";
}
else if (tags.IsMessaging())
{
operationName = $"{tags.MessagingSystem}.{tags.MessagingOperation}";
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

More a question for the RFC rather than this PR, but is it still preferable to use tags.MessagingSystem (for example) if we're missing tags.MessagingOperation?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think the semantics that we are going off of from OpenTelemetry (still says their status is Experimental) have it that messaging.operation and messaging.system are required tags: https://opentelemetry.io/docs/specs/semconv/messaging/messaging-spans/#messaging-attributes

But after I got the AzureServiceBus stuff ported over yesterday I think I saw messaging.system tags with no messaging.operation and they led to client.request operation names. So maybe it would be preferable, but I'm not 100% sure what to do.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

A question for @zacharycmontoya perhaps then

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think it makes sense for the other operation names to be client.request. messaging.operation should be one of a fixed, well-known set, so if a span doesn't have a corresponding operation then it's not really interacting with a message. I left comments below about the existing AzureServiceBus "Message" span where we may intelligently decorate it. Otherwise, client.request (low-cardinality) with different resource names seems fine to me.

}
else if (tags.IsAwsClient())
{
operationName = !string.IsNullOrEmpty(tags.RpcService) ? $"aws.{tags.RpcService}.request" : "aws.client.request";
}
else if (tags.IsRpcClient())
{
operationName = $"{tags.RpcSystem}.client.request";
}
else if (tags.IsRpcServer())
{
operationName = $"{tags.RpcSystem}.server.request";
}
else if (tags.IsFaasServer())
{
operationName = $"{tags.FaasTrigger}.invoke";
}
else if (tags.IsFaasClient())
{
operationName = $"{tags.FaasInvokedProvider}.{tags.FaasInvokedName}.invoke";
}
else if (tags.IsGraphQLServer())
{
operationName = "graphql.server.request";
}
Comment on lines +71 to +74
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We can't recognize IsGraphQLClient()?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good question, the semantics for this just have it as GraphQL Server: https://opentelemetry.io/docs/specs/semconv/database/graphql/

When I made the parametric tests I looked at the PR that added the spec (OpenTelemetry) for this and there was some discussion around whether or not it should also be for client open-telemetry/opentelemetry-specification#2456 (comment)

else if (tags.IsGenericServer())
{
operationName = !string.IsNullOrEmpty(tags.NetworkProtocolName) ? $"{tags.NetworkProtocolName}.server.request" : "server.request";
}
else if (tags.IsGenericClient())
{
operationName = !string.IsNullOrEmpty(tags.NetworkProtocolName) ? $"{tags.NetworkProtocolName}.client.request" : "client.request";
}
else
{
operationName = !string.IsNullOrEmpty(tags.SpanKind) ? tags.SpanKind : "otel_unknown";
}

span.OperationName = operationName.ToLower();
bouwkast marked this conversation as resolved.
Show resolved Hide resolved
}

private static bool IsHttpServer(this OpenTelemetryTags tags)
{
return tags.SpanKind == SpanKinds.Server && !string.IsNullOrEmpty(tags.HttpRequestMethod);
}

private static bool IsHttpClient(this OpenTelemetryTags tags)
{
return tags.SpanKind == SpanKinds.Client && !string.IsNullOrEmpty(tags.HttpRequestMethod);
}

private static bool IsDatabase(this OpenTelemetryTags tags)
{
return tags.SpanKind == SpanKinds.Client && !string.IsNullOrEmpty(tags.DbSystem);
}

private static bool IsMessaging(this OpenTelemetryTags tags)
{
return (tags.SpanKind == SpanKinds.Client ||
tags.SpanKind == SpanKinds.Server ||
tags.SpanKind == SpanKinds.Producer ||
tags.SpanKind == SpanKinds.Consumer)
&& !string.IsNullOrEmpty(tags.MessagingSystem) &&
!string.IsNullOrEmpty(tags.MessagingOperation);
}

private static bool IsAwsClient(this OpenTelemetryTags tags)
{
return tags.SpanKind == SpanKinds.Client && string.Equals(tags.RpcSystem, "aws-api", StringComparison.OrdinalIgnoreCase);
}

private static bool IsRpcClient(this OpenTelemetryTags tags)
{
return tags.SpanKind == SpanKinds.Client && !string.IsNullOrEmpty(tags.RpcSystem);
}

private static bool IsRpcServer(this OpenTelemetryTags tags)
{
return tags.SpanKind == SpanKinds.Server && !string.IsNullOrEmpty(tags.RpcSystem);
}

private static bool IsFaasServer(this OpenTelemetryTags tags)
{
return tags.SpanKind == SpanKinds.Server && !string.IsNullOrEmpty(tags.FaasTrigger);
}

private static bool IsFaasClient(this OpenTelemetryTags tags)
{
return tags.SpanKind == SpanKinds.Client && !string.IsNullOrEmpty(tags.FaasInvokedProvider) && !string.IsNullOrEmpty(tags.FaasInvokedName);
}

private static bool IsGraphQLServer(this OpenTelemetryTags tags)
{
return tags.SpanKind == SpanKinds.Server && !string.IsNullOrEmpty(tags.GraphQlOperationType);
}

private static bool IsGenericServer(this OpenTelemetryTags tags)
{
return tags.SpanKind == SpanKinds.Server;
}

private static bool IsGenericClient(this OpenTelemetryTags tags)
{
return tags.SpanKind == SpanKinds.Client;
}
}
}
50 changes: 24 additions & 26 deletions tracer/src/Datadog.Trace/Activity/OtlpHelpers.cs
Original file line number Diff line number Diff line change
Expand Up @@ -29,14 +29,6 @@ internal static void UpdateSpanFromActivity<TInner>(TInner activity, Span span)
var activity5 = activity as IActivity5;

AgentConvertSpan(activity, span);

// Additional Datadog policy: Set tag "span.kind"
// Since the ActivityKind can only be one of a fixed set of values, always set the tag as prescribed by Datadog practices
// even though the tag is not present on normal OTLP spans
if (activity5 is not null)
{
span.SetTag(Tags.SpanKind, GetSpanKind(activity5.Kind));
}
}

// See trace agent func convertSpan: https://github.com/DataDog/datadog-agent/blob/67c353cff1a6a275d7ce40059aad30fc6a3a0bc1/pkg/trace/api/otlp.go#L459
Expand Down Expand Up @@ -93,6 +85,30 @@ private static void AgentConvertSpan<TInner>(TInner activity, Span span)
}
}

// Additional Datadog policy: Set tag "span.kind"
// Since the ActivityKind can only be one of a fixed set of values, always set the tag as prescribed by Datadog practices
// even though the tag is not present on normal OTLP spans
// Note that the "span.kind" is used to help craft an OperationName for the span (if present)
if (activity5 is not null)
{
span.SetTag(Tags.SpanKind, GetSpanKind(activity5.Kind));
}

// Later: Support config 'span_name_as_resource_name'
// Later: Support config 'span_name_remappings'
if (Tracer.Instance.Settings.OpenTelemetryLegacyOperationNameEnabled && activity5 is not null && string.IsNullOrEmpty(span.OperationName))
{
span.OperationName = activity5.Source.Name switch
{
string libName when !string.IsNullOrEmpty(libName) => $"{libName}.{GetSpanKind(activity5.Kind)}",
_ => $"opentelemetry.{GetSpanKind(activity5.Kind)}",
};
}
else
{
OperationNameMapper.MapToOperationName(span);
}

// TODO: Add container tags from attributes if the tag isn't already in the span

// Fixup "env" tag
Expand Down Expand Up @@ -152,24 +168,6 @@ private static void AgentConvertSpan<TInner>(TInner activity, Span span)
// Map the OTEL status to error tags
AgentStatus2Error(activity, span);

if (span.OperationName is null)
{
// Later: Support config 'span_name_as_resource_name'
// Later: Support config 'span_name_remappings'
if (Tracer.Instance.Settings.OpenTelemetryLegacyOperationNameEnabled && activity5 is not null)
{
span.OperationName = activity5.Source.Name switch
{
string libName when !string.IsNullOrEmpty(libName) => $"{libName}.{GetSpanKind(activity5.Kind)}",
_ => $"opentelemetry.{GetSpanKind(activity5.Kind)}",
};
}
else
{
span.OperationName = activity.OperationName;
}
}

// Update Service with a reasonable default
if (span.ServiceName is null)
{
Expand Down
15 changes: 9 additions & 6 deletions tracer/src/Datadog.Trace/Configuration/ConfigurationKeys.cs
Original file line number Diff line number Diff line change
Expand Up @@ -599,12 +599,15 @@ internal static class FeatureFlags
public const string OpenTelemetryEnabled = "DD_TRACE_OTEL_ENABLED";

/// <summary>
/// Enables the use of the <see cref="ISpan.OperationName"/> being set to the legacy value.
/// This flag defaults to <see langword="false"/> and is intended to allow beta users of OpenTelemetry support
/// to toggle on to give them time to upgrade to the new format. This additionally requires that
/// the <c>ActivitySource</c> has a <c>Name</c> property which was introduced in .NET 5 and/or v5 of
/// <c>System.Diagnostics</c> library.
/// Note: This feature flag may be dropped when our OpenTelemetry support becomes generally available.
/// Enables the use of the <see cref="ISpan.OperationName"/> being set to the legacy value of:
/// <c>$"{Activity.Source.Name}.{Activity.Kind}"</c> when instrumenting OpenTelemetry Spans and Activities.
/// This will override the default mapping that is used to create an operation name based on
/// <c>Activity.Kind</c> and various tags on the <c>Activity</c>.
/// <para>
/// This flag defaults to <see langword="false"/> and is intended to allow previous beta customers
/// of the .NET Tracer's OpenTelemetry instrumentation (and System.Diagnostics)
/// to have an easier migration path.
/// </para>
/// </summary>
public const string OpenTelemetryLegacyOperationNameEnabled = "DD_TRACE_OTEL_LEGACY_OPERATION_NAME_ENABLED";

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,27 +38,13 @@ partial class AzureServiceBusTags
#else
private static readonly byte[] MessagingDestinationNameBytes = new byte[] { 186, 109, 101, 115, 115, 97, 103, 105, 110, 103, 46, 100, 101, 115, 116, 105, 110, 97, 116, 105, 111, 110, 46, 110, 97, 109, 101 };
#endif
// MessagingOperationBytes = MessagePack.Serialize("messaging.operation");
#if NETCOREAPP
private static ReadOnlySpan<byte> MessagingOperationBytes => new byte[] { 179, 109, 101, 115, 115, 97, 103, 105, 110, 103, 46, 111, 112, 101, 114, 97, 116, 105, 111, 110 };
#else
private static readonly byte[] MessagingOperationBytes = new byte[] { 179, 109, 101, 115, 115, 97, 103, 105, 110, 103, 46, 111, 112, 101, 114, 97, 116, 105, 111, 110 };
#endif
// SpanKindBytes = MessagePack.Serialize("span.kind");
#if NETCOREAPP
private static ReadOnlySpan<byte> SpanKindBytes => new byte[] { 169, 115, 112, 97, 110, 46, 107, 105, 110, 100 };
#else
private static readonly byte[] SpanKindBytes = new byte[] { 169, 115, 112, 97, 110, 46, 107, 105, 110, 100 };
#endif

public override string? GetTag(string key)
{
return key switch
{
"messaging.source.name" => MessagingSourceName,
"messaging.destination.name" => MessagingDestinationName,
"messaging.operation" => MessagingOperation,
"span.kind" => SpanKind,
_ => base.GetTag(key),
};
}
Expand All @@ -73,12 +59,6 @@ public override void SetTag(string key, string value)
case "messaging.destination.name":
MessagingDestinationName = value;
break;
case "messaging.operation":
MessagingOperation = value;
break;
case "span.kind":
SpanKind = value;
break;
default:
base.SetTag(key, value);
break;
Expand All @@ -97,16 +77,6 @@ public override void EnumerateTags<TProcessor>(ref TProcessor processor)
processor.Process(new TagItem<string>("messaging.destination.name", MessagingDestinationName, MessagingDestinationNameBytes));
}

if (MessagingOperation is not null)
{
processor.Process(new TagItem<string>("messaging.operation", MessagingOperation, MessagingOperationBytes));
}

if (SpanKind is not null)
{
processor.Process(new TagItem<string>("span.kind", SpanKind, SpanKindBytes));
}

base.EnumerateTags(ref processor);
}

Expand All @@ -126,20 +96,6 @@ protected override void WriteAdditionalTags(System.Text.StringBuilder sb)
.Append(',');
}

if (MessagingOperation is not null)
{
sb.Append("messaging.operation (tag):")
.Append(MessagingOperation)
.Append(',');
}

if (SpanKind is not null)
{
sb.Append("span.kind (tag):")
.Append(SpanKind)
.Append(',');
}

base.WriteAdditionalTags(sb);
}
public override double? GetMetric(string key)
Expand Down
Loading
Loading