Added configurable activity tracing

build/build.cs

@@ -151,6 +151,7 @@ private static void Main(string[] args)
                 "./src/Extensions/Wolverine.FluentValidation",
                 "./src/Extensions/Wolverine.MemoryPack",
                 "./src/Extensions/Wolverine.MessagePack",
+                "./src/Extensions/Wolverine.OpenTelemetry",
                 "./src/Http/Wolverine.Http",
                 "./src/Http/Wolverine.Http.FluentValidation",
             };

src/Extensions/Wolverine.OpenTelemetry/OpenTelemetryInstrumentationServiceCollectionExtensions.cs

@@ -0,0 +1,27 @@
+using Microsoft.Extensions.DependencyInjection;
+using Wolverine.Runtime.Tracing;
+
+namespace Wolverine.OpenTelemetry;
+
+internal static class OpenTelemetryInstrumentationServiceCollectionExtensions
+{
+    public static void AddTelemetryExtension(this IServiceCollection services, Action<WolverineTracingOptions> configureWolverineTracingOptions)
+    {
+        checkConfigMarker(services);
+        var telemetryExtension = new TelemetryWolverineExtension(configureWolverineTracingOptions);
+        services.Add(new ServiceDescriptor(typeof(IWolverineExtension), telemetryExtension));
+    }
+
+    private static void checkConfigMarker(IServiceCollection services)
+    {
+        var marker = services.FirstOrDefault(s => s.ServiceType == typeof(TelemetryExtensionConfigMarker));
+        if (marker == null)
+        {
+            services.AddSingleton(new TelemetryExtensionConfigMarker());
+            return;
+        }
+
+        throw new InvalidOperationException(
+            "Wolverine instrumentation has already been registered and doesn't support multiple registrations.");
+    }
+}

src/Extensions/Wolverine.OpenTelemetry/TelemetryExtensionConfigMarker.cs

@@ -0,0 +1,3 @@
+namespace Wolverine.OpenTelemetry;
+
+internal class TelemetryExtensionConfigMarker{}

src/Extensions/Wolverine.OpenTelemetry/TelemetryWolverineExtension.cs

@@ -0,0 +1,17 @@
+using Wolverine.Runtime.Tracing;
+
+namespace Wolverine.OpenTelemetry;
+
+internal class TelemetryWolverineExtension : IWolverineExtension
+{
+    private readonly Action<WolverineTracingOptions> _configureWolverineTracingOptions;
+
+    public TelemetryWolverineExtension(Action<WolverineTracingOptions> configureWolverineTracingOptions)
+    {
+        _configureWolverineTracingOptions = configureWolverineTracingOptions;
+    }
+    public void Configure(WolverineOptions options)
+    {
+        _configureWolverineTracingOptions?.Invoke(options.Tracing);
+    }
+}

src/Extensions/Wolverine.OpenTelemetry/TracerProviderBuilderExtensions.cs

@@ -0,0 +1,40 @@
+using OpenTelemetry.Trace;
+using Wolverine.Runtime;
+using Wolverine.Runtime.Tracing;
+
+namespace Wolverine.OpenTelemetry;
+/// <summary>
+/// Extension methods to simplify registering of Wolverine instrumentation.
+/// </summary>
+public static class TracerProviderBuilderExtensions
+{
+    private static readonly string _activitySourceName = typeof(WolverineRuntime).Assembly.GetName().Name;
+
+    /// <summary>
+    /// Enables envelope event data collection for Wolverine
+    /// </summary>
+    /// <param name="builder"><see cref="TracerProviderBuilder"/> being configured.</param>
+    /// <returns>The instance of <see cref="TracerProviderBuilder"/> to chain the calls.</returns>
+    public static TracerProviderBuilder AddWolverineInstrumentation(this TracerProviderBuilder builder)
+        => AddWolverineInstrumentation(builder, configureWolverineTracingOptions: null);
+    /// <summary>
+    /// Enables envelope event data collection for Wolverine
+    /// </summary>
+    /// <param name="builder"><see cref="TracerProviderBuilder"/> being configured.</param>
+    /// <param name="configureWolverineTracingOptions">Callback action for configuring <see cref="WolverineTracingOptions"/>.</param>
+    /// <returns>The instance of <see cref="TracerProviderBuilder"/> to chain the calls.</returns>
+    public static TracerProviderBuilder AddWolverineInstrumentation(this TracerProviderBuilder builder,
+        Action<WolverineTracingOptions> configureWolverineTracingOptions)
+    {
+        if (builder is null)
+        {
+            throw new ArgumentNullException(nameof(builder), "Must not be null");
+        }
+
+        builder.ConfigureServices(services =>
+        {
+            services.AddTelemetryExtension(configureWolverineTracingOptions);
+        });
+        return builder.AddSource(_activitySourceName);
+    }
+}

src/Extensions/Wolverine.OpenTelemetry/Wolverine.OpenTelemetry.csproj

@@ -0,0 +1,23 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+    <PropertyGroup>
+        <Description>OpenTelemetry Instrumentation and Extensions for Wolverine Applications</Description>
+        <DebugType>portable</DebugType>
+        <PackageId>WolverineFx.OpenTelemetry</PackageId>
+        <GenerateAssemblyProductAttribute>false</GenerateAssemblyProductAttribute>
+        <GenerateAssemblyCopyrightAttribute>false</GenerateAssemblyCopyrightAttribute>
+        <GenerateAssemblyVersionAttribute>false</GenerateAssemblyVersionAttribute>
+        <GenerateAssemblyFileVersionAttribute>false</GenerateAssemblyFileVersionAttribute>
+        <GenerateAssemblyInformationalVersionAttribute>false</GenerateAssemblyInformationalVersionAttribute>
+        <ImplicitUsings>enable</ImplicitUsings>
+    </PropertyGroup>
+
+    <ItemGroup>
+        <ProjectReference Include="..\..\Wolverine\Wolverine.csproj"/>
+    </ItemGroup>
+
+    <ItemGroup>
+        <PackageReference Include="OpenTelemetry" Version="1.6.0"/>
+    </ItemGroup>
+
+</Project>

src/Testing/CoreTests/Runtime/WolverineTracingTests.cs

@@ -1,10 +1,13 @@
-using System;
 using System.Diagnostics;
 using CoreTests.Messaging;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Logging.Abstractions;
+using NSubstitute;
 using OpenTelemetry;
 using OpenTelemetry.Resources;
 using OpenTelemetry.Trace;
-using Wolverine.Runtime;
+using Wolverine.Runtime.Agents;
+using Wolverine.Runtime.Tracing;
 using Xunit;
 
 namespace CoreTests.Runtime;
@@ -14,23 +17,149 @@ public class WolverineTracingTests
     [Fact]
     public void use_parent_id_from_envelope_when_exists()
     {
+        WolverineActivitySource.Options = new();
         var envelope = ObjectMother.Envelope();
         envelope.ParentId = "00-25d8f5709b569a1f61bcaf79b9450ed4-f293c0545fc237a1-01";
 
-        using var tracerProvider = Sdk.CreateTracerProviderBuilder()
-            .AddSource("Wolverine")
-            .SetResourceBuilder(
-                ResourceBuilder.CreateDefault()
-                    .AddService("Wolverine", serviceVersion: "1.0"))
-            .AddConsoleExporter()
-            .Build();
+        using var tracerProvider = BuildTracerProvider();
 
-        using var activity = WolverineTracing.StartEnvelopeActivity("process", envelope);
+        using var activity = WolverineTracing.StartEnvelopeActivity("process", envelope, NullLogger.Instance);
         activity.ShouldNotBeNull();
         activity.Start();
 
         activity.ParentId.ShouldBe(envelope.ParentId);
     }
+
+    [Fact]
+    public void can_filter_process_activities()
+    {
+        WolverineActivitySource.Options = new();
+        using var tracerProvider = BuildTracerProvider();
+        const string messageTypeToFilter = "Wolverine.Runtime.Agents.CheckAgentHealth";
+        var envelope = ObjectMother.Envelope();
+        envelope.MessageType = messageTypeToFilter;
+        WolverineActivitySource.Options.ExecuteEnvelopeFilter = env => env.MessageType != messageTypeToFilter;
+        using var activity = WolverineTracing.StartExecuting(envelope, NullLogger.Instance);
+        activity.ShouldBeNull();
+    }
+
+    [Fact]
+    public void can_filter_send_activities()
+    {
+        WolverineActivitySource.Options = new();
+        using var tracerProvider = BuildTracerProvider();
+        const string messageTypeToFilter = "Wolverine.Runtime.Agents.TryAssumeLeadership";
+        var envelope = ObjectMother.Envelope();
+        envelope.MessageType = messageTypeToFilter;
+        WolverineActivitySource.Options.SendEnvelopeFilter = env => env.MessageType != messageTypeToFilter;
+        using var activity = WolverineTracing.StartSending(envelope, NullLogger.Instance);
+        activity.ShouldBeNull();
+    }
+    [Fact]
+    public void can_filter_out_internal_messages()
+    {
+        WolverineActivitySource.Options = new();
+        using var tracerProvider = BuildTracerProvider();
+        var envelope = ObjectMother.Envelope();
+        envelope.Message = new CheckAgentHealth();
+        WolverineActivitySource.Options.SuppressInternalMessageTypes = true;
+        using var activity = WolverineTracing.StartReceiving(envelope, NullLogger.Instance);
+        activity.ShouldBeNull();
+    }
+
+    [Fact]
+    public void can_filter_receive_activities()
+    {
+        WolverineActivitySource.Options = new();
+        using var tracerProvider = BuildTracerProvider();
+        const string messageTypeToFilter = "Wolverine.Runtime.Agents.CheckAgentHealth";
+        var envelope = ObjectMother.Envelope();
+        envelope.MessageType = messageTypeToFilter;
+        WolverineActivitySource.Options.ReceiveEnvelopeFilter = env => env.MessageType != messageTypeToFilter;
+        using var activity = WolverineTracing.StartReceiving(envelope, NullLogger.Instance);
+        activity.ShouldBeNull();
+    }
+
+    [Fact]
+    public void can_filter_with_global_rule()
+    {
+        WolverineActivitySource.Options = new();
+        using var tracerProvider = BuildTracerProvider();
+        const string messageTypeToFilter = "Wolverine.Runtime.Agents.CheckAgentHealth";
+        var envelope = ObjectMother.Envelope();
+        envelope.MessageType = messageTypeToFilter;
+        WolverineActivitySource.Options.GlobalFilter = env => env.MessageType != messageTypeToFilter;
+        using var receiveActivity = WolverineTracing.StartReceiving(envelope, NullLogger.Instance);
+        using var sendActivity = WolverineTracing.StartSending(envelope, NullLogger.Instance);
+        using var executeActivity = WolverineTracing.StartExecuting(envelope, NullLogger.Instance);
+        using var someOtherActivity =
+            WolverineTracing.StartEnvelopeActivity("some other", envelope, NullLogger.Instance);
+        new[] { receiveActivity, sendActivity, executeActivity, someOtherActivity }.ShouldAllBe(x => x == null);
+    }
+
+    [Fact]
+    public void can_enrich_activities_by_configuration()
+    {
+        WolverineActivitySource.Options = new();
+        using var tracerProvider = BuildTracerProvider();
+        const string expectedTagName = "EnrichedTagForSend";
+        const string expectedTagValue = "Enriched Send Tag Value";
+        const string notExpectedTagName = "EnrichedTagForExecute";
+        const string notExpectedTagValue = "Enriched Execute Tag Value";
+        var envelope = ObjectMother.Envelope();
+        envelope.Headers[expectedTagName] = expectedTagValue;
+        envelope.Headers[notExpectedTagName] = notExpectedTagValue;
+        WolverineActivitySource.Options.Enrich = (activity, eventType, env) =>
+        {
+            if (eventType == WolverineEnrichEventNames.StartSendEnvelope)
+            {
+                activity.SetTag(expectedTagName, env.Headers[expectedTagName]!);
+            }
+
+            if (eventType == WolverineEnrichEventNames.StartExecutingEnvelope)
+            {
+                activity.SetTag(notExpectedTagName, env.Headers[notExpectedTagName]!);
+            }
+        };
+        using var activity = WolverineTracing.StartSending(envelope, NullLogger.Instance);
+        activity.ShouldSatisfyAllConditions(
+            x => x.GetTagItem(expectedTagName).ShouldBe(expectedTagValue),
+            x => x.GetTagItem(notExpectedTagName).ShouldBeNull());
+    }
+
+    [Fact]
+    public void can_handle_filtering_exceptions()
+    {
+        using var tracerProvider = BuildTracerProvider();
+        var envelope = ObjectMother.Envelope();
+        WolverineActivitySource.Options.GlobalFilter = _ => throw new Exception("The exception");
+        var logger = Substitute.For<ILogger>();
+        using var activity = WolverineTracing.StartSending(envelope, logger);
+        activity.ShouldBeNull();
+        logger.ReceivedWithAnyArgs().LogError(default(Exception), default);
+    }
+
+    [Fact]
+    public void can_handle_enrichment_exceptions()
+    {
+        WolverineActivitySource.Options = new();
+        using var tracerProvider = BuildTracerProvider();
+        var envelope = ObjectMother.Envelope();
+        WolverineActivitySource.Options.Enrich = (_, __, ___) => throw new Exception("the exception");
+        var logger = Substitute.For<ILogger>();
+        using var activity = WolverineTracing.StartSending(envelope, logger);
+        activity.ShouldNotBeNull();
+        logger.ReceivedWithAnyArgs().LogError(default(Exception), default);
+    }
+
+    private TracerProvider BuildTracerProvider()
+    =>Sdk.CreateTracerProviderBuilder()
+        .AddSource("Wolverine")
+        .SetResourceBuilder(
+            ResourceBuilder.CreateDefault()
+                .AddService("Wolverine", serviceVersion: "1.0"))
+        .AddConsoleExporter()
+        .Build();
 }
 
 public class when_creating_an_execution_activity

src/Wolverine/Envelope.Internals.cs

@@ -1,8 +1,10 @@
 using System.Diagnostics;
 using System.Runtime.InteropServices;
+using Microsoft.Extensions.Logging.Abstractions;
 using Wolverine.Persistence.Durability;
 using Wolverine.Runtime;
 using Wolverine.Runtime.Serialization;
+using Wolverine.Runtime.Tracing;
 using Wolverine.Transports;
 using Wolverine.Transports.Sending;
 using Wolverine.Util;
@@ -212,7 +214,7 @@ internal async ValueTask QuickSendAsync()
         _enqueued = true;
 
         if (Sender.Latched) return;
-        using var activity = WolverineTracing.StartSending(this);
+        using var activity = WolverineTracing.StartSending(this, NullLogger.Instance);
         try
         {
             await Sender.EnqueueOutgoingAsync(this);

src/Wolverine/ErrorHandling/DiscardEnvelope.cs

@@ -1,6 +1,7 @@
 using System.Diagnostics;
 using Microsoft.Extensions.Logging;
 using Wolverine.Runtime;
+using Wolverine.Runtime.Tracing;
 
 namespace Wolverine.ErrorHandling;
 

src/Wolverine/ErrorHandling/MoveToErrorQueue.cs

@@ -1,5 +1,6 @@
 using System.Diagnostics;
 using Wolverine.Runtime;
+using Wolverine.Runtime.Tracing;
 
 namespace Wolverine.ErrorHandling;
 

src/Wolverine/ErrorHandling/NoHandlerContinuation.cs

@@ -2,6 +2,7 @@
 using JasperFx.Core;
 using Microsoft.Extensions.Logging;
 using Wolverine.Runtime;
+using Wolverine.Runtime.Tracing;
 
 namespace Wolverine.ErrorHandling;
 

src/Wolverine/ErrorHandling/PauseListenerContinuation.cs

@@ -1,6 +1,7 @@
 using System.Diagnostics;
 using Microsoft.Extensions.Logging;
 using Wolverine.Runtime;
+using Wolverine.Runtime.Tracing;
 using Wolverine.Transports;
 
 namespace Wolverine.ErrorHandling;

src/Wolverine/ErrorHandling/RequeueContinuation.cs

@@ -1,5 +1,6 @@
 using System.Diagnostics;
 using Wolverine.Runtime;
+using Wolverine.Runtime.Tracing;
 
 namespace Wolverine.ErrorHandling;
 

src/Wolverine/ErrorHandling/RetryInlineContinuation.cs

@@ -1,5 +1,6 @@
 using System.Diagnostics;
 using Wolverine.Runtime;
+using Wolverine.Runtime.Tracing;
 
 namespace Wolverine.ErrorHandling;
 

src/Wolverine/ErrorHandling/ScheduledRetryContinuation.cs

@@ -1,5 +1,6 @@
 using System.Diagnostics;
 using Wolverine.Runtime;
+using Wolverine.Runtime.Tracing;
 
 namespace Wolverine.ErrorHandling;
 

src/Wolverine/Persistence/Durability/DurableSendingAgent.cs

@@ -2,6 +2,7 @@
 using Wolverine.Configuration;
 using Wolverine.Logging;
 using Wolverine.Runtime;
+using Wolverine.Runtime.Tracing;
 using Wolverine.Transports;
 using Wolverine.Transports.Sending;
 using Wolverine.Util.Dataflow;
@@ -125,7 +126,7 @@ public override Task MarkSuccessfulAsync(Envelope outgoing)
 
     protected override async Task storeAndForwardAsync(Envelope envelope)
     {
-        using var activity = WolverineTracing.StartSending(envelope);
+        using var activity = WolverineTracing.StartSending(envelope, _logger);
         await _storeAndForward.PostAsync(envelope);
         activity?.Stop();
     }