Add feature to extract extra values from the request and store in keys automatically

Author: gregsinclair42

libraries/src/AWS.Lambda.Powertools.Logging/Internal/LoggingAspect.cs

@@ -14,6 +14,7 @@
  */
 
 using System;
+using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using System.Reflection;
@@ -53,6 +54,11 @@ public class LoggingAspect
     ///     The correlation identifier path
     /// </summary>
     private string _correlationIdPath;
+    
+    /// <summary>
+    ///     Paths to arbitrary values to extract from the input object and the names of the keys to store them in 
+    /// </summary>
+    private IEnumerable<(string Path, string KeyName)> _extractedKeyPaths;
 
     /// <summary>
     ///     The Powertools for AWS Lambda (.NET) configurations
@@ -136,6 +142,7 @@ public void OnEntry(
 
             var logEvent = trigger.LogEvent;
             _correlationIdPath = trigger.CorrelationIdPath;
+            _extractedKeyPaths = trigger.ExtractedKeyPaths;
             _clearState = trigger.ClearState;
 
             Logger.LoggerProvider = new LoggerProvider(_config, _powertoolsConfigurations, _systemWrapper);
@@ -153,6 +160,7 @@ public void OnEntry(
             CaptureXrayTraceId();
             CaptureLambdaContext(eventArgs);
             CaptureCorrelationId(eventObject);
+            CaptureExtractedKeyPaths(eventObject);
             if (logEvent || _powertoolsConfigurations.LoggerLogEvent)
                 LogEvent(eventObject);
         }
@@ -224,48 +232,73 @@ private void CaptureLambdaContext(AspectEventArgs eventArgs)
     /// <param name="eventArg">The event argument.</param>
     private void CaptureCorrelationId(object eventArg)
     {
-        if (string.IsNullOrWhiteSpace(_correlationIdPath))
+        CaptureKeyFromPath(eventArg, _correlationIdPath, LoggingConstants.KeyCorrelationId);
+    }
+
+    /// <summary>
+    ///     Captures the extracted key paths.
+    /// </summary>
+    /// <param name="eventArg">The event argument.</param>
+    private void CaptureExtractedKeyPaths(object eventArg)
+    {
+        if (!_extractedKeyPaths?.Any() ?? true)
+            return;
+        foreach (var (path, keyName) in _extractedKeyPaths)
+        {
+            CaptureKeyFromPath(eventArg, path, keyName);
+        }
+    }
+
+    /// <summary>
+    /// Captures an arbitrary value from the path and stores it in a key
+    /// </summary>
+    /// <param name="eventArg">The event argument.</param>
+    /// <param name="path">The path to the input data.</param>
+    /// <param name="keyName">The name of the key to store the data in.</param>
+    private void CaptureKeyFromPath(object eventArg, string path, string keyName)
+    {
+        if (string.IsNullOrWhiteSpace(path) || string.IsNullOrWhiteSpace(keyName))
             return;
 
-        var correlationIdPaths = _correlationIdPath
+        var paths = path
             .Split(CorrelationIdPaths.Separator, StringSplitOptions.RemoveEmptyEntries);
 
-        if (!correlationIdPaths.Any())
+        if (!paths.Any())
             return;
 
         if (eventArg is null)
         {
             if (IsDebug())
                 _systemWrapper.LogLine(
-                    "Skipping CorrelationId capture because event parameter not found.");
+                    $"Skipping {keyName} capture because event parameter not found.");
             return;
         }
 
         try
         {
-            var correlationId = string.Empty;
+            var keyValue = string.Empty;
 
             var jsonDoc =
                 JsonDocument.Parse(PowertoolsLoggingSerializer.Serialize(eventArg, eventArg.GetType()));
 
             var element = jsonDoc.RootElement;
 
-            for (var i = 0; i < correlationIdPaths.Length; i++)
+            for (var i = 0; i < paths.Length; i++)
             {
                 // For casing parsing to be removed from Logging v2 when we get rid of outputcase
                 // without this CorrelationIdPaths.ApiGatewayRest would not work
                 var pathWithOutputCase =
-                    _powertoolsConfigurations.ConvertToOutputCase(correlationIdPaths[i], _config.LoggerOutputCase);
+                    _powertoolsConfigurations.ConvertToOutputCase(paths[i], _config.LoggerOutputCase);
                 if (!element.TryGetProperty(pathWithOutputCase, out var childElement))
                     break;
 
                 element = childElement;
-                if (i == correlationIdPaths.Length - 1)
-                    correlationId = element.ToString();
+                if (i == paths.Length - 1)
+                    keyValue = element.ToString();
             }
 
-            if (!string.IsNullOrWhiteSpace(correlationId))
-                Logger.AppendKey(LoggingConstants.KeyCorrelationId, correlationId);
+            if (!string.IsNullOrWhiteSpace(keyValue))
+                Logger.AppendKey(keyName, keyValue);
         }
         catch (Exception e)
         {

libraries/src/AWS.Lambda.Powertools.Logging/LoggingAttribute.cs

@@ -14,6 +14,8 @@
  */
 
 using System;
+using System.Collections;
+using System.Collections.Generic;
 using AspectInjector.Broker;
 using AWS.Lambda.Powertools.Logging.Internal;
 using Microsoft.Extensions.Logging;
@@ -93,6 +95,10 @@ namespace AWS.Lambda.Powertools.Logging;
 ///             <description>string, pointer path to extract correlation id from input parameter</description>
 ///         </item>
 ///         <item>
+///             <term>ExtractedKeyPaths</term>
+///             <description>IEnumerable of pairs of pointer path to extract a value from input parameter, and the name of the key to store it in</description>
+///         </item>
+///         <item>
 ///             <term>ClearState</term>
 ///             <description>bool, clear all custom keys on each request, by default false</description>
 ///         </item>
@@ -106,7 +112,8 @@ namespace AWS.Lambda.Powertools.Logging;
 ///             ClearState = true,
 ///             LogLevel = LogLevel.Debug,
 ///             LoggerOutputCase = LoggerOutputCase.SnakeCase,
-///             CorrelationIdPath = "/headers/my_request_id_header")
+///             CorrelationIdPath = "/headers/my_request_id_header"),
+///             ExtractedKeyPaths = [("/headers/my_request_id", device_id"), ("/headers/user_id", "user_id")]
 ///         ]
 ///         public async Task&lt;APIGatewayProxyResponse&gt; FunctionHandler
 ///              (APIGatewayProxyRequest apigProxyEvent, ILambdaContext context)
@@ -156,6 +163,18 @@ public class LoggingAttribute : Attribute
     /// </summary>
     /// <value>The correlation identifier path.</value>
     public string CorrelationIdPath { get; set; }
+    
+    /// <summary>
+    ///     Pointer paths to extract other information from input parameter and the name of the key store it in.
+    ///     For example ("/headers/device_id", "DeviceId") will extract the value from that path and add the value to
+    ///     the key named "DeviceId".
+    ///     The first handler parameter is the input to the handler, which can be
+    ///     event data (published by an event source) or custom input that you provide
+    ///     such as a string or any custom data object.
+    ///     Use the same path naming convention as CorrelationIdPath.
+    /// </summary>
+    /// <value>IEnumerable of pairs of the Path to the data, and the name of the key to store it in.</value>
+    public IEnumerable<(string Path, string KeyName)> ExtractedKeyPaths { get; set; }
 
     /// <summary>
     ///     Logger is commonly initialized in the global scope.

libraries/tests/AWS.Lambda.Powertools.Logging.Tests/Attributes/LoggerAspectTests.cs

@@ -263,7 +263,17 @@ public void OnEntry_Should_LogDebug_WhenSet_EnvironmentVariable()
         var name = "TestMethod";
         var args = new object[]
         {
-            new TestObject { FullName = "Powertools", Age = 20, Headers = new Header { MyRequestIdHeader = "test" } }
+            new TestObject
+            {
+                FullName = "Powertools", 
+                Age = 20, 
+                Headers = new Header
+                {
+                    MyRequestIdHeader = "test", 
+                    MySecondRequestIdHeader = "test2", 
+                    MyThirdRequestIdHeader = "test3"
+                }
+            }
         };
         var hostType = typeof(string);
         var method = typeof(TestHandlers).GetMethod("TestMethod");
@@ -275,7 +285,9 @@ public void OnEntry_Should_LogDebug_WhenSet_EnvironmentVariable()
                 Service = "TestService",
                 LoggerOutputCase = LoggerOutputCase.PascalCase,
                 LogEvent = true,
-                CorrelationIdPath = "/Headers/MyRequestIdHeader"
+                CorrelationIdPath = "/Headers/MyRequestIdHeader",
+                ExtractedKeyPaths = [("/Headers/MySecondRequestIdHeader", "second"), 
+                    ("/Headers/MyThirdRequestIdHeader", "third")]
             }
         };
 
@@ -298,8 +310,10 @@ public void OnEntry_Should_LogDebug_WhenSet_EnvironmentVariable()
 
         _mockSystemWrapper.Received(1).LogLine(Arg.Is<string>(s =>
             s.Contains("\"CorrelationId\":\"test\"") &&
+            s.Contains("\"Second\":\"test2\"") &&
+            s.Contains("\"Third\":\"test3\"") &&
             s.Contains(
-                "\"Message\":{\"FullName\":\"Powertools\",\"Age\":20,\"Headers\":{\"MyRequestIdHeader\":\"test\"}")
+                "\"Message\":{\"FullName\":\"Powertools\",\"Age\":20,\"Headers\":{\"MyRequestIdHeader\":\"test\",\"MySecondRequestIdHeader\":\"test2\",\"MyThirdRequestIdHeader\":\"test3\"}")
         ));
     }
 

libraries/tests/AWS.Lambda.Powertools.Logging.Tests/Serializers/TestJsonContext.cs

@@ -26,4 +26,6 @@ internal class TestObject
 internal class Header
 {
     public string MyRequestIdHeader { get; set; }
+    public string MySecondRequestIdHeader { get; set; }
+    public string MyThirdRequestIdHeader { get; set; }
 }