From 8a3d631379a4e395fa0549305a1ce30ec4612c26 Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Fri, 21 Nov 2025 10:40:43 +0000 Subject: [PATCH] Redacting log data unless opting out. --- .../Microsoft.Agents.AI.Mem0/Mem0Provider.cs | 17 ++- .../Mem0ProviderOptions.cs | 6 + .../Memory/ChatHistoryMemoryProvider.cs | 16 ++- .../ChatHistoryMemoryProviderOptions.cs | 6 + .../Mem0ProviderTests.cs | 105 ++++++++++++++- .../Memory/ChatHistoryMemoryProviderTests.cs | 127 ++++++++++++++++++ 6 files changed, 264 insertions(+), 13 deletions(-) diff --git a/dotnet/src/Microsoft.Agents.AI.Mem0/Mem0Provider.cs b/dotnet/src/Microsoft.Agents.AI.Mem0/Mem0Provider.cs index d18ed2b460..98bed507d5 100644 --- a/dotnet/src/Microsoft.Agents.AI.Mem0/Mem0Provider.cs +++ b/dotnet/src/Microsoft.Agents.AI.Mem0/Mem0Provider.cs @@ -28,6 +28,7 @@ public sealed class Mem0Provider : AIContextProvider private const string DefaultContextPrompt = "## Memories\nConsider the following memories when answering user questions:"; private readonly string _contextPrompt; + private readonly bool _enableSensitiveTelemetryData; private readonly Mem0Client _client; private readonly ILogger? _logger; @@ -64,6 +65,7 @@ public Mem0Provider(HttpClient httpClient, Mem0ProviderScope storageScope, Mem0P this._client = new Mem0Client(httpClient); this._contextPrompt = options?.ContextPrompt ?? DefaultContextPrompt; + this._enableSensitiveTelemetryData = options?.EnableSensitiveTelemetryData ?? false; this._storageScope = new Mem0ProviderScope(Throw.IfNull(storageScope)); this._searchScope = searchScope ?? storageScope; @@ -114,6 +116,7 @@ public Mem0Provider(HttpClient httpClient, JsonElement serializedState, JsonSeri this._client = new Mem0Client(httpClient); this._contextPrompt = options?.ContextPrompt ?? DefaultContextPrompt; + this._enableSensitiveTelemetryData = options?.EnableSensitiveTelemetryData ?? false; var jso = jsonSerializerOptions ?? Mem0JsonUtilities.DefaultOptions; var state = serializedState.Deserialize(jso.GetTypeInfo(typeof(Mem0State))) as Mem0State; @@ -158,17 +161,17 @@ public override async ValueTask InvokingAsync(InvokingContext context this._searchScope.ApplicationId, this._searchScope.AgentId, this._searchScope.ThreadId, - this._searchScope.UserId); + this.SanitizeLogData(this._searchScope.UserId)); if (outputMessageText is not null) { this._logger.LogTrace( "Mem0AIContextProvider: Search Results\nInput:{Input}\nOutput:{MessageText}\nApplicationId: '{ApplicationId}', AgentId: '{AgentId}', ThreadId: '{ThreadId}', UserId: '{UserId}'.", - queryText, - outputMessageText, + this.SanitizeLogData(queryText), + this.SanitizeLogData(outputMessageText), this._searchScope.ApplicationId, this._searchScope.AgentId, this._searchScope.ThreadId, - this._searchScope.UserId); + this.SanitizeLogData(this._searchScope.UserId)); } } @@ -189,7 +192,7 @@ public override async ValueTask InvokingAsync(InvokingContext context this._searchScope.ApplicationId, this._searchScope.AgentId, this._searchScope.ThreadId, - this._searchScope.UserId); + this.SanitizeLogData(this._searchScope.UserId)); return new AIContext(); } } @@ -215,7 +218,7 @@ public override async ValueTask InvokedAsync(InvokedContext context, Cancellatio this._storageScope.ApplicationId, this._storageScope.AgentId, this._storageScope.ThreadId, - this._storageScope.UserId); + this.SanitizeLogData(this._storageScope.UserId)); } } @@ -282,4 +285,6 @@ public Mem0State(Mem0ProviderScope storageScope, Mem0ProviderScope searchScope) public Mem0ProviderScope StorageScope { get; set; } public Mem0ProviderScope SearchScope { get; set; } } + + private string? SanitizeLogData(string? data) => this._enableSensitiveTelemetryData ? data : ""; } diff --git a/dotnet/src/Microsoft.Agents.AI.Mem0/Mem0ProviderOptions.cs b/dotnet/src/Microsoft.Agents.AI.Mem0/Mem0ProviderOptions.cs index 34b0392bec..f2d3d89e16 100644 --- a/dotnet/src/Microsoft.Agents.AI.Mem0/Mem0ProviderOptions.cs +++ b/dotnet/src/Microsoft.Agents.AI.Mem0/Mem0ProviderOptions.cs @@ -12,4 +12,10 @@ public sealed class Mem0ProviderOptions /// /// Defaults to "## Memories\nConsider the following memories when answering user questions:". public string? ContextPrompt { get; set; } + + /// + /// Gets or sets a value indicating whether sensitive data such as user ids and user messages may appear in logs. + /// + /// Defaults to . + public bool EnableSensitiveTelemetryData { get; set; } } diff --git a/dotnet/src/Microsoft.Agents.AI/Memory/ChatHistoryMemoryProvider.cs b/dotnet/src/Microsoft.Agents.AI/Memory/ChatHistoryMemoryProvider.cs index 6d90c877e8..9d629df53a 100644 --- a/dotnet/src/Microsoft.Agents.AI/Memory/ChatHistoryMemoryProvider.cs +++ b/dotnet/src/Microsoft.Agents.AI/Memory/ChatHistoryMemoryProvider.cs @@ -46,6 +46,7 @@ public sealed class ChatHistoryMemoryProvider : AIContextProvider, IDisposable private readonly VectorStoreCollection> _collection; private readonly int _maxResults; private readonly string _contextPrompt; + private readonly bool _enableSensitiveTelemetryData; private readonly ChatHistoryMemoryProviderOptions.SearchBehavior _searchTime; private readonly AITool[] _tools; private readonly ILogger? _logger; @@ -130,6 +131,7 @@ private ChatHistoryMemoryProvider( options ??= new ChatHistoryMemoryProviderOptions(); this._maxResults = options.MaxResults.HasValue ? Throw.IfLessThanOrEqual(options.MaxResults.Value, 0) : DefaultMaxResults; this._contextPrompt = options.ContextPrompt ?? DefaultContextPrompt; + this._enableSensitiveTelemetryData = options.EnableSensitiveTelemetryData; this._searchTime = options.SearchTime; this._logger = loggerFactory?.CreateLogger(); @@ -216,7 +218,7 @@ public override async ValueTask InvokingAsync(InvokingContext context this._searchScope.ApplicationId, this._searchScope.AgentId, this._searchScope.ThreadId, - this._searchScope.UserId); + this.SanitizeLogData(this._searchScope.UserId)); return new AIContext(); } } @@ -268,7 +270,7 @@ public override async ValueTask InvokedAsync(InvokedContext context, Cancellatio this._searchScope.ApplicationId, this._searchScope.AgentId, this._searchScope.ThreadId, - this._searchScope.UserId); + this.SanitizeLogData(this._searchScope.UserId)); } } @@ -302,12 +304,12 @@ internal async Task SearchTextAsync(string userQuestion, CancellationTok this._logger?.LogTrace( "ChatHistoryMemoryProvider: Search Results\nInput:{Input}\nOutput:{MessageText}\n ApplicationId: '{ApplicationId}', AgentId: '{AgentId}', ThreadId: '{ThreadId}', UserId: '{UserId}'.", - userQuestion, - formatted, + this.SanitizeLogData(userQuestion), + this.SanitizeLogData(formatted), this._searchScope.ApplicationId, this._searchScope.AgentId, this._searchScope.ThreadId, - this._searchScope.UserId); + this.SanitizeLogData(this._searchScope.UserId)); return formatted; } @@ -387,7 +389,7 @@ internal async Task SearchTextAsync(string userQuestion, CancellationTok this._searchScope.ApplicationId, this._searchScope.AgentId, this._searchScope.ThreadId, - this._searchScope.UserId); + this.SanitizeLogData(this._searchScope.UserId)); return results; } @@ -475,6 +477,8 @@ public override JsonElement Serialize(JsonSerializerOptions? jsonSerializerOptio return serializedState.Deserialize(jso.GetTypeInfo(typeof(ChatHistoryMemoryProviderState))) as ChatHistoryMemoryProviderState; } + private string? SanitizeLogData(string? data) => this._enableSensitiveTelemetryData ? data : ""; + internal sealed class ChatHistoryMemoryProviderState { public ChatHistoryMemoryProviderScope? StorageScope { get; set; } diff --git a/dotnet/src/Microsoft.Agents.AI/Memory/ChatHistoryMemoryProviderOptions.cs b/dotnet/src/Microsoft.Agents.AI/Memory/ChatHistoryMemoryProviderOptions.cs index 55f06d7429..e09de68a59 100644 --- a/dotnet/src/Microsoft.Agents.AI/Memory/ChatHistoryMemoryProviderOptions.cs +++ b/dotnet/src/Microsoft.Agents.AI/Memory/ChatHistoryMemoryProviderOptions.cs @@ -38,6 +38,12 @@ public sealed class ChatHistoryMemoryProviderOptions /// public int? MaxResults { get; set; } + /// + /// Gets or sets a value indicating whether sensitive data such as user ids and user messages may appear in logs. + /// + /// Defaults to . + public bool EnableSensitiveTelemetryData { get; set; } + /// /// Behavior choices for the provider. /// diff --git a/dotnet/tests/Microsoft.Agents.AI.Mem0.UnitTests/Mem0ProviderTests.cs b/dotnet/tests/Microsoft.Agents.AI.Mem0.UnitTests/Mem0ProviderTests.cs index 46a5482f15..8cfc0bf401 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Mem0.UnitTests/Mem0ProviderTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Mem0.UnitTests/Mem0ProviderTests.cs @@ -91,7 +91,7 @@ public async Task InvokingAsync_PerformsSearch_AndReturnsContextMessageAsync() ThreadId = "thread", UserId = "user" }; - var sut = new Mem0Provider(this._httpClient, storageScope, loggerFactory: this._loggerFactoryMock.Object); + var sut = new Mem0Provider(this._httpClient, storageScope, options: new() { EnableSensitiveTelemetryData = true }, loggerFactory: this._loggerFactoryMock.Object); var invokingContext = new AIContextProvider.InvokingContext(new[] { new ChatMessage(ChatRole.User, "What is my name?") }); // Act @@ -130,6 +130,60 @@ public async Task InvokingAsync_PerformsSearch_AndReturnsContextMessageAsync() Times.Once); } + [Theory] + [InlineData(false, false, 2)] + [InlineData(true, false, 2)] + [InlineData(false, true, 1)] + [InlineData(true, true, 1)] + public async Task InvokingAsync_LogsUserIdBasedOnEnableSensitiveTelemetryDataAsync(bool enableSensitiveTelemetryData, bool requestThrows, int expectedLogInvocations) + { + // Arrange + if (requestThrows) + { + this._handler.EnqueueEmptyInternalServerError(); + } + else + { + this._handler.EnqueueJsonResponse("[ { \"id\": \"1\", \"memory\": \"Name is Caoimhe\", \"hash\": \"h\", \"metadata\": null, \"score\": 0.9, \"created_at\": \"2023-01-01T00:00:00Z\", \"updated_at\": null, \"user_id\": \"u\", \"app_id\": null, \"agent_id\": \"agent\", \"session_id\": \"thread\" } ]"); + } + + var storageScope = new Mem0ProviderScope + { + ApplicationId = "app", + AgentId = "agent", + ThreadId = "thread", + UserId = "user" + }; + var options = new Mem0ProviderOptions { EnableSensitiveTelemetryData = enableSensitiveTelemetryData }; + + var sut = new Mem0Provider(this._httpClient, storageScope, options: options, loggerFactory: this._loggerFactoryMock.Object); + var invokingContext = new AIContextProvider.InvokingContext(new[] { new ChatMessage(ChatRole.User, "Who am I?") }); + + // Act + await sut.InvokingAsync(invokingContext, CancellationToken.None); + + // Assert + Assert.Equal(expectedLogInvocations, this._loggerMock.Invocations.Count); + foreach (var logInvocation in this._loggerMock.Invocations) + { + var state = Assert.IsAssignableFrom>>(logInvocation.Arguments[2]); + var userIdValue = state.First(kvp => kvp.Key == "UserId").Value; + Assert.Equal(enableSensitiveTelemetryData ? "user" : "", userIdValue); + + var inputValue = state.FirstOrDefault(kvp => kvp.Key == "Input").Value; + if (inputValue != null) + { + Assert.Equal(enableSensitiveTelemetryData ? "Who am I?" : "", inputValue); + } + + var messageTextValue = state.FirstOrDefault(kvp => kvp.Key == "MessageText").Value; + if (messageTextValue != null) + { + Assert.Equal(enableSensitiveTelemetryData ? "## Memories\nConsider the following memories when answering user questions:\nName is Caoimhe" : "", messageTextValue); + } + } + } + [Fact] public async Task InvokedAsync_PersistsAllowedMessagesAsync() { @@ -218,6 +272,55 @@ public async Task InvokedAsync_ShouldNotThrow_WhenStorageFailsAsync() Times.Once); } + [Theory] + [InlineData(false, false, 0)] + [InlineData(true, false, 0)] + [InlineData(false, true, 1)] + [InlineData(true, true, 1)] + public async Task InvokedAsync_LogsUserIdBasedOnEnableSensitiveTelemetryDataAsync(bool enableSensitiveTelemetryData, bool requestThrows, int expectedLogCount) + { + // Arrange + if (requestThrows) + { + this._handler.EnqueueEmptyInternalServerError(); + } + else + { + this._handler.EnqueueJsonResponse("[ { \"id\": \"1\", \"memory\": \"Name is Caoimhe\", \"hash\": \"h\", \"metadata\": null, \"score\": 0.9, \"created_at\": \"2023-01-01T00:00:00Z\", \"updated_at\": null, \"user_id\": \"u\", \"app_id\": null, \"agent_id\": \"agent\", \"session_id\": \"thread\" } ]"); + } + + var storageScope = new Mem0ProviderScope + { + ApplicationId = "app", + AgentId = "agent", + ThreadId = "thread", + UserId = "user" + }; + + var options = new Mem0ProviderOptions { EnableSensitiveTelemetryData = enableSensitiveTelemetryData }; + var sut = new Mem0Provider(this._httpClient, storageScope, options: options, loggerFactory: this._loggerFactoryMock.Object); + var requestMessages = new List + { + new(ChatRole.User, "User text") + }; + var responseMessages = new List + { + new(ChatRole.Assistant, "Assistant text") + }; + + // Act + await sut.InvokedAsync(new AIContextProvider.InvokedContext(requestMessages, aiContextProviderMessages: null) { ResponseMessages = responseMessages }); + + // Assert + Assert.Equal(expectedLogCount, this._loggerMock.Invocations.Count); + foreach (var logInvocation in this._loggerMock.Invocations) + { + var state = Assert.IsAssignableFrom>>(logInvocation.Arguments[2]); + var userIdValue = state.First(kvp => kvp.Key == "UserId").Value; + Assert.Equal(enableSensitiveTelemetryData ? "user" : "", userIdValue); + } + } + [Fact] public async Task ClearStoredMemoriesAsync_SendsDeleteWithQueryAsync() { diff --git a/dotnet/tests/Microsoft.Agents.AI.UnitTests/Memory/ChatHistoryMemoryProviderTests.cs b/dotnet/tests/Microsoft.Agents.AI.UnitTests/Memory/ChatHistoryMemoryProviderTests.cs index 49e5a5d29c..1cb841bd2c 100644 --- a/dotnet/tests/Microsoft.Agents.AI.UnitTests/Memory/ChatHistoryMemoryProviderTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.UnitTests/Memory/ChatHistoryMemoryProviderTests.cs @@ -2,6 +2,7 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Text.Json; using System.Threading; using System.Threading.Tasks; @@ -214,6 +215,56 @@ public async Task InvokedAsync_DoesNotThrow_WhenUpsertThrowsAsync() Times.Once); } + [Theory] + [InlineData(false, false, 0)] + [InlineData(true, false, 0)] + [InlineData(false, true, 1)] + [InlineData(true, true, 1)] + public async Task InvokedAsync_LogsUserIdBasedOnEnableSensitiveTelemetryDataAsync(bool enableSensitiveTelemetryData, bool requestThrows, int expectedLogInvocations) + { + // Arrange + var options = new ChatHistoryMemoryProviderOptions + { + EnableSensitiveTelemetryData = enableSensitiveTelemetryData + }; + + if (requestThrows) + { + this._vectorStoreCollectionMock + .Setup(c => c.UpsertAsync(It.IsAny>>(), It.IsAny())) + .ThrowsAsync(new InvalidOperationException("Upsert failed")); + } + else + { + this._vectorStoreCollectionMock + .Setup(c => c.UpsertAsync(It.IsAny>>(), It.IsAny())) + .Returns(Task.CompletedTask); + } + + var provider = new ChatHistoryMemoryProvider( + this._vectorStoreMock.Object, + TestCollectionName, + 1, + new ChatHistoryMemoryProviderScope { UserId = "user1" }, + options: options, + loggerFactory: this._loggerFactoryMock.Object); + + var requestMsg = new ChatMessage(ChatRole.User, "request text"); + var invokedContext = new AIContextProvider.InvokedContext([requestMsg], aiContextProviderMessages: null); + + // Act + await provider.InvokedAsync(invokedContext, CancellationToken.None); + + // Assert + Assert.Equal(expectedLogInvocations, this._loggerMock.Invocations.Count); + foreach (var logInvocation in this._loggerMock.Invocations) + { + var state = Assert.IsAssignableFrom>>(logInvocation.Arguments[2]); + var userIdValue = state.First(kvp => kvp.Key == "UserId").Value; + Assert.Equal(enableSensitiveTelemetryData ? "user1" : "", userIdValue); + } + } + #endregion #region InvokingAsync Tests @@ -333,6 +384,82 @@ public async Task InvokedAsync_CreatesFilter_WhenSearchScopeProvidedAsync() Times.Once); } + [Theory] + [InlineData(false, false, 1)] + [InlineData(true, false, 1)] + [InlineData(false, true, 1)] + [InlineData(true, true, 1)] + public async Task InvokingAsync_LogsUserIdBasedOnEnableSensitiveTelemetryDataAsync(bool enableSensitiveTelemetryData, bool requestThrows, int expectedLogInvocations) + { + // Arrange + var options = new ChatHistoryMemoryProviderOptions + { + SearchTime = ChatHistoryMemoryProviderOptions.SearchBehavior.BeforeAIInvoke, + EnableSensitiveTelemetryData = enableSensitiveTelemetryData + }; + + var scope = new ChatHistoryMemoryProviderScope + { + UserId = "user1" + }; + + if (requestThrows) + { + this._vectorStoreCollectionMock + .Setup(c => c.SearchAsync( + It.IsAny(), + It.IsAny(), + It.IsAny>>(), + It.IsAny())) + .Throws(new InvalidOperationException("Search failed")); + } + else + { + this._vectorStoreCollectionMock + .Setup(c => c.SearchAsync( + It.IsAny(), + It.IsAny(), + It.IsAny>>(), + It.IsAny())) + .Returns(ToAsyncEnumerableAsync(new List>>())); + } + + var provider = new ChatHistoryMemoryProvider( + this._vectorStoreMock.Object, + TestCollectionName, + 1, + storageScope: scope, + searchScope: scope, + options: options, + loggerFactory: this._loggerFactoryMock.Object); + + var invokingContext = new AIContextProvider.InvokingContext([new ChatMessage(ChatRole.User, "requesting relevant history")]); + + // Act + await provider.InvokingAsync(invokingContext, CancellationToken.None); + + // Assert + Assert.Equal(expectedLogInvocations, this._loggerMock.Invocations.Count); + foreach (var logInvocation in this._loggerMock.Invocations) + { + var state = Assert.IsAssignableFrom>>(logInvocation.Arguments[2]); + var userIdValue = state.First(kvp => kvp.Key == "UserId").Value; + Assert.Equal(enableSensitiveTelemetryData ? "user1" : "", userIdValue); + + var inputValue = state.FirstOrDefault(kvp => kvp.Key == "Input").Value; + if (inputValue != null) + { + Assert.Equal(enableSensitiveTelemetryData ? "Who am I?" : "", inputValue); + } + + var messageTextValue = state.FirstOrDefault(kvp => kvp.Key == "MessageText").Value; + if (messageTextValue != null) + { + Assert.Equal(enableSensitiveTelemetryData ? "## Memories\nConsider the following memories when answering user questions:\nName is Caoimhe" : "", messageTextValue); + } + } + } + #endregion #region Serialization Tests