diff --git a/dotnet/src/Microsoft.Agents.AI.Workflows/HandoffToolCallFilteringBehavior.cs b/dotnet/src/Microsoft.Agents.AI.Workflows/HandoffToolCallFilteringBehavior.cs new file mode 100644 index 0000000000..a2d269278e --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.Workflows/HandoffToolCallFilteringBehavior.cs @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Microsoft.Extensions.AI; + +namespace Microsoft.Agents.AI.Workflows; + +/// +/// Specifies the behavior for filtering and contents from +/// s flowing through a handoff workflow. This can be used to prevent agents from seeing external +/// tool calls. +/// +public enum HandoffToolCallFilteringBehavior +{ + /// + /// Do not filter and contents. + /// + None, + + /// + /// Filter only handoff-related and contents. + /// + HandoffOnly, + + /// + /// Filter all and contents. + /// + All +} diff --git a/dotnet/src/Microsoft.Agents.AI.Workflows/HandoffsWorkflowBuilder.cs b/dotnet/src/Microsoft.Agents.AI.Workflows/HandoffsWorkflowBuilder.cs index 9a3abfe960..bd0b3114f1 100644 --- a/dotnet/src/Microsoft.Agents.AI.Workflows/HandoffsWorkflowBuilder.cs +++ b/dotnet/src/Microsoft.Agents.AI.Workflows/HandoffsWorkflowBuilder.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.Linq; using Microsoft.Agents.AI.Workflows.Specialized; +using Microsoft.Extensions.AI; using Microsoft.Shared.Diagnostics; namespace Microsoft.Agents.AI.Workflows; @@ -16,6 +17,7 @@ public sealed class HandoffsWorkflowBuilder private readonly AIAgent _initialAgent; private readonly Dictionary> _targets = []; private readonly HashSet _allAgents = new(AIAgentIDEqualityComparer.Instance); + private HandoffToolCallFilteringBehavior _toolCallFilteringBehavior = HandoffToolCallFilteringBehavior.HandoffOnly; /// /// Initializes a new instance of the class with no handoff relationships. @@ -34,14 +36,38 @@ internal HandoffsWorkflowBuilder(AIAgent initialAgent) /// By default, simple instructions are included. This may be set to to avoid including /// any additional instructions, or may be customized to provide more specific guidance. /// - public string? HandoffInstructions { get; set; } = - $""" + public string? HandoffInstructions { get; private set; } = DefaultHandoffInstructions; + + private const string DefaultHandoffInstructions = + $""" You are one agent in a multi-agent system. You can hand off the conversation to another agent if appropriate. Handoffs are achieved by calling a handoff function, named in the form `{FunctionPrefix}`; the description of the function provides details on the target agent of that handoff. Handoffs between agents are handled seamlessly in the background; never mention or narrate these handoffs in your conversation with the user. """; + /// + /// Sets additional instructions to provide to an agent that has handoffs about how and when to + /// perform them. + /// + /// The instructions to provide, or to restore the default instructions. + public HandoffsWorkflowBuilder WithHandoffInstructions(string? instructions) + { + this.HandoffInstructions = instructions ?? DefaultHandoffInstructions; + return this; + } + + /// + /// Sets the behavior for filtering and contents from + /// s flowing through the handoff workflow. Defaults to . + /// + /// The filtering behavior to apply. + public HandoffsWorkflowBuilder WithToolCallFilteringBehavior(HandoffToolCallFilteringBehavior behavior) + { + this._toolCallFilteringBehavior = behavior; + return this; + } + /// /// Adds handoff relationships from a source agent to one or more target agents. /// @@ -149,8 +175,10 @@ public Workflow Build() HandoffsEndExecutor end = new(); WorkflowBuilder builder = new(start); + HandoffAgentExecutorOptions options = new(this.HandoffInstructions, this._toolCallFilteringBehavior); + // Create an AgentExecutor for each again. - Dictionary executors = this._allAgents.ToDictionary(a => a.Id, a => new HandoffAgentExecutor(a, this.HandoffInstructions)); + Dictionary executors = this._allAgents.ToDictionary(a => a.Id, a => new HandoffAgentExecutor(a, options)); // Connect the start executor to the initial agent. builder.AddEdge(start, executors[this._initialAgent.Id]); diff --git a/dotnet/src/Microsoft.Agents.AI.Workflows/Specialized/HandoffAgentExecutor.cs b/dotnet/src/Microsoft.Agents.AI.Workflows/Specialized/HandoffAgentExecutor.cs index 1627b50e4b..d1367b83ad 100644 --- a/dotnet/src/Microsoft.Agents.AI.Workflows/Specialized/HandoffAgentExecutor.cs +++ b/dotnet/src/Microsoft.Agents.AI.Workflows/Specialized/HandoffAgentExecutor.cs @@ -12,10 +12,155 @@ namespace Microsoft.Agents.AI.Workflows.Specialized; +internal sealed class HandoffAgentExecutorOptions +{ + public HandoffAgentExecutorOptions(string? handoffInstructions, HandoffToolCallFilteringBehavior toolCallFilteringBehavior) + { + this.HandoffInstructions = handoffInstructions; + this.ToolCallFilteringBehavior = toolCallFilteringBehavior; + } + + public string? HandoffInstructions { get; set; } + + public HandoffToolCallFilteringBehavior ToolCallFilteringBehavior { get; set; } = HandoffToolCallFilteringBehavior.HandoffOnly; +} + +internal sealed class HandoffMessagesFilter +{ + private readonly HandoffToolCallFilteringBehavior _filteringBehavior; + + public HandoffMessagesFilter(HandoffToolCallFilteringBehavior filteringBehavior) + { + this._filteringBehavior = filteringBehavior; + } + + internal static bool IsHandoffFunctionName(string name) + { + return name.StartsWith(HandoffsWorkflowBuilder.FunctionPrefix, StringComparison.Ordinal); + } + + public IEnumerable FilterMessages(List messages) + { + if (this._filteringBehavior == HandoffToolCallFilteringBehavior.None) + { + return messages; + } + + Dictionary filteringCandidates = new(); + List filteredMessages = []; + HashSet messagesToRemove = []; + + bool filterHandoffOnly = this._filteringBehavior == HandoffToolCallFilteringBehavior.HandoffOnly; + foreach (ChatMessage unfilteredMessage in messages) + { + ChatMessage filteredMessage = unfilteredMessage.Clone(); + + // .Clone() is shallow, so we cannot modify the contents of the cloned message in place. + List contents = []; + contents.Capacity = unfilteredMessage.Contents?.Count ?? 0; + filteredMessage.Contents = contents; + + // Because this runs after the role changes from assistant to user for the target agent, we cannot rely on tool calls + // originating only from messages with the Assistant role. Instead, we need to inspect the contents of all non-Tool (result) + // FunctionCallContent. + if (unfilteredMessage.Role != ChatRole.Tool) + { + for (int i = 0; i < unfilteredMessage.Contents!.Count; i++) + { + AIContent content = unfilteredMessage.Contents[i]; + if (content is not FunctionCallContent fcc || (filterHandoffOnly && !IsHandoffFunctionName(fcc.Name))) + { + filteredMessage.Contents.Add(content); + + // Track non-handoff function calls so their tool results are preserved in HandoffOnly mode + if (filterHandoffOnly && content is FunctionCallContent nonHandoffFcc) + { + filteringCandidates[nonHandoffFcc.CallId] = new FilterCandidateState(nonHandoffFcc.CallId) + { + IsHandoffFunction = false, + }; + } + } + else if (filterHandoffOnly) + { + if (!filteringCandidates.TryGetValue(fcc.CallId, out FilterCandidateState? candidateState)) + { + filteringCandidates[fcc.CallId] = new FilterCandidateState(fcc.CallId) + { + IsHandoffFunction = true, + }; + } + else + { + candidateState.IsHandoffFunction = true; + (int messageIndex, int contentIndex) = candidateState.FunctionCallResultLocation!.Value; + ChatMessage messageToFilter = filteredMessages[messageIndex]; + messageToFilter.Contents.RemoveAt(contentIndex); + if (messageToFilter.Contents.Count == 0) + { + messagesToRemove.Add(messageIndex); + } + } + } + else + { + // All mode: strip all FunctionCallContent + } + } + } + else + { + if (!filterHandoffOnly) + { + continue; + } + + for (int i = 0; i < unfilteredMessage.Contents!.Count; i++) + { + AIContent content = unfilteredMessage.Contents[i]; + if (content is not FunctionResultContent frc + || (filteringCandidates.TryGetValue(frc.CallId, out FilterCandidateState? candidateState) + && candidateState.IsHandoffFunction is false)) + { + // Either this is not a function result content, so we should let it through, or it is a FRC that + // we know is not related to a handoff call. In either case, we should include it. + filteredMessage.Contents.Add(content); + } + else if (candidateState is null) + { + // We haven't seen the corresponding function call yet, so add it as a candidate to be filtered later + filteringCandidates[frc.CallId] = new FilterCandidateState(frc.CallId) + { + FunctionCallResultLocation = (filteredMessages.Count, filteredMessage.Contents.Count), + }; + } + // else we have seen the corresponding function call and it is a handoff, so we should filter it out. + } + } + + if (filteredMessage.Contents.Count > 0) + { + filteredMessages.Add(filteredMessage); + } + } + + return filteredMessages.Where((_, index) => !messagesToRemove.Contains(index)); + } + + private class FilterCandidateState(string callId) + { + public (int MessageIndex, int ContentIndex)? FunctionCallResultLocation { get; set; } + + public string CallId => callId; + + public bool? IsHandoffFunction { get; set; } + } +} + /// Executor used to represent an agent in a handoffs workflow, responding to events. internal sealed class HandoffAgentExecutor( AIAgent agent, - string? handoffInstructions) : Executor(agent.GetDescriptiveId(), declareCrossRunShareable: true), IResettableExecutor + HandoffAgentExecutorOptions options) : Executor(agent.GetDescriptiveId(), declareCrossRunShareable: true), IResettableExecutor { private static readonly JsonElement s_handoffSchema = AIFunctionFactory.Create( ([Description("The reason for the handoff")] string? reasonForHandoff) => { }).JsonSchema; @@ -39,7 +184,7 @@ public void Initialize( ChatOptions = new() { AllowMultipleToolCalls = false, - Instructions = handoffInstructions, + Instructions = options.HandoffInstructions, Tools = [], }, }; @@ -69,10 +214,19 @@ public override async ValueTask HandleAsync(HandoffState message, List? roleChanges = allMessages.ChangeAssistantToUserForOtherParticipants(this._agent.Name ?? this._agent.Id); - await foreach (var update in this._agent.RunStreamingAsync(allMessages, + // If a handoff was invoked by a previous agent, filter out the handoff function + // call and tool result messages before sending to the underlying agent. These + // are internal workflow mechanics that confuse the target model into ignoring the + // original user question. + HandoffMessagesFilter handoffMessagesFilter = new(options.ToolCallFilteringBehavior); + IEnumerable messagesForAgent = message.InvokedHandoff is not null + ? handoffMessagesFilter.FilterMessages(allMessages) + : allMessages; + + await foreach (var update in this._agent.RunStreamingAsync(messagesForAgent, options: this._agentOptions, cancellationToken: cancellationToken) - .ConfigureAwait(false)) + .ConfigureAwait(false)) { await AddUpdateAsync(update, cancellationToken).ConfigureAwait(false); diff --git a/dotnet/tests/Microsoft.Agents.AI.Workflows.UnitTests/AgentWorkflowBuilderTests.cs b/dotnet/tests/Microsoft.Agents.AI.Workflows.UnitTests/AgentWorkflowBuilderTests.cs index 7056bb172c..ba969d95c7 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Workflows.UnitTests/AgentWorkflowBuilderTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Workflows.UnitTests/AgentWorkflowBuilderTests.cs @@ -274,6 +274,257 @@ public async Task Handoffs_OneTransfer_ResponseServedBySecondAgentAsync() Assert.Contains("nextAgent", result[3].AuthorName); } + [Fact] + public async Task Handoffs_OneTransfer_HandoffTargetDoesNotReceiveHandoffFunctionMessagesAsync() + { + // Regression test for https://github.com/microsoft/agent-framework/issues/3161 + // When a handoff occurs, the target agent should receive the original user message + // but should NOT receive the handoff function call or tool result messages from the + // source agent, as these confuse the target LLM into ignoring the user's question. + + List? capturedNextAgentMessages = null; + + var initialAgent = new ChatClientAgent(new MockChatClient((messages, options) => + { + string? transferFuncName = options?.Tools?.FirstOrDefault(t => t.Name.StartsWith("handoff_to_", StringComparison.Ordinal))?.Name; + Assert.NotNull(transferFuncName); + + return new(new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("call1", transferFuncName)])); + }), name: "initialAgent"); + + var nextAgent = new ChatClientAgent(new MockChatClient((messages, options) => + { + capturedNextAgentMessages = messages.ToList(); + return new(new ChatMessage(ChatRole.Assistant, "The derivative of x^2 is 2x.")); + }), + name: "nextAgent", + description: "The second agent"); + + var workflow = + AgentWorkflowBuilder.CreateHandoffBuilderWith(initialAgent) + .WithHandoff(initialAgent, nextAgent) + .Build(); + + _ = await RunWorkflowAsync(workflow, [new ChatMessage(ChatRole.User, "What is the derivative of x^2?")]); + + Assert.NotNull(capturedNextAgentMessages); + + // The target agent should see the original user message + Assert.Contains(capturedNextAgentMessages, m => m.Role == ChatRole.User && m.Text == "What is the derivative of x^2?"); + + // The target agent should NOT see the handoff function call or tool result from the source agent + Assert.DoesNotContain(capturedNextAgentMessages, m => m.Contents.Any(c => c is FunctionCallContent fcc && fcc.Name.StartsWith("handoff_to_", StringComparison.Ordinal))); + Assert.DoesNotContain(capturedNextAgentMessages, m => m.Role == ChatRole.Tool && m.Contents.Any(c => c is FunctionResultContent frc && frc.Result?.ToString() == "Transferred.")); + } + + [Fact] + public async Task Handoffs_TwoTransfers_HandoffTargetsDoNotReceiveHandoffFunctionMessagesAsync() + { + // Regression test for https://github.com/microsoft/agent-framework/issues/3161 + // With two hops (initial -> second -> third), each target agent should receive the + // original user message and text responses from prior agents (as User role), but + // NOT any handoff function call or tool result messages. + + List? capturedSecondAgentMessages = null; + List? capturedThirdAgentMessages = null; + + var initialAgent = new ChatClientAgent(new MockChatClient((messages, options) => + { + string? transferFuncName = options?.Tools?.FirstOrDefault(t => t.Name.StartsWith("handoff_to_", StringComparison.Ordinal))?.Name; + Assert.NotNull(transferFuncName); + + // Return both a text message and a handoff function call + return new(new ChatMessage(ChatRole.Assistant, [new TextContent("Routing to second agent"), new FunctionCallContent("call1", transferFuncName)])); + }), name: "initialAgent"); + + var secondAgent = new ChatClientAgent(new MockChatClient((messages, options) => + { + capturedSecondAgentMessages = messages.ToList(); + + string? transferFuncName = options?.Tools?.FirstOrDefault(t => t.Name.StartsWith("handoff_to_", StringComparison.Ordinal))?.Name; + Assert.NotNull(transferFuncName); + + // Return both a text message and a handoff function call + return new(new ChatMessage(ChatRole.Assistant, [new TextContent("Routing to third agent"), new FunctionCallContent("call2", transferFuncName)])); + }), name: "secondAgent", description: "The second agent"); + + var thirdAgent = new ChatClientAgent(new MockChatClient((messages, options) => + { + capturedThirdAgentMessages = messages.ToList(); + return new(new ChatMessage(ChatRole.Assistant, "Hello from agent3")); + }), + name: "thirdAgent", + description: "The third / final agent"); + + var workflow = + AgentWorkflowBuilder.CreateHandoffBuilderWith(initialAgent) + .WithHandoff(initialAgent, secondAgent) + .WithHandoff(secondAgent, thirdAgent) + .Build(); + + (string updateText, _) = await RunWorkflowAsync(workflow, [new ChatMessage(ChatRole.User, "abc")]); + + Assert.Contains("Hello from agent3", updateText); + + // Second agent should see the original user message and initialAgent's text as context + Assert.NotNull(capturedSecondAgentMessages); + Assert.Contains(capturedSecondAgentMessages, m => m.Text == "abc"); + Assert.Contains(capturedSecondAgentMessages, m => m.Text!.Contains("Routing to second agent")); + Assert.DoesNotContain(capturedSecondAgentMessages, m => m.Contents.Any(c => c is FunctionCallContent fcc && fcc.Name.StartsWith("handoff_to_", StringComparison.Ordinal))); + Assert.DoesNotContain(capturedSecondAgentMessages, m => m.Role == ChatRole.Tool && m.Contents.Any(c => c is FunctionResultContent)); + + // Third agent should see the original user message and both prior agents' text as context + Assert.NotNull(capturedThirdAgentMessages); + Assert.Contains(capturedThirdAgentMessages, m => m.Text == "abc"); + Assert.Contains(capturedThirdAgentMessages, m => m.Text!.Contains("Routing to second agent")); + Assert.Contains(capturedThirdAgentMessages, m => m.Text!.Contains("Routing to third agent")); + Assert.DoesNotContain(capturedThirdAgentMessages, m => m.Contents.Any(c => c is FunctionCallContent fcc && fcc.Name.StartsWith("handoff_to_", StringComparison.Ordinal))); + Assert.DoesNotContain(capturedThirdAgentMessages, m => m.Role == ChatRole.Tool && m.Contents.Any(c => c is FunctionResultContent)); + } + + [Fact] + public async Task Handoffs_FilteringNone_HandoffTargetReceivesAllMessagesIncludingToolCallsAsync() + { + // With filtering set to None, the target agent should see everything including + // handoff function calls and tool results. + + List? capturedNextAgentMessages = null; + + var initialAgent = new ChatClientAgent(new MockChatClient((messages, options) => + { + string? transferFuncName = options?.Tools?.FirstOrDefault(t => t.Name.StartsWith("handoff_to_", StringComparison.Ordinal))?.Name; + Assert.NotNull(transferFuncName); + + return new(new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("call1", transferFuncName)])); + }), name: "initialAgent"); + + var nextAgent = new ChatClientAgent(new MockChatClient((messages, options) => + { + capturedNextAgentMessages = messages.ToList(); + return new(new ChatMessage(ChatRole.Assistant, "response")); + }), + name: "nextAgent", + description: "The second agent"); + + var workflow = + AgentWorkflowBuilder.CreateHandoffBuilderWith(initialAgent) + .WithHandoff(initialAgent, nextAgent) + .WithToolCallFilteringBehavior(HandoffToolCallFilteringBehavior.None) + .Build(); + + _ = await RunWorkflowAsync(workflow, [new ChatMessage(ChatRole.User, "hello")]); + + Assert.NotNull(capturedNextAgentMessages); + Assert.Contains(capturedNextAgentMessages, m => m.Text == "hello"); + + // With None filtering, handoff function calls and tool results should be visible + Assert.Contains(capturedNextAgentMessages, m => m.Contents.Any(c => c is FunctionCallContent fcc && fcc.Name.StartsWith("handoff_to_", StringComparison.Ordinal))); + Assert.Contains(capturedNextAgentMessages, m => m.Contents.Any(c => c is FunctionResultContent)); + } + + [Fact] + public async Task Handoffs_FilteringAll_HandoffTargetDoesNotReceiveAnyToolCallsAsync() + { + // With filtering set to All, the target agent should see no function calls or tool + // results at all — not even non-handoff ones from prior conversation history. + + List? capturedNextAgentMessages = null; + + var initialAgent = new ChatClientAgent(new MockChatClient((messages, options) => + { + string? transferFuncName = options?.Tools?.FirstOrDefault(t => t.Name.StartsWith("handoff_to_", StringComparison.Ordinal))?.Name; + Assert.NotNull(transferFuncName); + + return new(new ChatMessage(ChatRole.Assistant, [new TextContent("Routing you now"), new FunctionCallContent("call1", transferFuncName)])); + }), name: "initialAgent"); + + var nextAgent = new ChatClientAgent(new MockChatClient((messages, options) => + { + capturedNextAgentMessages = messages.ToList(); + return new(new ChatMessage(ChatRole.Assistant, "response")); + }), + name: "nextAgent", + description: "The second agent"); + + var workflow = + AgentWorkflowBuilder.CreateHandoffBuilderWith(initialAgent) + .WithHandoff(initialAgent, nextAgent) + .WithToolCallFilteringBehavior(HandoffToolCallFilteringBehavior.All) + .Build(); + + // Input includes a pre-existing non-handoff tool call in the conversation history + List input = + [ + new(ChatRole.User, "What's the weather? Also help me with math."), + new(ChatRole.Assistant, [new FunctionCallContent("toolcall1", "get_weather")]) { AuthorName = "initialAgent" }, + new(ChatRole.Tool, [new FunctionResultContent("toolcall1", "sunny")]), + new(ChatRole.Assistant, "The weather is sunny. Now let me route your math question.") { AuthorName = "initialAgent" }, + ]; + + _ = await RunWorkflowAsync(workflow, input); + + Assert.NotNull(capturedNextAgentMessages); + + // With All filtering, NO function calls or tool results should be visible + Assert.DoesNotContain(capturedNextAgentMessages, m => m.Contents.Any(c => c is FunctionCallContent)); + Assert.DoesNotContain(capturedNextAgentMessages, m => m.Role == ChatRole.Tool); + + // But text content should still be visible + Assert.Contains(capturedNextAgentMessages, m => m.Text!.Contains("What's the weather")); + Assert.Contains(capturedNextAgentMessages, m => m.Text!.Contains("Routing you now")); + } + + [Fact] + public async Task Handoffs_FilteringHandoffOnly_PreservesNonHandoffToolCallsAsync() + { + // With HandoffOnly filtering (the default), non-handoff function calls and tool + // results should be preserved while handoff ones are stripped. + + List? capturedNextAgentMessages = null; + + var initialAgent = new ChatClientAgent(new MockChatClient((messages, options) => + { + string? transferFuncName = options?.Tools?.FirstOrDefault(t => t.Name.StartsWith("handoff_to_", StringComparison.Ordinal))?.Name; + Assert.NotNull(transferFuncName); + + return new(new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("call1", transferFuncName)])); + }), name: "initialAgent"); + + var nextAgent = new ChatClientAgent(new MockChatClient((messages, options) => + { + capturedNextAgentMessages = messages.ToList(); + return new(new ChatMessage(ChatRole.Assistant, "response")); + }), + name: "nextAgent", + description: "The second agent"); + + var workflow = + AgentWorkflowBuilder.CreateHandoffBuilderWith(initialAgent) + .WithHandoff(initialAgent, nextAgent) + .WithToolCallFilteringBehavior(HandoffToolCallFilteringBehavior.HandoffOnly) + .Build(); + + // Input includes a pre-existing non-handoff tool call in the conversation history + List input = + [ + new(ChatRole.User, "What's the weather? Also help me with math."), + new(ChatRole.Assistant, [new FunctionCallContent("toolcall1", "get_weather")]) { AuthorName = "initialAgent" }, + new(ChatRole.Tool, [new FunctionResultContent("toolcall1", "sunny")]), + new(ChatRole.Assistant, "The weather is sunny. Now let me route your math question.") { AuthorName = "initialAgent" }, + ]; + + _ = await RunWorkflowAsync(workflow, input); + + Assert.NotNull(capturedNextAgentMessages); + + // Handoff function calls and their tool results should be filtered + Assert.DoesNotContain(capturedNextAgentMessages, m => m.Contents.Any(c => c is FunctionCallContent fcc && fcc.Name.StartsWith("handoff_to_", StringComparison.Ordinal))); + + // Non-handoff function calls and their tool results should be preserved + Assert.Contains(capturedNextAgentMessages, m => m.Contents.Any(c => c is FunctionCallContent fcc && fcc.Name == "get_weather")); + Assert.Contains(capturedNextAgentMessages, m => m.Role == ChatRole.Tool && m.Contents.Any(c => c is FunctionResultContent frc && frc.CallId == "toolcall1")); + } + [Fact] public async Task Handoffs_TwoTransfers_ResponseServedByThirdAgentAsync() {