diff --git a/tests/AgentMemory.Tests.Unit.LongMemEval/MafAgentTaskRunnerTests.cs b/tests/AgentMemory.Tests.Unit.LongMemEval/MafAgentTaskRunnerTests.cs new file mode 100644 index 00000000..83af869f --- /dev/null +++ b/tests/AgentMemory.Tests.Unit.LongMemEval/MafAgentTaskRunnerTests.cs @@ -0,0 +1,106 @@ +using AgentMemory.LongMemEval; +using FluentAssertions; +using Microsoft.Extensions.AI; +using Xunit; + +namespace AgentMemory.Tests.Unit.LongMemEval; + +/// +/// Counting what an agent actually did (7.6). +/// +/// +/// +/// The harness decides what a benefit is; this decides what the numbers are. Both are +/// read off the returned messages rather than from the agent's own account of itself — asking a model +/// how many tools it used yields a figure that tracks how talkative it is, and a procedure that made +/// the agent merely more confident would show up as an efficiency win. +/// +/// +/// Provider-free: these assert the counting rules against constructed transcripts, which is the part +/// that can be quietly wrong. The part that costs money is the run. +/// +/// +public sealed class MafAgentTaskRunnerTests +{ + private static ChatMessage Assistant(params AIContent[] contents) => + new(ChatRole.Assistant, [.. contents]); + + private static FunctionCallContent Call(string id, string name) => new(id, name, null); + + [Fact] + public void StepsCountAssistantTurns() + { + var messages = new[] + { + new ChatMessage(ChatRole.User, "do the task"), + Assistant(new TextContent("thinking")), + Assistant(new TextContent("done")), + }; + + MafAgentTaskRunner.CountSteps(messages).Should().Be(2); + } + + [Fact] + public void ToolResultsAreNotSteps() + { + // The environment answering is not the agent acting. Counting tool results would make a + // procedure that batches its calls look like it took MORE steps rather than fewer -- inverting + // the exact signal the harness is looking for. + var messages = new[] + { + Assistant(Call("c1", "search")), + new ChatMessage(ChatRole.Tool, [new FunctionResultContent("c1", "result")]), + Assistant(new TextContent("done")), + }; + + MafAgentTaskRunner.CountSteps(messages).Should().Be(2); + } + + [Fact] + public void ToolCallsAreCountedAcrossEveryMessage() + { + // Several calls can share one assistant turn -- that is what batching looks like, and counting + // per message instead of per call would hide it. + var messages = new[] + { + Assistant(Call("c1", "search"), Call("c2", "lookup")), + new ChatMessage(ChatRole.Tool, [new FunctionResultContent("c1", "a")]), + Assistant(Call("c3", "book")), + }; + + MafAgentTaskRunner.CountToolCalls(messages).Should().Be(3); + } + + [Fact] + public void AToolResultIsNotAToolCall() + { + var messages = new[] + { + Assistant(Call("c1", "search")), + new ChatMessage(ChatRole.Tool, [new FunctionResultContent("c1", "result")]), + }; + + MafAgentTaskRunner.CountToolCalls(messages).Should().Be(1); + } + + [Fact] + public void AnEmptyTranscriptCountsZeroRatherThanThrowing() + { + // A run that failed before producing anything must score as an expensive nothing, not crash + // the arm and take the other attempts' data with it. + MafAgentTaskRunner.CountSteps([]).Should().Be(0); + MafAgentTaskRunner.CountToolCalls([]).Should().Be(0); + } + + [Fact] + public void CompletionIsDecidedByTheCallerNotTheAgent() + { + // An agent that learned a WRONG procedure reports success fluently. Completion has to be + // checked against the world, which is why the predicate is supplied rather than inferred from + // the agent claiming it finished -- and why 7.7 measures wrong-procedure rate separately. + Func strict = text => text.Contains("BOOKING-CONFIRMED", StringComparison.Ordinal); + + strict("I have completed the task successfully!").Should().BeFalse(); + strict("BOOKING-CONFIRMED ref 91821").Should().BeTrue(); + } +} diff --git a/tools/AgentMemory.LongMemEval/MafAgentTaskRunner.cs b/tools/AgentMemory.LongMemEval/MafAgentTaskRunner.cs new file mode 100644 index 00000000..a3717fbc --- /dev/null +++ b/tools/AgentMemory.LongMemEval/MafAgentTaskRunner.cs @@ -0,0 +1,89 @@ +using Microsoft.Agents.AI; +using Microsoft.Extensions.AI; + +namespace AgentMemory.LongMemEval; + +/// +/// Drives a repeated multi-step task through a real Agent Framework agent and reports what it cost +/// (7.6). +/// +/// +/// +/// The half of the procedural-benefit harness that cannot be scripted. The harness decides what a +/// benefit is — completion first, efficiency second, learning proved across attempts — and is +/// unit-tested against a scripted runner. This is the part that actually calls an agent, so it is +/// where an honest step and tool-call count has to come from. +/// +/// +/// Steps and tool calls are counted from the run, never from the agent's own account of itself. +/// Asking a model how many tools it used produces a number that correlates with how talkative it is. +/// Both figures here are read off the returned messages, which is what makes them a measurement +/// rather than a self-report — and which is what would otherwise quietly flatter a procedure that +/// merely made the agent more confident. +/// +/// +/// Completion is decided by a caller-supplied predicate over the transcript, not by the agent saying +/// it is done. An agent that has learned a wrong procedure will report success fluently, and 7.7's +/// wrong-procedure rate exists precisely because that failure is invisible to efficiency numbers. +/// +/// +internal sealed class MafAgentTaskRunner : IAgentTaskRunner +{ + private readonly Func _agentFactory; + private readonly string _taskPrompt; + private readonly Func _isComplete; + + /// + /// Builds an agent with procedural memory on or off. A factory rather than two instances because + /// the arms must not share conversation state — a second run reusing the first arm's thread would + /// measure the thread, not the memory. + /// + /// The task, issued identically on every attempt. + /// Decides completion from the final transcript. + public MafAgentTaskRunner( + Func agentFactory, + string taskPrompt, + Func isComplete) + { + _agentFactory = agentFactory; + _taskPrompt = taskPrompt; + _isComplete = isComplete; + } + + /// + public async Task RunAsync( + string taskId, + bool procedureMemoryEnabled, + int attempt, + CancellationToken cancellationToken = default) + { + var agent = _agentFactory(procedureMemoryEnabled); + + // A fresh session per attempt. Procedural memory is supposed to carry across attempts through + // the STORE; a shared session would carry it through the context window instead, and the + // measurement would credit memory for what the transcript did. + var session = await agent.CreateSessionAsync(cancellationToken).ConfigureAwait(false); + + var response = await agent.RunAsync(_taskPrompt, session, cancellationToken: cancellationToken) + .ConfigureAwait(false); + + var messages = response.Messages ?? []; + return new AgentTaskRun( + Completed: _isComplete(response.Text ?? string.Empty), + Steps: CountSteps(messages), + ToolCalls: CountToolCalls(messages)); + } + + /// Assistant turns taken — the agent's own reasoning steps. + /// + /// Tool-result messages are excluded: they are the environment answering, not the agent acting, + /// and counting them would make a procedure that batches tool calls look like it took more steps + /// rather than fewer. + /// + internal static int CountSteps(IEnumerable messages) => + messages.Count(m => m.Role == ChatRole.Assistant); + + /// Tool invocations across the whole run. + internal static int CountToolCalls(IEnumerable messages) => + messages.Sum(m => m.Contents.OfType().Count()); +}