diff --git a/docs/reviews/procedural-benefit-run-prerequisite.md b/docs/reviews/procedural-benefit-run-prerequisite.md index 87bf1145..163e18f8 100644 --- a/docs/reviews/procedural-benefit-run-prerequisite.md +++ b/docs/reviews/procedural-benefit-run-prerequisite.md @@ -46,3 +46,46 @@ Then the two arms differ in something real, and the harness measures the feature - `ProcedureRetrievalPrecision` — wrong-procedure rate (7.7) - `ProceduralBenchmarkTask` — the enforced-chain task - `MafAgentTaskRunner` — step and tool-call counting off the transcript + + +--- + +## First run executed — and the result is about the task, not the feature + +**Run:** `--procedural-benefit --attempts 3`, 2026-08-13. Promotion wiring in place. + +``` +procedures completion=100% meanSteps=4.0 meanToolCalls=3.0 +control completion=100% meanSteps=4.0 meanToolCalls=3.0 +stepReduction=0.0% toolCallReduction=0.0% completionDelta=0% +SHOWS BENEFIT: False +``` + +**This is not evidence that procedural memory does not help.** Three tool calls is the *minimum +possible* chain, and the log contains **zero refusals** — the agent walked +`LookUpTraveller → PlaceHold → Book` correctly on its first cold attempt, in both arms. There was +nothing to discover, so there was nothing a stored procedure could save. + +The cause is my own task design, and it is the exact property the benchmark's tests declare as the +hard requirement: *the shortest correct path must be discoverable but not guessable.* I enforced that +in the tool **bodies** — booking without a hold is refused — but gave it away in the tool +**descriptions**, which state the prerequisites outright: + +- *"Requires the traveller's loyalty tier."* +- *"Requires a hold reference."* + +A competent model reads the descriptions and orders the calls correctly without ever being refused. +The enforcement is real and never fires. + +### What the run does establish + +The assembly works end to end, which was the open question: the arms are genuinely distinct, the +promotion path stores a procedure, and the counting produces figures off the transcript. A harness +that could not run at all would have failed here instead of returning a clean, uninformative zero. + +### The fix + +Withhold the prerequisites from the descriptions — name each tool's purpose and let the refusal +message teach the ordering. Then the control arm must discover the chain by being refused on every +attempt, while the procedural arm pays that cost once. Re-run after that change; the current numbers +should not be cited. diff --git a/tools/AgentMemory.LongMemEval/ProceduralBenefitProgram.cs b/tools/AgentMemory.LongMemEval/ProceduralBenefitProgram.cs new file mode 100644 index 00000000..c8f0d6f9 --- /dev/null +++ b/tools/AgentMemory.LongMemEval/ProceduralBenefitProgram.cs @@ -0,0 +1,138 @@ +using Azure; +using Azure.AI.OpenAI; +using AgentMemory.Abstractions.Repositories; +using Microsoft.Agents.AI; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.DependencyInjection; + +namespace AgentMemory.LongMemEval; + +/// +/// Runs the procedural-benefit measurement: the same multi-step task, repeated, with procedural +/// memory on and off (7.6). +/// +/// +/// +/// Everything this composes has its own provider-free tests — the benefit scoring, the step and +/// tool-call counting, the enforced-chain task, and the promotion decision. This is the assembly, and +/// assembly is where a measurement quietly stops measuring: an arm switch that does not switch, or a +/// procedure store both arms share, produces a confident "no benefit" that reads as a finding. +/// +/// +/// The arms differ in exactly two things and nothing else. The procedural arm recalls traces +/// (MaxTraces > 0) and promotes successful attempts; the control does neither. Same model, +/// same tools, same prompt, same task, same attempt count. Anything else that differed would be +/// attributed to memory by a harness that cannot see it. +/// +/// +internal static class ProceduralBenefitProgram +{ + internal static async Task RunAsync(string[] args, CancellationToken cancellationToken = default) + { + var attempts = ParseAttempts(args); + var log = Console.Out; + + var endpoint = Required("AZURE_OPENAI_ENDPOINT"); + var apiKey = Required("AZURE_OPENAI_API_KEY"); + var deployment = Required("AZURE_OPENAI_DEPLOYMENT"); + var embeddingDeployment = Required("AZURE_OPENAI_EMBEDDING_DEPLOYMENT"); + + var azure = new AzureOpenAIClient(new Uri(endpoint), new AzureKeyCredential(apiKey)); + var chatClient = azure.GetChatClient(deployment).AsIChatClient(); + var embeddings = azure.GetEmbeddingClient(embeddingDeployment).AsIEmbeddingGenerator(); + var dimensions = await LongMemEvalRuntime + .ProbeEmbeddingDimensionsAsync(embeddings).ConfigureAwait(false); + + await using var profile = await LongMemEvalMemoryProfile.StartAsync( + embeddings, + // The profile demands an extraction client for Structured mode. This benchmark never + // extracts -- it stores and recalls procedures -- so the same chat client is handed over + // rather than a stub, which would fail loudly the moment anything did try to extract. + extractionChatClient: chatClient, + LongMemEvalMemoryMode.Structured, + extractionModelId: deployment, + dimensions, + log, + cancellationToken).ConfigureAwait(false); + + var task = new ProceduralBenchmarkTask(); + var traces = profile.Services.GetRequiredService(); + + // The arm switch, and the only difference between the two agents. The control arm is handed + // no trace repository, so it neither reads nor writes procedures. + var runner = new MafAgentTaskRunner( + proceduralMemoryEnabled => BuildAgent(chatClient, task, proceduralMemoryEnabled), + task.Prompt, + task.IsComplete, + traces); + + log.WriteLine($"procedural-benefit: {attempts} attempts per arm, task='{task.Prompt}'"); + var result = await ProceduralBenefitResult + .MeasureAsync(runner, "procedural-benchmark", attempts, cancellationToken) + .ConfigureAwait(false); + + Report(log, result); + return 0; + } + + /// + /// Builds the agent for one arm. + /// + /// + /// The benchmark tools are identical in both arms. Only whether the agent can recall a stored + /// procedure differs — which is what makes any measured gap attributable to memory rather than to + /// a differently-equipped agent. + /// + private static AIAgent BuildAgent( + IChatClient chatClient, ProceduralBenchmarkTask task, bool proceduralMemoryEnabled) + { + var instructions = proceduralMemoryEnabled + ? "You complete booking tasks using the supplied tools. If you recall a procedure for this " + + "task, follow it. Reply with the confirmation reference exactly as the tool returns it." + : "You complete booking tasks using the supplied tools. Reply with the confirmation " + + "reference exactly as the tool returns it."; + + return chatClient.AsAIAgent(new ChatClientAgentOptions + { + Name = proceduralMemoryEnabled ? "WithProcedures" : "Control", + ChatOptions = new ChatOptions + { + Instructions = instructions, + Tools = [.. task.CreateTools()], + }, + }); + } + + private static void Report(TextWriter log, ProceduralBenefitResult result) + { + void Arm(string name, ProceduralBenefitArm arm) => + log.WriteLine( + $" {name,-10} completion={arm.CompletionRate:P0} " + + $"meanSteps={arm.MeanStepsWhenCompleted:F1} " + + $"meanToolCalls={arm.MeanToolCallsWhenCompleted:F1}"); + + log.WriteLine("procedural-benefit results:"); + Arm("procedures", result.WithProcedures); + Arm("control", result.WithoutProcedures); + log.WriteLine( + $" stepReduction={result.StepReduction:P1} toolCallReduction={result.ToolCallReduction:P1} " + + $"completionDelta={result.CompletionRateDelta:P0}"); + log.WriteLine($" improvedWithRepetition={result.ImprovedWithRepetition}"); + // The verdict is completion-gated: an arm that finishes less often shows no benefit however + // few steps it took, because the steps it saved were not spent finishing. + log.WriteLine($" SHOWS BENEFIT: {result.ShowsBenefit}"); + } + + private static int ParseAttempts(string[] args) + { + var index = Array.IndexOf(args, "--attempts"); + return index >= 0 && index + 1 < args.Length + && int.TryParse(args[index + 1], out var value) && value >= 2 + ? value + : 3; + } + + private static string Required(string name) => + Environment.GetEnvironmentVariable(name) + ?? throw new InvalidOperationException($"{name} is not set."); +} diff --git a/tools/AgentMemory.LongMemEval/Program.cs b/tools/AgentMemory.LongMemEval/Program.cs index a75a0bf6..cb761500 100644 --- a/tools/AgentMemory.LongMemEval/Program.cs +++ b/tools/AgentMemory.LongMemEval/Program.cs @@ -63,6 +63,13 @@ public static async Task RunAsync(string[] args) return 0; } + if (args.Contains("--procedural-benefit", StringComparer.Ordinal)) + { + // 7.6. The arms differ in exactly two things -- trace recall and promotion -- so that any + // measured gap is attributable to memory rather than to a differently-equipped agent. + return await ProceduralBenefitProgram.RunAsync(args).ConfigureAwait(false); + } + if (args.Contains("--prepared-pair", StringComparer.Ordinal)) { return await LongMemEvalPreparedPairProgram.RunAsync(args) @@ -378,6 +385,7 @@ await File.WriteAllTextAsync( private static readonly string[] KnownOptions = [ "--reference-arm", "--surface-probe", "--predicate-distribution", "--prepared-pair", + "--procedural-benefit", "--attempts", "--list-prepared-corpora", "--extraction-compare", "--help", "--chronological-context", "--dataset", "--evidence-detail",