diff --git a/docs/performance/README.md b/docs/performance/README.md index b175e6f7..69a89b5b 100644 --- a/docs/performance/README.md +++ b/docs/performance/README.md @@ -111,6 +111,10 @@ dotnet run --project tools/AgentMemory.Cli -- perf --label mine --latency remote dotnet run --project tools/AgentMemory.Cli -- perf --label degraded \ --scenarios PERF-R-07 --iterations 3 +# Measures full memory + deterministic GraphRAG orchestration +dotnet run --project tools/AgentMemory.Cli -- perf --label graphrag \ + --scenarios PERF-R-08 --iterations 3 + # Compare two in-process recall configurations, with quality in the same report dotnet run --project tools/AgentMemory.Cli -- perf ab \ --control default \ @@ -159,10 +163,12 @@ The judged quality fixtures had zero observed variance across five complete runs tolerance is zero rather than a guessed allowance. All measured scenarios also **self-assert**. The full and degraded recall scenarios fail if they -retrieve fewer items than the configured limits; the degraded scenario additionally verifies that its -embedding and database waits occurred inside recorded stage spans. The greeting scenario locks its -current default-policy item shape, while both ingestion scenarios verify message persistence and -extraction outcomes. Those failures are otherwise silent and would produce a confident, wrong number. +retrieve fewer items than the configured limits; the degraded scenario additionally verifies that +its embedding and database waits occurred inside recorded stage spans. The GraphRAG scenario requires +its source call, two known items, configured wait, stage span, and rendered marker text while preserving +the complete memory result. The greeting scenario locks its current default-policy item shape, while +both ingestion scenarios verify message persistence and extraction outcomes. Those failures are +otherwise silent and would produce a confident, wrong number. ### Pull-request regression gate diff --git a/docs/performance/baseline-1.3.0.md b/docs/performance/baseline-1.3.0.md index 95d19ac4..aaa13cbe 100644 --- a/docs/performance/baseline-1.3.0.md +++ b/docs/performance/baseline-1.3.0.md @@ -84,6 +84,32 @@ entire 43-item result is preserved and both degraded stages are now observable. can use this control to prove bounded completion; graceful-degradation work must additionally report which categories were omitted rather than silently returning less. +### GraphRAG orchestration control + +`PERF-R-08` adds a scenario-only deterministic GraphRAG source to the complete `PERF-R-04` memory +shape. It returns two known context items after a configured 300 ms wait: + +| Counter | Full recall (`R-04`) | GraphRAG (`R-08`) | +|---|---:|---:| +| Memory items retrieved | 43 | **43** | +| GraphRAG items | – | **2** | +| Prompt messages | 14 | **15** | +| Embedding requests | 1 | **1** | +| Neo4j read / write transactions | 6 / 1 | **6 / 1** | +| Neo4j queries | 9 | **9** | +| Access timestamps updated | 25 | **25** | +| Configured GraphRAG wait | – | **1 × 300 ms** | + +The portable result is that enabling this optional path adds its two known items without changing or +dropping any of the 43 memory items. One `memory.recall.graphrag` span and the marker text in the final +Agent Framework context prove the source was registered, enabled, invoked, and materialized. + +The remote-shaped hermetic run also validates rank 17's orchestration target. The provider embedding +span had a 126 ms median and completed before the 312 ms GraphRAG span because the provider currently +awaits embedding before entering the assembler; median turn elapsed was 468 ms. Rank 17 should overlap +those controlled stages so their contribution tends toward the slower stage instead of their sum. +These are deterministic stimulus figures for an A/B control—not deployment performance. + ### Six-message tool-heavy ingestion control `PERF-W-03` holds the scripted extraction result constant while increasing response messages from one diff --git a/eng/perf/baselines/hermetic-S.json b/eng/perf/baselines/hermetic-S.json index f9f7793f..dfe380d7 100644 --- a/eng/perf/baselines/hermetic-S.json +++ b/eng/perf/baselines/hermetic-S.json @@ -70,6 +70,31 @@ "recall.chars": 3823 } }, + "PERF-R-08": { + "counters": { + "access_tracking.items": 25, + "context.chars": 4072, + "context.messages": 15, + "embed.chars": 95, + "embed.items": 1, + "embed.requests": 1, + "graphrag.calls": 1, + "injected.graphrag_delay.calls": 1, + "injected.graphrag_delay.ms": 300, + "items.entities": 10, + "items.facts": 10, + "items.graphrag": 2, + "items.preferences": 5, + "items.recent": 10, + "items.relevant": 5, + "items.retrieved": 43, + "items.traces": 3, + "neo4j.queries": 9, + "neo4j.tx.read": 6, + "neo4j.tx.write": 1, + "recall.chars": 3934 + } + }, "PERF-W-02": { "counters": { "embed.chars": 201, diff --git a/tests/AgentMemory.Tests.Unit/Cli/PerfScenarioCatalogTests.cs b/tests/AgentMemory.Tests.Unit/Cli/PerfScenarioCatalogTests.cs index dff7d209..0dbc0a4f 100644 --- a/tests/AgentMemory.Tests.Unit/Cli/PerfScenarioCatalogTests.cs +++ b/tests/AgentMemory.Tests.Unit/Cli/PerfScenarioCatalogTests.cs @@ -64,4 +64,24 @@ public void Select_DegradedRecallScenario_ReturnsOnlyThatScenario() selected.Should().ContainSingle(); selected[0].Id.Should().Be("PERF-R-07"); } + + [Fact] + public void Catalog_ContainsGraphRagRecallScenario_WithStableContract() + { + var scenario = PerfScenarios.All.Single(item => item.Id == "PERF-R-08"); + + scenario.Description.Should().ContainEquivalentOf("GraphRAG"); + scenario.Description.Should().ContainEquivalentOf("recall"); + scenario.SupportsInterleavedAb.Should().BeTrue( + "the GraphRAG source is deterministic and the recall fixture is isolated per A/B arm"); + } + + [Fact] + public void Select_GraphRagRecallScenario_ReturnsOnlyThatScenario() + { + var selected = PerfScenarios.Select("PERF-R-08"); + + selected.Should().ContainSingle(); + selected[0].Id.Should().Be("PERF-R-08"); + } } diff --git a/tools/AgentMemory.Cli/Commands/PerfCommand.cs b/tools/AgentMemory.Cli/Commands/PerfCommand.cs index 481b11d5..3c313729 100644 --- a/tools/AgentMemory.Cli/Commands/PerfCommand.cs +++ b/tools/AgentMemory.Cli/Commands/PerfCommand.cs @@ -351,6 +351,7 @@ private static object BuildSummary( "context.chars" or "recall.chars" => "characters", "items.retrieved" => "memory items", _ when counter.StartsWith("items.", StringComparison.Ordinal) => "memory items", + "graphrag.calls" => "GraphRAG requests", _ when counter.EndsWith("_delay.calls", StringComparison.Ordinal) => "injected waits", _ when counter.EndsWith("_delay.ms", StringComparison.Ordinal) => "configured milliseconds", _ => "count", diff --git a/tools/AgentMemory.Cli/Perf/DeterministicGraphRagContextSource.cs b/tools/AgentMemory.Cli/Perf/DeterministicGraphRagContextSource.cs new file mode 100644 index 00000000..f047e930 --- /dev/null +++ b/tools/AgentMemory.Cli/Perf/DeterministicGraphRagContextSource.cs @@ -0,0 +1,60 @@ +using AgentMemory.Abstractions.Domain; +using AgentMemory.Abstractions.Services; + +namespace AgentMemory.Cli.Perf; + +/// +/// Scenario-only GraphRAG source with a known result and delay. It lets the harness prove that the +/// optional path ran without coupling a GraphRAG orchestration measurement to a second search/index +/// implementation whose own behavior belongs in integration tests. +/// +internal sealed class DeterministicGraphRagContextSource : IGraphRagContextSource +{ + internal const long DelayMilliseconds = 300; + internal const string FirstMarker = + "Project Atlas depends on the Beacon identity service."; + internal const string SecondMarker = + "The platform team owns Beacon and its incident response."; + + public async Task GetContextAsync( + GraphRagContextRequest request, + CancellationToken cancellationToken = default) + { + PerfCollector.Current?.Add("graphrag.calls"); + + await Task.Delay( + TimeSpan.FromMilliseconds(DelayMilliseconds), + cancellationToken) + .ConfigureAwait(false); + + PerfCollector.Current?.Add("injected.graphrag_delay.calls"); + PerfCollector.Current?.Add( + "injected.graphrag_delay.ms", + DelayMilliseconds); + + IReadOnlyList all = + [ + new() + { + Text = FirstMarker, + Score = 1.0, + Metadata = new Dictionary + { + ["source"] = "deterministic-perf-fixture", + }, + }, + new() + { + Text = SecondMarker, + Score = 0.9, + Metadata = new Dictionary + { + ["source"] = "deterministic-perf-fixture", + }, + }, + ]; + var items = all.Take(Math.Max(0, request.TopK)).ToList(); + PerfCollector.Current?.Add("items.graphrag", items.Count); + return new GraphRagContextResult { Items = items }; + } +} diff --git a/tools/AgentMemory.Cli/Perf/HermeticProfile.cs b/tools/AgentMemory.Cli/Perf/HermeticProfile.cs index 3d3176b3..5957e09b 100644 --- a/tools/AgentMemory.Cli/Perf/HermeticProfile.cs +++ b/tools/AgentMemory.Cli/Perf/HermeticProfile.cs @@ -95,6 +95,11 @@ private async Task InitializeAsync( // stay registered and a post-turn scenario would measure extraction that never happens. llm => { }); + // Registered exactly as a host source would be, but the shared MemoryOptions keep GraphRAG + // disabled. PERF-R-08 builds an isolated production assembler with EnableGraphRag=true; every + // other scenario continues to exercise shipped defaults and cannot call this source. + services.AddSingleton(); + DecorateTransactionRunner(services, DependencyLatency); // Counting wrappers sit outermost so they observe every call the product makes, including the diff --git a/tools/AgentMemory.Cli/Perf/PerfScenarios.cs b/tools/AgentMemory.Cli/Perf/PerfScenarios.cs index e662c206..e313928c 100644 --- a/tools/AgentMemory.Cli/Perf/PerfScenarios.cs +++ b/tools/AgentMemory.Cli/Perf/PerfScenarios.cs @@ -1,6 +1,8 @@ using AgentMemory.Abstractions.Options; +using AgentMemory.Abstractions.Repositories; using AgentMemory.Abstractions.Services; using AgentMemory.AgentFramework; +using AgentMemory.Core.Services; using Microsoft.Agents.AI; using Microsoft.Extensions.AI; using Microsoft.Extensions.DependencyInjection; @@ -64,6 +66,10 @@ public static class PerfScenarios "Degraded dependency recall (embedding 2 s, database transaction 250 ms)", DegradedRecallAsync, DependencyLatency: PerfDependencyLatencyPreset.Degraded), + new( + "PERF-R-08", + "GraphRAG-enabled full recall with deterministic context", + GraphRagRecallAsync), new( "PERF-W-02", "Single response message, extraction enabled (shipped defaults)", @@ -201,14 +207,65 @@ private static async Task DegradedRecallAsync(ScenarioContext ctx) } } - private static async Task RunDefaultRecallAsync(ScenarioContext ctx, string scenarioId) + /// + /// PERF-R-08 — the full memory recall plus a scenario-only deterministic GraphRAG source. The + /// Agent Framework provider currently finishes its query embedding before the assembler can start + /// GraphRAG; rank 17 uses this control to prove those two stages overlap after orchestration moves. + /// + private static async Task GraphRagRecallAsync(ScenarioContext ctx) + { + var provider = CreateProvider(ctx.Profile, ctx.RecallOptions, enableGraphRag: true); + var context = await RunDefaultRecallAsync(ctx, "PERF-R-08", provider).ConfigureAwait(false); + + var graphRagSpans = ctx.Turn.SpanCounts.TryGetValue( + "memory.recall.graphrag", out var graphRagSpanCount) + ? graphRagSpanCount + : 0; + var graphRagCalls = ctx.Turn.Counter("graphrag.calls"); + var graphRagItems = ctx.Turn.Counter("items.graphrag"); + var graphRagDelayCalls = ctx.Turn.Counter("injected.graphrag_delay.calls"); + var graphRagDelayMs = ctx.Turn.Counter("injected.graphrag_delay.ms"); + var embeddings = ctx.Turn.Counter("embed.requests"); + var reads = ctx.Turn.Counter("neo4j.tx.read"); + var writes = ctx.Turn.Counter("neo4j.tx.write"); + var queries = ctx.Turn.Counter("neo4j.queries"); + var accessTracked = ctx.Turn.Counter("access_tracking.items"); + var materialized = context.Messages?.Any(message => + message.Text?.Contains( + DeterministicGraphRagContextSource.FirstMarker, + StringComparison.Ordinal) == true && + message.Text.Contains( + DeterministicGraphRagContextSource.SecondMarker, + StringComparison.Ordinal)) == true; + + if (graphRagSpans != 1 || graphRagCalls != 1 || graphRagItems != 2 || + graphRagDelayCalls != 1 || + graphRagDelayMs != DeterministicGraphRagContextSource.DelayMilliseconds || + embeddings != 1 || reads != 6 || writes != 1 || queries != 9 || + accessTracked != 25 || !materialized) + { + throw new InvalidOperationException( + $"PERF-R-08 did not exercise its GraphRAG contract " + + $"(spans={graphRagSpans}/1, calls={graphRagCalls}/1, items={graphRagItems}/2, " + + $"delay calls/ms={graphRagDelayCalls}/{graphRagDelayMs}, expected " + + $"1/{DeterministicGraphRagContextSource.DelayMilliseconds}; embed.requests=" + + $"{embeddings}/1; neo4j read/write/queries={reads}/{writes}/{queries}, expected " + + $"6/1/9; access_tracking.items={accessTracked}/25; materialized={materialized}/true). " + + "A disabled or unregistered GraphRAG source would make this measurement a no-op."); + } + } + + private static async Task RunDefaultRecallAsync( + ScenarioContext ctx, + string scenarioId, + Neo4jMemoryContextProvider? provider = null) { var identity = ctx.Variant is null ? PerfFixture.DefaultIdentity : PerfFixture.ForVariant(ctx.Variant); var messages = new[] { new ChatMessage(ChatRole.User, PerfFixture.ProbeQueryFor(identity)) }; - var context = await ctx.Provider.BuildContextAsync( + var context = await (provider ?? ctx.Provider).BuildContextAsync( messages, identity.SessionId, identity.ConversationId, @@ -231,6 +288,8 @@ private static async Task RunDefaultRecallAsync(ScenarioContext ctx, string scen $"({breakdown}). The fixture is not exercising the configured recall shape, so this " + "measurement would be misleading. Check MinSimilarityScore and the seeded embeddings."); } + + return context; } private static void RecordContext(TurnRecord turn, AIContext context) @@ -351,19 +410,63 @@ private static void AssertScriptedExtraction(ScenarioContext ctx, string scenari /// public static Neo4jMemoryContextProvider CreateProvider( HermeticProfile profile, - RecallOptions? recallOptions = null) + RecallOptions? recallOptions = null, + bool enableGraphRag = false) { var services = profile.Services; var configuredMemory = services.GetRequiredService>().Value; var selectedRecall = recallOptions ?? configuredMemory.Recall; + var selectedMemory = configuredMemory with + { + Recall = selectedRecall, + EnableGraphRag = enableGraphRag || configuredMemory.EnableGraphRag, + }; + var memoryService = enableGraphRag + ? CreateGraphRagMemoryService(services, selectedMemory) + : services.GetRequiredService(); return new Neo4jMemoryContextProvider( - services.GetRequiredService(), + memoryService, services.GetRequiredService(), services.GetRequiredService(), services.GetRequiredService(), - Options.Create(configuredMemory with { Recall = selectedRecall }), + Options.Create(selectedMemory), Options.Create(new ContextFormatOptions()), Options.Create(new AgentFrameworkOptions()), services.GetRequiredService>()); } + + private static IMemoryService CreateGraphRagMemoryService( + IServiceProvider services, + MemoryOptions memoryOptions) + { + var options = Options.Create(memoryOptions); + var shortTerm = services.GetRequiredService(); + var embedding = services.GetRequiredService(); + var assembler = new MemoryContextAssembler( + shortTerm, + services.GetRequiredService(), + services.GetRequiredService(), + services.GetRequiredService(), + embedding, + services.GetRequiredService(), + options, + services.GetRequiredService>(), + services.GetRequiredService(), + services.GetService()); + + return new MemoryService( + shortTerm, + assembler, + services.GetRequiredService(), + services.GetRequiredService(), + services.GetRequiredService(), + services.GetRequiredService(), + embedding, + options, + services.GetRequiredService(), + services.GetRequiredService(), + services.GetRequiredService>(), + services.GetService(), + services.GetService()); + } }