From b20d337f39a41b18dd230a50e055daf8caa69d3b Mon Sep 17 00:00:00 2001 From: Jose Luis Latorre Millas Date: Sat, 25 Jul 2026 22:48:23 +0200 Subject: [PATCH 1/2] perf(M-21): preregister interleaved A/B From 2854b70f9a0edd791aa9d8e5a94429cce4d3ea48 Mon Sep 17 00:00:00 2001 From: Jose Luis Latorre Millas Date: Sun, 26 Jul 2026 04:30:50 +0200 Subject: [PATCH 2/2] Add trustworthy interleaved performance A/B --- docs/performance/README.md | 22 + .../AgentMemory.Tests.Unit/Cli/PerfAbTests.cs | 161 +++++ tools/AgentMemory.Cli/CliArgs.cs | 25 +- .../AgentMemory.Cli/Commands/PerfAbCommand.cs | 649 ++++++++++++++++++ tools/AgentMemory.Cli/Commands/PerfCommand.cs | 6 +- .../Perf/PairedRatioBootstrap.cs | 113 +++ .../AgentMemory.Cli/Perf/PerfConfiguration.cs | 120 ++++ tools/AgentMemory.Cli/Perf/PerfFixture.cs | 100 ++- tools/AgentMemory.Cli/Perf/PerfScenarios.cs | 43 +- .../AgentMemory.Cli/Perf/QualityEvaluator.cs | 12 +- tools/AgentMemory.Cli/Program.cs | 21 + 11 files changed, 1230 insertions(+), 42 deletions(-) create mode 100644 tests/AgentMemory.Tests.Unit/Cli/PerfAbTests.cs create mode 100644 tools/AgentMemory.Cli/Commands/PerfAbCommand.cs create mode 100644 tools/AgentMemory.Cli/Perf/PairedRatioBootstrap.cs create mode 100644 tools/AgentMemory.Cli/Perf/PerfConfiguration.cs diff --git a/docs/performance/README.md b/docs/performance/README.md index 425e0fe4..fca2ba1a 100644 --- a/docs/performance/README.md +++ b/docs/performance/README.md @@ -105,6 +105,12 @@ dotnet run --project tools/AgentMemory.Cli -- perf --label mine --iterations 10 # Reproduces the shape of a remote deployment (embedding 120 ms, model 900 ms) dotnet run --project tools/AgentMemory.Cli -- perf --label mine --latency remote --iterations 10 + +# Compare two in-process recall configurations, with quality in the same report +dotnet run --project tools/AgentMemory.Cli -- perf ab \ + --control default \ + --candidate Recall.MaxEntities=2 \ + --iterations 30 ``` Each run writes a dated directory containing a manifest with the full environment fingerprint, an @@ -115,6 +121,22 @@ marks those scores as report-only. Run it at both latency settings. A change that improves only the `remote` shape is an ordering or overlap win; one that improves both removed work. +### Trustworthy A/B comparisons + +`perf ab` counterbalances execution order and crosses each configuration over two equivalent, +owner-isolated fixture copies in the same database. Six consecutive paired iterations form one +bootstrap unit, preserving the Docker/driver timing correlation instead of treating adjacent samples +as independent. `--iterations` must therefore be a multiple of six and at least 12; 30 is the default. + +The initial configuration grammar accepts `default` plus `Recall.Max*` and +`Recall.MinSimilarityScore` assignments. The default scenario is `PERF-R-04`. State-mutating +scenarios such as `PERF-W-02` are rejected because shared write state would invalidate paired-sample +independence. + +The rendered markdown reports exact counter ranges, retrieval and extraction quality, and the +candidate/control bootstrap interval together. Only `iteration total` is the pre-registered timing +headline; subspan intervals are exploratory and are not corrected for multiple comparisons. + ### Determinism The harness uses a deterministic embedding function and a scripted model, so counters are reproducible. diff --git a/tests/AgentMemory.Tests.Unit/Cli/PerfAbTests.cs b/tests/AgentMemory.Tests.Unit/Cli/PerfAbTests.cs new file mode 100644 index 00000000..f4252ef5 --- /dev/null +++ b/tests/AgentMemory.Tests.Unit/Cli/PerfAbTests.cs @@ -0,0 +1,161 @@ +using AgentMemory.Abstractions.Options; +using AgentMemory.Cli; +using AgentMemory.Cli.Commands; +using AgentMemory.Cli.Perf; +using FluentAssertions; + +namespace AgentMemory.Tests.Unit.Cli; + +public sealed class PerfAbTests +{ + [Fact] + public void CliArgs_PerfAb_ExposesSubcommand() + { + var args = CliArgs.Parse(["perf", "ab", "--control", "default"]); + + args.Command.Should().Be("perf"); + args.Subcommand.Should().Be("ab"); + args.Get("control").Should().Be("default"); + } + + [Fact] + public void Configuration_Default_UsesShippedRecallOptions() + { + var configuration = PerfConfiguration.Parse("default"); + + configuration.Recall.Should().Be(RecallOptions.Default); + configuration.CanonicalSpec.Should().Be("default"); + } + + [Fact] + public void Configuration_MaxEntitiesOverride_ChangesOnlyMaxEntities() + { + var configuration = PerfConfiguration.Parse("Recall.MaxEntities=2"); + + configuration.Recall.Should().Be(RecallOptions.Default with { MaxEntities = 2 }); + configuration.CanonicalSpec.Should().Be("Recall.MaxEntities=2"); + PerfFixture.ExpectedRecall(configuration.Recall).Total.Should().Be(35); + } + + [Fact] + public void Configuration_UnknownKey_IsRejected() + { + var act = () => PerfConfiguration.Parse("Recall.DoesNotExist=2"); + + act.Should().Throw() + .WithMessage("*DoesNotExist*"); + } + + [Fact] + public void StatefulWriteScenario_IsNotEligibleForSharedDatabaseAb() + { + var scenario = PerfScenarios.All.Single(item => item.Id == "PERF-W-02"); + + scenario.SupportsInterleavedAb.Should().BeFalse(); + } + + [Fact] + public void AbDatasetIdentities_AreEquivalentButDisjoint() + { + var control = PerfFixture.ForVariant("control"); + var candidate = PerfFixture.ForVariant("candidate"); + + control.OwnerId.Should().NotBe(candidate.OwnerId); + control.SessionId.Should().NotBe(candidate.SessionId); + control.ConversationId.Should().NotBe(candidate.ConversationId); + control.TopicToken.Length.Should().Be(candidate.TopicToken.Length); + PerfFixture.ProbeQueryFor(control).Length.Should().Be(PerfFixture.ProbeQueryFor(candidate).Length); + + control.IdPrefix.Should().NotBe(candidate.IdPrefix); + } + + [Fact] + public void DatasetCrossover_BalancesBothConfigurationsAcrossBothCopies() + { + PerfAbCommand.DatasetVariantFor("control", 0).Should().Be("control"); + PerfAbCommand.DatasetVariantFor("candidate", 0).Should().Be("candidate"); + PerfAbCommand.DatasetVariantFor("control", 1).Should().Be("candidate"); + PerfAbCommand.DatasetVariantFor("candidate", 1).Should().Be("control"); + } + + [Fact] + public void PairedBootstrap_IdenticalSamples_ReportNoSignificantDifference() + { + var result = PairedRatioBootstrap.Analyze( + [10, 11, 9, 12, 10], + [10, 11, 9, 12, 10], + resamples: 2_000, + seed: 21); + + result.Estimate.Should().Be(1); + result.Lower95.Should().Be(1); + result.Upper95.Should().Be(1); + result.Verdict.Should().Be(TimingVerdict.NoSignificantDifference); + } + + [Fact] + public void PairedBootstrap_UniformHalving_DeclaresImprovement() + { + var result = PairedRatioBootstrap.Analyze( + [10, 20, 30, 40, 50], + [5, 10, 15, 20, 25], + resamples: 2_000, + seed: 21); + + result.Estimate.Should().BeApproximately(0.5, 1e-12); + result.Upper95.Should().BeLessThan(1); + result.Verdict.Should().Be(TimingVerdict.Improvement); + } + + [Fact] + public void PairedBootstrap_UniformDoubling_DeclaresRegression() + { + var result = PairedRatioBootstrap.Analyze( + [10, 20, 30, 40, 50], + [20, 40, 60, 80, 100], + resamples: 2_000, + seed: 21); + + result.Estimate.Should().BeApproximately(2, 1e-12); + result.Lower95.Should().BeGreaterThan(1); + result.Verdict.Should().Be(TimingVerdict.Regression); + } + + [Fact] + public void CounterbalancedBootstrap_FirstSecondEffect_ReportsNoSignificantDifference() + { + // The second runner is 10% faster: candidate benefits in AB, control benefits in BA. + // Each six-pair cluster has three of each order, so execution position cancels exactly. + var result = PairedRatioBootstrap.AnalyzeCounterbalanced( + [100, 90, 100, 90, 100, 90, 100, 90, 100, 90, 100, 90], + [90, 100, 90, 100, 90, 100, 90, 100, 90, 100, 90, 100], + resamples: 2_000, + seed: 21); + + result.Pairs.Should().Be(2); + result.Estimate.Should().Be(1); + result.Lower95.Should().Be(1); + result.Upper95.Should().Be(1); + result.Verdict.Should().Be(TimingVerdict.NoSignificantDifference); + } + + [Fact] + public void CounterbalancedBootstrap_RequiresCompleteAbBaBlocks() + { + var act = () => PairedRatioBootstrap.AnalyzeCounterbalanced( + [100, 90, 100, 90, 100, 90, 100, 90, 100, 90], + [90, 100, 90, 100, 90, 100, 90, 100, 90, 100]); + + act.Should().Throw() + .WithMessage("*multiple of six*"); + } + + [Fact] + public void PairedBootstrap_RejectsNonPositiveTimings() + { + var act = () => PairedRatioBootstrap.Analyze([10, 0], [9, 1]); + + act.Should().Throw() + .WithMessage("*positive*"); + } +} diff --git a/tools/AgentMemory.Cli/CliArgs.cs b/tools/AgentMemory.Cli/CliArgs.cs index a91fa876..02f7525d 100644 --- a/tools/AgentMemory.Cli/CliArgs.cs +++ b/tools/AgentMemory.Cli/CliArgs.cs @@ -8,11 +8,17 @@ namespace AgentMemory.Cli; public sealed class CliArgs { public string? Command { get; } + public string? Subcommand => Positionals.FirstOrDefault(); + public IReadOnlyList Positionals { get; } public IReadOnlyDictionary Options { get; } - private CliArgs(string? command, IReadOnlyDictionary options) + private CliArgs( + string? command, + IReadOnlyList positionals, + IReadOnlyDictionary options) { Command = command; + Positionals = positionals; Options = options; } @@ -25,6 +31,7 @@ private CliArgs(string? command, IReadOnlyDictionary options) public static CliArgs Parse(string[] args) { string? command = null; + var positionals = new List(); var options = new Dictionary(StringComparer.OrdinalIgnoreCase); for (var i = 0; i < args.Length; i++) @@ -54,11 +61,14 @@ public static CliArgs Parse(string[] args) } else { - command ??= token; + if (command is null) + command = token; + else + positionals.Add(token); } } - return new CliArgs(command, options); + return new CliArgs(command, positionals, options); } } @@ -100,6 +110,13 @@ Neo4j via Testcontainers (Docker required) with deterministic embeddings and a scripted model, so counters are reproducible. Quality gate defaults on. Writes a dated run directory under artifacts/perf by default. + perf ab --control --candidate [--scenarios ] + [--iterations ] [--warmup ] [--latency ] + Run counterbalanced control/candidate pairs in one process/database. + Iterations must be a multiple of 6 and at least 12 (default 30). + Specs use `default` or recall assignments such as + `Recall.MaxEntities=2`. Reports exact counter deltas, paired + timing ratios with bootstrap 95% CIs, and quality side by side. decay [--owner ] Decay-prune memories: soft-invalidate by default (kept + recoverable; set MemoryDecay:NonDestructive=false to hard-delete). Owner-scoped, or global. schema-parity [--upstream-version ] @@ -125,6 +142,8 @@ agentmemory history --type fact --owner user-42 --limit 20 agentmemory evaluate --iterations 3 --output artifacts/evaluation/local.json agentmemory perf --label baseline --iterations 10 agentmemory perf --label feat-01-access-tracking --latency remote + agentmemory perf ab --control default --candidate default --scenarios PERF-R-04 + agentmemory perf ab --control default --candidate Recall.MaxEntities=2 """); } } diff --git a/tools/AgentMemory.Cli/Commands/PerfAbCommand.cs b/tools/AgentMemory.Cli/Commands/PerfAbCommand.cs new file mode 100644 index 00000000..c273a8ec --- /dev/null +++ b/tools/AgentMemory.Cli/Commands/PerfAbCommand.cs @@ -0,0 +1,649 @@ +using System.Diagnostics; +using System.Globalization; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using AgentMemory.Cli.Perf; + +namespace AgentMemory.Cli.Commands; + +/// +/// Runs control and candidate configurations alternately in one process against one hermetic profile. +/// Structural counters remain exact; timings are compared only as paired within-run ratios. +/// +public sealed class PerfAbCommand +{ + private static readonly JsonSerializerOptions Json = new() + { + WriteIndented = true, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + }; + + private readonly TextWriter _output; + + public PerfAbCommand(TextWriter output) => _output = output; + + public async Task ExecuteAsync( + string? controlValue, + string? candidateValue, + string? scenarioFilter, + string? iterationsValue, + string? warmupValue, + string? dimensionsValue, + string? latency, + string? outputRoot, + CancellationToken cancellationToken = default) + { + if (string.IsNullOrWhiteSpace(controlValue) || string.IsNullOrWhiteSpace(candidateValue)) + { + _output.WriteLine("error: perf ab requires --control and --candidate ."); + return 1; + } + + PerfConfiguration control; + PerfConfiguration candidate; + IReadOnlyList scenarios; + try + { + control = PerfConfiguration.Parse(controlValue); + candidate = PerfConfiguration.Parse(candidateValue); + scenarios = PerfScenarios.Select(scenarioFilter ?? "PERF-R-04"); + var unsupported = scenarios.Where(s => !s.SupportsInterleavedAb).Select(s => s.Id).ToList(); + if (unsupported.Count > 0) + { + throw new ArgumentException( + $"interleaved A/B does not support state-mutating scenarios ({string.Join(", ", unsupported)}); " + + "their shared database state violates paired-sample independence."); + } + } + catch (ArgumentException ex) + { + _output.WriteLine($"error: {ex.Message}"); + return 1; + } + + var iterations = ParsePositive(iterationsValue, 30, "iterations"); + if (iterations < 12 || iterations % 6 != 0) + { + _output.WriteLine("error: --iterations must be a multiple of six and at least 12 for block inference."); + return 1; + } + var warmup = ParseNonNegative(warmupValue, 3, "warmup"); + var dimensions = ParsePositive(dimensionsValue, 384, "embedding-dimensions"); + var (embeddingLatency, modelLatency, latencyName) = ResolveLatency(latency); + + var startedAt = DateTimeOffset.UtcNow; + var runId = $"{startedAt:yyyyMMdd'T'HHmmss'Z'}__ab__hermetic-S-{latencyName}"; + var runDir = Path.Combine(outputRoot ?? Path.Combine("artifacts", "perf"), runId); + Directory.CreateDirectory(runDir); + + var manifest = new + { + runId, + mode = "ab", + startedAtUtc = startedAt, + control = control.CanonicalSpec, + candidate = candidate.CanonicalSpec, + scenarios = scenarios.Select(s => s.Id).ToArray(), + iterations, + warmup, + environment = new + { + commit = TryGetCommit(), + embeddingDimensions = dimensions, + embeddingLatencyMs = embeddingLatency.TotalMilliseconds, + modelLatencyMs = modelLatency.TotalMilliseconds, + neo4jImage = "neo4j:5.26", + os = Environment.OSVersion.ToString(), + processorCount = Environment.ProcessorCount, + runtime = Environment.Version.ToString(), + serverGc = System.Runtime.GCSettings.IsServerGC, + machineName = Environment.MachineName, + }, + }; + + _output.WriteLine($"perf ab: run {runId}"); + _output.WriteLine($"perf ab: control {control.CanonicalSpec}"); + _output.WriteLine($"perf ab: candidate {candidate.CanonicalSpec}"); + + await File.WriteAllTextAsync( + Path.Combine(runDir, "run.json"), + JsonSerializer.Serialize(manifest, Json), + cancellationToken).ConfigureAwait(false); + + using var trace = new TraceLogWriter(Path.Combine(runDir, "trace.ndjson")); + trace.RunStart(runId, manifest); + var runStopwatch = Stopwatch.StartNew(); + using var collector = new PerfCollector(trace); + + var extractionFixture = ExtractionQualityFixture.Load(); + var scriptedRules = extractionFixture.ScriptedRules().Concat(PerfScenarios.ScriptedRules).ToList(); + + await using var profile = await HermeticProfile + .StartAsync( + dimensions, + embeddingLatency, + modelLatency, + _output, + scriptedRules, + cancellationToken) + .ConfigureAwait(false); + + await PerfFixture.SeedAsync( + profile, _output, cancellationToken, PerfFixture.ForVariant("control")).ConfigureAwait(false); + await PerfFixture.SeedAsync( + profile, _output, cancellationToken, PerfFixture.ForVariant("candidate")).ConfigureAwait(false); + + var qualityFixture = QualityFixture.Load(); + var qualitySeeder = new QualityEvaluator(qualityFixture, profile.Services, dimensions); + await qualitySeeder.SeedAsync(_output, cancellationToken).ConfigureAwait(false); + + var controlProvider = PerfScenarios.CreateProvider(profile, control.Recall); + var candidateProvider = PerfScenarios.CreateProvider(profile, candidate.Recall); + var samples = new List(); + + try + { + foreach (var scenario in scenarios) + { + _output.WriteLine($"perf ab: {scenario.Id} — counterbalanced AB/BA"); + for (var i = 0; i < warmup; i++) + { + await RunPairAsync( + scenario, control, candidate, controlProvider, candidateProvider, + profile, collector, i, "warmup", samples, cancellationToken).ConfigureAwait(false); + } + + for (var i = 0; i < iterations; i++) + { + await RunPairAsync( + scenario, control, candidate, controlProvider, candidateProvider, + profile, collector, i, "measure", samples, cancellationToken).ConfigureAwait(false); + } + } + } + catch (Exception ex) + { + runStopwatch.Stop(); + trace.RunEnd(collector.Records.Count, runStopwatch.Elapsed.TotalMilliseconds); + _output.WriteLine($"error: A/B scenario failed: {ex.Message}"); + return 1; + } + + _output.WriteLine("perf ab: scoring control and candidate retrieval quality…"); + var controlQuality = await new QualityEvaluator( + qualityFixture, profile.Services, dimensions, control.Recall) + .EvaluateAsync(cancellationToken).ConfigureAwait(false); + var candidateQuality = await new QualityEvaluator( + qualityFixture, profile.Services, dimensions, candidate.Recall) + .EvaluateAsync(cancellationToken).ConfigureAwait(false); + + _output.WriteLine("perf ab: scoring shared extraction quality…"); + var extractionQuality = await new ExtractionQualityEvaluator( + extractionFixture, profile.Services, profile.Driver) + .EvaluateAsync(cancellationToken).ConfigureAwait(false); + + var baseline = QualityGate.LoadBaseline(); + var controlGate = QualityGate.Evaluate( + baseline, controlQuality, extractionQuality, QualityGate.DefaultBaselinePath); + var candidateGate = QualityGate.Evaluate( + baseline, candidateQuality, extractionQuality, QualityGate.DefaultBaselinePath); + + runStopwatch.Stop(); + trace.RunEnd(collector.Records.Count, runStopwatch.Elapsed.TotalMilliseconds); + + var measured = samples.Where(s => s.Record.Phase == "measure").ToList(); + var comparisons = BuildComparisons(measured, scenarios); + var nullViolations = ValidateNullExperiment(control, candidate, comparisons); + + await WriteSamplesAsync(runDir, measured, cancellationToken).ConfigureAwait(false); + var summary = new + { + manifest, + quality = new + { + control = controlQuality, + candidate = candidateQuality, + extractionShared = extractionQuality, + controlGate, + candidateGate, + }, + scenarios = comparisons, + nullExperiment = new + { + applicable = control.Recall == candidate.Recall, + passed = nullViolations.Count == 0, + violations = nullViolations, + }, + }; + await File.WriteAllTextAsync( + Path.Combine(runDir, "summary.json"), + JsonSerializer.Serialize(summary, Json), + cancellationToken).ConfigureAwait(false); + + var report = RenderReport( + runId, + control, + candidate, + comparisons, + controlQuality, + candidateQuality, + extractionQuality, + controlGate, + candidateGate, + nullViolations); + await File.WriteAllTextAsync( + Path.Combine(runDir, "report.md"), report, cancellationToken).ConfigureAwait(false); + + _output.WriteLine(); + _output.Write(report); + _output.WriteLine($"perf ab: wrote {runDir}"); + + if (nullViolations.Count == 0) + return 0; + + foreach (var violation in nullViolations) + _output.WriteLine($"error: {violation}"); + return 1; + } + + private static async Task RunPairAsync( + PerfScenario scenario, + PerfConfiguration control, + PerfConfiguration candidate, + AgentMemory.AgentFramework.Neo4jMemoryContextProvider controlProvider, + AgentMemory.AgentFramework.Neo4jMemoryContextProvider candidateProvider, + HermeticProfile profile, + PerfCollector collector, + int iteration, + string phase, + ICollection samples, + CancellationToken cancellationToken) + { + if (iteration % 2 != 0) + { + await RunSideAsync( + scenario, "candidate", candidate, candidateProvider, profile, collector, + iteration, phase, samples, cancellationToken).ConfigureAwait(false); + await RunSideAsync( + scenario, "control", control, controlProvider, profile, collector, + iteration, phase, samples, cancellationToken).ConfigureAwait(false); + return; + } + + await RunSideAsync( + scenario, "control", control, controlProvider, profile, collector, + iteration, phase, samples, cancellationToken).ConfigureAwait(false); + await RunSideAsync( + scenario, "candidate", candidate, candidateProvider, profile, collector, + iteration, phase, samples, cancellationToken).ConfigureAwait(false); + } + + private static async Task RunSideAsync( + PerfScenario scenario, + string variant, + PerfConfiguration configuration, + AgentMemory.AgentFramework.Neo4jMemoryContextProvider provider, + HermeticProfile profile, + PerfCollector collector, + int iteration, + string phase, + ICollection samples, + CancellationToken cancellationToken) + { + using var turn = collector.BeginTurn($"{scenario.Id}/{variant}", iteration, phase); + await scenario.RunAsync(new ScenarioContext( + profile, + provider, + turn.Record, + iteration, + phase, + DatasetVariantFor(variant, iteration), + configuration.Recall, + cancellationToken)).ConfigureAwait(false); + samples.Add(new AbSample(scenario.Id, variant, turn.Record)); + } + + internal static string DatasetVariantFor(string configurationVariant, int iteration) => + iteration % 2 == 0 + ? configurationVariant + : configurationVariant switch + { + "control" => "candidate", + "candidate" => "control", + _ => throw new ArgumentException($"unknown configuration variant '{configurationVariant}'."), + }; + + private static IReadOnlyList BuildComparisons( + IReadOnlyList samples, + IReadOnlyList scenarios) + { + var results = new List(); + foreach (var scenario in scenarios) + { + var control = samples + .Where(s => s.Scenario == scenario.Id && s.Variant == "control") + .OrderBy(s => s.Record.Iteration) + .Select(s => s.Record) + .ToList(); + var candidate = samples + .Where(s => s.Scenario == scenario.Id && s.Variant == "candidate") + .OrderBy(s => s.Record.Iteration) + .Select(s => s.Record) + .ToList(); + + var counterNames = control + .SelectMany(r => r.Counters.Keys) + .Concat(candidate.SelectMany(r => r.Counters.Keys)) + .Distinct(StringComparer.Ordinal) + .OrderBy(name => name, StringComparer.Ordinal); + var counters = counterNames.Select(name => new CounterComparison( + name, + Values(control, name), + Values(candidate, name))).ToList(); + + var timings = new List + { + Timing("iteration total", control.Select(r => r.DurationMs), candidate.Select(r => r.DurationMs)), + }; + + var spanNames = control + .SelectMany(r => r.SpanMilliseconds.Keys) + .Concat(candidate.SelectMany(r => r.SpanMilliseconds.Keys)) + .Distinct(StringComparer.Ordinal) + .OrderBy(name => name, StringComparer.Ordinal); + foreach (var spanName in spanNames) + { + var controlValues = control.Select(r => r.SpanMilliseconds.GetValueOrDefault(spanName)).ToList(); + var candidateValues = candidate.Select(r => r.SpanMilliseconds.GetValueOrDefault(spanName)).ToList(); + if (controlValues.All(v => v > 0) && candidateValues.All(v => v > 0)) + timings.Add(Timing(spanName, controlValues, candidateValues)); + } + + results.Add(new ScenarioComparison(scenario.Id, control.Count, counters, timings)); + } + + return results; + } + + private static TimingComparison Timing( + string metric, + IEnumerable controlValues, + IEnumerable candidateValues) + { + var control = controlValues.ToList(); + var candidate = candidateValues.ToList(); + return new TimingComparison( + metric, + Median(control), + Median(candidate), + PairedRatioBootstrap.AnalyzeCounterbalanced(control, candidate)); + } + + private static ExactValues Values(IReadOnlyList records, string name) + { + var values = records.Select(r => r.Counter(name)).ToList(); + return new ExactValues(values.Min(), values.Max()); + } + + private static IReadOnlyList ValidateNullExperiment( + PerfConfiguration control, + PerfConfiguration candidate, + IReadOnlyList comparisons) + { + if (control.Recall != candidate.Recall) + return []; + + var violations = new List(); + foreach (var scenario in comparisons) + { + foreach (var counter in scenario.Counters) + { + if (counter.Control != counter.Candidate) + { + violations.Add( + $"{scenario.Scenario} identical configurations changed counter {counter.Name}: " + + $"{counter.Control.Display} vs {counter.Candidate.Display}."); + } + } + + var total = scenario.Timings.Single(t => t.Metric == "iteration total"); + if (total.Ratio.Verdict != TimingVerdict.NoSignificantDifference) + { + violations.Add( + $"{scenario.Scenario} identical configurations reported " + + $"{VerdictText(total.Ratio.Verdict)} for iteration total; " + + $"95% CI {total.Ratio.Lower95:F3}–{total.Ratio.Upper95:F3} must include 1.0."); + } + } + + return violations; + } + + private static async Task WriteSamplesAsync( + string runDir, + IReadOnlyList samples, + CancellationToken cancellationToken) + { + var lines = samples.Select(sample => JsonSerializer.Serialize(new + { + scenario = sample.Scenario, + variant = sample.Variant, + iteration = sample.Record.Iteration, + durMs = sample.Record.DurationMs, + counters = sample.Record.Counters, + spansMs = sample.Record.SpanMilliseconds, + })); + await File.WriteAllLinesAsync( + Path.Combine(runDir, "samples.ndjson"), lines, cancellationToken).ConfigureAwait(false); + } + + private static string RenderReport( + string runId, + PerfConfiguration control, + PerfConfiguration candidate, + IReadOnlyList comparisons, + QualityResult controlQuality, + QualityResult candidateQuality, + ExtractionQualityResult extraction, + QualityGateResult controlGate, + QualityGateResult candidateGate, + IReadOnlyList nullViolations) + { + var sb = new StringBuilder(); + sb.AppendLine(CultureInfo.InvariantCulture, $"## Interleaved A/B — `{runId}`"); + sb.AppendLine(); + sb.AppendLine(CultureInfo.InvariantCulture, $"- Control: `{control.CanonicalSpec}`"); + sb.AppendLine(CultureInfo.InvariantCulture, $"- Candidate: `{candidate.CanonicalSpec}`"); + sb.AppendLine("- Schedule: counterbalanced adjacent pairs `AB, BA, AB, BA…` in one process/database"); + sb.AppendLine( + "- Timing inference: six-pair (three `AB/BA` cycle) blocks are bootstrapped as independent units"); + sb.AppendLine("- Dataset crossover: each configuration runs equally against both disjoint fixture copies"); + sb.AppendLine(); + + if (control.Recall == candidate.Recall) + { + sb.AppendLine(nullViolations.Count == 0 + ? "**Null experiment: PASS — no significant difference.**" + : "**Null experiment: FAIL.**"); + if (nullViolations.Count > 0) + { + foreach (var violation in nullViolations) + sb.AppendLine($"- {violation}"); + } + sb.AppendLine(); + } + + sb.AppendLine("### Quality"); + sb.AppendLine(); + sb.AppendLine("| Metric | Control | Candidate | Delta |"); + sb.AppendLine("|---|---:|---:|---:|"); + QualityRow(sb, "Retrieval Recall@K", controlQuality.RecallAtK, candidateQuality.RecallAtK); + QualityRow(sb, "Retrieval MRR", controlQuality.Mrr, candidateQuality.Mrr); + sb.AppendLine(CultureInfo.InvariantCulture, + $"| Forbidden-retrieval cases | {controlQuality.CasesWithViolations} " + + $"| {candidateQuality.CasesWithViolations} " + + $"| {candidateQuality.CasesWithViolations - controlQuality.CasesWithViolations:+#;-#;0} |"); + QualityRow(sb, "Extraction entity precision (shared)", extraction.EntityPrecision, extraction.EntityPrecision); + QualityRow(sb, "Extraction entity recall (shared)", extraction.EntityRecall, extraction.EntityRecall); + QualityRow(sb, "Extraction fact precision (shared)", extraction.FactPrecision, extraction.FactPrecision); + QualityRow(sb, "Extraction fact recall (shared)", extraction.FactRecall, extraction.FactRecall); + QualityRow(sb, "Extraction preference precision (shared)", extraction.PreferencePrecision, extraction.PreferencePrecision); + QualityRow(sb, "Extraction preference recall (shared)", extraction.PreferenceRecall, extraction.PreferenceRecall); + sb.AppendLine(CultureInfo.InvariantCulture, + $"| Extraction false-positive rate (shared) | {extraction.FalsePositiveRate:F3} " + + $"| {extraction.FalsePositiveRate:F3} | 0.000 |"); + sb.AppendLine(); + sb.AppendLine(CultureInfo.InvariantCulture, + $"Quality baseline: control **{(controlGate.Passed ? "PASS" : "FAIL")}**, " + + $"candidate **{(candidateGate.Passed ? "PASS" : "FAIL")}**."); + if (!candidateGate.Passed) + { + foreach (var violation in candidateGate.Violations) + sb.AppendLine($"- Candidate: {violation}"); + } + sb.AppendLine(); + + foreach (var scenario in comparisons) + { + sb.AppendLine(CultureInfo.InvariantCulture, + $"### {scenario.Scenario} ({scenario.Pairs} paired iterations)"); + sb.AppendLine(); + sb.AppendLine("#### Structural counters"); + sb.AppendLine(); + sb.AppendLine("| Counter | Control | Candidate | Delta | Deterministic |"); + sb.AppendLine("|---|---:|---:|---:|---|"); + foreach (var counter in scenario.Counters) + { + var delta = counter.Control.Deterministic && counter.Candidate.Deterministic + ? (counter.Candidate.Min - counter.Control.Min) + .ToString("+#;-#;0", CultureInfo.InvariantCulture) + : "–"; + sb.AppendLine(CultureInfo.InvariantCulture, + $"| `{counter.Name}` | {counter.Control.Display} | {counter.Candidate.Display} " + + $"| {delta} | {(counter.Control.Deterministic && counter.Candidate.Deterministic ? "yes" : "**NO**")} |"); + } + sb.AppendLine(); + + sb.AppendLine("#### Paired timings — candidate/control"); + sb.AppendLine(); + sb.AppendLine("| Metric | Control p50 ms | Candidate p50 ms | Ratio | Bootstrap 95% CI | Verdict |"); + sb.AppendLine("|---|---:|---:|---:|---:|---|"); + foreach (var timing in scenario.Timings) + { + var verdict = timing.Metric == "iteration total" + ? VerdictText(timing.Ratio.Verdict) + : "exploratory"; + sb.AppendLine(CultureInfo.InvariantCulture, + $"| `{timing.Metric}` | {timing.ControlP50Ms:F2} | {timing.CandidateP50Ms:F2} " + + $"| {timing.Ratio.Estimate:F3} | {timing.Ratio.Lower95:F3}–{timing.Ratio.Upper95:F3} " + + $"| {verdict} |"); + } + sb.AppendLine(); + } + + sb.AppendLine( + "Iteration total is the pre-registered headline: a win is declared only when its confidence " + + "interval’s upper bound is below 1.0. Subspan intervals are exploratory and are not corrected " + + "for multiple comparisons. " + + "These timings are meaningful only within this interleaved run; structural counters are portable."); + return sb.ToString(); + } + + private static void QualityRow(StringBuilder sb, string label, double control, double candidate) => + sb.AppendLine(CultureInfo.InvariantCulture, + $"| {label} | {control:F3} | {candidate:F3} | {candidate - control:+0.000;-0.000;0.000} |"); + + private static string VerdictText(TimingVerdict verdict) => verdict switch + { + TimingVerdict.Improvement => "**improvement**", + TimingVerdict.Regression => "**regression**", + _ => "no significant difference", + }; + + private static double Median(List values) + { + values.Sort(); + var middle = values.Count / 2; + return values.Count % 2 == 0 + ? (values[middle - 1] + values[middle]) / 2 + : values[middle]; + } + + private static int ParsePositive(string? value, int fallback, string name) + { + if (string.IsNullOrWhiteSpace(value)) return fallback; + if (!int.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed) || + parsed <= 0) + { + throw new ArgumentException($"--{name} must be a positive integer."); + } + + return parsed; + } + + private static int ParseNonNegative(string? value, int fallback, string name) + { + if (string.IsNullOrWhiteSpace(value)) return fallback; + if (!int.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed) || + parsed < 0) + { + throw new ArgumentException($"--{name} must be zero or a positive integer."); + } + + return parsed; + } + + private static (TimeSpan Embedding, TimeSpan Model, string Name) ResolveLatency(string? latency) => + latency?.ToLowerInvariant() switch + { + null or "zero" => (TimeSpan.Zero, TimeSpan.Zero, "zero"), + "remote" => (TimeSpan.FromMilliseconds(120), TimeSpan.FromMilliseconds(900), "remote"), + _ => throw new ArgumentException($"unknown --latency '{latency}'. Use 'zero' or 'remote'."), + }; + + private static string? TryGetCommit() + { + try + { + using var process = Process.Start(new ProcessStartInfo("git", "rev-parse HEAD") + { + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + }); + if (process is null) return null; + var stdout = process.StandardOutput.ReadToEnd(); + process.WaitForExit(5_000); + return process.ExitCode == 0 ? stdout.Trim() : null; + } + catch + { + return null; + } + } + + private sealed record AbSample(string Scenario, string Variant, TurnRecord Record); + + private sealed record ExactValues(long Min, long Max) + { + public bool Deterministic => Min == Max; + public string Display => Deterministic + ? Min.ToString(CultureInfo.InvariantCulture) + : $"{Min.ToString(CultureInfo.InvariantCulture)}–{Max.ToString(CultureInfo.InvariantCulture)}"; + } + + private sealed record CounterComparison(string Name, ExactValues Control, ExactValues Candidate); + + private sealed record TimingComparison( + string Metric, + double ControlP50Ms, + double CandidateP50Ms, + PairedRatioResult Ratio); + + private sealed record ScenarioComparison( + string Scenario, + int Pairs, + IReadOnlyList Counters, + IReadOnlyList Timings); +} diff --git a/tools/AgentMemory.Cli/Commands/PerfCommand.cs b/tools/AgentMemory.Cli/Commands/PerfCommand.cs index a5fb5dbc..0647bda6 100644 --- a/tools/AgentMemory.Cli/Commands/PerfCommand.cs +++ b/tools/AgentMemory.Cli/Commands/PerfCommand.cs @@ -177,14 +177,16 @@ private static async Task RunScenarioAsync( { using var turn = collector.BeginTurn(scenario.Id, i, "warmup"); await scenario.RunAsync(new ScenarioContext( - profile, provider, turn.Record, i, "warmup", cancellationToken)).ConfigureAwait(false); + profile, provider, turn.Record, i, "warmup", null, + AgentMemory.Abstractions.Options.RecallOptions.Default, cancellationToken)).ConfigureAwait(false); } for (var i = 0; i < iterations; i++) { using var turn = collector.BeginTurn(scenario.Id, i, "measure"); await scenario.RunAsync(new ScenarioContext( - profile, provider, turn.Record, i, "measure", cancellationToken)).ConfigureAwait(false); + profile, provider, turn.Record, i, "measure", null, + AgentMemory.Abstractions.Options.RecallOptions.Default, cancellationToken)).ConfigureAwait(false); } } diff --git a/tools/AgentMemory.Cli/Perf/PairedRatioBootstrap.cs b/tools/AgentMemory.Cli/Perf/PairedRatioBootstrap.cs new file mode 100644 index 00000000..6413ed9f --- /dev/null +++ b/tools/AgentMemory.Cli/Perf/PairedRatioBootstrap.cs @@ -0,0 +1,113 @@ +namespace AgentMemory.Cli.Perf; + +internal enum TimingVerdict +{ + Improvement, + NoSignificantDifference, + Regression, +} + +internal sealed record PairedRatioResult( + double Estimate, + double Lower95, + double Upper95, + TimingVerdict Verdict, + int Pairs); + +/// +/// Deterministic percentile bootstrap over paired candidate/control ratios. The point estimate is the +/// geometric mean, which treats a 2x slowdown and a 2x speedup symmetrically in log space. +/// +internal static class PairedRatioBootstrap +{ + /// + /// Clusters three consecutive AB/BA cycles as one bootstrap unit. Each six-pair block balances + /// execution position while preserving the longer-lived Docker/driver timing correlation that made + /// a two-pair bootstrap produce confidently wrong null results. + /// + public static PairedRatioResult AnalyzeCounterbalanced( + IReadOnlyList control, + IReadOnlyList candidate, + int resamples = 10_000, + int seed = 21) + { + if (control.Count != candidate.Count || control.Count == 0) + throw new ArgumentException("control and candidate must contain the same non-zero number of pairs."); + if (control.Count < 12 || control.Count % 6 != 0) + { + throw new ArgumentException( + "counterbalanced timings require a multiple of six and at least twelve AB/BA pairs."); + } + if (control.Any(v => !double.IsFinite(v) || v <= 0) || + candidate.Any(v => !double.IsFinite(v) || v <= 0)) + { + throw new ArgumentException("paired timings must be finite and positive."); + } + + const int pairsPerBlock = 6; + var blockControl = Enumerable.Repeat(1d, control.Count / pairsPerBlock).ToArray(); + var blockCandidate = new double[blockControl.Length]; + for (var block = 0; block < blockCandidate.Length; block++) + { + var start = block * pairsPerBlock; + var meanLogRatio = Enumerable.Range(start, pairsPerBlock) + .Average(i => Math.Log(candidate[i] / control[i])); + blockCandidate[block] = Math.Exp(meanLogRatio); + } + + return Analyze(blockControl, blockCandidate, resamples, seed); + } + + public static PairedRatioResult Analyze( + IReadOnlyList control, + IReadOnlyList candidate, + int resamples = 10_000, + int seed = 21) + { + if (control.Count != candidate.Count || control.Count == 0) + throw new ArgumentException("control and candidate must contain the same non-zero number of pairs."); + if (resamples <= 0) + throw new ArgumentOutOfRangeException(nameof(resamples), "resamples must be positive."); + if (control.Any(v => !double.IsFinite(v) || v <= 0) || + candidate.Any(v => !double.IsFinite(v) || v <= 0)) + { + throw new ArgumentException("paired timings must be finite and positive."); + } + + var logRatios = control + .Zip(candidate, (a, b) => Math.Log(b / a)) + .ToArray(); + var estimate = Math.Exp(logRatios.Average()); + + var random = new Random(seed); + var bootstrap = new double[resamples]; + for (var sample = 0; sample < resamples; sample++) + { + var sum = 0d; + for (var pair = 0; pair < logRatios.Length; pair++) + sum += logRatios[random.Next(logRatios.Length)]; + bootstrap[sample] = Math.Exp(sum / logRatios.Length); + } + + Array.Sort(bootstrap); + var lower = Quantile(bootstrap, 0.025); + var upper = Quantile(bootstrap, 0.975); + var verdict = upper < 1 + ? TimingVerdict.Improvement + : lower > 1 + ? TimingVerdict.Regression + : TimingVerdict.NoSignificantDifference; + + return new PairedRatioResult(estimate, lower, upper, verdict, control.Count); + } + + private static double Quantile(IReadOnlyList sorted, double probability) + { + if (sorted.Count == 1) return sorted[0]; + var position = (sorted.Count - 1) * probability; + var lower = (int)Math.Floor(position); + var upper = (int)Math.Ceiling(position); + if (lower == upper) return sorted[lower]; + return sorted[lower] + ((sorted[upper] - sorted[lower]) * (position - lower)); + } +} diff --git a/tools/AgentMemory.Cli/Perf/PerfConfiguration.cs b/tools/AgentMemory.Cli/Perf/PerfConfiguration.cs new file mode 100644 index 00000000..20e6d087 --- /dev/null +++ b/tools/AgentMemory.Cli/Perf/PerfConfiguration.cs @@ -0,0 +1,120 @@ +using System.Globalization; +using AgentMemory.Abstractions.Options; + +namespace AgentMemory.Cli.Perf; + +/// +/// One side of a hermetic A/B comparison. Specs intentionally cover options that can be varied inside +/// one process; code-only changes still require two builds plus a ledger diff. +/// +internal sealed record PerfConfiguration(string CanonicalSpec, RecallOptions Recall) +{ + public static PerfConfiguration Parse(string? value) + { + if (string.IsNullOrWhiteSpace(value) || + value.Equals("default", StringComparison.OrdinalIgnoreCase)) + { + return new PerfConfiguration("default", RecallOptions.Default); + } + + var recall = RecallOptions.Default; + var canonical = new List(); + var assignments = value.Split( + [',', ';'], + StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + + if (assignments.Length == 0) + throw new ArgumentException("configuration spec is empty. Use 'default' or key=value."); + + foreach (var assignment in assignments) + { + var equals = assignment.IndexOf('='); + if (equals <= 0 || equals == assignment.Length - 1) + throw new ArgumentException( + $"invalid configuration assignment '{assignment}'. Use key=value."); + + var suppliedKey = assignment[..equals].Trim(); + var normalizedKey = Normalize(suppliedKey); + var suppliedValue = assignment[(equals + 1)..].Trim(); + + switch (normalizedKey) + { + case "recallmaxrecentmessages": + var maxRecent = ParseNonNegativeInt(suppliedValue, suppliedKey); + recall = recall with { MaxRecentMessages = maxRecent }; + canonical.Add($"Recall.MaxRecentMessages={maxRecent}"); + break; + case "recallmaxrelevantmessages": + var maxRelevant = ParseNonNegativeInt(suppliedValue, suppliedKey); + recall = recall with { MaxRelevantMessages = maxRelevant }; + canonical.Add($"Recall.MaxRelevantMessages={maxRelevant}"); + break; + case "recallmaxentities": + var maxEntities = ParseNonNegativeInt(suppliedValue, suppliedKey); + recall = recall with { MaxEntities = maxEntities }; + canonical.Add($"Recall.MaxEntities={maxEntities}"); + break; + case "recallmaxfacts": + var maxFacts = ParseNonNegativeInt(suppliedValue, suppliedKey); + recall = recall with { MaxFacts = maxFacts }; + canonical.Add($"Recall.MaxFacts={maxFacts}"); + break; + case "recallmaxpreferences": + var maxPreferences = ParseNonNegativeInt(suppliedValue, suppliedKey); + recall = recall with { MaxPreferences = maxPreferences }; + canonical.Add($"Recall.MaxPreferences={maxPreferences}"); + break; + case "recallmaxtraces": + var maxTraces = ParseNonNegativeInt(suppliedValue, suppliedKey); + recall = recall with { MaxTraces = maxTraces }; + canonical.Add($"Recall.MaxTraces={maxTraces}"); + break; + case "recallmaxgraphragitems": + var maxGraphRag = ParseNonNegativeInt(suppliedValue, suppliedKey); + recall = recall with { MaxGraphRagItems = maxGraphRag }; + canonical.Add($"Recall.MaxGraphRagItems={maxGraphRag}"); + break; + case "recallminsimilarityscore": + var minimum = ParseUnitDouble(suppliedValue, suppliedKey); + recall = recall with { MinSimilarityScore = minimum }; + canonical.Add( + $"Recall.MinSimilarityScore={minimum.ToString("G17", CultureInfo.InvariantCulture)}"); + break; + default: + throw new ArgumentException( + $"unknown configuration key '{suppliedKey}'. Supported keys are Recall.MaxRecentMessages, " + + "Recall.MaxRelevantMessages, Recall.MaxEntities, Recall.MaxFacts, " + + "Recall.MaxPreferences, Recall.MaxTraces, Recall.MaxGraphRagItems, and " + + "Recall.MinSimilarityScore."); + } + } + + return new PerfConfiguration(string.Join(';', canonical), recall); + } + + private static string Normalize(string key) => + new(key.Where(char.IsLetterOrDigit).Select(char.ToLowerInvariant).ToArray()); + + private static int ParseNonNegativeInt(string value, string key) + { + if (!int.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed) || + parsed < 0) + { + throw new ArgumentException($"{key} must be a non-negative integer."); + } + + return parsed; + } + + private static double ParseUnitDouble(string value, string key) + { + if (!double.TryParse(value, NumberStyles.Float, CultureInfo.InvariantCulture, out var parsed) || + !double.IsFinite(parsed) || + parsed is < 0 or > 1) + { + throw new ArgumentException($"{key} must be a number between 0 and 1."); + } + + return parsed; + } +} diff --git a/tools/AgentMemory.Cli/Perf/PerfFixture.cs b/tools/AgentMemory.Cli/Perf/PerfFixture.cs index 561a0259..3d1400fd 100644 --- a/tools/AgentMemory.Cli/Perf/PerfFixture.cs +++ b/tools/AgentMemory.Cli/Perf/PerfFixture.cs @@ -38,11 +38,38 @@ public static class PerfFixture /// Conversation used by the recall scenarios. public const string ConversationId = "perf-session-conv"; + public sealed record DatasetIdentity( + string OwnerId, + string SessionId, + string ConversationId, + string IdPrefix, + string TopicToken); + + public static DatasetIdentity DefaultIdentity { get; } = + new(OwnerId, SessionId, ConversationId, "perf", string.Empty); + + public static DatasetIdentity ForVariant(string variant) => + new( + $"{OwnerId}-{variant}", + $"{SessionId}-{variant}", + $"{ConversationId}-{variant}", + $"perf-{variant}", + variant switch + { + "control" => "alpha", + "candidate" => "bravo", + _ => throw new ArgumentException($"unknown A/B fixture variant '{variant}'.", nameof(variant)), + }); + + public static string ProbeQueryFor(DatasetIdentity identity) => + Qualify(ProbeQuery, identity); + // Sized above the shipped RecallOptions defaults (10 entities / 10 facts / 5 preferences / // 10 recent / 5 relevant / 3 traces) so the limits, not the fixture, decide what comes back. private const int EntityCount = 20; private const int FactCount = 20; private const int MessageCount = 30; + private const int PreferenceCount = 12; private const int TraceCount = 8; /// @@ -64,8 +91,13 @@ public static class PerfFixture /// Total items a default recall should return once seeded. public static readonly int ExpectedRecalledItems = ExpectedByCategory.Values.Sum(); - public static async Task SeedAsync(HermeticProfile profile, TextWriter log, CancellationToken cancellationToken) + public static async Task SeedAsync( + HermeticProfile profile, + TextWriter log, + CancellationToken cancellationToken, + DatasetIdentity? identity = null) { + identity ??= DefaultIdentity; var services = profile.Services; var shortTerm = services.GetRequiredService(); var longTerm = services.GetRequiredService(); @@ -74,20 +106,21 @@ public static async Task SeedAsync(HermeticProfile profile, TextWriter log, Canc log.WriteLine("perf: seeding scale-S fixture…"); - await shortTerm.AddConversationAsync(ConversationId, SessionId, OwnerId, null, cancellationToken) + await shortTerm.AddConversationAsync( + identity.ConversationId, identity.SessionId, identity.OwnerId, null, cancellationToken) .ConfigureAwait(false); var now = clock.UtcNow; for (var i = 0; i < MessageCount; i++) { - var text = $"Alice Martin at Acme Corporation discussed platform work item {i} " + - "and her preference for concise written communication."; + var text = Qualify($"Alice Martin at Acme Corporation discussed platform work item {i} " + + "and her preference for concise written communication.", identity); await shortTerm.AddMessageAsync(new Message { - MessageId = $"perf-msg-{i}", - SessionId = SessionId, - ConversationId = ConversationId, + MessageId = $"{identity.IdPrefix}-msg-{i}", + SessionId = identity.SessionId, + ConversationId = identity.ConversationId, Role = i % 2 == 0 ? "user" : "assistant", Content = text, TimestampUtc = now.AddSeconds(i), @@ -99,14 +132,15 @@ await shortTerm.AddMessageAsync(new Message var name = $"Acme Corporation platform team {i}"; await longTerm.AddEntityAsync(new Entity { - EntityId = $"perf-entity-{i}", + EntityId = $"{identity.IdPrefix}-entity-{i}", Name = name, Type = "ORGANIZATION", Description = $"Alice Martin communication work at Acme Corporation, area {i}.", Confidence = 0.95, - OwnerId = OwnerId, + OwnerId = identity.OwnerId, CreatedAtUtc = now, - Embedding = Embed($"{name} Alice Martin work communication preferences", profile.Dimensions), + Embedding = Embed( + Qualify($"{name} Alice Martin work communication preferences", identity), profile.Dimensions), }, cancellationToken).ConfigureAwait(false); } @@ -114,15 +148,16 @@ await longTerm.AddEntityAsync(new Entity { await longTerm.AddFactAsync(new Fact { - FactId = $"perf-fact-{i}", + FactId = $"{identity.IdPrefix}-fact-{i}", Subject = "Alice Martin", Predicate = i % 2 == 0 ? "works_at" : "prefers", Object = i % 2 == 0 ? $"Acme Corporation platform team {i}" : $"concise communication style {i}", Confidence = 0.9, - OwnerId = OwnerId, + OwnerId = identity.OwnerId, CreatedAtUtc = now, Embedding = Embed( - $"Alice Martin work Acme Corporation communication preferences {i}", profile.Dimensions), + Qualify($"Alice Martin work Acme Corporation communication preferences {i}", identity), + profile.Dimensions), }, cancellationToken).ConfigureAwait(false); } @@ -153,29 +188,48 @@ await longTerm.AddFactAsync(new Fact { await longTerm.AddPreferenceAsync(new Preference { - PreferenceId = $"perf-pref-{i}", + PreferenceId = $"{identity.IdPrefix}-pref-{i}", Category = "communication", PreferenceText = text, Confidence = 0.9, - OwnerId = OwnerId, + OwnerId = identity.OwnerId, CreatedAtUtc = now, - Embedding = Embed(text, profile.Dimensions), + Embedding = Embed(Qualify(text, identity), profile.Dimensions), }, cancellationToken).ConfigureAwait(false); } for (var i = 0; i < TraceCount; i++) { - var task = $"Summarize Alice Martin communication preferences for Acme Corporation work {i}"; + var task = Qualify( + $"Summarize Alice Martin communication preferences for Acme Corporation work {i}", identity); var trace = await reasoning.StartTraceAsync( - SessionId, task, Embed(task, profile.Dimensions), null, OwnerId, cancellationToken) + identity.SessionId, task, Embed(task, profile.Dimensions), null, identity.OwnerId, cancellationToken) .ConfigureAwait(false); await reasoning.CompleteTraceAsync(trace.TraceId, "done", true, cancellationToken) .ConfigureAwait(false); } log.WriteLine( - $"perf: seeded {EntityCount} entities, {FactCount} facts, {preferenceTexts.Length} preferences, " + - $"{MessageCount} messages, {TraceCount} traces."); + $"perf: seeded {identity.IdPrefix}: {EntityCount} entities, {FactCount} facts, " + + $"{preferenceTexts.Length} preferences, {MessageCount} messages, {TraceCount} traces."); + } + + public sealed record ExpectedRecallShape(IReadOnlyDictionary ByCategory, int Total); + + /// Expected shape for a configured recall, capped by what scale S seeds. + public static ExpectedRecallShape ExpectedRecall(RecallOptions options) + { + var byCategory = new Dictionary(StringComparer.Ordinal) + { + ["recent"] = Math.Min(options.MaxRecentMessages, MessageCount), + ["relevant"] = Math.Min(options.MaxRelevantMessages, MessageCount), + ["entities"] = Math.Min(options.MaxEntities, EntityCount), + ["facts"] = Math.Min(options.MaxFacts, FactCount), + ["preferences"] = Math.Min(options.MaxPreferences, PreferenceCount), + ["traces"] = Math.Min(options.MaxTraces, TraceCount), + }; + + return new ExpectedRecallShape(byCategory, byCategory.Values.Sum()); } /// @@ -185,4 +239,10 @@ await reasoning.CompleteTraceAsync(trace.TraceId, "done", true, cancellationToke /// private static float[] Embed(string text, int dimensions) => DeterministicEmbeddingGenerator.Vector(text, dimensions); + + private static string Qualify(string text, DatasetIdentity identity) => + string.IsNullOrEmpty(identity.TopicToken) + ? text + : $"{text} {identity.TopicToken}"; + } diff --git a/tools/AgentMemory.Cli/Perf/PerfScenarios.cs b/tools/AgentMemory.Cli/Perf/PerfScenarios.cs index 3ebc00f8..8c838ff4 100644 --- a/tools/AgentMemory.Cli/Perf/PerfScenarios.cs +++ b/tools/AgentMemory.Cli/Perf/PerfScenarios.cs @@ -13,7 +13,11 @@ namespace AgentMemory.Cli.Perf; /// Ids are stable forever. Adding a scenario is additive; changing one requires a new id, so a /// number recorded today stays comparable to one recorded a year from now. /// -public sealed record PerfScenario(string Id, string Description, Func RunAsync); +public sealed record PerfScenario( + string Id, + string Description, + Func RunAsync, + bool SupportsInterleavedAb = true); /// Everything a scenario body needs for one iteration. /// @@ -26,6 +30,8 @@ public sealed record ScenarioContext( TurnRecord Turn, int Iteration, string Phase, + string? Variant, + RecallOptions RecallOptions, CancellationToken CancellationToken); /// @@ -38,7 +44,11 @@ public static class PerfScenarios public static IReadOnlyList All { get; } = [ new("PERF-R-04", "Full multi-category recall at shipped defaults", RecallAsync), - new("PERF-W-02", "Single response message, extraction enabled (shipped defaults)", StoreAndExtractAsync), + new( + "PERF-W-02", + "Single response message, extraction enabled (shipped defaults)", + StoreAndExtractAsync, + SupportsInterleavedAb: false), ]; private const string StoreProbeUserMessage = @@ -70,14 +80,17 @@ public static IReadOnlyList Select(string? filter) /// private static async Task RecallAsync(ScenarioContext ctx) { - var messages = new[] { new ChatMessage(ChatRole.User, PerfFixture.ProbeQuery) }; + var identity = ctx.Variant is null + ? PerfFixture.DefaultIdentity + : PerfFixture.ForVariant(ctx.Variant); + var messages = new[] { new ChatMessage(ChatRole.User, PerfFixture.ProbeQueryFor(identity)) }; var context = await ctx.Provider.BuildContextAsync( messages, - PerfFixture.SessionId, - PerfFixture.ConversationId, + identity.SessionId, + identity.ConversationId, ctx.CancellationToken, - PerfFixture.OwnerId).ConfigureAwait(false); + identity.OwnerId).ConfigureAwait(false); // Materialized once: AIContext.Messages is an enumerable, so counting and summing it separately // would enumerate it twice. @@ -88,14 +101,15 @@ private static async Task RecallAsync(ScenarioContext ctx) // Self-check, not decoration. A fixture whose vectors drift below MinSimilarityScore produces an // empty recall that still "succeeds" — and a baseline recorded from that would understate the // real cost by an order of magnitude and be quietly wrong forever after. + var expected = PerfFixture.ExpectedRecall(ctx.RecallOptions); var retrieved = ctx.Turn.Counter("items.retrieved"); - if (retrieved < PerfFixture.ExpectedRecalledItems) + if (retrieved != expected.Total) { - var breakdown = string.Join(", ", PerfFixture.ExpectedByCategory + var breakdown = string.Join(", ", expected.ByCategory .Select(kv => $"{kv.Key}={ctx.Turn.Counter($"items.{kv.Key}")}/{kv.Value}")); throw new InvalidOperationException( - $"PERF-R-04 recalled {retrieved} items but expected {PerfFixture.ExpectedRecalledItems} " + - $"({breakdown}). The fixture is not exercising the default recall shape, so this " + + $"PERF-R-04 recalled {retrieved} items but expected {expected.Total} " + + $"({breakdown}). The fixture is not exercising the configured recall shape, so this " + "measurement would be misleading. Check MinSimilarityScore and the seeded embeddings."); } } @@ -115,7 +129,6 @@ private static async Task StoreAndExtractAsync(ScenarioContext ctx) // iteration 0 had already populated — the one measured turn that was not a clean turn. var sessionId = $"perf-w02-{ctx.Phase}-{ctx.Iteration}"; var conversationId = $"{sessionId}-conv"; - var requestMessages = new[] { new ChatMessage(ChatRole.User, StoreProbeUserMessage), @@ -160,15 +173,19 @@ await ctx.Provider.PerformStoreAsync( /// AddNeo4jAgentMemory, and passing explicit defaults here documents that the scenarios run /// against shipped defaults rather than a tuned configuration. /// - public static Neo4jMemoryContextProvider CreateProvider(HermeticProfile profile) + public static Neo4jMemoryContextProvider CreateProvider( + HermeticProfile profile, + RecallOptions? recallOptions = null) { var services = profile.Services; + var configuredMemory = services.GetRequiredService>().Value; + var selectedRecall = recallOptions ?? configuredMemory.Recall; return new Neo4jMemoryContextProvider( services.GetRequiredService(), services.GetRequiredService(), services.GetRequiredService(), services.GetRequiredService(), - services.GetRequiredService>(), + Options.Create(configuredMemory with { Recall = selectedRecall }), Options.Create(new ContextFormatOptions()), Options.Create(new AgentFrameworkOptions()), services.GetRequiredService>()); diff --git a/tools/AgentMemory.Cli/Perf/QualityEvaluator.cs b/tools/AgentMemory.Cli/Perf/QualityEvaluator.cs index 4ed1c986..204ec057 100644 --- a/tools/AgentMemory.Cli/Perf/QualityEvaluator.cs +++ b/tools/AgentMemory.Cli/Perf/QualityEvaluator.cs @@ -52,12 +52,18 @@ public sealed class QualityEvaluator private readonly QualityFixture _fixture; private readonly IServiceProvider _services; private readonly int _dimensions; + private readonly RecallOptions _recallOptions; - public QualityEvaluator(QualityFixture fixture, IServiceProvider services, int dimensions) + public QualityEvaluator( + QualityFixture fixture, + IServiceProvider services, + int dimensions, + RecallOptions? recallOptions = null) { _fixture = fixture; _services = services; _dimensions = dimensions; + _recallOptions = recallOptions ?? RecallOptions.Default; } public async Task SeedAsync(TextWriter log, CancellationToken cancellationToken) @@ -172,9 +178,7 @@ public async Task EvaluateAsync(CancellationToken cancellationTok UserId = _fixture.OwnerId, Query = testCase.Query, QueryEmbedding = embedding, - // Shipped defaults, deliberately: the guard must measure what users get, not a - // configuration chosen to make the fixture look good. - Options = RecallOptions.Default, + Options = _recallOptions, }, cancellationToken).ConfigureAwait(false); results.Add(Score(testCase, RankedIds(recall.Context, testCase.Kind))); diff --git a/tools/AgentMemory.Cli/Program.cs b/tools/AgentMemory.Cli/Program.cs index 8a297faf..87f6843b 100644 --- a/tools/AgentMemory.Cli/Program.cs +++ b/tools/AgentMemory.Cli/Program.cs @@ -42,6 +42,27 @@ { try { + if (string.Equals(cli.Subcommand, "ab", StringComparison.OrdinalIgnoreCase)) + { + return await new AgentMemory.Cli.Commands.PerfAbCommand(Console.Out).ExecuteAsync( + cli.Get("control"), + cli.Get("candidate"), + cli.Get("scenarios"), + cli.Get("iterations"), + cli.Get("warmup"), + cli.Get("embedding-dimensions"), + cli.Get("latency"), + cli.Get("output")); + } + + if (cli.Subcommand is not null && + !string.Equals(cli.Subcommand, "run", StringComparison.OrdinalIgnoreCase)) + { + Console.Error.WriteLine( + $"error: unknown perf subcommand '{cli.Subcommand}'. Use 'run' or 'ab'."); + return 1; + } + return await new AgentMemory.Cli.Commands.PerfCommand(Console.Out).ExecuteAsync( cli.Get("label"), cli.Get("scenarios"),