perf(M-21): add interleaved A/B comparisons

docs/performance/README.md

@@ -105,6 +105,12 @@ dotnet run --project tools/AgentMemory.Cli -- perf --label mine --iterations 10
 
 # Reproduces the shape of a remote deployment (embedding 120 ms, model 900 ms)
 dotnet run --project tools/AgentMemory.Cli -- perf --label mine --latency remote --iterations 10
+
+# Compare two in-process recall configurations, with quality in the same report
+dotnet run --project tools/AgentMemory.Cli -- perf ab \
+  --control default \
+  --candidate Recall.MaxEntities=2 \
+  --iterations 30
 ```
 
 Each run writes a dated directory containing a manifest with the full environment fingerprint, an
@@ -115,6 +121,22 @@ marks those scores as report-only.
 Run it at both latency settings. A change that improves only the `remote` shape is an ordering or
 overlap win; one that improves both removed work.
 
+### Trustworthy A/B comparisons
+
+`perf ab` counterbalances execution order and crosses each configuration over two equivalent,
+owner-isolated fixture copies in the same database. Six consecutive paired iterations form one
+bootstrap unit, preserving the Docker/driver timing correlation instead of treating adjacent samples
+as independent. `--iterations` must therefore be a multiple of six and at least 12; 30 is the default.
+
+The initial configuration grammar accepts `default` plus `Recall.Max*` and
+`Recall.MinSimilarityScore` assignments. The default scenario is `PERF-R-04`. State-mutating
+scenarios such as `PERF-W-02` are rejected because shared write state would invalidate paired-sample
+independence.
+
+The rendered markdown reports exact counter ranges, retrieval and extraction quality, and the
+candidate/control bootstrap interval together. Only `iteration total` is the pre-registered timing
+headline; subspan intervals are exploratory and are not corrected for multiple comparisons.
+
 ### Determinism
 
 The harness uses a deterministic embedding function and a scripted model, so counters are reproducible.

tests/AgentMemory.Tests.Unit/Cli/PerfAbTests.cs

@@ -0,0 +1,161 @@
+using AgentMemory.Abstractions.Options;
+using AgentMemory.Cli;
+using AgentMemory.Cli.Commands;
+using AgentMemory.Cli.Perf;
+using FluentAssertions;
+
+namespace AgentMemory.Tests.Unit.Cli;
+
+public sealed class PerfAbTests
+{
+    [Fact]
+    public void CliArgs_PerfAb_ExposesSubcommand()
+    {
+        var args = CliArgs.Parse(["perf", "ab", "--control", "default"]);
+
+        args.Command.Should().Be("perf");
+        args.Subcommand.Should().Be("ab");
+        args.Get("control").Should().Be("default");
+    }
+
+    [Fact]
+    public void Configuration_Default_UsesShippedRecallOptions()
+    {
+        var configuration = PerfConfiguration.Parse("default");
+
+        configuration.Recall.Should().Be(RecallOptions.Default);
+        configuration.CanonicalSpec.Should().Be("default");
+    }
+
+    [Fact]
+    public void Configuration_MaxEntitiesOverride_ChangesOnlyMaxEntities()
+    {
+        var configuration = PerfConfiguration.Parse("Recall.MaxEntities=2");
+
+        configuration.Recall.Should().Be(RecallOptions.Default with { MaxEntities = 2 });
+        configuration.CanonicalSpec.Should().Be("Recall.MaxEntities=2");
+        PerfFixture.ExpectedRecall(configuration.Recall).Total.Should().Be(35);
+    }
+
+    [Fact]
+    public void Configuration_UnknownKey_IsRejected()
+    {
+        var act = () => PerfConfiguration.Parse("Recall.DoesNotExist=2");
+
+        act.Should().Throw<ArgumentException>()
+            .WithMessage("*DoesNotExist*");
+    }
+
+    [Fact]
+    public void StatefulWriteScenario_IsNotEligibleForSharedDatabaseAb()
+    {
+        var scenario = PerfScenarios.All.Single(item => item.Id == "PERF-W-02");
+
+        scenario.SupportsInterleavedAb.Should().BeFalse();
+    }
+
+    [Fact]
+    public void AbDatasetIdentities_AreEquivalentButDisjoint()
+    {
+        var control = PerfFixture.ForVariant("control");
+        var candidate = PerfFixture.ForVariant("candidate");
+
+        control.OwnerId.Should().NotBe(candidate.OwnerId);
+        control.SessionId.Should().NotBe(candidate.SessionId);
+        control.ConversationId.Should().NotBe(candidate.ConversationId);
+        control.TopicToken.Length.Should().Be(candidate.TopicToken.Length);
+        PerfFixture.ProbeQueryFor(control).Length.Should().Be(PerfFixture.ProbeQueryFor(candidate).Length);
+
+        control.IdPrefix.Should().NotBe(candidate.IdPrefix);
+    }
+
+    [Fact]
+    public void DatasetCrossover_BalancesBothConfigurationsAcrossBothCopies()
+    {
+        PerfAbCommand.DatasetVariantFor("control", 0).Should().Be("control");
+        PerfAbCommand.DatasetVariantFor("candidate", 0).Should().Be("candidate");
+        PerfAbCommand.DatasetVariantFor("control", 1).Should().Be("candidate");
+        PerfAbCommand.DatasetVariantFor("candidate", 1).Should().Be("control");
+    }
+
+    [Fact]
+    public void PairedBootstrap_IdenticalSamples_ReportNoSignificantDifference()
+    {
+        var result = PairedRatioBootstrap.Analyze(
+            [10, 11, 9, 12, 10],
+            [10, 11, 9, 12, 10],
+            resamples: 2_000,
+            seed: 21);
+
+        result.Estimate.Should().Be(1);
+        result.Lower95.Should().Be(1);
+        result.Upper95.Should().Be(1);
+        result.Verdict.Should().Be(TimingVerdict.NoSignificantDifference);
+    }
+
+    [Fact]
+    public void PairedBootstrap_UniformHalving_DeclaresImprovement()
+    {
+        var result = PairedRatioBootstrap.Analyze(
+            [10, 20, 30, 40, 50],
+            [5, 10, 15, 20, 25],
+            resamples: 2_000,
+            seed: 21);
+
+        result.Estimate.Should().BeApproximately(0.5, 1e-12);
+        result.Upper95.Should().BeLessThan(1);
+        result.Verdict.Should().Be(TimingVerdict.Improvement);
+    }
+
+    [Fact]
+    public void PairedBootstrap_UniformDoubling_DeclaresRegression()
+    {
+        var result = PairedRatioBootstrap.Analyze(
+            [10, 20, 30, 40, 50],
+            [20, 40, 60, 80, 100],
+            resamples: 2_000,
+            seed: 21);
+
+        result.Estimate.Should().BeApproximately(2, 1e-12);
+        result.Lower95.Should().BeGreaterThan(1);
+        result.Verdict.Should().Be(TimingVerdict.Regression);
+    }
+
+    [Fact]
+    public void CounterbalancedBootstrap_FirstSecondEffect_ReportsNoSignificantDifference()
+    {
+        // The second runner is 10% faster: candidate benefits in AB, control benefits in BA.
+        // Each six-pair cluster has three of each order, so execution position cancels exactly.
+        var result = PairedRatioBootstrap.AnalyzeCounterbalanced(
+            [100, 90, 100, 90, 100, 90, 100, 90, 100, 90, 100, 90],
+            [90, 100, 90, 100, 90, 100, 90, 100, 90, 100, 90, 100],
+            resamples: 2_000,
+            seed: 21);
+
+        result.Pairs.Should().Be(2);
+        result.Estimate.Should().Be(1);
+        result.Lower95.Should().Be(1);
+        result.Upper95.Should().Be(1);
+        result.Verdict.Should().Be(TimingVerdict.NoSignificantDifference);
+    }
+
+    [Fact]
+    public void CounterbalancedBootstrap_RequiresCompleteAbBaBlocks()
+    {
+        var act = () => PairedRatioBootstrap.AnalyzeCounterbalanced(
+            [100, 90, 100, 90, 100, 90, 100, 90, 100, 90],
+            [90, 100, 90, 100, 90, 100, 90, 100, 90, 100]);
+
+        act.Should().Throw<ArgumentException>()
+            .WithMessage("*multiple of six*");
+    }
+
+    [Fact]
+    public void PairedBootstrap_RejectsNonPositiveTimings()
+    {
+        var act = () => PairedRatioBootstrap.Analyze([10, 0], [9, 1]);
+
+        act.Should().Throw<ArgumentException>()
+            .WithMessage("*positive*");
+    }
+}

tools/AgentMemory.Cli/CliArgs.cs

@@ -8,11 +8,17 @@ namespace AgentMemory.Cli;
 public sealed class CliArgs
 {
     public string? Command { get; }
+    public string? Subcommand => Positionals.FirstOrDefault();
+    public IReadOnlyList<string> Positionals { get; }
     public IReadOnlyDictionary<string, string?> Options { get; }
 
-    private CliArgs(string? command, IReadOnlyDictionary<string, string?> options)
+    private CliArgs(
+        string? command,
+        IReadOnlyList<string> positionals,
+        IReadOnlyDictionary<string, string?> options)
     {
         Command = command;
+        Positionals = positionals;
         Options = options;
     }
 
@@ -25,6 +31,7 @@ private CliArgs(string? command, IReadOnlyDictionary<string, string?> options)
     public static CliArgs Parse(string[] args)
     {
         string? command = null;
+        var positionals = new List<string>();
         var options = new Dictionary<string, string?>(StringComparer.OrdinalIgnoreCase);
 
         for (var i = 0; i < args.Length; i++)
@@ -54,11 +61,14 @@ public static CliArgs Parse(string[] args)
             }
             else
             {
-                command ??= token;
+                if (command is null)
+                    command = token;
+                else
+                    positionals.Add(token);
             }
         }
 
-        return new CliArgs(command, options);
+        return new CliArgs(command, positionals, options);
     }
 }
 
@@ -100,6 +110,13 @@ Neo4j via Testcontainers (Docker required) with deterministic
                                      embeddings and a scripted model, so counters are reproducible.
                                      Quality gate defaults on. Writes a dated run directory under
                                      artifacts/perf by default.
+              perf ab --control <spec> --candidate <spec> [--scenarios <ids|all>]
+                      [--iterations <n>] [--warmup <n>] [--latency <zero|remote>]
+                                     Run counterbalanced control/candidate pairs in one process/database.
+                                     Iterations must be a multiple of 6 and at least 12 (default 30).
+                                     Specs use `default` or recall assignments such as
+                                     `Recall.MaxEntities=2`. Reports exact counter deltas, paired
+                                     timing ratios with bootstrap 95% CIs, and quality side by side.
               decay [--owner <id>]   Decay-prune memories: soft-invalidate by default (kept + recoverable;
                                      set MemoryDecay:NonDestructive=false to hard-delete). Owner-scoped, or global.
               schema-parity [--upstream-version <v>]
@@ -125,6 +142,8 @@ agentmemory history --type fact --owner user-42 --limit 20
               agentmemory evaluate --iterations 3 --output artifacts/evaluation/local.json
               agentmemory perf --label baseline --iterations 10
               agentmemory perf --label feat-01-access-tracking --latency remote
+              agentmemory perf ab --control default --candidate default --scenarios PERF-R-04
+              agentmemory perf ab --control default --candidate Recall.MaxEntities=2
             """);
     }
 }