diff --git a/.github/workflows/performance-gate.yml b/.github/workflows/performance-gate.yml new file mode 100644 index 00000000..16e87b75 --- /dev/null +++ b/.github/workflows/performance-gate.yml @@ -0,0 +1,82 @@ +name: Performance Gate + +on: + pull_request: + branches: [main] + types: [opened, synchronize, reopened, ready_for_review, labeled, unlabeled, edited] + +permissions: + contents: read + +concurrency: + group: performance-gate-${{ github.event.pull_request.number }} + cancel-in-progress: true + +env: + DOTNET_NOLOGO: 'true' + DOTNET_CLI_TELEMETRY_OPTOUT: 'true' + +jobs: + gate: + name: Hermetic counter + quality gate + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + + - name: Setup .NET + uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0 + with: + global-json-file: global.json + cache: true + cache-dependency-path: '**/*.csproj' + + - name: Restore + run: dotnet restore tools/AgentMemory.Cli/AgentMemory.Cli.csproj + + - name: Build performance CLI + run: > + dotnet build tools/AgentMemory.Cli/AgentMemory.Cli.csproj + -c Release --no-restore + + - name: Run zero and remote profiles through the gate + shell: bash + env: + PERF_COUNTER_CHANGE_ALLOWED: ${{ contains(github.event.pull_request.labels.*.name, 'perf-counter-change') }} + PERF_COUNTER_CHANGE_JUSTIFICATION: ${{ github.event.pull_request.body }} + run: | + set -euo pipefail + + for latency in zero remote; do + output="artifacts/perf-ci/$latency" + dotnet run --project tools/AgentMemory.Cli/AgentMemory.Cli.csproj \ + -c Release --no-build -- \ + perf \ + --label "ci-$latency" \ + --iterations 3 \ + --warmup 1 \ + --latency "$latency" \ + --output "$output" + + report="$(find "$output" -type f -name summary.json -print -quit)" + if [[ -z "$report" ]]; then + echo "::error::perf did not produce summary.json for $latency" + exit 1 + fi + + dotnet run --project tools/AgentMemory.Cli/AgentMemory.Cli.csproj \ + -c Release --no-build -- \ + perf gate \ + --baseline eng/perf/baselines/hermetic-S.json \ + --report "$report" \ + --allow-counter-change "$PERF_COUNTER_CHANGE_ALLOWED" \ + --counter-change-justification "$PERF_COUNTER_CHANGE_JUSTIFICATION" + done + + - name: Upload performance reports + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: performance-gate-reports + path: artifacts/perf-ci + if-no-files-found: ignore diff --git a/docs/performance/README.md b/docs/performance/README.md index fca2ba1a..5cce8bd6 100644 --- a/docs/performance/README.md +++ b/docs/performance/README.md @@ -5,6 +5,7 @@ What AgentMemory costs you per agent turn, how that cost is measured, and how to | Doc | What it covers | |---|---| | **README.md** (this file) | The two phases, what is and isn't measured, how to run it yourself | +| [`hermetic-S.json`](../../eng/perf/baselines/hermetic-S.json) | Machine-readable counter + quality baseline enforced on pull requests | | [baseline-1.3.0.md](baseline-1.3.0.md) | The measured cost model for release 1.3.0 | --- @@ -111,6 +112,15 @@ dotnet run --project tools/AgentMemory.Cli -- perf ab \ --control default \ --candidate Recall.MaxEntities=2 \ --iterations 30 + +# Check one completed run against the committed counter + quality baseline +dotnet run --project tools/AgentMemory.Cli -- perf gate \ + --baseline eng/perf/baselines/hermetic-S.json \ + --report + +# Deliberately refresh the reviewable baseline from a completed run +dotnet run --project tools/AgentMemory.Cli -- perf baseline --update \ + --report ``` Each run writes a dated directory containing a manifest with the full environment fingerprint, an @@ -147,6 +157,27 @@ Both measured scenarios also **self-assert**: the recall scenario fails the run items than the configured limits, and the ingestion scenario fails if extraction never ran. Those failures are otherwise silent and would produce a confident, wrong number. +### Pull-request regression gate + +A dedicated GitHub Actions job runs the hermetic scale-S profile with both zero and remote-like +provider latency. Each report is compared with +[`eng/perf/baselines/hermetic-S.json`](../../eng/perf/baselines/hermetic-S.json). The gate rejects: + +- any increase in Neo4j transactions or queries, embedding requests, or model calls; +- an estimated payload-byte increase above 5%, once that counter is available; and +- any retrieval or extraction quality regression beyond the committed tolerance (currently zero). + +A deliberate structural-counter increase needs both the `perf-counter-change` pull-request label and +a non-empty pull-request-body line in this exact form: + +```text +Perf counter change justification: +``` + +That acknowledgement cannot override payload or quality failures. Hermetic elapsed milliseconds are +reported for diagnosis but are deliberately excluded from the CI decision because runner timings are +not portable. + --- ## Tuning starting points diff --git a/docs/performance/baseline-1.3.0.md b/docs/performance/baseline-1.3.0.md index c16c1f3e..9d1e7e68 100644 --- a/docs/performance/baseline-1.3.0.md +++ b/docs/performance/baseline-1.3.0.md @@ -70,8 +70,11 @@ The cost counters are only accepted when deterministic quality remains at this c Every value above was identical across five fresh-container runs: maximum observed variance **0.000**. The derived tolerance is therefore **zero**, recorded in -`eng/perf/baselines/quality.json`. The `perf` command gates by default and exits non-zero on a drop or -forbidden retrieval. This guard is about deterministic pipeline behavior; it does not claim to score +`eng/perf/baselines/quality.json`. The combined reviewable counter + quality snapshot used by pull +request CI is [`eng/perf/baselines/hermetic-S.json`](../../eng/perf/baselines/hermetic-S.json). +The `perf` command gates quality by default, while `perf gate` also rejects structural-counter +regressions against that snapshot. Neither command grades hermetic elapsed milliseconds. This guard is +about deterministic pipeline behavior; it does not claim to score the quality of a live model's prose. --- diff --git a/eng/perf/baselines/hermetic-S.json b/eng/perf/baselines/hermetic-S.json new file mode 100644 index 00000000..1cd7a211 --- /dev/null +++ b/eng/perf/baselines/hermetic-S.json @@ -0,0 +1,59 @@ +{ + "schemaVersion": 1, + "profile": "hermetic-S", + "qualityTolerance": 0, + "scenarios": { + "PERF-R-04": { + "counters": { + "access_tracking.items": 25, + "context.chars": 3906, + "context.messages": 14, + "embed.chars": 95, + "embed.items": 1, + "embed.requests": 1, + "items.entities": 10, + "items.facts": 10, + "items.preferences": 5, + "items.recent": 10, + "items.relevant": 5, + "items.retrieved": 43, + "items.traces": 3, + "neo4j.queries": 9, + "neo4j.tx.read": 6, + "neo4j.tx.write": 1, + "recall.chars": 3823 + } + }, + "PERF-W-02": { + "counters": { + "embed.chars": 201, + "embed.items": 4, + "embed.requests": 4, + "extract.candidate_entities": 2, + "llm.calls": 4, + "llm.tokens_in": 947, + "llm.tokens_out": 668, + "neo4j.queries": 43, + "neo4j.tx.read": 4, + "neo4j.tx.write": 18, + "persist.entities": 2, + "persist.facts": 2, + "persist.preferences": 1, + "persist.relationships": 0, + "store.messages": 1 + } + } + }, + "quality": { + "recallAtK": 1, + "mrr": 1, + "casesWithViolations": 0, + "entityPrecision": 1, + "entityRecall": 1, + "factPrecision": 1, + "factRecall": 1, + "preferencePrecision": 1, + "preferenceRecall": 1, + "extractionFalsePositiveRate": 0 + } +} diff --git a/tests/AgentMemory.Tests.Unit/Cli/PerfBaselineGateTests.cs b/tests/AgentMemory.Tests.Unit/Cli/PerfBaselineGateTests.cs new file mode 100644 index 00000000..87b2e9b3 --- /dev/null +++ b/tests/AgentMemory.Tests.Unit/Cli/PerfBaselineGateTests.cs @@ -0,0 +1,270 @@ +using System.Text.Json; +using AgentMemory.Cli.Commands; +using AgentMemory.Cli.Perf; +using FluentAssertions; + +namespace AgentMemory.Tests.Unit.Cli; + +public sealed class PerfBaselineGateTests +{ + [Fact] + public void BaselineFromSummary_StoresPortableCountersAndQuality() + { + var baseline = PerfBaselineDocument.FromSummary(Report()); + + baseline.Scenarios["PERF-R-04"].Counters["neo4j.queries"].Should().Be(9); + baseline.Scenarios["PERF-W-02"].Counters["llm.calls"].Should().Be(4); + baseline.Quality.RecallAtK.Should().Be(1); + baseline.Quality.ExtractionFalsePositiveRate.Should().Be(0); + } + + [Fact] + public void Gate_IdenticalReport_Passes() + { + var report = Report(); + var baseline = PerfBaselineDocument.FromSummary(report); + + var result = PerfRegressionGate.Evaluate( + baseline, + report, + new PerfCounterChangeOverride(false, null)); + + result.Passed.Should().BeTrue(); + result.Violations.Should().BeEmpty(); + } + + [Fact] + public void Gate_OneExtraQuery_FailsWithoutOverride() + { + var baseline = PerfBaselineDocument.FromSummary(Report()); + + var result = PerfRegressionGate.Evaluate( + baseline, + Report(recallQueries: 10), + new PerfCounterChangeOverride(false, null)); + + result.Passed.Should().BeFalse(); + result.Violations.Should().ContainSingle(message => + message.Contains("PERF-R-04", StringComparison.Ordinal) && + message.Contains("neo4j.queries", StringComparison.Ordinal) && + message.Contains("9 -> 10", StringComparison.Ordinal)); + } + + [Fact] + public void Gate_OneExtraQuery_LabelWithoutJustification_StillFails() + { + var baseline = PerfBaselineDocument.FromSummary(Report()); + + var result = PerfRegressionGate.Evaluate( + baseline, + Report(recallQueries: 10), + new PerfCounterChangeOverride(true, "ordinary PR description")); + + result.Passed.Should().BeFalse(); + result.Violations.Should().Contain(message => + message.Contains("justification", StringComparison.OrdinalIgnoreCase)); + } + + [Fact] + public void Gate_OneExtraQuery_LabelAndJustification_Passes() + { + var baseline = PerfBaselineDocument.FromSummary(Report()); + + var result = PerfRegressionGate.Evaluate( + baseline, + Report(recallQueries: 10), + new PerfCounterChangeOverride( + true, + "Summary\nPerf counter change justification: deliberate query for freshness")); + + result.Passed.Should().BeTrue(); + result.AcknowledgedCounterChanges.Should().ContainSingle(); + } + + [Fact] + public void Gate_QualityDrop_CannotBeOverridden() + { + var baseline = PerfBaselineDocument.FromSummary(Report()); + + var result = PerfRegressionGate.Evaluate( + baseline, + Report(recallAtK: 0.99), + new PerfCounterChangeOverride( + true, + "Perf counter change justification: deliberate query for freshness")); + + result.Passed.Should().BeFalse(); + result.Violations.Should().Contain(message => + message.Contains("recallAtK", StringComparison.Ordinal)); + } + + [Fact] + public void Gate_BytesIncreaseAboveFivePercent_Fails() + { + var baseline = PerfBaselineDocument.FromSummary(Report(bytesEstimate: 100)); + + var result = PerfRegressionGate.Evaluate( + baseline, + Report(bytesEstimate: 106), + new PerfCounterChangeOverride(false, null)); + + result.Passed.Should().BeFalse(); + result.Violations.Should().Contain(message => + message.Contains("bytes_est", StringComparison.Ordinal) && + message.Contains("5%", StringComparison.Ordinal)); + } + + [Fact] + public void BaselineFromSummary_NondeterministicCounter_IsRejected() + { + var act = () => PerfBaselineDocument.FromSummary(Report(recallQueriesMin: 8)); + + act.Should().Throw() + .WithMessage("*neo4j.queries*deterministic*"); + } + + [Fact] + public void BaselineFromSummary_NonScaleSReport_IsRejected() + { + var act = () => PerfBaselineDocument.FromSummary(Report(scale: "M")); + + act.Should().Throw() + .WithMessage("*profile hermetic, scale S*"); + } + + [Fact] + public void Baseline_SaveAndLoad_RoundTrips() + { + var path = Path.Combine( + Path.GetTempPath(), + $"agentmemory-perf-baseline-{Guid.NewGuid():N}.json"); + try + { + PerfBaselineDocument.FromSummary(Report()).Save(path); + + var loaded = PerfBaselineDocument.Load(path); + + loaded.Profile.Should().Be("hermetic-S"); + loaded.Scenarios["PERF-R-04"].Counters["neo4j.queries"].Should().Be(9); + loaded.Quality.RecallAtK.Should().Be(1); + } + finally + { + File.Delete(path); + } + } + + [Fact] + public async Task GateCommand_ExtraQuery_ReturnsNonZero() + { + var baselinePath = Path.Combine( + Path.GetTempPath(), + $"agentmemory-perf-baseline-{Guid.NewGuid():N}.json"); + var reportPath = Path.Combine( + Path.GetTempPath(), + $"agentmemory-perf-report-{Guid.NewGuid():N}.json"); + try + { + PerfBaselineDocument.FromSummary(Report()).Save(baselinePath); + await File.WriteAllTextAsync(reportPath, Report(recallQueries: 10).GetRawText()); + var output = new StringWriter(); + + var exitCode = await new PerfGateCommand(output).ExecuteAsync( + baselinePath, + reportPath, + allowCounterChangeValue: null, + pullRequestBody: null); + + exitCode.Should().Be(1); + output.ToString().Should().Contain("perf gate: FAIL"); + output.ToString().Should().Contain("neo4j.queries increased: 9 -> 10"); + } + finally + { + File.Delete(baselinePath); + File.Delete(reportPath); + } + } + + [Fact] + public async Task BaselineCommand_WithoutExplicitUpdate_ReturnsNonZero() + { + var output = new StringWriter(); + + var exitCode = await new PerfBaselineCommand(output).ExecuteAsync(null, null, null); + + exitCode.Should().Be(1); + output.ToString().Should().Contain("--update"); + } + + private static JsonElement Report( + long recallQueries = 9, + long? recallQueriesMin = null, + double recallAtK = 1, + long? bytesEstimate = null, + string scale = "S") + { + static object Counter(long min, long max) => new + { + median = (double)max, + min, + max, + deterministic = min == max, + }; + + var recallCounters = new Dictionary(StringComparer.Ordinal) + { + ["neo4j.tx.read"] = Counter(6, 6), + ["neo4j.tx.write"] = Counter(1, 1), + ["neo4j.queries"] = Counter(recallQueriesMin ?? recallQueries, recallQueries), + ["embed.requests"] = Counter(1, 1), + ["items.retrieved"] = Counter(43, 43), + }; + if (bytesEstimate is not null) + recallCounters["neo4j.bytes_est"] = Counter(bytesEstimate.Value, bytesEstimate.Value); + + var writeCounters = new Dictionary(StringComparer.Ordinal) + { + ["neo4j.tx.read"] = Counter(4, 4), + ["neo4j.tx.write"] = Counter(18, 18), + ["neo4j.queries"] = Counter(43, 43), + ["embed.requests"] = Counter(4, 4), + ["llm.calls"] = Counter(4, 4), + }; + + return JsonSerializer.SerializeToElement(new + { + manifest = new { profile = "hermetic", scale }, + qualityGate = new { tolerance = 0d }, + quality = new + { + recallAtK, + mrr = 1d, + casesWithViolations = 0, + }, + extractionQuality = new + { + entityPrecision = 1d, + entityRecall = 1d, + factPrecision = 1d, + factRecall = 1d, + preferencePrecision = 1d, + preferenceRecall = 1d, + falsePositiveRate = 0d, + }, + scenarios = new object[] + { + new + { + scenario = "PERF-R-04", + counters = recallCounters, + }, + new + { + scenario = "PERF-W-02", + counters = writeCounters, + }, + }, + }); + } +} diff --git a/tools/AgentMemory.Cli/CliArgs.cs b/tools/AgentMemory.Cli/CliArgs.cs index 02f7525d..6df70166 100644 --- a/tools/AgentMemory.Cli/CliArgs.cs +++ b/tools/AgentMemory.Cli/CliArgs.cs @@ -117,6 +117,15 @@ Iterations must be a multiple of 6 and at least 12 (default 30). Specs use `default` or recall assignments such as `Recall.MaxEntities=2`. Reports exact counter deltas, paired timing ratios with bootstrap 95% CIs, and quality side by side. + perf baseline --update --report [--output ] + Write deterministic counters and quality to a reviewable baseline. + Default output: eng/perf/baselines/hermetic-S.json. + perf gate --baseline --report + [--allow-counter-change ] + [--counter-change-justification ] + Fail on a structural counter increase or quality regression. + A counter increase needs both the perf-counter-change label + (passed as allow-counter-change) and a PR-body justification. decay [--owner ] Decay-prune memories: soft-invalidate by default (kept + recoverable; set MemoryDecay:NonDestructive=false to hard-delete). Owner-scoped, or global. schema-parity [--upstream-version ] @@ -144,6 +153,9 @@ agentmemory perf --label baseline --iterations 10 agentmemory perf --label feat-01-access-tracking --latency remote agentmemory perf ab --control default --candidate default --scenarios PERF-R-04 agentmemory perf ab --control default --candidate Recall.MaxEntities=2 + agentmemory perf baseline --update --report artifacts/perf/run/summary.json + agentmemory perf gate --baseline eng/perf/baselines/hermetic-S.json \ + --report artifacts/perf/run/summary.json """); } } diff --git a/tools/AgentMemory.Cli/Commands/PerfGateCommands.cs b/tools/AgentMemory.Cli/Commands/PerfGateCommands.cs new file mode 100644 index 00000000..cd1e238b --- /dev/null +++ b/tools/AgentMemory.Cli/Commands/PerfGateCommands.cs @@ -0,0 +1,76 @@ +using System.Text.Json; +using AgentMemory.Cli.Perf; + +namespace AgentMemory.Cli.Commands; + +public sealed class PerfBaselineCommand(TextWriter output) +{ + public async Task ExecuteAsync( + string? updateValue, + string? reportPath, + string? outputPath, + CancellationToken cancellationToken = default) + { + if (!bool.TryParse(updateValue, out var update) || !update) + { + output.WriteLine("error: perf baseline requires --update."); + return 1; + } + if (string.IsNullOrWhiteSpace(reportPath) || !File.Exists(reportPath)) + { + output.WriteLine($"error: performance summary not found: {reportPath ?? "(missing --report)"}"); + return 1; + } + + await using var stream = File.OpenRead(reportPath); + using var report = await JsonDocument.ParseAsync( + stream, + cancellationToken: cancellationToken).ConfigureAwait(false); + var baseline = PerfBaselineDocument.FromSummary(report.RootElement); + var path = outputPath ?? PerfBaselineDocument.DefaultPath; + baseline.Save(path); + + output.WriteLine($"perf baseline: wrote {path}"); + return 0; + } +} + +public sealed class PerfGateCommand(TextWriter output) +{ + public async Task ExecuteAsync( + string? baselinePath, + string? reportPath, + string? allowCounterChangeValue, + string? pullRequestBody, + CancellationToken cancellationToken = default) + { + var path = baselinePath ?? PerfBaselineDocument.DefaultPath; + if (string.IsNullOrWhiteSpace(reportPath) || !File.Exists(reportPath)) + { + output.WriteLine($"error: performance summary not found: {reportPath ?? "(missing --report)"}"); + return 1; + } + + var allowCounterChange = + bool.TryParse(allowCounterChangeValue, out var allowed) && allowed; + var baseline = PerfBaselineDocument.Load(path); + await using var stream = File.OpenRead(reportPath); + using var report = await JsonDocument.ParseAsync( + stream, + cancellationToken: cancellationToken).ConfigureAwait(false); + + var result = PerfRegressionGate.Evaluate( + baseline, + report.RootElement, + new PerfCounterChangeOverride(allowCounterChange, pullRequestBody)); + + output.WriteLine(result.Passed + ? "perf gate: PASS" + : "perf gate: FAIL"); + foreach (var change in result.AcknowledgedCounterChanges) + output.WriteLine($"acknowledged: {change}"); + foreach (var violation in result.Violations) + output.WriteLine($"error: {violation}"); + return result.Passed ? 0 : 1; + } +} diff --git a/tools/AgentMemory.Cli/Perf/PerfBaselineDocument.cs b/tools/AgentMemory.Cli/Perf/PerfBaselineDocument.cs new file mode 100644 index 00000000..c6cce612 --- /dev/null +++ b/tools/AgentMemory.Cli/Perf/PerfBaselineDocument.cs @@ -0,0 +1,298 @@ +using System.Text.Json; +using System.Text.RegularExpressions; + +namespace AgentMemory.Cli.Perf; + +internal sealed record PerfBaselineDocument( + int SchemaVersion, + string Profile, + double QualityTolerance, + IReadOnlyDictionary Scenarios, + PerfQualityBaseline Quality) +{ + public const string DefaultPath = "eng/perf/baselines/hermetic-S.json"; + + private static readonly JsonSerializerOptions Json = new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + PropertyNameCaseInsensitive = true, + WriteIndented = true, + }; + + public static PerfBaselineDocument FromSummary(JsonElement summary) + { + var manifest = Required(summary, "manifest"); + var profile = Required(manifest, "profile").GetString(); + var scale = Required(manifest, "scale").GetString(); + if (profile != "hermetic" || scale != "S") + { + throw new InvalidDataException( + $"performance baselines require profile hermetic, scale S; got {profile ?? "(null)"}, " + + $"{scale ?? "(null)"}."); + } + + var scenarios = new SortedDictionary(StringComparer.Ordinal); + foreach (var scenario in Required(summary, "scenarios").EnumerateArray()) + { + var id = Required(scenario, "scenario").GetString() + ?? throw new InvalidDataException("scenario id must be a string."); + var counters = new SortedDictionary(StringComparer.Ordinal); + foreach (var counter in Required(scenario, "counters").EnumerateObject()) + { + var value = counter.Value; + var minimum = Required(value, "min").GetInt64(); + var maximum = Required(value, "max").GetInt64(); + var deterministic = Required(value, "deterministic").GetBoolean(); + if (!deterministic || minimum != maximum) + { + throw new InvalidDataException( + $"{id} counter {counter.Name} must be deterministic to enter a baseline " + + $"(min={minimum}, max={maximum})."); + } + + counters[counter.Name] = maximum; + } + + if (!scenarios.TryAdd(id, new PerfScenarioBaseline(counters))) + throw new InvalidDataException($"summary contains duplicate scenario '{id}'."); + } + + if (scenarios.Count == 0) + throw new InvalidDataException("summary contains no performance scenarios."); + + var quality = Required(summary, "quality"); + var extraction = Required(summary, "extractionQuality"); + var tolerance = Required(Required(summary, "qualityGate"), "tolerance").GetDouble(); + + return new PerfBaselineDocument( + SchemaVersion: 1, + Profile: $"{profile}-{scale}", + QualityTolerance: tolerance, + Scenarios: scenarios, + Quality: new PerfQualityBaseline( + RecallAtK: Required(quality, "recallAtK").GetDouble(), + Mrr: Required(quality, "mrr").GetDouble(), + CasesWithViolations: Required(quality, "casesWithViolations").GetInt32(), + EntityPrecision: Required(extraction, "entityPrecision").GetDouble(), + EntityRecall: Required(extraction, "entityRecall").GetDouble(), + FactPrecision: Required(extraction, "factPrecision").GetDouble(), + FactRecall: Required(extraction, "factRecall").GetDouble(), + PreferencePrecision: Required(extraction, "preferencePrecision").GetDouble(), + PreferenceRecall: Required(extraction, "preferenceRecall").GetDouble(), + ExtractionFalsePositiveRate: Required(extraction, "falsePositiveRate").GetDouble())); + } + + public static PerfBaselineDocument Load(string path) + { + if (!File.Exists(path)) + throw new FileNotFoundException($"performance baseline not found: {path}", path); + + var baseline = JsonSerializer.Deserialize(File.ReadAllText(path), Json) + ?? throw new InvalidDataException($"performance baseline is empty: {path}"); + if (baseline.SchemaVersion != 1) + { + throw new InvalidDataException( + $"unsupported performance baseline schema {baseline.SchemaVersion}; expected 1."); + } + + return baseline; + } + + public void Save(string path) + { + var directory = Path.GetDirectoryName(path); + if (!string.IsNullOrEmpty(directory)) + Directory.CreateDirectory(directory); + File.WriteAllText(path, JsonSerializer.Serialize(this, Json) + Environment.NewLine); + } + + private static JsonElement Required(JsonElement parent, string property) + { + if (!parent.TryGetProperty(property, out var value)) + throw new InvalidDataException($"performance summary is missing '{property}'."); + return value; + } +} + +internal sealed record PerfScenarioBaseline(IReadOnlyDictionary Counters); + +internal sealed record PerfQualityBaseline( + double RecallAtK, + double Mrr, + int CasesWithViolations, + double EntityPrecision, + double EntityRecall, + double FactPrecision, + double FactRecall, + double PreferencePrecision, + double PreferenceRecall, + double ExtractionFalsePositiveRate); + +internal sealed record PerfCounterChangeOverride(bool Allowed, string? PullRequestBody) +{ + private static readonly Regex JustificationPattern = new( + @"^\s*Perf counter change justification:\s*(?\S.*?)\s*$", + RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase | + RegexOptions.Multiline); + + public string? Justification => + PullRequestBody is null + ? null + : JustificationPattern.Match(PullRequestBody) is { Success: true } match + ? match.Groups["reason"].Value + : null; +} + +internal sealed record PerfGateResult( + bool Passed, + IReadOnlyList Violations, + IReadOnlyList AcknowledgedCounterChanges); + +internal static class PerfRegressionGate +{ + private const double BytesIncreaseLimit = 0.05; + + public static PerfGateResult Evaluate( + PerfBaselineDocument baseline, + JsonElement report, + PerfCounterChangeOverride counterOverride) + { + var actual = PerfBaselineDocument.FromSummary(report); + var violations = new List(); + var acknowledged = new List(); + + foreach (var (scenarioId, expectedScenario) in baseline.Scenarios) + { + if (!actual.Scenarios.TryGetValue(scenarioId, out var actualScenario)) + { + violations.Add($"missing required scenario {scenarioId}."); + continue; + } + + var counterNames = expectedScenario.Counters.Keys + .Concat(actualScenario.Counters.Keys) + .Distinct(StringComparer.Ordinal) + .OrderBy(name => name, StringComparer.Ordinal); + foreach (var counter in counterNames) + { + var expected = expectedScenario.Counters.GetValueOrDefault(counter); + var observed = actualScenario.Counters.GetValueOrDefault(counter); + if (counter is "bytes_est" or "neo4j.bytes_est") + { + if (BytesIncreasedTooFar(expected, observed)) + { + violations.Add( + $"{scenarioId} bytes_est increased more than 5%: {expected} -> {observed}."); + } + continue; + } + + if (!IsGuardedCounter(counter) || observed <= expected) + continue; + + var message = $"{scenarioId} {counter} increased: {expected} -> {observed}."; + if (counterOverride.Allowed && counterOverride.Justification is not null) + acknowledged.Add($"{message} {counterOverride.Justification}"); + else + violations.Add(message); + } + } + + if (counterOverride.Allowed && + counterOverride.Justification is null && + HasCounterIncrease(baseline, actual)) + { + violations.Add( + "perf-counter-change requires a non-empty 'Perf counter change justification:' line."); + } + + CompareQuality(baseline, actual, violations); + return new PerfGateResult(violations.Count == 0, violations, acknowledged); + } + + private static bool HasCounterIncrease( + PerfBaselineDocument baseline, + PerfBaselineDocument actual) => + baseline.Scenarios.Any(pair => + actual.Scenarios.TryGetValue(pair.Key, out var actualScenario) && + pair.Value.Counters.Keys + .Concat(actualScenario.Counters.Keys) + .Distinct(StringComparer.Ordinal) + .Any(counter => + IsGuardedCounter(counter) && + actualScenario.Counters.GetValueOrDefault(counter) > + pair.Value.Counters.GetValueOrDefault(counter))); + + private static bool IsGuardedCounter(string counter) => + counter.StartsWith("neo4j.tx", StringComparison.Ordinal) || + counter is "neo4j.queries" or "embed.requests" or "llm.calls"; + + private static bool BytesIncreasedTooFar(long expected, long actual) => + expected == 0 + ? actual > 0 + : actual > expected * (1 + BytesIncreaseLimit); + + private static void CompareQuality( + PerfBaselineDocument baseline, + PerfBaselineDocument actual, + ICollection violations) + { + var expected = baseline.Quality; + var observed = actual.Quality; + var tolerance = baseline.QualityTolerance; + + Minimum("recallAtK", expected.RecallAtK, observed.RecallAtK, tolerance, violations); + Minimum("mrr", expected.Mrr, observed.Mrr, tolerance, violations); + Maximum( + "casesWithViolations", + expected.CasesWithViolations, + observed.CasesWithViolations, + tolerance, + violations); + Minimum( + "entityPrecision", expected.EntityPrecision, observed.EntityPrecision, tolerance, violations); + Minimum("entityRecall", expected.EntityRecall, observed.EntityRecall, tolerance, violations); + Minimum("factPrecision", expected.FactPrecision, observed.FactPrecision, tolerance, violations); + Minimum("factRecall", expected.FactRecall, observed.FactRecall, tolerance, violations); + Minimum( + "preferencePrecision", + expected.PreferencePrecision, + observed.PreferencePrecision, + tolerance, + violations); + Minimum( + "preferenceRecall", + expected.PreferenceRecall, + observed.PreferenceRecall, + tolerance, + violations); + Maximum( + "extractionFalsePositiveRate", + expected.ExtractionFalsePositiveRate, + observed.ExtractionFalsePositiveRate, + tolerance, + violations); + } + + private static void Minimum( + string metric, + double expected, + double actual, + double tolerance, + ICollection violations) + { + if (actual < expected - tolerance) + violations.Add($"{metric} decreased: {expected:F6} -> {actual:F6}."); + } + + private static void Maximum( + string metric, + double expected, + double actual, + double tolerance, + ICollection violations) + { + if (actual > expected + tolerance) + violations.Add($"{metric} increased: {expected:F6} -> {actual:F6}."); + } +} diff --git a/tools/AgentMemory.Cli/Program.cs b/tools/AgentMemory.Cli/Program.cs index 87f6843b..a400e6fd 100644 --- a/tools/AgentMemory.Cli/Program.cs +++ b/tools/AgentMemory.Cli/Program.cs @@ -42,6 +42,23 @@ { try { + if (string.Equals(cli.Subcommand, "baseline", StringComparison.OrdinalIgnoreCase)) + { + return await new AgentMemory.Cli.Commands.PerfBaselineCommand(Console.Out).ExecuteAsync( + cli.HasFlag("update") ? cli.Get("update") ?? bool.TrueString : null, + cli.Get("report"), + cli.Get("output")); + } + + if (string.Equals(cli.Subcommand, "gate", StringComparison.OrdinalIgnoreCase)) + { + return await new AgentMemory.Cli.Commands.PerfGateCommand(Console.Out).ExecuteAsync( + cli.Get("baseline"), + cli.Get("report"), + cli.Get("allow-counter-change"), + cli.Get("counter-change-justification")); + } + if (string.Equals(cli.Subcommand, "ab", StringComparison.OrdinalIgnoreCase)) { return await new AgentMemory.Cli.Commands.PerfAbCommand(Console.Out).ExecuteAsync( @@ -59,7 +76,7 @@ !string.Equals(cli.Subcommand, "run", StringComparison.OrdinalIgnoreCase)) { Console.Error.WriteLine( - $"error: unknown perf subcommand '{cli.Subcommand}'. Use 'run' or 'ab'."); + $"error: unknown perf subcommand '{cli.Subcommand}'. Use 'run', 'ab', 'baseline', or 'gate'."); return 1; }