diff --git a/docs/performance/README.md b/docs/performance/README.md index 93d2942a..425e0fe4 100644 --- a/docs/performance/README.md +++ b/docs/performance/README.md @@ -59,6 +59,16 @@ consecutive runs of the published baseline produced identical values for all 27 **These are safe to reason about, budget against, and compare across versions.** They are what [baseline-1.3.0.md](baseline-1.3.0.md) leads with. +### Quality guards — deterministic and enforced + +Every performance run also executes 19 judged retrieval cases and 20 judged extraction cases. Retrieval +is scored with Recall@K, MRR, and forbidden-result checks; extraction is scored with precision and +recall per memory kind plus false positives on six turns that should teach the system nothing. + +Five fresh-container runs produced identical values for every guarded metric, so the committed +tolerance is the observed variance: **zero**. The gate is on by default and returns a non-zero exit when +a score falls below `eng/perf/baselines/quality.json` or a forbidden retrieval appears. + ### Timings — indicative only Latency figures published here come from a local container with in-process stand-ins for the embedding @@ -98,7 +108,9 @@ dotnet run --project tools/AgentMemory.Cli -- perf --label mine --latency remote ``` Each run writes a dated directory containing a manifest with the full environment fingerprint, an -append-only trace log, per-iteration samples, a machine-readable summary, and a rendered report. +append-only trace log, per-iteration samples, a machine-readable summary, and a rendered report. The +quality gate is on by default; `--quality-gate=false` is available for diagnostic runs, and the report +marks those scores as report-only. Run it at both latency settings. A change that improves only the `remote` shape is an ordering or overlap win; one that improves both removed work. @@ -106,9 +118,12 @@ overlap win; one that improves both removed work. ### Determinism The harness uses a deterministic embedding function and a scripted model, so counters are reproducible. -Both measured scenarios **self-assert**: the recall scenario fails the run if it retrieves fewer items -than the configured limits, and the ingestion scenario fails if extraction never ran. Both failures are -otherwise silent and would produce a confident, wrong number. +The judged quality fixtures had zero observed variance across five complete runs, which is why their +tolerance is zero rather than a guessed allowance. + +Both measured scenarios also **self-assert**: the recall scenario fails the run if it retrieves fewer +items than the configured limits, and the ingestion scenario fails if extraction never ran. Those +failures are otherwise silent and would produce a confident, wrong number. --- diff --git a/docs/performance/baseline-1.3.0.md b/docs/performance/baseline-1.3.0.md index 5300bbc3..c16c1f3e 100644 --- a/docs/performance/baseline-1.3.0.md +++ b/docs/performance/baseline-1.3.0.md @@ -55,6 +55,25 @@ Worth knowing before you tune anything: | Turn produces more memories | – | writes and embedding requests grow linearly | | Grow the graph | read cost grows with index behaviour (not yet characterised) | resolution cost grows | +### Quality guard applied beside this cost baseline + +The cost counters are only accepted when deterministic quality remains at this committed baseline: + +| Guard | Baseline | +|---|---:| +| Retrieval Recall@K / MRR | 1.000 / 1.000 | +| Retrieval cases with forbidden results | 0 of 19 | +| Entity precision / recall | 1.000 / 1.000 | +| Fact precision / recall | 1.000 / 1.000 | +| Preference precision / recall | 1.000 / 1.000 | +| Extraction false positives on learn-nothing turns | 0 of 6 (20 total cases) | + +Every value above was identical across five fresh-container runs: maximum observed variance **0.000**. +The derived tolerance is therefore **zero**, recorded in +`eng/perf/baselines/quality.json`. The `perf` command gates by default and exits non-zero on a drop or +forbidden retrieval. This guard is about deterministic pipeline behavior; it does not claim to score +the quality of a live model's prose. + --- ## 2. Where the time goes — proportions, not expectations diff --git a/eng/perf/baselines/quality.json b/eng/perf/baselines/quality.json new file mode 100644 index 00000000..981477b6 --- /dev/null +++ b/eng/perf/baselines/quality.json @@ -0,0 +1,30 @@ +{ + "schemaVersion": 1, + "tolerance": 0.0, + "evidence": { + "measuredAtUtc": "2026-07-25", + "runs": 5, + "profile": "hermetic-S-zero", + "iterationsPerRun": 1, + "judgedFixturesRunInFull": true, + "maxObservedVariance": 0.0, + "toleranceDerivation": "Every guarded metric was identical across five fresh containers, so tolerance equals observed variance: zero." + }, + "retrieval": { + "recallAtK": 1.0, + "mrr": 1.0, + "cases": 19, + "maxCasesWithViolations": 0 + }, + "extraction": { + "entityPrecision": 1.0, + "entityRecall": 1.0, + "factPrecision": 1.0, + "factRecall": 1.0, + "preferencePrecision": 1.0, + "preferenceRecall": 1.0, + "cases": 20, + "expectNothingCases": 6, + "maxFalsePositiveRate": 0.0 + } +} diff --git a/tests/AgentMemory.Tests.Unit/Cli/QualityGateTests.cs b/tests/AgentMemory.Tests.Unit/Cli/QualityGateTests.cs new file mode 100644 index 00000000..1bddba50 --- /dev/null +++ b/tests/AgentMemory.Tests.Unit/Cli/QualityGateTests.cs @@ -0,0 +1,140 @@ +using AgentMemory.Cli.Perf; +using FluentAssertions; + +namespace AgentMemory.Tests.Unit.Cli; + +public sealed class QualityGateTests +{ + [Fact] + public void Evaluate_ExactBaseline_PassesAtZeroTolerance() + { + var result = QualityGate.Evaluate(Baseline(), Retrieval(), Extraction(), "quality.json"); + + result.Passed.Should().BeTrue(); + result.Tolerance.Should().Be(0); + result.Violations.Should().BeEmpty(); + } + + [Fact] + public void Evaluate_RetrievalBelowBaseline_Fails() + { + var result = QualityGate.Evaluate( + Baseline(), + Retrieval(recallAtK: 18d / 19d, mrr: 18d / 19d), + Extraction(), + "quality.json"); + + result.Passed.Should().BeFalse(); + result.Violations.Should() + .Contain(v => v.Contains("retrieval.recallAtK", StringComparison.Ordinal)) + .And.Contain(v => v.Contains("retrieval.mrr", StringComparison.Ordinal)); + } + + [Fact] + public void Evaluate_ForbiddenRetrieval_FailsEvenWhenScoresMeetBaseline() + { + var result = QualityGate.Evaluate( + Baseline(), + Retrieval(casesWithViolations: 1), + Extraction(), + "quality.json"); + + result.Passed.Should().BeFalse(); + result.Violations.Should() + .ContainSingle(v => v.Contains("forbidden retrieval", StringComparison.OrdinalIgnoreCase)); + } + + [Fact] + public void Evaluate_ExtractionBelowBaseline_Fails() + { + var result = QualityGate.Evaluate( + Baseline(), + Retrieval(), + Extraction(factRecall: 0.95), + "quality.json"); + + result.Passed.Should().BeFalse(); + result.Violations.Should() + .ContainSingle(v => v.Contains("extraction.factRecall", StringComparison.Ordinal)); + } + + [Fact] + public void Evaluate_ExtractionFalsePositiveIncrease_Fails() + { + var result = QualityGate.Evaluate( + Baseline(), + Retrieval(), + Extraction(falsePositives: 1), + "quality.json"); + + result.Passed.Should().BeFalse(); + result.Violations.Should() + .Contain(v => v.Contains("extraction.falsePositiveRate", StringComparison.Ordinal)); + } + + [Theory] + [InlineData(18, 20, 6)] + [InlineData(19, 19, 6)] + [InlineData(19, 20, 5)] + public void Evaluate_FixtureCaseCountChanged_Fails( + int retrievalCases, + int extractionCases, + int expectNothingCases) + { + var result = QualityGate.Evaluate( + Baseline(), + Retrieval(cases: retrievalCases), + Extraction(cases: extractionCases, expectNothingCases: expectNothingCases), + "quality.json"); + + result.Passed.Should().BeFalse(); + result.Violations.Should().NotBeEmpty(); + } + + private static QualityBaseline Baseline() => new( + SchemaVersion: 1, + Tolerance: 0, + Retrieval: new RetrievalQualityBaseline( + RecallAtK: 1, + Mrr: 1, + Cases: 19, + MaxCasesWithViolations: 0), + Extraction: new ExtractionQualityBaseline( + EntityPrecision: 1, + EntityRecall: 1, + FactPrecision: 1, + FactRecall: 1, + PreferencePrecision: 1, + PreferenceRecall: 1, + Cases: 20, + ExpectNothingCases: 6, + MaxFalsePositiveRate: 0)); + + private static QualityResult Retrieval( + double recallAtK = 1, + double mrr = 1, + int cases = 19, + int casesWithViolations = 0) => new( + RecallAtK: recallAtK, + Mrr: mrr, + Cases: cases, + CasesWithViolations: casesWithViolations, + RecallByCategory: new Dictionary(), + CaseResults: []); + + private static ExtractionQualityResult Extraction( + double factRecall = 1, + int cases = 20, + int expectNothingCases = 6, + int falsePositives = 0) => new( + EntityPrecision: 1, + EntityRecall: 1, + FactPrecision: 1, + FactRecall: factRecall, + PreferencePrecision: 1, + PreferenceRecall: 1, + Cases: cases, + ExpectNothingCases: expectNothingCases, + FalsePositives: falsePositives, + CaseResults: []); +} diff --git a/tools/AgentMemory.Cli/AgentMemory.Cli.csproj b/tools/AgentMemory.Cli/AgentMemory.Cli.csproj index 385b1c63..1073d96d 100644 --- a/tools/AgentMemory.Cli/AgentMemory.Cli.csproj +++ b/tools/AgentMemory.Cli/AgentMemory.Cli.csproj @@ -34,7 +34,9 @@ retrieval" means, so it must version with the assembly that scores against it and be reviewed in the diff like source. A loose file could differ per machine and silently invalidate every comparison. --> - + + diff --git a/tools/AgentMemory.Cli/CliArgs.cs b/tools/AgentMemory.Cli/CliArgs.cs index cb036bc0..a91fa876 100644 --- a/tools/AgentMemory.Cli/CliArgs.cs +++ b/tools/AgentMemory.Cli/CliArgs.cs @@ -93,11 +93,13 @@ Run deterministic memory-layer quality/performance scenarios and write a JSON report under artifacts/evaluation by default. perf [--label ] [--scenarios ] [--iterations ] [--warmup ] [--latency ] [--embedding-dimensions ] [--output ] + [--quality-gate ] Measure a complete agent TURN: database round trips, embedding requests, model calls, and per-stage timing. Provisions its own Neo4j via Testcontainers (Docker required) with deterministic embeddings and a scripted model, so counters are reproducible. - Writes a dated run directory under performance/runs by default. + Quality gate defaults on. Writes a dated run directory under + artifacts/perf by default. decay [--owner ] Decay-prune memories: soft-invalidate by default (kept + recoverable; set MemoryDecay:NonDestructive=false to hard-delete). Owner-scoped, or global. schema-parity [--upstream-version ] diff --git a/tools/AgentMemory.Cli/Commands/PerfCommand.cs b/tools/AgentMemory.Cli/Commands/PerfCommand.cs index 1c57e108..a5fb5dbc 100644 --- a/tools/AgentMemory.Cli/Commands/PerfCommand.cs +++ b/tools/AgentMemory.Cli/Commands/PerfCommand.cs @@ -42,12 +42,15 @@ public async Task ExecuteAsync( string? dimensionsValue, string? latency, string? outputRoot, + string? qualityGateValue, CancellationToken cancellationToken = default) { var runLabel = Sanitize(label) ?? "baseline"; var iterations = ParsePositive(iterationsValue, 10, "iterations"); var warmup = ParseNonNegative(warmupValue, 3, "warmup"); var dimensions = ParsePositive(dimensionsValue, 384, "embedding-dimensions"); + var qualityGateEnabled = ParseDefaultTrue(qualityGateValue, "quality-gate"); + var qualityBaseline = qualityGateEnabled ? QualityGate.LoadBaseline() : null; var (embeddingLatency, modelLatency) = ResolveLatency(latency); @@ -83,8 +86,14 @@ await File.WriteAllTextAsync( var runStopwatch = Stopwatch.StartNew(); using var collector = new PerfCollector(trace); + // The extraction fixture and cost scenarios supply input-keyed model answers, so they must be + // assembled before the profile that wires the chat client. + var extractionFixture = ExtractionQualityFixture.Load(); + var scriptedRules = extractionFixture.ScriptedRules().Concat(PerfScenarios.ScriptedRules).ToList(); + await using var profile = await HermeticProfile - .StartAsync(dimensions, embeddingLatency, modelLatency, _output, cancellationToken) + .StartAsync(dimensions, embeddingLatency, modelLatency, _output, + scriptedRules, cancellationToken) .ConfigureAwait(false); await PerfFixture.SeedAsync(profile, _output, cancellationToken).ConfigureAwait(false); @@ -118,24 +127,37 @@ await RunScenarioAsync(scenario, profile, provider, collector, warmup, iteration _output.WriteLine("perf: scoring retrieval quality…"); var qualityResult = await quality.EvaluateAsync(cancellationToken).ConfigureAwait(false); + _output.WriteLine("perf: scoring extraction quality…"); + var extractionQuality = await new ExtractionQualityEvaluator(extractionFixture, profile.Services, profile.Driver) + .EvaluateAsync(cancellationToken).ConfigureAwait(false); + var qualityGate = qualityBaseline is null + ? QualityGateResult.Disabled() + : QualityGate.Evaluate(qualityBaseline, qualityResult, extractionQuality); + runStopwatch.Stop(); trace.RunEnd(collector.Records.Count, runStopwatch.Elapsed.TotalMilliseconds); var measured = collector.Records.Where(r => r.Phase == "measure").ToList(); await WriteSamplesAsync(runDir, measured, cancellationToken).ConfigureAwait(false); - var summary = BuildSummary(manifest, measured, qualityResult); + var summary = BuildSummary(manifest, measured, qualityGate, qualityResult, extractionQuality); await File.WriteAllTextAsync( Path.Combine(runDir, "summary.json"), JsonSerializer.Serialize(summary, Json), cancellationToken) .ConfigureAwait(false); - var report = RenderReport(runId, measured, qualityResult); + var report = RenderReport(runId, measured, qualityGate, qualityResult, extractionQuality); await File.WriteAllTextAsync(Path.Combine(runDir, "report.md"), report, cancellationToken) .ConfigureAwait(false); _output.WriteLine(); _output.Write(report); _output.WriteLine($"perf: wrote {runDir}"); - return 0; + if (qualityGate.Passed) + return 0; + + _output.WriteLine($"error: quality gate failed with {qualityGate.Violations.Count} violation(s)."); + foreach (var violation in qualityGate.Violations) + _output.WriteLine($"error: {violation}"); + return 1; } private static async Task RunScenarioAsync( @@ -230,9 +252,34 @@ private static object BuildManifest( } private static object BuildSummary( - object manifest, IReadOnlyList measured, QualityResult quality) => new + object manifest, IReadOnlyList measured, QualityGateResult qualityGate, + QualityResult quality, + ExtractionQualityResult extraction) => new { manifest, + qualityGate = new + { + enabled = qualityGate.Enabled, + passed = qualityGate.Passed, + baseline = qualityGate.BaselinePath, + tolerance = qualityGate.Tolerance, + violations = qualityGate.Violations, + }, + extractionQuality = new + { + entityPrecision = extraction.EntityPrecision, + entityRecall = extraction.EntityRecall, + factPrecision = extraction.FactPrecision, + factRecall = extraction.FactRecall, + preferencePrecision = extraction.PreferencePrecision, + preferenceRecall = extraction.PreferenceRecall, + cases = extraction.Cases, + expectNothingCases = extraction.ExpectNothingCases, + falsePositives = extraction.FalsePositives, + falsePositiveRate = extraction.FalsePositiveRate, + clean = extraction.Clean, + caseResults = extraction.CaseResults, + }, quality = new { recallAtK = quality.RecallAtK, @@ -346,7 +393,9 @@ await File.WriteAllLinesAsync(Path.Combine(runDir, "samples.ndjson"), lines, can } private static string RenderReport( - string runId, IReadOnlyList measured, QualityResult quality) + string runId, IReadOnlyList measured, QualityGateResult qualityGate, + QualityResult quality, + ExtractionQualityResult extraction) { var sb = new StringBuilder(); sb.AppendLine(CultureInfo.InvariantCulture, $"# Performance run — {runId}"); @@ -354,6 +403,26 @@ private static string RenderReport( // Quality first, deliberately. A speed number read without the quality number beside it is how // "we got 96% faster" ships without the second sentence. + sb.AppendLine("## Quality gate"); + sb.AppendLine(); + if (!qualityGate.Enabled) + { + sb.AppendLine("**DISABLED** by `--quality-gate=false`; scores below are report-only."); + } + else + { + sb.AppendLine(CultureInfo.InvariantCulture, + $"**{(qualityGate.Passed ? "PASS ✅" : "FAIL ❌")}** · baseline " + + $"`{qualityGate.BaselinePath}` · tolerance {qualityGate.Tolerance:F6}"); + if (!qualityGate.Passed) + { + sb.AppendLine(); + foreach (var violation in qualityGate.Violations) + sb.AppendLine($"- {violation}"); + } + } + sb.AppendLine(); + sb.AppendLine("## Retrieval quality (deterministic — no model involved)"); sb.AppendLine(); sb.AppendLine(CultureInfo.InvariantCulture, @@ -385,6 +454,38 @@ private static string RenderReport( sb.AppendLine(); } + sb.AppendLine("## Extraction quality (deterministic — scripted model, no judging model)"); + sb.AppendLine(); + sb.AppendLine(CultureInfo.InvariantCulture, + $"{extraction.Cases} judged cases · **false-positive rate {extraction.FalsePositiveRate:P0}** " + + $"({extraction.FalsePositives}/{extraction.ExpectNothingCases} cases that should learn nothing) · " + + $"{(extraction.Clean ? "✅ clean" : "⚠️ **see failures below**")}"); + sb.AppendLine(); + sb.AppendLine("| Kind | Precision | Recall |"); + sb.AppendLine("|---|---:|---:|"); + sb.AppendLine(CultureInfo.InvariantCulture, + $"| entities | {extraction.EntityPrecision:F3} | {extraction.EntityRecall:F3} |"); + sb.AppendLine(CultureInfo.InvariantCulture, + $"| facts | {extraction.FactPrecision:F3} | {extraction.FactRecall:F3} |"); + sb.AppendLine(CultureInfo.InvariantCulture, + $"| preferences | {extraction.PreferencePrecision:F3} | {extraction.PreferenceRecall:F3} |"); + sb.AppendLine(); + + var dirty = extraction.CaseResults.Where(c => !c.Clean).ToList(); + if (dirty.Count > 0) + { + sb.AppendLine("| Case | Missing | Unexpected | False positive |"); + sb.AppendLine("|---|---|---|---|"); + foreach (var c in dirty) + { + sb.AppendLine(CultureInfo.InvariantCulture, + $"| `{c.CaseId}` | {(c.Missing.Count == 0 ? "–" : string.Join(", ", c.Missing))} " + + $"| {(c.Unexpected.Count == 0 ? "–" : string.Join(", ", c.Unexpected))} " + + $"| {(c.FalsePositive ? "**YES**" : "–")} |"); + } + sb.AppendLine(); + } + sb.AppendLine("---"); sb.AppendLine(); @@ -531,6 +632,17 @@ private static int ParseNonNegative(string? value, int fallback, string name) return parsed; } + private static bool ParseDefaultTrue(string? value, string name) + { + if (string.IsNullOrWhiteSpace(value)) return true; + return value.ToLowerInvariant() switch + { + "true" or "1" or "on" or "yes" => true, + "false" or "0" or "off" or "no" => false, + _ => throw new ArgumentException($"--{name} must be true or false."), + }; + } + /// Keeps the label safe for a directory name without silently mangling it. private static string? Sanitize(string? label) { diff --git a/tools/AgentMemory.Cli/Perf/ExtractionQualityEvaluator.cs b/tools/AgentMemory.Cli/Perf/ExtractionQualityEvaluator.cs new file mode 100644 index 00000000..a8aa6557 --- /dev/null +++ b/tools/AgentMemory.Cli/Perf/ExtractionQualityEvaluator.cs @@ -0,0 +1,242 @@ +using AgentMemory.Abstractions.Domain; +using AgentMemory.Abstractions.Services; +using Microsoft.Extensions.DependencyInjection; +using Neo4j.Driver; + +namespace AgentMemory.Cli.Perf; + +/// Precision and recall for one memory kind. +public sealed record KindScore(int Expected, int Produced, int Matched) +{ + /// Of what was produced, how much was wanted. 1.0 when nothing was produced and none was wanted. + public double Precision => Produced == 0 ? (Expected == 0 ? 1.0 : 0.0) : (double)Matched / Produced; + + /// Of what was wanted, how much was produced. 1.0 when nothing was wanted. + public double Recall => Expected == 0 ? 1.0 : (double)Matched / Expected; +} + +/// Scores for one judged extraction case. +public sealed record ExtractionCaseResult( + string CaseId, + string Category, + bool ExpectNothing, + KindScore Entities, + KindScore Facts, + KindScore Preferences, + bool FalsePositive, + IReadOnlyList Missing, + IReadOnlyList Unexpected) +{ + public bool Clean => Missing.Count == 0 && Unexpected.Count == 0 && !FalsePositive; +} + +/// Aggregate extraction-quality scores for a run. +public sealed record ExtractionQualityResult( + double EntityPrecision, double EntityRecall, + double FactPrecision, double FactRecall, + double PreferencePrecision, double PreferenceRecall, + int Cases, + int ExpectNothingCases, + int FalsePositives, + IReadOnlyList CaseResults) +{ + /// False-positive rate over the cases where the correct behaviour is to learn nothing. + public double FalsePositiveRate => + ExpectNothingCases == 0 ? 0.0 : (double)FalsePositives / ExpectNothingCases; + + public bool Clean => CaseResults.All(c => c.Clean); +} + +/// +/// Runs the judged extraction fixture and scores it. Deterministic and free — the model is scripted +/// and matching is normalized string comparison. +/// +public sealed class ExtractionQualityEvaluator +{ + private readonly ExtractionQualityFixture _fixture; + private readonly IServiceProvider _services; + private readonly IDriver _driver; + + public ExtractionQualityEvaluator( + ExtractionQualityFixture fixture, IServiceProvider services, IDriver driver) + { + _fixture = fixture; + _services = services; + _driver = driver; + } + + public async Task EvaluateAsync(CancellationToken cancellationToken) + { + var memory = _services.GetRequiredService(); + var clock = _services.GetRequiredService(); + var results = new List(); + + foreach (var testCase in _fixture.Cases) + { + // Own owner and session per case. Extraction deduplicates and resolves against existing + // memory, so sharing an owner would let case N's facts merge into case N-1's and the score + // would depend on fixture ORDER — which is exactly the kind of hidden coupling that makes a + // quality number untrustworthy. + var owner = $"{_fixture.OwnerPrefix}-{testCase.Id}"; + var sessionId = $"{owner}-session"; + var now = clock.UtcNow; + + var messages = testCase.Messages.Select((m, i) => new Message + { + MessageId = $"{owner}-msg-{i}", + SessionId = sessionId, + ConversationId = $"{owner}-conv", + Role = m.Role, + Content = m.Content, + TimestampUtc = now.AddSeconds(i), + }).ToList(); + + await memory.ExtractAndPersistAsync(new ExtractionRequest + { + Messages = messages, + SessionId = sessionId, + UserId = owner, + }, cancellationToken).ConfigureAwait(false); + + var learned = await ReadLearnedAsync(owner, cancellationToken).ConfigureAwait(false); + results.Add(Score(testCase, learned)); + } + + return Aggregate(results); + } + + /// What the system actually ended up knowing for this case's owner. + /// + /// + /// Read back from the graph, deliberately, not from ExtractionResult. That type's + /// Entities/Facts/Preferences are populated from the pipeline's Raw* + /// collections — everything the extractor returned, before confidence filtering, entity + /// resolution and dedup. Scoring against it would mean a sub-threshold item counted as "learned" + /// when the pipeline correctly discarded it, and would make the whole persistence half of the + /// pipeline invisible to this guard. + /// + /// + /// That distinction is the entire point here: ranks 2, 4 and 8 change what the system learns, + /// not what an extractor proposes. + /// + /// + private async Task ReadLearnedAsync(string owner, CancellationToken cancellationToken) + { + await using var session = _driver.AsyncSession(); + var cursor = await session.RunAsync( + @"MATCH (n) + WHERE n.owner_id = $owner AND (n:Entity OR n:Fact OR n:Preference) + RETURN labels(n)[0] AS label, + n.name AS name, + n.subject AS subject, + n.predicate AS predicate, + n.object AS object, + n.preference AS preference", + new { owner }).ConfigureAwait(false); + + var entities = new List(); + var facts = new List(); + var preferences = new List(); + + var records = await cursor.ToListAsync(cancellationToken).ConfigureAwait(false); + foreach (var record in records) + { + switch (record["label"]?.ToString()) + { + case "Entity": + if (record["name"]?.ToString() is { Length: > 0 } name) entities.Add(Normalize(name)); + break; + case "Fact": + facts.Add(Normalize( + $"{record["subject"]}|{record["predicate"]}|{record["object"]}")); + break; + case "Preference": + if (record["preference"]?.ToString() is { Length: > 0 } pref) preferences.Add(Normalize(pref)); + break; + } + } + + return new LearnedMemory(entities, facts, preferences); + } + + private sealed record LearnedMemory( + List Entities, List Facts, List Preferences); + + /// + /// Scores one case against what the system actually learned. See + /// for why that is read from the graph rather than taken from ExtractionResult. + /// + private static ExtractionCaseResult Score(ExtractionCase testCase, LearnedMemory learned) + { + var producedEntities = learned.Entities; + var producedFacts = learned.Facts; + var producedPrefs = learned.Preferences; + + var expectedEntities = testCase.ExpectEntities.Select(Normalize).ToList(); + var expectedFacts = testCase.ExpectFacts.Select(f => Normalize(f.ToString())).ToList(); + var expectedPrefs = testCase.ExpectPreferences.Select(Normalize).ToList(); + + var entities = ScoreKind(expectedEntities, producedEntities); + var facts = ScoreKind(expectedFacts, producedFacts); + var prefs = ScoreKind(expectedPrefs, producedPrefs); + + var missing = Missing(expectedEntities, producedEntities, "entity") + .Concat(Missing(expectedFacts, producedFacts, "fact")) + .Concat(Missing(expectedPrefs, producedPrefs, "preference")) + .ToList(); + + var unexpected = Missing(producedEntities, expectedEntities, "entity") + .Concat(Missing(producedFacts, expectedFacts, "fact")) + .Concat(Missing(producedPrefs, expectedPrefs, "preference")) + .ToList(); + + // A false positive is learning ANYTHING on a turn that should have taught us nothing. This is + // the number a salience gate must not move: skipping turns is only safe if the turns skipped + // were genuinely empty. + var producedAnything = producedEntities.Count + producedFacts.Count + producedPrefs.Count > 0; + var falsePositive = testCase.ExpectNothing && producedAnything; + + return new ExtractionCaseResult( + testCase.Id, testCase.Category, testCase.ExpectNothing, + entities, facts, prefs, falsePositive, missing, unexpected); + } + + private static KindScore ScoreKind(List expected, List produced) + { + var producedSet = produced.ToHashSet(StringComparer.Ordinal); + var matched = expected.Count(producedSet.Contains); + return new KindScore(expected.Count, produced.Count, matched); + } + + private static IEnumerable Missing(List wanted, List got, string kind) + { + var gotSet = got.ToHashSet(StringComparer.Ordinal); + return wanted.Where(w => !gotSet.Contains(w)).Select(w => $"{kind}:{w}"); + } + + /// + /// Case- and whitespace-insensitive comparison. Deliberately simple and documented: a matcher that + /// is clever is a matcher whose verdicts cannot be predicted from the fixture, and a quality gate + /// whose result cannot be predicted is not a gate. + /// + private static string Normalize(string value) => + string.Join(' ', value.ToLowerInvariant().Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries)); + + private static ExtractionQualityResult Aggregate(List results) + { + double Avg(Func selector) => + results.Count == 0 ? 0 : results.Average(selector); + + return new ExtractionQualityResult( + EntityPrecision: Avg(r => r.Entities.Precision), + EntityRecall: Avg(r => r.Entities.Recall), + FactPrecision: Avg(r => r.Facts.Precision), + FactRecall: Avg(r => r.Facts.Recall), + PreferencePrecision: Avg(r => r.Preferences.Precision), + PreferenceRecall: Avg(r => r.Preferences.Recall), + Cases: results.Count, + ExpectNothingCases: results.Count(r => r.ExpectNothing), + FalsePositives: results.Count(r => r.FalsePositive), + CaseResults: results); + } +} diff --git a/tools/AgentMemory.Cli/Perf/ExtractionQualityFixture.cs b/tools/AgentMemory.Cli/Perf/ExtractionQualityFixture.cs new file mode 100644 index 00000000..7b726b85 --- /dev/null +++ b/tools/AgentMemory.Cli/Perf/ExtractionQualityFixture.cs @@ -0,0 +1,107 @@ +using System.Reflection; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace AgentMemory.Cli.Perf; + +/// +/// The judged extraction-quality fixture: turns, the answer the model is scripted to give for each, +/// and what the pipeline should end up having learned. +/// +/// +/// +/// The write-path counterpart to . Ranks 2, 4 and 8 on the optimization +/// matrix all change extraction — skipping turns, merging four prompts into one, batching across a +/// window — and every one of them could quietly learn less. Nothing else detects that. +/// +/// +/// What this does and does not measure. The model is scripted, so this measures the +/// pipeline: given what the extractor returned, did resolution, confidence filtering and +/// persistence do the right thing. It deliberately does not measure whether a prompt is good +/// at reading a conversation — that is model-dependent, needs a real LLM, and belongs to the optional +/// comparative benchmark track. The distinction matters most for rank 4, which changes the prompt +/// itself and therefore needs both. +/// +/// +public sealed class ExtractionQualityFixture +{ + private static readonly JsonSerializerOptions Json = new() + { + PropertyNameCaseInsensitive = true, + ReadCommentHandling = JsonCommentHandling.Skip, + }; + + [JsonPropertyName("schemaVersion")] public int SchemaVersion { get; init; } + + /// Owner id prefix. Each case gets its own owner — see . + [JsonPropertyName("ownerPrefix")] public string OwnerPrefix { get; init; } = "perf-xq"; + + [JsonPropertyName("cases")] public List Cases { get; init; } = new(); + + public static ExtractionQualityFixture Load() + { + var assembly = Assembly.GetExecutingAssembly(); + var name = assembly.GetManifestResourceNames() + .Single(n => n.EndsWith("extraction-quality.json", StringComparison.Ordinal)); + using var stream = assembly.GetManifestResourceStream(name)!; + return JsonSerializer.Deserialize(stream, Json) + ?? throw new InvalidOperationException("extraction-quality.json deserialized to null."); + } + + /// + /// The scripted-response rules this fixture needs, for wiring into the chat client. + /// + public IReadOnlyList ScriptedRules() => + Cases.Select(c => new ScriptedChatClient.Rule( + c.MatchOn, + JsonSerializer.Serialize(c.ScriptedResponse, Json))) + .ToList(); +} + +/// One judged extraction case. +public sealed class ExtractionCase +{ + [JsonPropertyName("id")] public string Id { get; init; } = ""; + [JsonPropertyName("category")] public string Category { get; init; } = ""; + + /// + /// A distinctive phrase from this case's conversation. The scripted chat client uses it to return + /// this case's answer rather than another's. + /// + [JsonPropertyName("matchOn")] public string MatchOn { get; init; } = ""; + + [JsonPropertyName("messages")] public List Messages { get; init; } = new(); + + /// What the model is scripted to reply. Shape must match what the LLM extractors parse. + [JsonPropertyName("scriptedResponse")] public JsonElement ScriptedResponse { get; init; } + + [JsonPropertyName("expectEntities")] public List ExpectEntities { get; init; } = new(); + [JsonPropertyName("expectFacts")] public List ExpectFacts { get; init; } = new(); + [JsonPropertyName("expectPreferences")] public List ExpectPreferences { get; init; } = new(); + [JsonPropertyName("expectRelationships")] public int? ExpectRelationships { get; init; } + + /// + /// True when the correct behaviour is to learn nothing at all — an acknowledgement, a command, a + /// pure transformation. These are the cases a salience gate (rank 2) must not get wrong in the + /// other direction, and they are why a false-positive rate is reported separately. + /// + [JsonPropertyName("expectNothing")] public bool ExpectNothing { get; init; } + + /// Free-text rationale, for cases whose expectation is not self-evident. + [JsonPropertyName("note")] public string? Note { get; init; } +} + +public sealed class ExtractionMessage +{ + [JsonPropertyName("role")] public string Role { get; init; } = "user"; + [JsonPropertyName("content")] public string Content { get; init; } = ""; +} + +public sealed class ExpectedFact +{ + [JsonPropertyName("subject")] public string Subject { get; init; } = ""; + [JsonPropertyName("predicate")] public string Predicate { get; init; } = ""; + [JsonPropertyName("object")] public string Object { get; init; } = ""; + + public override string ToString() => $"{Subject}|{Predicate}|{Object}"; +} diff --git a/tools/AgentMemory.Cli/Perf/Fixtures/extraction-quality.json b/tools/AgentMemory.Cli/Perf/Fixtures/extraction-quality.json new file mode 100644 index 00000000..42383844 --- /dev/null +++ b/tools/AgentMemory.Cli/Perf/Fixtures/extraction-quality.json @@ -0,0 +1,289 @@ +{ + "schemaVersion": 1, + "description": "Judged extraction-quality fixture. Each case is a turn plus the response the model is scripted to give for it, plus what the pipeline should end up having learned. Scored with precision and recall per memory kind, and a false-positive rate over the cases where the correct behaviour is to learn NOTHING. Deterministic: the model is scripted, matching is normalized string comparison, no LLM is involved in scoring.", + "ownerPrefix": "perf-xq", + + "cases": [ + { + "id": "XQ-001", + "category": "employment", + "matchOn": "joined the platform team at Northwind Traders", + "messages": [{ "role": "user", "content": "I just joined the platform team at Northwind Traders as a staff engineer." }], + "scriptedResponse": { + "entities": [{ "name": "Northwind Traders", "type": "ORGANIZATION", "confidence": 0.93 }], + "facts": [{ "subject": "user", "predicate": "works_at", "object": "Northwind Traders", "confidence": 0.92 }], + "preferences": [], + "relations": [] + }, + "expectEntities": ["Northwind Traders"], + "expectFacts": [{ "subject": "user", "predicate": "works_at", "object": "Northwind Traders" }], + "expectPreferences": [] + }, + { + "id": "XQ-002", + "category": "preference", + "matchOn": "send me short written summaries", + "messages": [{ "role": "user", "content": "Please send me short written summaries, not long documents." }], + "scriptedResponse": { + "entities": [], + "facts": [], + "preferences": [{ "category": "communication", "preference": "short written summaries over long documents", "confidence": 0.9 }], + "relations": [] + }, + "expectEntities": [], + "expectFacts": [], + "expectPreferences": ["short written summaries over long documents"] + }, + { + "id": "XQ-003", + "category": "mixed", + "matchOn": "allergic to shellfish and I cook on Sundays", + "messages": [{ "role": "user", "content": "I'm allergic to shellfish and I cook on Sundays." }], + "scriptedResponse": { + "entities": [{ "name": "shellfish", "type": "CONCEPT", "confidence": 0.88 }], + "facts": [{ "subject": "user", "predicate": "allergic_to", "object": "shellfish", "confidence": 0.94 }], + "preferences": [{ "category": "lifestyle", "preference": "cooks on Sundays", "confidence": 0.82 }], + "relations": [] + }, + "expectEntities": ["shellfish"], + "expectFacts": [{ "subject": "user", "predicate": "allergic_to", "object": "shellfish" }], + "expectPreferences": ["cooks on Sundays"] + }, + { + "id": "XQ-004", + "category": "relationship", + "matchOn": "Priya is my manager at Northwind", + "messages": [{ "role": "user", "content": "Priya is my manager at Northwind." }], + "scriptedResponse": { + "entities": [{ "name": "Priya", "type": "PERSON", "confidence": 0.91 }], + "facts": [{ "subject": "Priya", "predicate": "manages", "object": "user", "confidence": 0.9 }], + "preferences": [], + "relations": [{ "source": "Priya", "target": "user", "relationType": "MANAGES", "confidence": 0.88 }] + }, + "expectEntities": ["Priya"], + "expectFacts": [{ "subject": "Priya", "predicate": "manages", "object": "user" }], + "expectPreferences": [], + "expectRelationships": 1 + }, + { + "id": "XQ-005", + "category": "employment", + "matchOn": "moved from Contoso to Fabrikam last March", + "messages": [{ "role": "user", "content": "I moved from Contoso to Fabrikam last March." }], + "scriptedResponse": { + "entities": [ + { "name": "Contoso", "type": "ORGANIZATION", "confidence": 0.9 }, + { "name": "Fabrikam", "type": "ORGANIZATION", "confidence": 0.9 } + ], + "facts": [{ "subject": "user", "predicate": "works_at", "object": "Fabrikam", "confidence": 0.89 }], + "preferences": [], + "relations": [] + }, + "expectEntities": ["Contoso", "Fabrikam"], + "expectFacts": [{ "subject": "user", "predicate": "works_at", "object": "Fabrikam" }], + "expectPreferences": [] + }, + { + "id": "XQ-006", + "category": "preference", + "matchOn": "prefer dark mode and keyboard shortcuts", + "messages": [{ "role": "user", "content": "I prefer dark mode and keyboard shortcuts everywhere." }], + "scriptedResponse": { + "entities": [], + "facts": [], + "preferences": [{ "category": "interface", "preference": "dark mode and keyboard shortcuts", "confidence": 0.87 }], + "relations": [] + }, + "expectEntities": [], + "expectFacts": [], + "expectPreferences": ["dark mode and keyboard shortcuts"] + }, + { + "id": "XQ-007", + "category": "location", + "matchOn": "based in Porto now, moved from Berlin", + "messages": [{ "role": "user", "content": "I'm based in Porto now, moved from Berlin." }], + "scriptedResponse": { + "entities": [ + { "name": "Porto", "type": "LOCATION", "confidence": 0.92 }, + { "name": "Berlin", "type": "LOCATION", "confidence": 0.9 } + ], + "facts": [{ "subject": "user", "predicate": "lives_in", "object": "Porto", "confidence": 0.91 }], + "preferences": [], + "relations": [] + }, + "expectEntities": ["Porto", "Berlin"], + "expectFacts": [{ "subject": "user", "predicate": "lives_in", "object": "Porto" }], + "expectPreferences": [] + }, + { + "id": "XQ-008", + "category": "skill", + "matchOn": "been writing Rust for about six years", + "messages": [{ "role": "user", "content": "I've been writing Rust for about six years." }], + "scriptedResponse": { + "entities": [{ "name": "Rust", "type": "TECHNOLOGY", "confidence": 0.93 }], + "facts": [{ "subject": "user", "predicate": "skilled_in", "object": "Rust", "confidence": 0.9 }], + "preferences": [], + "relations": [] + }, + "expectEntities": ["Rust"], + "expectFacts": [{ "subject": "user", "predicate": "skilled_in", "object": "Rust" }], + "expectPreferences": [] + }, + { + "id": "XQ-009", + "category": "multi-turn", + "matchOn": "quarterly planning offsite in Valencia", + "messages": [ + { "role": "user", "content": "We're doing the quarterly planning offsite in Valencia." }, + { "role": "assistant", "content": "Noted." }, + { "role": "user", "content": "Book me a window seat, I always take window seats." } + ], + "scriptedResponse": { + "entities": [{ "name": "Valencia", "type": "LOCATION", "confidence": 0.9 }], + "facts": [{ "subject": "team", "predicate": "meets_in", "object": "Valencia", "confidence": 0.85 }], + "preferences": [{ "category": "travel", "preference": "window seats when flying", "confidence": 0.88 }], + "relations": [] + }, + "expectEntities": ["Valencia"], + "expectFacts": [{ "subject": "team", "predicate": "meets_in", "object": "Valencia" }], + "expectPreferences": ["window seats when flying"] + }, + { + "id": "XQ-010", + "category": "confidence-filtering", + "matchOn": "might possibly be interested in woodworking", + "messages": [{ "role": "user", "content": "I might possibly be interested in woodworking, not sure yet." }], + "scriptedResponse": { + "entities": [], + "facts": [], + "preferences": [{ "category": "hobby", "preference": "possibly woodworking", "confidence": 0.2 }], + "relations": [] + }, + "expectEntities": [], + "expectFacts": [], + "expectPreferences": [], + "note": "Confidence 0.2 is below MinConfidenceThreshold (0.5). The pipeline must DROP it. If this case ever reports a learned preference, confidence filtering has regressed." + }, + + { + "id": "XQ-020", + "category": "nothing-to-learn", + "matchOn": "thanks that works", + "messages": [{ "role": "user", "content": "thanks, that works" }], + "scriptedResponse": { "entities": [], "facts": [], "preferences": [], "relations": [] }, + "expectNothing": true + }, + { + "id": "XQ-021", + "category": "nothing-to-learn", + "matchOn": "run the tests again please", + "messages": [{ "role": "user", "content": "run the tests again please" }], + "scriptedResponse": { "entities": [], "facts": [], "preferences": [], "relations": [] }, + "expectNothing": true + }, + { + "id": "XQ-022", + "category": "nothing-to-learn", + "matchOn": "translate that paragraph into German", + "messages": [{ "role": "user", "content": "translate that paragraph into German" }], + "scriptedResponse": { "entities": [], "facts": [], "preferences": [], "relations": [] }, + "expectNothing": true + }, + { + "id": "XQ-023", + "category": "nothing-to-learn", + "matchOn": "yes go ahead", + "messages": [{ "role": "user", "content": "yes, go ahead" }], + "scriptedResponse": { "entities": [], "facts": [], "preferences": [], "relations": [] }, + "expectNothing": true + }, + { + "id": "XQ-024", + "category": "nothing-to-learn", + "matchOn": "what was the second point again", + "messages": [{ "role": "user", "content": "what was the second point again?" }], + "scriptedResponse": { "entities": [], "facts": [], "preferences": [], "relations": [] }, + "expectNothing": true + }, + { + "id": "XQ-025", + "category": "nothing-to-learn", + "matchOn": "reformat this as a bulleted list", + "messages": [{ "role": "user", "content": "reformat this as a bulleted list" }], + "scriptedResponse": { "entities": [], "facts": [], "preferences": [], "relations": [] }, + "expectNothing": true + }, + + { + "id": "XQ-030", + "category": "partial-extraction", + "matchOn": "drink oat milk and my sister works at Adventure Works", + "messages": [{ "role": "user", "content": "I drink oat milk and my sister works at Adventure Works." }], + "scriptedResponse": { + "entities": [{ "name": "Adventure Works", "type": "ORGANIZATION", "confidence": 0.9 }], + "facts": [{ "subject": "sister", "predicate": "works_at", "object": "Adventure Works", "confidence": 0.88 }], + "preferences": [{ "category": "food", "preference": "drinks oat milk", "confidence": 0.85 }], + "relations": [] + }, + "expectEntities": ["Adventure Works"], + "expectFacts": [{ "subject": "sister", "predicate": "works_at", "object": "Adventure Works" }], + "expectPreferences": ["drinks oat milk"] + }, + { + "id": "XQ-031", + "category": "multi-entity", + "matchOn": "Tailspin and Wingtip both use our SDK", + "messages": [{ "role": "user", "content": "Tailspin and Wingtip both use our SDK." }], + "scriptedResponse": { + "entities": [ + { "name": "Tailspin", "type": "ORGANIZATION", "confidence": 0.9 }, + { "name": "Wingtip", "type": "ORGANIZATION", "confidence": 0.9 } + ], + "facts": [ + { "subject": "Tailspin", "predicate": "uses", "object": "SDK", "confidence": 0.87 }, + { "subject": "Wingtip", "predicate": "uses", "object": "SDK", "confidence": 0.87 } + ], + "preferences": [], + "relations": [] + }, + "expectEntities": ["Tailspin", "Wingtip"], + "expectFacts": [ + { "subject": "Tailspin", "predicate": "uses", "object": "SDK" }, + { "subject": "Wingtip", "predicate": "uses", "object": "SDK" } + ], + "expectPreferences": [] + }, + { + "id": "XQ-032", + "category": "preference", + "matchOn": "never schedule anything before ten in the morning", + "messages": [{ "role": "user", "content": "Never schedule anything before ten in the morning for me." }], + "scriptedResponse": { + "entities": [], + "facts": [], + "preferences": [{ "category": "scheduling", "preference": "no meetings before 10am", "confidence": 0.91 }], + "relations": [] + }, + "expectEntities": [], + "expectFacts": [], + "expectPreferences": ["no meetings before 10am"] + }, + { + "id": "XQ-033", + "category": "skill", + "matchOn": "certified in Kubernetes administration", + "messages": [{ "role": "user", "content": "I got certified in Kubernetes administration last month." }], + "scriptedResponse": { + "entities": [{ "name": "Kubernetes", "type": "TECHNOLOGY", "confidence": 0.92 }], + "facts": [{ "subject": "user", "predicate": "certified_in", "object": "Kubernetes administration", "confidence": 0.9 }], + "preferences": [], + "relations": [] + }, + "expectEntities": ["Kubernetes"], + "expectFacts": [{ "subject": "user", "predicate": "certified_in", "object": "Kubernetes administration" }], + "expectPreferences": [] + } + ] +} diff --git a/tools/AgentMemory.Cli/Perf/HermeticProfile.cs b/tools/AgentMemory.Cli/Perf/HermeticProfile.cs index 64a5be15..a195bf9c 100644 --- a/tools/AgentMemory.Cli/Perf/HermeticProfile.cs +++ b/tools/AgentMemory.Cli/Perf/HermeticProfile.cs @@ -53,15 +53,18 @@ public static async Task StartAsync( TimeSpan embeddingLatency, TimeSpan modelLatency, TextWriter log, + IReadOnlyList? scriptedRules = null, CancellationToken cancellationToken = default) { var profile = new HermeticProfile(dimensions); - await profile.InitializeAsync(embeddingLatency, modelLatency, log, cancellationToken).ConfigureAwait(false); + await profile.InitializeAsync(embeddingLatency, modelLatency, log, scriptedRules, cancellationToken) + .ConfigureAwait(false); return profile; } private async Task InitializeAsync( - TimeSpan embeddingLatency, TimeSpan modelLatency, TextWriter log, CancellationToken cancellationToken) + TimeSpan embeddingLatency, TimeSpan modelLatency, TextWriter log, + IReadOnlyList? scriptedRules, CancellationToken cancellationToken) { log.WriteLine($"perf: starting {Image} (Testcontainers)…"); _container = new Neo4jBuilder(Image) @@ -98,7 +101,8 @@ private async Task InitializeAsync( new DeterministicEmbeddingGenerator(Dimensions), embeddingLatency))); services.RemoveAll(); - services.AddSingleton(new CountingChatClient(new ScriptedChatClient(modelLatency))); + services.AddSingleton( + new CountingChatClient(new ScriptedChatClient(modelLatency, payload: null, rules: scriptedRules))); _provider = services.BuildServiceProvider(); _scope = _provider.CreateAsyncScope(); diff --git a/tools/AgentMemory.Cli/Perf/PerfScenarios.cs b/tools/AgentMemory.Cli/Perf/PerfScenarios.cs index 2a910372..3ebc00f8 100644 --- a/tools/AgentMemory.Cli/Perf/PerfScenarios.cs +++ b/tools/AgentMemory.Cli/Perf/PerfScenarios.cs @@ -41,6 +41,17 @@ public static class PerfScenarios new("PERF-W-02", "Single response message, extraction enabled (shipped defaults)", StoreAndExtractAsync), ]; + private const string StoreProbeUserMessage = + "Alice Martin just moved to the Acme Corporation platform team and prefers concise updates."; + + /// + /// Input-keyed model responses required by cost scenarios. Kept separate from judged fixture rules: + /// an unmatched rule deliberately returns an empty extraction, so omitting this entry turns W-02 + /// into a no-op that its self-assertion rejects. + /// + internal static IReadOnlyList ScriptedRules { get; } = + [new(StoreProbeUserMessage, ScriptedChatClient.ExtractionPayload)]; + public static IReadOnlyList Select(string? filter) { if (string.IsNullOrWhiteSpace(filter) || filter.Equals("all", StringComparison.OrdinalIgnoreCase)) @@ -107,8 +118,7 @@ private static async Task StoreAndExtractAsync(ScenarioContext ctx) var requestMessages = new[] { - new ChatMessage(ChatRole.User, - "Alice Martin just moved to the Acme Corporation platform team and prefers concise updates."), + new ChatMessage(ChatRole.User, StoreProbeUserMessage), }; var responseMessages = new[] { @@ -125,12 +135,20 @@ await ctx.Provider.PerformStoreAsync( PerfFixture.OwnerId).ConfigureAwait(false); // The mirror of the recall self-check: a scripted model returning unparseable output would make - // extraction yield nothing, persistence write nothing, and this scenario measure a no-op. - if (ctx.Turn.Counter("llm.calls") == 0) + // extraction yield nothing, persistence write nothing, and this scenario measure a no-op. Model + // calls alone are not evidence: an empty-but-valid response still records all four calls. + var modelCalls = ctx.Turn.Counter("llm.calls"); + var entities = ctx.Turn.Counter("persist.entities"); + var facts = ctx.Turn.Counter("persist.facts"); + var preferences = ctx.Turn.Counter("persist.preferences"); + if (modelCalls == 0 || entities == 0 || facts == 0 || preferences == 0) { throw new InvalidOperationException( - "PERF-W-02 recorded zero LLM calls, so automatic extraction did not run. Check that LLM " + - "extraction is opted in (AddNeo4jAgentMemory's configureLlm) and AutoExtractOnPersist is true."); + $"PERF-W-02 did not persist the scripted extraction (llm.calls={modelCalls}, " + + $"persist.entities={entities}, persist.facts={facts}, " + + $"persist.preferences={preferences}). The scenario would measure a no-op. Check that " + + "LLM extraction is opted in, AutoExtractOnPersist is true, and the scripted client " + + "returned its cost-scenario payload."); } } diff --git a/tools/AgentMemory.Cli/Perf/QualityGate.cs b/tools/AgentMemory.Cli/Perf/QualityGate.cs new file mode 100644 index 00000000..8e6c6018 --- /dev/null +++ b/tools/AgentMemory.Cli/Perf/QualityGate.cs @@ -0,0 +1,204 @@ +using System.Globalization; +using System.Text.Json; + +namespace AgentMemory.Cli.Perf; + +/// +/// Reviewable quality thresholds committed with the harness. +/// +internal sealed record QualityBaseline( + int SchemaVersion, + double Tolerance, + RetrievalQualityBaseline Retrieval, + ExtractionQualityBaseline Extraction); + +internal sealed record RetrievalQualityBaseline( + double RecallAtK, + double Mrr, + int Cases, + int MaxCasesWithViolations); + +internal sealed record ExtractionQualityBaseline( + double EntityPrecision, + double EntityRecall, + double FactPrecision, + double FactRecall, + double PreferencePrecision, + double PreferenceRecall, + int Cases, + int ExpectNothingCases, + double MaxFalsePositiveRate); + +/// The enforceable verdict written beside every quality report. +internal sealed record QualityGateResult( + bool Enabled, + bool Passed, + string BaselinePath, + double Tolerance, + IReadOnlyList Violations) +{ + public static QualityGateResult Disabled() => + new(false, true, QualityGate.DefaultBaselinePath, 0, []); +} + +/// +/// Compares deterministic retrieval and extraction scores with the committed baseline. +/// +/// +/// Lower-is-worse metrics use actual + tolerance < baseline. False positives use the inverse +/// upper bound. Fixture case counts are exact: silently deleting a difficult judged case must not make +/// a run pass. +/// +internal static class QualityGate +{ + internal const string DefaultBaselinePath = "eng/perf/baselines/quality.json"; + + private static readonly JsonSerializerOptions Json = new() + { + PropertyNameCaseInsensitive = true, + }; + + public static QualityBaseline LoadBaseline() + { + if (!File.Exists(DefaultBaselinePath)) + { + throw new FileNotFoundException( + $"Quality baseline not found at '{DefaultBaselinePath}'. Run perf from the repository root.", + DefaultBaselinePath); + } + + var baseline = JsonSerializer.Deserialize( + File.ReadAllText(DefaultBaselinePath), Json) + ?? throw new InvalidOperationException( + $"Quality baseline '{DefaultBaselinePath}' deserialized to null."); + + Validate(baseline); + return baseline; + } + + public static QualityGateResult Evaluate( + QualityBaseline baseline, + QualityResult retrieval, + ExtractionQualityResult extraction, + string baselinePath = DefaultBaselinePath) + { + Validate(baseline); + var violations = new List(); + + RequireExact("retrieval.cases", retrieval.Cases, baseline.Retrieval.Cases, violations); + RequireMinimum( + "retrieval.recallAtK", retrieval.RecallAtK, baseline.Retrieval.RecallAtK, + baseline.Tolerance, violations); + RequireMinimum( + "retrieval.mrr", retrieval.Mrr, baseline.Retrieval.Mrr, + baseline.Tolerance, violations); + + if (retrieval.CasesWithViolations > baseline.Retrieval.MaxCasesWithViolations) + { + violations.Add( + $"retrieval forbidden retrievals: actual {retrieval.CasesWithViolations}, " + + $"allowed {baseline.Retrieval.MaxCasesWithViolations}"); + } + + RequireExact("extraction.cases", extraction.Cases, baseline.Extraction.Cases, violations); + RequireExact( + "extraction.expectNothingCases", + extraction.ExpectNothingCases, + baseline.Extraction.ExpectNothingCases, + violations); + RequireMinimum( + "extraction.entityPrecision", extraction.EntityPrecision, + baseline.Extraction.EntityPrecision, baseline.Tolerance, violations); + RequireMinimum( + "extraction.entityRecall", extraction.EntityRecall, + baseline.Extraction.EntityRecall, baseline.Tolerance, violations); + RequireMinimum( + "extraction.factPrecision", extraction.FactPrecision, + baseline.Extraction.FactPrecision, baseline.Tolerance, violations); + RequireMinimum( + "extraction.factRecall", extraction.FactRecall, + baseline.Extraction.FactRecall, baseline.Tolerance, violations); + RequireMinimum( + "extraction.preferencePrecision", extraction.PreferencePrecision, + baseline.Extraction.PreferencePrecision, baseline.Tolerance, violations); + RequireMinimum( + "extraction.preferenceRecall", extraction.PreferenceRecall, + baseline.Extraction.PreferenceRecall, baseline.Tolerance, violations); + + if (extraction.FalsePositiveRate > + baseline.Extraction.MaxFalsePositiveRate + baseline.Tolerance) + { + violations.Add( + $"extraction.falsePositiveRate: actual {Format(extraction.FalsePositiveRate)}, " + + $"maximum {Format(baseline.Extraction.MaxFalsePositiveRate)}, " + + $"tolerance {Format(baseline.Tolerance)}"); + } + + return new QualityGateResult( + Enabled: true, + Passed: violations.Count == 0, + BaselinePath: baselinePath.Replace('\\', '/'), + Tolerance: baseline.Tolerance, + Violations: violations); + } + + private static void RequireMinimum( + string metric, + double actual, + double expected, + double tolerance, + List violations) + { + if (actual + tolerance < expected) + { + violations.Add( + $"{metric}: actual {Format(actual)}, baseline {Format(expected)}, " + + $"tolerance {Format(tolerance)}"); + } + } + + private static void RequireExact( + string metric, + int actual, + int expected, + List violations) + { + if (actual != expected) + violations.Add($"{metric}: actual {actual}, baseline {expected} (must match exactly)"); + } + + private static string Format(double value) => + value.ToString("F6", CultureInfo.InvariantCulture); + + private static void Validate(QualityBaseline baseline) + { + if (baseline.SchemaVersion != 1) + throw new InvalidOperationException( + $"Unsupported quality baseline schemaVersion {baseline.SchemaVersion}; expected 1."); + if (!double.IsFinite(baseline.Tolerance) || baseline.Tolerance < 0) + throw new InvalidOperationException("Quality baseline tolerance must be finite and non-negative."); + if (baseline.Retrieval.Cases <= 0 || baseline.Extraction.Cases <= 0 || + baseline.Extraction.ExpectNothingCases <= 0) + { + throw new InvalidOperationException("Quality baseline case counts must be positive."); + } + + var scores = new[] + { + baseline.Retrieval.RecallAtK, + baseline.Retrieval.Mrr, + baseline.Extraction.EntityPrecision, + baseline.Extraction.EntityRecall, + baseline.Extraction.FactPrecision, + baseline.Extraction.FactRecall, + baseline.Extraction.PreferencePrecision, + baseline.Extraction.PreferenceRecall, + baseline.Extraction.MaxFalsePositiveRate, + }; + if (scores.Any(score => !double.IsFinite(score) || score is < 0 or > 1)) + throw new InvalidOperationException("Quality baseline scores must be finite values from 0 through 1."); + if (baseline.Retrieval.MaxCasesWithViolations < 0) + throw new InvalidOperationException( + "Quality baseline maxCasesWithViolations must be non-negative."); + } +} diff --git a/tools/AgentMemory.Cli/Perf/ScriptedChatClient.cs b/tools/AgentMemory.Cli/Perf/ScriptedChatClient.cs index fb9301c7..ce91aff0 100644 --- a/tools/AgentMemory.Cli/Perf/ScriptedChatClient.cs +++ b/tools/AgentMemory.Cli/Perf/ScriptedChatClient.cs @@ -45,13 +45,19 @@ public sealed class ScriptedChatClient : IChatClient } """; + /// A per-input scripted answer: when appears in the prompt, return + /// . + public sealed record Rule(string MatchOn, string Payload); + private readonly TimeSpan _delay; private readonly string _payload; + private readonly IReadOnlyList _rules; - public ScriptedChatClient(TimeSpan delay, string? payload = null) + public ScriptedChatClient(TimeSpan delay, string? payload = null, IReadOnlyList? rules = null) { _delay = delay; _payload = payload ?? ExtractionPayload; + _rules = rules ?? Array.Empty(); } public async Task GetResponseAsync( @@ -62,19 +68,53 @@ public async Task GetResponseAsync( if (_delay > TimeSpan.Zero) await Task.Delay(_delay, cancellationToken).ConfigureAwait(false); + var materialized = messages as IList ?? messages.ToList(); + var payload = SelectPayload(materialized); + // Token counts are approximated from character length rather than invented, so cost accounting // stays proportional to real prompt growth as scenarios change. - var inputChars = messages.Sum(m => m.Text?.Length ?? 0); - return new ChatResponse(new ChatMessage(ChatRole.Assistant, _payload)) + var inputChars = materialized.Sum(m => m.Text?.Length ?? 0); + return new ChatResponse(new ChatMessage(ChatRole.Assistant, payload)) { Usage = new UsageDetails { InputTokenCount = inputChars / 4, - OutputTokenCount = _payload.Length / 4, + OutputTokenCount = payload.Length / 4, }, }; } + /// + /// Picks the scripted answer for this prompt. + /// + /// + /// Cost scenarios do not care what comes back, so they use the single default payload. Extraction + /// quality scenarios do: a client that answers identically regardless of input would make + /// every judged case extract the same facts, and the fixture would measure nothing at all. Rules + /// key on a distinctive phrase from the case's own conversation — deterministic, and readable in the + /// fixture, unlike a hash. + /// + private string SelectPayload(IEnumerable messages) + { + if (_rules.Count == 0) return _payload; + + var prompt = string.Join("\n", messages.Select(m => m.Text ?? string.Empty)); + foreach (var rule in _rules) + { + if (prompt.Contains(rule.MatchOn, StringComparison.OrdinalIgnoreCase)) + return rule.Payload; + } + + // No rule matched. Return an EMPTY extraction rather than the default payload: silently + // substituting facts from an unrelated case would make a mis-keyed fixture case look like it + // extracted correctly, which is the one failure mode this client must not hide. + return EmptyPayload; + } + + /// A well-formed response that extracts nothing. + public const string EmptyPayload = + """{"entities": [], "facts": [], "preferences": [], "relations": []}"""; + public async IAsyncEnumerable GetStreamingResponseAsync( IEnumerable messages, ChatOptions? options = null, diff --git a/tools/AgentMemory.Cli/Program.cs b/tools/AgentMemory.Cli/Program.cs index df63d9f2..8a297faf 100644 --- a/tools/AgentMemory.Cli/Program.cs +++ b/tools/AgentMemory.Cli/Program.cs @@ -49,7 +49,8 @@ cli.Get("warmup"), cli.Get("embedding-dimensions"), cli.Get("latency"), - cli.Get("output")); + cli.Get("output"), + cli.Get("quality-gate")); } catch (Exception ex) {