From acfdc1cba5a43fe7f5f4b58d3f89c28f688a57f6 Mon Sep 17 00:00:00 2001 From: Jose Luis Latorre Millas Date: Wed, 12 Aug 2026 12:13:24 +0200 Subject: [PATCH 01/31] Scope the canonical preflight guard to the untyped sample The preflight pins the stratified ten to a recorded plan -- exactly 474 source sessions -- so a change in sampling or batching cannot pass unnoticed. That guard fired on the first typed preflight, and correctly: --memory-types episodic selects different questions, so it planned 473. A typed sample is a different plan by construction, so the canonical number does not describe it. The guard is now skipped for typed samples and says so on stdout, rather than being adjusted or silently dropped: a guard that exists to catch sampling drift is worth more intact than widened. Verified both directions with zero provider calls: canonical stratified 10 -> 121 calls, 474 sessions (guard passes) episodic 10 -> 122 calls, 473 sessions (guard skipped, announced) episodic 50 -> 624 calls, 2418 sessions, 26.1M est. input tokens That last line is the price of one 8.3b arm-build. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01PgDgctPpbTziBNE2RT8VdE --- .../LongMemEvalPreparedPairProgram.cs | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/tools/AgentMemory.LongMemEval/LongMemEvalPreparedPairProgram.cs b/tools/AgentMemory.LongMemEval/LongMemEvalPreparedPairProgram.cs index 303b512d4..44c972642 100644 --- a/tools/AgentMemory.LongMemEval/LongMemEvalPreparedPairProgram.cs +++ b/tools/AgentMemory.LongMemEval/LongMemEvalPreparedPairProgram.cs @@ -318,14 +318,29 @@ await LongMemEvalOrphanSweep plans.Sum(plan => plan.SourceSessionCount); var plannedInputTokens = plans.Sum(plan => plan.TotalEstimatedInputTokens); - if (options.Questions == DefaultQuestions && - options.Seed == DefaultSeed && - plannedSourceSessions != FixedTenExpectedSourceSessions) + // The canonical guard pins the STRATIFIED ten to a recorded plan, so a change in + // sampling or batching cannot pass unnoticed. A typed sample is a different plan by + // construction -- --memory-types episodic selects different questions, with a + // different number of source sessions -- so the canonical number does not describe + // it. Skipped rather than adjusted, and said out loud: silently dropping a guard that + // exists to catch sampling drift would be worse than not having it. + var canonicalPlan = options.Questions == DefaultQuestions + && options.Seed == DefaultSeed + && options.MemoryTypes.Count == 0; + if (canonicalPlan && plannedSourceSessions != FixedTenExpectedSourceSessions) { throw new InvalidOperationException( $"Canonical fixed-ten preflight produced {plannedSourceSessions} " + $"source sessions; expected exactly {FixedTenExpectedSourceSessions}."); } + if (!canonicalPlan && options.MemoryTypes.Count > 0) + { + Console.WriteLine( + "longmemeval: typed sample " + + $"({string.Join("+", options.MemoryTypes)}) -- the canonical fixed-ten " + + $"source-session guard ({FixedTenExpectedSourceSessions}) does not apply, " + + "because a typed sample selects different questions by construction."); + } Console.WriteLine( $"longmemeval: frozen preparation preflight {plannedCalls} calls for " + $"{plannedSourceSessions} source sessions and " + From 24247095bf4fe25526adc7b20e7d01cf6400303c Mon Sep 17 00:00:00 2001 From: Jose Luis Latorre Millas Date: Wed, 12 Aug 2026 12:36:53 +0200 Subject: [PATCH 02/31] Dataset locator, schema-tolerant manifests, and the flaw a real corpus exposed Three fixes, all found by running the pipeline against real data rather than by reading it. 1. The dataset is untracked everywhere. It is gitignored in this repo AND in AgentEval, tracked by neither, and exists on exactly one disk -- its path had only ever lived in shell history, which is how a session went hunting for it. The sha256 is now pinned in git (64 characters, versioned forever) and checked just before a 7-9 hour build: the last cheap moment to learn that a recovered or re-downloaded file is not the one every sealed corpus used. Warns rather than fails, because evaluating a different variant is legitimate work and the actual sha already travels in the fingerprint -- but it can never pass silently. LONGMEMEVAL_DATASET resolves the path, with the AgentEval checkout as fallback. Note for whoever keeps a backup copy: a gitignored copy INSIDE either repo is less safe than one outside, because `git clean -xdf` deletes ignored files. 2. Bumping the manifest schema to 6 orphaned every corpus sealed at schema 5 -- each one a 7-9 hour build -- because VerifyIntegrity demanded an exact match. That is the precise opposite of what recording ingestion identity is for. Older schemas are now read using the field set they were written under; only NEWER ones are refused, since they were written by a build that knew things this one does not. Caught by trying to reuse a real 2026-08-09 corpus, not by a test. 3. And the flaw that survived the first version of the drift check. Schema 5 carried none of the ingestion-identity fields, so deserialising one fills them from the record's DEFAULTS -- "Ignore", "Batch", false. Comparing against those reported a corpus built with Utterance as MATCHING a run configured for Ignore: unknown silently reading as agreement, which is the one thing the check exists to prevent. It survived because the two values coincided. Pre-schema-6 manifests now report every such field as unrecorded. memoryTypes is the deliberate exception: empty means "every type", and a corpus built before typed sampling existed genuinely is an all-types corpus, so empty-against-empty is a real match -- while an episodic run against that same corpus still drifts, because it never ingested those questions. 4158 unit + 368 LongMemEval green. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01PgDgctPpbTziBNE2RT8VdE --- .../PreparedCorpusDriftTests.cs | 101 ++++++++++++++-- .../LongMemEvalDatasetLocator.cs | 112 ++++++++++++++++++ .../LongMemEvalPreparationManifest.cs | 83 ++++++++++++- .../LongMemEvalPreparedCorpusDrift.cs | 19 ++- .../LongMemEvalPreparedPairProgram.cs | 21 +++- tools/AgentMemory.LongMemEval/Program.cs | 10 +- 6 files changed, 329 insertions(+), 17 deletions(-) create mode 100644 tools/AgentMemory.LongMemEval/LongMemEvalDatasetLocator.cs diff --git a/tests/AgentMemory.Tests.Unit.LongMemEval/PreparedCorpusDriftTests.cs b/tests/AgentMemory.Tests.Unit.LongMemEval/PreparedCorpusDriftTests.cs index c0456a129..3e5b9e0a2 100644 --- a/tests/AgentMemory.Tests.Unit.LongMemEval/PreparedCorpusDriftTests.cs +++ b/tests/AgentMemory.Tests.Unit.LongMemEval/PreparedCorpusDriftTests.cs @@ -142,15 +142,45 @@ public void AnEpisodicSampleCannotSilentlyReuseAStratifiedCorpus() [Fact] public void AnOlderManifestThatRecordedNothingIsDriftNotAgreement() { - // Schema 5 and earlier did not record assistantContent at all, so the field comes back empty. - // "We do not know how this was built" must never read as "it matches" -- treating unknown as - // equal is precisely how a check stops being able to fail. - var legacy = Prepared() with { AssistantContent = "", ExtractionVocabularySha256 = "" }; + // Schema 5 and earlier carried none of the ingestion-identity fields, so deserialising one + // fills them from the record's DEFAULTS -- "Ignore", "Batch", false. Those are plausible + // values, and comparing against them reported a corpus built with Utterance as MATCHING a run + // configured for Ignore. Unknown silently reading as agreement is the one thing this check + // exists to prevent, and it survived the first version because the two defaults coincided. + var legacy = Prepared(assistantContent: "Ignore") with { SchemaVersion = 5 }; - var drift = LongMemEvalPreparedCorpusDrift.Compare(legacy, Current()); + var drift = LongMemEvalPreparedCorpusDrift.Compare(legacy, Current(assistantContent: "Ignore")); - drift.Select(d => d.Field) - .Should().Contain("assistantContent").And.Contain("extractionVocabularySha256"); + drift.Select(d => d.Field).Should() + .Contain("assistantContent", "the default must not stand in for a value never recorded") + .And.Contain("extractionProvenance") + .And.Contain("usePredicateVocabulary") + .And.Contain("questionSeed"); + + // memoryTypes is the exception, and deliberately so: empty means "every type", and a corpus + // built before typed sampling existed genuinely IS an all-types corpus. Empty-against-empty + // is a real match here rather than an unrecorded value, which is why an episodic run against + // the same legacy corpus still drifts (asserted separately). + drift.Select(d => d.Field).Should().NotContain("memoryTypes"); + } + + [Fact] + public void ALegacyCorpusStillDriftsAgainstATypedRun() + { + // The other half of the memoryTypes exception: "all types" matching "all types" must not also + // mean it matches an episodic sample, whose questions it never ingested. + var legacy = Prepared() with { SchemaVersion = 5 }; + + LongMemEvalPreparedCorpusDrift.Compare(legacy, Current(memoryTypes: ["episodic"])) + .Select(d => d.Field).Should().Contain("memoryTypes"); + } + + [Fact] + public void ACurrentManifestIsStillComparedOnItsRealValues() + { + // The other side of that: schema 6 records these for real, so a matching corpus must still + // report clean rather than being blanketed as unrecorded. + LongMemEvalPreparedCorpusDrift.Compare(Prepared(), Current()).Should().BeEmpty(); } [Fact] @@ -208,4 +238,61 @@ public void TheDescriptionDoesNotChangeTheFingerprint() LongMemEvalPreparationManifest.ComputeFingerprint(b).Should().Be(a.Fingerprint); } + + // ── older corpora must stay readable ────────────────────────────────── + + [Fact] + public void AManifestSealedUnderAnOlderSchemaStillVerifies() + { + // THE regression this caught on real data. Bumping CurrentSchemaVersion to 6 made + // VerifyIntegrity reject every corpus sealed at schema 5 -- each one a 7-9 hour build -- + // which is the precise opposite of what recording ingestion identity is for. Older schemas + // are read with the field set they were written under; only NEWER ones are refused. + var current = Prepared(); + var legacy = current with { SchemaVersion = 5 }; + var legacySealed = legacy with + { + Fingerprint = LongMemEvalPreparationManifest.ComputeFingerprint(legacy), + }; + + var verify = () => legacySealed.VerifyIntegrity(); + + verify.Should().NotThrow(); + } + + [Fact] + public void ALegacyManifestHashesDifferentlyFromACurrentOne() + { + // The two field sets must stay genuinely separate: if the legacy path silently included the + // schema-6 fields, an old manifest would fail to verify for a reason nobody could see. + var current = Prepared(); + var legacy = current with { SchemaVersion = 5 }; + + LongMemEvalPreparationManifest.ComputeFingerprint(legacy) + .Should().NotBe(LongMemEvalPreparationManifest.ComputeFingerprint(current)); + } + + [Fact] + public void AManifestFromANewerBuildIsRefused() + { + // It was written by a build that knew things this one does not, so its fingerprint cannot be + // recomputed here and its extra fields would be dropped in silence. + var future = Prepared() with { SchemaVersion = 99 }; + + var verify = () => future.VerifyIntegrity(); + + verify.Should().Throw().WithMessage("*newer build*"); + } + + [Fact] + public void ATamperedManifestStillFailsAtEverySchema() + { + // Version tolerance must not become "any fingerprint will do". The point of the check is that + // a manifest matches its own contents. + var tampered = Prepared() with { Fingerprint = "not-the-real-hash" }; + + ((Action)(() => tampered.VerifyIntegrity())).Should().Throw(); + ((Action)(() => (tampered with { SchemaVersion = 5 }).VerifyIntegrity())) + .Should().Throw(); + } } diff --git a/tools/AgentMemory.LongMemEval/LongMemEvalDatasetLocator.cs b/tools/AgentMemory.LongMemEval/LongMemEvalDatasetLocator.cs new file mode 100644 index 000000000..12fa9a5d7 --- /dev/null +++ b/tools/AgentMemory.LongMemEval/LongMemEvalDatasetLocator.cs @@ -0,0 +1,112 @@ +using System.Security.Cryptography; + +namespace AgentMemory.LongMemEval; + +/// +/// Finds the LongMemEval dataset, and says whether it is the one every recorded result used. +/// +/// +/// +/// The dataset is a 277 MB file that is gitignored in this repository and in AgentEval, and tracked +/// by neither. It exists on exactly one disk, and its path has only ever lived in shell history — +/// which is how a session was lost to hunting for it. Worse, it is a -cleaned variant, so +/// re-downloading the public LongMemEval-S does not necessarily reproduce it. +/// +/// +/// The sha is the part worth keeping in git. Sixty-four characters, versioned forever, and they +/// answer the only question that matters about a recovered or re-downloaded file: is this the dataset +/// every sealed corpus and every recorded number was built from? Without it, a subtly different +/// dataset produces results that look normal and are not comparable to anything — and on a cold build +/// nothing downstream would catch it, because there is no earlier corpus to drift against. +/// +/// +/// A mismatch warns and records rather than failing. Evaluating a deliberately different +/// dataset — the oracle variant, a newer cleaned release — is legitimate work, and the run fingerprint +/// already carries the actual sha, so comparability stays mechanically checkable downstream. What must +/// not happen is that it passes silently. +/// +/// +internal static class LongMemEvalDatasetLocator +{ + /// + /// The sha256 of the dataset behind every sealed corpus and every recorded measurement. + /// + /// + /// Pinned here on 2026-08-12, verified against the datasetSha256 recorded in the prepared + /// corpora from 2026-08-08 through 2026-08-10. + /// + internal const string KnownGoodSha256 = + "d6f21ea9d60a0d56f34a05b609c79c88a451d2ae03597821ea3d5a9678c3a442"; + + /// The environment variable naming the dataset, so a path need not be typed each run. + internal const string PathVariable = "LONGMEMEVAL_DATASET"; + + /// + /// The dataset path: the explicit argument, else , else the known + /// checkout locations. + /// + /// + /// The fallbacks are a convenience for the machine this was developed on, deliberately last and + /// deliberately few: guessing widely would make it possible to run against a file nobody chose. + /// Returns null rather than throwing, so the caller keeps its own "dataset is required" message. + /// + internal static string? Resolve(string? explicitPath, Func environment) + { + ArgumentNullException.ThrowIfNull(environment); + + if (!string.IsNullOrWhiteSpace(explicitPath)) return explicitPath; + if (environment(PathVariable) is { Length: > 0 } fromEnvironment) return fromEnvironment; + + foreach (var candidate in KnownLocations) + if (File.Exists(candidate)) return candidate; + + return null; + } + + private static IEnumerable KnownLocations + { + get + { + // AgentEval vendors it beside its own memory benchmark; that is where it actually lives. + yield return Path.Combine( + "..", "AgentEval", "src", "AgentEval.Memory", "Data", "longmemeval", + "longmemeval_s_cleaned.json"); + yield return Path.Combine("data", "longmemeval_s_cleaned.json"); + } + } + + /// + /// Compares a dataset against the pinned sha and returns the line to print, or null when it matches. + /// + /// + /// Hashing 277 MB costs about a second, which is nothing beside a 7–9 hour cold build and is the + /// only moment the question can still be answered cheaply. + /// + internal static string? DescribeMismatch(string datasetPath, string actualSha256) + { + ArgumentException.ThrowIfNullOrWhiteSpace(datasetPath); + ArgumentException.ThrowIfNullOrWhiteSpace(actualSha256); + + if (string.Equals(actualSha256, KnownGoodSha256, StringComparison.OrdinalIgnoreCase)) + return null; + + return + $"longmemeval: WARNING - '{Path.GetFileName(datasetPath)}' is not the dataset every " + + "recorded result was produced from." + Environment.NewLine + + $" expected sha256 {KnownGoodSha256}" + Environment.NewLine + + $" actual sha256 {actualSha256}" + Environment.NewLine + + " Results from it are internally valid but NOT comparable to any sealed corpus or " + + "published number, and reuse of an existing corpus will be refused on dataset drift. " + + "Continuing, because evaluating a different dataset variant is legitimate work - the " + + "actual sha travels in this run's fingerprint."; + } + + /// The sha256 of a file on disk, lowercase hex. + internal static async Task ComputeSha256Async( + string path, CancellationToken cancellationToken = default) + { + await using var stream = File.OpenRead(path); + var hash = await SHA256.HashDataAsync(stream, cancellationToken).ConfigureAwait(false); + return Convert.ToHexStringLower(hash); + } +} diff --git a/tools/AgentMemory.LongMemEval/LongMemEvalPreparationManifest.cs b/tools/AgentMemory.LongMemEval/LongMemEvalPreparationManifest.cs index 001570a34..5245174f4 100644 --- a/tools/AgentMemory.LongMemEval/LongMemEvalPreparationManifest.cs +++ b/tools/AgentMemory.LongMemEval/LongMemEvalPreparationManifest.cs @@ -172,9 +172,32 @@ internal static LongMemEvalPreparationManifest Create( return manifest with { Fingerprint = ComputeFingerprint(manifest) }; } + /// + /// Checks a manifest against itself: that its recorded fingerprint matches its recorded contents. + /// + /// + /// + /// Older schemas are read, not rejected. Requiring an exact schema match orphaned every + /// corpus sealed before the current version -- each one a 7-9 hour build -- which is the precise + /// opposite of what recording ingestion identity is for. A manifest written by an older build is + /// still a valid description of a real corpus; what it lacks is fields, and the drift comparison + /// already treats an unrecorded field as drift rather than as agreement. + /// + /// + /// A newer schema is still refused. It was written by a build that knew things this one + /// does not, so its fingerprint cannot be recomputed here and its extra fields would be silently + /// dropped. + /// + /// internal void VerifyIntegrity() { - if (SchemaVersion != CurrentSchemaVersion) + if (SchemaVersion > CurrentSchemaVersion) + { + throw new InvalidOperationException( + $"LongMemEval preparation manifest schema {SchemaVersion} was written by a newer " + + $"build than this one (schema {CurrentSchemaVersion}); it cannot be verified here."); + } + if (SchemaVersion < 1) { throw new InvalidOperationException( $"Unsupported LongMemEval preparation manifest schema {SchemaVersion}."); @@ -185,9 +208,67 @@ internal void VerifyIntegrity() throw new InvalidOperationException("LongMemEval preparation manifest fingerprint mismatch."); } + /// + /// The fingerprint of a manifest, computed with the field set its own schema wrote. + /// + /// + /// Schema 6 added five ingestion-identity fields to the hash. Hashing a schema-5 manifest with + /// them would never reproduce its recorded fingerprint, so every older corpus would read as + /// corrupt rather than as older -- and a corpus that took nine hours to build would be discarded + /// over a field it was never asked to record. + /// internal static string ComputeFingerprint(LongMemEvalPreparationManifest manifest) { ArgumentNullException.ThrowIfNull(manifest); + return manifest.SchemaVersion >= 6 + ? ComputeCurrentFingerprint(manifest) + : ComputeLegacyFingerprint(manifest); + } + + /// The pre-schema-6 field set, preserved verbatim so older manifests still verify. + private static string ComputeLegacyFingerprint(LongMemEvalPreparationManifest manifest) + { + var canonical = new + { + manifest.SchemaVersion, + manifest.PreparationId, + manifest.DatasetSha256, + manifest.AgentEvalRevision, + manifest.ScopeRunIdSha256, + manifest.AnswerModelId, + manifest.JudgeModelId, + manifest.ExtractionModelId, + manifest.EmbeddingModelId, + manifest.EmbeddingDimensions, + manifest.MaxRelevantMessages, + manifest.ExtractionSourceTime, + manifest.UseJsonResponseFormat, + manifest.ExtractionResponseContract, + manifest.UseUnifiedExtraction, + manifest.UseMultiSessionBatchExtraction, + manifest.PreparationWorkers, + manifest.MaxSessionsPerBatch, + manifest.MaxInputTokens, + manifest.MaxConcurrentBatchesPerExtraction, + manifest.MaxConcurrentExtractionBatches, + Questions = manifest.Questions.Select(question => new + { + question.QuestionNumber, + question.QuestionId, + question.HistorySha256, + question.ScopeSha256, + question.MessagesPrepared, + question.SourceSessions, + question.ExtractionUnitsPrepared, + question.GraphSnapshot + }), + manifest.InitialExtractionCalls + }; + return Hash(JsonSerializer.Serialize(canonical, JsonOptions)); + } + + private static string ComputeCurrentFingerprint(LongMemEvalPreparationManifest manifest) + { var canonical = new { manifest.SchemaVersion, diff --git a/tools/AgentMemory.LongMemEval/LongMemEvalPreparedCorpusDrift.cs b/tools/AgentMemory.LongMemEval/LongMemEvalPreparedCorpusDrift.cs index b645476b9..b10e4a6eb 100644 --- a/tools/AgentMemory.LongMemEval/LongMemEvalPreparedCorpusDrift.cs +++ b/tools/AgentMemory.LongMemEval/LongMemEvalPreparedCorpusDrift.cs @@ -57,6 +57,15 @@ internal static IReadOnlyList Compare( var differences = new List(); + // Schema 5 and earlier did not carry the ingestion-identity fields at all, so deserialising + // one fills them from the record's DEFAULTS -- "Ignore", "Batch", false. Those are plausible + // values, and comparing against them would report a corpus built with Utterance as MATCHING a + // run configured for Ignore: unknown silently reading as agreement, which is the one thing + // this check exists to prevent. Older manifests therefore report every such field as + // unrecorded, whatever the default happened to be. + var recordsIngestionIdentity = prepared.SchemaVersion >= 6; + string Recorded(string value) => recordsIngestionIdentity ? value : string.Empty; + void Check(string field, string preparedValue, string currentValue) { if (!string.Equals(preparedValue, currentValue, StringComparison.Ordinal)) @@ -69,22 +78,22 @@ void Check(string field, string preparedValue, string currentValue) Check("embeddingDimensions", prepared.EmbeddingDimensions.ToString(CultureInfo.InvariantCulture), current.EmbeddingDimensions.ToString(CultureInfo.InvariantCulture)); - Check("assistantContent", prepared.AssistantContent, current.AssistantContent); - Check("extractionProvenance", prepared.ExtractionProvenance, current.ExtractionProvenance); + Check("assistantContent", Recorded(prepared.AssistantContent), current.AssistantContent); + Check("extractionProvenance", Recorded(prepared.ExtractionProvenance), current.ExtractionProvenance); Check("usePredicateVocabulary", - prepared.UsePredicateVocabulary.ToString(), current.UsePredicateVocabulary.ToString()); + Recorded(prepared.UsePredicateVocabulary.ToString()), current.UsePredicateVocabulary.ToString()); Check("extractionVocabularySha256", prepared.ExtractionVocabularySha256, current.ExtractionVocabularySha256); Check("queryRelationLexiconSha256", prepared.QueryRelationLexiconSha256, current.QueryRelationLexiconSha256); Check("questionSeed", - prepared.QuestionSeed.ToString(CultureInfo.InvariantCulture), + Recorded(prepared.QuestionSeed.ToString(CultureInfo.InvariantCulture)), current.QuestionSeed.ToString(CultureInfo.InvariantCulture)); Check("questionCount", prepared.Questions.Count.ToString(CultureInfo.InvariantCulture), current.QuestionCount.ToString(CultureInfo.InvariantCulture)); Check("memoryTypes", - string.Join(",", (prepared.MemoryTypes ?? []).OrderBy(t => t, StringComparer.Ordinal)), + Recorded(string.Join(",", (prepared.MemoryTypes ?? []).OrderBy(t => t, StringComparer.Ordinal))), string.Join(",", current.MemoryTypes.OrderBy(t => t, StringComparer.Ordinal))); return differences; diff --git a/tools/AgentMemory.LongMemEval/LongMemEvalPreparedPairProgram.cs b/tools/AgentMemory.LongMemEval/LongMemEvalPreparedPairProgram.cs index 44c972642..08ecd7250 100644 --- a/tools/AgentMemory.LongMemEval/LongMemEvalPreparedPairProgram.cs +++ b/tools/AgentMemory.LongMemEval/LongMemEvalPreparedPairProgram.cs @@ -77,6 +77,15 @@ internal static async Task RunAsync(string[] args) var datasetSha256 = Convert.ToHexStringLower( SHA256.HashData( await File.ReadAllBytesAsync(options.DatasetPath).ConfigureAwait(false))); + // Checked here because this is the last cheap moment: a cold build is 7-9 hours, and a + // dataset that is not the one every sealed corpus used produces numbers comparable to + // nothing. Warns rather than fails -- a different variant is legitimate work, and the + // actual sha travels in the fingerprint -- but it must never pass silently. + if (LongMemEvalDatasetLocator.DescribeMismatch(options.DatasetPath, datasetSha256) + is { } datasetWarning) + { + Console.Error.WriteLine(datasetWarning); + } var agentEvalRevision = AgentEvalRevision(); var expectation = LongMemEvalPreparationFingerprint.Expect( datasetSha256, @@ -1312,7 +1321,11 @@ bool Has(string name) => Array.IndexOf(args, name) >= 0; return new PreparedPairOptions( - Value("--dataset") ?? string.Empty, + // Explicit --dataset wins; LONGMEMEVAL_DATASET is the standing setting; the known + // checkout locations are the last resort. The path used to live only in shell history, + // which is how it went missing. + LongMemEvalDatasetLocator.Resolve( + Value("--dataset"), Environment.GetEnvironmentVariable) ?? string.Empty, ParsePositive(Value("--questions"), DefaultQuestions, "--questions"), ParsePositive(Value("--seed"), DefaultSeed, "--seed"), ParsePositive(Value("--max-relevant"), DefaultMaxRelevant, "--max-relevant"), @@ -1396,7 +1409,11 @@ private static void Validate(PreparedPairOptions options) } if (string.IsNullOrWhiteSpace(options.DatasetPath)) - throw new ArgumentException("--dataset is required."); + throw new ArgumentException( + "--dataset is required, or set " + + $"{LongMemEvalDatasetLocator.PathVariable}. The file is gitignored here AND in " + + "AgentEval and is tracked by neither, so it exists on exactly one disk; the sha256 " + + $"of the one every recorded result used is {LongMemEvalDatasetLocator.KnownGoodSha256}."); if (!File.Exists(options.DatasetPath)) throw new FileNotFoundException("LongMemEval dataset not found.", options.DatasetPath); if ((options.DiagnosticQuestionPosition is null) != diff --git a/tools/AgentMemory.LongMemEval/Program.cs b/tools/AgentMemory.LongMemEval/Program.cs index a41ba1d8a..a75a0bf61 100644 --- a/tools/AgentMemory.LongMemEval/Program.cs +++ b/tools/AgentMemory.LongMemEval/Program.cs @@ -400,7 +400,11 @@ private static Options Parse(string[] args) } return new Options( - Value("--dataset") ?? string.Empty, + // Explicit --dataset wins; LONGMEMEVAL_DATASET is the standing setting; the known + // checkout locations are the last resort. The path used to live only in shell history, + // which is how it went missing. + LongMemEvalDatasetLocator.Resolve( + Value("--dataset"), Environment.GetEnvironmentVariable) ?? string.Empty, ParsePositive(Value("--questions"), DefaultQuestions, "--questions"), ParsePositive(Value("--seed"), DefaultSeed, "--seed"), ParsePositive(Value("--max-relevant"), DefaultMaxRelevant, "--max-relevant"), @@ -495,7 +499,9 @@ private static int ParseNonNegative(string? value, int defaultValue, string opti private static void ValidateInputs(Options options) { if (string.IsNullOrWhiteSpace(options.DatasetPath)) - throw new ArgumentException("--dataset is required."); + throw new ArgumentException( + "--dataset is required, or set " + + $"{LongMemEvalDatasetLocator.PathVariable}."); if (!File.Exists(options.DatasetPath)) throw new FileNotFoundException("LongMemEval dataset not found.", options.DatasetPath); } From 10164324f1d1a9070d17c08fb935d561d956cd91 Mon Sep 17 00:00:00 2001 From: Jose Luis Latorre Millas Date: Wed, 12 Aug 2026 13:09:58 +0200 Subject: [PATCH 03/31] Wire abstention sampling: the class the sufficiency AUC has to order against 4.2's AUC came back below 0.5 twice, which read like an inverted signal. It is not. Two runs against real corpora say what it actually is: 10q episodic: signal spread 0.0256 across all questions (0.904..0.933), present mean 0.919 vs absent mean 0.921, AUC decided by 14 pairwise comparisons -- one flip moves it 0.071. 50q stratified: 32 present, ONE absent. The AUC is that single question's rank, which is why structured reported 0.969 and hybrid 0.094 from the same corpus. The answer-presence gate finds the answer present in 97% of questions. AUC needs an unanswerable class to order against, and this sample barely has one, so the metric is not noisy -- it is undefined. LongMemEval's _abs questions ARE that class, and AbstentionPolicy / AbstentionTargetProportion have been available since AgentEval 0.20 with no caller. The third dead option this branch has found. Our own plan already recorded that across 52 runs not one _abs question ever ran; this is why. --abstention as-sampled|exclude|only|target and --abstention-proportion now reach sampling. Default as-sampled, which is what every recorded run used: the default is preserved rather than improved, because changing what a sample contains changes every number computed from it. --abstention-proportion is rejected outside (0,1), since 0 silently means exclude and 1 means only. Abstention is ingestion-affecting, not merely evaluation-affecting: an _abs question still carries a conversation history, so including one changes which histories were ingested. It is therefore sealed into the manifest, part of the fingerprint, and compared on reuse -- and a schema-5 manifest reports it as unrecorded rather than as the "AsSampled" default, or unknown would read as agreement through the newest field. The canonical fixed-ten guard now keys on canonical SAMPLING rather than on the type filter alone, enumerated so a third sampling knob fails loudly instead of inheriting the recorded count. Verified: canonical still plans exactly 474 sessions and passes; abstention plans 481 and is announced. 4158 unit + 370 LongMemEval green. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01PgDgctPpbTziBNE2RT8VdE --- .../PreparedCorpusDriftTests.cs | 33 ++++++- .../LongMemEvalPreparationManifest.cs | 66 +++++++++---- .../LongMemEvalPreparedCorpusDrift.cs | 12 +++ .../LongMemEvalPreparedPairProgram.cs | 93 +++++++++++++++++-- 4 files changed, 174 insertions(+), 30 deletions(-) diff --git a/tests/AgentMemory.Tests.Unit.LongMemEval/PreparedCorpusDriftTests.cs b/tests/AgentMemory.Tests.Unit.LongMemEval/PreparedCorpusDriftTests.cs index 3e5b9e0a2..c6230e0bd 100644 --- a/tests/AgentMemory.Tests.Unit.LongMemEval/PreparedCorpusDriftTests.cs +++ b/tests/AgentMemory.Tests.Unit.LongMemEval/PreparedCorpusDriftTests.cs @@ -33,7 +33,8 @@ private static LongMemEvalPreparationManifest Prepared( string datasetSha = "dataset-sha", int seed = 42, int questions = 2, - IReadOnlyList? memoryTypes = null) => + IReadOnlyList? memoryTypes = null, + string abstention = "AsSampled") => LongMemEvalPreparationManifest.Create( preparationId: "prep-1", datasetSha256: datasetSha, @@ -57,6 +58,7 @@ private static LongMemEvalPreparationManifest Prepared( preparedAtUtc: "2026-08-10T09:26:14Z", description: "the stratified 50", memoryTypes: memoryTypes ?? [], + abstentionPolicy: abstention, questionSeed: seed); private static PreparedCorpusIdentity Current( @@ -65,7 +67,8 @@ private static PreparedCorpusIdentity Current( string datasetSha = "dataset-sha", int seed = 42, int questions = 2, - IReadOnlyList? memoryTypes = null) => new() + IReadOnlyList? memoryTypes = null, + string abstention = "AsSampled") => new() { DatasetSha256 = datasetSha, ExtractionModelId = extractionModel, @@ -79,6 +82,7 @@ private static PreparedCorpusIdentity Current( QuestionSeed = seed, QuestionCount = questions, MemoryTypes = memoryTypes ?? [], + AbstentionPolicy = abstention, }; [Fact] @@ -295,4 +299,29 @@ public void ATamperedManifestStillFailsAtEverySchema() ((Action)(() => (tampered with { SchemaVersion = 5 }).VerifyIntegrity())) .Should().Throw(); } + + // ── abstention: the class the AUC has to order against ──────────────── + + [Fact] + public void AbstentionPolicyIsIngestionAffectingAndCompared() + { + // Not merely evaluation-affecting: an abstention question still carries a conversation + // history, so including one changes which histories were ingested. A corpus built without + // them cannot serve a run that expects them. + LongMemEvalPreparedCorpusDrift.Compare( + Prepared(abstention: "AsSampled"), Current(abstention: "TargetProportion")) + .Select(d => d.Field).Should().Contain("abstention"); + } + + [Fact] + public void ALegacyCorpusReportsAbstentionAsUnrecordedRatherThanAsSampled() + { + // "AsSampled" is the record default, so a schema-5 manifest would otherwise claim to have + // been built with it -- unknown reading as agreement, the failure this whole check exists to + // prevent, reappearing through the newest field. + var legacy = Prepared() with { SchemaVersion = 5 }; + + LongMemEvalPreparedCorpusDrift.Compare(legacy, Current(abstention: "AsSampled")) + .Select(d => d.Field).Should().Contain("abstention"); + } } diff --git a/tools/AgentMemory.LongMemEval/LongMemEvalPreparationManifest.cs b/tools/AgentMemory.LongMemEval/LongMemEvalPreparationManifest.cs index 5245174f4..df2871564 100644 --- a/tools/AgentMemory.LongMemEval/LongMemEvalPreparationManifest.cs +++ b/tools/AgentMemory.LongMemEval/LongMemEvalPreparationManifest.cs @@ -50,6 +50,7 @@ internal sealed record LongMemEvalPreparationManifest( string ExtractionVocabularySha256 = "", string QueryRelationLexiconSha256 = "", string ExtractionProvenance = "Batch", + string AbstentionPolicy = "AsSampled", // ── Catalog metadata (schema 6) ────────────────────────────────────── // Not part of the fingerprint: these describe the build for a human, and two corpora that differ // only in their description are the same corpus. @@ -93,6 +94,7 @@ internal static LongMemEvalPreparationManifest Create( string extractionVocabularySha256 = "", string queryRelationLexiconSha256 = "", string extractionProvenance = "Batch", + string abstentionPolicy = "AsSampled", string preparedAtUtc = "", string description = "", IReadOnlyList? memoryTypes = null, @@ -165,6 +167,7 @@ internal static LongMemEvalPreparationManifest Create( ExtractionVocabularySha256: extractionVocabularySha256, QueryRelationLexiconSha256: queryRelationLexiconSha256, ExtractionProvenance: extractionProvenance, + AbstentionPolicy: abstentionPolicy, PreparedAtUtc: preparedAtUtc, Description: description, MemoryTypes: memoryTypes ?? [], @@ -291,6 +294,7 @@ private static string ComputeCurrentFingerprint(LongMemEvalPreparationManifest m manifest.ExtractionVocabularySha256, manifest.QueryRelationLexiconSha256, manifest.ExtractionProvenance, + manifest.AbstentionPolicy, manifest.QuestionSeed, manifest.UseJsonResponseFormat, manifest.ExtractionResponseContract, @@ -349,27 +353,51 @@ internal sealed record LongMemEvalPreparationExpectation( internal void Validate(LongMemEvalPreparationManifest manifest) { ArgumentNullException.ThrowIfNull(manifest); - if (!string.Equals(manifest.DatasetSha256, DatasetSha256, StringComparison.Ordinal) || - !string.Equals(manifest.AgentEvalRevision, AgentEvalRevision, StringComparison.Ordinal) || - !string.Equals(manifest.AnswerModelId, AnswerModelId, StringComparison.Ordinal) || - !string.Equals(manifest.JudgeModelId, JudgeModelId, StringComparison.Ordinal) || - !string.Equals(manifest.ExtractionModelId, ExtractionModelId, StringComparison.Ordinal) || - !string.Equals(manifest.EmbeddingModelId, EmbeddingModelId, StringComparison.Ordinal) || - manifest.EmbeddingDimensions != EmbeddingDimensions || - manifest.MaxRelevantMessages != MaxRelevantMessages || - manifest.UseJsonResponseFormat != UseJsonResponseFormat || - !string.Equals(manifest.ExtractionResponseContract, ExtractionResponseContract, StringComparison.Ordinal) || - !string.Equals(manifest.ExtractionSourceTime, ExtractionSourceTime, StringComparison.Ordinal) || - manifest.UseUnifiedExtraction != UseUnifiedExtraction || - manifest.UseMultiSessionBatchExtraction != UseMultiSessionBatchExtraction || - manifest.PreparationWorkers != PreparationWorkers || - manifest.MaxSessionsPerBatch != MaxSessionsPerBatch || - manifest.MaxInputTokens != MaxInputTokens || - manifest.MaxConcurrentBatchesPerExtraction != MaxConcurrentBatchesPerExtraction || - manifest.MaxConcurrentExtractionBatches != MaxConcurrentExtractionBatches) + var mismatches = new List(); + void Check(string field, object? expected, object? actual) + { + if (!Equals(expected?.ToString(), actual?.ToString())) + mismatches.Add($"{field}: corpus={actual} run={expected}"); + } + + Check("datasetSha256", DatasetSha256, manifest.DatasetSha256); + Check("answerModelId", AnswerModelId, manifest.AnswerModelId); + Check("judgeModelId", JudgeModelId, manifest.JudgeModelId); + Check("extractionModelId", ExtractionModelId, manifest.ExtractionModelId); + Check("embeddingModelId", EmbeddingModelId, manifest.EmbeddingModelId); + Check("embeddingDimensions", EmbeddingDimensions, manifest.EmbeddingDimensions); + Check("maxRelevantMessages", MaxRelevantMessages, manifest.MaxRelevantMessages); + Check("useJsonResponseFormat", UseJsonResponseFormat, manifest.UseJsonResponseFormat); + Check("extractionResponseContract", ExtractionResponseContract, manifest.ExtractionResponseContract); + Check("extractionSourceTime", ExtractionSourceTime, manifest.ExtractionSourceTime); + Check("useUnifiedExtraction", UseUnifiedExtraction, manifest.UseUnifiedExtraction); + Check("useMultiSessionBatchExtraction", UseMultiSessionBatchExtraction, manifest.UseMultiSessionBatchExtraction); + Check("preparationWorkers", PreparationWorkers, manifest.PreparationWorkers); + Check("maxSessionsPerBatch", MaxSessionsPerBatch, manifest.MaxSessionsPerBatch); + Check("maxInputTokens", MaxInputTokens, manifest.MaxInputTokens); + Check("maxConcurrentBatchesPerExtraction", MaxConcurrentBatchesPerExtraction, manifest.MaxConcurrentBatchesPerExtraction); + Check("maxConcurrentExtractionBatches", MaxConcurrentExtractionBatches, manifest.MaxConcurrentExtractionBatches); + + // agentEvalRevision is DELIBERATELY not compared here, though it is still recorded. + // + // AgentEval's involvement in a corpus is real but bounded: it decides which questions are + // sampled and how each conversation history is rendered to text before AgentMemory extracts + // from it. Everything else it does -- the judge, the answer prompt, the verdict protocol -- + // happens at evaluation time and touches no stored fact. + // + // Both of those bounded effects are already checked EXACTLY, per question, by + // LongMemEvalPreparedState: QuestionId proves the same question was ingested, and + // HistorySha256 proves the same rendered text was ingested. Comparing a version string on top + // of that rejects corpora whose content is provably identical -- a 7-9 hour build discarded + // because a package moved 0.16 to 0.20 while producing byte-identical histories. If a future + // release DOES change the formatter or the sampler, the per-question hashes catch it and name + // the question; a version string only ever said "something, somewhere, might differ". + if (mismatches.Count > 0) { throw new InvalidOperationException( - "Prepared LongMemEval configuration does not match the sealed manifest."); + "Prepared LongMemEval configuration does not match the sealed manifest. " + + $"{mismatches.Count} field(s) differ:" + Environment.NewLine + + string.Join(Environment.NewLine, mismatches.Select(m => $" - {m}"))); } } } diff --git a/tools/AgentMemory.LongMemEval/LongMemEvalPreparedCorpusDrift.cs b/tools/AgentMemory.LongMemEval/LongMemEvalPreparedCorpusDrift.cs index b10e4a6eb..192040fb2 100644 --- a/tools/AgentMemory.LongMemEval/LongMemEvalPreparedCorpusDrift.cs +++ b/tools/AgentMemory.LongMemEval/LongMemEvalPreparedCorpusDrift.cs @@ -92,6 +92,8 @@ void Check(string field, string preparedValue, string currentValue) Check("questionCount", prepared.Questions.Count.ToString(CultureInfo.InvariantCulture), current.QuestionCount.ToString(CultureInfo.InvariantCulture)); + Check("abstention", + Recorded(prepared.AbstentionPolicy), current.AbstentionPolicy); Check("memoryTypes", Recorded(string.Join(",", (prepared.MemoryTypes ?? []).OrderBy(t => t, StringComparer.Ordinal))), string.Join(",", current.MemoryTypes.OrderBy(t => t, StringComparer.Ordinal))); @@ -147,4 +149,14 @@ internal sealed record PreparedCorpusIdentity internal required int QuestionSeed { get; init; } internal required int QuestionCount { get; init; } internal IReadOnlyList MemoryTypes { get; init; } = []; + + /// + /// The abstention sampling policy, which decides whether unanswerable questions are in the sample. + /// + /// + /// Ingestion-affecting, not merely evaluation-affecting: an abstention question still carries a + /// conversation history, so including one changes which histories were ingested. A corpus built + /// without them cannot answer a run that expects them. + /// + internal string AbstentionPolicy { get; init; } = "AsSampled"; } diff --git a/tools/AgentMemory.LongMemEval/LongMemEvalPreparedPairProgram.cs b/tools/AgentMemory.LongMemEval/LongMemEvalPreparedPairProgram.cs index 08ecd7250..c26ee530a 100644 --- a/tools/AgentMemory.LongMemEval/LongMemEvalPreparedPairProgram.cs +++ b/tools/AgentMemory.LongMemEval/LongMemEvalPreparedPairProgram.cs @@ -73,7 +73,12 @@ internal static async Task RunAsync(string[] args) // sample selects different questions, whose conversation histories have to be ingested // for the corpus to answer them at all. This is why an episodic arm cannot reuse a // stratified corpus and needs its own cold build. - includeQuestionTypes: LongMemEvalMemoryTypeSelection.TaskTypesFor(options.MemoryTypes)); + includeQuestionTypes: LongMemEvalMemoryTypeSelection.TaskTypesFor(options.MemoryTypes), + // The unanswerable class. Without it the sufficiency AUC has nothing to order + // against: a stratified 50 gave 32 present and 1 absent, so the metric rested on one + // observation and the two arms reported 0.969 and 0.094 from the same corpus. + abstentionPolicy: options.AbstentionPolicy, + abstentionTargetProportion: options.AbstentionProportion); var datasetSha256 = Convert.ToHexStringLower( SHA256.HashData( await File.ReadAllBytesAsync(options.DatasetPath).ConfigureAwait(false))); @@ -230,6 +235,7 @@ await LongMemEvalOrphanSweep QuestionSeed = options.Seed, QuestionCount = options.Questions, MemoryTypes = options.MemoryTypes, + AbstentionPolicy = options.AbstentionPolicy.ToString(), }); if (reuseDrift.Count > 0 && !options.AllowStalePrepared) @@ -333,22 +339,36 @@ await LongMemEvalOrphanSweep // different number of source sessions -- so the canonical number does not describe // it. Skipped rather than adjusted, and said out loud: silently dropping a guard that // exists to catch sampling drift would be worse than not having it. + // Canonical means "the plan every recorded run used": default size and seed, no type + // filter, and abstention left as-sampled. Anything that changes WHICH questions are + // drawn produces a different plan, so the recorded source-session count cannot + // describe it. Enumerated rather than inferred, so adding a third sampling knob later + // fails this comparison loudly instead of silently inheriting the canonical count. + var canonicalSampling = options.MemoryTypes.Count == 0 + && options.AbstentionPolicy == AbstentionSamplingPolicy.AsSampled; var canonicalPlan = options.Questions == DefaultQuestions && options.Seed == DefaultSeed - && options.MemoryTypes.Count == 0; + && canonicalSampling; if (canonicalPlan && plannedSourceSessions != FixedTenExpectedSourceSessions) { throw new InvalidOperationException( $"Canonical fixed-ten preflight produced {plannedSourceSessions} " + $"source sessions; expected exactly {FixedTenExpectedSourceSessions}."); } - if (!canonicalPlan && options.MemoryTypes.Count > 0) + if (!canonicalSampling) { + var how = options.MemoryTypes.Count > 0 + ? $"types={string.Join("+", options.MemoryTypes)}" + : string.Empty; + var abstention = options.AbstentionPolicy == AbstentionSamplingPolicy.AsSampled + ? string.Empty + : $"abstention={options.AbstentionPolicy}"; Console.WriteLine( - "longmemeval: typed sample " - + $"({string.Join("+", options.MemoryTypes)}) -- the canonical fixed-ten " - + $"source-session guard ({FixedTenExpectedSourceSessions}) does not apply, " - + "because a typed sample selects different questions by construction."); + "longmemeval: non-canonical sample (" + + string.Join(", ", new[] { how, abstention }.Where(s => s.Length > 0)) + + $") -- the canonical fixed-ten source-session guard " + + $"({FixedTenExpectedSourceSessions}) does not apply, because this sample " + + "selects different questions by construction."); } Console.WriteLine( $"longmemeval: frozen preparation preflight {plannedCalls} calls for " + @@ -611,6 +631,7 @@ await LongMemEvalOrphanSweep usePredicateVocabulary: options.UsePredicateVocabulary, extractionVocabularySha256: MemoryPredicateSeedVocabulary.Fingerprint, queryRelationLexiconSha256: MemoryRelationSeedTable.Fingerprint, + abstentionPolicy: options.AbstentionPolicy.ToString(), preparedAtUtc: DateTimeOffset.UtcNow.ToString("O"), description: options.Description ?? string.Empty, memoryTypes: options.MemoryTypes, @@ -1302,6 +1323,7 @@ private static string AgentEvalRevision() "--questions", "--resolve-query-relations", "--retain-prepared-volumes", "--reuse-prepared-volumes", "--seed", "--single-session-unified", "--description", "--memory-types", "--allow-stale-prepared", + "--abstention", "--abstention-proportion", "--use-predicate-vocabulary", ]; @@ -1369,7 +1391,9 @@ bool Has(string name) => ParseNonNegative(Value("--graphrag-items"), 0, "--graphrag-items"), Value("--description"), ParseMemoryTypes(Value("--memory-types")), - Has("--allow-stale-prepared")); + Has("--allow-stale-prepared"), + ParseAbstention(Value("--abstention")), + ParseAbstentionProportion(Value("--abstention-proportion"))); } /// @@ -1390,6 +1414,55 @@ private static IReadOnlyList ParseMemoryTypes(string? value) return types; } + /// + /// Parses --abstention exclude|as-sampled|only|target. + /// + /// + /// + /// Abstention questions are the only unanswerable questions in this dataset, and they are + /// what the retrieval-sufficiency AUC has to order against. Measured on a stratified 50: 32 of 33 + /// checkable questions had their answer present in memory, so the AUC rested on a single + /// observation and the two arms landed at 0.969 and 0.094 from the same data. Without an absent + /// class the metric is not noisy -- it is undefined. + /// + /// + /// Default as-sampled, which is what every recorded run used: LongMemEval-S contains few + /// _abs questions and stratified sampling almost never draws one, so across 52 recorded runs not + /// one ever ran. That default is preserved exactly rather than improved, because changing what a + /// sample contains changes every number computed from it. + /// + /// + private static AbstentionSamplingPolicy ParseAbstention(string? value) => + value?.ToLowerInvariant() switch + { + null or "" or "as-sampled" => AbstentionSamplingPolicy.AsSampled, + "exclude" => AbstentionSamplingPolicy.Exclude, + "only" => AbstentionSamplingPolicy.Only, + "target" => AbstentionSamplingPolicy.TargetProportion, + _ => throw new ArgumentException( + "--abstention must be one of: as-sampled, exclude, only, target."), + }; + + /// Parses --abstention-proportion, the share of the sample that must abstain. + /// + /// Only meaningful with --abstention target. Rejected outside (0,1): a proportion of 0 + /// silently means "exclude" and 1 means "only", and expressing either by accident would produce a + /// sample nobody chose. + /// + private static double? ParseAbstentionProportion(string? value) + { + if (string.IsNullOrWhiteSpace(value)) return null; + if (!double.TryParse(value, System.Globalization.NumberStyles.Float, + System.Globalization.CultureInfo.InvariantCulture, out var parsed) + || parsed is <= 0 or >= 1) + { + throw new ArgumentException( + "--abstention-proportion must be strictly between 0 and 1; use --abstention exclude " + + "or --abstention only for the endpoints."); + } + return parsed; + } + private static void Validate(PreparedPairOptions options) { if (!string.IsNullOrWhiteSpace(options.ReusePreparedVolume) && @@ -1639,7 +1712,9 @@ internal sealed record PreparedPairOptions( // reuse later, when nobody remembers what it was built for. string? Description = null, IReadOnlyList? MemoryTypesRequested = null, - bool AllowStalePrepared = false) + bool AllowStalePrepared = false, + AbstentionSamplingPolicy AbstentionPolicy = AbstentionSamplingPolicy.AsSampled, + double? AbstentionProportion = null) { /// The memory types this corpus was sampled for; empty means every type. internal IReadOnlyList MemoryTypes => MemoryTypesRequested ?? []; From 44203f72ad548021fc2d148d90b990da92707a7d Mon Sep 17 00:00:00 2001 From: Jose Luis Latorre Millas Date: Wed, 12 Aug 2026 13:34:03 +0200 Subject: [PATCH 04/31] Label answerability from the dataset, not from the extraction-failure heuristic Wiring abstention sampling did not fix the AUC, and the reason is the finding. Abstention questions turned out to be perfectly checkable -- and the presence gate reported 3 of 4 of them as PRESENT. Their topic is discussed in the conversation even though the specific fact is not, so the gold answer's distinctive tokens are there to find. The gate is a deliberately cheap floor for spotting extraction failure; it was never a ground truth for "was this question answerable", and using it as one put most of the unanswerable class on the wrong side of every pairwise comparison. The dataset already knows. An _abs question is unanswerable by definition, and AgentEval surfaces IsAbstention on every entry. That label is free, exact, and needs no gate -- so it now takes precedence, and the gate keeps its real job: deciding answerability for ORDINARY questions, where the dataset asserts nothing about whether our extraction happened to store the answer. An abstention question is also no longer required to have a checkable gold answer, since an abstention answer is often phrased in common words with no distinctive tokens; requiring the gate to confirm a dataset fact would drop exactly the questions that make the metric computable. Measured, on the same corpus, free (a reuse, no extraction): before structured AUC 0.969 / hybrid 0.094 over 32 present, 1 absent -- one observation, so each arm reported that question's luck after structured 0.600 / hybrid 0.600 over 5 present, 4 absent Both arms agreeing is the signal that it is now measuring the signal rather than a single draw. 0.600 sits exactly on the justification line fixed in code before any number existed -- at n=9, where one flip moves it 0.05. Weakly above a coin, and not yet worth building on. abstentionQuestions is reported alongside the denominators, so a zero -- meaning the unanswerable class came entirely from accidental extraction misses, as in the 50-question run -- is visible without recounting. 4158 unit + 376 LongMemEval green. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01PgDgctPpbTziBNE2RT8VdE --- .../SufficiencyLabellingTests.cs | 127 ++++++++++++++++++ .../AgentMemoryLongMemEvalAdapter.cs | 17 +++ .../LongMemEvalSufficiencyReport.cs | 20 ++- 3 files changed, 162 insertions(+), 2 deletions(-) create mode 100644 tests/AgentMemory.Tests.Unit.LongMemEval/SufficiencyLabellingTests.cs diff --git a/tests/AgentMemory.Tests.Unit.LongMemEval/SufficiencyLabellingTests.cs b/tests/AgentMemory.Tests.Unit.LongMemEval/SufficiencyLabellingTests.cs new file mode 100644 index 000000000..84b52d415 --- /dev/null +++ b/tests/AgentMemory.Tests.Unit.LongMemEval/SufficiencyLabellingTests.cs @@ -0,0 +1,127 @@ +using AgentMemory.LongMemEval; +using FluentAssertions; +using Xunit; + +namespace AgentMemory.Tests.Unit.LongMemEval; + +/// +/// Where "was this question answerable?" comes from — and why it is not the presence gate. +/// +/// +/// +/// The sufficiency AUC asks whether retrieval confidence ranks answerable questions above +/// unanswerable ones, so it needs a ground truth for answerability. The first version used the +/// answer-presence gate, which asks whether the gold answer's distinctive tokens appear anywhere in +/// memory — a deliberately cheap floor for spotting extraction failure. +/// +/// +/// Measured on an abstention-enriched sample, that gate reported 3 of 4 abstention questions as +/// "present". Their topic is discussed in the conversation even though the specific fact is +/// not, so the tokens are there. A heuristic floor for extraction failure is not a ground truth for +/// answerability, and using it as one put most of the unanswerable class on the wrong side of every +/// comparison. +/// +/// +/// The dataset already knows: an _abs question is unanswerable by definition. That label is +/// free, exact, and needs no gate. +/// +/// +public sealed class SufficiencyLabellingTests +{ + private static LongMemEvalQuestionTelemetry Question( + int number, + double? signal, + bool isAbstention = false, + bool checkable = true, + bool present = true) => + new(number, MessagesStored: 10, ItemsRetrieved: 5, RecallTruncated: false, Status: "ok") + { + SufficiencySignal = signal, + IsAbstention = isAbstention, + // (Checkable, Present) -- in that order. Getting it backwards here made a checkable + // missing answer read as uncheckable, and the test caught it. + AnswerPresence = new LongMemEvalAnswerPresenceResult(checkable, present, [], 0), + }; + + private static dynamic Report(params LongMemEvalQuestionTelemetry[] questions) => + LongMemEvalSufficiencyReport.From(questions); + + [Fact] + public void AnAbstentionQuestionCountsAsUnanswerableEvenWhenTheGateSaysPresent() + { + // THE case, and it is the common one: 3 of 4 abstention questions in a real run had their + // gold answer's tokens present in memory. Deferring to the gate put them on the answerable + // side and destroyed the class the AUC exists to order against. + var report = Report( + Question(1, 0.5, isAbstention: true, present: true), + Question(2, 0.9, present: true)); + + ((int)report.presentCount).Should().Be(1); + ((int)report.absentCount).Should().Be(1); + ((double?)report.auc).Should().Be(1.0, "the answerable question scored higher"); + } + + [Fact] + public void AnAbstentionQuestionNeedsNoCheckableGoldAnswer() + { + // Its unanswerability is a dataset fact, not a measurement. Requiring the gate to confirm it + // would drop exactly the questions that make the metric computable -- an abstention answer is + // often phrased in common words with no distinctive tokens to find. + var report = Report( + Question(1, 0.5, isAbstention: true, checkable: false, present: false), + Question(2, 0.9, present: true)); + + ((int)report.absentCount).Should().Be(1); + ((int)report.abstentionQuestions).Should().Be(1); + } + + [Fact] + public void AnOrdinaryQuestionStillNeedsTheGate() + { + // The other half. For a normal question the dataset asserts nothing about whether OUR + // extraction stored the answer, so the gate remains the only evidence -- and an unmeasurable + // one is excluded rather than guessed. + var report = Report( + Question(1, 0.9, present: true), + Question(2, 0.5, checkable: false, present: false), + Question(3, 0.4, present: false)); + + ((int)report.presentCount).Should().Be(1); + ((int)report.absentCount).Should().Be(1); + ((int)report.excludedNotCheckable).Should().Be(1); + } + + [Fact] + public void AnAbstentionQuestionIsNeverCountedAsUncheckable() + { + // It was not excluded, so reporting it under an exclusion count would make the denominators + // stop adding up -- and the denominators are the only way to judge whether the AUC means + // anything. + var report = Report(Question(1, 0.5, isAbstention: true, checkable: false)); + + ((int)report.excludedNotCheckable).Should().Be(0); + } + + [Fact] + public void TheAbstentionCountIsReportedSoAZeroIsVisible() + { + // When it is 0 the unanswerable class came entirely from accidental extraction misses, which + // is how a 50-question run ended up with ONE absent observation and two arms reporting 0.969 + // and 0.094 from the same corpus. A reader must be able to see that without recounting. + ((int)Report(Question(1, 0.9), Question(2, 0.4, present: false)).abstentionQuestions) + .Should().Be(0); + } + + [Fact] + public void AQuestionWithNoSignalIsExcludedWhateverItsLabel() + { + // Diagnostics off means the signal was never collected. An abstention label cannot substitute + // for the measurement the comparison is actually about. + var report = Report( + Question(1, null, isAbstention: true), + Question(2, 0.9, present: true)); + + ((int)report.absentCount).Should().Be(0); + ((int)report.excludedNoSignal).Should().Be(1); + } +} diff --git a/tools/AgentMemory.LongMemEval/AgentMemoryLongMemEvalAdapter.cs b/tools/AgentMemory.LongMemEval/AgentMemoryLongMemEvalAdapter.cs index dbacd8e01..3db9b0a98 100644 --- a/tools/AgentMemory.LongMemEval/AgentMemoryLongMemEvalAdapter.cs +++ b/tools/AgentMemory.LongMemEval/AgentMemoryLongMemEvalAdapter.cs @@ -868,6 +868,7 @@ _chatClient is LongMemEvalChatCallMeter callMeter unionGraphTotal: relationUnionGraphTotal), answerPresence: answerPresence, questionType: evidenceQuestion?.QuestionType, + isAbstention: evidenceQuestion?.IsAbstention ?? false, retrievedGoldCoverage: RetrievedGoldCoverage( recall.Context.RelevantFacts.Items, originsByMessageId @@ -984,6 +985,7 @@ private void RecordTelemetry( LongMemEvalRelationCompleteness? relationCompleteness = null, LongMemEvalAnswerPresenceResult? answerPresence = null, string? questionType = null, + bool isAbstention = false, string? answerPromptText = null) { lock (_stateLock) @@ -1020,6 +1022,7 @@ private void RecordTelemetry( // work can skip and none of it has ever asked. SufficiencySignal = SufficiencySignalOf(context), QuestionType = questionType, + IsAbstention = isAbstention, // Makes "expansion had nothing to expand" visible per question, instead of // requiring the lexicon to be consulted by hand after a run. ResolvedQueryRelations = context?.ResolvedQueryRelations ?? [], @@ -1725,6 +1728,20 @@ public sealed record LongMemEvalQuestionTelemetry( /// public double? SufficiencySignal { get; init; } + /// + /// Whether the dataset declares this question unanswerable (an _abs question). + /// + /// + /// Ground truth, and the reason it is carried separately from + /// . The presence gate asks whether the gold answer's distinctive + /// tokens appear anywhere in memory -- a deliberately cheap floor for spotting extraction + /// failure. Measured on an abstention-enriched sample, it reported 3 of 4 abstention + /// questions as "present": their topic is discussed in the conversation even though the specific + /// fact is not, so the tokens are there. For "was this answerable?" the dataset's own label is + /// correct and the heuristic is not. + /// + public bool IsAbstention { get; init; } + /// /// The benchmark's own question type, carried so the gate can be read per type. /// diff --git a/tools/AgentMemory.LongMemEval/LongMemEvalSufficiencyReport.cs b/tools/AgentMemory.LongMemEval/LongMemEvalSufficiencyReport.cs index e32891e7c..a5d4b8812 100644 --- a/tools/AgentMemory.LongMemEval/LongMemEvalSufficiencyReport.cs +++ b/tools/AgentMemory.LongMemEval/LongMemEvalSufficiencyReport.cs @@ -23,13 +23,24 @@ internal static object From(IReadOnlyList telemetr { ArgumentNullException.ThrowIfNull(telemetry); + // An abstention question is unanswerable BY DEFINITION -- the dataset says so -- and needs no + // gate to establish it. Everything else is labelled by the presence gate, which requires a + // checkable gold answer. + // + // The order matters and was wrong at first. Deferring to the gate for abstention questions + // labelled 3 of 4 of them "answerable", because their topic IS discussed in the conversation + // even though the specific fact is not, so the gold answer's tokens are present. A heuristic + // floor for extraction failure is not a ground truth for answerability, and using it as one + // put most of the unanswerable class on the wrong side of the comparison. var usable = telemetry - .Where(item => item.SufficiencySignal is not null && item.AnswerPresence is { Checkable: true }) + .Where(item => item.SufficiencySignal is not null + && (item.IsAbstention || item.AnswerPresence is { Checkable: true })) .ToArray(); var result = LongMemEvalSufficiencyAuc.Compute( [.. usable.Select(item => new LongMemEvalSufficiencyAuc.Observation( - item.SufficiencySignal!.Value, item.AnswerPresence!.Present))]); + item.SufficiencySignal!.Value, + AnswerPresent: !item.IsAbstention && item.AnswerPresence!.Present))]); return new { @@ -42,9 +53,14 @@ internal static object From(IReadOnlyList telemetr // Every question that could not contribute, and why. Without these the AUC's denominator // is unauditable -- and it is the denominator, not the number, that decides whether the // result means anything. + // How much of the unanswerable class came from the dataset's own label rather than from + // an extraction failure. When this is 0 the AUC is resting on accidental misses, which is + // how a 50-question run ended up with a single absent observation. + abstentionQuestions = usable.Count(item => item.IsAbstention), excludedNoSignal = telemetry.Count(item => item.SufficiencySignal is null), excludedNotCheckable = telemetry.Count( item => item.SufficiencySignal is not null + && !item.IsAbstention && item.AnswerPresence is null or { Checkable: false }), questionsConsidered = telemetry.Count, }; From ac7e7c69ea2724fbe869bc20cdbf19844b7793a9 Mon Sep 17 00:00:00 2001 From: Jose Luis Latorre Millas Date: Wed, 12 Aug 2026 13:35:43 +0200 Subject: [PATCH 05/31] Show the abstention policy in the corpus listing The two abstention-enriched corpora were indistinguishable from an ordinary one in the listing, which defeats the point of having a catalog to pick from: it is the difference between a corpus the sufficiency AUC can be computed on and one it cannot. Sealed in the manifest already; only the display was missing. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01PgDgctPpbTziBNE2RT8VdE --- .../LongMemEvalPreparedCorpusRegistry.cs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/tools/AgentMemory.LongMemEval/LongMemEvalPreparedCorpusRegistry.cs b/tools/AgentMemory.LongMemEval/LongMemEvalPreparedCorpusRegistry.cs index 0352366b8..9d5e1f3e4 100644 --- a/tools/AgentMemory.LongMemEval/LongMemEvalPreparedCorpusRegistry.cs +++ b/tools/AgentMemory.LongMemEval/LongMemEvalPreparedCorpusRegistry.cs @@ -1,4 +1,4 @@ -using System.Globalization; +using System.Globalization; using System.Text.Json; using System.Text.Json.Serialization; @@ -75,6 +75,7 @@ internal static void Record( EmbeddingDimensions = manifest.EmbeddingDimensions, AssistantContent = manifest.AssistantContent, ExtractionProvenance = manifest.ExtractionProvenance, + AbstentionPolicy = manifest.AbstentionPolicy, ExtractionVocabularySha256 = manifest.ExtractionVocabularySha256, AgentEvalRevision = manifest.AgentEvalRevision, }); @@ -135,12 +136,18 @@ internal static string Describe(IReadOnlyList entries, Date ? $"{(now - when).TotalDays:0} day(s) old" : "age unknown"; var types = entry.MemoryTypes is { Count: > 0 } t ? string.Join("+", t) : "all"; + // Shown because it is the difference between a corpus the sufficiency AUC can be computed + // on and one it cannot: without abstention questions the unanswerable class comes only + // from accidental extraction misses, which on a 50-question run meant one observation. + var abstention = entry.AbstentionPolicy is { Length: > 0 } policy && policy != "AsSampled" + ? $" abstention={policy}" + : string.Empty; var schema = entry.ManifestSchemaVersion < LongMemEvalPreparationManifest.CurrentSchemaVersion ? $" [manifest schema {entry.ManifestSchemaVersion} < {LongMemEvalPreparationManifest.CurrentSchemaVersion}" + " - some ingestion settings were not recorded, so reuse will report them as drift]" : string.Empty; return $" {entry.Volume}\n" - + $" {entry.Questions}q seed={entry.Seed} types={types} | {age} ({entry.PreparedAtUtc})\n" + + $" {entry.Questions}q seed={entry.Seed} types={types}{abstention} | {age} ({entry.PreparedAtUtc})\n" + $" extraction={entry.ExtractionModel} embedding={entry.EmbeddingModel}/{entry.EmbeddingDimensions}" + $" assistantContent={entry.AssistantContent} provenance={entry.ExtractionProvenance}\n" + $" {(string.IsNullOrWhiteSpace(entry.Description) ? "(no description)" : entry.Description)}{schema}"; @@ -172,6 +179,9 @@ internal sealed record PreparedCorpusEntry public int EmbeddingDimensions { get; init; } public string AssistantContent { get; init; } = ""; public string ExtractionProvenance { get; init; } = ""; + + /// Whether unanswerable questions were drawn into this corpus, and how. + public string AbstentionPolicy { get; init; } = ""; public string ExtractionVocabularySha256 { get; init; } = ""; public string AgentEvalRevision { get; init; } = ""; } From 5028859632a5b00240621405df41f26eadeadd52 Mon Sep 17 00:00:00 2001 From: Jose Luis Latorre Millas Date: Wed, 12 Aug 2026 14:34:11 +0200 Subject: [PATCH 06/31] 2.11: rescue the short-but-non-empty owner result with the scan, not more widening The vector index is global, so an owner filter is a POST-filter on a top-K drawn from every tenant: measured, a mean of 7 of 60 candidates reached the querying owner. Only a totally empty result triggered a rescue, on the argument that a short-but-non-empty result still answers the question. That argument has a measured counter-example. Question 5d3d2817 returned 2 facts from a 710-fact graph with the gold answer present at coverage 1.00, and both arms answered it wrongly. The claim is true for a small tenant and false for a crowded one, and the returned count cannot separate them. The plan asked for a ratio gate. That is the wrong shape: a fraction of the limit needs a cutoff nobody can justify, and any shortfall might be crowding. The gate is simply "short". The rescue is the SCAN, not another widening -- and this is what dissolves the original objection rather than trading against it. Widening is a second draw on the same global index, and a tenant losing to 50 neighbours at top-60 usually loses again at top-480. The scan is bounded by one owner rows, so the small tenant the "do not tax them" argument protected pays LESS for it than for a wider index query. It takes whichever result is larger, so a genuinely sparse owner never loses indexed rows to it. Off by default; it trades latency for recall on every short result and every recorded measurement was taken without it. Fact path only for now -- that is where the counter-example is -- with the other three tracked as 2.12, because "a setting only some components respect" is the exact defect this session already found twice. The scan is now one extracted helper reached by both the empty and short paths; two copies of a fallback drift. 4166 unit tests green. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01PgDgctPpbTziBNE2RT8VdE --- .../Options/MemoryOptions.cs | 23 ++++ .../Repositories/Neo4jFactRepository.cs | 87 ++++++++++---- .../Repositories/OwnerVectorOverFetch.cs | 38 ++++++ .../ShortOwnerResultRescueTests.cs | 113 ++++++++++++++++++ 4 files changed, 241 insertions(+), 20 deletions(-) create mode 100644 tests/AgentMemory.Tests.Unit/Repositories/ShortOwnerResultRescueTests.cs diff --git a/src/AgentMemory.Abstractions/Options/MemoryOptions.cs b/src/AgentMemory.Abstractions/Options/MemoryOptions.cs index 68fc61138..8f6b27948 100644 --- a/src/AgentMemory.Abstractions/Options/MemoryOptions.cs +++ b/src/AgentMemory.Abstractions/Options/MemoryOptions.cs @@ -23,6 +23,29 @@ public sealed record MemoryOptions /// Whether to enable GraphRAG integration. public bool EnableGraphRag { get; init; } + /// + /// Falls back to an owner-bounded similarity scan when an owner-scoped vector search returns + /// fewer rows than were asked for. + /// + /// + /// + /// Neo4j's vector index is global, so an owner filter is a POST-filter on a top-K drawn from every + /// tenant. Measured on a 50-owner corpus, a mean of 7 of 60 candidates reached the querying + /// owner. Today only a totally empty result triggers a rescue; a search returning 2 rows of + /// a requested 10 is accepted as the answer. + /// + /// + /// That is sometimes right and sometimes badly wrong: question 5d3d2817 returned 2 facts + /// from a 710-fact graph with the answer present, and was answered incorrectly in both arms. + /// + /// + /// Off by default. It costs one extra query per short result — bounded by the owner's own rows, + /// not the corpus — and every recorded measurement was taken without it, so turning it on is a + /// stated decision rather than an inherited one. + /// + /// + public bool RescueShortOwnerResults { get; init; } + // NOTE: extraction at the Core layer is explicit (call ExtractAndPersistAsync / // ExtractFromSessionAsync). Automatic extraction on message persist is an adapter concern, configured // by AgentFrameworkOptions.AutoExtractOnPersist. The former EnableAutoExtraction flag here was read diff --git a/src/AgentMemory.Neo4j/Repositories/Neo4jFactRepository.cs b/src/AgentMemory.Neo4j/Repositories/Neo4jFactRepository.cs index 3b34245c8..7c5e193f9 100644 --- a/src/AgentMemory.Neo4j/Repositories/Neo4jFactRepository.cs +++ b/src/AgentMemory.Neo4j/Repositories/Neo4jFactRepository.cs @@ -23,6 +23,7 @@ internal sealed partial class Neo4jFactRepository : IFactRepository, IUpsertPers internal const string OwnerKeyShared = "*"; private readonly INeo4jTransactionRunner _tx; + private readonly bool _rescueShortOwnerResults; private readonly ILogger _logger; private readonly MemoryRankingOptions _ranking; private readonly MemoryDecayOptions _decay; @@ -33,8 +34,10 @@ public Neo4jFactRepository( ILogger logger, IOptions? ranking = null, IOptions? decay = null, - IMemoryRankingContext? rankingContext = null) + IMemoryRankingContext? rankingContext = null, + IOptions? memoryOptions = null) { + _rescueShortOwnerResults = memoryOptions?.Value.RescueShortOwnerResults ?? false; _tx = tx; _logger = logger; _ranking = ranking?.Value ?? MemoryRankingOptions.Default; @@ -293,6 +296,43 @@ public async Task> GetBySubjectAsync( }, cancellationToken).ConfigureAwait(false); } + /// + /// Scores this owner's OWN facts directly, bypassing the global vector index. + /// + /// + /// Extracted because two different conditions now reach it — a totally empty scoped result, and + /// (opt-in) a short one — and two copies of a fallback drift. Bounded by one owner's rows rather + /// than by the corpus, which is what makes it affordable for the small tenant and effective for + /// the crowded one. + /// + private async Task> OwnerScopedScanAsync( + float[] queryEmbedding, + int limit, + double minScore, + MemoryScope scope, + bool includeShared, + bool currentValidTime, + CancellationToken cancellationToken) => + await _tx.ReadAsync(async runner => + { + var cursor = await runner.RunAsync( + FactQueries.SearchByVectorOwnerScopedFallback(includeShared, currentValidTime), + new Dictionary + { + ["embedding"] = queryEmbedding.ToList(), + ["limit"] = limit, + ["minScore"] = minScore, + ["ownerId"] = scope.OwnerId, + }).ConfigureAwait(false); + var records = await cursor.ToListAsync().ConfigureAwait(false); + return records.Select(r => + { + var node = r["node"].As(); + var score = r["score"].As(); + return (MapToFact(node, ReadEmbedding(node)), score); + }).ToList(); + }, cancellationToken).ConfigureAwait(false) ?? []; + public Task> SearchByVectorAsync( float[] queryEmbedding, int limit = 10, @@ -387,27 +427,34 @@ public async Task> GetBySubjectAsync( _logger.LogDebug( "Owner-scoped fact vector search still empty after widening to {Widened}; " + "falling back to a scoped similarity scan.", escalatedTopK ?? topK); - results = await _tx.ReadAsync(async runner => - { - var cursor = await runner.RunAsync( - FactQueries.SearchByVectorOwnerScopedFallback(includeShared, currentValidTime), - new Dictionary - { - ["embedding"] = queryEmbedding.ToList(), - ["limit"] = limit, - ["minScore"] = minScore, - ["ownerId"] = scope!.OwnerId, - }).ConfigureAwait(false); - var records = await cursor.ToListAsync().ConfigureAwait(false); - return records.Select(r => - { - var node = r["node"].As(); - var score = r["score"].As(); - return (MapToFact(node, ReadEmbedding(node)), score); - }).ToList(); - }, cancellationToken).ConfigureAwait(false) ?? []; + results = await OwnerScopedScanAsync( + queryEmbedding, limit, minScore, scope!, includeShared, currentValidTime, + cancellationToken).ConfigureAwait(false); } } + else if (_rescueShortOwnerResults + && OwnerVectorOverFetch.ShouldRescueShortResult(results.Count, limit, hasOwner)) + { + // 2.11. "Short still answers the question" is true for a small tenant and false for a + // crowded one, and the returned count cannot tell them apart: question 5d3d2817 returned + // 2 facts from a 710-fact graph with the answer present, and was answered wrongly. + // + // The rescue is the SCAN, not another widening. Widening is a second draw on the same + // global index, and a tenant losing to 50 neighbours at top-60 usually loses again at + // top-480. The scan is bounded by one owner's rows, so the small tenant this heuristic + // was protecting pays less than it would for a wider index query. + // + // Takes whichever result is larger rather than replacing outright: the scan applies the + // same minScore and orders by the same similarity, so a shorter scan result means the + // owner genuinely holds no more, and discarding indexed rows for it would be a loss. + _logger.LogDebug( + "Owner-scoped fact vector search returned {Returned} of {Limit}; rescuing with a " + + "scoped similarity scan.", results.Count, limit); + var scanned = await OwnerScopedScanAsync( + queryEmbedding, limit, minScore, scope!, includeShared, currentValidTime, + cancellationToken).ConfigureAwait(false); + if (scanned.Count > results.Count) results = scanned; + } if (activity is not null) { diff --git a/src/AgentMemory.Neo4j/Repositories/OwnerVectorOverFetch.cs b/src/AgentMemory.Neo4j/Repositories/OwnerVectorOverFetch.cs index 63257a270..4cb419a52 100644 --- a/src/AgentMemory.Neo4j/Repositories/OwnerVectorOverFetch.cs +++ b/src/AgentMemory.Neo4j/Repositories/OwnerVectorOverFetch.cs @@ -27,6 +27,20 @@ namespace AgentMemory.Neo4j.Repositories; /// with an extra query on every recall forever, because an owner holding three facts can never fill a /// ten-row limit. So: one extra query, only when the first pass found nothing, and bounded. /// +/// +/// That last argument has a measured counter-example. Question 5d3d2817 returned +/// 2 facts from a 710-fact graph with the gold answer present at coverage 1.00, and both arms +/// answered it wrongly. "Short still answers the question" is true for a small tenant and false for a +/// crowded one, and the returned count alone cannot tell them apart. +/// +/// +/// The rescue is therefore not more widening. Widening is another draw on the same global +/// index, and a tenant losing to 50 neighbours at top-60 usually loses again at top-480. The +/// owner-scoped similarity scan already used as the last resort is the right instrument: its cost +/// scales with one owner's data rather than with the corpus, so the small tenant this argument +/// was protecting pays almost nothing — scanning three facts is cheaper than a second index query — +/// while the crowded tenant gets its true top-K instead of whatever survived the neighbours. +/// /// internal static class OwnerVectorOverFetch { @@ -57,6 +71,30 @@ internal static int InitialTopK(int limit, bool hasOwner) => /// internal static bool ShouldEscalate(int returned, bool hasOwner) => hasOwner && returned == 0; + /// + /// Whether a scoped search that returned something, but less than asked for, should fall + /// back to the owner-bounded scan. + /// + /// + /// + /// Opt-in (MemoryOptions.RescueShortOwnerResults), because it trades latency for recall on + /// every short result and every recorded measurement was taken without it. + /// + /// + /// Deliberately not a fraction of the limit. A ratio invites a threshold nobody can + /// justify — is 4 of 10 starved? 6 of 10? — and the honest answer is that any shortfall might be + /// crowding, since the index gave the owner whatever the neighbours left. The scan is bounded by + /// the owner's own rows either way, so the gate is simply "short", and the cost question is + /// answered by the scan's shape rather than by guessing a cutoff. + /// + /// + /// Excludes the empty case, which already owns and which reaches the + /// same scan through a path that first tries one widened query. + /// + /// + internal static bool ShouldRescueShortResult(int returned, int limit, bool hasOwner) => + hasOwner && returned > 0 && returned < limit; + /// The widened width, capped. internal static int EscalatedTopK(int currentTopK) { diff --git a/tests/AgentMemory.Tests.Unit/Repositories/ShortOwnerResultRescueTests.cs b/tests/AgentMemory.Tests.Unit/Repositories/ShortOwnerResultRescueTests.cs new file mode 100644 index 000000000..3c5a4984a --- /dev/null +++ b/tests/AgentMemory.Tests.Unit/Repositories/ShortOwnerResultRescueTests.cs @@ -0,0 +1,113 @@ +using AgentMemory.Neo4j.Repositories; +using FluentAssertions; +using Xunit; + +namespace AgentMemory.Tests.Unit.Repositories; + +/// +/// When an owner-scoped vector search comes back short but not empty (PLAN 2.11). +/// +/// +/// +/// Neo4j's vector index is global, so an owner filter is a POST-filter on a top-K drawn from every +/// tenant. Measured on a 50-owner corpus, a mean of 7 of 60 candidates reached the querying +/// owner. Only a totally empty result triggered a rescue, on the argument that "a short-but-non-empty +/// result still answers the question". +/// +/// +/// That argument has a measured counter-example. Question 5d3d2817 returned 2 facts from +/// a 710-fact graph with the gold answer present at coverage 1.00, and both arms answered it wrongly. +/// The claim is true for a small tenant and false for a crowded one, and the returned count alone +/// cannot separate them. +/// +/// +/// The rescue is the owner-bounded scan, not another widening — widening is a second draw on +/// the same global index, and a tenant losing to 50 neighbours at top-60 usually loses again at +/// top-480. The scan's cost scales with one owner's rows, so the small tenant the original argument +/// protected pays less for it than for a wider index query. +/// +/// +public sealed class ShortOwnerResultRescueTests +{ + [Fact] + public void AShortScopedResultIsRescued() + { + // The measured case: 2 rows of a requested 10. + OwnerVectorOverFetch.ShouldRescueShortResult(returned: 2, limit: 10, hasOwner: true) + .Should().BeTrue(); + } + + [Fact] + public void AFullResultIsNotRescued() + { + // Nothing was crowded out: the limit was met, so there is no shortfall to explain. + OwnerVectorOverFetch.ShouldRescueShortResult(returned: 10, limit: 10, hasOwner: true) + .Should().BeFalse(); + } + + [Fact] + public void AnEmptyResultIsLeftToTheEscalationPath() + { + // Zero already has an owner: ShouldEscalate tries one widened index query first and only then + // falls back to the scan. Claiming it here too would run the scan twice on the same search. + OwnerVectorOverFetch.ShouldRescueShortResult(returned: 0, limit: 10, hasOwner: true) + .Should().BeFalse(); + OwnerVectorOverFetch.ShouldEscalate(returned: 0, hasOwner: true).Should().BeTrue(); + } + + [Fact] + public void AnUnscopedSearchIsNeverRescued() + { + // With no owner filter there is no post-filter and therefore no crowding: a short result means + // the corpus genuinely held that little above the floor, and a scan returns the same rows more + // slowly. + OwnerVectorOverFetch.ShouldRescueShortResult(returned: 2, limit: 10, hasOwner: false) + .Should().BeFalse(); + } + + [Fact] + public void TheTwoRescuePathsAreMutuallyExclusiveAtEveryCount() + { + // A count that satisfied both would run the widening AND the scan for one search. Asserted + // across the whole range rather than at the boundary, because the two predicates are written + // independently and nothing else forces them apart. + for (var returned = 0; returned <= 12; returned++) + { + var escalate = OwnerVectorOverFetch.ShouldEscalate(returned, hasOwner: true); + var rescue = OwnerVectorOverFetch.ShouldRescueShortResult(returned, limit: 10, hasOwner: true); + + (escalate && rescue).Should().BeFalse($"returned={returned} must not trigger both paths"); + } + } + + [Fact] + public void EveryShortfallCounts_ThereIsNoRatioThreshold() + { + // A fraction of the limit would need a cutoff nobody can justify -- is 4 of 10 starved? 6? -- + // and the honest answer is that any shortfall might be crowding, because the index gave the + // owner whatever the neighbours left. The scan is bounded by the owner's rows either way, so + // the cost question is answered by its shape rather than by guessing a threshold. + for (var returned = 1; returned < 10; returned++) + { + OwnerVectorOverFetch.ShouldRescueShortResult(returned, limit: 10, hasOwner: true) + .Should().BeTrue($"returned={returned} of 10 is short"); + } + } + + [Fact] + public void TheRescueIsOffByDefault() + { + // It trades latency for recall on every short result, and every recorded measurement was taken + // without it, so enabling it is a stated decision rather than an inherited one. + new Abstractions.Options.MemoryOptions().RescueShortOwnerResults.Should().BeFalse(); + } + + [Fact] + public void EscalationStillOnlyFiresOnEmpty() + { + // The byte-identical guarantee: 2.11 adds a path, it does not widen the existing one. A host + // that leaves the option off gets exactly today's behaviour. + OwnerVectorOverFetch.ShouldEscalate(returned: 1, hasOwner: true).Should().BeFalse(); + OwnerVectorOverFetch.ShouldEscalate(returned: 0, hasOwner: false).Should().BeFalse(); + } +} From 9c9b00512168b90267cbbab8fa360791fe3493a1 Mon Sep 17 00:00:00 2001 From: Jose Luis Latorre Millas Date: Wed, 12 Aug 2026 14:37:31 +0200 Subject: [PATCH 07/31] 6.3: give reasoning traces a trust level Traces were the one recall category with no trust signal. StartTraceAsync strips any caller-supplied trust_level -- correctly, since a self-assigned ApplicationTrusted would bypass the admission policy instruction-detection for the trace own Task text on recall -- and then nothing stamped one. Every trace read back as Untrusted, making "the agent generated this" and "no signal was recorded" the same value, which is what the enum exists to prevent. ModelGenerated is the accurate label: a trace is the agent own record of what it did, not something a user asserted. Stamped from configuration, after the strip, never from the caller. Safe at shipped defaults, and asserted rather than claimed: MinimumTrustForAdmissionBypass defaults to ApplicationTrusted, which ModelGenerated does not reach, so no trace gains a bypass; MinimumTrustForSystemRole defaults to Untrusted, which everything already met. A host that lowered the bypass threshold is the one case this changes, and it can set the level back. An existing test proved "stripped" by asserting the result was Untrusted. That was the same thing until now, so the proxy stopped tracking the property; it now asserts what it always meant -- the caller ApplicationTrusted is not what got stored. 4171 unit tests green. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01PgDgctPpbTziBNE2RT8VdE --- .../Options/ReasoningMemoryOptions.cs | 29 ++++ .../Services/ReasoningMemoryService.cs | 8 +- .../Services/ReasoningMemoryServiceTests.cs | 9 +- .../Services/TraceTrustStampingTests.cs | 127 ++++++++++++++++++ 4 files changed, 171 insertions(+), 2 deletions(-) create mode 100644 tests/AgentMemory.Tests.Unit/Services/TraceTrustStampingTests.cs diff --git a/src/AgentMemory.Abstractions/Options/ReasoningMemoryOptions.cs b/src/AgentMemory.Abstractions/Options/ReasoningMemoryOptions.cs index e31ce4a13..07183c322 100644 --- a/src/AgentMemory.Abstractions/Options/ReasoningMemoryOptions.cs +++ b/src/AgentMemory.Abstractions/Options/ReasoningMemoryOptions.cs @@ -1,3 +1,5 @@ +using AgentMemory.Abstractions.Domain; + namespace AgentMemory.Abstractions.Options; /// @@ -13,4 +15,31 @@ public sealed record ReasoningMemoryOptions /// Maximum number of traces to retain per session. public int? MaxTracesPerSession { get; init; } + + /// + /// The trust level stamped on every reasoning trace. + /// + /// + /// + /// Traces were the one recall category with no trust level at all: StartTraceAsync strips + /// any caller-supplied trust_level — correctly, since a caller must never self-assign a + /// level that bypasses the admission policy — and then nothing stamped one, so every trace read + /// back as . That is indistinguishable from "no signal + /// recorded", which is exactly what the enum exists to avoid. + /// + /// + /// is the accurate label: a trace is the agent's own + /// record of what it did, not something a user asserted. + /// + /// + /// Safe at shipped defaults, and worth stating why. MinimumTrustForAdmissionBypass + /// defaults to , which + /// does not reach, so no trace gains an admission + /// bypass. MinimumTrustForSystemRole defaults to , + /// which everything already met. A host that has lowered the bypass threshold to + /// or below is the one case where this changes + /// behaviour, and it can set this back to . + /// + /// + public MemoryTrustLevel DefaultTraceTrustLevel { get; init; } = MemoryTrustLevel.ModelGenerated; } diff --git a/src/AgentMemory.Core/Services/ReasoningMemoryService.cs b/src/AgentMemory.Core/Services/ReasoningMemoryService.cs index c5ac8229b..70f2a3209 100644 --- a/src/AgentMemory.Core/Services/ReasoningMemoryService.cs +++ b/src/AgentMemory.Core/Services/ReasoningMemoryService.cs @@ -84,7 +84,13 @@ public async Task StartTraceAsync( // #92 Phase 3: trust_level is a framework-reserved metadata key -- a caller of this public // service method must never be able to self-assign a trust level that bypasses the admission // policy's instruction-like-content detection for this trace's Task text on recall. - Metadata = (metadata ?? new Dictionary()).WithoutCallerSuppliedTrustLevel() + // Stripped first, then stamped from CONFIGURATION -- never from the caller. The strip is + // the security property (#92 Phase 3); the stamp closes 6.3, where every trace read back + // as Untrusted because nothing ever assigned one, making "the agent generated this" and + // "no signal was recorded" the same value. + Metadata = (metadata ?? new Dictionary()) + .WithoutCallerSuppliedTrustLevel() + .WithTrustLevel(_options.DefaultTraceTrustLevel) }; _logger.LogDebug("Starting trace {TraceId} for session {SessionId}", trace.TraceId, sessionId); diff --git a/tests/AgentMemory.Tests.Unit/Services/ReasoningMemoryServiceTests.cs b/tests/AgentMemory.Tests.Unit/Services/ReasoningMemoryServiceTests.cs index 9037714d9..1bd295b0e 100644 --- a/tests/AgentMemory.Tests.Unit/Services/ReasoningMemoryServiceTests.cs +++ b/tests/AgentMemory.Tests.Unit/Services/ReasoningMemoryServiceTests.cs @@ -89,7 +89,14 @@ public async Task StartTraceAsync_CallerSuppliedTrustLevel_IsStripped() var result = await sut.StartTraceAsync("session-1", "Test task", metadata: callerMetadata); - result.Metadata.GetTrustLevel().Should().Be(MemoryTrustLevel.Untrusted); + // Asserts the caller's value was discarded, NOT that the result is Untrusted. Those were the + // same thing until 6.3, when traces began carrying a configured trust level -- so the old + // assertion was checking a proxy that has since stopped tracking the property. What must hold + // is that ApplicationTrusted, which the caller asked for and which would bypass the admission + // policy, is not what got stored. + result.Metadata.GetTrustLevel().Should().NotBe(MemoryTrustLevel.ApplicationTrusted); + result.Metadata.GetTrustLevel().Should() + .Be(new ReasoningMemoryOptions().DefaultTraceTrustLevel); } [Fact] diff --git a/tests/AgentMemory.Tests.Unit/Services/TraceTrustStampingTests.cs b/tests/AgentMemory.Tests.Unit/Services/TraceTrustStampingTests.cs new file mode 100644 index 000000000..8b915738c --- /dev/null +++ b/tests/AgentMemory.Tests.Unit/Services/TraceTrustStampingTests.cs @@ -0,0 +1,127 @@ +using AgentMemory.Abstractions.Domain; +using AgentMemory.Abstractions.Options; +using AgentMemory.Abstractions.Repositories; +using AgentMemory.Abstractions.Services; +using AgentMemory.Core.Services; +using FluentAssertions; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using NSubstitute; +using Xunit; + +namespace AgentMemory.Tests.Unit.Services; + +/// +/// Reasoning traces carry a trust level (PLAN 6.3) — stamped from configuration, never from the caller. +/// +/// +/// +/// Traces were the one recall category with no trust signal at all. StartTraceAsync strips any +/// caller-supplied trust_level — correctly, since a caller must never self-assign a level that +/// bypasses the admission policy — and then nothing stamped one, so every trace read back as +/// . "The agent generated this" and "no signal was recorded" +/// were the same value, which is precisely what the enum exists to prevent. +/// +/// +/// Both halves have to hold at once, and they pull in opposite directions: the caller must not be able +/// to set it, and it must not stay unset. +/// +/// +public sealed class TraceTrustStampingTests +{ + private readonly IReasoningTraceRepository _traceRepo = Substitute.For(); + private readonly IReasoningStepRepository _stepRepo = Substitute.For(); + private readonly IToolCallRepository _toolRepo = Substitute.For(); + private readonly IEmbeddingOrchestrator _embeddings = Substitute.For(); + private readonly IClock _clock = Substitute.For(); + private readonly IIdGenerator _idGen = Substitute.For(); + + private readonly List _written = []; + + public TraceTrustStampingTests() + { + _clock.UtcNow.Returns(DateTimeOffset.UtcNow); + _idGen.GenerateId().Returns(_ => Guid.NewGuid().ToString("N")); + _embeddings.EmbedAsync(Arg.Any(), Arg.Any()).Returns(new float[8]); + _traceRepo.AddAsync(Arg.Any(), Arg.Any()) + .Returns(ci => + { + _written.Add(ci.Arg()); + return Task.FromResult(ci.Arg()); + }); + } + + private ReasoningMemoryService CreateSut(MemoryTrustLevel? level = null) => + new(_traceRepo, _stepRepo, _toolRepo, _embeddings, _clock, _idGen, + Options.Create(level is { } value + ? new ReasoningMemoryOptions { DefaultTraceTrustLevel = value } + : new ReasoningMemoryOptions()), + NullLogger.Instance, + new DefaultMemoryIsolationPolicy( + Options.Create(new MemoryIsolationOptions()), + NullLogger.Instance)); + + [Fact] + public async Task ATraceIsStampedRatherThanLeftUntrusted() + { + await CreateSut().StartTraceAsync("s-1", "deploy the service"); + + _written.Should().ContainSingle() + .Which.Metadata.GetTrustLevel().Should().Be(MemoryTrustLevel.ModelGenerated, + "a trace is the agent's own record of what it did, and Untrusted is indistinguishable " + + "from no signal at all"); + } + + [Fact] + public async Task ACallerStillCannotSetItsOwnTrustLevel() + { + // The security half, and it must survive the stamping. A caller that could self-assign + // ApplicationTrusted would bypass the admission policy's instruction-like-content detection + // for this trace's own Task text on recall. + await CreateSut().StartTraceAsync("s-1", "deploy the service", + metadata: new Dictionary + { + ["trust_level"] = nameof(MemoryTrustLevel.ApplicationTrusted), + }); + + _written.Should().ContainSingle() + .Which.Metadata.GetTrustLevel().Should().Be(MemoryTrustLevel.ModelGenerated, + "the caller's value is stripped before the configured one is stamped"); + } + + [Fact] + public async Task TheLevelIsConfigurable() + { + // The escape hatch for a host that lowered MinimumTrustForAdmissionBypass to ModelGenerated + // or below -- the one configuration where this stamp changes behaviour. + await CreateSut(MemoryTrustLevel.Untrusted).StartTraceAsync("s-1", "deploy the service"); + + _written.Should().ContainSingle() + .Which.Metadata.GetTrustLevel().Should().Be(MemoryTrustLevel.Untrusted); + } + + [Fact] + public async Task OtherCallerMetadataSurvives() + { + // Only the reserved key is removed. Stripping the whole dictionary would silently discard + // application data that has nothing to do with trust. + await CreateSut().StartTraceAsync("s-1", "deploy the service", + metadata: new Dictionary { ["ticket"] = "OPS-42" }); + + var stored = _written.Should().ContainSingle().Subject; + stored.Metadata.Should().ContainKey("ticket"); + stored.Metadata["ticket"].Should().Be("OPS-42"); + } + + [Fact] + public void TheDefaultDoesNotReachTheAdmissionBypassThreshold() + { + // Why stamping by default is safe: the shipped bypass threshold is ApplicationTrusted, which + // ModelGenerated does not reach. If this ever inverts, a trace would start bypassing the + // instruction-detection it is currently subject to. + var shipped = new ReasoningMemoryOptions().DefaultTraceTrustLevel; + var bypass = new AgentMemory.AgentFramework.ContextFormatOptions().MinimumTrustForAdmissionBypass; + + ((int)shipped).Should().BeLessThan((int)bypass); + } +} From 82f46f54663ec92f8f3b8716b0566bd4b0bc5a61 Mon Sep 17 00:00:00 2001 From: Jose Luis Latorre Millas Date: Wed, 12 Aug 2026 14:40:11 +0200 Subject: [PATCH 08/31] 1.5/1.7: the valid-time falsifier, and what it found 1.5 asks whether turning the gate on changes results over a corpus nobody set validity bounds on -- if it does, an unaudited writer exists. The inventory is the answer, so the tests pin it rather than asserting a vague "nothing changed". Two writers, both deliberate: temporal extraction when TemporalValidityMode .Extract asks the model for a window, and supersession, which stamps valid_until as it closes a fact. The second matters because 9.1 made it reachable on the ingestion path rather than only from an offline hygiene pass. And it changes nothing. Supersession stamps invalidated_at too, and the transaction-clock filter already removes the fact from live recall whatever this option says. So the valid-time gate is REDUNDANT for superseded facts and load-bearing only for windows the conversation actually stated -- worth recording before someone reaches for it to hide superseded data. Six live-Neo4j tests: both modes identical with no bounds; each bound filtering when set (including valid_from in the future, which a valid_until-only implementation would return as current); null meaning unbounded rather than "not yet valid"; and the supersession interaction. 1.7 documents all three of this session opt-in behaviours in CHANGELOG with the measurement that motivated each. TCK exposure confirmed zero. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01PgDgctPpbTziBNE2RT8VdE --- CHANGELOG.md | 36 ++++ .../ValidTimeGateIntegrationTests.cs | 157 ++++++++++++++++++ 2 files changed, 193 insertions(+) create mode 100644 tests/AgentMemory.Tests.Integration/Repositories/ValidTimeGateIntegrationTests.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 7310c9a61..d601b299f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,42 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- **Valid-time recall (opt-in).** `RecallOptions.ValidTime = ValidTimeMode.Current` filters facts on + their real-world window (`valid_from`/`valid_until`) rather than only on the transaction clock. + Default `Ignore`, which is byte-for-byte today's behaviour, and `MemoryProfile.Parity` resolves to + `Ignore` — parity means parity. **TCK exposure is zero**: the bridge exposes neither `/search_facts` + nor `/get_facts_about`, so no conformance surface can observe this. + + Two things write validity bounds, and only two: temporal extraction, when + `TemporalValidityMode.Extract` asks the model for a window, and **supersession**, which stamps + `valid_until` as it closes a fact. The second is worth knowing about because it changes nothing + here — supersession also stamps `invalidated_at`, and the transaction-clock filter already removes + the fact from live recall whatever this option says. **So the gate is redundant for superseded + facts and load-bearing only for windows the conversation actually stated.** Reaching for it to hide + superseded data solves a problem that is already solved. + +- **Reasoning traces carry a trust level.** `ReasoningMemoryOptions.DefaultTraceTrustLevel`, default + `ModelGenerated` — a trace is the agent's own record of what it did. Traces were the one recall + category with no trust signal: caller-supplied `trust_level` is stripped (a caller must never + self-assign a level that bypasses admission checks) and nothing then stamped one, so every trace + read back as `Untrusted` — indistinguishable from "no signal recorded". Safe at shipped defaults: + `MinimumTrustForAdmissionBypass` is `ApplicationTrusted`, which `ModelGenerated` does not reach. + +- **Owner-starvation rescue for short results (opt-in).** `MemoryOptions.RescueShortOwnerResults`. + Neo4j's vector index is global, so an owner filter is a post-filter on a top-K drawn from every + tenant — measured, a mean of **7 of 60** candidates reached the querying owner. Previously only a + *totally empty* result triggered a rescue, on the argument that a short result still answers the + question. It does for a small tenant and does not for a crowded one: one measured question returned + **2 facts from a 710-fact graph** with the answer present, and was answered wrongly. + + The rescue is the owner-bounded **scan**, not a wider index query — widening is another draw on the + same global index, and a tenant losing to 50 neighbours at top-60 usually loses again at top-480. + The scan's cost scales with one owner's rows, so the small tenant the original argument protected + pays *less* for it than for a wider index query. Off by default, and it takes whichever result is + larger, so a genuinely sparse owner never loses indexed rows. + ### Changed - **A greeting no longer costs a full recall.** Measured on the hermetic `PERF-R-01` scenario — a diff --git a/tests/AgentMemory.Tests.Integration/Repositories/ValidTimeGateIntegrationTests.cs b/tests/AgentMemory.Tests.Integration/Repositories/ValidTimeGateIntegrationTests.cs new file mode 100644 index 000000000..d5869cffb --- /dev/null +++ b/tests/AgentMemory.Tests.Integration/Repositories/ValidTimeGateIntegrationTests.cs @@ -0,0 +1,157 @@ +using FluentAssertions; +using Microsoft.Extensions.Logging.Abstractions; +using AgentMemory.Abstractions.Domain; +using AgentMemory.Abstractions.Options; +using AgentMemory.Neo4j.Repositories; +using AgentMemory.Tests.Integration.Fixtures; + +namespace AgentMemory.Tests.Integration.Repositories; + +/// +/// The valid-time gate's falsifier (PLAN 1.5): with the gate off and on, results must be identical +/// unless something actually wrote a validity bound. +/// +/// +/// +/// The gate filters on valid_from/valid_until. If turning it on changes results over a +/// corpus that nobody set validity bounds on, then an unaudited writer of validity bounds exists +/// and must be found before the option ships. The audit is only as good as the inventory, so these +/// tests pin the inventory rather than assert a vague "nothing changed". +/// +/// +/// Two writers exist, both deliberate. Temporal extraction sets bounds when +/// TemporalValidityMode.Extract asks the model for them, and supersession stamps +/// valid_until when a fact is closed. The second is the one worth testing, because write-time +/// supersession (9.1) made it reachable on the ingestion path rather than only from an offline +/// hygiene pass. +/// +/// +/// The interesting result is that supersession changes nothing here: it stamps invalidated_at +/// too, and the existing transaction-clock filter already removes the fact from live recall. So the +/// valid-time gate is redundant for superseded facts and load-bearing only for facts whose +/// real-world window was stated. Worth knowing before anyone reaches for it to hide superseded data. +/// +/// +[Collection("Neo4j Integration")] +[Trait("Category", "Integration")] +public class ValidTimeGateIntegrationTests : IAsyncLifetime +{ + private readonly Neo4jIntegrationFixture _fixture; + private readonly Neo4jFactRepository _facts; + + private static readonly float[] Embedding = [0.2f, 0.4f, 0.4f, 0.2f]; + + public ValidTimeGateIntegrationTests(Neo4jIntegrationFixture fixture) + { + _fixture = fixture; + _facts = new Neo4jFactRepository( + fixture.TransactionRunner, NullLogger.Instance); + } + + public Task InitializeAsync() => _fixture.CleanDatabaseAsync(); + public Task DisposeAsync() => Task.CompletedTask; + + private Task StoreAsync( + string @object, DateTimeOffset? validFrom = null, DateTimeOffset? validUntil = null) => + _facts.UpsertAsync(new Fact + { + FactId = $"fact-{Guid.NewGuid():N}", + Subject = "user", + Predicate = "lives in", + Object = @object, + Confidence = 0.9, + OwnerId = "alice", + Embedding = Embedding, + ValidFrom = validFrom, + ValidUntil = validUntil, + CreatedAtUtc = DateTimeOffset.UtcNow, + }); + + private Task> SearchAsync(ValidTimeMode mode) => + _facts.SearchByVectorAsync( + Embedding, limit: 20, minScore: 0.0, + scope: MemoryScope.For("alice", includeShared: false), validTime: mode); + + private static string[] Objects(IReadOnlyList<(Fact Fact, double Score)> results) => + [.. results.Select(r => r.Fact.Object).OrderBy(o => o, StringComparer.Ordinal)]; + + [Fact] + public async Task WithNoValidityBoundsAnywhereBothModesReturnTheSameSet() + { + // THE falsifier. If these differ, something wrote a bound nobody audited. + await StoreAsync("Zurich"); + await StoreAsync("Basel"); + await StoreAsync("Bern"); + + Objects(await SearchAsync(ValidTimeMode.Ignore)) + .Should().Equal(Objects(await SearchAsync(ValidTimeMode.Current))); + } + + [Fact] + public async Task AnExpiredFactIsExcludedOnlyWhenTheGateIsOn() + { + // The gate doing its job, and the proof that the test above is not passing vacuously: if the + // gate never filtered anything, both assertions would hold for the wrong reason. + await StoreAsync("Zurich"); + await StoreAsync("Basel", validUntil: DateTimeOffset.UtcNow.AddDays(-1)); + + Objects(await SearchAsync(ValidTimeMode.Ignore)).Should().Equal("Basel", "Zurich"); + Objects(await SearchAsync(ValidTimeMode.Current)).Should().Equal("Zurich"); + } + + [Fact] + public async Task AFactNotYetValidIsExcludedOnlyWhenTheGateIsOn() + { + // Both bounds, not just the upper one: a valid_from in the future is the case an + // implementation that only checked valid_until would return as current. + await StoreAsync("Zurich"); + await StoreAsync("Geneva", validFrom: DateTimeOffset.UtcNow.AddDays(1)); + + Objects(await SearchAsync(ValidTimeMode.Ignore)).Should().Equal("Geneva", "Zurich"); + Objects(await SearchAsync(ValidTimeMode.Current)).Should().Equal("Zurich"); + } + + [Fact] + public async Task AnOpenEndedFactIsCurrentUnderBothModes() + { + // Null must mean "unbounded", never "not yet valid". An implementation comparing nulls with + // ordinary operators would drop every fact that never stated a window -- which is nearly all + // of them. + await StoreAsync("Zurich", validFrom: DateTimeOffset.UtcNow.AddYears(-1), validUntil: null); + + Objects(await SearchAsync(ValidTimeMode.Current)).Should().Equal("Zurich"); + } + + // ── the second writer: supersession ─────────────────────────────────── + + [Fact] + public async Task ASupersededFactIsAlreadyGoneBeforeTheGateIsConsulted() + { + // Supersession stamps valid_until AND invalidated_at, and the transaction-clock filter runs + // regardless of this option. So the gate is redundant for superseded facts and load-bearing + // only for stated real-world windows. Anyone reaching for valid-time to hide superseded data + // is solving a problem that is already solved. + var loser = await StoreAsync("Basel"); + var winner = await StoreAsync("Zurich"); + await _facts.SupersedeAsync( + loser.FactId, winner.FactId, MemoryScope.For("alice", includeShared: false)); + + Objects(await SearchAsync(ValidTimeMode.Ignore)).Should().Equal("Zurich"); + Objects(await SearchAsync(ValidTimeMode.Current)).Should().Equal("Zurich"); + } + + [Fact] + public async Task SupersessionIsTheOnlyWriterThatFiresWithoutTemporalExtraction() + { + // The inventory, asserted. Ordinary ingestion writes null bounds; only an explicitly supplied + // window (temporal extraction) or a supersession sets one. If a third writer appears, the + // first test in this class starts failing and this one says where to look. + var untouched = await StoreAsync("Zurich"); + + var stored = await _facts.GetByIdAsync(untouched.FactId); + + stored.Should().NotBeNull(); + stored!.ValidFrom.Should().BeNull("ordinary ingestion states no window"); + stored.ValidUntil.Should().BeNull(); + } +} From fe54e76995a36f8e7ac253d86b0462725b6747c2 Mon Sep 17 00:00:00 2001 From: Jose Luis Latorre Millas Date: Wed, 12 Aug 2026 14:50:57 +0200 Subject: [PATCH 09/31] 4.6a: survive provider content refusals instead of losing the whole build The 50-question abstention build died after 270 of 616 extraction calls. Two sessions of a public research dataset tripped an Azure content policy; the batch-split path treated the 400 as a shape failure, halved the batch, re-sent the same text to the same filter, and propagated once the batch reached size 1. An hour of work lost to content that was never going to be accepted. A refusal is terminal. Neither a retry nor a split changes the text or the policy, so the only useful responses are to skip the affected sessions or abandon the run -- and skipping is overwhelmingly better PROVIDED the loss is recorded. A corpus with gaps that looks complete would have those gaps attributed to recall, which is the failure this whole track exists to prevent. So: content refusals are classified separately from shape failures (matched on the provider own vocabulary, since 400 covers both "your request is malformed", which splitting legitimately diagnoses, and "I will not process this", which it cannot); the refused sessions are skipped with an empty result; the count is recorded in diagnostics, warned loudly, sealed into the manifest and folded into its fingerprint. And refused above a 2hare of planned sessions, because the failure changes character with scale: a handful out of 2,418 is noise, hundreds is a different corpus wearing the same name. The observed rate here is a fraction of one percent, so a run approaching the limit has something new happening. Transient failures (408/429/5xx) are neither refusals nor shape failures and keep their existing handling -- the same text may well succeed later. 4185 unit tests green. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01PgDgctPpbTziBNE2RT8VdE --- .../LlmExtractionBatchDiagnostics.cs | 37 ++++- .../LlmMultiSessionUnifiedMemoryExtractor.cs | 51 +++++++ .../ContentRefusalResilienceTests.cs | 141 ++++++++++++++++++ .../LongMemEvalPreparationManifest.cs | 6 + .../LongMemEvalPreparedPairProgram.cs | 43 +++++- 5 files changed, 273 insertions(+), 5 deletions(-) create mode 100644 tests/AgentMemory.Tests.Unit/Extraction/ContentRefusalResilienceTests.cs diff --git a/src/AgentMemory.Extraction.Llm/LlmExtractionBatchDiagnostics.cs b/src/AgentMemory.Extraction.Llm/LlmExtractionBatchDiagnostics.cs index 5ce9235ee..20573050c 100644 --- a/src/AgentMemory.Extraction.Llm/LlmExtractionBatchDiagnostics.cs +++ b/src/AgentMemory.Extraction.Llm/LlmExtractionBatchDiagnostics.cs @@ -8,6 +8,8 @@ internal sealed class LlmExtractionBatchDiagnostics private readonly ConcurrentQueue _details = new(); private long _splits; private long _droppedDetails; + private long _contentRejections; + private long _sessionsRefused; internal void RecordSplit(Exception exception, int sourceSessions) { @@ -22,11 +24,36 @@ internal void RecordSplit(Exception exception, int sourceSessions) Interlocked.Increment(ref _droppedDetails); } + /// + /// Records a batch the provider refused on content grounds, and the sessions lost with it. + /// + /// + /// Counted separately from splits because it is a different event with a different remedy: a + /// split is a recoverable shape problem, a refusal is terminal for that text. The session count + /// is what a reader needs — "2 refusals" says nothing about whether the corpus is usable, "2 + /// refusals costing 6 sessions of 2,418" does. + /// + internal void RecordContentRejection(Exception exception, int sourceSessions) + { + ArgumentNullException.ThrowIfNull(exception); + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(sourceSessions); + Interlocked.Increment(ref _contentRejections); + Interlocked.Add(ref _sessionsRefused, sourceSessions); + _details.Enqueue(new LlmExtractionBatchSplitDetail( + "content-rejected", + sourceSessions, + exception.GetType().FullName ?? exception.GetType().Name)); + while (_details.Count > MaximumDetails && _details.TryDequeue(out _)) + Interlocked.Increment(ref _droppedDetails); + } + internal LlmExtractionBatchDiagnosticsSnapshot Snapshot() => new( Interlocked.Read(ref _splits), _details.ToArray(), - Interlocked.Read(ref _droppedDetails)); + Interlocked.Read(ref _droppedDetails), + Interlocked.Read(ref _contentRejections), + Interlocked.Read(ref _sessionsRefused)); private static string Classify(Exception exception) => exception.Message switch @@ -46,14 +73,18 @@ private static string Classify(Exception exception) => internal sealed record LlmExtractionBatchDiagnosticsSnapshot( long Splits, IReadOnlyList Details, - long DroppedDetails) + long DroppedDetails, + long ContentRejections = 0, + long SessionsRefused = 0) { internal LlmExtractionBatchDiagnosticsSnapshot Delta( LlmExtractionBatchDiagnosticsSnapshot baseline) => new( Splits - baseline.Splits, Details.Skip(Math.Min(Details.Count, baseline.Details.Count)).ToArray(), - DroppedDetails - baseline.DroppedDetails); + DroppedDetails - baseline.DroppedDetails, + ContentRejections - baseline.ContentRejections, + SessionsRefused - baseline.SessionsRefused); } internal sealed record LlmExtractionBatchSplitDetail( diff --git a/src/AgentMemory.Extraction.Llm/LlmMultiSessionUnifiedMemoryExtractor.cs b/src/AgentMemory.Extraction.Llm/LlmMultiSessionUnifiedMemoryExtractor.cs index c20368894..0715c23d7 100644 --- a/src/AgentMemory.Extraction.Llm/LlmMultiSessionUnifiedMemoryExtractor.cs +++ b/src/AgentMemory.Extraction.Llm/LlmMultiSessionUnifiedMemoryExtractor.cs @@ -247,6 +247,23 @@ private async Task> Extract // count. Removing it does not remove a working recovery; it removes a misleading one. A real // transport retry is tracked separately, because it must be reconciled with the harness's // exact-call-count invariant rather than quietly breaking it. + // The provider refused this content and will refuse it again, at any batch size. Skipping the + // affected sessions costs their extraction; propagating costs the ENTIRE preparation, which + // for 50 questions is ~616 calls and over an hour. Recorded, never silent: the count reaches + // the manifest, so a corpus with gaps says so instead of looking complete. + catch (Exception ex) when (IsContentRejection(ex)) + { + _batchDiagnostics?.RecordContentRejection(ex, batch.Count); + _logger.LogWarning( + ex, + "Provider refused the content of {Count} source session(s); their extraction is " + + "skipped and recorded. The corpus will be missing whatever those sessions held.", + batch.Count); + return batch.ToDictionary( + request => request.SessionId, + _ => new UnifiedExtractionResult(), + StringComparer.Ordinal); + } catch (Exception ex) when (batch.Count > 1 && IsBatchShapeFailure(ex)) { _batchDiagnostics?.RecordSplit(ex, batch.Count); @@ -508,9 +525,43 @@ internal static bool IsBatchShapeFailure(Exception exception) if (exception is FormatException) return true; + // A content-filter rejection is a 4xx, but it is NOT a shape failure: the request is + // well-formed and the provider is refusing the content. Splitting re-sends the same text to + // the same filter, so it can only fail again, one session at a time, until the batch reaches + // size 1 and the exception escapes. + if (IsContentRejection(exception)) return false; + var status = Internal.LlmExtractionRunner.TryGetStatus(exception); return status is >= 400 and < 500 and not 408 and not 429; } + /// + /// Whether the provider refused this content outright, rather than failing to process it. + /// + /// + /// + /// Terminal by nature. A retry sends the same text to the same policy, and a split sends smaller + /// pieces of the same text; neither can succeed. Measured cost of treating it as retryable: a + /// 50-question preparation died after 270 of 616 calls because two sessions of a public + /// research dataset tripped an Azure content policy. + /// + /// + /// Matched on the provider's own vocabulary rather than on status alone, because 400 covers both + /// "your request is malformed" (which splitting legitimately diagnoses) and "I will not process + /// this" (which it cannot). + /// + /// + internal static bool IsContentRejection(Exception exception) + { + var status = Internal.LlmExtractionRunner.TryGetStatus(exception); + if (status != 400) return false; + + var message = exception.Message; + return message.Contains("content_filter", StringComparison.OrdinalIgnoreCase) + || message.Contains("content filter", StringComparison.OrdinalIgnoreCase) + || message.Contains("cyber_policy", StringComparison.OrdinalIgnoreCase) + || message.Contains("ResponsibleAIPolicyViolation", StringComparison.OrdinalIgnoreCase); + } + private sealed class BatchValidationException(string message) : FormatException(message); } diff --git a/tests/AgentMemory.Tests.Unit/Extraction/ContentRefusalResilienceTests.cs b/tests/AgentMemory.Tests.Unit/Extraction/ContentRefusalResilienceTests.cs new file mode 100644 index 000000000..150376355 --- /dev/null +++ b/tests/AgentMemory.Tests.Unit/Extraction/ContentRefusalResilienceTests.cs @@ -0,0 +1,141 @@ +using System.Net; +using AgentMemory.Extraction.Llm; +using FluentAssertions; +using Xunit; + +namespace AgentMemory.Tests.Unit.Extraction; + +/// +/// A provider that refuses some content must not cost the whole preparation. +/// +/// +/// +/// Measured, 2026-08-12. A 50-question cold build died after 270 of 616 extraction calls +/// because two sessions of a public research dataset tripped an Azure content policy. The batch-split +/// path treated the 400 as a shape failure, halved the batch, re-sent the same text to the same +/// filter, and propagated once the batch reached size 1 — losing an hour of work to content that was +/// never going to be accepted. +/// +/// +/// A refusal is terminal: neither a retry nor a split changes the text or the policy. The only +/// useful responses are to skip the affected sessions or to abandon the run, and skipping is +/// overwhelmingly better provided the loss is recorded — a corpus with gaps that looks complete +/// would attribute those gaps to recall. +/// +/// +public sealed class ContentRefusalResilienceTests +{ + // HttpRequestException rather than ClientResultException: TryGetStatus reads the status + // reflectively from either, and the test should not take a provider package dependency to + // classify an error the production code deliberately classifies without one. + private static Exception Refusal(string body) => + new HttpRequestException($"HTTP 400 ({body})", null, HttpStatusCode.BadRequest); + + private static Exception Status(int code) => + new HttpRequestException($"HTTP {code} (something else)", null, (HttpStatusCode)code); + + [Theory] + [InlineData("content_filter")] + [InlineData("invalid_request_error: cyber_policy")] + [InlineData("ResponsibleAIPolicyViolation")] + [InlineData("The response was filtered due to the content filter")] + public void TheProvidersRefusalVocabularyIsRecognised(string body) + { + // Matched on the provider's own words, not on status alone: 400 covers both "your request is + // malformed" -- which splitting legitimately diagnoses -- and "I will not process this", + // which it cannot. + LlmMultiSessionUnifiedMemoryExtractor.IsContentRejection(Refusal(body)).Should().BeTrue(); + } + + [Fact] + public void ARefusalIsNotTreatedAsABatchShapeFailure() + { + // THE fix. Classifying it as a shape failure is what made the extractor split, re-send the + // same text to the same policy, and eventually propagate. + var refusal = Refusal("content_filter"); + + LlmMultiSessionUnifiedMemoryExtractor.IsContentRejection(refusal).Should().BeTrue(); + LlmMultiSessionUnifiedMemoryExtractor.IsBatchShapeFailure(refusal).Should().BeFalse(); + } + + [Fact] + public void AnOrdinaryBadRequestIsStillASplittableShapeFailure() + { + // The other direction, and the one that must not regress: a genuine 400 -- an over-long batch, + // a malformed schema -- is exactly what halving the batch diagnoses. + var malformed = Status(400); + + LlmMultiSessionUnifiedMemoryExtractor.IsContentRejection(malformed).Should().BeFalse(); + LlmMultiSessionUnifiedMemoryExtractor.IsBatchShapeFailure(malformed).Should().BeTrue(); + } + + [Theory] + [InlineData(408)] + [InlineData(429)] + [InlineData(500)] + [InlineData(503)] + public void TransientAndServerFailuresAreNeitherRefusalsNorShapeFailures(int status) + { + // A timeout, a throttle or a server error may succeed on the same text later, so neither + // skipping the content nor splitting the batch is the right response. + LlmMultiSessionUnifiedMemoryExtractor.IsContentRejection(Status(status)).Should().BeFalse(); + LlmMultiSessionUnifiedMemoryExtractor.IsBatchShapeFailure(Status(status)).Should().BeFalse(); + } + + [Fact] + public void AFormatFailureRemainsASplittableShapeFailure() + { + LlmMultiSessionUnifiedMemoryExtractor.IsBatchShapeFailure(new FormatException("bad json")) + .Should().BeTrue(); + LlmMultiSessionUnifiedMemoryExtractor.IsContentRejection(new FormatException("bad json")) + .Should().BeFalse(); + } + + [Fact] + public void ARefusalIsRecordedWithTheSessionsItCost() + { + // "2 refusals" says nothing about whether the corpus is usable. "2 refusals costing 6 of + // 2,418 sessions" does, and it is the session count that the tolerance and the manifest need. + var diagnostics = new LlmExtractionBatchDiagnostics(); + + diagnostics.RecordContentRejection(Refusal("content_filter"), sourceSessions: 4); + diagnostics.RecordContentRejection(Refusal("content_filter"), sourceSessions: 2); + + var snapshot = diagnostics.Snapshot(); + snapshot.ContentRejections.Should().Be(2); + snapshot.SessionsRefused.Should().Be(6); + } + + [Fact] + public void RefusalsAreCountedSeparatelyFromSplits() + { + // Different events with different remedies: a split is a recoverable shape problem, a refusal + // is terminal for that text. Folding them together would hide a corpus losing content behind + // a counter that looks like ordinary batching noise. + var diagnostics = new LlmExtractionBatchDiagnostics(); + + diagnostics.RecordSplit(new FormatException("bad json"), sourceSessions: 4); + diagnostics.RecordContentRejection(Refusal("content_filter"), sourceSessions: 1); + + var snapshot = diagnostics.Snapshot(); + snapshot.Splits.Should().Be(1); + snapshot.ContentRejections.Should().Be(1); + snapshot.Details.Should().Contain(d => d.Reason == "content-rejected"); + } + + [Fact] + public void ADeltaSubtractsRefusalsToo() + { + // The harness reconciles per-question deltas against a baseline. A field that did not subtract + // would report every earlier refusal again on every subsequent question. + var diagnostics = new LlmExtractionBatchDiagnostics(); + diagnostics.RecordContentRejection(Refusal("content_filter"), sourceSessions: 3); + var baseline = diagnostics.Snapshot(); + + diagnostics.RecordContentRejection(Refusal("content_filter"), sourceSessions: 2); + var delta = diagnostics.Snapshot().Delta(baseline); + + delta.ContentRejections.Should().Be(1); + delta.SessionsRefused.Should().Be(2); + } +} diff --git a/tools/AgentMemory.LongMemEval/LongMemEvalPreparationManifest.cs b/tools/AgentMemory.LongMemEval/LongMemEvalPreparationManifest.cs index df2871564..520d84816 100644 --- a/tools/AgentMemory.LongMemEval/LongMemEvalPreparationManifest.cs +++ b/tools/AgentMemory.LongMemEval/LongMemEvalPreparationManifest.cs @@ -51,6 +51,9 @@ internal sealed record LongMemEvalPreparationManifest( string QueryRelationLexiconSha256 = "", string ExtractionProvenance = "Batch", string AbstentionPolicy = "AsSampled", + // Sessions whose content the provider refused, so their material is absent from this corpus. + // Part of the fingerprint: a corpus with gaps is not the same corpus as one without. + int RefusedSourceSessions = 0, // ── Catalog metadata (schema 6) ────────────────────────────────────── // Not part of the fingerprint: these describe the build for a human, and two corpora that differ // only in their description are the same corpus. @@ -95,6 +98,7 @@ internal static LongMemEvalPreparationManifest Create( string queryRelationLexiconSha256 = "", string extractionProvenance = "Batch", string abstentionPolicy = "AsSampled", + int refusedSourceSessions = 0, string preparedAtUtc = "", string description = "", IReadOnlyList? memoryTypes = null, @@ -168,6 +172,7 @@ internal static LongMemEvalPreparationManifest Create( QueryRelationLexiconSha256: queryRelationLexiconSha256, ExtractionProvenance: extractionProvenance, AbstentionPolicy: abstentionPolicy, + RefusedSourceSessions: refusedSourceSessions, PreparedAtUtc: preparedAtUtc, Description: description, MemoryTypes: memoryTypes ?? [], @@ -295,6 +300,7 @@ private static string ComputeCurrentFingerprint(LongMemEvalPreparationManifest m manifest.QueryRelationLexiconSha256, manifest.ExtractionProvenance, manifest.AbstentionPolicy, + manifest.RefusedSourceSessions, manifest.QuestionSeed, manifest.UseJsonResponseFormat, manifest.ExtractionResponseContract, diff --git a/tools/AgentMemory.LongMemEval/LongMemEvalPreparedPairProgram.cs b/tools/AgentMemory.LongMemEval/LongMemEvalPreparedPairProgram.cs index c26ee530a..13acab088 100644 --- a/tools/AgentMemory.LongMemEval/LongMemEvalPreparedPairProgram.cs +++ b/tools/AgentMemory.LongMemEval/LongMemEvalPreparedPairProgram.cs @@ -32,6 +32,16 @@ internal static class LongMemEvalPreparedPairProgram private const double ColdBuildSpeedTargetMilliseconds = 900_000d; private const int FixedTenExpectedSourceSessions = 474; + + /// + /// How much of a corpus may be missing to provider content refusals before it stops being usable. + /// + /// + /// A handful of sessions out of thousands is noise; hundreds is a different corpus wearing the + /// same name. 2% is deliberately low — the observed rate on this dataset is a fraction of one + /// percent, so a run approaching this limit has something new happening and should stop. + /// + private const double MaximumRefusedSessionShare = 0.02; internal static async Task RunAsync(string[] args) { try @@ -546,8 +556,36 @@ await LongMemEvalOrphanSweep // retries 0. Split sub-calls run under their own Activity, so the activity-based // retry counter cannot see them - a split appears as bare extra calls that neither // counter accounts for. One guard accepted that shape and the other rejected it. - var recordedSplits = baseProfile.Services - .GetRequiredService().Snapshot().Splits; + var batchDiagnostics = baseProfile.Services + .GetRequiredService().Snapshot(); + var recordedSplits = batchDiagnostics.Splits; + + // A corpus with gaps must never look complete. The provider refuses some content + // outright -- two sessions of this public research dataset tripped an Azure policy -- + // and skipping them is far better than losing a 616-call preparation to it. But the + // loss is real: those sessions contributed nothing to the graph, so any question + // depending on them is unanswerable for a reason that has nothing to do with recall. + // + // Tolerated up to a small share and REFUSED beyond it, because the failure changes + // character with scale: a handful of sessions out of 2,418 is noise, and hundreds is a + // different corpus wearing the same name. + if (batchDiagnostics.SessionsRefused > 0) + { + var refusedShare = (double)batchDiagnostics.SessionsRefused / Math.Max(1, plannedSourceSessions); + Console.Error.WriteLine( + $"longmemeval: WARNING - the provider refused {batchDiagnostics.ContentRejections} " + + $"batch(es) covering {batchDiagnostics.SessionsRefused} of {plannedSourceSessions} " + + $"source sessions ({refusedShare:P1}). Their content is absent from this corpus, " + + "and it is recorded in the manifest so a later reuse cannot mistake it for complete."); + if (refusedShare > MaximumRefusedSessionShare) + { + throw new InvalidOperationException( + $"Provider content refusals cost {batchDiagnostics.SessionsRefused} of " + + $"{plannedSourceSessions} source sessions ({refusedShare:P1}), above the " + + $"{MaximumRefusedSessionShare:P0} tolerance. A corpus missing that much is a " + + "different corpus, and measuring against it would attribute the loss to recall."); + } + } var successfulCalls = extractionSnapshot.Calls - extractionSnapshot.Failures; var accountingAcceptable = AgentMemoryLongMemEvalAdapter.IsBatchAccountingAcceptable( @@ -632,6 +670,7 @@ await LongMemEvalOrphanSweep extractionVocabularySha256: MemoryPredicateSeedVocabulary.Fingerprint, queryRelationLexiconSha256: MemoryRelationSeedTable.Fingerprint, abstentionPolicy: options.AbstentionPolicy.ToString(), + refusedSourceSessions: checked((int)batchDiagnostics.SessionsRefused), preparedAtUtc: DateTimeOffset.UtcNow.ToString("O"), description: options.Description ?? string.Empty, memoryTypes: options.MemoryTypes, From cc3482498846e1f13e5c1e4c2be207a7d752f7c8 Mon Sep 17 00:00:00 2001 From: Jose Luis Latorre Millas Date: Wed, 12 Aug 2026 15:29:33 +0200 Subject: [PATCH 10/31] Isolate a refused batch instead of skipping it, and record which session Two gaps in yesterday hour-old fix, both exposed by asking the obvious operator questions: which model, and which prompt. 1. The fix skipped the WHOLE batch. Measured, the batch that killed the 616-call preparation went 4 -> 2 -> 1 and still failed at 1, so one session content was responsible and its three batch-mates were innocent. A multi-session refusal is now split first -- not to retry, since the policy is deterministic for the same text, but to ISOLATE -- and only the individual session that is still refused at size 1 is skipped. 2. Nothing recorded WHICH session. The batch diagnostics are deliberately content-free, which is right, but they recorded only the exception type and batch size -- so a refusal that will recur left no trace of what caused it, and could not be investigated or raised with the provider. The session ID is now recorded and printed. The identifier, never the text: the id is the handle and the dataset is the lookup, which keeps the diagnostics free of dataset content while still being actionable. On retry specifically: there is nothing to retry. An Azure content filter is deterministic for the same input, so re-sending cannot succeed. Isolation is the only recovery that gains anything, and it gains the batch-mates. 4187 unit tests green. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01PgDgctPpbTziBNE2RT8VdE --- .../LlmExtractionBatchDiagnostics.cs | 17 ++++++-- .../LlmMultiSessionUnifiedMemoryExtractor.cs | 43 +++++++++++++++---- .../ContentRefusalResilienceTests.cs | 30 +++++++++++++ .../LongMemEvalPreparedPairProgram.cs | 11 +++-- 4 files changed, 86 insertions(+), 15 deletions(-) diff --git a/src/AgentMemory.Extraction.Llm/LlmExtractionBatchDiagnostics.cs b/src/AgentMemory.Extraction.Llm/LlmExtractionBatchDiagnostics.cs index 20573050c..f3d5278f7 100644 --- a/src/AgentMemory.Extraction.Llm/LlmExtractionBatchDiagnostics.cs +++ b/src/AgentMemory.Extraction.Llm/LlmExtractionBatchDiagnostics.cs @@ -10,6 +10,7 @@ internal sealed class LlmExtractionBatchDiagnostics private long _droppedDetails; private long _contentRejections; private long _sessionsRefused; + private readonly ConcurrentQueue _refusedSessionIds = new(); internal void RecordSplit(Exception exception, int sourceSessions) { @@ -33,12 +34,17 @@ internal void RecordSplit(Exception exception, int sourceSessions) /// is what a reader needs — "2 refusals" says nothing about whether the corpus is usable, "2 /// refusals costing 6 sessions of 2,418" does. /// - internal void RecordContentRejection(Exception exception, int sourceSessions) + internal void RecordContentRejection( + Exception exception, int sourceSessions, string? sessionId = null) { ArgumentNullException.ThrowIfNull(exception); ArgumentOutOfRangeException.ThrowIfNegativeOrZero(sourceSessions); Interlocked.Increment(ref _contentRejections); Interlocked.Add(ref _sessionsRefused, sourceSessions); + // The identifier, never the text. A refusal that leaves no trace of what caused it cannot be + // investigated or reported to the provider -- and this recurs -- but the diagnostics are + // deliberately free of dataset content, so the id is the handle and the dataset is the lookup. + if (!string.IsNullOrEmpty(sessionId)) _refusedSessionIds.Enqueue(sessionId); _details.Enqueue(new LlmExtractionBatchSplitDetail( "content-rejected", sourceSessions, @@ -53,7 +59,8 @@ internal LlmExtractionBatchDiagnosticsSnapshot Snapshot() => _details.ToArray(), Interlocked.Read(ref _droppedDetails), Interlocked.Read(ref _contentRejections), - Interlocked.Read(ref _sessionsRefused)); + Interlocked.Read(ref _sessionsRefused), + _refusedSessionIds.ToArray()); private static string Classify(Exception exception) => exception.Message switch @@ -75,7 +82,8 @@ internal sealed record LlmExtractionBatchDiagnosticsSnapshot( IReadOnlyList Details, long DroppedDetails, long ContentRejections = 0, - long SessionsRefused = 0) + long SessionsRefused = 0, + IReadOnlyList? RefusedSessionIds = null) { internal LlmExtractionBatchDiagnosticsSnapshot Delta( LlmExtractionBatchDiagnosticsSnapshot baseline) => @@ -84,7 +92,8 @@ internal LlmExtractionBatchDiagnosticsSnapshot Delta( Details.Skip(Math.Min(Details.Count, baseline.Details.Count)).ToArray(), DroppedDetails - baseline.DroppedDetails, ContentRejections - baseline.ContentRejections, - SessionsRefused - baseline.SessionsRefused); + SessionsRefused - baseline.SessionsRefused, + (RefusedSessionIds ?? []).Skip((baseline.RefusedSessionIds ?? []).Count).ToArray()); } internal sealed record LlmExtractionBatchSplitDetail( diff --git a/src/AgentMemory.Extraction.Llm/LlmMultiSessionUnifiedMemoryExtractor.cs b/src/AgentMemory.Extraction.Llm/LlmMultiSessionUnifiedMemoryExtractor.cs index 0715c23d7..778f06e94 100644 --- a/src/AgentMemory.Extraction.Llm/LlmMultiSessionUnifiedMemoryExtractor.cs +++ b/src/AgentMemory.Extraction.Llm/LlmMultiSessionUnifiedMemoryExtractor.cs @@ -247,18 +247,45 @@ private async Task> Extract // count. Removing it does not remove a working recovery; it removes a misleading one. A real // transport retry is tracked separately, because it must be reconciled with the harness's // exact-call-count invariant rather than quietly breaking it. - // The provider refused this content and will refuse it again, at any batch size. Skipping the - // affected sessions costs their extraction; propagating costs the ENTIRE preparation, which - // for 50 questions is ~616 calls and over an hour. Recorded, never silent: the count reaches - // the manifest, so a corpus with gaps says so instead of looking complete. - catch (Exception ex) when (IsContentRejection(ex)) + // A refusal on a MULTI-session batch is split first, exactly like a shape failure -- not to + // retry (the policy is deterministic for the same text) but to ISOLATE. Measured: the batch + // that killed a 616-call preparation went 4 -> 2 -> 1 and still failed at 1, so one session's + // content was responsible and its three batch-mates were innocent. Skipping the whole batch + // would have discarded all four. + catch (Exception ex) when (batch.Count > 1 && IsContentRejection(ex)) { - _batchDiagnostics?.RecordContentRejection(ex, batch.Count); + _batchDiagnostics?.RecordSplit(ex, batch.Count); _logger.LogWarning( ex, - "Provider refused the content of {Count} source session(s); their extraction is " - + "skipped and recorded. The corpus will be missing whatever those sessions held.", + "Provider refused a batch of {Count} source sessions; splitting to isolate which " + + "one(s) it objects to, so the rest are not lost with it.", batch.Count); + var midpoint = batch.Count / 2; + var isolatedLeft = await ExtractOrSplitAsync( + batch.Take(midpoint).ToArray(), maxInputTokens, cancellationToken).ConfigureAwait(false); + var isolatedRight = await ExtractOrSplitAsync( + batch.Skip(midpoint).ToArray(), maxInputTokens, cancellationToken).ConfigureAwait(false); + return isolatedLeft.Concat(isolatedRight) + .ToDictionary(pair => pair.Key, pair => pair.Value, StringComparer.Ordinal); + } + + // Isolated to one session, and the provider still refuses it: terminal. No retry and no + // further split changes the text or the policy. Skipping costs this session's extraction; + // propagating costs the ENTIRE preparation -- ~616 calls and over an hour for 50 questions. + // + // The SESSION ID is recorded, not the content. A refusal that leaves no trace of what caused + // it cannot be investigated or reported to the provider, and this will recur: the id is the + // handle an operator needs to look the conversation up in the dataset themselves, without the + // diagnostics carrying dataset text they are deliberately free of. + catch (Exception ex) when (IsContentRejection(ex)) + { + var refused = batch[0].SessionId; + _batchDiagnostics?.RecordContentRejection(ex, batch.Count, refused); + _logger.LogWarning( + ex, + "Provider refused source session '{SessionId}' on content grounds; its extraction is " + + "skipped and recorded. Whatever that conversation held is absent from this corpus.", + refused); return batch.ToDictionary( request => request.SessionId, _ => new UnifiedExtractionResult(), diff --git a/tests/AgentMemory.Tests.Unit/Extraction/ContentRefusalResilienceTests.cs b/tests/AgentMemory.Tests.Unit/Extraction/ContentRefusalResilienceTests.cs index 150376355..17b577422 100644 --- a/tests/AgentMemory.Tests.Unit/Extraction/ContentRefusalResilienceTests.cs +++ b/tests/AgentMemory.Tests.Unit/Extraction/ContentRefusalResilienceTests.cs @@ -47,6 +47,23 @@ public void TheProvidersRefusalVocabularyIsRecognised(string body) LlmMultiSessionUnifiedMemoryExtractor.IsContentRejection(Refusal(body)).Should().BeTrue(); } + [Fact] + public void AMultiSessionRefusalIsSplitToIsolateTheOffender() + { + // Not a retry -- the policy is deterministic for the same text -- but isolation. Measured: the + // batch that killed a 616-call preparation went 4 -> 2 -> 1 and still failed at 1, so ONE + // session was responsible and its three batch-mates were innocent. Skipping the whole batch + // would have discarded all four. + // + // Asserted through the classification the split path keys on: a refusal must not be a shape + // failure (or it would split forever and then propagate), while the extractor splits it for + // its own reason at batch > 1. + var refusal = Refusal("content_filter"); + + LlmMultiSessionUnifiedMemoryExtractor.IsContentRejection(refusal).Should().BeTrue(); + LlmMultiSessionUnifiedMemoryExtractor.IsBatchShapeFailure(refusal).Should().BeFalse(); + } + [Fact] public void ARefusalIsNotTreatedAsABatchShapeFailure() { @@ -91,6 +108,19 @@ public void AFormatFailureRemainsASplittableShapeFailure() .Should().BeFalse(); } + [Fact] + public void TheRefusedSessionIdIsRecordedSoItCanBeInvestigated() + { + // "Something was refused" is not a report anyone can act on, and content filters recur. The + // ID is the handle: an operator looks the conversation up in the dataset themselves, which + // keeps the diagnostics free of dataset text while still being actionable. + var diagnostics = new LlmExtractionBatchDiagnostics(); + + diagnostics.RecordContentRejection(Refusal("content_filter"), 1, sessionId: "session-42"); + + diagnostics.Snapshot().RefusedSessionIds.Should().Equal("session-42"); + } + [Fact] public void ARefusalIsRecordedWithTheSessionsItCost() { diff --git a/tools/AgentMemory.LongMemEval/LongMemEvalPreparedPairProgram.cs b/tools/AgentMemory.LongMemEval/LongMemEvalPreparedPairProgram.cs index 13acab088..d66d4ccf1 100644 --- a/tools/AgentMemory.LongMemEval/LongMemEvalPreparedPairProgram.cs +++ b/tools/AgentMemory.LongMemEval/LongMemEvalPreparedPairProgram.cs @@ -574,9 +574,14 @@ await LongMemEvalOrphanSweep var refusedShare = (double)batchDiagnostics.SessionsRefused / Math.Max(1, plannedSourceSessions); Console.Error.WriteLine( $"longmemeval: WARNING - the provider refused {batchDiagnostics.ContentRejections} " - + $"batch(es) covering {batchDiagnostics.SessionsRefused} of {plannedSourceSessions} " - + $"source sessions ({refusedShare:P1}). Their content is absent from this corpus, " - + "and it is recorded in the manifest so a later reuse cannot mistake it for complete."); + + $"source session(s) on content grounds, {batchDiagnostics.SessionsRefused} of " + + $"{plannedSourceSessions} ({refusedShare:P1}). Their content is absent from this " + + "corpus, and it is recorded in the manifest so a later reuse cannot mistake it " + + "for complete."); + // The ids, so the refusal is investigable. Content filters recur, and "something + // was refused" is not a report anyone can act on or raise with a provider. + foreach (var refusedId in batchDiagnostics.RefusedSessionIds ?? []) + Console.Error.WriteLine($" - refused session: {refusedId}"); if (refusedShare > MaximumRefusedSessionShare) { throw new InvalidOperationException( From e9ef85db31b251ff2a952007324d7c58be03ee53 Mon Sep 17 00:00:00 2001 From: Jose Luis Latorre Millas Date: Wed, 12 Aug 2026 16:09:57 +0200 Subject: [PATCH 11/31] 2.12: the short-owner rescue on all four scoped vector paths The fact path got the rescue first because that is where the measured counter-example lives (2 facts of a 710-fact graph, answer present, answered wrongly). Leaving the other three would have meant a host enabling RescueShortOwnerResults gets it on facts and silently not on entities, preferences or traces -- the "a setting only some components respect" shape this project has already found twice, in TemporalValidityMode inert on the batch rung and in a read-only filter that removed nothing. Each path now has its owner-scoped scan extracted into one helper reached by both the empty and the short condition; two copies of a fallback drift. The trace path needed its own signature because successFilter and proceduresOnly travel with it -- and proceduresOnly reaching the scan is the fix from 7.5, preserved. Off by default everywhere, as on the fact path. 4187 unit + 332 non-NAMS integration green. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01PgDgctPpbTziBNE2RT8VdE --- .../Repositories/Neo4jEntityRepository.cs | 73 +++++++++++++----- .../Repositories/Neo4jPreferenceRepository.cs | 73 +++++++++++++----- .../Neo4jReasoningTraceRepository.cs | 75 +++++++++++++++---- 3 files changed, 165 insertions(+), 56 deletions(-) diff --git a/src/AgentMemory.Neo4j/Repositories/Neo4jEntityRepository.cs b/src/AgentMemory.Neo4j/Repositories/Neo4jEntityRepository.cs index b7618b2b2..afb62fabb 100644 --- a/src/AgentMemory.Neo4j/Repositories/Neo4jEntityRepository.cs +++ b/src/AgentMemory.Neo4j/Repositories/Neo4jEntityRepository.cs @@ -19,9 +19,42 @@ internal sealed partial class Neo4jEntityRepository : IEntityRepository, IUpsert { private readonly INeo4jTransactionRunner _tx; + private readonly bool _rescueShortOwnerResults; private readonly ILogger _logger; private readonly MemoryRankingOptions _ranking; private readonly MemoryDecayOptions _decay; + /// Scores this owner's OWN entitys directly, bypassing the global vector index. + /// + /// Extracted because two conditions reach it -- an empty scoped result, and (opt-in) a short one + /// -- and two copies of a fallback drift. + /// + private async Task> OwnerScopedScanAsync( + float[] queryEmbedding, + int limit, + double minScore, + MemoryScope scope, + bool includeShared, + CancellationToken cancellationToken) => + await _tx.ReadAsync(async runner => + { + var cursor = await runner.RunAsync( + EntityQueries.SearchByVectorOwnerScopedFallback(includeShared), + new Dictionary + { + ["embedding"] = queryEmbedding.ToList(), + ["limit"] = limit, + ["minScore"] = minScore, + ["ownerId"] = scope.OwnerId, + }).ConfigureAwait(false); + var records = await cursor.ToListAsync().ConfigureAwait(false); + return records.Select(r => + { + var node = r["node"].As(); + var score = r["score"].As(); + return (MapToEntity(node, ReadEmbedding(node)), score); + }).ToList(); + }, cancellationToken).ConfigureAwait(false) ?? []; + private readonly IMemoryRankingContext? _rankingContext; public Neo4jEntityRepository( @@ -29,8 +62,10 @@ public Neo4jEntityRepository( ILogger logger, IOptions? ranking = null, IOptions? decay = null, - IMemoryRankingContext? rankingContext = null) + IMemoryRankingContext? rankingContext = null, + IOptions? memoryOptions = null) { + _rescueShortOwnerResults = memoryOptions?.Value.RescueShortOwnerResults ?? false; _tx = tx; _logger = logger; _ranking = ranking?.Value ?? MemoryRankingOptions.Default; @@ -234,27 +269,25 @@ public async Task> GetByNameAsync( _logger.LogDebug( "Owner-scoped entity vector search still empty after widening; falling back to a " + "scoped similarity scan."); - results = await _tx.ReadAsync(async runner => - { - var cursor = await runner.RunAsync( - EntityQueries.SearchByVectorOwnerScopedFallback(includeShared), - new Dictionary - { - ["embedding"] = queryEmbedding.ToList(), - ["limit"] = limit, - ["minScore"] = minScore, - ["ownerId"] = scope!.OwnerId, - }).ConfigureAwait(false); - var records = await cursor.ToListAsync().ConfigureAwait(false); - return records.Select(r => - { - var node = r["node"].As(); - var score = r["score"].As(); - return (MapToEntity(node, ReadEmbedding(node)), score); - }).ToList(); - }, cancellationToken).ConfigureAwait(false) ?? []; + results = await OwnerScopedScanAsync( + queryEmbedding, limit, minScore, scope!, includeShared, cancellationToken) + .ConfigureAwait(false); } } + else if (_rescueShortOwnerResults + && OwnerVectorOverFetch.ShouldRescueShortResult(results.Count, limit, hasOwner)) + { + // 2.12, the same rescue the fact path carries. Extended here because "a setting only some + // components respect" is the defect shape this project has found twice -- a host enabling + // RescueShortOwnerResults would otherwise get it on facts and silently not on entitys. + _logger.LogDebug( + "Owner-scoped entity vector search returned {Returned} of {Limit}; rescuing with a " + + "scoped similarity scan.", results.Count, limit); + var scanned = await OwnerScopedScanAsync( + queryEmbedding, limit, minScore, scope!, includeShared, cancellationToken) + .ConfigureAwait(false); + if (scanned.Count > results.Count) results = scanned; + } EmitVectorYield( activity, hasOwner, limit, escalatedTopK ?? topK, results.Count, diff --git a/src/AgentMemory.Neo4j/Repositories/Neo4jPreferenceRepository.cs b/src/AgentMemory.Neo4j/Repositories/Neo4jPreferenceRepository.cs index 430e847c5..502dd0d26 100644 --- a/src/AgentMemory.Neo4j/Repositories/Neo4jPreferenceRepository.cs +++ b/src/AgentMemory.Neo4j/Repositories/Neo4jPreferenceRepository.cs @@ -19,9 +19,42 @@ internal sealed partial class Neo4jPreferenceRepository : IPreferenceRepository, { private readonly INeo4jTransactionRunner _tx; + private readonly bool _rescueShortOwnerResults; private readonly ILogger _logger; private readonly MemoryRankingOptions _ranking; private readonly MemoryDecayOptions _decay; + /// Scores this owner's OWN preferences directly, bypassing the global vector index. + /// + /// Extracted because two conditions reach it -- an empty scoped result, and (opt-in) a short one + /// -- and two copies of a fallback drift. + /// + private async Task> OwnerScopedScanAsync( + float[] queryEmbedding, + int limit, + double minScore, + MemoryScope scope, + bool includeShared, + CancellationToken cancellationToken) => + await _tx.ReadAsync(async runner => + { + var cursor = await runner.RunAsync( + PreferenceQueries.SearchByVectorOwnerScopedFallback(includeShared), + new Dictionary + { + ["embedding"] = queryEmbedding.ToList(), + ["limit"] = limit, + ["minScore"] = minScore, + ["ownerId"] = scope.OwnerId, + }).ConfigureAwait(false); + var records = await cursor.ToListAsync().ConfigureAwait(false); + return records.Select(r => + { + var node = r["node"].As(); + var score = r["score"].As(); + return (MapToPreference(node, ReadEmbedding(node)), score); + }).ToList(); + }, cancellationToken).ConfigureAwait(false) ?? []; + private readonly IMemoryRankingContext? _rankingContext; public Neo4jPreferenceRepository( @@ -29,8 +62,10 @@ public Neo4jPreferenceRepository( ILogger logger, IOptions? ranking = null, IOptions? decay = null, - IMemoryRankingContext? rankingContext = null) + IMemoryRankingContext? rankingContext = null, + IOptions? memoryOptions = null) { + _rescueShortOwnerResults = memoryOptions?.Value.RescueShortOwnerResults ?? false; _tx = tx; _logger = logger; _ranking = ranking?.Value ?? MemoryRankingOptions.Default; @@ -248,27 +283,25 @@ public async Task> GetByCategoryAsync( _logger.LogDebug( "Owner-scoped preference vector search still empty after widening; falling back to a " + "scoped similarity scan."); - results = await _tx.ReadAsync(async runner => - { - var cursor = await runner.RunAsync( - PreferenceQueries.SearchByVectorOwnerScopedFallback(includeShared), - new Dictionary - { - ["embedding"] = queryEmbedding.ToList(), - ["limit"] = limit, - ["minScore"] = minScore, - ["ownerId"] = scope!.OwnerId, - }).ConfigureAwait(false); - var records = await cursor.ToListAsync().ConfigureAwait(false); - return records.Select(r => - { - var node = r["node"].As(); - var score = r["score"].As(); - return (MapToPreference(node, ReadEmbedding(node)), score); - }).ToList(); - }, cancellationToken).ConfigureAwait(false) ?? []; + results = await OwnerScopedScanAsync( + queryEmbedding, limit, minScore, scope!, includeShared, cancellationToken) + .ConfigureAwait(false); } } + else if (_rescueShortOwnerResults + && OwnerVectorOverFetch.ShouldRescueShortResult(results.Count, limit, hasOwner)) + { + // 2.12, the same rescue the fact path carries. A host enabling RescueShortOwnerResults + // would otherwise get it on facts and silently not on preferences -- the "setting only some + // components respect" shape this project has found twice. + _logger.LogDebug( + "Owner-scoped preference vector search returned {Returned} of {Limit}; rescuing with " + + "a scoped similarity scan.", results.Count, limit); + var scanned = await OwnerScopedScanAsync( + queryEmbedding, limit, minScore, scope!, includeShared, cancellationToken) + .ConfigureAwait(false); + if (scanned.Count > results.Count) results = scanned; + } // Explicit null check rather than `activity?.SetTag(...)` per AgentMemoryDiagnostics' remarks: the // tag block exists purely to produce telemetry, so it is skipped whole when nobody is listening. diff --git a/src/AgentMemory.Neo4j/Repositories/Neo4jReasoningTraceRepository.cs b/src/AgentMemory.Neo4j/Repositories/Neo4jReasoningTraceRepository.cs index b58d10136..dfe0475a6 100644 --- a/src/AgentMemory.Neo4j/Repositories/Neo4jReasoningTraceRepository.cs +++ b/src/AgentMemory.Neo4j/Repositories/Neo4jReasoningTraceRepository.cs @@ -1,5 +1,6 @@ using System.Diagnostics; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using AgentMemory.Abstractions.Diagnostics; using AgentMemory.Abstractions.Domain; using AgentMemory.Abstractions.Options; @@ -15,10 +16,50 @@ namespace AgentMemory.Neo4j.Repositories; internal sealed class Neo4jReasoningTraceRepository : IReasoningTraceRepository { private readonly INeo4jTransactionRunner _tx; + private readonly bool _rescueShortOwnerResults; + /// Scores this owner's OWN traces directly, bypassing the global vector index. + /// + /// Extracted because two conditions reach it -- an empty scoped result, and (opt-in) a short one + /// -- and two copies of a fallback drift. + /// + private async Task> OwnerScopedScanAsync( + float[] queryEmbedding, + int limit, + double minScore, + MemoryScope scope, + bool includeShared, + bool? successFilter, + bool? proceduresOnly, + CancellationToken cancellationToken) => + await _tx.ReadAsync(async runner => + { + var cursor = await runner.RunAsync( + ReasoningQueries.SearchByTaskVectorOwnerScopedFallback( + successFilter.HasValue, includeShared, proceduresOnly), + new Dictionary + { + ["embedding"] = queryEmbedding.ToList(), + ["limit"] = limit, + ["minScore"] = minScore, + ["ownerId"] = scope.OwnerId, + }).ConfigureAwait(false); + var records = await cursor.ToListAsync().ConfigureAwait(false); + return records.Select(r => + { + var node = r["node"].As(); + var score = r["score"].As(); + return (MapToTrace(node, ReadEmbedding(node)), score); + }).ToList(); + }, cancellationToken).ConfigureAwait(false) ?? []; + private readonly ILogger _logger; - public Neo4jReasoningTraceRepository(INeo4jTransactionRunner tx, ILogger logger) + public Neo4jReasoningTraceRepository( + INeo4jTransactionRunner tx, + ILogger logger, + IOptions? memoryOptions = null) { + _rescueShortOwnerResults = memoryOptions?.Value.RescueShortOwnerResults ?? false; _tx = tx; _logger = logger; } @@ -243,23 +284,25 @@ public async Task> ListAllAsync(int limit = 50, int ["ownerId"] = scope!.OwnerId!, }; if (successFilter.HasValue) fallbackParameters["successFilter"] = successFilter.Value; - - results = await _tx.ReadAsync(async runner => - { - var cursor = await runner.RunAsync( - ReasoningQueries.SearchByTaskVectorOwnerScopedFallback( - successFilter.HasValue, includeShared, proceduresOnly), - fallbackParameters).ConfigureAwait(false); - var records = await cursor.ToListAsync().ConfigureAwait(false); - return records.Select(r => - { - var node = r["node"].As(); - var score = r["score"].As(); - return (MapToTrace(node, ReadEmbedding(node)), score); - }).ToList(); - }, cancellationToken).ConfigureAwait(false) ?? []; + results = await OwnerScopedScanAsync( + taskEmbedding, limit, minScore, scope!, includeShared, successFilter, + proceduresOnly, cancellationToken).ConfigureAwait(false); } } + else if (_rescueShortOwnerResults + && OwnerVectorOverFetch.ShouldRescueShortResult(results.Count, limit, hasOwner)) + { + // 2.12, the same rescue the fact path carries. A host enabling RescueShortOwnerResults + // would otherwise get it on facts and silently not on traces -- the "setting only some + // components respect" shape this project has found twice. + _logger.LogDebug( + "Owner-scoped trace vector search returned {Returned} of {Limit}; rescuing with " + + "a scoped similarity scan.", results.Count, limit); + var scanned = await OwnerScopedScanAsync( + taskEmbedding, limit, minScore, scope!, includeShared, successFilter, + proceduresOnly, cancellationToken).ConfigureAwait(false); + if (scanned.Count > results.Count) results = scanned; + } // Explicit null check rather than `activity?.SetTag(...)` per AgentMemoryDiagnostics' remarks: the // tag block exists purely to produce telemetry, so it is skipped whole when nobody is listening. From e563b6bec33abe7bb0048ebedc4f48473f8cd65e Mon Sep 17 00:00:00 2001 From: Jose Luis Latorre Millas Date: Wed, 12 Aug 2026 20:30:40 +0200 Subject: [PATCH 12/31] 4.6/4.6b: the AUC reports, and whether the refusals cost anything 4.6. The 50-question abstention-enriched run: structured AUC 0.709, hybrid 0.768, over 22 present and 20 absent. That is a real class balance -- 440 pairwise comparisons instead of the 32 the stratified run managed with its single absent observation -- and both arms agree while both clear the 0.6 line that was fixed in code before any number existed. The progression is the story: 32/1 gave 0.969 and 0.094 from the same corpus (one question rank); 5/4 gave 0.600 twice (real but n=9); 22/20 gives 0.709/0.768. Phase 10 gate is open -- reranking is no longer reordering a candidate set whose starvation is unmeasured. Accuracy 87.8tructured, 88.0 --- .../RefusedEvidenceTests.cs | 189 ++++++++++++++++++ .../LongMemEvalPreparedPairProgram.cs | 26 ++- .../LongMemEvalRefusedEvidence.cs | 121 +++++++++++ 3 files changed, 334 insertions(+), 2 deletions(-) create mode 100644 tests/AgentMemory.Tests.Unit.LongMemEval/RefusedEvidenceTests.cs create mode 100644 tools/AgentMemory.LongMemEval/LongMemEvalRefusedEvidence.cs diff --git a/tests/AgentMemory.Tests.Unit.LongMemEval/RefusedEvidenceTests.cs b/tests/AgentMemory.Tests.Unit.LongMemEval/RefusedEvidenceTests.cs new file mode 100644 index 000000000..85f8ca2f1 --- /dev/null +++ b/tests/AgentMemory.Tests.Unit.LongMemEval/RefusedEvidenceTests.cs @@ -0,0 +1,189 @@ +using AgentMemory.LongMemEval; +using FluentAssertions; +using Xunit; + +namespace AgentMemory.Tests.Unit.LongMemEval; + +/// +/// Whether a content refusal cost a question its answer, or merely some context. +/// +/// +/// +/// A refused source session is skipped so the build survives — 3 of 2,386 sessions is well inside +/// tolerance. But tolerance is a statement about volume, not about consequence. If the +/// refused session held a question's gold evidence, that question is unanswerable from memory for a +/// reason unrelated to retrieval, and counting it wrong attributes a content-policy decision to +/// recall quality. +/// +/// +/// Recording that sessions were refused is not enough. What matters is whether they mattered. +/// +/// +public sealed class RefusedEvidenceTests +{ + private static LongMemEvalMessageOrigin Message(int sourceSession, bool hasAnswer) => + new( + MessageOrdinal: 0, + SourceSessionId: $"s{sourceSession}", + SourceSessionOrdinal: sourceSession, + SourceTurnOrdinal: 0, + SourceTimestamp: "2026-01-01", + Role: "user", + FormattedContent: "", + IsSyntheticBoundary: false, + IsSyntheticFormatterPadding: false, + HasAnswer: hasAnswer); + + private static LongMemEvalEvidenceQuestion Question( + string id, params LongMemEvalMessageOrigin[] messages) => + new( + QuestionId: id, + QuestionType: "single-session-user", + Question: "q", + InvocationPrompt: "q", + GoldAnswer: "a", + QuestionDate: "2026-01-01", + IsAbstention: false, + AnswerSessionIds: new HashSet(StringComparer.Ordinal), + AnnotatedGoldTurnCount: 1, + Messages: messages); + + private static string SessionId(int question, int source) => + $"longmemeval-prepared-20260812T140253Z-session-{question:D4}-source-{source:D4}"; + + [Fact] + public void ARefusedSessionHoldingGoldEvidenceIsFlagged() + { + // THE case. This question can no longer be answered from memory, and nothing about that is a + // retrieval failure. + var questions = new[] { Question("q-1", Message(25, hasAnswer: true)) }; + + var analysed = LongMemEvalRefusedEvidence.Analyse([SessionId(1, 25)], questions); + + analysed.Should().ContainSingle(); + analysed[0].HeldGoldEvidence.Should().BeTrue(); + analysed[0].QuestionId.Should().Be("q-1"); + } + + [Fact] + public void ARefusedSessionHoldingOnlyContextIsNotFlagged() + { + // The common and genuinely harmless case: a conversation the question never depended on. It + // is still reported by id, because "harmless" is a judgement the reader should be able to + // check rather than one buried in a filter. + var questions = new[] { Question("q-1", Message(25, hasAnswer: false)) }; + + var analysed = LongMemEvalRefusedEvidence.Analyse([SessionId(1, 25)], questions); + + analysed[0].HeldGoldEvidence.Should().BeFalse(); + analysed[0].QuestionId.Should().Be("q-1"); + } + + [Fact] + public void TheGoldCheckIsScopedToTheRefusedSourceSession() + { + // A question whose answer lives in session 30 is untouched by a refusal of session 25. Without + // the ordinal comparison every refusal would look catastrophic for its whole question. + var questions = new[] + { + Question("q-1", Message(25, hasAnswer: false), Message(30, hasAnswer: true)), + }; + + LongMemEvalRefusedEvidence.Analyse([SessionId(1, 25)], questions)[0] + .HeldGoldEvidence.Should().BeFalse(); + } + + [Fact] + public void TheQuestionIndexInTheIdIsOneBased() + { + // Preparation numbers questions from 1 and the id carries that. An off-by-one here would + // attribute every refusal to the neighbouring question -- and still produce a plausible + // report. + var questions = new[] + { + Question("q-1", Message(3, hasAnswer: true)), + Question("q-2", Message(3, hasAnswer: true)), + }; + + LongMemEvalRefusedEvidence.Analyse([SessionId(2, 3)], questions)[0] + .QuestionId.Should().Be("q-2"); + } + + [Fact] + public void SyntheticFormatterMessagesNeverCountAsGoldEvidence() + { + // Session boundaries and padding are the harness's own text. Treating one as gold would report + // a refusal as costing an answer that never lived there. + var padding = Message(25, hasAnswer: true) with { IsSyntheticFormatterPadding = true }; + var boundary = Message(26, hasAnswer: true) with { IsSyntheticBoundary = true }; + var questions = new[] { Question("q-1", padding, boundary) }; + + var analysed = LongMemEvalRefusedEvidence.Analyse( + [SessionId(1, 25), SessionId(1, 26)], questions); + + analysed.Should().OnlyContain(r => !r.HeldGoldEvidence); + } + + [Fact] + public void AnUnparseableIdIsUnknownRatherThanHarmless() + { + // The id shape is a convention, not a contract. If it changes, the analysis must degrade to + // "we cannot tell" -- reported, with no question attached -- rather than silently answering + // "nothing was lost". + var analysed = LongMemEvalRefusedEvidence.Analyse(["some-other-session-naming"], []); + + analysed.Should().ContainSingle(); + analysed[0].QuestionId.Should().BeNull(); + analysed[0].SessionId.Should().Be("some-other-session-naming"); + } + + [Fact] + public void AQuestionIndexOutsideTheSampleIsUnknownToo() + { + var analysed = LongMemEvalRefusedEvidence.Analyse([SessionId(99, 3)], [Question("q-1")]); + + analysed[0].QuestionId.Should().BeNull(); + analysed[0].HeldGoldEvidence.Should().BeFalse(); + } + + [Fact] + public void NoCompromisedQuestionsProducesNoValidationIssue() + { + // A clean run must stay clean. An issue raised on every refusal would train the reader to + // ignore it, which is how the one that matters gets missed. + var questions = new[] { Question("q-1", Message(25, hasAnswer: false)) }; + var analysed = LongMemEvalRefusedEvidence.Analyse([SessionId(1, 25)], questions); + + LongMemEvalRefusedEvidence.DescribeCompromisedQuestions(analysed).Should().BeNull(); + } + + [Fact] + public void ACompromisedQuestionNamesItselfAndTheConsequence() + { + // The message has to say what changes about reading the number, not just that something + // happened -- the count is already reported elsewhere. + var questions = new[] { Question("q-7", Message(25, hasAnswer: true)) }; + var analysed = LongMemEvalRefusedEvidence.Analyse([SessionId(1, 25)], questions); + + var issue = LongMemEvalRefusedEvidence.DescribeCompromisedQuestions(analysed); + + issue.Should().NotBeNull(); + issue.Should().Contain("q-7").And.Contain("unrelated to retrieval"); + } + + [Fact] + public void EachCompromisedQuestionIsNamedOnce() + { + // Two refused sessions of the same question is one compromised question, not two. A count + // that double-reported would overstate the damage. + var questions = new[] + { + Question("q-1", Message(3, hasAnswer: true), Message(4, hasAnswer: true)), + }; + var analysed = LongMemEvalRefusedEvidence.Analyse( + [SessionId(1, 3), SessionId(1, 4)], questions); + + LongMemEvalRefusedEvidence.DescribeCompromisedQuestions(analysed) + .Should().Contain("1 question(s)"); + } +} diff --git a/tools/AgentMemory.LongMemEval/LongMemEvalPreparedPairProgram.cs b/tools/AgentMemory.LongMemEval/LongMemEvalPreparedPairProgram.cs index d66d4ccf1..c1b0cdd50 100644 --- a/tools/AgentMemory.LongMemEval/LongMemEvalPreparedPairProgram.cs +++ b/tools/AgentMemory.LongMemEval/LongMemEvalPreparedPairProgram.cs @@ -167,6 +167,7 @@ await LongMemEvalOrphanSweep new ProviderCompatibleExtractionChatClient( azureClient.GetChatClient(extractionDeployment).AsIChatClient())); LongMemEvalPreparationManifest manifest; + IReadOnlyList refusedEvidence = []; IReadOnlyList reuseDrift = []; IReadOnlyList preparationTelemetry; LongMemEvalPreparedBatchExecution? batchExecution = null; @@ -580,8 +581,22 @@ await LongMemEvalOrphanSweep + "for complete."); // The ids, so the refusal is investigable. Content filters recur, and "something // was refused" is not a report anyone can act on or raise with a provider. - foreach (var refusedId in batchDiagnostics.RefusedSessionIds ?? []) - Console.Error.WriteLine($" - refused session: {refusedId}"); + // + // And, more important than the count: did it COST anything? A refused session + // holding a question's gold evidence makes that question unanswerable from memory + // for a reason unrelated to retrieval, so scoring it wrong attributes a + // content-policy decision to recall quality. + refusedEvidence = LongMemEvalRefusedEvidence.Analyse( + batchDiagnostics.RefusedSessionIds ?? [], questions); + foreach (var refused in refusedEvidence) + { + var cost = refused.HeldGoldEvidence + ? $"HELD GOLD EVIDENCE for question {refused.QuestionId}" + : refused.QuestionId is null + ? "unresolved question - treat as unknown, not harmless" + : $"context only for question {refused.QuestionId}"; + Console.Error.WriteLine($" - refused session: {refused.SessionId} ({cost})"); + } if (refusedShare > MaximumRefusedSessionShare) { throw new InvalidOperationException( @@ -852,6 +867,13 @@ await LongMemEvalOrphanSweep // --allow-stale-prepared was used, so a result taken over a drifted corpus carries // that fact with it instead of looking like any other run. preparedCorpusDrift = reuseDrift.Select(d => d.ToString()).ToArray(), + // Empty on a clean build. Non-empty means questions in THIS sample lost evidence + // to a content policy, which is the difference between a number that is low and a + // number that is measuring the wrong thing. + refusedSessionsHoldingGoldEvidence = refusedEvidence + .Where(r => r.HeldGoldEvidence) + .Select(r => new { r.SessionId, r.QuestionId }) + .ToArray(), preparedCorpusPreparedAtUtc = reusing ? manifest.PreparedAtUtc : null, preparedCorpusDescription = reusing ? manifest.Description : options.Description, evidenceDetail = options.EvidenceDetail.ToString().ToLowerInvariant(), diff --git a/tools/AgentMemory.LongMemEval/LongMemEvalRefusedEvidence.cs b/tools/AgentMemory.LongMemEval/LongMemEvalRefusedEvidence.cs new file mode 100644 index 000000000..ee66234e9 --- /dev/null +++ b/tools/AgentMemory.LongMemEval/LongMemEvalRefusedEvidence.cs @@ -0,0 +1,121 @@ +using System.Globalization; +using System.Text.RegularExpressions; + +namespace AgentMemory.LongMemEval; + +/// +/// Whether a provider's content refusal cost a question its answer, or merely some context. +/// +/// +/// +/// A refused source session is skipped so the build survives, and 3 of 2,386 sessions is well inside +/// tolerance. But "inside tolerance" is a statement about volume, not about consequence: if the +/// refused session is the one holding a question's gold evidence, that question becomes +/// unanswerable from memory and will score as wrong — attributing an Azure content-policy +/// decision to our recall quality. +/// +/// +/// That is the exact failure this track exists to prevent: a number that is internally consistent, +/// reproducible, and measuring something other than what it claims. Recording that sessions +/// were refused is not enough; what matters is whether they mattered. +/// +/// +/// This does not change the score. Excluding a question from accuracy would require reaching into +/// AgentEval's scoring, and silently dropping questions is its own way to make a metric lie. It +/// raises a validation issue instead, so an affected run is flagged rather than quietly accepted, and +/// the reader can decide. +/// +/// +internal static class LongMemEvalRefusedEvidence +{ + // {preparationId}-session-{questionIndex:D4}-source-{sourceSessionOrdinal:D4} + private static readonly Regex SessionIdPattern = new( + @"-session-(?\d{4})-source-(?\d{4})$", + RegexOptions.CultureInvariant | RegexOptions.ExplicitCapture, TimeSpan.FromSeconds(1)); + + /// One refused session, and what it cost. + /// The scoped session id the provider refused. + /// The dataset question whose history it belonged to, when resolvable. + /// + /// Whether that source session contained a message the dataset marks as answer-bearing. True means + /// the question can no longer be answered from memory for a reason unrelated to retrieval. + /// + internal readonly record struct RefusedSession( + string SessionId, string? QuestionId, bool HeldGoldEvidence); + + /// + /// Resolves each refused session against the sampled questions' gold evidence. + /// + /// + /// A session id that does not parse, or names a question outside the sample, yields + /// HeldGoldEvidence = false with a null question — unknown rather than assumed harmless, + /// and visible as such because the id is still reported. + /// + internal static IReadOnlyList Analyse( + IReadOnlyList refusedSessionIds, + IReadOnlyList questions) + { + ArgumentNullException.ThrowIfNull(refusedSessionIds); + ArgumentNullException.ThrowIfNull(questions); + + var analysed = new List(refusedSessionIds.Count); + foreach (var sessionId in refusedSessionIds) + { + var match = SessionIdPattern.Match(sessionId); + if (!match.Success) + { + analysed.Add(new RefusedSession(sessionId, null, false)); + continue; + } + + // The question index is 1-based in the id, matching how preparation numbers them. + var questionIndex = int.Parse( + match.Groups["question"].Value, CultureInfo.InvariantCulture) - 1; + var sourceOrdinal = int.Parse( + match.Groups["source"].Value, CultureInfo.InvariantCulture); + + if (questionIndex < 0 || questionIndex >= questions.Count) + { + analysed.Add(new RefusedSession(sessionId, null, false)); + continue; + } + + var question = questions[questionIndex]; + var heldGold = question.Messages.Any(message => + message.SourceSessionOrdinal == sourceOrdinal + && message.HasAnswer + && !message.IsSyntheticBoundary + && !message.IsSyntheticFormatterPadding); + + analysed.Add(new RefusedSession(sessionId, question.QuestionId, heldGold)); + } + + return analysed; + } + + /// + /// The validation issue for a run whose refusals cost gold evidence, or null when none did. + /// + /// + /// Phrased around the consequence rather than the count, because the count is already reported and + /// the consequence is what changes how the accuracy number should be read. + /// + internal static string? DescribeCompromisedQuestions(IReadOnlyList analysed) + { + ArgumentNullException.ThrowIfNull(analysed); + + var compromised = analysed + .Where(session => session.HeldGoldEvidence && session.QuestionId is not null) + .Select(session => session.QuestionId!) + .Distinct(StringComparer.Ordinal) + .OrderBy(id => id, StringComparer.Ordinal) + .ToArray(); + + return compromised.Length == 0 + ? null + : $"The provider refused source sessions holding the gold evidence for " + + $"{compromised.Length} question(s): {string.Join(", ", compromised)}. Those questions " + + "are unanswerable from this corpus for a reason unrelated to retrieval, so counting " + + "them as wrong attributes a content-policy decision to recall quality."; + } +} From 0a43bbafe1896a8e2a4a9b6e47cdaad3a0d5d2d9 Mon Sep 17 00:00:00 2001 From: Jose Luis Latorre Millas Date: Wed, 12 Aug 2026 20:44:33 +0200 Subject: [PATCH 13/31] Report abstention accuracy: the meta-memory number, measured for the first time Two different things share the word "absent", and conflating them is easy enough that it happened in conversation. The sufficiency AUC's absentCount is a ground-truth INPUT -- the label retrieval confidence is ordered against -- and it is identical across arms by construction. Abstention accuracy is an OUTCOME: whether the agent actually behaved correctly on those questions. Reporting only the first invites reading a class balance as a result. On the 50q abstention-enriched run: 18 of 20 correct in BOTH arms (90%), against 25/30 structured and 26/30 hybrid on ordinary questions. The system declines to answer what memory does not hold more reliably than it retrieves what it does. That is also an independent cross-check on the AUC. A system scoring 90% on abstention is genuinely separating answerable from unanswerable, which is what an AUC of 0.709-0.768 claims; two instruments built on different evidence agreeing is worth more than either alone. Worth its own line because the project's own taxonomy names abstention questions as the only place this dataset scores meta-memory at all -- and across 52 recorded runs before typed sampling shipped, not one had ever been drawn. This is the first meta-memory measurement the project has taken. The failure is named for the behaviour rather than as "incorrect": answeredWhenItShouldHaveAbstained is the agent asserting something memory did not hold, which is the failure a memory system is least able to detect in itself. Two of twenty. No abstention questions reports null, not zero -- a rate of 0 would read as "it never abstains correctly" when the truth is "it was never asked to". An unjudged question leaves both denominators alone rather than counting wrong, since judge-parse failures are a known recurring class here. 4187 unit + 391 LongMemEval green. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01PgDgctPpbTziBNE2RT8VdE --- docs/architecture.md | 2 +- .../Services/IMemoryReranker.cs | 93 +++++++++++++ .../AbstentionAccuracyTests.cs | 128 ++++++++++++++++++ .../AbstractionsContractGuardTests.cs | 2 +- .../LongMemEvalAbstentionAccuracy.cs | 89 ++++++++++++ .../LongMemEvalPreparedPairProgram.cs | 31 +++++ 6 files changed, 343 insertions(+), 2 deletions(-) create mode 100644 src/AgentMemory.Abstractions/Services/IMemoryReranker.cs create mode 100644 tests/AgentMemory.Tests.Unit.LongMemEval/AbstentionAccuracyTests.cs create mode 100644 tools/AgentMemory.LongMemEval/LongMemEvalAbstentionAccuracy.cs diff --git a/docs/architecture.md b/docs/architecture.md index c9407fb97..053248477 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -142,7 +142,7 @@ graph TD | **Purpose** | Domain contracts — all models, interfaces, and configuration types shared across the system | | **Dependencies** | **Microsoft.Extensions.AI.Abstractions** 10.8.0 (approved, D-AR2-1) — .NET BCL otherwise (multi-targets net8.0/net9.0/net10.0) | | **MUST NOT reference** | Neo4j.Driver, Microsoft.Agents.*, any GraphRAG SDK, any MCP SDK, any NuGet package **except** Microsoft.Extensions.AI.Abstractions | -| **Key types** | 52 domain records (Conversation, Message, Entity, Fact, Preference, Relationship, MemoryHistoryQuery, MemoryHistoryRecord, ReasoningTrace, ReasoningStep, ToolCall, ToolCallStats, IngestionItemOutcome, MemoryContextRankedItem, MemoryContextSectionDiagnostics, UnifiedExtractionResult, etc.), 41 service interfaces (incl. `IMemoryIsolationPolicy`, `IUnifiedMemoryExtractor`, and `IMultiSessionUnifiedMemoryExtractor`), 11 repository interfaces, 16 configuration types (incl. `MemoryRankingOptions`, `MemoryIsolationOptions`), 26 enums (incl. `MemoryProfile`, `RankingIntent`, `DuplicateStatus`, `EntityMatchType`, `MemoryNodeKind`, `MemoryOperationAccess`, `MemoryIsolationMode`, `IngestionStatus`, `IngestionStage`, `IngestionItemStatus`, `MemoryItemKind`, `IngestionFailureMode`, `MemoryTrustLevel`, `AssistantContentMode`, `TemporalValidityMode`, `TraceKind`, `ExtractionProvenanceMode`) | +| **Key types** | 52 domain records (Conversation, Message, Entity, Fact, Preference, Relationship, MemoryHistoryQuery, MemoryHistoryRecord, ReasoningTrace, ReasoningStep, ToolCall, ToolCallStats, IngestionItemOutcome, MemoryContextRankedItem, MemoryContextSectionDiagnostics, UnifiedExtractionResult, etc.), 42 service interfaces (incl. `IMemoryIsolationPolicy`, `IUnifiedMemoryExtractor`, `IMultiSessionUnifiedMemoryExtractor`, and `IMemoryReranker`), 11 repository interfaces, 16 configuration types (incl. `MemoryRankingOptions`, `MemoryIsolationOptions`), 26 enums (incl. `MemoryProfile`, `RankingIntent`, `DuplicateStatus`, `EntityMatchType`, `MemoryNodeKind`, `MemoryOperationAccess`, `MemoryIsolationMode`, `IngestionStatus`, `IngestionStage`, `IngestionItemStatus`, `MemoryItemKind`, `IngestionFailureMode`, `MemoryTrustLevel`, `AssistantContentMode`, `TemporalValidityMode`, `TraceKind`, `ExtractionProvenanceMode`) | **Namespace structure:** ``` diff --git a/src/AgentMemory.Abstractions/Services/IMemoryReranker.cs b/src/AgentMemory.Abstractions/Services/IMemoryReranker.cs new file mode 100644 index 000000000..b9634b83d --- /dev/null +++ b/src/AgentMemory.Abstractions/Services/IMemoryReranker.cs @@ -0,0 +1,93 @@ +using AgentMemory.Abstractions.Domain; +using AgentMemory.Abstractions.Options; + +namespace AgentMemory.Abstractions.Services; + +/// +/// Reorders the candidates a retrieval returned, before context budgeting truncates them. +/// +/// +/// +/// This seam is deliberately narrower than the task that proposed it. R2 asked for a reranker +/// in order to fuse GraphRAG as a distinct retrieval channel, and that premise was killed on +/// measurement: of 132 GraphRAG items, 132 were duplicates of what the structured surface had +/// already retrieved and 0 verdicts changed. Fusing a channel that returns the same rows in a +/// different order cannot help, and the trigger for revisiting it — a corpus with a genuinely separate +/// knowledge graph — has not fired. +/// +/// +/// What survives is the extension point, which R6 (node-distance) and R7 (mention frequency) +/// both need and neither of which is about fusion. Keeping the seam and dropping the fusion is the +/// honest reading of that measurement: the mechanism was worth building, the use case was not. +/// +/// +/// Reranking reorders survivors; it cannot rescue what retrieval never returned. That is why +/// this ships behind the retrieval-sufficiency work rather than before it — a reranker over a starved +/// candidate set reorders seven of an owner's 504 facts and reports success. The signal now measures +/// AUC 0.709–0.768, which is what made this safe to build. +/// +/// +/// Implementations must be pure and fast: this runs inside recall, on the blocking path, once +/// per section. Anything needing a database round trip should issue one bounded query, never +/// one per candidate. +/// +/// +public interface IMemoryReranker +{ + /// + /// Whether this reranker should run at all for the current request. + /// + /// + /// Checked before the candidates are materialised, so a disabled reranker costs nothing — not even + /// the allocation of the list it would have reordered. + /// + bool IsEnabled { get; } + + /// + /// Returns reordered, or the same sequence when nothing changed. + /// + /// + /// The retrieval's own ranking, best first. Scores are the provider's, comparable within this + /// section and meaningless across sections. + /// + /// What the request knows: the query, its embedding, and the owner scope. + /// Cancels the reordering; a cancelled rerank leaves the provider order. + /// + /// + /// Returning a different set — adding or dropping candidates — is not supported and not + /// checked for cheaply, so it would corrupt the section's diagnostics silently. A reranker + /// reorders; a filter belongs in retrieval, where its effect on recall is measurable. + /// + /// + /// A reranker that throws must not fail the recall: callers treat an exception as "no reordering" + /// and continue with the provider's own ranking, because a degraded order is recoverable and a + /// failed recall is not. + /// + /// + Task> RerankAsync( + IReadOnlyList candidates, + MemoryRerankContext context, + CancellationToken cancellationToken = default); +} + +/// +/// What a reranker is told about the request it is reordering for. +/// +/// The user's query text, for a reranker that reads language. +/// +/// The query vector, when one was generated. Null on a turn narrowed to categories that need no +/// vector — a reranker depending on it must handle that rather than assume. +/// +/// +/// The owner scope the retrieval ran under. A reranker issuing its own query must carry this: +/// a reordering pass that reads another owner's graph is an isolation breach wearing a ranking hat. +/// +/// +/// Which recall category is being reordered, so one reranker can serve several and behave differently +/// per kind rather than pretending a fact and a trace rank alike. +/// +public sealed record MemoryRerankContext( + string? Query, + float[]? QueryEmbedding, + MemoryScope? Scope, + MemoryItemKind SectionKind); diff --git a/tests/AgentMemory.Tests.Unit.LongMemEval/AbstentionAccuracyTests.cs b/tests/AgentMemory.Tests.Unit.LongMemEval/AbstentionAccuracyTests.cs new file mode 100644 index 000000000..b35b3a2ce --- /dev/null +++ b/tests/AgentMemory.Tests.Unit.LongMemEval/AbstentionAccuracyTests.cs @@ -0,0 +1,128 @@ +using AgentMemory.LongMemEval; +using FluentAssertions; +using Xunit; + +namespace AgentMemory.Tests.Unit.LongMemEval; + +/// +/// Meta-memory: how well the agent declines to answer what memory does not hold. +/// +/// +/// +/// This exists because two different things share the word "absent", and the difference is easy to +/// miss. The sufficiency AUC's absentCount is a ground-truth input — the label +/// retrieval confidence is ordered against — and it is identical across arms by construction. +/// Abstention accuracy is an outcome: whether the agent behaved correctly on those +/// questions. Reporting only the first invites reading a class balance as a result. +/// +/// +/// It is worth its own line because the project's taxonomy names abstention questions as the only +/// place this dataset scores meta-memory at all, and across 52 recorded runs before typed sampling +/// shipped, none had ever been drawn. +/// +/// +public sealed class AbstentionAccuracyTests +{ + private static LongMemEvalQuestionTelemetry Question(int number, string id, bool isAbstention) => + new(number, MessagesStored: 10, ItemsRetrieved: 5, RecallTruncated: false, Status: "ok") + { + QuestionId = id, + IsAbstention = isAbstention, + }; + + private static Func Verdicts(params (string Id, bool? Correct)[] verdicts) + { + var map = verdicts.ToDictionary(v => v.Id, v => v.Correct, StringComparer.Ordinal); + return id => id is not null && map.TryGetValue(id, out var v) ? v : null; + } + + [Fact] + public void TheTwoHalvesAreScoredSeparately() + { + // The whole point: an aggregate hides whether the system is good at knowing what it does not + // know, which is a different capability from retrieving what it does. + var telemetry = new[] + { + Question(1, "abs-1", isAbstention: true), + Question(2, "abs-2", isAbstention: true), + Question(3, "ord-1", isAbstention: false), + }; + + var scored = LongMemEvalAbstentionAccuracy.Score(telemetry, Verdicts( + ("abs-1", true), ("abs-2", false), ("ord-1", true))); + + scored.AbstentionCorrect.Should().Be(1); + scored.AbstentionTotal.Should().Be(2); + scored.OrdinaryCorrect.Should().Be(1); + scored.OrdinaryTotal.Should().Be(1); + } + + [Fact] + public void AnsweringWhenItShouldHaveAbstainedIsNamedForTheBehaviour() + { + // Not "incorrect": this specific wrong answer is the interesting one, because the agent + // asserted something memory did not hold. That is the failure a memory system is least able + // to detect in itself. + var telemetry = new[] + { + Question(1, "abs-1", isAbstention: true), + Question(2, "abs-2", isAbstention: true), + }; + + var scored = LongMemEvalAbstentionAccuracy.Score(telemetry, Verdicts( + ("abs-1", true), ("abs-2", false))); + + scored.AnsweredWhenItShouldHaveAbstained.Should().Be(1); + } + + [Fact] + public void NoAbstentionQuestionsMeansNotMeasuredRatherThanZero() + { + // Before typed sampling this was every run. A rate of 0 would read as "it never abstains + // correctly"; null reads as "it was never asked to", which is the truth. + var scored = LongMemEvalAbstentionAccuracy.Score( + [Question(1, "ord-1", isAbstention: false)], Verdicts(("ord-1", true))); + + scored.AbstentionAccuracy.Should().BeNull(); + scored.AbstentionTotal.Should().Be(0); + scored.OrdinaryAccuracy.Should().Be(1.0); + } + + [Fact] + public void AnUnjudgedQuestionLeavesBothDenominatorsAlone() + { + // Counting it wrong would understate every rate by however many the judge could not parse -- + // and judge-parse failures are a known, recurring class here. + var telemetry = new[] + { + Question(1, "abs-1", isAbstention: true), + Question(2, "abs-2", isAbstention: true), + }; + + var scored = LongMemEvalAbstentionAccuracy.Score(telemetry, Verdicts( + ("abs-1", true), ("abs-2", null))); + + scored.AbstentionTotal.Should().Be(1, "an unjudged question measured nothing"); + scored.AbstentionAccuracy.Should().Be(1.0); + } + + [Fact] + public void TheMeasuredRunReproduces() + { + // The 50q abstention-enriched run of 2026-08-12: 18 of 20 abstention questions correct in both + // arms, against 25/30 and 26/30 ordinary. Pinned as a shape check on the scorer -- if this + // ever computes 20/20 or 0/20 from the same inputs, the scorer changed, not the system. + var telemetry = Enumerable.Range(1, 20) + .Select(i => Question(i, $"abs-{i}", isAbstention: true)) + .Concat(Enumerable.Range(21, 30).Select(i => Question(i, $"ord-{i}", isAbstention: false))) + .ToArray(); + + var scored = LongMemEvalAbstentionAccuracy.Score(telemetry, id => + id is null ? null + : id.StartsWith("abs-", StringComparison.Ordinal) ? id is not ("abs-1" or "abs-2") + : id is not ("ord-21" or "ord-22" or "ord-23" or "ord-24" or "ord-25")); + + scored.AbstentionAccuracy.Should().BeApproximately(0.90, 1e-9); + scored.OrdinaryAccuracy.Should().BeApproximately(25.0 / 30.0, 1e-9); + } +} diff --git a/tests/AgentMemory.Tests.Unit/Infrastructure/AbstractionsContractGuardTests.cs b/tests/AgentMemory.Tests.Unit/Infrastructure/AbstractionsContractGuardTests.cs index 6104145c6..e21deb25f 100644 --- a/tests/AgentMemory.Tests.Unit/Infrastructure/AbstractionsContractGuardTests.cs +++ b/tests/AgentMemory.Tests.Unit/Infrastructure/AbstractionsContractGuardTests.cs @@ -16,7 +16,7 @@ public sealed class AbstractionsContractGuardTests private static readonly Assembly Abstractions = typeof(IMemoryService).Assembly; // Counts mirrored in docs/architecture.md §3.1 and docs/design.md §5/§6. - private const int DocumentedServiceInterfaces = 41; // +IMultiSessionUnifiedMemoryExtractor (M-27-V2 LAB-B1) + private const int DocumentedServiceInterfaces = 42; // +IMemoryReranker (PLAN 10.1, seam only) private const int DocumentedRepositoryInterfaces = 11; private const int DocumentedDomainRecords = 52; // +MemoryContextSectionDiagnostics (PLAN 4.1) private const int DocumentedEnums = 29; // +TraceKind (7.1), +ExtractionProvenanceMode (9.3) diff --git a/tools/AgentMemory.LongMemEval/LongMemEvalAbstentionAccuracy.cs b/tools/AgentMemory.LongMemEval/LongMemEvalAbstentionAccuracy.cs new file mode 100644 index 000000000..87be58d09 --- /dev/null +++ b/tools/AgentMemory.LongMemEval/LongMemEvalAbstentionAccuracy.cs @@ -0,0 +1,89 @@ +namespace AgentMemory.LongMemEval; + +/// +/// How well the agent says "I don't know" — the only meta-memory score this dataset offers. +/// +/// +/// +/// An abstention question is one the dataset declares unanswerable. Getting it right means declining +/// to answer; getting it wrong means producing an answer anyway, which is hallucination under absence +/// — the failure mode a memory system is least able to detect in itself. +/// +/// +/// Not the same thing as the AUC's "absent" count, though both concern unanswerable questions, +/// and conflating them is easy. The AUC's absent count is an input: the ground-truth label +/// against which retrieval confidence is ordered, and it is identical across arms by construction. +/// This is an outcome: whether the agent behaved correctly on those questions. Reporting only +/// the first invites reading a class balance as a result. +/// +/// +/// Worth reporting on its own line because the project's taxonomy names abstention as the only place +/// LongMemEval scores meta-memory at all — and across 52 recorded runs before typed sampling shipped, +/// not one abstention question had ever been drawn, so this had never been measured. +/// +/// +internal static class LongMemEvalAbstentionAccuracy +{ + /// Correct-answer rates for the abstention and ordinary halves of a run. + /// Abstention questions the judge scored correct. + /// Abstention questions judged at all. + /// Ordinary questions the judge scored correct. + /// Ordinary questions judged at all. + internal readonly record struct Result( + int AbstentionCorrect, int AbstentionTotal, int OrdinaryCorrect, int OrdinaryTotal) + { + /// Abstention accuracy, or null when no abstention question was drawn. + /// + /// Null rather than 0 or 1: a sample containing no abstention question has not measured + /// meta-memory badly, it has not measured it. Before typed sampling shipped that was every run. + /// + internal double? AbstentionAccuracy => + AbstentionTotal == 0 ? null : (double)AbstentionCorrect / AbstentionTotal; + + internal double? OrdinaryAccuracy => + OrdinaryTotal == 0 ? null : (double)OrdinaryCorrect / OrdinaryTotal; + + /// + /// Abstention questions answered when they should have been declined. + /// + /// + /// Named for the behaviour rather than as "incorrect", because this specific wrong answer is + /// the interesting one: the agent asserted something memory did not hold. + /// + internal int AnsweredWhenItShouldHaveAbstained => AbstentionTotal - AbstentionCorrect; + } + + /// + /// Scores the two halves from per-question telemetry and the judge's own verdicts. + /// + /// + /// A question with no judge verdict is excluded from both denominators rather than counted wrong: + /// an unjudged question measured nothing, and folding it in as a failure would understate every + /// rate by however many the judge could not parse. + /// + internal static Result Score( + IReadOnlyList telemetry, + Func verdictFor) + { + ArgumentNullException.ThrowIfNull(telemetry); + ArgumentNullException.ThrowIfNull(verdictFor); + + int abstentionCorrect = 0, abstentionTotal = 0, ordinaryCorrect = 0, ordinaryTotal = 0; + foreach (var item in telemetry) + { + if (verdictFor(item.QuestionId) is not { } correct) continue; + if (item.IsAbstention) + { + abstentionTotal++; + if (correct) abstentionCorrect++; + } + else + { + ordinaryTotal++; + if (correct) ordinaryCorrect++; + } + } + + return new Result(abstentionCorrect, abstentionTotal, ordinaryCorrect, ordinaryTotal); + } +} diff --git a/tools/AgentMemory.LongMemEval/LongMemEvalPreparedPairProgram.cs b/tools/AgentMemory.LongMemEval/LongMemEvalPreparedPairProgram.cs index c1b0cdd50..df0b52b7a 100644 --- a/tools/AgentMemory.LongMemEval/LongMemEvalPreparedPairProgram.cs +++ b/tools/AgentMemory.LongMemEval/LongMemEvalPreparedPairProgram.cs @@ -1123,6 +1123,30 @@ private static async Task RunArmAsync( LongMemEvalVectorYieldSummary.From(vectorYield.Samples)); } + /// The meta-memory half of an arm's result, or nulls when no abstention question ran. + private static object ProjectAbstentionAccuracy(PreparedArmExecution arm) + { + var verdicts = arm.Result.QuestionResults + .Where(q => q.QuestionId is not null) + .GroupBy(q => q.QuestionId!, StringComparer.Ordinal) + .ToDictionary(g => g.Key, g => (bool?)g.First().Correct, StringComparer.Ordinal); + + var scored = LongMemEvalAbstentionAccuracy.Score( + arm.Telemetry, + questionId => questionId is not null && verdicts.TryGetValue(questionId, out var v) ? v : null); + + return new + { + abstentionCorrect = scored.AbstentionCorrect, + abstentionTotal = scored.AbstentionTotal, + abstentionAccuracy = scored.AbstentionAccuracy, + answeredWhenItShouldHaveAbstained = scored.AnsweredWhenItShouldHaveAbstained, + ordinaryCorrect = scored.OrdinaryCorrect, + ordinaryTotal = scored.OrdinaryTotal, + ordinaryAccuracy = scored.OrdinaryAccuracy, + }; + } + private static object ProjectArm( PreparedArmExecution arm, LongMemEvalEvidenceDetail evidenceDetail) => @@ -1171,6 +1195,13 @@ private static object ProjectArm( // holds the two prices, never their difference). Reported as a group so a derived-answer // type's absences are not read as extraction failures. answerPresenceByType = LongMemEvalAnswerPresence.SummariseByType(arm.Telemetry), + // Meta-memory: how well the agent declines to answer what memory does not hold. + // Reported separately from the AUC's "absent" count, which is a ground-truth INPUT + // identical across arms -- reading a class balance as a result is the easy mistake, + // and only one of these two numbers is an outcome. The dataset's abstention questions + // are the only place it scores meta-memory, and until typed sampling shipped none had + // ever been drawn. + abstentionAccuracy = ProjectAbstentionAccuracy(arm), // PLAN 4.2. Emitted here as well as on the plain verb, because THIS is where the // 50-question runs happen: the plain verb's cold build costs ~684 extraction calls and // 7-9 hours, so every real measurement goes through the prepared pair. An instrument that From 30bc647b073a47dae70494397147bab99d0b10f1 Mon Sep 17 00:00:00 2001 From: Jose Luis Latorre Millas Date: Wed, 12 Aug 2026 20:54:31 +0200 Subject: [PATCH 14/31] 10.1/10.2: the reranker seam, and node-distance reranking on it 10.1's premise was killed by measurement and is not being resurrected. R2 wanted a reranker in order to fuse GraphRAG as a distinct channel; of 132 GraphRAG items 132 were duplicates of what the structured surface already returned and 0 verdicts changed. Fusing a channel that returns the same rows in a different order cannot help, and its revival trigger -- a corpus with a genuinely separate knowledge graph -- has not fired. But 10.2 and 10.3 need the extension point, which is a different thing from the fusion. So the seam ships and the use case stays dead: the mechanism was worth building, the application was not. The contract states what actually matters. Reorder-only, because a filter belongs in retrieval where its effect on recall is measurable and a reranker that changes the set corrupts the section's diagnostics silently. One bounded query rather than one per candidate. Owner scope carried explicitly, because a reordering pass that reads another owner's graph is an isolation breach wearing a ranking hat. And a thrown reranker degrades to the provider's order rather than failing the recall, since a degraded order is recoverable and a failed recall is not. 10.2 is the first user of it: find the entity the query is about by the SAME embedding the retrieval used -- so "near the centroid" and "similar to the query" are commensurable rather than two unrelated notions of relevance -- then boost by gamma^hops over ABOUT/RELATED_TO. One decay constant, not two: it reuses EffectiveStructuralDecayGamma, the same one GraphRAG hop-decay uses, and is inert when that decay is off. A second independently-tuned constant for the same idea would drift and nobody would know which result came from which. The boost is multiplicative, so graph closeness is evidence about relevance rather than a replacement for it -- a candidate retrieval ranked badly is not promoted past a strong one by adjacency alone. EXTRACTED_FROM is excluded from the traversal on purpose: it links to source messages, so following it would make every fact from a shared conversation look adjacent regardless of subject. Off by default, and gated hard: non-fact sections, missing embeddings and single-candidate sets all return before any query is issued. Shipped behind the sufficiency work deliberately -- reranking reorders survivors, so over a starved candidate set it reorders seven of an owner's 504 facts and reports success. The AUC measured 0.709-0.768 first. 4198 unit tests green. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01PgDgctPpbTziBNE2RT8VdE --- .../Options/MemoryOptions.cs | 22 ++ .../Queries/NodeDistanceQueries.cs | 68 +++++ .../Services/NodeDistanceReranker.cs | 170 +++++++++++++ .../Services/NodeDistanceRerankerTests.cs | 235 ++++++++++++++++++ 4 files changed, 495 insertions(+) create mode 100644 src/AgentMemory.Neo4j/Queries/NodeDistanceQueries.cs create mode 100644 src/AgentMemory.Neo4j/Services/NodeDistanceReranker.cs create mode 100644 tests/AgentMemory.Tests.Unit/Services/NodeDistanceRerankerTests.cs diff --git a/src/AgentMemory.Abstractions/Options/MemoryOptions.cs b/src/AgentMemory.Abstractions/Options/MemoryOptions.cs index 8f6b27948..a5ac6aeed 100644 --- a/src/AgentMemory.Abstractions/Options/MemoryOptions.cs +++ b/src/AgentMemory.Abstractions/Options/MemoryOptions.cs @@ -46,6 +46,28 @@ public sealed record MemoryOptions /// public bool RescueShortOwnerResults { get; init; } + /// + /// Boosts recalled facts that sit close, in the graph, to the entity the query is about (R6). + /// + /// + /// + /// Vector similarity asks whether a row looks like the query; this asks whether it is + /// about what the query is about. A plainly-worded fact hanging directly off the named + /// entity is often the answer, while a fact that paraphrases the query beautifully may concern + /// someone else entirely. + /// + /// + /// Reuses MemoryRankingOptions.StructuralDecayGamma as its decay constant rather than + /// introducing a second one, and does nothing unless that decay is also enabled -- at gamma = 1 + /// every boost is 1.0, so the two extra queries could not change an ordering. + /// + /// + /// Off by default: it changes recall ordering, costs two bounded queries per fact section, and + /// every recorded measurement was taken without it. + /// + /// + public bool NodeDistanceReranking { get; init; } + // NOTE: extraction at the Core layer is explicit (call ExtractAndPersistAsync / // ExtractFromSessionAsync). Automatic extraction on message persist is an adapter concern, configured // by AgentFrameworkOptions.AutoExtractOnPersist. The former EnableAutoExtraction flag here was read diff --git a/src/AgentMemory.Neo4j/Queries/NodeDistanceQueries.cs b/src/AgentMemory.Neo4j/Queries/NodeDistanceQueries.cs new file mode 100644 index 000000000..f131bc383 --- /dev/null +++ b/src/AgentMemory.Neo4j/Queries/NodeDistanceQueries.cs @@ -0,0 +1,68 @@ +namespace AgentMemory.Neo4j.Queries; + +/// +/// Graph distance from a query-matched centroid entity to a set of candidate facts (R6). +/// +/// +/// +/// Vector similarity asks "does this row look like the query?"; graph distance asks "is this row +/// about what the query is about?". They disagree usefully: a fact whose wording is unremarkable +/// but which hangs directly off the entity the question names is often the answer, and a fact that +/// merely paraphrases the query can be about someone else entirely. +/// +/// +/// One bounded query for the whole candidate set, never one per candidate. A per-candidate +/// traversal would multiply a recall's database round trips by the limit — and it runs on the blocking +/// path, so a reranker that costs more than the retrieval it reorders is not an improvement. +/// +/// +internal static class NodeDistanceQueries +{ + /// + /// The entity whose name or aliases best match the query, by vector similarity. + /// + /// + /// The centroid is chosen by the same embedding the retrieval used, so "near the centroid" + /// and "similar to the query" are commensurable rather than two unrelated notions of relevance. + /// Owner-scoped: a centroid drawn from another tenant's graph would silently reorder this tenant's + /// results around a node they cannot see. + /// + internal static string CentroidEntity(bool hasOwnerFilter, bool includeShared) + { + var owner = !hasOwnerFilter ? string.Empty + : includeShared ? " AND (node.owner_id = $ownerId OR node.owner_id IS NULL)" + : " AND node.owner_id = $ownerId"; + return $@" + CALL db.index.vector.queryNodes('entity_embedding_idx', $topK, $embedding) + YIELD node, score + WHERE score >= $minScore{owner} + RETURN node.id AS entityId, score AS score + ORDER BY score DESC + LIMIT 1"; + } + + /// + /// Hop distance from the centroid entity to each candidate fact, for the whole set at once. + /// + /// + /// + /// Bounded three ways, because an unbounded traversal on a dense graph is how a reranker becomes + /// the slowest thing in a recall: the candidate list is the starting set (not the corpus), the + /// path length is capped at $maxHops, and only RELATED_TO/ABOUT edges are + /// followed — the two that carry meaning between an entity and a fact. EXTRACTED_FROM is + /// deliberately excluded: it links to source messages, so following it would make every fact from + /// a shared conversation look adjacent regardless of subject. + /// + /// + /// Candidates with no path within the cap are simply absent from the result, which the caller + /// reads as "no boost" — the alternative, returning a sentinel distance, invites arithmetic on a + /// number that means "unreachable". + /// + /// + internal static string DistanceToCandidates => @" + MATCH (centroid:Entity {id: $centroidId}) + MATCH (f:Fact) WHERE f.id IN $candidateIds + MATCH path = shortestPath((centroid)-[:RELATED_TO|ABOUT*..4]-(f)) + WHERE length(path) <= $maxHops + RETURN f.id AS candidateId, length(path) AS hops"; +} diff --git a/src/AgentMemory.Neo4j/Services/NodeDistanceReranker.cs b/src/AgentMemory.Neo4j/Services/NodeDistanceReranker.cs new file mode 100644 index 000000000..2c7e10bdc --- /dev/null +++ b/src/AgentMemory.Neo4j/Services/NodeDistanceReranker.cs @@ -0,0 +1,170 @@ +using AgentMemory.Abstractions.Domain; +using AgentMemory.Abstractions.Options; +using AgentMemory.Abstractions.Services; +using AgentMemory.Neo4j.Infrastructure; +using AgentMemory.Neo4j.Queries; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Neo4j.Driver; + +namespace AgentMemory.Neo4j.Services; + +/// +/// Boosts candidates that sit close, in the graph, to the entity the query is about (R6). +/// +/// +/// +/// Vector similarity asks whether a row looks like the query. This asks whether it is +/// about what the query is about, and the two disagree in useful ways: a plainly-worded fact +/// hanging directly off the named entity is often the answer, while a fact that paraphrases the query +/// beautifully may concern someone else entirely. +/// +/// +/// One decay constant, not two. The boost is γ^hops using +/// — the same γ GraphRAG's hop decay +/// already uses. A second, independently-tuned constant for the same idea would drift from the first +/// and nobody would notice which one a given result came from. +/// +/// +/// Cost is bounded to two queries per section, regardless of candidate count: one to find the +/// centroid, one to measure distance to every candidate at once. This runs on the blocking recall +/// path, so a per-candidate traversal would make the reranker cost more than the retrieval it +/// reorders. +/// +/// +/// Off by default. It changes recall ordering, every recorded measurement was taken without it, and +/// the hermetic perf harness counts queries — so enabling it is a stated decision with a visible cost. +/// +/// +internal sealed class NodeDistanceReranker : IMemoryReranker +{ + private readonly INeo4jTransactionRunner _tx; + private readonly MemoryRankingOptions _ranking; + private readonly MemoryOptions _memory; + private readonly ILogger _logger; + + public NodeDistanceReranker( + INeo4jTransactionRunner tx, + ILogger logger, + IOptions? ranking = null, + IOptions? memory = null) + { + _tx = tx; + _logger = logger; + _ranking = ranking?.Value ?? MemoryRankingOptions.Default; + _memory = memory?.Value ?? new MemoryOptions(); + } + + /// Candidate ceiling for the centroid lookup, matching the ordinary over-fetch shape. + private const int CentroidTopK = 20; + + /// Hops beyond which the boost is indistinguishable from noise at any sane γ. + /// + /// At γ = 0.5 the fourth hop contributes 0.0625; the query caps at the same depth, so raising this + /// alone would silently do nothing. Kept next to the Cypher's *..4 for that reason. + /// + private const int MaxHops = 4; + + /// + /// + /// Requires the structural decay to be switched on as well as the reranker: with γ = 1 every boost + /// is 1.0, so running the two queries could not change an ordering and would be pure cost. + /// + public bool IsEnabled => _memory.NodeDistanceReranking && _ranking.StructuralDecayEnabled; + + /// + public async Task> RerankAsync( + IReadOnlyList candidates, + MemoryRerankContext context, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(candidates); + ArgumentNullException.ThrowIfNull(context); + + // Facts are the only kind this understands: the distance query walks ABOUT/RELATED_TO from an + // entity to a Fact. Silently reordering another kind by a measure that does not apply to it + // would be worse than not running. + if (context.SectionKind != MemoryItemKind.Fact) return candidates; + if (candidates.Count < 2 || context.QueryEmbedding is not { Length: > 0 }) return candidates; + + var centroidId = await FindCentroidAsync(context, cancellationToken).ConfigureAwait(false); + if (centroidId is null) return candidates; + + var hops = await MeasureHopsAsync(centroidId, candidates, cancellationToken).ConfigureAwait(false); + if (hops.Count == 0) return candidates; + + var gamma = _ranking.EffectiveStructuralDecayGamma; + + // Multiplicative on the provider score, so a candidate the retrieval already ranked badly is + // not promoted past a strong one by proximity alone -- graph closeness is evidence about + // relevance, not a replacement for it. Unreachable candidates keep their score exactly. + var reordered = candidates + .Select(item => new + { + Item = item, + Boosted = hops.TryGetValue(item.ItemId, out var h) + ? item.Score * (1.0 + Math.Pow(gamma, h)) + : item.Score, + }) + .OrderByDescending(scored => scored.Boosted) + // Provider rank breaks ties, so an unreachable candidate and a 4-hop one with equal + // boosted scores stay in the order retrieval put them, rather than an arbitrary one. + .ThenBy(scored => scored.Item.RetrievalRank) + .Select((scored, index) => scored.Item with { ContextRank = index + 1 }) + .ToList(); + + _logger.LogDebug( + "Node-distance rerank: {Reached} of {Total} candidates within {MaxHops} hops of centroid " + + "{Centroid} (gamma {Gamma}).", + hops.Count, candidates.Count, MaxHops, centroidId, gamma); + + return reordered; + } + + private async Task FindCentroidAsync( + MemoryRerankContext context, CancellationToken cancellationToken) + { + var hasOwner = context.Scope?.HasOwnerFilter == true; + var includeShared = context.Scope?.IncludeShared ?? true; + var parameters = new Dictionary + { + ["embedding"] = context.QueryEmbedding!.ToList(), + ["topK"] = CentroidTopK, + ["minScore"] = 0.0, + }; + if (hasOwner) parameters["ownerId"] = context.Scope!.OwnerId; + + return await _tx.ReadAsync(async runner => + { + var cursor = await runner.RunAsync( + NodeDistanceQueries.CentroidEntity(hasOwner, includeShared), parameters) + .ConfigureAwait(false); + var records = await cursor.ToListAsync().ConfigureAwait(false); + return records.Count == 0 ? null : records[0]["entityId"].As(); + }, cancellationToken).ConfigureAwait(false); + } + + private async Task> MeasureHopsAsync( + string centroidId, + IReadOnlyList candidates, + CancellationToken cancellationToken) + { + var parameters = new Dictionary + { + ["centroidId"] = centroidId, + ["candidateIds"] = candidates.Select(item => item.ItemId).ToList(), + ["maxHops"] = MaxHops, + }; + + return await _tx.ReadAsync(async runner => + { + var cursor = await runner.RunAsync( + NodeDistanceQueries.DistanceToCandidates, parameters).ConfigureAwait(false); + var records = await cursor.ToListAsync().ConfigureAwait(false); + return (IReadOnlyDictionary)records.ToDictionary( + r => r["candidateId"].As(), + r => r["hops"].As(), + StringComparer.Ordinal); + }, cancellationToken).ConfigureAwait(false) ?? new Dictionary(StringComparer.Ordinal); + } +} diff --git a/tests/AgentMemory.Tests.Unit/Services/NodeDistanceRerankerTests.cs b/tests/AgentMemory.Tests.Unit/Services/NodeDistanceRerankerTests.cs new file mode 100644 index 000000000..5cb5da287 --- /dev/null +++ b/tests/AgentMemory.Tests.Unit/Services/NodeDistanceRerankerTests.cs @@ -0,0 +1,235 @@ +using AgentMemory.Abstractions.Domain; +using AgentMemory.Abstractions.Options; +using AgentMemory.Abstractions.Services; +using AgentMemory.Neo4j.Infrastructure; +using AgentMemory.Neo4j.Services; +using FluentAssertions; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Neo4j.Driver; +using NSubstitute; +using Xunit; + +namespace AgentMemory.Tests.Unit.Services; + +/// +/// Node-distance reranking (R6): boost what the query is about, not only what it looks like. +/// +/// +/// +/// The gates matter as much as the boost. This runs on the blocking recall path and issues two +/// queries per fact section, so every condition under which it declines to run is a cost avoided — +/// and a reranker that fires when it cannot change anything is pure overhead wearing a feature's name. +/// +/// +/// Shipped behind the retrieval-sufficiency work deliberately: reranking reorders survivors, so over a +/// starved candidate set it reorders seven of an owner's 504 facts and reports success. The signal +/// measured AUC 0.709–0.768 before this was built. +/// +/// +public sealed class NodeDistanceRerankerTests +{ + private readonly INeo4jTransactionRunner _tx = Substitute.For(); + + private NodeDistanceReranker CreateSut(bool enabled = true, double gamma = 0.5) => + new(_tx, + NullLogger.Instance, + Options.Create(new MemoryRankingOptions { StructuralDecayGamma = gamma }), + Options.Create(new MemoryOptions { NodeDistanceReranking = enabled })); + + private static MemoryRerankContext Context( + MemoryItemKind kind = MemoryItemKind.Fact, float[]? embedding = null) => + new("who does alice work for", embedding ?? [0.1f, 0.2f], MemoryScope.For("alice"), kind); + + private static IReadOnlyList Candidates() => + [ + new("fact-a", Score: 0.90, RetrievalRank: 1, ContextRank: 1), + new("fact-b", Score: 0.80, RetrievalRank: 2, ContextRank: 2), + ]; + + // ── the gates ───────────────────────────────────────────────────────── + + [Fact] + public void ItIsOffByDefault() + { + // It changes recall ordering and costs two queries per section; every recorded measurement was + // taken without it. + new MemoryOptions().NodeDistanceReranking.Should().BeFalse(); + CreateSut(enabled: false).IsEnabled.Should().BeFalse(); + } + + [Fact] + public void ItStaysOffWhenTheDecayConstantIsOne() + { + // At gamma = 1 every boost is 1.0, so the two queries could not change an ordering. Running + // them anyway would be pure cost -- and it is the same constant GraphRAG hop-decay uses, so + // "structural decay off" means off for both rather than for one of them. + CreateSut(enabled: true, gamma: 1.0).IsEnabled.Should().BeFalse(); + CreateSut(enabled: true, gamma: 0.5).IsEnabled.Should().BeTrue(); + } + + [Fact] + public async Task ANonFactSectionIsLeftAlone() + { + // The distance query walks ABOUT/RELATED_TO from an entity to a Fact. Reordering traces or + // preferences by a measure that does not apply to them would be worse than not running. + var result = await CreateSut().RerankAsync( + Candidates(), Context(kind: MemoryItemKind.Preference)); + + result.Should().BeSameAs(await Task.FromResult(result)); + result.Select(r => r.ItemId).Should().Equal("fact-a", "fact-b"); + await _tx.DidNotReceive().ReadAsync( + Arg.Any>>(), Arg.Any()); + } + + [Fact] + public async Task AMissingQueryEmbeddingCostsNothing() + { + // A turn narrowed to categories needing no vector has no embedding, and the centroid lookup is + // a vector search. Asking for one anyway would reintroduce exactly the round trip the trivial + // turn policy exists to elide. + var result = await CreateSut().RerankAsync(Candidates(), Context(embedding: [])); + + result.Select(r => r.ItemId).Should().Equal("fact-a", "fact-b"); + await _tx.DidNotReceive().ReadAsync( + Arg.Any>>(), Arg.Any()); + } + + [Fact] + public async Task ASingleCandidateIsNotWorthTwoQueries() + { + // Nothing to reorder. The cheapest correct answer is the one already held. + var single = new[] { new MemoryContextRankedItem("fact-a", 0.9, 1, 1) }; + + var result = await CreateSut().RerankAsync(single, Context()); + + result.Should().BeEquivalentTo(single); + await _tx.DidNotReceive().ReadAsync( + Arg.Any>>(), Arg.Any()); + } + + // ── the boost ───────────────────────────────────────────────────────── + + [Fact] + public async Task ACloserCandidateOvertakesAHigherScoringDistantOne() + { + // The whole point. fact-b scores lower on similarity but sits one hop from the entity the + // query names, while fact-a is four hops away: 0.80*(1+0.5^1)=1.20 beats 0.90*(1+0.5^4)=0.956. + _tx.ReadAsync(Arg.Any>>(), Arg.Any()) + .Returns("entity-1"); + _tx.ReadAsync( + Arg.Any>>>(), + Arg.Any()) + .Returns((IReadOnlyDictionary)new Dictionary(StringComparer.Ordinal) + { + ["fact-a"] = 4, + ["fact-b"] = 1, + }); + + var result = await CreateSut().RerankAsync(Candidates(), Context()); + + result.Select(r => r.ItemId).Should().Equal("fact-b", "fact-a"); + result[0].ContextRank.Should().Be(1); + result[1].ContextRank.Should().Be(2); + } + + [Fact] + public async Task ProximityDoesNotOverrideAStrongSimilarityGap() + { + // Graph closeness is evidence about relevance, not a replacement for it. The boost is + // multiplicative so a candidate retrieval ranked badly is not promoted past a strong one by + // adjacency alone: 0.90*(1+0.5^4)=0.956 still beats 0.20*(1+0.5)=0.30. + _tx.ReadAsync(Arg.Any>>(), Arg.Any()) + .Returns("entity-1"); + _tx.ReadAsync( + Arg.Any>>>(), + Arg.Any()) + .Returns((IReadOnlyDictionary)new Dictionary(StringComparer.Ordinal) + { + ["fact-a"] = 4, + ["weak"] = 1, + }); + + var candidates = new[] + { + new MemoryContextRankedItem("fact-a", 0.90, 1, 1), + new MemoryContextRankedItem("weak", 0.20, 2, 2), + }; + + var result = await CreateSut().RerankAsync(candidates, Context()); + + result.Select(r => r.ItemId).Should().Equal("fact-a", "weak"); + } + + [Fact] + public async Task AnUnreachableCandidateKeepsItsScoreExactly() + { + // Absent from the distance result means "no path within the cap", which the caller reads as + // no boost. A sentinel distance would invite arithmetic on a number meaning "unreachable". + _tx.ReadAsync(Arg.Any>>(), Arg.Any()) + .Returns("entity-1"); + _tx.ReadAsync( + Arg.Any>>>(), + Arg.Any()) + .Returns((IReadOnlyDictionary)new Dictionary(StringComparer.Ordinal) + { + ["fact-b"] = 1, + }); + + var result = await CreateSut().RerankAsync(Candidates(), Context()); + + // fact-b: 0.80*1.5 = 1.20; fact-a unreachable, keeps 0.90. + result.Select(r => r.ItemId).Should().Equal("fact-b", "fact-a"); + result.Should().OnlyContain(r => r.Score == 0.90 || r.Score == 0.80, + "the reranker reorders; it must not rewrite the provider's scores"); + } + + [Fact] + public async Task NoCentroidMeansNoReordering() + { + // An owner whose graph holds no entity resembling the query. Returning the provider order is + // correct and costs one query rather than two. + _tx.ReadAsync(Arg.Any>>(), Arg.Any()) + .Returns((string?)null); + + var result = await CreateSut().RerankAsync(Candidates(), Context()); + + result.Select(r => r.ItemId).Should().Equal("fact-a", "fact-b"); + } + + [Fact] + public async Task NoCandidateWithinRangeLeavesTheOrderUntouched() + { + _tx.ReadAsync(Arg.Any>>(), Arg.Any()) + .Returns("entity-1"); + _tx.ReadAsync( + Arg.Any>>>(), + Arg.Any()) + .Returns((IReadOnlyDictionary)new Dictionary(StringComparer.Ordinal)); + + var result = await CreateSut().RerankAsync(Candidates(), Context()); + + result.Select(r => r.ItemId).Should().Equal("fact-a", "fact-b"); + } + + [Fact] + public async Task TheCandidateSetIsNeverChanged() + { + // A reranker reorders. Adding or dropping a candidate would corrupt the section's diagnostics + // silently, because Returned is counted before this runs. + _tx.ReadAsync(Arg.Any>>(), Arg.Any()) + .Returns("entity-1"); + _tx.ReadAsync( + Arg.Any>>>(), + Arg.Any()) + .Returns((IReadOnlyDictionary)new Dictionary(StringComparer.Ordinal) + { + ["fact-b"] = 2, + }); + + var result = await CreateSut().RerankAsync(Candidates(), Context()); + + result.Select(r => r.ItemId).Should().BeEquivalentTo(["fact-a", "fact-b"]); + result.Should().HaveCount(2); + } +} From 176a65b1fbf14c847cc984a84c55fce40fb4004c Mon Sep 17 00:00:00 2001 From: Jose Luis Latorre Millas Date: Wed, 12 Aug 2026 21:19:32 +0200 Subject: [PATCH 15/31] 10.3: mention-frequency reranking, on the ingestion-side signal A fact the world asserted five times is usually more central than one mentioned once, and similarity cannot see that: two facts phrased alike score alike however often either was actually said. The signal's provenance IS the task. mention_count is incremented by the ON MATCH of the fact triple MERGE -- once per ingestion that re-states an existing triple -- so it measures how often the world said something. The tempting substitute, :MemoryReadAudit, counts how often WE surfaced it, and ranking on that closes a loop: high rank causes retrieval, retrieval raises the count, the count raises the rank. It would look like learning and be self-reinforcement. Refused, and tested for by name. Logarithmic rather than linear, because the gap between one mention and three is real while the gap between thirty and thirty-two is noise -- and under a linear boost a chatty topic would bury a precise answer. The weight is small enough that salience breaks near-ties among plausible answers rather than substituting for matching the question: a 50-mention weak match still loses to a strong one. Incremented on all three upsert paths -- single, batch and fused -- because a counter that depends on which write path ran is the "setting only some components respect" shape in a different costume. Pre-existing facts coalesce to 1, the honest reading of an absent counter, which leaves their ranking unchanged since log(1) is 0. Six live-database tests pin what the Cypher text cannot: that re-assertion increments, that a different fact about the same subject does not, that a different object is a different fact, that another owner's assertion counts separately -- and that reading a fact five times leaves the counter at 1. Phase 10 is complete. Its gate was 4.6, which reported AUC 0.709/0.768. 4212 unit + 338 non-NAMS integration green. Cypher snapshot regenerated (BOM stripped) and the query-count constant updated for the one new const. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01PgDgctPpbTziBNE2RT8VdE --- .../Options/MemoryOptions.cs | 21 ++ src/AgentMemory.Neo4j/Queries/FactQueries.cs | 44 +++- .../Queries/FusedPersistenceQueries.cs | 2 + .../Services/MentionFrequencyReranker.cs | 119 +++++++++++ .../MentionCountIntegrationTests.cs | 148 +++++++++++++ .../Queries/CypherQuerySnapshot.snap | 26 ++- .../Queries/CypherQuerySnapshotTests.cs | 2 +- .../Services/MentionFrequencyRerankerTests.cs | 199 ++++++++++++++++++ 8 files changed, 551 insertions(+), 10 deletions(-) create mode 100644 src/AgentMemory.Neo4j/Services/MentionFrequencyReranker.cs create mode 100644 tests/AgentMemory.Tests.Integration/Repositories/MentionCountIntegrationTests.cs create mode 100644 tests/AgentMemory.Tests.Unit/Services/MentionFrequencyRerankerTests.cs diff --git a/src/AgentMemory.Abstractions/Options/MemoryOptions.cs b/src/AgentMemory.Abstractions/Options/MemoryOptions.cs index a5ac6aeed..cafd5d8eb 100644 --- a/src/AgentMemory.Abstractions/Options/MemoryOptions.cs +++ b/src/AgentMemory.Abstractions/Options/MemoryOptions.cs @@ -68,6 +68,27 @@ public sealed record MemoryOptions /// public bool NodeDistanceReranking { get; init; } + /// + /// Boosts recalled facts the conversation keeps returning to (R7). + /// + /// + /// + /// A fact the world asserted five times is usually more central than one mentioned once, and + /// similarity cannot see that: two facts phrased alike score alike however often either was said. + /// + /// + /// Ingestion-side. It counts how often the WORLD re-asserted a fact, not how often we + /// retrieved it. Ranking on our own retrievals -- the :MemoryReadAudit trail -- would be a + /// rich-get-richer loop: whatever ranks highly gets retrieved, which raises its count, which + /// raises its rank. That looks like learning and is self-reinforcement. + /// + /// + /// Logarithmic, so the gap between one mention and three counts while the gap between thirty and + /// thirty-two does not. Off by default; every recorded measurement was taken without it. + /// + /// + public bool MentionFrequencyReranking { get; init; } + // NOTE: extraction at the Core layer is explicit (call ExtractAndPersistAsync / // ExtractFromSessionAsync). Automatic extraction on message persist is an adapter concern, configured // by AgentFrameworkOptions.AutoExtractOnPersist. The former EnableAutoExtraction flag here was read diff --git a/src/AgentMemory.Neo4j/Queries/FactQueries.cs b/src/AgentMemory.Neo4j/Queries/FactQueries.cs index a9f9c7b92..c949dec58 100644 --- a/src/AgentMemory.Neo4j/Queries/FactQueries.cs +++ b/src/AgentMemory.Neo4j/Queries/FactQueries.cs @@ -109,7 +109,13 @@ ON CREATE SET f.valid_until = CASE WHEN $validUntil IS NOT NULL THEN datetime($validUntil) ELSE null END, f.source_message_ids = $sourceMessageIds, f.created_at = datetime($createdAtUtc), - f.metadata = $metadata + f.metadata = $metadata, + // R7 salience. Counts how often THE WORLD asserted this fact, incremented only here + // and in ON MATCH -- i.e. once per ingestion that re-states the same triple. + // Deliberately NOT the :MemoryReadAudit trail, which counts how often WE surfaced it: + // ranking on our own retrievals is a rich-get-richer loop that reinforces whatever + // already ranks highly and calls it learning. + f.mention_count = 1 ON MATCH SET f.category = $category, f.confidence = $confidence, @@ -118,7 +124,8 @@ ON MATCH SET f.source_message_ids = $sourceMessageIds, f.updated_at = datetime($updatedAtUtc), f.metadata = $metadata, - f.invalidated_at = null + f.invalidated_at = null, + f.mention_count = coalesce(f.mention_count, 1) + 1 RETURN f"; // ── UpsertBatchAsync ─────────────────────────────────────────────── @@ -148,7 +155,8 @@ ON CREATE SET f.valid_until = CASE WHEN item.valid_until IS NOT NULL THEN datetime(item.valid_until) ELSE null END, f.source_message_ids = item.source_message_ids, f.created_at = datetime(item.created_at), - f.metadata = item.metadata + f.metadata = item.metadata, + f.mention_count = 1 ON MATCH SET f.category = item.category, f.confidence = item.confidence, @@ -157,9 +165,37 @@ ON MATCH SET f.source_message_ids = item.source_message_ids, f.updated_at = datetime(item.updated_at), f.metadata = item.metadata, - f.invalidated_at = null + f.invalidated_at = null, + f.mention_count = coalesce(f.mention_count, 1) + 1 RETURN f"; + // ── R7 mention frequency ─────────────────────────────────────────── + + /// + /// How often the world asserted each of these facts, for the whole candidate set at once. + /// + /// + /// + /// mention_count is incremented only by the upsert paths, i.e. once per ingestion that + /// re-states the same triple. It is a salience signal: a fact the conversation keeps + /// returning to matters more than one mentioned in passing. + /// + /// + /// Explicitly not the read audit. :MemoryReadAudit records how often we + /// surfaced a fact, and ranking on that is a rich-get-richer loop: whatever ranks highly gets + /// retrieved, which raises its count, which raises its rank. It would look like learning and be + /// self-reinforcement. + /// + /// + /// Facts written before this property existed report 1 via coalesce — asserted once, + /// which is what a single ingestion means and the only honest reading of an absent counter. + /// + /// + public const string MentionCounts = @" + MATCH (f:Fact) + WHERE f.id IN $candidateIds + RETURN f.id AS candidateId, coalesce(f.mention_count, 1) AS mentions"; + // ── GetByIdAsync ─────────────────────────────────────────────────── /// Get a single fact by id. diff --git a/src/AgentMemory.Neo4j/Queries/FusedPersistenceQueries.cs b/src/AgentMemory.Neo4j/Queries/FusedPersistenceQueries.cs index fa9622ca4..de78a519f 100644 --- a/src/AgentMemory.Neo4j/Queries/FusedPersistenceQueries.cs +++ b/src/AgentMemory.Neo4j/Queries/FusedPersistenceQueries.cs @@ -65,6 +65,7 @@ ON CREATE SET f.confidence = item.confidence, f.valid_from = CASE WHEN item.valid_from IS NOT NULL THEN datetime(item.valid_from) ELSE null END, f.valid_until = CASE WHEN item.valid_until IS NOT NULL THEN datetime(item.valid_until) ELSE null END, + f.mention_count = 1, f.source_message_ids = item.source_message_ids, f.created_at = datetime(item.created_at), f.metadata = item.metadata @@ -73,6 +74,7 @@ ON MATCH SET f.confidence = item.confidence, f.valid_from = CASE WHEN item.valid_from IS NOT NULL THEN datetime(item.valid_from) ELSE f.valid_from END, f.valid_until = CASE WHEN item.valid_until IS NOT NULL THEN datetime(item.valid_until) ELSE f.valid_until END, + f.mention_count = coalesce(f.mention_count, 1) + 1, f.source_message_ids = item.source_message_ids, f.updated_at = datetime(item.updated_at), f.metadata = item.metadata, diff --git a/src/AgentMemory.Neo4j/Services/MentionFrequencyReranker.cs b/src/AgentMemory.Neo4j/Services/MentionFrequencyReranker.cs new file mode 100644 index 000000000..74dad34d8 --- /dev/null +++ b/src/AgentMemory.Neo4j/Services/MentionFrequencyReranker.cs @@ -0,0 +1,119 @@ +using AgentMemory.Abstractions.Domain; +using AgentMemory.Abstractions.Options; +using AgentMemory.Abstractions.Services; +using AgentMemory.Neo4j.Infrastructure; +using AgentMemory.Neo4j.Queries; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Neo4j.Driver; + +namespace AgentMemory.Neo4j.Services; + +/// +/// Boosts facts the conversation keeps returning to (R7). +/// +/// +/// +/// A fact the world asserted five times is usually more central than one mentioned once. Similarity +/// cannot see that: two facts phrased alike score alike however often either was actually said. +/// +/// +/// The signal is ingestion-side, and that distinction is the whole task. It counts how often +/// the world re-asserted a fact — mention_count, incremented by the upsert paths when a +/// new ingestion re-states an existing triple. The tempting substitute, :MemoryReadAudit, +/// counts how often we surfaced it, and ranking on that is a rich-get-richer loop: whatever +/// ranks highly gets retrieved, retrieval raises the count, the count raises the rank. It would look +/// like learning and be self-reinforcement, so it is not used here and must not be. +/// +/// +/// Logarithmic, not linear. The gap between one mention and three is real; the gap between +/// thirty and thirty-two is noise, and a linear boost would let a chatty topic bury a precise answer. +/// +/// +/// One bounded query for the whole candidate set. Off by default: it changes recall ordering and +/// every recorded measurement was taken without it. +/// +/// +internal sealed class MentionFrequencyReranker : IMemoryReranker +{ + private readonly INeo4jTransactionRunner _tx; + private readonly MemoryOptions _memory; + private readonly ILogger _logger; + + public MentionFrequencyReranker( + INeo4jTransactionRunner tx, + ILogger logger, + IOptions? memory = null) + { + _tx = tx; + _logger = logger; + _memory = memory?.Value ?? new MemoryOptions(); + } + + /// + /// How much a well-attested fact may outrank a better-matching one. + /// + /// + /// At 0.15, a fact mentioned eight times gains ~1.31× while a once-mentioned one gains 1.0 — enough + /// to break a near-tie, not enough to promote a weak match over a strong one. Salience is a + /// tiebreaker among plausible answers, not a substitute for matching the question. + /// + private const double BoostWeight = 0.15; + + /// + public bool IsEnabled => _memory.MentionFrequencyReranking; + + /// + public async Task> RerankAsync( + IReadOnlyList candidates, + MemoryRerankContext context, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(candidates); + ArgumentNullException.ThrowIfNull(context); + + // mention_count is maintained on Fact only. Boosting another kind by a counter nothing + // increments would be a no-op dressed as a feature. + if (context.SectionKind != MemoryItemKind.Fact || candidates.Count < 2) return candidates; + + var mentions = await MentionCountsAsync(candidates, cancellationToken).ConfigureAwait(false); + if (mentions.Count == 0) return candidates; + + var reordered = candidates + .Select(item => new + { + Item = item, + Boosted = item.Score * (1.0 + (BoostWeight * Math.Log( + mentions.TryGetValue(item.ItemId, out var count) ? Math.Max(count, 1) : 1))), + }) + .OrderByDescending(scored => scored.Boosted) + // Retrieval rank breaks ties, so equal salience leaves the provider's order intact rather + // than an arbitrary one. + .ThenBy(scored => scored.Item.RetrievalRank) + .Select((scored, index) => scored.Item with { ContextRank = index + 1 }) + .ToList(); + + _logger.LogDebug( + "Mention-frequency rerank over {Count} candidates; most-mentioned {Max}.", + candidates.Count, mentions.Count == 0 ? 0 : mentions.Values.Max()); + + return reordered; + } + + private async Task> MentionCountsAsync( + IReadOnlyList candidates, CancellationToken cancellationToken) => + await _tx.ReadAsync(async runner => + { + var cursor = await runner.RunAsync( + FactQueries.MentionCounts, + new Dictionary + { + ["candidateIds"] = candidates.Select(item => item.ItemId).ToList(), + }).ConfigureAwait(false); + var records = await cursor.ToListAsync().ConfigureAwait(false); + return (IReadOnlyDictionary)records.ToDictionary( + r => r["candidateId"].As(), + r => r["mentions"].As(), + StringComparer.Ordinal); + }, cancellationToken).ConfigureAwait(false) ?? new Dictionary(StringComparer.Ordinal); +} diff --git a/tests/AgentMemory.Tests.Integration/Repositories/MentionCountIntegrationTests.cs b/tests/AgentMemory.Tests.Integration/Repositories/MentionCountIntegrationTests.cs new file mode 100644 index 000000000..51e34f26b --- /dev/null +++ b/tests/AgentMemory.Tests.Integration/Repositories/MentionCountIntegrationTests.cs @@ -0,0 +1,148 @@ +using FluentAssertions; +using Microsoft.Extensions.Logging.Abstractions; +using AgentMemory.Abstractions.Domain; +using AgentMemory.Abstractions.Options; +using AgentMemory.Neo4j.Queries; +using AgentMemory.Neo4j.Repositories; +using AgentMemory.Tests.Integration.Fixtures; +using Neo4j.Driver; + +namespace AgentMemory.Tests.Integration.Repositories; + +/// +/// The salience counter behind R7, verified in the database rather than in the Cypher text. +/// +/// +/// +/// mention_count must record how often the world asserted a fact. Unit tests can prove +/// the query says so; only a live database proves the MERGE actually increments on re-assertion and +/// not on, say, every write of any fact about the same subject. +/// +/// +/// The failure this guards against is silent and slow: a counter that never moves makes the reranker +/// a no-op that looks enabled, and one that moves on the wrong event makes it rank on noise. +/// +/// +[Collection("Neo4j Integration")] +[Trait("Category", "Integration")] +public class MentionCountIntegrationTests : IAsyncLifetime +{ + private readonly Neo4jIntegrationFixture _fixture; + private readonly Neo4jFactRepository _facts; + + public MentionCountIntegrationTests(Neo4jIntegrationFixture fixture) + { + _fixture = fixture; + _facts = new Neo4jFactRepository( + fixture.TransactionRunner, NullLogger.Instance); + } + + public Task InitializeAsync() => _fixture.CleanDatabaseAsync(); + public Task DisposeAsync() => Task.CompletedTask; + + private Task AssertAsync(string subject, string predicate, string @object) => + _facts.UpsertAsync(new Fact + { + FactId = $"fact-{Guid.NewGuid():N}", + Subject = subject, + Predicate = predicate, + Object = @object, + Confidence = 0.9, + OwnerId = "alice", + CreatedAtUtc = DateTimeOffset.UtcNow, + }); + + private Task MentionsOfAsync(string factId) => + _fixture.TransactionRunner.ReadAsync(async runner => + { + var cursor = await runner.RunAsync( + FactQueries.MentionCounts, new { candidateIds = new[] { factId } }); + var records = await cursor.ToListAsync(); + // Fully qualified: FluentAssertions and the Neo4j driver both offer As, and the + // ambiguity is a compile error rather than a silent wrong conversion (the precedent is + // ReasoningTracePruneIntegrationTests). + return records.Count == 0 + ? 0 + : global::Neo4j.Driver.ValueExtensions.As(records[0]["mentions"]); + }); + + [Fact] + public async Task AFirstAssertionCountsOnce() + { + var stored = await AssertAsync("user", "lives in", "Zurich"); + + (await MentionsOfAsync(stored.FactId)).Should().Be(1); + } + + [Fact] + public async Task ReAssertingTheSameFactIncrements() + { + // THE behaviour: the world said it again. The triple MERGEs to the same node, so this is the + // one moment that distinguishes a fact the conversation keeps returning to from one mentioned + // in passing. + var stored = await AssertAsync("user", "lives in", "Zurich"); + await AssertAsync("user", "lives in", "Zurich"); + await AssertAsync("user", "lives in", "Zurich"); + + (await MentionsOfAsync(stored.FactId)).Should().Be(3); + } + + [Fact] + public async Task ADifferentFactAboutTheSameSubjectDoesNotIncrement() + { + // Salience is per fact, not per subject. Counting every write about "user" would make one + // chatty subject's facts all look equally central. + var lives = await AssertAsync("user", "lives in", "Zurich"); + await AssertAsync("user", "works at", "Acme"); + await AssertAsync("user", "likes", "tea"); + + (await MentionsOfAsync(lives.FactId)).Should().Be(1); + } + + [Fact] + public async Task ADifferentObjectIsADifferentFact() + { + // "lives in Zurich" and "lives in Basel" are separate assertions, not one fact asserted twice + // -- the MERGE key includes the object precisely so contradiction stays visible. + var zurich = await AssertAsync("user", "lives in", "Zurich"); + var basel = await AssertAsync("user", "lives in", "Basel"); + + (await MentionsOfAsync(zurich.FactId)).Should().Be(1); + (await MentionsOfAsync(basel.FactId)).Should().Be(1); + } + + [Fact] + public async Task ReadingAFactNeverIncrementsIt() + { + // The loop this must not close. Retrieval must leave the counter alone, or ranking starts + // reinforcing whatever already ranks highly on evidence it generated itself. + var stored = await AssertAsync("user", "lives in", "Zurich"); + + for (var i = 0; i < 5; i++) + await _facts.GetByIdAsync(stored.FactId); + await _facts.GetBySubjectAsync("user", MemoryScope.For("alice")); + + (await MentionsOfAsync(stored.FactId)).Should().Be(1, + "reads are not mentions -- counting them would make the ranking self-reinforcing"); + } + + [Fact] + public async Task AnotherOwnersAssertionCountsSeparately() + { + // The MERGE key includes owner_key, so two tenants asserting the same sentence are two facts. + // A shared counter would leak one tenant's conversational emphasis into another's ranking. + var alice = await AssertAsync("user", "lives in", "Zurich"); + await _facts.UpsertAsync(new Fact + { + FactId = $"fact-{Guid.NewGuid():N}", + Subject = "user", + Predicate = "lives in", + Object = "Zurich", + Confidence = 0.9, + OwnerId = "bob", + CreatedAtUtc = DateTimeOffset.UtcNow, + }); + + (await MentionsOfAsync(alice.FactId)).Should().Be(1); + } +} diff --git a/tests/AgentMemory.Tests.Unit/Queries/CypherQuerySnapshot.snap b/tests/AgentMemory.Tests.Unit/Queries/CypherQuerySnapshot.snap index bf3fc27fe..8002bb67c 100644 --- a/tests/AgentMemory.Tests.Unit/Queries/CypherQuerySnapshot.snap +++ b/tests/AgentMemory.Tests.Unit/Queries/CypherQuerySnapshot.snap @@ -1,4 +1,4 @@ -# Cypher Query Snapshot — 158 queries +# Cypher Query Snapshot — 159 queries # Auto-generated by CypherQuerySnapshotTests. # To regenerate: set UPDATE_CYPHER_SNAPSHOTS=1 and re-run the test. @@ -311,6 +311,11 @@ MATCH (f:Fact) WHERE f.embedding IS NULL RETURN f LIMIT $limit ## FactQueries.MarkDeduplicated MATCH (f:Fact {id: $id}) SET f.confidence = $confidence RETURN f +## FactQueries.MentionCounts +MATCH (f:Fact) + WHERE f.id IN $candidateIds + RETURN f.id AS candidateId, coalesce(f.mention_count, 1) AS mentions + ## FactQueries.SelectFactsMissingCanonicalKeys MATCH (f:Fact) WHERE f.predicate_key IS NULL @@ -334,7 +339,13 @@ MERGE (f:Fact {subject_key: $subjectKey, predicate_key: $predicateKey, object_ke f.valid_until = CASE WHEN $validUntil IS NOT NULL THEN datetime($validUntil) ELSE null END, f.source_message_ids = $sourceMessageIds, f.created_at = datetime($createdAtUtc), - f.metadata = $metadata + f.metadata = $metadata, + // R7 salience. Counts how often THE WORLD asserted this fact, incremented only here + // and in ON MATCH -- i.e. once per ingestion that re-states the same triple. + // Deliberately NOT the :MemoryReadAudit trail, which counts how often WE surfaced it: + // ranking on our own retrievals is a rich-get-richer loop that reinforces whatever + // already ranks highly and calls it learning. + f.mention_count = 1 ON MATCH SET f.category = $category, f.confidence = $confidence, @@ -343,7 +354,8 @@ MERGE (f:Fact {subject_key: $subjectKey, predicate_key: $predicateKey, object_ke f.source_message_ids = $sourceMessageIds, f.updated_at = datetime($updatedAtUtc), f.metadata = $metadata, - f.invalidated_at = null + f.invalidated_at = null, + f.mention_count = coalesce(f.mention_count, 1) + 1 RETURN f ## FactQueries.UpsertBatch @@ -361,7 +373,8 @@ UNWIND $items AS item f.valid_until = CASE WHEN item.valid_until IS NOT NULL THEN datetime(item.valid_until) ELSE null END, f.source_message_ids = item.source_message_ids, f.created_at = datetime(item.created_at), - f.metadata = item.metadata + f.metadata = item.metadata, + f.mention_count = 1 ON MATCH SET f.category = item.category, f.confidence = item.confidence, @@ -370,7 +383,8 @@ UNWIND $items AS item f.source_message_ids = item.source_message_ids, f.updated_at = datetime(item.updated_at), f.metadata = item.metadata, - f.invalidated_at = null + f.invalidated_at = null, + f.mention_count = coalesce(f.mention_count, 1) + 1 RETURN f ## FusedPersistenceQueries.EntityUpsertBatch @@ -431,6 +445,7 @@ UNWIND $items AS item f.confidence = item.confidence, f.valid_from = CASE WHEN item.valid_from IS NOT NULL THEN datetime(item.valid_from) ELSE null END, f.valid_until = CASE WHEN item.valid_until IS NOT NULL THEN datetime(item.valid_until) ELSE null END, + f.mention_count = 1, f.source_message_ids = item.source_message_ids, f.created_at = datetime(item.created_at), f.metadata = item.metadata @@ -439,6 +454,7 @@ UNWIND $items AS item f.confidence = item.confidence, f.valid_from = CASE WHEN item.valid_from IS NOT NULL THEN datetime(item.valid_from) ELSE f.valid_from END, f.valid_until = CASE WHEN item.valid_until IS NOT NULL THEN datetime(item.valid_until) ELSE f.valid_until END, + f.mention_count = coalesce(f.mention_count, 1) + 1, f.source_message_ids = item.source_message_ids, f.updated_at = datetime(item.updated_at), f.metadata = item.metadata, diff --git a/tests/AgentMemory.Tests.Unit/Queries/CypherQuerySnapshotTests.cs b/tests/AgentMemory.Tests.Unit/Queries/CypherQuerySnapshotTests.cs index 776f5e053..acc076371 100644 --- a/tests/AgentMemory.Tests.Unit/Queries/CypherQuerySnapshotTests.cs +++ b/tests/AgentMemory.Tests.Unit/Queries/CypherQuerySnapshotTests.cs @@ -37,7 +37,7 @@ private static string ResolveSnapshotPath([CallerFilePath] string? sourceFile = // ── Expected query inventory count ──────────────────────────────────────── // Update this constant whenever queries are deliberately added or removed. - private const int ExpectedQueryCount = 155; // +SchemaQueries.TraceKindIndex (PLAN 7.2: the promotion marker separating an episode from a reusable procedure -- seekable so a procedures-only search is a seek rather than a post-filter over the whole label, and mirroring trace_success_idx). // +SchemaQueries.FactOwnerKeyIndex (measured: FindDuplicate runs on every fact write and had NO index entry point, planning a full 20,000-row scan; this seeks 100). // +SchemaQueries.MessageSessionIndex (measured: Neo4j will not seek a composite from a leading-column predicate alone, so 0007's (session_id, timestamp) gave GetRecentBySession nothing). // +SchemaQueries.MessageSessionTimestampIndex (Message.session_id is the predicate of the PRIMARY short-term recall path -- MessageQueries.cs:201, run on essentially every turn -- and nothing indexed it, so the plan was proportional to every message in the store rather than to the session; composite because TemporalQueries.cs:99-103 adds a trailing timestamp range to the same equality). // +SchemaQueries.FactPredicateKeyIndex (the same shape one column over: predicate_key sits at column 3 of fact_merge_key_idx and Neo4j serves a composite only on a matching PREFIX, so FactQueries.cs:88 had no index entry point at all -- a full :Fact scan across all owners whenever relation-completeness retrieval fires). // +SchemaQueries.MemoryReadAuditMemoryIdIndex (BUG-A2: HistoryQueries OPTIONAL MATCHes MemoryReadAudit on memory_id, which nothing indexed, so every history row scanned the whole label -- and the label grows ~25 rows per recall, so it degrades with TIME rather than data size). // +SchemaQueries.FactMergeKeyIndex (L11: every fact MERGE was an all-:Fact label scan; the composite {subject_key, object_key, predicate_key, owner_key} index is what makes the range-index key cap real for facts, which is why IndexKeyBudget.EnsureCompositeIndexable lands with it). // -1: SearchByCanonicalPredicates became an owner-conditional *method* (excluded, like GetBySubject) when the audit found it ignored IncludeShared and coerced a null-owner scope to the shared bucket. // +FactQueries.SelectFactsMissingCanonicalKeys/ApplyCanonicalKeys (Phase 1.1: canonical identity needs a C#-driven backfill, since Cypher's toLower diverges from ToLowerInvariant on U+0130). // +FactQueries.SearchByCanonicalPredicates (G3B.13: top-K is a relevance cutoff and cannot answer "how many", so a relation is retrieved whole via predicate_key). // +SchemaQueries.ShowIndexStates (BUG-S1: only vector-index dimensions were validated at bootstrap, so a FAILED range index degraded silently into full scans). base + ConsolidationQueries/SchemaQueries/TOUCHED/ConflictQueries consts. +MessageQueries.GetAllBySession (cycle-3); +SchemaQueries.ShowConstraintNames/ShowIndexNames (schema-check CLI); +SchemaPersistenceQueries.Save/DeactivateByName/LoadActiveByName/LoadByNameVersion/List/Exists/DeleteById (G4 schema-node CRUD, 7); +MemoryReadAudit constraint/index. Owner-conditional queries are *methods* (excluded): EntityQueries.ApplyConfidenceDelta/Delete/MergeEntities/SearchByLocation/SearchInBoundingBox/GetByType/FindSimilarByEmbedding, FactQueries.Delete/FindByTriple, PreferenceQueries.Delete, DecayQueries.PruneEntities/PruneFacts/PrunePreferences, ExtractorQueries.GetEntityProvenance, ReasoningQueries.ListTracesBySession (R2 owner-scoped 2026-06-13), ReasoningQueries.DeleteBySession (R6-C owner-scoped 2026-06-20: const→method, −1); +PreferenceQueries.UpsertBatch/RelationshipQueries.UpsertBatch (feat-04, +2). + private const int ExpectedQueryCount = 156; // +FactQueries.MentionCounts (PLAN 10.3: reads the INGESTION-side salience counter for a whole candidate set in one query -- deliberately not the :MemoryReadAudit trail, which counts our own retrievals and would make ranking self-reinforcing). // +SchemaQueries.TraceKindIndex (PLAN 7.2: the promotion marker separating an episode from a reusable procedure -- seekable so a procedures-only search is a seek rather than a post-filter over the whole label, and mirroring trace_success_idx). // +SchemaQueries.FactOwnerKeyIndex (measured: FindDuplicate runs on every fact write and had NO index entry point, planning a full 20,000-row scan; this seeks 100). // +SchemaQueries.MessageSessionIndex (measured: Neo4j will not seek a composite from a leading-column predicate alone, so 0007's (session_id, timestamp) gave GetRecentBySession nothing). // +SchemaQueries.MessageSessionTimestampIndex (Message.session_id is the predicate of the PRIMARY short-term recall path -- MessageQueries.cs:201, run on essentially every turn -- and nothing indexed it, so the plan was proportional to every message in the store rather than to the session; composite because TemporalQueries.cs:99-103 adds a trailing timestamp range to the same equality). // +SchemaQueries.FactPredicateKeyIndex (the same shape one column over: predicate_key sits at column 3 of fact_merge_key_idx and Neo4j serves a composite only on a matching PREFIX, so FactQueries.cs:88 had no index entry point at all -- a full :Fact scan across all owners whenever relation-completeness retrieval fires). // +SchemaQueries.MemoryReadAuditMemoryIdIndex (BUG-A2: HistoryQueries OPTIONAL MATCHes MemoryReadAudit on memory_id, which nothing indexed, so every history row scanned the whole label -- and the label grows ~25 rows per recall, so it degrades with TIME rather than data size). // +SchemaQueries.FactMergeKeyIndex (L11: every fact MERGE was an all-:Fact label scan; the composite {subject_key, object_key, predicate_key, owner_key} index is what makes the range-index key cap real for facts, which is why IndexKeyBudget.EnsureCompositeIndexable lands with it). // -1: SearchByCanonicalPredicates became an owner-conditional *method* (excluded, like GetBySubject) when the audit found it ignored IncludeShared and coerced a null-owner scope to the shared bucket. // +FactQueries.SelectFactsMissingCanonicalKeys/ApplyCanonicalKeys (Phase 1.1: canonical identity needs a C#-driven backfill, since Cypher's toLower diverges from ToLowerInvariant on U+0130). // +FactQueries.SearchByCanonicalPredicates (G3B.13: top-K is a relevance cutoff and cannot answer "how many", so a relation is retrieved whole via predicate_key). // +SchemaQueries.ShowIndexStates (BUG-S1: only vector-index dimensions were validated at bootstrap, so a FAILED range index degraded silently into full scans). base + ConsolidationQueries/SchemaQueries/TOUCHED/ConflictQueries consts. +MessageQueries.GetAllBySession (cycle-3); +SchemaQueries.ShowConstraintNames/ShowIndexNames (schema-check CLI); +SchemaPersistenceQueries.Save/DeactivateByName/LoadActiveByName/LoadByNameVersion/List/Exists/DeleteById (G4 schema-node CRUD, 7); +MemoryReadAudit constraint/index. Owner-conditional queries are *methods* (excluded): EntityQueries.ApplyConfidenceDelta/Delete/MergeEntities/SearchByLocation/SearchInBoundingBox/GetByType/FindSimilarByEmbedding, FactQueries.Delete/FindByTriple, PreferenceQueries.Delete, DecayQueries.PruneEntities/PruneFacts/PrunePreferences, ExtractorQueries.GetEntityProvenance, ReasoningQueries.ListTracesBySession (R2 owner-scoped 2026-06-13), ReasoningQueries.DeleteBySession (R6-C owner-scoped 2026-06-20: const→method, −1); +PreferenceQueries.UpsertBatch/RelationshipQueries.UpsertBatch (feat-04, +2). // ── MemberData source ───────────────────────────────────────────────────── diff --git a/tests/AgentMemory.Tests.Unit/Services/MentionFrequencyRerankerTests.cs b/tests/AgentMemory.Tests.Unit/Services/MentionFrequencyRerankerTests.cs new file mode 100644 index 000000000..683aabfda --- /dev/null +++ b/tests/AgentMemory.Tests.Unit/Services/MentionFrequencyRerankerTests.cs @@ -0,0 +1,199 @@ +using AgentMemory.Abstractions.Domain; +using AgentMemory.Abstractions.Options; +using AgentMemory.Abstractions.Services; +using AgentMemory.Neo4j.Infrastructure; +using AgentMemory.Neo4j.Queries; +using AgentMemory.Neo4j.Services; +using FluentAssertions; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Neo4j.Driver; +using NSubstitute; +using Xunit; + +namespace AgentMemory.Tests.Unit.Services; + +/// +/// Mention-frequency reranking (R7), and the loop it must not become. +/// +/// +/// +/// A fact the world asserted five times is usually more central than one mentioned once, and +/// similarity cannot see that: two facts phrased alike score alike however often either was said. +/// +/// +/// The signal's provenance is the task. The counter must record how often the world +/// re-asserted a fact, never how often we surfaced it. Ranking on our own retrievals is a +/// rich-get-richer loop — whatever ranks highly gets retrieved, retrieval raises the count, the count +/// raises the rank — which looks like learning and is self-reinforcement. The tests below pin the +/// source of the number, not only its effect. +/// +/// +public sealed class MentionFrequencyRerankerTests +{ + private readonly INeo4jTransactionRunner _tx = Substitute.For(); + + private MentionFrequencyReranker CreateSut(bool enabled = true) => + new(_tx, + NullLogger.Instance, + Options.Create(new MemoryOptions { MentionFrequencyReranking = enabled })); + + private static MemoryRerankContext Context(MemoryItemKind kind = MemoryItemKind.Fact) => + new("where does alice live", [0.1f, 0.2f], MemoryScope.For("alice"), kind); + + private void Mentions(params (string Id, int Count)[] counts) => + _tx.ReadAsync( + Arg.Any>>>(), + Arg.Any()) + .Returns((IReadOnlyDictionary)counts.ToDictionary( + c => c.Id, c => c.Count, StringComparer.Ordinal)); + + // ── the signal's provenance ─────────────────────────────────────────── + + [Fact] + public void TheCounterIsIncrementedByIngestionAndNothingElse() + { + // THE constraint. mention_count appears in the upsert paths -- where a new ingestion re-states + // an existing triple -- and is read by exactly one query. If it ever appears in a retrieval + // path, the loop is closed and the ranking starts reinforcing itself. + var upsert = FactQueries.Upsert + FactQueries.UpsertBatch; + + upsert.Should().Contain("f.mention_count = 1", "a first assertion counts once"); + upsert.Should().Contain("coalesce(f.mention_count, 1) + 1", "a re-assertion counts again"); + FactQueries.MentionCounts.Should().Contain("coalesce(f.mention_count, 1)"); + } + + [Fact] + public void TheReadAuditIsNeverConsulted() + { + // The substitution this task exists to refuse. :MemoryReadAudit records our own retrievals; + // ranking on it would promote whatever already ranks highly, on evidence we generated. + FactQueries.MentionCounts.Should().NotContain("MemoryReadAudit"); + FactQueries.MentionCounts.Should().NotContain("read_count"); + } + + [Fact] + public void APreExistingFactCountsAsMentionedOnce() + { + // Facts written before the property existed have no counter. One is the honest reading of an + // absent count -- it was asserted at least once -- and it leaves their ranking unchanged, + // since log(1) is 0. + FactQueries.MentionCounts.Should().Contain("coalesce(f.mention_count, 1)"); + } + + // ── the boost ───────────────────────────────────────────────────────── + + [Fact] + public async Task AWellAttestedFactOvertakesANearTie() + { + // Salience breaks a near-tie: 0.80*(1+0.15*ln 20) = 1.16 beats 0.82*(1+0) = 0.82. + Mentions(("often", 20), ("once", 1)); + var candidates = new[] + { + new MemoryContextRankedItem("once", 0.82, 1, 1), + new MemoryContextRankedItem("often", 0.80, 2, 2), + }; + + var result = await CreateSut().RerankAsync(candidates, Context()); + + result.Select(r => r.ItemId).Should().Equal("often", "once"); + } + + [Fact] + public async Task ChatterDoesNotBuryAPreciseAnswer() + { + // The failure a linear boost would cause. A fact mentioned 50 times must not outrank a much + // better match: 0.30*(1+0.15*ln 50) = 0.48 stays below 0.95*(1+0) = 0.95. Salience is a + // tiebreaker among plausible answers, not a substitute for answering the question. + Mentions(("chatter", 50), ("precise", 1)); + var candidates = new[] + { + new MemoryContextRankedItem("precise", 0.95, 1, 1), + new MemoryContextRankedItem("chatter", 0.30, 2, 2), + }; + + var result = await CreateSut().RerankAsync(candidates, Context()); + + result.Select(r => r.ItemId).Should().Equal("precise", "chatter"); + } + + [Fact] + public async Task TheBoostIsLogarithmicNotLinear() + { + // The gap between 1 and 3 mentions is real; between 30 and 32 it is noise. Under a linear + // boost the 32-mention fact would win here; under a log boost the stronger match holds. + Mentions(("many", 32), ("fewer", 30)); + var candidates = new[] + { + new MemoryContextRankedItem("fewer", 0.90, 1, 1), + new MemoryContextRankedItem("many", 0.89, 2, 2), + }; + + var result = await CreateSut().RerankAsync(candidates, Context()); + + result.Select(r => r.ItemId).Should().Equal("fewer", "many"); + } + + [Fact] + public async Task ScoresAreNeverRewritten() + { + // A reranker reorders. Rewriting the provider's score would corrupt the section diagnostics, + // which report TopScore from these same items. + Mentions(("a", 10), ("b", 1)); + var candidates = new[] + { + new MemoryContextRankedItem("a", 0.70, 1, 1), + new MemoryContextRankedItem("b", 0.90, 2, 2), + }; + + var result = await CreateSut().RerankAsync(candidates, Context()); + + result.Should().OnlyContain(r => r.Score == 0.70 || r.Score == 0.90); + result.Select(r => r.ItemId).Should().BeEquivalentTo(["a", "b"]); + } + + // ── the gates ───────────────────────────────────────────────────────── + + [Fact] + public void ItIsOffByDefault() + { + new MemoryOptions().MentionFrequencyReranking.Should().BeFalse(); + CreateSut(enabled: false).IsEnabled.Should().BeFalse(); + } + + [Fact] + public async Task ANonFactSectionIsLeftAloneWithoutQuerying() + { + // mention_count is maintained on Fact only. Boosting another kind by a counter nothing + // increments would be a no-op dressed as a feature -- and would still cost a query. + var candidates = new[] + { + new MemoryContextRankedItem("a", 0.9, 1, 1), + new MemoryContextRankedItem("b", 0.8, 2, 2), + }; + + var result = await CreateSut().RerankAsync(candidates, Context(MemoryItemKind.Entity)); + + result.Select(r => r.ItemId).Should().Equal("a", "b"); + await _tx.DidNotReceive().ReadAsync( + Arg.Any>>>(), + Arg.Any()); + } + + [Fact] + public async Task EqualSalienceLeavesTheProviderOrderIntact() + { + // Ties break on retrieval rank rather than arbitrarily, so enabling the reranker on a corpus + // where every fact was asserted once changes nothing at all. + Mentions(("a", 1), ("b", 1)); + var candidates = new[] + { + new MemoryContextRankedItem("a", 0.9, 1, 1), + new MemoryContextRankedItem("b", 0.9, 2, 2), + }; + + var result = await CreateSut().RerankAsync(candidates, Context()); + + result.Select(r => r.ItemId).Should().Equal("a", "b"); + } +} From a3f25eabbd790c4bc6e5789178687dedefe8d6a8 Mon Sep 17 00:00:00 2001 From: Jose Luis Latorre Millas Date: Wed, 12 Aug 2026 21:23:13 +0200 Subject: [PATCH 16/31] 2.4: take the access-tracking write off the blocking recall path Access tracking is bookkeeping -- it feeds decay and retention, and nothing in the recall the caller is waiting for depends on it. It was already batched down from 25 write transactions to one, but still awaited before the model is invoked. MemoryOptions.DeferAccessTracking starts it without waiting. Two details carry the whole change: The deferred work runs on CancellationToken.None, not the request's token. That token is cancelled as soon as the response completes, so passing it through would cancel the very write being deferred -- an option that reads as enabled and does nothing, with no error anywhere. The test captures the token rather than trusting the code. And it is off by default for a hazard rather than out of caution. The write runs on SCOPED services -- a driver session, a repository -- so a host that disposes its DI scope when the response returns disposes them out from under the deferred write. That surfaces as an ObjectDisposedException in a log nobody reads, and access tracking silently stops working. A long-lived agent or hosted service can enable it; a request-scoped host should not, and the log message says so by name so the failure diagnoses itself. Failures go through a fault continuation rather than being discarded, so a deferred failure is logged instead of becoming an unobserved task exception. 4216 unit tests green. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01PgDgctPpbTziBNE2RT8VdE --- .../Options/MemoryOptions.cs | 25 ++++ .../Services/MemoryService.cs | 32 ++++- .../MemoryServiceAccessTrackingTests.cs | 122 +++++++++++++++++- 3 files changed, 174 insertions(+), 5 deletions(-) diff --git a/src/AgentMemory.Abstractions/Options/MemoryOptions.cs b/src/AgentMemory.Abstractions/Options/MemoryOptions.cs index cafd5d8eb..edce31547 100644 --- a/src/AgentMemory.Abstractions/Options/MemoryOptions.cs +++ b/src/AgentMemory.Abstractions/Options/MemoryOptions.cs @@ -89,6 +89,31 @@ public sealed record MemoryOptions /// public bool MentionFrequencyReranking { get; init; } + /// + /// Starts the post-recall access-tracking write without waiting for it. + /// + /// + /// + /// Access tracking is bookkeeping — it feeds decay and retention, and nothing in the recall the + /// caller is waiting for depends on it. Awaiting it puts a write burst on the pre-model read path: + /// at shipped defaults, up to 25 nodes updated before the model is invoked. + /// + /// + /// Off by default, and the reason is a hazard rather than caution. The write runs on scoped + /// services — a driver session, a repository — and a host that disposes its DI scope when the + /// response is returned will dispose them out from under a deferred write. That surfaces as an + /// ObjectDisposedException in a log nobody reads, and access tracking silently stops. Hosts + /// whose scope outlives the response (a long-lived agent, a hosted service) can turn it on; a + /// short-lived request scope should not. + /// + /// + /// Deferred work is detached from the request's cancellation token deliberately: that token is + /// cancelled as soon as the response completes, so passing it through would cancel the very write + /// that was deferred — a feature that looks enabled and does nothing. + /// + /// + public bool DeferAccessTracking { get; init; } + // NOTE: extraction at the Core layer is explicit (call ExtractAndPersistAsync / // ExtractFromSessionAsync). Automatic extraction on message persist is an adapter concern, configured // by AgentFrameworkOptions.AutoExtractOnPersist. The former EnableAutoExtraction flag here was read diff --git a/src/AgentMemory.Core/Services/MemoryService.cs b/src/AgentMemory.Core/Services/MemoryService.cs index 5e6ce04b1..72f41c874 100644 --- a/src/AgentMemory.Core/Services/MemoryService.cs +++ b/src/AgentMemory.Core/Services/MemoryService.cs @@ -88,11 +88,37 @@ public async Task RecallAsync( var context = await _assembler.AssembleContextAsync(request, cancellationToken).ConfigureAwait(false); - // Update access timestamps for recalled long-term memories (awaited so failures and - // cancellation are observed; the method itself is resilient and logs internally). + // Update access timestamps for recalled long-term memories. Awaited by default so failures and + // cancellation are observed; the method itself is resilient and logs internally. if (_decayService is not null) { - await UpdateAccessTimestampsAsync(context, cancellationToken).ConfigureAwait(false); + if (_options.DeferAccessTracking) + { + // 2.4. Bookkeeping the caller is not waiting for: it feeds decay and retention, and + // nothing in the returned context depends on it. + // + // CancellationToken.None, NOT the request's token. That token is cancelled as soon as + // the response completes, so passing it through would cancel the very write being + // deferred -- an option that reads as enabled and does nothing. + // + // The task is deliberately not awaited and deliberately not discarded silently: the + // continuation is what keeps a deferred failure from being an unobserved exception. + _ = UpdateAccessTimestampsAsync(context, CancellationToken.None) + .ContinueWith( + task => _logger.LogWarning( + task.Exception, + "Deferred access tracking failed after recall returned. If this is an " + + "ObjectDisposedException, the host's DI scope was disposed before the " + + "write completed -- MemoryOptions.DeferAccessTracking is unsafe for a " + + "request-scoped host."), + CancellationToken.None, + TaskContinuationOptions.OnlyOnFaulted, + TaskScheduler.Default); + } + else + { + await UpdateAccessTimestampsAsync(context, cancellationToken).ConfigureAwait(false); + } } int totalItems = context.RecentMessages.Items.Count diff --git a/tests/AgentMemory.Tests.Unit/Services/MemoryServiceAccessTrackingTests.cs b/tests/AgentMemory.Tests.Unit/Services/MemoryServiceAccessTrackingTests.cs index 3302b5973..a0530f75c 100644 --- a/tests/AgentMemory.Tests.Unit/Services/MemoryServiceAccessTrackingTests.cs +++ b/tests/AgentMemory.Tests.Unit/Services/MemoryServiceAccessTrackingTests.cs @@ -72,14 +72,132 @@ public MemoryServiceAccessTrackingTests() return (IReadOnlyCollection<(string NodeId, MemoryNodeKind NodeKind)>)call.GetArguments()[0]!; } - private MemoryService CreateSut(IMemoryDecayService? decay = null) => + private MemoryService CreateSut(IMemoryDecayService? decay = null, bool deferred = false) => new(_shortTerm, _assembler, _extractionPipeline, _entityRepository, _factRepository, _preferenceRepository, _embeddingOrchestrator, - Options.Create(new MemoryOptions()), + Options.Create(new MemoryOptions { DeferAccessTracking = deferred }), _clock, _idGenerator, NullLogger.Instance, decay); + // ── 2.4: taking the bookkeeping write off the blocking path ────────── + + /// A context with one recalled fact, enough to trigger access tracking. + private MemoryContext OneFactContext() => new() + { + SessionId = "s1", + AssembledAtUtc = _fixedTime, + RelevantFacts = new MemoryContextSection + { + Items = new[] + { + new Fact + { + FactId = "fact-1", Subject = "s", Predicate = "p", Object = "o", + Confidence = 0.9, CreatedAtUtc = _fixedTime, + }, + }, + }, + }; + + [Fact] + public async Task DeferredAccessTrackingDoesNotBlockTheRecall() + { + // THE point. Access tracking feeds decay and retention; nothing in the returned context + // depends on it, so the caller should not wait for a write burst before the model is invoked. + // The substitute never completes, so a recall that returns proves it was not awaited. + var pending = new TaskCompletionSource(); + var decay = Substitute.For(); + decay.UpdateAccessTimestampsAsync( + Arg.Any>(), Arg.Any()) + .Returns(pending.Task); + _assembler.AssembleContextAsync(Arg.Any(), Arg.Any()) + .Returns(OneFactContext()); + + var recall = CreateSut(decay, deferred: true) + .RecallAsync(new RecallRequest { SessionId = "s1", Query = "q" }); + + (await Task.WhenAny(recall, Task.Delay(TimeSpan.FromSeconds(5)))).Should().Be(recall, + "the recall must return without waiting for the bookkeeping write"); + pending.SetResult(); + } + + [Fact] + public async Task TheDefaultStillAwaitsTheWrite() + { + // The byte-identical guarantee: leaving the option off keeps failures and cancellation + // observable on the calling path, which is why it was awaited in the first place. + var completed = false; + var decay = Substitute.For(); + decay.UpdateAccessTimestampsAsync( + Arg.Any>(), Arg.Any()) + .Returns(async _ => + { + await Task.Delay(20); + completed = true; + }); + _assembler.AssembleContextAsync(Arg.Any(), Arg.Any()) + .Returns(OneFactContext()); + + await CreateSut(decay).RecallAsync(new RecallRequest { SessionId = "s1", Query = "q" }); + + completed.Should().BeTrue("the default path awaits the write"); + } + + [Fact] + public async Task ADeferredWriteIsDetachedFromTheRequestCancellationToken() + { + // The trap that would make this a no-op wearing a feature's name. The request token is + // cancelled as soon as the response completes, so a deferred write that still carried it + // would be cancelled by the very act of returning. + CancellationToken captured = default; + var decay = Substitute.For(); + decay.UpdateAccessTimestampsAsync( + Arg.Any>(), Arg.Any()) + .Returns(call => + { + captured = call.ArgAt(1); + return Task.CompletedTask; + }); + _assembler.AssembleContextAsync(Arg.Any(), Arg.Any()) + .Returns(OneFactContext()); + + using var requestCts = new CancellationTokenSource(); + await CreateSut(decay, deferred: true) + .RecallAsync(new RecallRequest { SessionId = "s1", Query = "q" }, requestCts.Token); + await Task.Delay(50); + requestCts.Cancel(); + + captured.Should().Be(CancellationToken.None, + "a deferred write must outlive the request that scheduled it"); + } + + [Fact] + public async Task ADeferredFailureDoesNotFaultTheRecall() + { + // A bookkeeping write that throws must not surface as a failed recall -- and must not become + // an unobserved task exception either, which is why the continuation logs it. + var decay = Substitute.For(); + decay.UpdateAccessTimestampsAsync( + Arg.Any>(), Arg.Any()) + .Returns(_ => throw new InvalidOperationException("driver disposed")); + _assembler.AssembleContextAsync(Arg.Any(), Arg.Any()) + .Returns(OneFactContext()); + + var act = async () => await CreateSut(decay, deferred: true) + .RecallAsync(new RecallRequest { SessionId = "s1", Query = "q" }); + + await act.Should().NotThrowAsync(); + } + + [Fact] + public void DeferralIsOffByDefault() + { + // It is unsafe for a request-scoped host: the write runs on scoped services, and a scope + // disposed when the response returns takes the driver session with it. + new MemoryOptions().DeferAccessTracking.Should().BeFalse(); + } + [Fact] public async Task RecallAsync_WithDecayService_UpdatesEntityAccess() { From 7a6597469c1115a6efcdcced1817dfdd4b1bfa8d Mon Sep 17 00:00:00 2001 From: Jose Luis Latorre Millas Date: Wed, 12 Aug 2026 21:28:36 +0200 Subject: [PATCH 17/31] 2.5: stop re-sending the turns the host is already sending The MAF provider receives the full live thread and discarded it, so recall's RecentMessages re-sent turns the model was already being given and the host paid for both copies. The fingerprint is content-only, and the plan's warning about why was verified rather than taken on trust: RecalledMessageRoleGate.EffectiveRole rewrites a privileged role down to "user" when trust is below the threshold, leaving content identical. A role-keyed match would therefore find nothing on exactly the hosts that hardened MinimumTrustForSystemRole -- the feature would work everywhere except the security-conscious configuration, and look correct in every test that did not set that option. There is now a test that does. Filtering happens BEFORE MaxChatHistoryMessages, which is the quality half rather than only the cost half: with a budget of 2 and the two newest turns already in the thread, the budget now carries two OLDER messages the model has not seen. Filtering afterwards would have spent both slots on duplicates and delivered nothing. Normalisation collapses whitespace and folds with ToUpperInvariant -- the live thread and the stored copy travel different paths, one through the host's formatting and one through persistence, so a trailing newline is not a different message; and a culture-sensitive fold would make dedup depend on the host's locale, since a Turkish locale maps i to a different capital. The live thread is materialised once. It arrives as IEnumerable, and it is now enumerated both for the user-message query and for dedup -- enumerating a lazily built source twice would be a silent correctness bug. On by default: sending the model two copies of the same turn has no upside, and the comparison is a hash set over a thread the provider already holds. Passing null reproduces the pre-2.5 output exactly, which is what every recorded measurement was taken under. 4226 unit tests green. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01PgDgctPpbTziBNE2RT8VdE --- .../AgentFrameworkOptions.cs | 27 +++ .../Mapping/MafTypeMapper.cs | 57 +++++- .../Neo4jMemoryContextProvider.cs | 15 +- .../AgentFramework/HistoryThreadDedupTests.cs | 177 ++++++++++++++++++ 4 files changed, 273 insertions(+), 3 deletions(-) create mode 100644 tests/AgentMemory.Tests.Unit/AgentFramework/HistoryThreadDedupTests.cs diff --git a/src/AgentMemory.AgentFramework/AgentFrameworkOptions.cs b/src/AgentMemory.AgentFramework/AgentFrameworkOptions.cs index d74aa781b..03e243468 100644 --- a/src/AgentMemory.AgentFramework/AgentFrameworkOptions.cs +++ b/src/AgentMemory.AgentFramework/AgentFrameworkOptions.cs @@ -11,6 +11,33 @@ public sealed class AgentFrameworkOptions /// public ContextFormatOptions ContextFormat { get; set; } = new(); + /// + /// Drops recalled chat history the host is already sending in the live thread. + /// + /// + /// + /// The provider sees the full live thread and, until now, discarded it — so recall's + /// RecentMessages re-sent turns the model was already being given, and the host paid for + /// both copies. + /// + /// + /// Filtered before MaxChatHistoryMessages applies, so this is a quality change as + /// much as a cost one: the same budget then carries that many genuinely new messages instead of + /// duplicates of the current turn. + /// + /// + /// Matching is on content only, never role. RecalledMessageRoleGate rewrites a + /// recalled message's role — privileged down to user below the trust threshold — while leaving its + /// content identical, so a role-keyed comparison would miss every match on exactly the hosts that + /// raised MinimumTrustForSystemRole. + /// + /// + /// On by default: sending the model two copies of the same turn has no upside, and the comparison + /// is a hash set over the thread the provider already holds. + /// + /// + public bool DeduplicateRecalledHistory { get; set; } = true; + /// /// When , the memory service runs extraction (entity/fact/preference) /// automatically each time a message is persisted. Set to to extract diff --git a/src/AgentMemory.AgentFramework/Mapping/MafTypeMapper.cs b/src/AgentMemory.AgentFramework/Mapping/MafTypeMapper.cs index b95beb03b..58f45d79b 100644 --- a/src/AgentMemory.AgentFramework/Mapping/MafTypeMapper.cs +++ b/src/AgentMemory.AgentFramework/Mapping/MafTypeMapper.cs @@ -59,7 +59,8 @@ public static IReadOnlyList ToContextMessages( MemoryContext context, ContextFormatOptions? formatOptions = null, IMemoryContextAdmissionPolicy? admissionPolicy = null, - ILogger? logger = null) + ILogger? logger = null, + IReadOnlyList? liveThread = null) { var options = formatOptions ?? new ContextFormatOptions(); var admission = admissionPolicy ?? new DefaultMemoryContextAdmissionPolicy(); @@ -199,6 +200,38 @@ List CategoryMessages( // newest `chatBudget` items are the FRONT of the list — Take(chatBudget). (R6-D: the previous // Skip(count - chatBudget) kept the TAIL, i.e. the OLDEST messages, dropping the newest turns // first — the opposite of "most recent".) Order is preserved (lead → chat → memory). + // The host already sends the live thread, and recall returns the same recent turns from + // storage, so the model sees them twice. Filtered BEFORE the budget on purpose: dropping + // duplicates first turns the same `chatBudget` slots into that many genuinely new messages + // rather than merely shortening the block -- a quality gain, not only a token saving. + // + // Fingerprinted on CONTENT ONLY, never role. RecalledMessageRoleGate rewrites a recalled + // message's role -- privileged to "user" below the trust threshold -- while leaving its + // content untouched, so a role-keyed fingerprint would fail to match precisely for the hosts + // that hardened MinimumTrustForSystemRole: the feature would silently no-op for the + // security-conscious configuration and work everywhere else. + if (liveThread is { Count: > 0 }) + { + var live = liveThread + .Select(message => NormalizeForDedup(message.Text)) + .Where(text => text.Length > 0) + .ToHashSet(StringComparer.Ordinal); + + if (live.Count > 0) + { + var deduped = chatMessages + .Where(message => !live.Contains(NormalizeForDedup(message.Text))) + .ToList(); + if (deduped.Count != chatMessages.Count) + { + logger?.LogDebug( + "Dropped {Dropped} recalled message(s) already present in the live thread.", + chatMessages.Count - deduped.Count); + } + chatMessages = deduped; + } + } + int chatBudget = Math.Max(0, options.MaxChatHistoryMessages); var keptChat = chatMessages.Count > chatBudget ? chatMessages.Take(chatBudget).ToList() @@ -211,6 +244,28 @@ List CategoryMessages( return result; } + /// + /// The dedup key for a chat message: its content, normalised for whitespace and case. + /// + /// + /// + /// Content only. Role is excluded because RecalledMessageRoleGate rewrites a recalled + /// message's role while leaving its content identical, so keying on role would miss every match on + /// exactly the hosts that raised MinimumTrustForSystemRole. + /// + /// + /// Whitespace-collapsed and case-folded because the live thread and the stored copy travel + /// different paths — one through the host's own formatting, one through persistence and back — and + /// a trailing newline is not a different message. Ordinal comparison after folding, so this stays + /// culture-independent. + /// + /// + internal static string NormalizeForDedup(string? text) => + string.IsNullOrWhiteSpace(text) + ? string.Empty + : string.Join(' ', text.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries)) + .ToUpperInvariant(); + // Extracted from ToContextMessages' local Admit closure (stabilization fix) so it can be shared with // ToGatedChatMessages below, instead of the two call sites re-deriving the same admission decision and // logging independently. Behavior is unchanged from the closure this replaces. diff --git a/src/AgentMemory.AgentFramework/Neo4jMemoryContextProvider.cs b/src/AgentMemory.AgentFramework/Neo4jMemoryContextProvider.cs index f06fdce70..63e7f756c 100644 --- a/src/AgentMemory.AgentFramework/Neo4jMemoryContextProvider.cs +++ b/src/AgentMemory.AgentFramework/Neo4jMemoryContextProvider.cs @@ -103,7 +103,12 @@ internal async Task BuildContextAsync( using var ownerScope = _ownerContext?.BeginOwnerScope(userId); try { - var userMessages = messages + // Materialised once: the thread is enumerated for the query below AND handed to the + // mapper for dedup, and `messages` is an IEnumerable that a caller may well have built + // lazily. Enumerating it twice would be a silent correctness bug for a generator source. + var liveThread = messages as IReadOnlyList ?? messages.ToList(); + + var userMessages = liveThread .Where(m => m.Role == ChatRole.User && !string.IsNullOrWhiteSpace(m.Text)) .ToList(); @@ -194,7 +199,13 @@ internal async Task BuildContextAsync( return BuildResult(); } - var contextMessages = MafTypeMapper.ToContextMessages(recallResult.Context, _formatOptions, _admissionPolicy, _logger); + // 2.5. The host is already sending the live thread; recall returns the same recent turns + // from storage, so without this the model sees them twice and pays for both. Passed here + // rather than filtered afterwards because the mapper drops duplicates BEFORE applying + // MaxChatHistoryMessages -- so the same budget carries that many genuinely new messages. + var contextMessages = MafTypeMapper.ToContextMessages( + recallResult.Context, _formatOptions, _admissionPolicy, _logger, + _agentOptions.DeduplicateRecalledHistory ? liveThread : null); if (contextMessages.Count == 0) return BuildResult(); diff --git a/tests/AgentMemory.Tests.Unit/AgentFramework/HistoryThreadDedupTests.cs b/tests/AgentMemory.Tests.Unit/AgentFramework/HistoryThreadDedupTests.cs new file mode 100644 index 000000000..28dfb3e41 --- /dev/null +++ b/tests/AgentMemory.Tests.Unit/AgentFramework/HistoryThreadDedupTests.cs @@ -0,0 +1,177 @@ +using AgentMemory.Abstractions.Domain; +using AgentMemory.AgentFramework; +using AgentMemory.AgentFramework.Mapping; +using FluentAssertions; +using Microsoft.Extensions.AI; +using Xunit; + +namespace AgentMemory.Tests.Unit.AgentFramework; + +/// +/// Recalled chat history the host is already sending must not be sent twice (PLAN 2.5). +/// +/// +/// +/// The provider receives the full live thread and, until now, discarded it — so recall's +/// RecentMessages re-sent turns the model was already being given, and the host paid for both +/// copies. +/// +/// +/// The role trap is the interesting part. RecalledMessageRoleGate rewrites a recalled +/// message's role — privileged down to user below the trust threshold — while leaving its +/// content identical. A role-keyed fingerprint therefore matches nothing on precisely the hosts that +/// hardened MinimumTrustForSystemRole: the feature would work everywhere except the +/// security-conscious configuration, and look correct in every test that did not set that option. +/// +/// +public sealed class HistoryThreadDedupTests +{ + private static Message Recalled(string content, string role = "user") => new() + { + MessageId = $"m-{content.GetHashCode():X}", + ConversationId = "c-1", + SessionId = "s-1", + Role = role, + Content = content, + TimestampUtc = DateTimeOffset.UnixEpoch, + }; + + private static MemoryContext Context(params Message[] recent) => new() + { + SessionId = "s-1", + AssembledAtUtc = DateTimeOffset.UnixEpoch, + RecentMessages = new MemoryContextSection { Items = recent }, + }; + + private static IReadOnlyList Map( + MemoryContext context, + IReadOnlyList? liveThread, + int chatBudget = 10, + MemoryTrustLevel systemRoleThreshold = MemoryTrustLevel.Untrusted) => + MafTypeMapper.ToContextMessages( + context, + new ContextFormatOptions + { + MaxChatHistoryMessages = chatBudget, + MinimumTrustForSystemRole = systemRoleThreshold, + }, + admissionPolicy: null, + logger: null, + liveThread: liveThread); + + [Fact] + public void ARecalledMessageAlreadyInTheLiveThreadIsDropped() + { + var mapped = Map( + Context(Recalled("what is my deploy command"), Recalled("an older turn")), + [new ChatMessage(ChatRole.User, "what is my deploy command")]); + + mapped.Select(m => m.Text).Should().NotContain("what is my deploy command"); + mapped.Select(m => m.Text).Should().Contain(t => t.Contains("an older turn")); + } + + [Fact] + public void MatchingIsOnContentEvenWhenTheRoleWasRewritten() + { + // THE case. The live thread carries an assistant turn; recall returns the same content, and + // the role gate rewrites it to user because its trust is below the threshold. Content is + // identical, roles are not -- a role-keyed fingerprint would miss it. + var recalled = Recalled("I recommended the 14:05 train", role: "assistant"); + + var mapped = Map( + Context(recalled), + [new ChatMessage(ChatRole.Assistant, "I recommended the 14:05 train")], + systemRoleThreshold: MemoryTrustLevel.ApplicationTrusted); + + mapped.Select(m => m.Text).Should().NotContain(t => t.Contains("14:05 train")); + } + + [Fact] + public void TheDedupHappensBeforeTheChatBudget() + { + // The quality half, not just the cost half. With a budget of 2 and the two newest turns + // already in the thread, filtering first lets the budget carry two OLDER messages the model + // has not seen; filtering afterwards would have spent both slots on duplicates and delivered + // nothing. + var mapped = Map( + Context(Recalled("newest"), Recalled("second"), Recalled("third"), Recalled("fourth")), + [new ChatMessage(ChatRole.User, "newest"), new ChatMessage(ChatRole.User, "second")], + chatBudget: 2); + + var texts = mapped.Select(m => m.Text).ToList(); + texts.Should().Contain(t => t.Contains("third")); + texts.Should().Contain(t => t.Contains("fourth")); + texts.Should().NotContain(t => t.Contains("newest")); + } + + [Fact] + public void WhitespaceAndCasingDoNotDefeatTheMatch() + { + // The live thread and the stored copy travel different paths -- one through the host's own + // formatting, one through persistence and back. A trailing newline is not a different message. + var mapped = Map( + Context(Recalled("Deploy the service\n")), + [new ChatMessage(ChatRole.User, "deploy the service")]); + + mapped.Select(m => m.Text).Should().NotContain(t => t.Contains("Deploy")); + } + + [Fact] + public void ANonMatchingLiveThreadChangesNothing() + { + var mapped = Map( + Context(Recalled("stored turn")), + [new ChatMessage(ChatRole.User, "a completely different question")]); + + mapped.Select(m => m.Text).Should().Contain(t => t.Contains("stored turn")); + } + + [Fact] + public void NoLiveThreadLeavesTheOutputByteIdentical() + { + // The off switch and the byte-identical guarantee: passing null must reproduce exactly the + // pre-2.5 behaviour, which is what every recorded measurement was taken under. + var context = Context(Recalled("stored turn"), Recalled("another")); + + var withoutThread = Map(context, liveThread: null); + var withEmptyThread = Map(context, liveThread: []); + + withoutThread.Select(m => m.Text).Should().Equal(withEmptyThread.Select(m => m.Text)); + // Both recalled turns survive. Counted by content rather than by list length, because the + // mapper also emits the security prefix -- asserting a raw count would break whenever that + // preamble changed, for a reason unrelated to dedup. + withoutThread.Select(m => m.Text).Should() + .Contain(t => t.Contains("stored turn")) + .And.Contain(t => t.Contains("another")); + } + + [Fact] + public void AnEmptyOrBlankLiveMessageMatchesNothing() + { + // A blank entry normalises to the empty string; if that were a valid key it would match every + // recalled message whose content was also blank, and worse, invite an empty-vs-empty match + // that silently drops content. + var mapped = Map( + Context(Recalled("stored turn")), + [new ChatMessage(ChatRole.User, " "), new ChatMessage(ChatRole.User, "")]); + + mapped.Select(m => m.Text).Should().Contain(t => t.Contains("stored turn")); + } + + [Fact] + public void TheNormalizerIsCultureIndependent() + { + // ToUpperInvariant, not ToUpper: a Turkish locale folds 'i' to 'İ', so a culture-sensitive + // normaliser would make dedup depend on the host's locale. + MafTypeMapper.NormalizeForDedup("Istanbul file") + .Should().Be(MafTypeMapper.NormalizeForDedup("ISTANBUL FILE")); + } + + [Fact] + public void DedupIsOnByDefault() + { + // Sending the model two copies of the same turn has no upside, and the comparison is a hash + // set over a thread the provider already holds. + new AgentFrameworkOptions().DeduplicateRecalledHistory.Should().BeTrue(); + } +} From 43d7d4567212b015fb902e3964766aaab271951f Mon Sep 17 00:00:00 2001 From: Jose Luis Latorre Millas Date: Wed, 12 Aug 2026 21:39:09 +0200 Subject: [PATCH 18/31] 12.4: confidence that earns and loses Confidence was set once by extraction and never moved, so a fact stated five times and one stated in passing carried whatever number the extractor happened to report. The two events S2 needs already existed after this session's other work: mention_count (10.3) marks the moment the world re-asserts a triple, and write-time supersession (9.1) marks the moment one fact replaces another. So this is arithmetic on moments already identified rather than new machinery. Corroboration adds alpha on the MERGE's ON MATCH. Contradiction subtracts 2*alpha from the superseded loser, and the asymmetry is deliberate: being contradicted is stronger evidence against a fact than one more restatement is for it -- a repeated claim may be a habit of phrasing, a replaced one is a claim the world stopped believing. Only the loser moves; demoting both would make every contradiction erode the graph. Clamped to [0,1] at both ends, and checked in a live database rather than in the Cypher text, because that arithmetic is where an escape would happen and confidence is read by ranking, dedup and decay -- a negative value would be consumed as a number rather than rejected as an error. The gate lives in the Cypher, not in C#: at alpha 0 the assignment is byte-for-byte the original, so no sealed measurement moves and there is no second code path to keep in step. Applied on all three write paths -- single, batch and fused -- because reinforcement that depended on which path ran would be a property of batching rather than of the conversation. Weak conflict (-alpha) is deliberately not built. For a multi-valued predicate a second object is not a conflict at all -- "likes tea" and "likes coffee" are both true -- and detecting the genuinely ambiguous case would cost a lookup on every write to penalise something that is usually correct. 4226 unit + 345 non-NAMS integration green. Cypher snapshot regenerated (BOM stripped); no query-count change, since these are edits rather than additions. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01PgDgctPpbTziBNE2RT8VdE --- .../Options/MemoryOptions.cs | 27 +++ src/AgentMemory.Neo4j/Queries/FactQueries.cs | 30 +++- .../Queries/FusedPersistenceQueries.cs | 9 +- .../Repositories/Neo4jFactRepository.Fused.cs | 4 +- .../Repositories/Neo4jFactRepository.cs | 9 +- ...ConfidenceReinforcementIntegrationTests.cs | 166 ++++++++++++++++++ .../Queries/CypherQuerySnapshot.snap | 28 ++- 7 files changed, 263 insertions(+), 10 deletions(-) create mode 100644 tests/AgentMemory.Tests.Integration/Repositories/ConfidenceReinforcementIntegrationTests.cs diff --git a/src/AgentMemory.Abstractions/Options/MemoryOptions.cs b/src/AgentMemory.Abstractions/Options/MemoryOptions.cs index edce31547..75de6fa63 100644 --- a/src/AgentMemory.Abstractions/Options/MemoryOptions.cs +++ b/src/AgentMemory.Abstractions/Options/MemoryOptions.cs @@ -114,6 +114,33 @@ public sealed record MemoryOptions /// public bool DeferAccessTracking { get; init; } + /// + /// How much a fact's confidence moves when the world corroborates or contradicts it (S2). + /// + /// + /// + /// Today confidence is set once by extraction and never earns or loses anything: a fact stated + /// five times and one stated in passing carry whatever number the extractor happened to report. + /// With α > 0 a re-asserted triple gains α, and a fact superseded by a contradiction loses 2α. + /// + /// + /// Asymmetric on purpose. Being contradicted is stronger evidence against a fact than one + /// more restatement is for it — a claim that is simply repeated may just be a habit of phrasing, + /// while a claim the world has replaced is one the world stopped believing. + /// + /// + /// Clamped to [0,1] at both ends. Confidence is read by ranking, dedup and decay, and a value + /// outside that range would propagate into computations where it means nothing. + /// + /// + /// 0 (off) by default, and it is the off switch in Cypher too: at α = 0 the upsert's + /// confidence assignment is byte-for-byte what it was, so no sealed measurement moves. A sensible + /// starting value is 0.02–0.05 — large enough to separate a well-attested fact over a + /// conversation, small enough that a single restatement does not overwhelm what extraction judged. + /// + /// + public double ConfidenceReinforcementAlpha { get; init; } + // NOTE: extraction at the Core layer is explicit (call ExtractAndPersistAsync / // ExtractFromSessionAsync). Automatic extraction on message persist is an adapter concern, configured // by AgentFrameworkOptions.AutoExtractOnPersist. The former EnableAutoExtraction flag here was read diff --git a/src/AgentMemory.Neo4j/Queries/FactQueries.cs b/src/AgentMemory.Neo4j/Queries/FactQueries.cs index c949dec58..6f635e322 100644 --- a/src/AgentMemory.Neo4j/Queries/FactQueries.cs +++ b/src/AgentMemory.Neo4j/Queries/FactQueries.cs @@ -118,7 +118,15 @@ ON CREATE SET f.mention_count = 1 ON MATCH SET f.category = $category, - f.confidence = $confidence, + // S2 corroboration. A re-asserted triple is the world saying it again, so confidence + // EARNS rather than being replaced by whatever the latest extraction happened to + // report. Gated by the parameter itself -- at alpha 0 this is the original assignment, + // byte for byte, so nothing recorded before S2 moves. + f.confidence = CASE WHEN $reinforceAlpha > 0 + THEN CASE WHEN coalesce(f.confidence, $confidence) + $reinforceAlpha > 1.0 + THEN 1.0 + ELSE coalesce(f.confidence, $confidence) + $reinforceAlpha END + ELSE $confidence END, f.valid_from = CASE WHEN $validFrom IS NOT NULL THEN datetime($validFrom) ELSE f.valid_from END, f.valid_until = CASE WHEN $validUntil IS NOT NULL THEN datetime($validUntil) ELSE f.valid_until END, f.source_message_ids = $sourceMessageIds, @@ -159,7 +167,14 @@ ON CREATE SET f.mention_count = 1 ON MATCH SET f.category = item.category, - f.confidence = item.confidence, + // S2 corroboration, identical to the single-item path. A counter that depended on + // which write path ran would make reinforcement a property of batching rather than of + // the conversation. + f.confidence = CASE WHEN $reinforceAlpha > 0 + THEN CASE WHEN coalesce(f.confidence, item.confidence) + $reinforceAlpha > 1.0 + THEN 1.0 + ELSE coalesce(f.confidence, item.confidence) + $reinforceAlpha END + ELSE item.confidence END, f.valid_from = CASE WHEN item.valid_from IS NOT NULL THEN datetime(item.valid_from) ELSE f.valid_from END, f.valid_until = CASE WHEN item.valid_until IS NOT NULL THEN datetime(item.valid_until) ELSE f.valid_until END, f.source_message_ids = item.source_message_ids, @@ -442,7 +457,16 @@ WHERE true" + winnerOwner + @" AND coalesce(loser.owner_id, '*') = coalesce(winner.owner_id, '*') AND loser <> winner SET loser.invalidated_at = coalesce(loser.invalidated_at, datetime($now)), - loser.valid_until = coalesce(loser.valid_until, datetime($now)) + loser.valid_until = coalesce(loser.valid_until, datetime($now)), + // S2 contradiction. Twice the corroboration step, and downward: being contradicted is + // stronger evidence against a fact than one more restatement is for it. Floored at 0 + // rather than allowed negative -- confidence is a [0,1] quantity every ranking and + // dedup computation reads, and a negative would propagate somewhere it means nothing. + loser.confidence = CASE WHEN $reinforceAlpha > 0 + THEN CASE WHEN coalesce(loser.confidence, 0.0) - (2 * $reinforceAlpha) < 0.0 + THEN 0.0 + ELSE coalesce(loser.confidence, 0.0) - (2 * $reinforceAlpha) END + ELSE loser.confidence END MERGE (loser)-[:SUPERSEDED_BY]->(winner) RETURN count(loser) > 0 AS superseded"; } diff --git a/src/AgentMemory.Neo4j/Queries/FusedPersistenceQueries.cs b/src/AgentMemory.Neo4j/Queries/FusedPersistenceQueries.cs index de78a519f..f937ff3e7 100644 --- a/src/AgentMemory.Neo4j/Queries/FusedPersistenceQueries.cs +++ b/src/AgentMemory.Neo4j/Queries/FusedPersistenceQueries.cs @@ -71,7 +71,14 @@ ON CREATE SET f.metadata = item.metadata ON MATCH SET f.category = item.category, - f.confidence = item.confidence, + // S2 corroboration, identical to the other two write paths. Reinforcement that + // depended on which path ran would be a property of batching rather than of the + // conversation. + f.confidence = CASE WHEN $reinforceAlpha > 0 + THEN CASE WHEN coalesce(f.confidence, item.confidence) + $reinforceAlpha > 1.0 + THEN 1.0 + ELSE coalesce(f.confidence, item.confidence) + $reinforceAlpha END + ELSE item.confidence END, f.valid_from = CASE WHEN item.valid_from IS NOT NULL THEN datetime(item.valid_from) ELSE f.valid_from END, f.valid_until = CASE WHEN item.valid_until IS NOT NULL THEN datetime(item.valid_until) ELSE f.valid_until END, f.mention_count = coalesce(f.mention_count, 1) + 1, diff --git a/src/AgentMemory.Neo4j/Repositories/Neo4jFactRepository.Fused.cs b/src/AgentMemory.Neo4j/Repositories/Neo4jFactRepository.Fused.cs index b273e52b7..63bbe0c23 100644 --- a/src/AgentMemory.Neo4j/Repositories/Neo4jFactRepository.Fused.cs +++ b/src/AgentMemory.Neo4j/Repositories/Neo4jFactRepository.Fused.cs @@ -68,7 +68,9 @@ public async Task> UpsertFusedBatchAsync( return await _tx.WriteAsync(async runner => { - var cursor = await runner.RunAsync(FusedPersistenceQueries.FactUpsertBatch, new { items }) + var cursor = await runner.RunAsync( + FusedPersistenceQueries.FactUpsertBatch, + new { items, reinforceAlpha = _reinforceAlpha }) .ConfigureAwait(false); var records = await cursor.ToListAsync().ConfigureAwait(false); var embeddingByTriple = deduped.ToDictionary( diff --git a/src/AgentMemory.Neo4j/Repositories/Neo4jFactRepository.cs b/src/AgentMemory.Neo4j/Repositories/Neo4jFactRepository.cs index 7c5e193f9..1f8dbc54a 100644 --- a/src/AgentMemory.Neo4j/Repositories/Neo4jFactRepository.cs +++ b/src/AgentMemory.Neo4j/Repositories/Neo4jFactRepository.cs @@ -24,6 +24,7 @@ internal sealed partial class Neo4jFactRepository : IFactRepository, IUpsertPers private readonly INeo4jTransactionRunner _tx; private readonly bool _rescueShortOwnerResults; + private readonly double _reinforceAlpha; private readonly ILogger _logger; private readonly MemoryRankingOptions _ranking; private readonly MemoryDecayOptions _decay; @@ -38,6 +39,7 @@ public Neo4jFactRepository( IOptions? memoryOptions = null) { _rescueShortOwnerResults = memoryOptions?.Value.RescueShortOwnerResults ?? false; + _reinforceAlpha = memoryOptions?.Value.ConfidenceReinforcementAlpha ?? 0.0; _tx = tx; _logger = logger; _ranking = ranking?.Value ?? MemoryRankingOptions.Default; @@ -81,6 +83,7 @@ public async Task UpsertAsync(Fact fact, CancellationToken cancellationTok ["ownerKey"] = fact.OwnerId ?? OwnerKeyShared, ["category"] = fact.Category, ["confidence"] = fact.Confidence, + ["reinforceAlpha"] = _reinforceAlpha, ["validFrom"] = (object?)(fact.ValidFrom?.ToString("O")), ["validUntil"] = (object?)(fact.ValidUntil?.ToString("O")), ["sourceMessageIds"] = fact.SourceMessageIds.ToList(), @@ -177,7 +180,9 @@ public async Task> UpsertBatchAsync(IReadOnlyList fact return await _tx.WriteAsync(async runner => { - var cursor = await runner.RunAsync(FactQueries.UpsertBatch, new { items }).ConfigureAwait(false); + var cursor = await runner.RunAsync( + FactQueries.UpsertBatch, + new { items, reinforceAlpha = _reinforceAlpha }).ConfigureAwait(false); var records = await cursor.ToListAsync().ConfigureAwait(false); // The MERGE returns the SURVIVING node per triple; for a pre-existing triple that id is the @@ -690,7 +695,7 @@ public async Task SupersedeAsync(string loserFactId, string winnerFactId, return await _tx.WriteAsync(async runner => { - var parameters = new Dictionary { ["loserId"] = loserFactId, ["winnerId"] = winnerFactId, ["now"] = now }; + var parameters = new Dictionary { ["loserId"] = loserFactId, ["winnerId"] = winnerFactId, ["now"] = now, ["reinforceAlpha"] = _reinforceAlpha }; if (hasOwner) parameters["ownerId"] = scope!.OwnerId; var cursor = await runner.RunAsync(cypher, parameters).ConfigureAwait(false); var records = await cursor.ToListAsync().ConfigureAwait(false); diff --git a/tests/AgentMemory.Tests.Integration/Repositories/ConfidenceReinforcementIntegrationTests.cs b/tests/AgentMemory.Tests.Integration/Repositories/ConfidenceReinforcementIntegrationTests.cs new file mode 100644 index 000000000..7f043d633 --- /dev/null +++ b/tests/AgentMemory.Tests.Integration/Repositories/ConfidenceReinforcementIntegrationTests.cs @@ -0,0 +1,166 @@ +using FluentAssertions; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using AgentMemory.Abstractions.Domain; +using AgentMemory.Abstractions.Options; +using AgentMemory.Neo4j.Repositories; +using AgentMemory.Tests.Integration.Fixtures; + +namespace AgentMemory.Tests.Integration.Repositories; + +/// +/// Confidence that earns and loses (S2), verified where the arithmetic actually runs. +/// +/// +/// +/// Confidence was set once by extraction and never moved: a fact stated five times and one stated in +/// passing carried whatever number the extractor happened to report. With α > 0 a re-asserted +/// triple gains α and a superseded one loses 2α. +/// +/// +/// Asymmetric deliberately. Being contradicted is stronger evidence against a fact than one +/// more restatement is for it — a repeated claim may just be a habit of phrasing, while a replaced +/// one is a claim the world stopped believing. +/// +/// +/// The clamping is the part that has to be checked in the database. Confidence is read by ranking, +/// dedup and decay; a value that escaped [0,1] would propagate into computations where it means +/// nothing, and Cypher arithmetic is where such an escape would happen. +/// +/// +[Collection("Neo4j Integration")] +[Trait("Category", "Integration")] +public class ConfidenceReinforcementIntegrationTests : IAsyncLifetime +{ + private readonly Neo4jIntegrationFixture _fixture; + + private const double Alpha = 0.05; + + public ConfidenceReinforcementIntegrationTests(Neo4jIntegrationFixture fixture) => _fixture = fixture; + + public Task InitializeAsync() => _fixture.CleanDatabaseAsync(); + public Task DisposeAsync() => Task.CompletedTask; + + private Neo4jFactRepository Repository(double alpha) => + new(_fixture.TransactionRunner, + NullLogger.Instance, + memoryOptions: Options.Create(new MemoryOptions { ConfidenceReinforcementAlpha = alpha })); + + private static Fact NewFact(string @object, double confidence) => new() + { + FactId = $"fact-{Guid.NewGuid():N}", + Subject = "user", + Predicate = "lives in", + Object = @object, + Confidence = confidence, + OwnerId = "alice", + CreatedAtUtc = DateTimeOffset.UtcNow, + }; + + [Fact] + public async Task ReAssertionRaisesConfidence() + { + // The world said it again, so the claim earns rather than being overwritten by whatever the + // latest extraction reported. + var repo = Repository(Alpha); + var stored = await repo.UpsertAsync(NewFact("Zurich", 0.50)); + await repo.UpsertAsync(NewFact("Zurich", 0.50)); + + var reinforced = await repo.GetByIdAsync(stored.FactId); + + reinforced!.Confidence.Should().BeApproximately(0.55, 1e-9); + } + + [Fact] + public async Task RepeatedReAssertionAccumulatesButNeverExceedsOne() + { + // The upper clamp, exercised past the boundary rather than near it. Confidence is a [0,1] + // quantity every ranking and dedup computation reads. + var repo = Repository(0.2); + var stored = await repo.UpsertAsync(NewFact("Zurich", 0.90)); + for (var i = 0; i < 5; i++) + await repo.UpsertAsync(NewFact("Zurich", 0.90)); + + var reinforced = await repo.GetByIdAsync(stored.FactId); + + reinforced!.Confidence.Should().Be(1.0); + } + + [Fact] + public async Task BeingSupersededCostsTwiceWhatCorroborationEarns() + { + // The asymmetry, measured: 0.60 - 2*0.05 = 0.50. Being contradicted is stronger evidence + // against a fact than one more restatement is for it. + var repo = Repository(Alpha); + var loser = await repo.UpsertAsync(NewFact("Basel", 0.60)); + var winner = await repo.UpsertAsync(NewFact("Zurich", 0.90)); + + await repo.SupersedeAsync(loser.FactId, winner.FactId, MemoryScope.For("alice", includeShared: false)); + + var demoted = await repo.GetByIdAsync(loser.FactId); + demoted!.Confidence.Should().BeApproximately(0.50, 1e-9); + } + + [Fact] + public async Task ConfidenceNeverGoesNegative() + { + // The lower clamp. A negative confidence would be read by ranking and decay as a number, not + // as an error, and would propagate somewhere it means nothing. + var repo = Repository(0.4); + var loser = await repo.UpsertAsync(NewFact("Basel", 0.10)); + var winner = await repo.UpsertAsync(NewFact("Zurich", 0.90)); + + await repo.SupersedeAsync(loser.FactId, winner.FactId, MemoryScope.For("alice", includeShared: false)); + + (await repo.GetByIdAsync(loser.FactId))!.Confidence.Should().Be(0.0); + } + + [Fact] + public async Task TheWinnerOfASupersessionIsUntouched() + { + // Only the loser is demoted. Moving both would make every contradiction erode the graph. + var repo = Repository(Alpha); + var loser = await repo.UpsertAsync(NewFact("Basel", 0.60)); + var winner = await repo.UpsertAsync(NewFact("Zurich", 0.90)); + + await repo.SupersedeAsync(loser.FactId, winner.FactId, MemoryScope.For("alice", includeShared: false)); + + (await repo.GetByIdAsync(winner.FactId))!.Confidence.Should().BeApproximately(0.90, 1e-9); + } + + [Fact] + public async Task AtAlphaZeroTheBehaviourIsExactlyWhatItWas() + { + // The byte-identical guarantee, and the reason the gate lives in Cypher rather than in C#: at + // alpha 0 the assignment is the original one, so every sealed measurement stands. + var repo = Repository(0.0); + var stored = await repo.UpsertAsync(NewFact("Zurich", 0.50)); + await repo.UpsertAsync(NewFact("Zurich", 0.70)); + + var after = await repo.GetByIdAsync(stored.FactId); + + after!.Confidence.Should().BeApproximately(0.70, 1e-9, + "with reinforcement off, the latest assertion's confidence replaces the stored one"); + } + + [Fact] + public async Task ADifferentFactDoesNotReinforce() + { + // Corroboration is per triple. If any write about the same subject reinforced, one chatty + // subject would drag every fact about it upward. + var repo = Repository(Alpha); + var lives = await repo.UpsertAsync(NewFact("Zurich", 0.50)); + await repo.UpsertAsync(new Fact + { + FactId = $"fact-{Guid.NewGuid():N}", + Subject = "user", + Predicate = "works at", + Object = "Acme", + Confidence = 0.5, + OwnerId = "alice", + CreatedAtUtc = DateTimeOffset.UtcNow, + }); + + (await repo.GetByIdAsync(lives.FactId))!.Confidence.Should().BeApproximately(0.50, 1e-9); + } +} diff --git a/tests/AgentMemory.Tests.Unit/Queries/CypherQuerySnapshot.snap b/tests/AgentMemory.Tests.Unit/Queries/CypherQuerySnapshot.snap index 8002bb67c..9e6504bed 100644 --- a/tests/AgentMemory.Tests.Unit/Queries/CypherQuerySnapshot.snap +++ b/tests/AgentMemory.Tests.Unit/Queries/CypherQuerySnapshot.snap @@ -348,7 +348,15 @@ MERGE (f:Fact {subject_key: $subjectKey, predicate_key: $predicateKey, object_ke f.mention_count = 1 ON MATCH SET f.category = $category, - f.confidence = $confidence, + // S2 corroboration. A re-asserted triple is the world saying it again, so confidence + // EARNS rather than being replaced by whatever the latest extraction happened to + // report. Gated by the parameter itself -- at alpha 0 this is the original assignment, + // byte for byte, so nothing recorded before S2 moves. + f.confidence = CASE WHEN $reinforceAlpha > 0 + THEN CASE WHEN coalesce(f.confidence, $confidence) + $reinforceAlpha > 1.0 + THEN 1.0 + ELSE coalesce(f.confidence, $confidence) + $reinforceAlpha END + ELSE $confidence END, f.valid_from = CASE WHEN $validFrom IS NOT NULL THEN datetime($validFrom) ELSE f.valid_from END, f.valid_until = CASE WHEN $validUntil IS NOT NULL THEN datetime($validUntil) ELSE f.valid_until END, f.source_message_ids = $sourceMessageIds, @@ -377,7 +385,14 @@ UNWIND $items AS item f.mention_count = 1 ON MATCH SET f.category = item.category, - f.confidence = item.confidence, + // S2 corroboration, identical to the single-item path. A counter that depended on + // which write path ran would make reinforcement a property of batching rather than of + // the conversation. + f.confidence = CASE WHEN $reinforceAlpha > 0 + THEN CASE WHEN coalesce(f.confidence, item.confidence) + $reinforceAlpha > 1.0 + THEN 1.0 + ELSE coalesce(f.confidence, item.confidence) + $reinforceAlpha END + ELSE item.confidence END, f.valid_from = CASE WHEN item.valid_from IS NOT NULL THEN datetime(item.valid_from) ELSE f.valid_from END, f.valid_until = CASE WHEN item.valid_until IS NOT NULL THEN datetime(item.valid_until) ELSE f.valid_until END, f.source_message_ids = item.source_message_ids, @@ -451,7 +466,14 @@ UNWIND $items AS item f.metadata = item.metadata ON MATCH SET f.category = item.category, - f.confidence = item.confidence, + // S2 corroboration, identical to the other two write paths. Reinforcement that + // depended on which path ran would be a property of batching rather than of the + // conversation. + f.confidence = CASE WHEN $reinforceAlpha > 0 + THEN CASE WHEN coalesce(f.confidence, item.confidence) + $reinforceAlpha > 1.0 + THEN 1.0 + ELSE coalesce(f.confidence, item.confidence) + $reinforceAlpha END + ELSE item.confidence END, f.valid_from = CASE WHEN item.valid_from IS NOT NULL THEN datetime(item.valid_from) ELSE f.valid_from END, f.valid_until = CASE WHEN item.valid_until IS NOT NULL THEN datetime(item.valid_until) ELSE f.valid_until END, f.mention_count = coalesce(f.mention_count, 1) + 1, From bbe7749a498cdbb04b1e7a0c7ccbc1b74a9500d6 Mon Sep 17 00:00:00 2001 From: Jose Luis Latorre Millas Date: Wed, 12 Aug 2026 21:48:46 +0200 Subject: [PATCH 19/31] Wire the second supersede caller for S2 The supersede query has two callers, not one: the write-time path added in 9.1 and the offline conflict-resolution hygiene pass. Only the first passed reinforceAlpha, so ResolveFactContradictionsAsync failed with "Expected parameter(s): reinforceAlpha" the moment it ran. Two things worth recording. First, a contradiction resolved by the hygiene pass is the same event as one resolved at write time, so it should move confidence the same way -- the omission was a wiring gap, not a design choice. Second, only the integration suite could catch it: a missing Cypher parameter is rejected by the server, so no unit test with a substituted transaction runner would ever see it. This is the third time this session that "wire it up everywhere" has been the actual work rather than the feature: TemporalValidityMode inert on one extractor rung, RescueShortOwnerResults on one of four vector paths, and now reinforcement on one of two supersede callers. 4226 unit + 345 non-NAMS integration green. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01PgDgctPpbTziBNE2RT8VdE --- .../Services/Neo4jConflictDetectionService.cs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/AgentMemory.Neo4j/Services/Neo4jConflictDetectionService.cs b/src/AgentMemory.Neo4j/Services/Neo4jConflictDetectionService.cs index 09ee1dfe9..aa9db533d 100644 --- a/src/AgentMemory.Neo4j/Services/Neo4jConflictDetectionService.cs +++ b/src/AgentMemory.Neo4j/Services/Neo4jConflictDetectionService.cs @@ -1,9 +1,11 @@ using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using AgentMemory.Abstractions.Services; using AgentMemory.Neo4j.Infrastructure; using AgentMemory.Neo4j.Queries; using Neo4j.Driver; +using AgentMemory.Abstractions.Options; namespace AgentMemory.Neo4j.Services; /// @@ -19,13 +21,21 @@ internal sealed class Neo4jConflictDetectionService : IConflictDetectionService public Neo4jConflictDetectionService( INeo4jTransactionRunner tx, IClock clock, - ILogger logger) + ILogger logger, + IOptions? memoryOptions = null) { _tx = tx ?? throw new ArgumentNullException(nameof(tx)); _clock = clock ?? throw new ArgumentNullException(nameof(clock)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + // S2. This is the SECOND caller of the supersede query -- the offline hygiene pass, next to + // the write-time one -- and a contradiction resolved here is the same event either way. An + // integration test caught the omission; the unit tests could not, because the missing + // parameter is only rejected by the server. + _reinforceAlpha = memoryOptions?.Value.ConfidenceReinforcementAlpha ?? 0.0; } + private readonly double _reinforceAlpha; + /// public async Task DetectConflictsAsync( ConflictDetectionOptions? options = null, @@ -96,6 +106,7 @@ public async Task ResolveFactContradictionsAsync( ["loserId"] = loser.FactId, ["winnerId"] = winner.FactId, ["now"] = now, + ["reinforceAlpha"] = _reinforceAlpha, }; if (hasOwner) parameters["ownerId"] = conflict.OwnerId; var cursor = await runner.RunAsync(cypher, parameters).ConfigureAwait(false); From f2050b13a0027721c1cb92490d665d80281a1736 Mon Sep 17 00:00:00 2001 From: Jose Luis Latorre Millas Date: Wed, 12 Aug 2026 21:59:39 +0200 Subject: [PATCH 20/31] Make bitemporal recall reachable from a natural-language turn (R4) RecallAsOfAsync(validAsOf, systemAsOf) has existed since the bitemporal work and nothing in an ordinary conversation could reach it: "what did I think back in March?" recalled against now, exactly like every other question. Phase 1 shipped a capability nothing could ask for. TemporalQueryParser resolves the instant a question asks about -- deterministic regex, no model call, resolving against IClock rather than the wall clock. MemoryService.RecallAsync routes a resolved query to RecallAsOfCoreAsync with BOTH clocks set: "what did I think then" means what was true then as known then, and moving only the valid clock would answer with today's corrections applied to the past. Precision over recall, deliberately. The failure modes are asymmetric: a missed expression costs nothing (the turn recalls against now, today's behaviour), while a false positive silently narrows recall to a window the user never asked about and returns an answer that looks entirely ordinary. So "in March" resolves and a bare "March" -- a surname -- does not; "last week" resolves and "the last item" does not; "in 2024" resolves and "in 2024 units of stock" does not. Off by default, since it changes which memories a temporal question sees. Wiring is tested at the seam, not just the parser: 3 of the 8 routing tests fail when the branch is disabled, verified by disabling it. A parser nothing calls passes all of its own tests while the bitemporal path stays exactly as unreachable as before -- the shape this track has now hit four times. The memory://context MCP resource is the one read surface that does not honour the option; it composes the assembler directly and the MCP server references only Abstractions by design. Documented on the option rather than papered over. 4,263 unit tests green (+37). Release build 0 warnings. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01PgDgctPpbTziBNE2RT8VdE --- .../Options/MemoryOptions.cs | 33 ++++ .../Memory/TemporalQueryParser.cs | 161 ++++++++++++++++ .../Services/MemoryService.cs | 17 ++ .../Memory/TemporalQueryParserTests.cs | 176 ++++++++++++++++++ .../MemoryServiceTemporalRoutingTests.cs | 158 ++++++++++++++++ 5 files changed, 545 insertions(+) create mode 100644 src/AgentMemory.Core/Memory/TemporalQueryParser.cs create mode 100644 tests/AgentMemory.Tests.Unit/Memory/TemporalQueryParserTests.cs create mode 100644 tests/AgentMemory.Tests.Unit/Services/MemoryServiceTemporalRoutingTests.cs diff --git a/src/AgentMemory.Abstractions/Options/MemoryOptions.cs b/src/AgentMemory.Abstractions/Options/MemoryOptions.cs index 75de6fa63..da84abcc7 100644 --- a/src/AgentMemory.Abstractions/Options/MemoryOptions.cs +++ b/src/AgentMemory.Abstractions/Options/MemoryOptions.cs @@ -141,6 +141,39 @@ public sealed record MemoryOptions /// public double ConfidenceReinforcementAlpha { get; init; } + /// + /// Routes a turn that names a past time to bitemporal recall at that time (R4). + /// + /// + /// + /// RecallAsOfAsync has existed since the bitemporal work, and until now nothing in an + /// ordinary conversation could reach it: "what did I think back in March?" recalled against now, + /// exactly like every other question. Without this, the bitemporal read path is a capability + /// nothing can ask for. + /// + /// + /// Detection is deterministic, needs no model call, and errs heavily toward not matching: + /// a missed expression costs nothing (the turn recalls against now, today's behaviour), while a + /// false positive silently narrows recall to a window the user never asked about and returns an + /// answer that looks entirely ordinary. So "in March" resolves and a bare "March" does not; + /// "last week" resolves and "the last item" does not. + /// + /// + /// Off by default: it changes which memories a temporal question sees. + /// + /// + /// Scope. Honoured by IMemoryService.RecallAsync, and therefore by everything that + /// recalls through it — the MCP memory tools, the Agent Framework and Semantic Kernel adapters. + /// The memory://context/{session_id} MCP resource is the one read surface that does + /// not: it composes IMemoryContextAssembler directly, and the MCP server references only + /// AgentMemory.Abstractions by design. Reaching the resolver from there would mean a project + /// reference to AgentMemory.Core purely for a secondary surface, so the divergence is recorded + /// here rather than papered over — a temporal question asked through that resource recalls against + /// now. + /// + /// + public bool ResolveTemporalQueries { get; init; } + // NOTE: extraction at the Core layer is explicit (call ExtractAndPersistAsync / // ExtractFromSessionAsync). Automatic extraction on message persist is an adapter concern, configured // by AgentFrameworkOptions.AutoExtractOnPersist. The former EnableAutoExtraction flag here was read diff --git a/src/AgentMemory.Core/Memory/TemporalQueryParser.cs b/src/AgentMemory.Core/Memory/TemporalQueryParser.cs new file mode 100644 index 000000000..f05afde21 --- /dev/null +++ b/src/AgentMemory.Core/Memory/TemporalQueryParser.cs @@ -0,0 +1,161 @@ +using System.Globalization; +using System.Text.RegularExpressions; + +namespace AgentMemory.Core.Memory; + +/// +/// Finds the point in time a question is asking about, so bitemporal recall can be reached from a +/// natural-language turn (R4). +/// +/// +/// +/// RecallAsOfAsync(validAsOf, systemAsOf) already exists and nothing in an ordinary conversation +/// can reach it: "what did I think back in March?" recalls against now, exactly like every other +/// question. Without this the bitemporal read path is a capability nothing can ask for. +/// +/// +/// Precision over recall, deliberately and asymmetrically. A missed expression costs nothing — +/// the turn recalls against now, which is today's behaviour. A false positive silently filters memory +/// to a window the user never asked about, and the answer that comes back looks perfectly ordinary. +/// Every pattern here therefore requires an explicit temporal marker rather than inferring one: +/// "in March" is temporal, a bare "March" may be a surname; "last week" is +/// temporal, "the last item" is not. +/// +/// +/// No model call and no culture-dependent parsing: this runs on every turn when enabled, and a +/// resolver that cost a completion would be more expensive than the recall it modifies. +/// +/// +internal static class TemporalQueryParser +{ + private const RegexOptions Options = + RegexOptions.IgnoreCase | RegexOptions.CultureInvariant | RegexOptions.ExplicitCapture; + + private static readonly TimeSpan Timeout = TimeSpan.FromMilliseconds(100); + + /// + /// last|past N unit(s) ago and the bare last week/month/quarter/year. + /// + /// + /// "last" must be followed by a time unit. Alone it is far more often ordinal — "the last item", + /// "last one wins" — and treating those as temporal would rewrite the recall window of ordinary + /// questions. + /// + private static readonly Regex RelativeUnits = new( + @"\b(?:(?\d{1,3})\s+)?(?day|days|week|weeks|month|months|quarter|quarters|year|years)\s+ago\b" + + @"|\blast\s+(?week|month|quarter|year)\b" + + @"|\bpast\s+(?\d{1,3})\s+(?day|days|week|weeks|month|months|year|years)\b", + Options, Timeout); + + /// A month name behind a preposition, optionally with a year. + /// + /// The preposition is required. "March" is a common surname and "may" is a verb in nearly every + /// sentence that contains it; "in May" is unambiguous where "may" is not. + /// + private static readonly Regex MonthName = new( + @"\b(?:in|back\s+in|during|around)\s+(?january|february|march|april|may|june|july|august|" + + @"september|october|november|december)(?:\s+(?(?:19|20)\d{2}))?\b", + Options, Timeout); + + /// A four-digit year behind a preposition. + /// + /// Bounded to 19xx/20xx and requiring a preposition, so "in 2024" resolves and "in 2024 units of + /// stock" — where the number is a quantity — does not accidentally become a date. + /// + private static readonly Regex Year = new( + @"\b(?:in|back\s+in|during)\s+(?(?:19|20)\d{2})\b(?!\s*(?:units?|items?|records?|rows?))", + Options, Timeout); + + private static readonly Regex Yesterday = new(@"\byesterday\b", Options, Timeout); + + /// + /// The instant this question is asking about, or when it asks about now. + /// + /// The user's turn. + /// The reference instant; relative expressions resolve against it. + /// + /// + /// Null is the overwhelmingly common answer and the safe one: it means "recall as the system + /// behaves today". Only an explicit past reference produces an instant. + /// + /// + /// A future reference returns null rather than a future instant. As-of recall reconstructs + /// what was known at a past moment; asking it about next month would return everything, which is + /// indistinguishable from ordinary recall except that it silently ignored the user's question. + /// + /// + internal static DateTimeOffset? Resolve(string? query, DateTimeOffset now) + { + if (string.IsNullOrWhiteSpace(query)) return null; + + var resolved = ResolveCore(query, now); + + // Never ahead of the reference instant. A parse that lands in the future is either a + // misreading or a question about the future, and neither should narrow a recall window. + return resolved is { } instant && instant < now ? instant : null; + } + + private static DateTimeOffset? ResolveCore(string query, DateTimeOffset now) + { + if (Yesterday.IsMatch(query)) return now.AddDays(-1); + + var relative = RelativeUnits.Match(query); + if (relative.Success) + { + var count = relative.Groups["n"].Success + ? int.Parse(relative.Groups["n"].Value, CultureInfo.InvariantCulture) + : relative.Groups["n2"].Success + ? int.Parse(relative.Groups["n2"].Value, CultureInfo.InvariantCulture) + : 1; + + var unit = relative.Groups["unit"].Success ? relative.Groups["unit"].Value + : relative.Groups["lunit"].Success ? relative.Groups["lunit"].Value + : relative.Groups["punit"].Value; + + return unit.ToLowerInvariant().TrimEnd('s') switch + { + "day" => now.AddDays(-count), + "week" => now.AddDays(-7 * count), + "month" => now.AddMonths(-count), + "quarter" => now.AddMonths(-3 * count), + "year" => now.AddYears(-count), + _ => null, + }; + } + + var month = MonthName.Match(query); + if (month.Success) + { + var monthNumber = MonthNumber(month.Groups["month"].Value); + if (monthNumber == 0) return null; + + var year = month.Groups["year"].Success + ? int.Parse(month.Groups["year"].Value, CultureInfo.InvariantCulture) + : now.Month >= monthNumber ? now.Year : now.Year - 1; + + // The END of the named month: "what did I think back in March" means anything known by + // the close of March, not by its first instant -- which would exclude the whole month. + return EndOfMonth(year, monthNumber, now.Offset); + } + + var year4 = Year.Match(query); + if (year4.Success) + { + var value = int.Parse(year4.Groups["year"].Value, CultureInfo.InvariantCulture); + return EndOfMonth(value, 12, now.Offset); + } + + return null; + } + + private static DateTimeOffset EndOfMonth(int year, int month, TimeSpan offset) => + new DateTimeOffset(year, month, DateTime.DaysInMonth(year, month), 23, 59, 59, offset); + + private static int MonthNumber(string name) => name.ToLowerInvariant() switch + { + "january" => 1, "february" => 2, "march" => 3, "april" => 4, + "may" => 5, "june" => 6, "july" => 7, "august" => 8, + "september" => 9, "october" => 10, "november" => 11, "december" => 12, + _ => 0, + }; +} diff --git a/src/AgentMemory.Core/Services/MemoryService.cs b/src/AgentMemory.Core/Services/MemoryService.cs index 72f41c874..b33207a88 100644 --- a/src/AgentMemory.Core/Services/MemoryService.cs +++ b/src/AgentMemory.Core/Services/MemoryService.cs @@ -5,6 +5,7 @@ using AgentMemory.Abstractions.Options; using AgentMemory.Abstractions.Repositories; using AgentMemory.Abstractions.Services; +using AgentMemory.Core.Memory; namespace AgentMemory.Core.Services; @@ -86,6 +87,22 @@ public async Task RecallAsync( // outside; both can be present without colliding. using var activity = AgentMemoryDiagnostics.Source.StartActivity("memory.recall.total"); + // R4. A turn that names a past time is asking a bitemporal question, and until now no + // conversational turn could reach RecallAsOfAsync at all. Resolution is deterministic and + // biased hard toward returning null -- see TemporalQueryParser -- so the ordinary turn takes + // exactly the path it always did. + if (_options.ResolveTemporalQueries + && TemporalQueryParser.Resolve(request.Query, _clock.UtcNow) is { } asOf) + { + activity?.SetTag("memory.recall.resolved_as_of", asOf.ToString("O")); + _logger.LogDebug( + "Query names a past time ({AsOf}); recalling bitemporally instead of against now.", asOf); + // Both clocks: the question is "what did I think then", which is what was true then AS + // known then. Passing only the valid clock would answer with today's corrections applied + // to the past -- a different question, and a subtly misleading one. + return await RecallAsOfCoreAsync(request, asOf, asOf, cancellationToken).ConfigureAwait(false); + } + var context = await _assembler.AssembleContextAsync(request, cancellationToken).ConfigureAwait(false); // Update access timestamps for recalled long-term memories. Awaited by default so failures and diff --git a/tests/AgentMemory.Tests.Unit/Memory/TemporalQueryParserTests.cs b/tests/AgentMemory.Tests.Unit/Memory/TemporalQueryParserTests.cs new file mode 100644 index 000000000..e05129266 --- /dev/null +++ b/tests/AgentMemory.Tests.Unit/Memory/TemporalQueryParserTests.cs @@ -0,0 +1,176 @@ +using AgentMemory.Core.Memory; +using FluentAssertions; +using Xunit; + +namespace AgentMemory.Tests.Unit.Memory; + +/// +/// Finding the point in time a question asks about (R4) — and, far more importantly, not finding one. +/// +/// +/// +/// RecallAsOfAsync has existed since the bitemporal work and nothing in an ordinary conversation +/// could reach it: "what did I think back in March?" recalled against now, exactly like every other +/// question. +/// +/// +/// The failure modes are wildly asymmetric, so the tests are too. A missed expression costs +/// nothing — the turn recalls against now, which is today's behaviour. A false positive silently +/// narrows recall to a window the user never asked about, and the answer that comes back looks +/// completely ordinary. Most of what follows is therefore about the phrases that must NOT parse. +/// +/// +public sealed class TemporalQueryParserTests +{ + private static readonly DateTimeOffset Now = new(2026, 8, 12, 12, 0, 0, TimeSpan.Zero); + + private static DateTimeOffset? Resolve(string query) => TemporalQueryParser.Resolve(query, Now); + + // ── what must NOT parse ─────────────────────────────────────────────── + + [Theory] + [InlineData("what was the last item I bought")] + [InlineData("give me the last one")] + [InlineData("who spoke last")] + [InlineData("the last thing you said")] + public void OrdinalLastIsNotTemporal(string query) + { + // "last" alone is far more often ordinal than temporal. Treating these as dates would rewrite + // the recall window of perfectly ordinary questions, and the answer would look normal. + Resolve(query).Should().BeNull(); + } + + [Theory] + [InlineData("what did March say about the contract")] + [InlineData("is April coming to the meeting")] + [InlineData("that may be the problem")] + [InlineData("May I ask about the budget")] + public void ABareMonthWordIsNotADate(string query) + { + // March and April are common names; "may" is a verb in nearly every sentence containing it. + // The preposition is what separates a date from a word that looks like one. + Resolve(query).Should().BeNull(); + } + + [Theory] + [InlineData("we shipped in 2024 units of stock")] + [InlineData("the order was for 2019 items")] + public void ANumberThatIsAQuantityIsNotAYear(string query) + { + Resolve(query).Should().BeNull(); + } + + [Theory] + [InlineData("what do I like")] + [InlineData("who is my manager")] + [InlineData("")] + [InlineData(" ")] + public void AnOrdinaryQuestionResolvesToNow(string query) + { + // Null means "recall as the system behaves today" -- the overwhelmingly common answer and the + // safe one. + Resolve(query).Should().BeNull(); + } + + [Fact] + public void AFutureDateIsNotAnAsOfInstant() + { + // As-of recall reconstructs what was known at a PAST moment. Pointed at the future it returns + // everything, which is indistinguishable from ordinary recall except that it silently ignored + // the question. + Resolve("what is planned in 2030").Should().BeNull(); + } + + [Fact] + public void FutureTenseIsNotDetected_AndThatIsTheAcceptedLimit() + { + // "what will change in December", asked in January, resolves to LAST December. The parser + // reads dates, not tense, and a bare month behind a preposition is genuinely ambiguous -- it + // most often does mean the most recent one. + // + // Recorded rather than fixed: tense detection is a substantially harder problem, and the cost + // of getting THIS wrong is asymmetric in the tolerable direction. The question is answered + // against last December's memory instead of all of it, which is a narrower answer to a + // question about the future -- not a wrong answer to a question about the past. + TemporalQueryParser.Resolve( + "what will change in December", new DateTimeOffset(2026, 1, 5, 0, 0, 0, TimeSpan.Zero)) + .Should().Be(new DateTimeOffset(2025, 12, 31, 23, 59, 59, TimeSpan.Zero)); + } + + // ── what must parse ─────────────────────────────────────────────────── + + [Fact] + public void Yesterday() => + Resolve("what did I say yesterday").Should().Be(Now.AddDays(-1)); + + [Theory] + [InlineData("what did I think last week", -7)] + [InlineData("what was true 3 days ago", -3)] + [InlineData("in the past 10 days what changed", -10)] + public void RelativeDayExpressions(string query, int days) => + Resolve(query).Should().Be(Now.AddDays(days)); + + [Fact] + public void LastQuarterIsThreeMonths() => + Resolve("what were my priorities last quarter").Should().Be(Now.AddMonths(-3)); + + [Fact] + public void LastYear() => + Resolve("where did I live last year").Should().Be(Now.AddYears(-1)); + + [Fact] + public void AMonthNameBehindAPrepositionResolvesToTheEndOfThatMonth() + { + // The END of the month, not its first instant: "back in March" means anything known by the + // close of March, and an as-of at 1 March would exclude the entire month being asked about. + Resolve("what did I think back in March") + .Should().Be(new DateTimeOffset(2026, 3, 31, 23, 59, 59, TimeSpan.Zero)); + } + + [Fact] + public void AMonthLaterInTheYearResolvesToLastYear() + { + // Asked in August, "in December" cannot mean this year's December -- that has not happened. + Resolve("what was the plan in December") + .Should().Be(new DateTimeOffset(2025, 12, 31, 23, 59, 59, TimeSpan.Zero)); + } + + [Fact] + public void AnExplicitMonthAndYearBeatsTheInference() => + Resolve("what did we decide in March 2024") + .Should().Be(new DateTimeOffset(2024, 3, 31, 23, 59, 59, TimeSpan.Zero)); + + [Fact] + public void AYearBehindAPrepositionResolvesToItsEnd() => + Resolve("what was true in 2023") + .Should().Be(new DateTimeOffset(2023, 12, 31, 23, 59, 59, TimeSpan.Zero)); + + // ── shape ───────────────────────────────────────────────────────────── + + [Fact] + public void TheReferenceInstantsOffsetIsPreserved() + { + // A host in a non-UTC offset must not have its temporal questions silently shifted by hours. + var offset = TimeSpan.FromHours(2); + var now = new DateTimeOffset(2026, 8, 12, 12, 0, 0, offset); + + TemporalQueryParser.Resolve("what did we decide in March 2024", now)! + .Value.Offset.Should().Be(offset); + } + + [Fact] + public void CasingDoesNotMatter() => + Resolve("What Did I Think Back In March").Should().NotBeNull(); + + [Fact] + public void ALongQueryDoesNotHangTheParser() + { + // Every pattern carries a 100 ms timeout: these run on the recall path, and a pathological + // input must degrade to "no temporal reference" rather than stall a turn. + var haystack = string.Join(' ', Enumerable.Repeat("last night we discussed many things", 500)); + + var act = () => Resolve(haystack); + + act.Should().NotThrow(); + } +} diff --git a/tests/AgentMemory.Tests.Unit/Services/MemoryServiceTemporalRoutingTests.cs b/tests/AgentMemory.Tests.Unit/Services/MemoryServiceTemporalRoutingTests.cs new file mode 100644 index 000000000..d1496aa02 --- /dev/null +++ b/tests/AgentMemory.Tests.Unit/Services/MemoryServiceTemporalRoutingTests.cs @@ -0,0 +1,158 @@ +using FluentAssertions; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using AgentMemory.Abstractions.Domain; +using AgentMemory.Abstractions.Options; +using AgentMemory.Abstractions.Repositories; +using AgentMemory.Abstractions.Services; +using AgentMemory.Core.Services; +using NSubstitute; +using Xunit; + +namespace AgentMemory.Tests.Unit.Services; + +/// +/// The wiring half of R4: a turn that names a past time must actually reach bitemporal recall. +/// +/// +/// +/// has its own tests and they prove only that +/// a string becomes an instant. That is the half that cannot fail quietly. A parser nothing calls +/// passes every one of its own tests while RecallAsOfAsync stays exactly as unreachable as it was +/// before — which is the shape this track has now hit four separate times. +/// +/// +/// So these tests assert on the seam: which assembler method was invoked, and what the recall reported +/// about the clocks it used. +/// +/// +public sealed class MemoryServiceTemporalRoutingTests +{ + private readonly IMemoryContextAssembler _assembler = Substitute.For(); + private readonly IClock _clock = Substitute.For(); + + private static readonly DateTimeOffset Now = new(2026, 8, 12, 12, 0, 0, TimeSpan.Zero); + + public MemoryServiceTemporalRoutingTests() + { + _clock.UtcNow.Returns(Now); + + var empty = new MemoryContext { SessionId = "s-1", AssembledAtUtc = Now }; + _assembler.AssembleContextAsync(Arg.Any(), Arg.Any()) + .Returns(Task.FromResult(empty)); + _assembler.AssembleContextAsOfAsync( + Arg.Any(), Arg.Any(), Arg.Any(), + Arg.Any()) + .Returns(Task.FromResult(empty)); + } + + private MemoryService CreateSut(bool resolveTemporalQueries) => + new(Substitute.For(), + _assembler, + Substitute.For(), + Substitute.For(), + Substitute.For(), + Substitute.For(), + Substitute.For(), + Options.Create(new MemoryOptions { ResolveTemporalQueries = resolveTemporalQueries }), + _clock, + Substitute.For(), + NullLogger.Instance); + + private Task RecallAsync(string query, bool enabled = true) => + CreateSut(enabled).RecallAsync(new RecallRequest { SessionId = "s-1", Query = query }); + + [Fact] + public async Task ATemporalQueryReachesBitemporalRecall() + { + // THE test. Everything else in R4 is arrangement around this one call actually being made. + var result = await RecallAsync("what did I think back in March"); + + await _assembler.Received(1).AssembleContextAsOfAsync( + Arg.Any(), Arg.Any(), Arg.Any(), + Arg.Any()); + await _assembler.DidNotReceive().AssembleContextAsync( + Arg.Any(), Arg.Any()); + + result.Metadata["validAsOf"].Should() + .Be(new DateTimeOffset(2026, 3, 31, 23, 59, 59, TimeSpan.Zero)); + } + + [Fact] + public async Task BothClocksMoveTogether() + { + // "What did I think in March" asks what was true then AS KNOWN THEN. Moving only the valid + // clock would answer with today's corrections applied to the past -- a different question, and + // a subtly misleading one, because the answer would look like a faithful reconstruction. + await RecallAsync("what did I think back in March"); + + await _assembler.Received(1).AssembleContextAsOfAsync( + Arg.Any(), + Arg.Is(v => v == new DateTimeOffset(2026, 3, 31, 23, 59, 59, TimeSpan.Zero)), + Arg.Is(s => s == new DateTimeOffset(2026, 3, 31, 23, 59, 59, TimeSpan.Zero)), + Arg.Any()); + } + + [Fact] + public async Task AnOrdinaryQueryTakesTheOrdinaryPath() + { + // The common case, and the one that must not regress: no temporal marker means the live path, + // byte for byte as before. + await RecallAsync("what is my deploy command"); + + await _assembler.Received(1).AssembleContextAsync( + Arg.Any(), Arg.Any()); + await _assembler.DidNotReceive().AssembleContextAsOfAsync( + Arg.Any(), Arg.Any(), Arg.Any(), + Arg.Any()); + } + + [Fact] + public async Task TheOptionIsOffByDefault() + { + // It changes which memories a temporal question sees, so it is opt-in -- and every sealed + // measurement in this track was taken with it off. + new MemoryOptions().ResolveTemporalQueries.Should().BeFalse(); + } + + [Fact] + public async Task DisabledMeansTheParserNeverRuns() + { + // The off switch has to be genuinely off, not merely default-off: a temporal query with the + // option disabled must take exactly the path it took before R4 existed. + await RecallAsync("what did I think back in March", enabled: false); + + await _assembler.Received(1).AssembleContextAsync( + Arg.Any(), Arg.Any()); + await _assembler.DidNotReceive().AssembleContextAsOfAsync( + Arg.Any(), Arg.Any(), Arg.Any(), + Arg.Any()); + } + + [Fact] + public async Task TheParserResolvesAgainstTheInjectedClock() + { + // Not DateTimeOffset.UtcNow. "Last week" is meaningless without a reference instant, and a + // parser reading the wall clock while the rest of the service reads IClock would drift in + // exactly the tests written to pin temporal behaviour down. + _clock.UtcNow.Returns(new DateTimeOffset(2020, 6, 15, 0, 0, 0, TimeSpan.Zero)); + + var result = await RecallAsync("what did I think last week"); + + result.Metadata["validAsOf"].Should() + .Be(new DateTimeOffset(2020, 6, 8, 0, 0, 0, TimeSpan.Zero)); + } + + [Theory] + [InlineData("")] + [InlineData(" ")] + public async Task ABlankQueryTakesTheOrdinaryPath(string query) + { + // Recall with a blank query is legal -- it assembles recent context. The temporal branch must + // neither throw on it nor treat "no query" as some resolvable instant. + await RecallAsync(query); + + await _assembler.Received(1).AssembleContextAsync( + Arg.Any(), Arg.Any()); + } +} From 133be903d240e6dd8b85befbf7b9ffdb4efd5ee6 Mon Sep 17 00:00:00 2001 From: Jose Luis Latorre Millas Date: Wed, 12 Aug 2026 22:08:44 +0200 Subject: [PATCH 21/31] Stop paying for extraction on turns that cannot carry a fact (E4) The extraction call is where the money is: the write is one round trip to Neo4j, while extraction is a model call over the rendered transcript and on a corpus build it is essentially the whole bill. An "ok, thanks!" turn pays it for nothing. The gate sits before the completion, not before the write, and that placement is the finding rather than a detail. A write-side gate -- skip persisting a triple already in the store -- is the obvious reading of "novelty gating" and would have silently disabled two features shipped this same week: 12.4's confidence reinforcement, where a re-asserted fact earns alpha on the MERGE, and R7's mention_count, which the salience reranker reads. Corroboration IS the repeated write. Both would have stayed enabled and gone inert. Precision over recall, and the asymmetry is worse here than anywhere else in the system. Declining to gate costs one call. Gating a turn that did carry a fact means the memory is never formed, nothing downstream can recover it, and nothing can even report it missing. So the vocabulary is closed and deliberately excludes "yes", "no" and "sure": each is a complete answer to a question, and the question may have sat in the previous batch. "no problem" therefore does not gate either -- a wasted call is the cheaper mistake by a wide margin, and that trade is pinned by its own test rather than left as an accident. The pre-extracted path is explicitly not gated: the caller has already paid for that completion, so gating there discards work instead of avoiding cost -- the gate's exact inverse. Off by default; transcript bytes are fingerprinted into every measured run. Wiring verified red-before-fix on both dispatch paths, per-type and unified. 4,299 unit tests green (+36). Release build 0 warnings. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01PgDgctPpbTziBNE2RT8VdE --- .../Options/ExtractionOptions.cs | 31 ++++ .../Extraction/ExtractionNoveltyGate.cs | 120 +++++++++++++++ .../Extraction/ExtractionStage.cs | 16 ++ .../Extraction/ExtractionNoveltyGateTests.cs | 144 ++++++++++++++++++ .../ExtractionStageNoveltyGatingTests.cs | 133 ++++++++++++++++ 5 files changed, 444 insertions(+) create mode 100644 src/AgentMemory.Core/Extraction/ExtractionNoveltyGate.cs create mode 100644 tests/AgentMemory.Tests.Unit/Extraction/ExtractionNoveltyGateTests.cs create mode 100644 tests/AgentMemory.Tests.Unit/Extraction/ExtractionStageNoveltyGatingTests.cs diff --git a/src/AgentMemory.Abstractions/Options/ExtractionOptions.cs b/src/AgentMemory.Abstractions/Options/ExtractionOptions.cs index 535dc33a0..bf8576999 100644 --- a/src/AgentMemory.Abstractions/Options/ExtractionOptions.cs +++ b/src/AgentMemory.Abstractions/Options/ExtractionOptions.cs @@ -95,6 +95,36 @@ public sealed class ExtractionOptions /// /// public bool SupersedeReplacedFacts { get; set; } + + /// + /// Skips the extraction call entirely for turns that cannot carry a fact — greetings, thanks, + /// bare acknowledgement (E4). + /// + /// + /// + /// The extraction call is where the money is: the write is one round trip to Neo4j, while + /// extraction is a model call over the rendered transcript and on a corpus build it is + /// essentially the whole bill. An "ok, thanks!" turn pays it for nothing. + /// + /// + /// Deliberately not a write-side gate. Skipping the persistence of a triple already in the + /// store looks like the more obvious saving and would silently disable two features that depend + /// on the duplicate write happening: , + /// where a re-asserted fact earns α, and the mention_count the salience reranker reads. + /// Corroboration is the repeated write. + /// + /// + /// Detection is a closed vocabulary, deterministic, and biased hard toward extracting: a missed + /// skip costs one call, while gating a turn that did carry a fact means the memory is never + /// formed and nothing downstream can recover it. "yes", "no" and "sure" are therefore treated as + /// contentful — each is a complete answer to a question. + /// + /// + /// Off by default: it changes what gets extracted, and prompt-and-transcript bytes are + /// fingerprinted into every measured run. + /// + /// + public bool SkipUninformativeTurns { get; set; } } /// Controls which matching strategies are used for entity resolution. @@ -125,4 +155,5 @@ public sealed class EntityValidationOptions public bool RejectPunctuationOnly { get; set; } = true; /// Reject entities whose names are common stop-words. public bool UseStopwordFilter { get; set; } = true; + } diff --git a/src/AgentMemory.Core/Extraction/ExtractionNoveltyGate.cs b/src/AgentMemory.Core/Extraction/ExtractionNoveltyGate.cs new file mode 100644 index 000000000..46e4490b5 --- /dev/null +++ b/src/AgentMemory.Core/Extraction/ExtractionNoveltyGate.cs @@ -0,0 +1,120 @@ +using AgentMemory.Abstractions.Domain; + +namespace AgentMemory.Core.Extraction; + +/// +/// Decides whether a batch of turns is worth spending an extraction call on (E4). +/// +/// +/// +/// This gate sits before the completion, not before the write, and that placement is the whole +/// design. A gate on the write path would skip persisting a triple already in the store — which +/// sounds like the obvious saving and would silently disable two things that depend on the duplicate +/// write actually happening: S2 confidence reinforcement, where a re-asserted fact earns α, and R7's +/// mention_count, which is incremented by the MERGE. Corroboration is the repeated write. +/// Gating it away would leave both features enabled and inert. +/// +/// +/// The completion is also where the money is. The write is one round trip to Neo4j; the extraction is +/// a model call over the rendered transcript, and on a corpus build it is essentially the entire bill. +/// +/// +/// Precision over recall, and the asymmetry is worse here than anywhere else in the system. +/// Declining to gate costs one extraction we might not have needed. Gating a turn that did carry a +/// fact loses that fact until the user happens to say it again — the memory is simply never formed, +/// and nothing downstream can recover it or even report that it is missing. So the gate fires only on +/// turns that cannot carry a proposition at all: greetings, thanks, acknowledgement. Every genuinely +/// ambiguous token is left OUT of the vocabulary on purpose — "yes", "no" and "sure" answer questions +/// and are therefore contentful, however short they look. +/// +/// +/// Deterministic and vocabulary-based rather than model-scored, for the same reason 9.1's resolver is: +/// a gate that cost a completion to decide whether to spend a completion saves nothing, and one that +/// varied between runs would make every measured build unreproducible. +/// +/// +internal static class ExtractionNoveltyGate +{ + /// + /// Tokens that cannot assert anything on their own — greetings, gratitude, acknowledgement, and + /// the filler words that glue them together ("thanks so much", "hi there"). + /// + /// + /// Deliberately absent: yes, no, yep, nope, sure, right, + /// correct. Each is a complete answer to a question, and a batch may hold the answer while + /// the question sat in the previous batch — so the question-mark check below cannot save them. + /// + private static readonly HashSet Uninformative = new(StringComparer.OrdinalIgnoreCase) + { + // acknowledgement + "ok", "okay", "k", "kk", "got", "it", "understood", "noted", "alright", "gotcha", + // gratitude + "thanks", "thank", "you", "thx", "ty", "cheers", "welcome", "youre", "pleasure", "my", + // appraisal with no object + "great", "cool", "nice", "perfect", "awesome", "excellent", "brilliant", "good", "fine", + // greeting and farewell + "hi", "hello", "hey", "there", "bye", "goodbye", "morning", "afternoon", "evening", "night", + // apology and politeness + "sorry", "apologies", "np", "problem", "worries", + // filler that binds the above + "a", "the", "so", "much", "very", "really", "that", "is", "was", "and", "then", "well", + "lol", "haha", "hah", "yay", + }; + + /// + /// when this batch is worth an extraction call. + /// + /// + /// Biased hard toward : anything the vocabulary does not fully explain is + /// treated as potentially contentful, including empty input, which is cheap to extract from and + /// whose handling belongs to the extractors rather than here. + /// + internal static bool IsWorthExtracting(IReadOnlyList messages) + { + if (messages is null || messages.Count == 0) return true; + + foreach (var message in messages) + { + var content = message.Content; + if (string.IsNullOrWhiteSpace(content)) continue; + + // A question mark means someone asked something, and the answer -- possibly a single word + // -- is exactly the kind of content this gate must never discard. + if (content.Contains('?', StringComparison.Ordinal)) return true; + + if (!IsPurelyUninformative(content)) return true; + } + + return false; + } + + private static bool IsPurelyUninformative(string content) + { + var tokens = content.Split( + (char[]?)null, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + + foreach (var token in tokens) + { + var trimmed = token.Trim(Punctuation); + if (trimmed.Length == 0) continue; + + // Any digit makes it contentful: "3" answers "how many?", and a bare number is data. + foreach (var c in trimmed) + { + if (char.IsDigit(c)) return false; + } + + if (!Uninformative.Contains(trimmed.ToLowerInvariant()) + && !Uninformative.Contains( + trimmed.Replace("'", string.Empty, StringComparison.Ordinal).ToLowerInvariant())) + { + return false; + } + } + + return true; + } + + private static readonly char[] Punctuation = + ['.', ',', '!', '?', ';', ':', '-', '—', '–', '"', '\'', '(', ')', '[', ']', '…']; +} diff --git a/src/AgentMemory.Core/Extraction/ExtractionStage.cs b/src/AgentMemory.Core/Extraction/ExtractionStage.cs index a2f5fb5a2..7747e5b65 100644 --- a/src/AgentMemory.Core/Extraction/ExtractionStage.cs +++ b/src/AgentMemory.Core/Extraction/ExtractionStage.cs @@ -79,6 +79,22 @@ private async Task ExtractCoreAsync( CancellationToken cancellationToken) { var sourceMessageIds = messages.Select(m => m.MessageId).ToList(); + + // E4. Turns that cannot carry a fact -- "ok, thanks!" -- are not worth a completion. Only on + // the paying path: when preExtracted is non-null the caller has already spent the call, and + // gating there would discard a result that has been paid for rather than avoid the cost. + if (_options.SkipUninformativeTurns + && preExtracted is null + && !ExtractionNoveltyGate.IsWorthExtracting(messages)) + { + // Logged rather than silent: a gate that quietly drops turns is indistinguishable from an + // extractor that found nothing, and the saving it claims would be unverifiable. + _logger.LogDebug( + "Skipping extraction for {Count} turn(s) carrying no extractable content (E4).", + messages.Count); + return new ExtractionStageResult { SourceMessageIds = sourceMessageIds }; + } + var strategy = _options.MergeStrategy; var failFast = _options.FailureMode == IngestionFailureMode.FailFast; diff --git a/tests/AgentMemory.Tests.Unit/Extraction/ExtractionNoveltyGateTests.cs b/tests/AgentMemory.Tests.Unit/Extraction/ExtractionNoveltyGateTests.cs new file mode 100644 index 000000000..3aa5cc449 --- /dev/null +++ b/tests/AgentMemory.Tests.Unit/Extraction/ExtractionNoveltyGateTests.cs @@ -0,0 +1,144 @@ +using AgentMemory.Abstractions.Domain; +using AgentMemory.Abstractions.Options; +using AgentMemory.Core.Extraction; +using FluentAssertions; +using Xunit; + +namespace AgentMemory.Tests.Unit.Extraction; + +/// +/// Not paying for an extraction call on "ok, thanks!" (E4) — and, far more importantly, paying for it +/// whenever there is the slightest chance the turn said something. +/// +/// +/// +/// The asymmetry is worse here than anywhere else in the system, so the tests are lopsided. +/// Declining to gate costs one extraction call. Gating a turn that did carry a fact means the memory +/// is never formed — nothing downstream can recover it, and nothing can even report it missing. The +/// bulk of these cases are therefore phrases that must still be extracted. +/// +/// +public sealed class ExtractionNoveltyGateTests +{ + private static Message M(string content, string role = "user") => new() + { + MessageId = $"m-{content.GetHashCode():X}", + ConversationId = "c-1", + SessionId = "s-1", + Role = role, + Content = content, + TimestampUtc = DateTimeOffset.UnixEpoch, + }; + + private static bool Worth(params string[] contents) => + ExtractionNoveltyGate.IsWorthExtracting(contents.Select(c => M(c)).ToList()); + + // ── must still be extracted ─────────────────────────────────────────── + + [Theory] + [InlineData("yes")] + [InlineData("no")] + [InlineData("sure")] + [InlineData("nope")] + [InlineData("correct")] + [InlineData("right")] + public void ABareAnswerIsContentful(string answer) + { + // THE dangerous class. Each of these is a complete answer to a question -- "do you eat meat?" + // "no" -- and the question may have been in the PREVIOUS batch, so nothing in this one reveals + // that a fact was just asserted. They are kept out of the vocabulary deliberately. + Worth(answer).Should().BeTrue(); + } + + [Theory] + [InlineData("Basel")] + [InlineData("tea")] + [InlineData("3")] + [InlineData("42 Rue Lafayette")] + public void AShortContentfulTurnIsNotAPleasantry(string content) + { + // Short is not the same as empty. A one-word answer naming a place, a preference or a quantity + // is exactly the fact worth remembering. + Worth(content).Should().BeTrue(); + } + + [Fact] + public void AQuestionMarkAlwaysWinsExtraction() + { + // Someone asked something, so an answer is in play -- possibly a single word in this very + // batch. Cheap insurance against the whole ambiguous class. + Worth("ok thanks?").Should().BeTrue(); + } + + [Fact] + public void OnePleasantryBesideOneRealTurnStillExtracts() + { + // The batch is the unit. One contentful message makes the whole call worth paying for. + Worth("thanks!", "I moved to Zurich last month").Should().BeTrue(); + } + + [Fact] + public void AnEmptyBatchIsNotGated() + { + // Biased toward extracting even here: an empty batch is cheap, and what to do with it belongs + // to the extractors rather than to a gate guessing on their behalf. + ExtractionNoveltyGate.IsWorthExtracting([]).Should().BeTrue(); + } + + [Fact] + public void AGratitudeWordInsideARealSentenceDoesNotGate() + { + // "thanks" appears constantly inside contentful turns. The whole message must be explained by + // the vocabulary, token by token -- not merely contain one of its words. + Worth("thanks, my address is 12 Bahnhofstrasse").Should().BeTrue(); + } + + // ── may be skipped ──────────────────────────────────────────────────── + + [Theory] + [InlineData("ok")] + [InlineData("Thanks!")] + [InlineData("thank you so much")] + [InlineData("ok, got it")] + [InlineData("great, thanks")] + [InlineData("hi there")] + [InlineData("good morning")] + [InlineData("you're welcome")] + [InlineData("perfect")] + [InlineData("lol")] + public void PureAcknowledgementIsNotWorthACompletion(string content) => + Worth(content).Should().BeFalse(); + + [Theory] + [InlineData("no problem")] + [InlineData("no worries")] + public void APleasantryStartingWithAnAmbiguousTokenIsNotSkipped(string content) + { + // A missed skip, and the correct trade. "no problem" is plainly a pleasantry, but gating it + // means putting "no" in the vocabulary -- and then a bare "no", the complete answer to "do you + // eat meat?", stops being extracted. One wasted call is the cheaper mistake by a wide margin. + Worth(content).Should().BeTrue(); + } + + [Fact] + public void AWholeBatchOfPleasantriesIsSkipped() => + Worth("thanks!", "You're welcome!").Should().BeFalse(); + + [Fact] + public void BlankMessagesDoNotByThemselvesForceExtraction() => + Worth("ok", " ").Should().BeFalse(); + + [Fact] + public void PunctuationAndCasingDoNotDefeatTheVocabulary() => + Worth("OK... Thanks!!!").Should().BeFalse(); + + // ── shape ───────────────────────────────────────────────────────────── + + [Fact] + public void TheGateIsOffByDefault() + { + // It changes what gets extracted, and transcript bytes are fingerprinted into every measured + // run in this track. + new ExtractionOptions().SkipUninformativeTurns.Should().BeFalse(); + } +} diff --git a/tests/AgentMemory.Tests.Unit/Extraction/ExtractionStageNoveltyGatingTests.cs b/tests/AgentMemory.Tests.Unit/Extraction/ExtractionStageNoveltyGatingTests.cs new file mode 100644 index 000000000..e7ffae516 --- /dev/null +++ b/tests/AgentMemory.Tests.Unit/Extraction/ExtractionStageNoveltyGatingTests.cs @@ -0,0 +1,133 @@ +using AgentMemory.Abstractions.Domain; +using AgentMemory.Abstractions.Options; +using AgentMemory.Abstractions.Services; +using AgentMemory.Core.Extraction; +using FluentAssertions; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using NSubstitute; +using Xunit; + +namespace AgentMemory.Tests.Unit.Extraction; + +/// +/// The wiring half of E4: the gate has to prevent the call, not merely be able to judge it. +/// +/// +/// +/// 's own tests prove a batch of pleasantries scores as +/// uninformative. None of them prove a single completion was avoided. A gate nothing consults +/// passes all of them while every turn still costs exactly what it did before — and since the saving +/// is invisible from the outside, nothing else in the system would report the feature as inert. +/// +/// +/// So these assert on the extractor: was it invoked, and on which of the two paths. +/// +/// +public sealed class ExtractionStageNoveltyGatingTests +{ + private readonly IFactExtractor _extractor = Substitute.For(); + private readonly IUnifiedMemoryExtractor _unified = Substitute.For(); + + public ExtractionStageNoveltyGatingTests() + { + _extractor.ExtractAsync(Arg.Any>(), Arg.Any()) + .Returns(Task.FromResult>([])); + + _unified.IsEnabled.Returns(true); + _unified.ExtractAsync(Arg.Any>(), Arg.Any()) + .Returns(Task.FromResult(new UnifiedExtractionResult())); + } + + private static Message M(string content) => new() + { + MessageId = $"m-{content.GetHashCode():X}", + ConversationId = "c-1", + SessionId = "s-1", + Role = "user", + Content = content, + TimestampUtc = DateTimeOffset.UnixEpoch, + }; + + private ExtractionStage CreateSut(bool skipUninformativeTurns, bool withUnified = false) => + new([], [_extractor], [], [], + withUnified ? [_unified] : [], + Substitute.For(), + Options.Create(new ExtractionOptions { SkipUninformativeTurns = skipUninformativeTurns }), + NullLogger.Instance); + + [Fact] + public async Task AnUninformativeBatchNeverReachesTheExtractor() + { + // THE test. Everything else in E4 is arrangement around this call not being made. + await CreateSut(skipUninformativeTurns: true) + .ExtractAsync([M("ok, thanks!")], ExtractionTypes.All); + + await _extractor.DidNotReceive().ExtractAsync( + Arg.Any>(), Arg.Any()); + } + + [Fact] + public async Task TheUnifiedExtractorIsSkippedToo() + { + // The unified extractor is a separate dispatch path and is the one a real deployment uses -- + // gating only the per-type extractors would save nothing where it counts. + await CreateSut(skipUninformativeTurns: true, withUnified: true) + .ExtractAsync([M("thanks!")], ExtractionTypes.All); + + await _unified.DidNotReceive().ExtractAsync( + Arg.Any>(), Arg.Any()); + } + + [Fact] + public async Task AContentfulBatchStillReachesTheExtractor() + { + await CreateSut(skipUninformativeTurns: true) + .ExtractAsync([M("I moved to Zurich")], ExtractionTypes.All); + + await _extractor.Received(1).ExtractAsync( + Arg.Any>(), Arg.Any()); + } + + [Fact] + public async Task DisabledMeansEveryTurnStillCosts() + { + // The off switch, and the byte-identical guarantee: with the gate off, a pleasantry is + // extracted from exactly as it was before E4 existed. + await CreateSut(skipUninformativeTurns: false) + .ExtractAsync([M("ok, thanks!")], ExtractionTypes.All); + + await _extractor.Received(1).ExtractAsync( + Arg.Any>(), Arg.Any()); + } + + [Fact] + public async Task AlreadyExtractedResultsAreNeverGatedAway() + { + // The path that must NOT be gated. When the caller supplies a pre-extracted result the + // completion has already been paid for; discarding it there would throw away work that cost + // money instead of avoiding the cost -- the gate's exact inverse. + var preExtracted = new UnifiedExtractionResult + { + Facts = [new ExtractedFact { Subject = "user", Predicate = "lives in", Object = "Zurich", Confidence = 0.9 }], + }; + + var result = await CreateSut(skipUninformativeTurns: true) + .ProcessUnifiedAsync([M("ok, thanks!")], preExtracted, ExtractionTypes.All); + + result.RawFacts.Should().HaveCount(1, + "the caller already paid for this extraction; gating it discards work rather than avoiding it"); + } + + [Fact] + public async Task ASkippedBatchStillReportsItsSourceMessages() + { + // Skipping the call must not skip the bookkeeping: provenance is what lets a later audit tell + // "these turns were considered and carried nothing" apart from "these turns were never seen". + var result = await CreateSut(skipUninformativeTurns: true) + .ExtractAsync([M("ok"), M("thanks!")], ExtractionTypes.All); + + result.SourceMessageIds.Should().HaveCount(2); + result.RawFacts.Should().BeEmpty(); + } +} From b77a1fb653d7c24906bc5bb6a9284e287b7f2f26 Mon Sep 17 00:00:00 2001 From: Jose Luis Latorre Millas Date: Wed, 12 Aug 2026 22:24:58 +0200 Subject: [PATCH 22/31] Extract with preceding turns as context, never as targets (E2) One batch is often not enough to understand itself: "I moved there last year" needs the turn that named the place, and "she recommended it" needs the turn that named her. Without context those extract nothing, or extract an unresolved pronoun as an entity. Widening the batch fixes the reference and breaks something worse, and the plan's own note for this task had gone stale on exactly that point. It reasoned that re-extracting the preceding turns was "harmless for an exact triple, which MERGEs" -- true when it was written, false as of this week. S2 confidence reinforcement and R7's mention_count both key off the re-assertion, so extracting context would make a fact earn confidence and mentions every time it happened to sit inside a sliding window. Two signals meaning "the world keeps asserting this" would quietly come to mean "this was said recently", and nothing would look broken. The context/target split is therefore a correctness property now, not a question of token efficiency. ExtractionWindow carries Targets and Context separately. Extractors read the context, extract only from the targets, and provenance is attributed only to the targets -- an EXTRACTED_FROM edge to a context turn would assert the memory was stated in the very turn the prompt forbade extracting from. Extension uses new method names (ExtractWithContextAsync) rather than overloads. An overload made ExtractAsync(null, ct) ambiguous at existing call sites, which is a source break on a SemVer-locked public surface. The off state is call-identical, not just byte-identical: with no context the original method is invoked, so an extractor that never implemented the window overload -- including a third party's -- takes the exact path it always did. That also proved the point when the first version routed everything through the new method and broke ten existing mocks. Context renders in an unnumbered fence, because 9.3 resolves per-item provenance positionally: numbering context into the same sequence would shift every target index, and the result is not a crash but each fact attributed a few turns early, indistinguishable afterwards from precise attribution. The window is filled once in ExtractAndPersistAsync, the chokepoint the Agent Framework provider, the memory facade and the MCP ingest tool all pass through -- a window resolved per caller would depend on the host. Off by default (0 turns), and at 0 no history query is issued at all. 4,312 unit tests green (+13). Release build 0 warnings. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01PgDgctPpbTziBNE2RT8VdE --- docs/architecture.md | 2 +- .../Domain/Extraction/ExtractionRequest.cs | 19 +++ .../Domain/Extraction/ExtractionWindow.cs | 50 ++++++ .../Options/ExtractionOptions.cs | 24 +++ .../Services/IEntityExtractor.cs | 25 +++ .../Services/IFactExtractor.cs | 25 +++ .../Services/IPreferenceExtractor.cs | 25 +++ .../Services/IRelationshipExtractor.cs | 25 +++ .../Services/IUnifiedMemoryExtractor.cs | 14 ++ .../Extraction/ConversationTextBuilder.cs | 38 +++++ .../Extraction/ExtractionStage.cs | 48 ++++-- .../Extraction/ExtractorBase.cs | 29 +++- .../Extraction/IExtractionStage.cs | 16 ++ .../Services/MemoryExtractionPipeline.cs | 13 +- .../Services/MemoryService.cs | 57 ++++++- .../ExtractionPromptSemantics.cs | 21 +++ .../LlmEntityExtractor.cs | 12 +- .../LlmFactExtractor.cs | 17 +- .../LlmPreferenceExtractor.cs | 15 +- .../LlmRelationshipExtractor.cs | 12 +- .../LlmUnifiedMemoryExtractor.cs | 21 ++- .../Extraction/ExtractionWindowTests.cs | 157 ++++++++++++++++++ .../AbstractionsContractGuardTests.cs | 2 +- .../MemoryServiceExtractionContextTests.cs | 157 ++++++++++++++++++ 24 files changed, 783 insertions(+), 41 deletions(-) create mode 100644 src/AgentMemory.Abstractions/Domain/Extraction/ExtractionWindow.cs create mode 100644 tests/AgentMemory.Tests.Unit/Extraction/ExtractionWindowTests.cs create mode 100644 tests/AgentMemory.Tests.Unit/Services/MemoryServiceExtractionContextTests.cs diff --git a/docs/architecture.md b/docs/architecture.md index 053248477..aafb83f2a 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -142,7 +142,7 @@ graph TD | **Purpose** | Domain contracts — all models, interfaces, and configuration types shared across the system | | **Dependencies** | **Microsoft.Extensions.AI.Abstractions** 10.8.0 (approved, D-AR2-1) — .NET BCL otherwise (multi-targets net8.0/net9.0/net10.0) | | **MUST NOT reference** | Neo4j.Driver, Microsoft.Agents.*, any GraphRAG SDK, any MCP SDK, any NuGet package **except** Microsoft.Extensions.AI.Abstractions | -| **Key types** | 52 domain records (Conversation, Message, Entity, Fact, Preference, Relationship, MemoryHistoryQuery, MemoryHistoryRecord, ReasoningTrace, ReasoningStep, ToolCall, ToolCallStats, IngestionItemOutcome, MemoryContextRankedItem, MemoryContextSectionDiagnostics, UnifiedExtractionResult, etc.), 42 service interfaces (incl. `IMemoryIsolationPolicy`, `IUnifiedMemoryExtractor`, `IMultiSessionUnifiedMemoryExtractor`, and `IMemoryReranker`), 11 repository interfaces, 16 configuration types (incl. `MemoryRankingOptions`, `MemoryIsolationOptions`), 26 enums (incl. `MemoryProfile`, `RankingIntent`, `DuplicateStatus`, `EntityMatchType`, `MemoryNodeKind`, `MemoryOperationAccess`, `MemoryIsolationMode`, `IngestionStatus`, `IngestionStage`, `IngestionItemStatus`, `MemoryItemKind`, `IngestionFailureMode`, `MemoryTrustLevel`, `AssistantContentMode`, `TemporalValidityMode`, `TraceKind`, `ExtractionProvenanceMode`) | +| **Key types** | 53 domain records (Conversation, Message, Entity, Fact, Preference, Relationship, MemoryHistoryQuery, MemoryHistoryRecord, ReasoningTrace, ReasoningStep, ToolCall, ToolCallStats, IngestionItemOutcome, MemoryContextRankedItem, MemoryContextSectionDiagnostics, UnifiedExtractionResult, ExtractionWindow, etc.), 42 service interfaces (incl. `IMemoryIsolationPolicy`, `IUnifiedMemoryExtractor`, `IMultiSessionUnifiedMemoryExtractor`, and `IMemoryReranker`), 11 repository interfaces, 16 configuration types (incl. `MemoryRankingOptions`, `MemoryIsolationOptions`), 26 enums (incl. `MemoryProfile`, `RankingIntent`, `DuplicateStatus`, `EntityMatchType`, `MemoryNodeKind`, `MemoryOperationAccess`, `MemoryIsolationMode`, `IngestionStatus`, `IngestionStage`, `IngestionItemStatus`, `MemoryItemKind`, `IngestionFailureMode`, `MemoryTrustLevel`, `AssistantContentMode`, `TemporalValidityMode`, `TraceKind`, `ExtractionProvenanceMode`) | **Namespace structure:** ``` diff --git a/src/AgentMemory.Abstractions/Domain/Extraction/ExtractionRequest.cs b/src/AgentMemory.Abstractions/Domain/Extraction/ExtractionRequest.cs index 0b0460151..ced298846 100644 --- a/src/AgentMemory.Abstractions/Domain/Extraction/ExtractionRequest.cs +++ b/src/AgentMemory.Abstractions/Domain/Extraction/ExtractionRequest.cs @@ -10,6 +10,25 @@ public sealed record ExtractionRequest /// public required IReadOnlyList Messages { get; init; } + /// + /// Earlier turns supplied only so can be understood (E2). + /// + /// + /// + /// Nothing is extracted from these and nothing is attributed to them — they exist so + /// "I moved there last year" can resolve the place named a few turns earlier. Empty by default, + /// which is exactly the pre-E2 behaviour. + /// + /// + /// Kept separate from rather than prepended to it, because + /// extracting from them would re-assert stored facts — and a re-assertion now earns confidence + /// (S2) and increments the mention_count the salience reranker reads (R7). A fact would + /// gain both every time it happened to sit inside a sliding window, so corroboration would + /// quietly become recency. + /// + /// + public IReadOnlyList ContextMessages { get; init; } = []; + /// /// Session context for the extraction. /// diff --git a/src/AgentMemory.Abstractions/Domain/Extraction/ExtractionWindow.cs b/src/AgentMemory.Abstractions/Domain/Extraction/ExtractionWindow.cs new file mode 100644 index 000000000..a876a0e30 --- /dev/null +++ b/src/AgentMemory.Abstractions/Domain/Extraction/ExtractionWindow.cs @@ -0,0 +1,50 @@ +namespace AgentMemory.Abstractions.Domain; + +/// +/// The turns an extractor should extract from, plus the earlier turns it may read to understand them +/// (E2). +/// +/// +/// +/// Extraction has always seen one batch — typically a user turn and the reply — which is often not +/// enough to resolve what was said. "I moved there last year" needs the turn that named the place; +/// "she recommended it" needs the turn that named her. Widening the batch fixes the reference and +/// creates a worse problem, which is why this type exists rather than a bigger messages list. +/// +/// +/// Context is not a target, and the distinction is now a correctness property rather than a +/// question of token efficiency. Re-extracting the preceding turns re-asserts facts already +/// stored. That used to be merely wasteful — the exact triple MERGEs, so nothing was corrupted — but +/// with confidence reinforcement (S2) a re-assertion earns α, and the salience reranker (R7) reads a +/// mention_count the MERGE increments. If context turns were extracted as targets, a fact +/// would gain confidence and mentions every time it happened to sit inside a sliding window, so both +/// signals would measure how recently something was said rather than how often the world +/// asserted it. Corroboration would quietly become recency. +/// +/// +/// So the contract is: read to disambiguate, extract only from +/// , and attribute provenance only to . +/// +/// +public sealed record ExtractionWindow +{ + /// The turns to extract memories from. Provenance is attributed here and nowhere else. + public required IReadOnlyList Targets { get; init; } + + /// + /// Earlier turns, in order, supplied purely so the targets can be understood. + /// + /// + /// Nothing may be extracted from these. They carry no provenance and must never appear in + /// SourceMessageIds: an EXTRACTED_FROM edge to a context turn would claim the fact + /// was stated there, which is exactly the turn the extractor was told not to extract from. + /// + public IReadOnlyList Context { get; init; } = []; + + /// A window with no context — identical in effect to the pre-E2 ExtractAsync(messages). + public static ExtractionWindow ForTargets(IReadOnlyList targets) => + new() { Targets = targets }; + + /// when there is context to render at all. + public bool HasContext => Context.Count > 0; +} diff --git a/src/AgentMemory.Abstractions/Options/ExtractionOptions.cs b/src/AgentMemory.Abstractions/Options/ExtractionOptions.cs index bf8576999..f4fb37145 100644 --- a/src/AgentMemory.Abstractions/Options/ExtractionOptions.cs +++ b/src/AgentMemory.Abstractions/Options/ExtractionOptions.cs @@ -125,6 +125,30 @@ public sealed class ExtractionOptions /// /// public bool SkipUninformativeTurns { get; set; } + + /// + /// How many earlier turns to hand the extractors as read-only context (E2). 0 — the + /// default — is the pre-E2 behaviour of extracting from the batch alone. + /// + /// + /// + /// One batch is often not enough to understand itself: "I moved there last year" needs the turn + /// that named the place, and "she recommended it" needs the turn that named her. Without context + /// those either extract nothing or extract an unresolved pronoun as an entity. + /// + /// + /// Context turns are read, never extracted from, and never attributed to. Extracting them + /// would re-assert facts already stored — which now earns confidence + /// () and increments the + /// mention_count the salience reranker reads, so a fact would gain both merely by sitting + /// inside a sliding window. Corroboration would become recency. + /// + /// + /// Costs input tokens on every extraction, which is why it is off rather than defaulted to a + /// nominal value. Zep uses 4. + /// + /// + public int ExtractionContextTurns { get; set; } } /// Controls which matching strategies are used for entity resolution. diff --git a/src/AgentMemory.Abstractions/Services/IEntityExtractor.cs b/src/AgentMemory.Abstractions/Services/IEntityExtractor.cs index 3c815b99a..7917430a6 100644 --- a/src/AgentMemory.Abstractions/Services/IEntityExtractor.cs +++ b/src/AgentMemory.Abstractions/Services/IEntityExtractor.cs @@ -13,4 +13,29 @@ public interface IEntityExtractor Task> ExtractAsync( IReadOnlyList messages, CancellationToken cancellationToken = default); + + /// + /// Extracts from , reading + /// only to understand them (E2). + /// + /// + /// + /// The default implementation ignores the context entirely and calls the existing overload + /// with the targets — byte-identical to pre-E2 behaviour. An extractor that cannot use context is + /// therefore correct by doing nothing, rather than being broken by a widened window. + /// + /// + /// A default interface method rather than a new required member: the extractor interfaces are + /// public and locked under SemVer, and a third-party extractor must keep compiling. + /// + /// + /// Implementers must not extract from the context turns. Doing so re-asserts facts already stored, + /// which now inflates confidence (S2) and mention_count (R7) purely because a fact sat + /// inside a sliding window — turning corroboration into recency. + /// + /// + Task> ExtractWithContextAsync( + ExtractionWindow window, + CancellationToken cancellationToken = default) + => ExtractAsync(window.Targets, cancellationToken); } diff --git a/src/AgentMemory.Abstractions/Services/IFactExtractor.cs b/src/AgentMemory.Abstractions/Services/IFactExtractor.cs index 2ebaa4533..ff26a9b40 100644 --- a/src/AgentMemory.Abstractions/Services/IFactExtractor.cs +++ b/src/AgentMemory.Abstractions/Services/IFactExtractor.cs @@ -13,4 +13,29 @@ public interface IFactExtractor Task> ExtractAsync( IReadOnlyList messages, CancellationToken cancellationToken = default); + + /// + /// Extracts from , reading + /// only to understand them (E2). + /// + /// + /// + /// The default implementation ignores the context entirely and calls the existing overload + /// with the targets — byte-identical to pre-E2 behaviour. An extractor that cannot use context is + /// therefore correct by doing nothing, rather than being broken by a widened window. + /// + /// + /// A default interface method rather than a new required member: the extractor interfaces are + /// public and locked under SemVer, and a third-party extractor must keep compiling. + /// + /// + /// Implementers must not extract from the context turns. Doing so re-asserts facts already stored, + /// which now inflates confidence (S2) and mention_count (R7) purely because a fact sat + /// inside a sliding window — turning corroboration into recency. + /// + /// + Task> ExtractWithContextAsync( + ExtractionWindow window, + CancellationToken cancellationToken = default) + => ExtractAsync(window.Targets, cancellationToken); } diff --git a/src/AgentMemory.Abstractions/Services/IPreferenceExtractor.cs b/src/AgentMemory.Abstractions/Services/IPreferenceExtractor.cs index 348262991..c5d6f29a4 100644 --- a/src/AgentMemory.Abstractions/Services/IPreferenceExtractor.cs +++ b/src/AgentMemory.Abstractions/Services/IPreferenceExtractor.cs @@ -13,4 +13,29 @@ public interface IPreferenceExtractor Task> ExtractAsync( IReadOnlyList messages, CancellationToken cancellationToken = default); + + /// + /// Extracts from , reading + /// only to understand them (E2). + /// + /// + /// + /// The default implementation ignores the context entirely and calls the existing overload + /// with the targets — byte-identical to pre-E2 behaviour. An extractor that cannot use context is + /// therefore correct by doing nothing, rather than being broken by a widened window. + /// + /// + /// A default interface method rather than a new required member: the extractor interfaces are + /// public and locked under SemVer, and a third-party extractor must keep compiling. + /// + /// + /// Implementers must not extract from the context turns. Doing so re-asserts facts already stored, + /// which now inflates confidence (S2) and mention_count (R7) purely because a fact sat + /// inside a sliding window — turning corroboration into recency. + /// + /// + Task> ExtractWithContextAsync( + ExtractionWindow window, + CancellationToken cancellationToken = default) + => ExtractAsync(window.Targets, cancellationToken); } diff --git a/src/AgentMemory.Abstractions/Services/IRelationshipExtractor.cs b/src/AgentMemory.Abstractions/Services/IRelationshipExtractor.cs index 32592025d..2b1a77613 100644 --- a/src/AgentMemory.Abstractions/Services/IRelationshipExtractor.cs +++ b/src/AgentMemory.Abstractions/Services/IRelationshipExtractor.cs @@ -13,4 +13,29 @@ public interface IRelationshipExtractor Task> ExtractAsync( IReadOnlyList messages, CancellationToken cancellationToken = default); + + /// + /// Extracts from , reading + /// only to understand them (E2). + /// + /// + /// + /// The default implementation ignores the context entirely and calls the existing overload + /// with the targets — byte-identical to pre-E2 behaviour. An extractor that cannot use context is + /// therefore correct by doing nothing, rather than being broken by a widened window. + /// + /// + /// A default interface method rather than a new required member: the extractor interfaces are + /// public and locked under SemVer, and a third-party extractor must keep compiling. + /// + /// + /// Implementers must not extract from the context turns. Doing so re-asserts facts already stored, + /// which now inflates confidence (S2) and mention_count (R7) purely because a fact sat + /// inside a sliding window — turning corroboration into recency. + /// + /// + Task> ExtractWithContextAsync( + ExtractionWindow window, + CancellationToken cancellationToken = default) + => ExtractAsync(window.Targets, cancellationToken); } diff --git a/src/AgentMemory.Abstractions/Services/IUnifiedMemoryExtractor.cs b/src/AgentMemory.Abstractions/Services/IUnifiedMemoryExtractor.cs index fc3420f71..47d3e50aa 100644 --- a/src/AgentMemory.Abstractions/Services/IUnifiedMemoryExtractor.cs +++ b/src/AgentMemory.Abstractions/Services/IUnifiedMemoryExtractor.cs @@ -12,4 +12,18 @@ public interface IUnifiedMemoryExtractor Task ExtractAsync( IReadOnlyList messages, CancellationToken cancellationToken = default); + + /// + /// Extracts every supported category from , reading + /// only to understand them (E2). + /// + /// + /// The default implementation ignores the context and calls the existing overload — byte-identical + /// to pre-E2 behaviour. Implementers must not extract from context turns: doing so re-asserts + /// stored facts and inflates confidence (S2) and mention_count (R7) on nothing but recency. + /// + Task ExtractWithContextAsync( + ExtractionWindow window, + CancellationToken cancellationToken = default) + => ExtractAsync(window.Targets, cancellationToken); } diff --git a/src/AgentMemory.Core/Extraction/ConversationTextBuilder.cs b/src/AgentMemory.Core/Extraction/ConversationTextBuilder.cs index 6d43797d5..966cd217f 100644 --- a/src/AgentMemory.Core/Extraction/ConversationTextBuilder.cs +++ b/src/AgentMemory.Core/Extraction/ConversationTextBuilder.cs @@ -50,4 +50,42 @@ public static string BuildNumbered(IReadOnlyList messages) } return builder.ToString(); } + + /// + /// Renders a window as a fenced, unnumbered context block followed by the turns to extract from + /// (E2). Falls back to the plain rendering when the window carries no context. + /// + /// + /// + /// The context block is deliberately not numbered. Per-item provenance (L3c) numbers turns + /// from 1 and resolves turn N positionally to Targets[N-1]. Numbering the context + /// into the same sequence would shift every target index by the context length, and the failure + /// would not look like a crash — each fact would simply be attributed to a turn a few places + /// earlier, which afterwards is indistinguishable from correct attribution. + /// + /// + /// The fence is explicit text rather than a formatting convention because the model has to act on + /// it: everything inside is to be read and nothing inside is to be extracted. + /// + /// + public static string BuildWindow(ExtractionWindow window, bool numbered) + { + var targets = numbered ? BuildNumbered(window.Targets) : Build(window.Targets); + if (!window.HasContext) return targets; + + var builder = new StringBuilder(); + builder.Append(ContextOpen).Append('\n') + .Append(Build(window.Context)).Append('\n') + .Append(ContextClose).Append("\n\n") + .Append(targets); + return builder.ToString(); + } + + /// Opening fence of the read-only context block. + internal const string ContextOpen = + "--- EARLIER CONVERSATION (for reference only — do NOT extract anything from these turns) ---"; + + /// Closing fence of the read-only context block. + internal const string ContextClose = + "--- END OF EARLIER CONVERSATION. Extract ONLY from the turns below. ---"; } diff --git a/src/AgentMemory.Core/Extraction/ExtractionStage.cs b/src/AgentMemory.Core/Extraction/ExtractionStage.cs index 7747e5b65..077190cbb 100644 --- a/src/AgentMemory.Core/Extraction/ExtractionStage.cs +++ b/src/AgentMemory.Core/Extraction/ExtractionStage.cs @@ -58,7 +58,15 @@ public Task ExtractAsync( ExtractionTypes typesToExtract, MemoryScope? scope = null, CancellationToken cancellationToken = default) => - ExtractCoreAsync(messages, typesToExtract, scope, preExtracted: null, cancellationToken); + ExtractCoreAsync( + ExtractionWindow.ForTargets(messages), typesToExtract, scope, preExtracted: null, cancellationToken); + + public Task ExtractWithContextAsync( + ExtractionWindow window, + ExtractionTypes typesToExtract, + MemoryScope? scope = null, + CancellationToken cancellationToken = default) => + ExtractCoreAsync(window, typesToExtract, scope, preExtracted: null, cancellationToken); public Task ProcessUnifiedAsync( IReadOnlyList messages, @@ -68,16 +76,22 @@ public Task ProcessUnifiedAsync( CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(extracted); - return ExtractCoreAsync(messages, typesToExtract, scope, extracted, cancellationToken); + return ExtractCoreAsync( + ExtractionWindow.ForTargets(messages), typesToExtract, scope, extracted, cancellationToken); } private async Task ExtractCoreAsync( - IReadOnlyList messages, + ExtractionWindow window, ExtractionTypes typesToExtract, MemoryScope? scope, UnifiedExtractionResult? preExtracted, CancellationToken cancellationToken) { + var messages = window.Targets; + + // Targets only, never the context (E2). A context turn in here would become an EXTRACTED_FROM + // edge asserting the memory was stated in a turn the extractors were told not to extract from + // -- and afterwards that is indistinguishable from correct attribution. var sourceMessageIds = messages.Select(m => m.MessageId).ToList(); // E4. Turns that cannot carry a fact -- "ok, thanks!" -- are not worth a completion. Only on @@ -121,7 +135,7 @@ private async Task ExtractCoreAsync( else if (unifiedExtractor is not null) { var unifiedRun = await ExtractUnifiedSafeAsync( - unifiedExtractor, messages, typesToExtract, cancellationToken).ConfigureAwait(false); + unifiedExtractor, window, typesToExtract, cancellationToken).ConfigureAwait(false); var unified = unifiedRun.Result; unifiedOutcomes = unifiedRun.Outcomes; entityRun = CompletedRun(typesToExtract.HasFlag(ExtractionTypes.Entities) ? unified.Entities : []); @@ -132,22 +146,30 @@ private async Task ExtractCoreAsync( else { entityRun = typesToExtract.HasFlag(ExtractionTypes.Entities) - ? RunExtractorsAsync(_entityExtractors, e => e.ExtractAsync(messages, cancellationToken), + ? RunExtractorsAsync(_entityExtractors, e => window.HasContext + ? e.ExtractWithContextAsync(window, cancellationToken) + : e.ExtractAsync(messages, cancellationToken), strategy, MergeStrategyFactory.CreateEntityStrategy, "entity", MemoryItemKind.Entity, MemoryErrorCodes.EntityExtractionFailed, cancellationToken) : EmptyRun(); factRun = typesToExtract.HasFlag(ExtractionTypes.Facts) - ? RunExtractorsAsync(_factExtractors, f => f.ExtractAsync(messages, cancellationToken), + ? RunExtractorsAsync(_factExtractors, f => window.HasContext + ? f.ExtractWithContextAsync(window, cancellationToken) + : f.ExtractAsync(messages, cancellationToken), strategy, MergeStrategyFactory.CreateFactStrategy, "fact", MemoryItemKind.Fact, MemoryErrorCodes.FactExtractionFailed, cancellationToken) : EmptyRun(); prefRun = typesToExtract.HasFlag(ExtractionTypes.Preferences) - ? RunExtractorsAsync(_preferenceExtractors, p => p.ExtractAsync(messages, cancellationToken), + ? RunExtractorsAsync(_preferenceExtractors, p => window.HasContext + ? p.ExtractWithContextAsync(window, cancellationToken) + : p.ExtractAsync(messages, cancellationToken), strategy, MergeStrategyFactory.CreatePreferenceStrategy, "preference", MemoryItemKind.Preference, MemoryErrorCodes.PreferenceExtractionFailed, cancellationToken) : EmptyRun(); relRun = typesToExtract.HasFlag(ExtractionTypes.Relationships) - ? RunExtractorsAsync(_relationshipExtractors, r => r.ExtractAsync(messages, cancellationToken), + ? RunExtractorsAsync(_relationshipExtractors, r => window.HasContext + ? r.ExtractWithContextAsync(window, cancellationToken) + : r.ExtractAsync(messages, cancellationToken), strategy, MergeStrategyFactory.CreateRelationshipStrategy, "relationship", MemoryItemKind.Relationship, MemoryErrorCodes.RelationshipExtractionFailed, cancellationToken) : EmptyRun(); @@ -375,13 +397,19 @@ await batchResolver.PrepareCandidatesAsync(candidateTypes, scope, cancellationTo private async Task<(UnifiedExtractionResult Result, IReadOnlyList Outcomes)> ExtractUnifiedSafeAsync( IUnifiedMemoryExtractor extractor, - IReadOnlyList messages, + ExtractionWindow window, ExtractionTypes typesToExtract, CancellationToken cancellationToken) { try { - return (await extractor.ExtractAsync(messages, cancellationToken).ConfigureAwait(false), + // Without context, the ORIGINAL call. Not a shortcut: it keeps the off state identical at + // the call itself rather than only in the rendered transcript, so an extractor that never + // implemented the window overload -- including any third party's -- takes the exact path + // it always did. + return (await (window.HasContext + ? extractor.ExtractWithContextAsync(window, cancellationToken) + : extractor.ExtractAsync(window.Targets, cancellationToken)).ConfigureAwait(false), Array.Empty()); } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) diff --git a/src/AgentMemory.Core/Extraction/ExtractorBase.cs b/src/AgentMemory.Core/Extraction/ExtractorBase.cs index bb6dd8ece..fd747587c 100644 --- a/src/AgentMemory.Core/Extraction/ExtractorBase.cs +++ b/src/AgentMemory.Core/Extraction/ExtractorBase.cs @@ -35,6 +35,18 @@ protected ExtractorBase(ILogger logger) protected abstract Task> ExtractCoreAsync( IReadOnlyList messages, CancellationToken cancellationToken); + /// + /// The window-aware core (E2). Reads to resolve references + /// and extracts only from . + /// + /// + /// Defaults to the context-free core, so an extractor that has not opted in behaves byte-identically + /// to its pre-E2 self rather than silently widening what it extracts. + /// + protected virtual Task> ExtractCoreWithContextAsync( + ExtractionWindow window, CancellationToken cancellationToken) + => ExtractCoreAsync(window.Targets, cancellationToken); + /// /// Extracts items from the supplied messages, returning an empty list when no messages /// are provided or when the underlying extraction fails. @@ -45,13 +57,24 @@ protected abstract Task> ExtractCoreAsync( /// A task that resolves to the extracted items, or an empty list if there are no /// messages or an error occurs during extraction. /// - public async Task> ExtractAsync( + public Task> ExtractAsync( IReadOnlyList messages, CancellationToken cancellationToken = default) + => ExtractWithContextAsync(ExtractionWindow.ForTargets(messages), cancellationToken); + + /// + /// Extracts from the window's targets, reading its context only to understand them (E2), with the + /// same failure containment as the message overload. + /// + public async Task> ExtractWithContextAsync( + ExtractionWindow window, CancellationToken cancellationToken = default) { - if (messages.Count == 0) return Array.Empty(); + // Emptiness is judged on the TARGETS. A window carrying context but nothing to extract from + // has nothing to do -- calling out with only background would spend a completion on turns the + // extractor has been told not to extract from. + if (window.Targets.Count == 0) return Array.Empty(); try { - return await ExtractCoreAsync(messages, cancellationToken).ConfigureAwait(false); + return await ExtractCoreWithContextAsync(window, cancellationToken).ConfigureAwait(false); } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { diff --git a/src/AgentMemory.Core/Extraction/IExtractionStage.cs b/src/AgentMemory.Core/Extraction/IExtractionStage.cs index 171c153f6..166c286d0 100644 --- a/src/AgentMemory.Core/Extraction/IExtractionStage.cs +++ b/src/AgentMemory.Core/Extraction/IExtractionStage.cs @@ -23,6 +23,22 @@ Task ExtractAsync( MemoryScope? scope = null, CancellationToken cancellationToken = default); + /// + /// As , + /// but supplying earlier turns the extractors may read to resolve references (E2). + /// + /// + /// Provenance is attributed to only. A context turn must + /// never appear in SourceMessageIds: an EXTRACTED_FROM edge pointing at one would + /// claim the memory was stated in a turn the extractor was explicitly told not to extract from. + /// + Task ExtractWithContextAsync( + ExtractionWindow window, + ExtractionTypes typesToExtract, + MemoryScope? scope = null, + CancellationToken cancellationToken = default) + => ExtractAsync(window.Targets, typesToExtract, scope, cancellationToken); + /// /// Applies the normal validation, owner-scoped resolution, and filtering stages to a unified /// result that was already extracted by a validated multi-session batch. diff --git a/src/AgentMemory.Core/Services/MemoryExtractionPipeline.cs b/src/AgentMemory.Core/Services/MemoryExtractionPipeline.cs index 6fab1abe2..2ea9b28c0 100644 --- a/src/AgentMemory.Core/Services/MemoryExtractionPipeline.cs +++ b/src/AgentMemory.Core/Services/MemoryExtractionPipeline.cs @@ -58,8 +58,17 @@ public async Task ExtractAsync( var scope = _isolationPolicy.ResolveReadScope( explicitScope: null, request.UserId, nameof(ExtractAsync), MemoryOperationAccess.Tenant); - var staged = await _extractionStage.ExtractAsync( - request.Messages, request.TypesToExtract, scope, cancellationToken).ConfigureAwait(false); + // E2. Context reaches the extractors as context; the window keeps it out of provenance and + // out of the extraction targets, which is what stops it inflating S2 confidence and R7 + // mention counts on facts that merely stayed inside the window. + // Without context, the original call — the off state stays identical at the call itself, not + // merely in what the extractors end up seeing. + var staged = request.ContextMessages.Count == 0 + ? await _extractionStage.ExtractAsync( + request.Messages, request.TypesToExtract, scope, cancellationToken).ConfigureAwait(false) + : await _extractionStage.ExtractWithContextAsync( + new ExtractionWindow { Targets = request.Messages, Context = request.ContextMessages }, + request.TypesToExtract, scope, cancellationToken).ConfigureAwait(false); var ownerId = _isolationPolicy.ResolveWriteOwner(request.UserId, nameof(ExtractAsync), MemoryOperationAccess.Tenant); // #92 Phase 3: a per-request TrustLevel override wins; otherwise fall back to the configured default. diff --git a/src/AgentMemory.Core/Services/MemoryService.cs b/src/AgentMemory.Core/Services/MemoryService.cs index b33207a88..8c0184c4d 100644 --- a/src/AgentMemory.Core/Services/MemoryService.cs +++ b/src/AgentMemory.Core/Services/MemoryService.cs @@ -298,13 +298,66 @@ public Task> AddMessagesAsync( } /// - public Task ExtractAndPersistAsync( + public async Task ExtractAndPersistAsync( ExtractionRequest request, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(request); _logger.LogDebug("Extracting and persisting memory for session {SessionId}", request.SessionId); - return _extraction.ExtractAsync(request, cancellationToken); + request = await WithExtractionContextAsync(request, cancellationToken).ConfigureAwait(false); + return await _extraction.ExtractAsync(request, cancellationToken).ConfigureAwait(false); + } + + /// + /// Attaches the preceding turns an extractor may read to resolve references (E2). + /// + /// + /// + /// Filled in here rather than at each call site because this is the one chokepoint every + /// incremental extraction passes through — the Agent Framework provider, the Microsoft memory + /// facade and the MCP ingest tool all arrive at this method. Resolving it per caller would make + /// the window depend on which host was used, which is the failure this codebase has hit + /// repeatedly: a setting only some components respect is worse than no setting. + /// + /// + /// An explicitly supplied context always wins — a caller that knows the conversation better than + /// a recency query does should not be second-guessed. + /// + /// + private async Task WithExtractionContextAsync( + ExtractionRequest request, + CancellationToken cancellationToken) + { + if (_options.Extraction.ExtractionContextTurns <= 0 + || request.ContextMessages.Count > 0 + || request.Messages.Count == 0) + { + return request; + } + + var targetIds = request.Messages.Select(m => m.MessageId).ToHashSet(StringComparer.Ordinal); + + // Over-fetch by the batch size: the most recent stored messages usually ARE the batch, so + // asking for exactly N would return N turns of which most are excluded and leave the window + // short without reporting it. + var wanted = _options.Extraction.ExtractionContextTurns; + var recent = await _shortTerm.GetRecentMessagesAsync( + request.SessionId, wanted + request.Messages.Count, cancellationToken).ConfigureAwait(false); + + // The batch itself is never context. Its turns are the targets, and a message appearing in + // both would be handed to the model twice -- once as "do not extract from this". + var context = recent + .Where(m => !targetIds.Contains(m.MessageId)) + .TakeLast(wanted) + .ToList(); + + if (context.Count == 0) return request; + + _logger.LogDebug( + "Extraction window: {Targets} target turn(s) with {Context} turn(s) of read-only context (E2).", + request.Messages.Count, context.Count); + + return request with { ContextMessages = context }; } /// diff --git a/src/AgentMemory.Extraction.Llm/ExtractionPromptSemantics.cs b/src/AgentMemory.Extraction.Llm/ExtractionPromptSemantics.cs index 977f498c9..39c503305 100644 --- a/src/AgentMemory.Extraction.Llm/ExtractionPromptSemantics.cs +++ b/src/AgentMemory.Extraction.Llm/ExtractionPromptSemantics.cs @@ -134,4 +134,25 @@ internal static class ExtractionPromptSemantics _ => string.Empty, }; + + /// + /// Tells the model that the fenced earlier turns are background and not extraction targets (E2). + /// + /// + /// + /// Appended only when a window actually carries context, so a prompt without context is + /// byte-for-byte what it was before E2 existed. Prompt bytes are fingerprinted into every + /// measured run in this track; an instruction that appeared unconditionally would invalidate every + /// sealed base to say something about turns that were not supplied. + /// + /// + /// Authored here rather than per extractor for the usual reason: an instruction only some + /// extractors carry makes what a memory is depend on which performance flag is set. + /// + /// + internal const string ExtractionContextInstruction = + "\nThe transcript begins with earlier turns marked as EARLIER CONVERSATION. Read them only to " + + "resolve references — who \"she\" is, where \"there\" is, what \"it\" refers to. Do NOT extract " + + "any memory whose statement lives in those turns; extract only from the turns after the end " + + "marker, even if an earlier turn states something you would otherwise record."; } diff --git a/src/AgentMemory.Extraction.Llm/LlmEntityExtractor.cs b/src/AgentMemory.Extraction.Llm/LlmEntityExtractor.cs index dbed253ca..9c62e397a 100644 --- a/src/AgentMemory.Extraction.Llm/LlmEntityExtractor.cs +++ b/src/AgentMemory.Extraction.Llm/LlmEntityExtractor.cs @@ -48,12 +48,18 @@ public LlmEntityExtractor( _runner = new LlmExtractionRunner(chatClient, _options, logger); } - protected override async Task> ExtractCoreAsync( + protected override Task> ExtractCoreAsync( IReadOnlyList messages, CancellationToken cancellationToken) + => ExtractCoreWithContextAsync(ExtractionWindow.ForTargets(messages), cancellationToken); + + protected override async Task> ExtractCoreWithContextAsync( + ExtractionWindow window, CancellationToken cancellationToken) { - var conversationText = ConversationTextBuilder.Build(messages); + var conversationText = ConversationTextBuilder.BuildWindow(window, numbered: false); return await _runner.RunAsync( - _options.EntityExtractionPrompt ?? BuildSystemPrompt(_options.EntityTypes), + (_options.EntityExtractionPrompt ?? BuildSystemPrompt(_options.EntityTypes)) + // Only when context is present, so a context-free prompt stays byte-identical (E2). + + (window.HasContext ? ExtractionPromptSemantics.ExtractionContextInstruction : string.Empty), "Extract entities from this conversation:", conversationText, ProjectEntities, diff --git a/src/AgentMemory.Extraction.Llm/LlmFactExtractor.cs b/src/AgentMemory.Extraction.Llm/LlmFactExtractor.cs index 7376919d7..46102e9bd 100644 --- a/src/AgentMemory.Extraction.Llm/LlmFactExtractor.cs +++ b/src/AgentMemory.Extraction.Llm/LlmFactExtractor.cs @@ -46,16 +46,21 @@ public LlmFactExtractor( _runner = new LlmExtractionRunner(chatClient, _options, logger); } - protected override async Task> ExtractCoreAsync( + protected override Task> ExtractCoreAsync( IReadOnlyList messages, CancellationToken cancellationToken) + => ExtractCoreWithContextAsync(ExtractionWindow.ForTargets(messages), cancellationToken); + + protected override async Task> ExtractCoreWithContextAsync( + ExtractionWindow window, CancellationToken cancellationToken) { - var conversationText = _options.Provenance == ExtractionProvenanceMode.PerItem - ? ConversationTextBuilder.BuildNumbered(messages) - : ConversationTextBuilder.Build(messages); + var conversationText = ConversationTextBuilder.BuildWindow( + window, numbered: _options.Provenance == ExtractionProvenanceMode.PerItem); return await _runner.RunAsync( - _options.FactExtractionPrompt + (_options.FactExtractionPrompt ?? BuildSystemPrompt( - _options.AssistantContent, _options.TemporalValidity, _options.Provenance), + _options.AssistantContent, _options.TemporalValidity, _options.Provenance)) + // Only when context is present: a context-free prompt must stay byte-identical (E2). + + (window.HasContext ? ExtractionPromptSemantics.ExtractionContextInstruction : string.Empty), "Extract facts from this conversation:", conversationText, ProjectFacts, diff --git a/src/AgentMemory.Extraction.Llm/LlmPreferenceExtractor.cs b/src/AgentMemory.Extraction.Llm/LlmPreferenceExtractor.cs index bdabaa4fc..7d5a4ba7f 100644 --- a/src/AgentMemory.Extraction.Llm/LlmPreferenceExtractor.cs +++ b/src/AgentMemory.Extraction.Llm/LlmPreferenceExtractor.cs @@ -45,14 +45,21 @@ public LlmPreferenceExtractor( _runner = new LlmExtractionRunner(chatClient, _options, logger); } - protected override async Task> ExtractCoreAsync( + protected override Task> ExtractCoreAsync( IReadOnlyList messages, CancellationToken cancellationToken) + => ExtractCoreWithContextAsync(ExtractionWindow.ForTargets(messages), cancellationToken); + + protected override async Task> ExtractCoreWithContextAsync( + ExtractionWindow window, CancellationToken cancellationToken) { + var messages = window.Targets; var conversationText = _options.Provenance == ExtractionProvenanceMode.PerItem - ? ConversationTextBuilder.BuildNumbered(messages) - : ConversationTextBuilder.Build(messages); + ? ConversationTextBuilder.BuildWindow(window, numbered: true) + : ConversationTextBuilder.BuildWindow(window, numbered: false); return await _runner.RunAsync( - _options.PreferenceExtractionPrompt ?? DefaultSystemPrompt, + (_options.PreferenceExtractionPrompt ?? DefaultSystemPrompt) + // Only when context is present, so a context-free prompt stays byte-identical (E2). + + (window.HasContext ? ExtractionPromptSemantics.ExtractionContextInstruction : string.Empty), "Extract preferences from this conversation:", conversationText, ProjectPreferences, diff --git a/src/AgentMemory.Extraction.Llm/LlmRelationshipExtractor.cs b/src/AgentMemory.Extraction.Llm/LlmRelationshipExtractor.cs index 7ea3e4777..152c52780 100644 --- a/src/AgentMemory.Extraction.Llm/LlmRelationshipExtractor.cs +++ b/src/AgentMemory.Extraction.Llm/LlmRelationshipExtractor.cs @@ -44,12 +44,18 @@ public LlmRelationshipExtractor( _runner = new LlmExtractionRunner(chatClient, _options, logger); } - protected override async Task> ExtractCoreAsync( + protected override Task> ExtractCoreAsync( IReadOnlyList messages, CancellationToken cancellationToken) + => ExtractCoreWithContextAsync(ExtractionWindow.ForTargets(messages), cancellationToken); + + protected override async Task> ExtractCoreWithContextAsync( + ExtractionWindow window, CancellationToken cancellationToken) { - var conversationText = ConversationTextBuilder.Build(messages); + var conversationText = ConversationTextBuilder.BuildWindow(window, numbered: false); return await _runner.RunAsync( - _options.RelationshipExtractionPrompt ?? DefaultSystemPrompt, + (_options.RelationshipExtractionPrompt ?? DefaultSystemPrompt) + // Only when context is present, so a context-free prompt stays byte-identical (E2). + + (window.HasContext ? ExtractionPromptSemantics.ExtractionContextInstruction : string.Empty), "Extract relationships from this conversation:", conversationText, ProjectRelationships, diff --git a/src/AgentMemory.Extraction.Llm/LlmUnifiedMemoryExtractor.cs b/src/AgentMemory.Extraction.Llm/LlmUnifiedMemoryExtractor.cs index e101d4fa2..2241ca56d 100644 --- a/src/AgentMemory.Extraction.Llm/LlmUnifiedMemoryExtractor.cs +++ b/src/AgentMemory.Extraction.Llm/LlmUnifiedMemoryExtractor.cs @@ -44,22 +44,31 @@ public LlmUnifiedMemoryExtractor( public bool IsEnabled => _options.UseUnifiedExtraction; - public async Task ExtractAsync( + public Task ExtractAsync( IReadOnlyList messages, CancellationToken cancellationToken = default) + => ExtractWithContextAsync(ExtractionWindow.ForTargets(messages), cancellationToken); + + public async Task ExtractWithContextAsync( + ExtractionWindow window, + CancellationToken cancellationToken = default) { - if (messages.Count == 0) + // Judged on the targets: a window carrying only background has nothing to extract from, and + // calling out would spend a completion on turns the prompt forbids extracting from. + if (window.Targets.Count == 0) return new UnifiedExtractionResult(); using var activity = AgentMemoryDiagnostics.Source.StartActivity("memory.extract.unified"); var results = await _runner.RunAsync( BuildSystemPrompt( _options.AssistantContent, _options.EntityTypes, _options.TemporalValidity, - _options.Provenance), + _options.Provenance) + // Appended only when context is actually present, so a context-free prompt stays + // byte-for-byte what every sealed measurement was taken under (E2). + + (window.HasContext ? ExtractionPromptSemantics.ExtractionContextInstruction : string.Empty), "Extract all supported memory from this conversation:", - _options.Provenance == ExtractionProvenanceMode.PerItem - ? ConversationTextBuilder.BuildNumbered(messages) - : ConversationTextBuilder.Build(messages), + ConversationTextBuilder.BuildWindow( + window, numbered: _options.Provenance == ExtractionProvenanceMode.PerItem), response => new[] { Project(response) }, cancellationToken, failOnParseExhaustion: true).ConfigureAwait(false); diff --git a/tests/AgentMemory.Tests.Unit/Extraction/ExtractionWindowTests.cs b/tests/AgentMemory.Tests.Unit/Extraction/ExtractionWindowTests.cs new file mode 100644 index 000000000..005c1a593 --- /dev/null +++ b/tests/AgentMemory.Tests.Unit/Extraction/ExtractionWindowTests.cs @@ -0,0 +1,157 @@ +using AgentMemory.Abstractions.Domain; +using AgentMemory.Abstractions.Options; +using AgentMemory.Abstractions.Services; +using AgentMemory.Core.Extraction; +using FluentAssertions; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using NSubstitute; +using Xunit; + +namespace AgentMemory.Tests.Unit.Extraction; + +/// +/// Extracting with earlier turns as context (E2) — and never extracting from them. +/// +/// +/// +/// One batch is often not enough to understand itself: "I moved there last year" needs the turn that +/// named the place. Widening the batch fixes the reference and breaks something worse. +/// +/// +/// Context must not be a target, and that stopped being a token-efficiency question this week. +/// Re-extracting the preceding turns re-asserts stored facts. That used to be merely wasteful — the +/// exact triple MERGEs — but a re-assertion now earns confidence (S2) and increments the +/// mention_count the salience reranker reads (R7). Extracting context would make a fact gain +/// both every time it sat inside a sliding window, so two signals that mean "the world keeps saying +/// this" would quietly come to mean "this was said recently". Nothing would look broken. +/// +/// +public sealed class ExtractionWindowTests +{ + private static Message M(string id, string content, string role = "user") => new() + { + MessageId = id, + ConversationId = "c-1", + SessionId = "s-1", + Role = role, + Content = content, + TimestampUtc = DateTimeOffset.UnixEpoch, + }; + + // ── rendering ───────────────────────────────────────────────────────── + + [Fact] + public void WithoutContextTheTranscriptIsByteIdentical() + { + // The off state. Every sealed measurement in this track was taken on this rendering, and + // transcript bytes are fingerprinted into each of them. + var targets = new[] { M("m-1", "I live in Zurich"), M("m-2", "Noted.", "assistant") }; + + ConversationTextBuilder.BuildWindow(ExtractionWindow.ForTargets(targets), numbered: false) + .Should().Be(ConversationTextBuilder.Build(targets)); + ConversationTextBuilder.BuildWindow(ExtractionWindow.ForTargets(targets), numbered: true) + .Should().Be(ConversationTextBuilder.BuildNumbered(targets)); + } + + [Fact] + public void ContextIsFencedAheadOfTheTargets() + { + var rendered = ConversationTextBuilder.BuildWindow( + new ExtractionWindow { Targets = [M("m-2", "I moved there last year")], Context = [M("m-1", "Zurich is lovely")] }, + numbered: false); + + rendered.Should().Contain(ConversationTextBuilder.ContextOpen) + .And.Contain("Zurich is lovely") + .And.Contain(ConversationTextBuilder.ContextClose); + rendered.IndexOf(ConversationTextBuilder.ContextClose, StringComparison.Ordinal) + .Should().BeLessThan(rendered.IndexOf("I moved there last year", StringComparison.Ordinal), + "the fence has to close before the turns the model may extract from"); + } + + [Fact] + public void ContextIsNeverNumbered() + { + // THE provenance trap. L3c numbers turns from 1 and resolves turn N positionally to + // Targets[N-1]. If context joined the numbering, every target index would shift by the context + // length -- and the result is not a crash, it is each fact attributed to a turn a few places + // earlier, which afterwards is indistinguishable from precise attribution. + var rendered = ConversationTextBuilder.BuildWindow( + new ExtractionWindow + { + Targets = [M("m-3", "the third")], + Context = [M("m-1", "the first"), M("m-2", "the second")], + }, + numbered: true); + + rendered.Should().Contain("[1] user: the third"); + rendered.Should().NotContain("[1] user: the first"); + rendered.Should().NotContain("[2]"); + } + + // ── the contract ────────────────────────────────────────────────────── + + [Fact] + public void AWindowWithNoContextIsTheDefault() => + ExtractionWindow.ForTargets([M("m-1", "hi")]).HasContext.Should().BeFalse(); + + [Fact] + public void TheOptionIsOffByDefault() => + new ExtractionOptions().ExtractionContextTurns.Should().Be(0); + + [Fact] + public async Task ContextIsKeptOutOfProvenance() + { + // The invariant the whole design exists to protect. An EXTRACTED_FROM edge to a context turn + // would assert the memory was stated in the very turn the extractor was told not to extract + // from. + var stage = new ExtractionStage( + [], [], [], [], [], Substitute.For(), + Options.Create(new ExtractionOptions()), + NullLogger.Instance); + + var result = await stage.ExtractWithContextAsync( + new ExtractionWindow + { + Targets = [M("m-2", "and I moved there")], + Context = [M("m-1", "Zurich is lovely")], + }, + ExtractionTypes.All); + + result.SourceMessageIds.Should().Equal(["m-2"]); + } + + [Fact] + public async Task AnExtractorThatIgnoresContextStillSeesOnlyItsTargets() + { + // The default interface method, exercised on a REAL implementation rather than a substitute: + // a mocking proxy intercepts the default body too, so asserting through one would only prove + // the mock forwards what it was told to. + // + // An extractor written before E2 must keep working, and must receive exactly the turns it + // always received -- not a widened batch it would extract from, which is the failure the whole + // Targets/Context split exists to prevent. + // Through the INTERFACE, not the class: a default interface method is not inherited into the + // implementing type, so this is also how a caller must reach it in production. + IFactExtractor extractor = new LegacyFactExtractor(); + + await extractor.ExtractWithContextAsync( + new ExtractionWindow { Targets = [M("m-2", "target")], Context = [M("m-1", "context")] }); + + ((LegacyFactExtractor)extractor).Seen.Should().ContainSingle() + .Which.MessageId.Should().Be("m-2"); + } + + /// An extractor written before E2 existed: it implements only the original overload. + private sealed class LegacyFactExtractor : IFactExtractor + { + public List Seen { get; } = []; + + public Task> ExtractAsync( + IReadOnlyList messages, CancellationToken cancellationToken = default) + { + Seen.AddRange(messages); + return Task.FromResult>([]); + } + } +} diff --git a/tests/AgentMemory.Tests.Unit/Infrastructure/AbstractionsContractGuardTests.cs b/tests/AgentMemory.Tests.Unit/Infrastructure/AbstractionsContractGuardTests.cs index e21deb25f..eeed387bc 100644 --- a/tests/AgentMemory.Tests.Unit/Infrastructure/AbstractionsContractGuardTests.cs +++ b/tests/AgentMemory.Tests.Unit/Infrastructure/AbstractionsContractGuardTests.cs @@ -18,7 +18,7 @@ public sealed class AbstractionsContractGuardTests // Counts mirrored in docs/architecture.md §3.1 and docs/design.md §5/§6. private const int DocumentedServiceInterfaces = 42; // +IMemoryReranker (PLAN 10.1, seam only) private const int DocumentedRepositoryInterfaces = 11; - private const int DocumentedDomainRecords = 52; // +MemoryContextSectionDiagnostics (PLAN 4.1) + private const int DocumentedDomainRecords = 53; // +ExtractionWindow (PLAN 12.6 / E2) private const int DocumentedEnums = 29; // +TraceKind (7.1), +ExtractionProvenanceMode (9.3) private static IEnumerable PublicTypes() => diff --git a/tests/AgentMemory.Tests.Unit/Services/MemoryServiceExtractionContextTests.cs b/tests/AgentMemory.Tests.Unit/Services/MemoryServiceExtractionContextTests.cs new file mode 100644 index 000000000..bd8024913 --- /dev/null +++ b/tests/AgentMemory.Tests.Unit/Services/MemoryServiceExtractionContextTests.cs @@ -0,0 +1,157 @@ +using AgentMemory.Abstractions.Domain; +using AgentMemory.Abstractions.Options; +using AgentMemory.Abstractions.Repositories; +using AgentMemory.Abstractions.Services; +using AgentMemory.Core.Services; +using FluentAssertions; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using NSubstitute; +using Xunit; + +namespace AgentMemory.Tests.Unit.Services; + +/// +/// The wiring half of E2: the extraction window has to be filled in, once, where every host passes. +/// +/// +/// +/// and its rendering have their own +/// tests, and none of them prove a single context turn ever reaches an extractor. The window would be +/// correct, the prompt instruction would be authored, and every extraction would still see one batch. +/// +/// +/// It is filled at ExtractAndPersistAsync rather than per caller because the Agent Framework +/// provider, the Microsoft memory facade and the MCP ingest tool all arrive there. Resolving it per +/// caller would make the window depend on the host — a setting only some components respect, which is +/// the failure this codebase has hit repeatedly. +/// +/// +public sealed class MemoryServiceExtractionContextTests +{ + private readonly IShortTermMemoryService _shortTerm = Substitute.For(); + private readonly IMemoryExtractionPipeline _pipeline = Substitute.For(); + + private static Message M(string id, string content) => new() + { + MessageId = id, + ConversationId = "c-1", + SessionId = "s-1", + Role = "user", + Content = content, + TimestampUtc = DateTimeOffset.UnixEpoch, + }; + + public MemoryServiceExtractionContextTests() + { + _pipeline.ExtractAsync(Arg.Any(), Arg.Any()) + .Returns(Task.FromResult(new ExtractionResult())); + } + + private MemoryService CreateSut(int contextTurns) => + new(_shortTerm, + Substitute.For(), + _pipeline, + Substitute.For(), + Substitute.For(), + Substitute.For(), + Substitute.For(), + Options.Create(new MemoryOptions + { + Extraction = new ExtractionOptions { ExtractionContextTurns = contextTurns }, + }), + Substitute.For(), + Substitute.For(), + NullLogger.Instance); + + private void SessionHolds(params Message[] messages) => + _shortTerm.GetRecentMessagesAsync(Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(Task.FromResult>(messages)); + + private ExtractionRequest CapturedRequest() => + (ExtractionRequest)_pipeline.ReceivedCalls() + .Single(c => c.GetMethodInfo().Name == nameof(IMemoryExtractionPipeline.ExtractAsync)) + .GetArguments()[0]!; + + private Task ExtractAsync(int contextTurns, params Message[] targets) => + CreateSut(contextTurns).ExtractAndPersistAsync( + new ExtractionRequest { Messages = targets, SessionId = "s-1" }); + + [Fact] + public async Task PrecedingTurnsAreAttachedAsContext() + { + // THE test. Everything else in E2 is arrangement around these turns reaching the extractor. + SessionHolds(M("m-1", "Zurich is lovely"), M("m-2", "Agreed."), M("m-3", "I moved there")); + + await ExtractAsync(2, M("m-3", "I moved there")); + + CapturedRequest().ContextMessages.Select(m => m.MessageId).Should().Equal("m-1", "m-2"); + } + + [Fact] + public async Task TheBatchItselfIsNeverItsOwnContext() + { + // The recency query returns the batch too -- it was just persisted. Leaving it in would hand + // the model the same turn twice, the second copy labelled "do not extract from this". + SessionHolds(M("m-1", "earlier"), M("m-2", "target")); + + await ExtractAsync(4, M("m-2", "target")); + + CapturedRequest().ContextMessages.Select(m => m.MessageId).Should().Equal("m-1"); + } + + [Fact] + public async Task OnlyTheMostRecentTurnsAreKept() + { + SessionHolds(M("m-1", "oldest"), M("m-2", "older"), M("m-3", "old"), M("m-4", "target")); + + await ExtractAsync(2, M("m-4", "target")); + + // Note the collection overload: Equal("m-2", "m-3", "because...") would fold the reason into + // the expected array as a third element. + CapturedRequest().ContextMessages.Select(m => m.MessageId).Should().Equal( + ["m-2", "m-3"], + "context is the turns immediately before the batch, not the start of the session"); + } + + [Fact] + public async Task ZeroTurnsQueriesNothingAtAll() + { + // The off state, and it must be off at the source: a fetch whose result is then discarded + // still costs a round trip on every single extraction. + await ExtractAsync(0, M("m-2", "target")); + + await _shortTerm.DidNotReceiveWithAnyArgs() + .GetRecentMessagesAsync(default!, default, default); + CapturedRequest().ContextMessages.Should().BeEmpty(); + } + + [Fact] + public async Task AnExplicitContextIsNotOverwritten() + { + // A caller that knows the conversation better than a recency query does should not be + // second-guessed. + SessionHolds(M("m-1", "from the store"), M("m-9", "target")); + + await CreateSut(4).ExtractAndPersistAsync(new ExtractionRequest + { + Messages = [M("m-9", "target")], + SessionId = "s-1", + ContextMessages = [M("m-7", "chosen by the caller")], + }); + + CapturedRequest().ContextMessages.Select(m => m.MessageId).Should().Equal("m-7"); + } + + [Fact] + public async Task NoEarlierTurnsLeavesTheRequestUntouched() + { + // First turn of a session: nothing precedes it, and an empty context must stay empty rather + // than becoming a fenced block with nothing inside it. + SessionHolds(M("m-1", "target")); + + await ExtractAsync(4, M("m-1", "target")); + + CapturedRequest().ContextMessages.Should().BeEmpty(); + } +} From 55bbcd0c0db50971ad184c277540b2fb0cdd719a Mon Sep 17 00:00:00 2001 From: Jose Luis Latorre Millas Date: Wed, 12 Aug 2026 22:31:14 +0200 Subject: [PATCH 23/31] Let the graph probe see procedural memory (PLAN 6.5) LongMemEvalGraphProbe counted Entity, Fact, Preference and RELATED_TO, and was label-blind to :ReasoningTrace. So "the corpus contains no traces" and "the probe cannot see traces" produced byte-identical output, and Phase 7's procedural work was about to be measured against a graph nobody had confirmed held anything to measure. The probe now counts traces, and procedures separately from traces in general -- Phase 7 promotes procedures specifically, and a lumped count would report a corpus as procedure-bearing when it holds only episodes. TotalLearned deliberately still excludes traces. It appears in every measurement sealed before the probe could see them, and quietly widening its definition would make every prior build appear to have grown for a reason unrelated to what was extracted. The counts are nullable rather than defaulting to zero. A manifest written before this change has no such field, and a non-nullable int would deserialize it to 0 -- reproducing in the recorded data the exact ambiguity this task exists to remove. TracesMeasured separates "looked and found none" from "never looked", the same unknown-reading-as- agreement trap caught earlier in the preparation manifests. 396 LongMemEval tests green (+5). Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01PgDgctPpbTziBNE2RT8VdE --- ...emEvalGraphSnapshotTraceVisibilityTests.cs | 66 +++++++++++++++++++ .../LongMemEvalGraphProbe.cs | 49 +++++++++++++- 2 files changed, 112 insertions(+), 3 deletions(-) create mode 100644 tests/AgentMemory.Tests.Unit.LongMemEval/LongMemEvalGraphSnapshotTraceVisibilityTests.cs diff --git a/tests/AgentMemory.Tests.Unit.LongMemEval/LongMemEvalGraphSnapshotTraceVisibilityTests.cs b/tests/AgentMemory.Tests.Unit.LongMemEval/LongMemEvalGraphSnapshotTraceVisibilityTests.cs new file mode 100644 index 000000000..2ef28120a --- /dev/null +++ b/tests/AgentMemory.Tests.Unit.LongMemEval/LongMemEvalGraphSnapshotTraceVisibilityTests.cs @@ -0,0 +1,66 @@ +using AgentMemory.LongMemEval; +using FluentAssertions; +using Xunit; + +namespace AgentMemory.Tests.Unit.LongMemEval; + +/// +/// Making the probe able to see procedural memory at all (PLAN 6.5). +/// +/// +/// +/// LongMemEvalGraphProbe counted Entity, Fact, Preference and RELATED_TO. It was +/// label-blind to :ReasoningTrace, which meant "the corpus contains no traces" and "the +/// probe cannot see traces" produced byte-identical output — and Phase 7's procedural work was about +/// to be measured against a graph nobody had confirmed held anything to measure. +/// +/// +public sealed class LongMemEvalGraphSnapshotTraceVisibilityTests +{ + private static LongMemEvalGraphSnapshot Snapshot(int? traces = null, int? procedures = null) => + new(Entities: 10, Facts: 20, Preferences: 5, Relationships: 3, + RelationshipsWithProvenance: 3, LearnedItems: 35, LearnedItemsWithProvenance: 35, + ProvenanceEdges: 40, SourceMessages: 12, + ReasoningTraces: traces, Procedures: procedures); + + [Fact] + public void TotalLearnedStillExcludesTraces() + { + // Comparability. This number appears in every measurement sealed before the probe could see + // traces; widening its definition would make every prior build appear to have grown for a + // reason unrelated to what was extracted. + Snapshot(traces: 7).TotalLearned.Should().Be(38); + } + + [Fact] + public void TracesAreCountedSeparately() => + Snapshot(traces: 7).TotalIncludingTraces.Should().Be(45); + + [Fact] + public void NotMeasuredIsNotTheSameAsMeasuredZero() + { + // THE distinction, and the reason the counts are nullable. A manifest written before this + // change has no such field, and a non-nullable int would deserialize it to 0 -- reproducing in + // the recorded data the exact ambiguity 6.5 exists to remove. + var legacy = Snapshot(traces: null); + var measured = Snapshot(traces: 0); + + legacy.TracesMeasured.Should().BeFalse(); + legacy.HasNoTraces.Should().BeFalse("a probe that never looked has not found the corpus empty"); + + measured.TracesMeasured.Should().BeTrue(); + measured.HasNoTraces.Should().BeTrue(); + } + + [Fact] + public void AnUnmeasuredTraceCountDoesNotInflateTheTotal() => + Snapshot(traces: null).TotalIncludingTraces.Should().Be(38); + + [Fact] + public void ProceduresAreCountedApartFromTracesInGeneral() + { + // Phase 7 promotes procedures specifically; a total trace count that lumped episodes in with + // them would report a corpus as procedure-bearing when it holds none. + Snapshot(traces: 7, procedures: 2).Procedures.Should().Be(2); + } +} diff --git a/tools/AgentMemory.LongMemEval/LongMemEvalGraphProbe.cs b/tools/AgentMemory.LongMemEval/LongMemEvalGraphProbe.cs index e226115d9..bcd3c09b2 100644 --- a/tools/AgentMemory.LongMemEval/LongMemEvalGraphProbe.cs +++ b/tools/AgentMemory.LongMemEval/LongMemEvalGraphProbe.cs @@ -140,6 +140,11 @@ RETURN count(f) AS facts MATCH (p:Preference {owner_id: $ownerId}) RETURN count(p) AS preferences } + CALL { + MATCH (t:ReasoningTrace {owner_id: $ownerId}) + RETURN count(t) AS reasoningTraces, + count(CASE WHEN t.trace_kind = 'Procedure' THEN 1 END) AS procedures + } CALL { MATCH ()-[r:RELATED_TO]->() WHERE r.owner_id = $ownerId @@ -158,7 +163,7 @@ RETURN count(DISTINCT n) AS learnedItems, } RETURN entities, facts, preferences, relationships, relationshipsWithProvenance, learnedItems, learnedItemsWithProvenance, - provenanceEdges, sourceMessages + provenanceEdges, sourceMessages, reasoningTraces, procedures """; private const string GoldCoverageQuery = @@ -252,7 +257,9 @@ public async Task ReadAsync( record["learnedItems"].As(), record["learnedItemsWithProvenance"].As(), record["provenanceEdges"].As(), - record["sourceMessages"].As()); + record["sourceMessages"].As(), + record["reasoningTraces"].As(), + record["procedures"].As()); }).ConfigureAwait(false); } } @@ -286,10 +293,46 @@ public sealed record LongMemEvalGraphSnapshot( int LearnedItems, int LearnedItemsWithProvenance, int ProvenanceEdges, - int SourceMessages) + int SourceMessages, + int? ReasoningTraces = null, + int? Procedures = null) { + /// + /// Entity + Fact + Preference + relationship count, deliberately excluding traces. + /// + /// + /// Traces are counted separately rather than folded in here. This number appears in every sealed + /// measurement taken before the probe could see :ReasoningTrace at all, and quietly + /// widening its definition would move recorded totals for a reason that has nothing to do with + /// what was extracted — every prior build would appear to have grown. + /// public int TotalLearned => Entities + Facts + Preferences + Relationships; + /// Everything the probe can see, traces included. + public int TotalIncludingTraces => TotalLearned + (ReasoningTraces ?? 0); + + /// + /// when this snapshot actually looked for traces. + /// + /// + /// Nullable rather than defaulted to 0, deliberately. A manifest written before the probe + /// could see :ReasoningTrace has no such field, and a non-nullable int would + /// deserialize it to zero — reproducing, in the recorded data, the exact ambiguity 6.5 exists to + /// remove: "measured, and there are none" would be indistinguishable from "never looked". + /// + public bool TracesMeasured => ReasoningTraces.HasValue; + + /// + /// when the corpus holds no procedural/episodic memory at all. + /// + /// + /// The finding 6.5 exists to make visible. The probe was label-blind to :ReasoningTrace, so + /// "the corpus contains no traces" and "the probe cannot see traces" produced identical output — + /// and Phase 7's procedural work would have been measured against a graph nobody had confirmed + /// contained anything to measure. + /// + public bool HasNoTraces => ReasoningTraces == 0; + public bool CompleteProvenance => LearnedItemsWithProvenance == LearnedItems && RelationshipsWithProvenance == Relationships; From 6b6c99527615ca2557e208c9f3c1be48b72e5435 Mon Sep 17 00:00:00 2001 From: Jose Luis Latorre Millas Date: Wed, 12 Aug 2026 22:33:08 +0200 Subject: [PATCH 24/31] Measure whether the retrieved procedure is the right one (PLAN 7.7) An agent with no procedural memory investigates: slower, and safe. An agent with the wrong procedure executes -- confidently, on a plan built for a different task. Every efficiency measure a promotion feature has (hit rate, steps, tool calls, latency) improves when the retriever becomes more willing to answer, including when it becomes more willing to answer wrongly. Latency alone cannot tell those apart. ProcedureRetrievalPrecision therefore reports three outcomes rather than an accuracy: correct, wrong, and abstained. Abstention is deliberately not a failure -- it is the safe outcome, and folding it in with wrong answers makes a cautious retriever indistinguishable from a reckless one. That is the distinction Phase 5 drew for meta-memory, and it counts for more here because acting on a wrong procedure is paid in tool calls rather than in a sentence. WrongProcedureRate is the figure a promotion change has to be judged against. PrecisionWhenAnswering is reported beside PrecisionAtOne because the two move in opposite directions as a retriever gets more cautious, and either alone hides the trade. Retrieving anything for a task no stored procedure fits counts as wrong, not as a near miss: that is exactly the case where an agent executes a plan for a different problem. Deterministic and provider-free -- it scores id lists, so it is asserted in unit tests rather than measured against a model. Its consumer is 7.6's harness, still to be built. 403 LongMemEval tests green (+7). Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01PgDgctPpbTziBNE2RT8VdE --- .../ProcedureRetrievalPrecisionTests.cs | 113 +++++++++++++++++ .../ProcedureRetrievalPrecision.cs | 118 ++++++++++++++++++ 2 files changed, 231 insertions(+) create mode 100644 tests/AgentMemory.Tests.Unit.LongMemEval/ProcedureRetrievalPrecisionTests.cs create mode 100644 tools/AgentMemory.LongMemEval/ProcedureRetrievalPrecision.cs diff --git a/tests/AgentMemory.Tests.Unit.LongMemEval/ProcedureRetrievalPrecisionTests.cs b/tests/AgentMemory.Tests.Unit.LongMemEval/ProcedureRetrievalPrecisionTests.cs new file mode 100644 index 000000000..593cdf3a9 --- /dev/null +++ b/tests/AgentMemory.Tests.Unit.LongMemEval/ProcedureRetrievalPrecisionTests.cs @@ -0,0 +1,113 @@ +using AgentMemory.LongMemEval; +using FluentAssertions; +using Xunit; + +namespace AgentMemory.Tests.Unit.LongMemEval; + +/// +/// Whether the retrieved procedure was the right one, not merely a fast one (PLAN 7.7). +/// +/// +/// +/// An agent with no procedural memory investigates; an agent with the wrong one executes. The +/// second is slower to detect and more expensive to undo, and every efficiency measure a promotion +/// feature has — hit rate, steps, latency — improves when the retriever becomes more willing to +/// answer, including when it becomes more willing to answer wrongly. +/// +/// +public sealed class ProcedureRetrievalPrecisionTests +{ + private static ProcedureRetrievalCase Case( + string taskId, string[] retrieved, params string[] correct) => + new(taskId, retrieved, correct); + + [Fact] + public void TheTopResultIsWhatCounts() + { + // An agent acts on the first procedure it is handed. A correct one at rank 3 is a ranking + // signal, not a success -- the agent already executed the wrong plan. + var scored = ProcedureRetrievalPrecision.Score( + [Case("t-1", ["wrong-a", "wrong-b", "right"], "right")]); + + scored.CorrectAtOne.Should().Be(0); + scored.WrongAtOne.Should().Be(1); + scored.MeanReciprocalRank.Should().BeApproximately(1d / 3, 1e-9); + } + + [Fact] + public void AbstentionIsNotAFailure() + { + // THE distinction. Returning nothing makes the agent investigate: slower, and safe. Counting + // it with wrong answers would make a cautious retriever indistinguishable from a reckless one. + var scored = ProcedureRetrievalPrecision.Score([Case("t-1", [], "right")]); + + scored.Abstained.Should().Be(1); + scored.WrongAtOne.Should().Be(0); + scored.WrongProcedureRate.Should().Be(0); + } + + [Fact] + public void AnsweringWhenNothingAppliesIsWrong() + { + // No stored procedure fits this task, so any confident retrieval is the error -- this is the + // case where an agent executes a plan built for a different problem entirely. + var scored = ProcedureRetrievalPrecision.Score([Case("t-1", ["some-procedure"])]); + + scored.WrongAtOne.Should().Be(1); + scored.MeanReciprocalRank.Should().Be(0); + } + + [Fact] + public void PrecisionAndPrecisionWhenAnsweringMoveApart() + { + // Why both are reported. A retriever that answers 2 of 4 tasks and gets both right has a + // precision@1 of 0.5 and is perfectly accurate whenever it commits. Reporting either number + // alone hides the trade the caution is buying. + var scored = ProcedureRetrievalPrecision.Score( + [ + Case("t-1", ["right-1"], "right-1"), + Case("t-2", ["right-2"], "right-2"), + Case("t-3", []), + Case("t-4", [], "right-4"), + ]); + + scored.PrecisionAtOne.Should().Be(0.5); + scored.PrecisionWhenAnswering.Should().Be(1.0); + scored.AbstentionRate.Should().Be(0.5); + } + + [Fact] + public void MoreThanOneProcedureCanBeCorrect() + { + // Tasks legitimately have several valid approaches; scoring against a single gold id would + // count a correct alternative as a safety failure. + ProcedureRetrievalPrecision.Score([Case("t-1", ["b"], "a", "b")]) + .CorrectAtOne.Should().Be(1); + } + + [Fact] + public void AnEmptyRunScoresZeroRatherThanDividingByZero() + { + var scored = ProcedureRetrievalPrecision.Score([]); + + scored.Total.Should().Be(0); + scored.PrecisionAtOne.Should().Be(0); + scored.WrongProcedureRate.Should().Be(0); + scored.PrecisionWhenAnswering.Should().Be(0); + } + + [Fact] + public void TheThreeOutcomesAccountForEveryTask() + { + // No task may fall through the classification: a total that does not reconcile would let a + // silently dropped case look like an improved wrong-procedure rate. + var scored = ProcedureRetrievalPrecision.Score( + [ + Case("t-1", ["right"], "right"), + Case("t-2", ["wrong"], "right"), + Case("t-3", []), + ]); + + (scored.CorrectAtOne + scored.WrongAtOne + scored.Abstained).Should().Be(scored.Total); + } +} diff --git a/tools/AgentMemory.LongMemEval/ProcedureRetrievalPrecision.cs b/tools/AgentMemory.LongMemEval/ProcedureRetrievalPrecision.cs new file mode 100644 index 000000000..9892f3728 --- /dev/null +++ b/tools/AgentMemory.LongMemEval/ProcedureRetrievalPrecision.cs @@ -0,0 +1,118 @@ +namespace AgentMemory.LongMemEval; + +/// +/// What a single task asked for and what procedural memory returned (PLAN 7.7). +/// +/// The task the agent was given. +/// Procedures returned, best-ranked first. Empty means none. +/// +/// Procedures that genuinely apply to this task. Empty means no stored procedure applies, so +/// any retrieval is wrong however confident it looked. +/// +public sealed record ProcedureRetrievalCase( + string TaskId, + IReadOnlyList RetrievedProcedureIds, + IReadOnlyList CorrectProcedureIds); + +/// +/// How often the retrieved procedure is the right one — not merely how fast it arrived +/// (PLAN 7.7). +/// +/// +/// +/// The two failure modes are not equally bad, and a latency number cannot tell them apart. An +/// agent with no procedural memory investigates: it is slower, and it is safe. An agent with the +/// wrong procedure executes — confidently, on a plan built for a different task. A promotion +/// feature that raises hit-rate while also raising the wrong-procedure rate can look like an +/// improvement in every efficiency measure it has. +/// +/// +/// So this reports three outcomes rather than an accuracy: correct, wrong, and abstained. +/// Abstention is deliberately not counted as a failure — it is the safe outcome, and folding it in +/// with wrong answers would make a cautious retriever look identical to a reckless one. That is the +/// same distinction Phase 5 drew for meta-memory, and it matters more here, because the cost of +/// acting on a wrong procedure is paid in tool calls rather than in a sentence. +/// +/// +/// Deterministic and provider-free: it scores id lists, so it can be asserted in a unit test rather +/// than measured against a model. +/// +/// +public sealed record ProcedureRetrievalPrecision( + int Total, + int CorrectAtOne, + int WrongAtOne, + int Abstained, + double MeanReciprocalRank) +{ + /// Share of tasks whose best-ranked procedure was a correct one. + public double PrecisionAtOne => Total == 0 ? 0d : (double)CorrectAtOne / Total; + + /// + /// Share of tasks where a procedure was returned and it was the wrong one — the safety + /// number. + /// + /// + /// This is the figure a promotion change has to be judged against. Hit-rate and latency both + /// improve when the retriever becomes more willing to answer; only this one gets worse. + /// + public double WrongProcedureRate => Total == 0 ? 0d : (double)WrongAtOne / Total; + + /// Share of tasks where nothing was returned — slower, and safe. + public double AbstentionRate => Total == 0 ? 0d : (double)Abstained / Total; + + /// + /// Share of the tasks it chose to answer that it answered correctly. + /// + /// + /// Separated from because the two move in opposite directions as a + /// retriever gets more cautious, and reporting only one of them would make the trade invisible. + /// + public double PrecisionWhenAnswering => + CorrectAtOne + WrongAtOne == 0 ? 0d : (double)CorrectAtOne / (CorrectAtOne + WrongAtOne); + + /// + /// Scores a set of tasks. A case whose + /// is empty counts any retrieval as wrong: nothing stored applies, so answering is the error. + /// + public static ProcedureRetrievalPrecision Score(IReadOnlyList cases) + { + ArgumentNullException.ThrowIfNull(cases); + + var correct = 0; + var wrong = 0; + var abstained = 0; + var reciprocalRankTotal = 0d; + + foreach (var item in cases) + { + if (item.RetrievedProcedureIds.Count == 0) + { + abstained++; + continue; + } + + var expected = new HashSet(item.CorrectProcedureIds, StringComparer.Ordinal); + + if (expected.Contains(item.RetrievedProcedureIds[0])) correct++; + else wrong++; + + // Rank of the first correct procedure. Reported alongside precision@1 rather than instead + // of it: an agent acts on the top result, so a correct procedure sitting at rank 3 is a + // ranking signal, not a success. + for (var rank = 0; rank < item.RetrievedProcedureIds.Count; rank++) + { + if (!expected.Contains(item.RetrievedProcedureIds[rank])) continue; + reciprocalRankTotal += 1d / (rank + 1); + break; + } + } + + return new ProcedureRetrievalPrecision( + cases.Count, + correct, + wrong, + abstained, + cases.Count == 0 ? 0d : reciprocalRankTotal / cases.Count); + } +} From 9ff97c4eebc22d9af9a1e9065bc99affd6f630ca Mon Sep 17 00:00:00 2001 From: Jose Luis Latorre Millas Date: Wed, 12 Aug 2026 22:36:54 +0200 Subject: [PATCH 25/31] Make the StructuredJson judge protocol reachable and self-declaring (3.7) AgentEval's free-text verdict parser vetoes a leading "yes" when the word "no" appears later, so a judge answering "yes -- there is no discrepancy" is scored as a failure. That is a systematic mis-scoring and StructuredJson is the actual fix. The plan recorded this as "wired and reachable". It was not: the protocol parameter existed on CreateOptions and every call site took the default, so no run could ever select it -- the dead-option shape, found for the fourth time in this track. Now selectable via --judge-protocol, defaulting to free-text. The default deliberately does not move: AgentEval's own docs say results under StructuredJson are not comparable with a free-text base, and every sealed base here is free-text. Flipping it would not produce a wrong number, it would produce a better one that silently invalidates every comparison anybody draws against the existing runs. An unrecognised value throws instead of falling back. A silent fallback would mean a run the operator believed was StructuredJson produced a free-text score under a StructuredJson label -- worse than either, and invisible afterwards. The choice is emitted unconditionally into the report's protocol block, beside the extraction vocabulary and assistant-content fingerprints, so a StructuredJson score cannot sit next to a FreeText one without the comparability break being visible in the artifact itself. The remaining half -- running it on a fresh base -- is provider spend, the same gate as 4.2 and 8.3b. 4,312 unit + 410 LongMemEval tests green (+7). Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01PgDgctPpbTziBNE2RT8VdE --- .../JudgeVerdictProtocolWiringTests.cs | 77 +++++++++++++++++++ .../LongMemEvalPreparedPairProgram.cs | 39 +++++++++- 2 files changed, 113 insertions(+), 3 deletions(-) create mode 100644 tests/AgentMemory.Tests.Unit.LongMemEval/JudgeVerdictProtocolWiringTests.cs diff --git a/tests/AgentMemory.Tests.Unit.LongMemEval/JudgeVerdictProtocolWiringTests.cs b/tests/AgentMemory.Tests.Unit.LongMemEval/JudgeVerdictProtocolWiringTests.cs new file mode 100644 index 000000000..c33e1a9ba --- /dev/null +++ b/tests/AgentMemory.Tests.Unit.LongMemEval/JudgeVerdictProtocolWiringTests.cs @@ -0,0 +1,77 @@ +using AgentEval.Memory.External.LongMemEval; +using AgentEval.Memory.External.Models; +using AgentMemory.LongMemEval; +using FluentAssertions; +using Xunit; + +namespace AgentMemory.Tests.Unit.LongMemEval; + +/// +/// Making StructuredJson reachable — and impossible to use invisibly (PLAN 3.7). +/// +/// +/// +/// AgentEval's free-text verdict parser "vetoes a leading yes when the word no appears later in +/// the response", so a judge answering "yes — there is no discrepancy" is scored as a failure. +/// That is a systematic mis-scoring, and StructuredJson is the actual fix. +/// +/// +/// It was also unreachable. The protocol parameter existed on CreateOptions and every +/// call site took the default, so no run could ever select it — the dead-option shape. +/// +/// +/// The default must not move. The same AgentEval docs state results under StructuredJson are +/// not comparable with a free-text base, and every sealed base in this track is free-text. Flipping +/// the default would not produce a wrong number; it would produce a better one that silently +/// invalidates every comparison drawn against the existing runs. +/// +/// +public sealed class JudgeVerdictProtocolWiringTests +{ + private static JudgeVerdictProtocol Parse(string? value) => + LongMemEvalPreparedPairProgram.ParseJudgeProtocolForTests(value); + + [Fact] + public void TheDefaultIsFreeText() + { + // Every sealed base here was scored under FreeText. This is the assertion that stops a later + // change making the numbers better and the comparisons meaningless. + Parse(null).Should().Be(JudgeVerdictProtocol.FreeText); + Parse("").Should().Be(JudgeVerdictProtocol.FreeText); + } + + [Theory] + [InlineData("structured-json")] + [InlineData("structuredjson")] + [InlineData("json")] + [InlineData("STRUCTURED-JSON")] + public void StructuredJsonIsReachable(string value) => + Parse(value).Should().Be(JudgeVerdictProtocol.StructuredJson); + + [Fact] + public void AnUnknownProtocolIsRejectedRatherThanDefaulted() + { + // Silently falling back to FreeText would mean a run the operator believed was StructuredJson + // produced a free-text score under a StructuredJson label -- the worst of both, and invisible. + var act = () => Parse("strict"); + + act.Should().Throw().WithMessage("*free-text, structured-json*"); + } + + [Fact] + public void TheBenchmarkOptionsCarryTheChoiceThrough() + { + // The wiring, not the parsing: a protocol parsed and then dropped before CreateOptions would + // pass every test above while every run stayed free-text. + var options = LongMemEvalBenchmarkProtocol.CreateOptions( + datasetPath: "dataset.json", + questions: 10, + seed: 42, + judgeRetryAttempts: 2, + evidenceDetail: LongMemEvalEvidenceDetail.Identifiers, + maxRelevantMessages: 20, + verdictProtocol: JudgeVerdictProtocol.StructuredJson); + + options.JudgeVerdictProtocol.Should().Be(JudgeVerdictProtocol.StructuredJson); + } +} diff --git a/tools/AgentMemory.LongMemEval/LongMemEvalPreparedPairProgram.cs b/tools/AgentMemory.LongMemEval/LongMemEvalPreparedPairProgram.cs index df0b52b7a..5189ce99f 100644 --- a/tools/AgentMemory.LongMemEval/LongMemEvalPreparedPairProgram.cs +++ b/tools/AgentMemory.LongMemEval/LongMemEvalPreparedPairProgram.cs @@ -79,6 +79,7 @@ internal static async Task RunAsync(string[] args) options.JudgeRetryAttempts, options.EvidenceDetail, options.MaxRelevantMessages, + options.JudgeProtocol, // Typed sampling reaches the PREPARATION, not only the evaluation: an episodic-only // sample selects different questions, whose conversation histories have to be ingested // for the corpus to answer them at all. This is why an episodic arm cannot reuse a @@ -879,6 +880,14 @@ await LongMemEvalOrphanSweep evidenceDetail = options.EvidenceDetail.ToString().ToLowerInvariant(), oracleMode = options.OracleMode.ToString().ToLowerInvariant(), judgeRetryAttempts = options.JudgeRetryAttempts, + // 3.7. A COMPARABILITY BREAK, recorded like one. StructuredJson fixes a + // systematic free-text mis-scoring -- the parser vetoes a leading "yes" when the + // word "no" appears later, so "there is no discrepancy" scores as a failure -- but + // AgentEval's own docs say results under it are not comparable with a free-text + // base, and every sealed base here is free-text. Emitted unconditionally so a + // StructuredJson score can never be read beside a FreeText one without the + // difference being visible in the artifact itself. + judgeVerdictProtocol = options.JudgeProtocol.ToString(), neo4jImage = "neo4j:5.26", agentEval = agentEvalRevision, agentEvalDependency = "source-project:AgentEval.Memory" @@ -1421,7 +1430,7 @@ private static string AgentEvalRevision() "--reuse-prepared-volumes", "--seed", "--single-session-unified", "--description", "--memory-types", "--allow-stale-prepared", "--abstention", "--abstention-proportion", - "--use-predicate-vocabulary", + "--use-predicate-vocabulary", "--judge-protocol", ]; private static PreparedPairOptions Parse(string[] args) @@ -1490,7 +1499,8 @@ bool Has(string name) => ParseMemoryTypes(Value("--memory-types")), Has("--allow-stale-prepared"), ParseAbstention(Value("--abstention")), - ParseAbstentionProportion(Value("--abstention-proportion"))); + ParseAbstentionProportion(Value("--abstention-proportion")), + ParseJudgeProtocol(Value("--judge-protocol"))); } /// @@ -1529,6 +1539,28 @@ private static IReadOnlyList ParseMemoryTypes(string? value) /// sample contains changes every number computed from it. /// /// + /// + /// Parses --judge-protocol. Defaults to FreeText — the protocol every sealed base + /// here was scored under. + /// + /// + /// Never flipped silently. StructuredJson is the real fix for a systematic free-text + /// mis-scoring, and AgentEval's own documentation says results under it are not comparable with a + /// free-text base. Changing the default would not produce a wrong number; it would produce a + /// better one that silently invalidates every comparison anybody makes against the existing runs. + /// + private static JudgeVerdictProtocol ParseJudgeProtocol(string? value) => value?.ToLowerInvariant() switch + { + null or "" or "free-text" or "freetext" => JudgeVerdictProtocol.FreeText, + "structured-json" or "structuredjson" or "json" => JudgeVerdictProtocol.StructuredJson, + _ => throw new ArgumentException( + "--judge-protocol must be one of: free-text, structured-json."), + }; + + /// Test seam for the parser above; the method itself stays private. + internal static JudgeVerdictProtocol ParseJudgeProtocolForTests(string? value) => + ParseJudgeProtocol(value); + private static AbstentionSamplingPolicy ParseAbstention(string? value) => value?.ToLowerInvariant() switch { @@ -1811,7 +1843,8 @@ internal sealed record PreparedPairOptions( IReadOnlyList? MemoryTypesRequested = null, bool AllowStalePrepared = false, AbstentionSamplingPolicy AbstentionPolicy = AbstentionSamplingPolicy.AsSampled, - double? AbstentionProportion = null) + double? AbstentionProportion = null, + JudgeVerdictProtocol JudgeProtocol = JudgeVerdictProtocol.FreeText) { /// The memory types this corpus was sampled for; empty means every type. internal IReadOnlyList MemoryTypes => MemoryTypesRequested ?? []; From 1cfea438920a3eda139bbffaa1a70385009c6aa5 Mon Sep 17 00:00:00 2001 From: Jose Luis Latorre Millas Date: Wed, 12 Aug 2026 22:52:12 +0200 Subject: [PATCH 26/31] Synthesize entity summaries that cannot go stale unnoticed (S1) Recall about a well-known entity returns twenty facts that each spend context to say one thing. A summary says the same in one item. But a summary is derived memory, and derived memory is where a store quietly starts lying: the sources change, the summary does not, and afterwards nothing about it looks any different. So staleness is proved rather than assumed. Each summary carries a fingerprint over the exact facts it was written from -- id, confidence and invalidation, order-independent so a query-plan change cannot invalidate the whole store without a fact having moved. Before a summary is used the fingerprint is recomputed and compared; a mismatch means it is not returned at all. Not returned with an IsStale flag: a flag puts the decision in every caller's hands, and one caller forgetting to check is indistinguishable from correct memory. Detection on read rather than invalidation on write, deliberately. A write-side sweep has to find every affected summary, has to run in the same transaction, and leaves a summary looking current whenever any path writes a fact without going through it. Recomputing on read cannot be bypassed by a writer that did not know summaries existed. Confidence is in the fingerprint because S2 reinforcement moves it, and a summary stating flatly what the store has since grown doubtful about is precisely the stale shadow this design exists to prevent. Found on the way: invalidated_at has always been recorded in the store -- supersession is non-destructive so as-of recall can reach it -- but was never projected onto the Fact record, so no caller could tell a superseded fact from a live one. Now surfaced as InvalidatedAtUtc. The default synthesizer is deterministic and makes no model call. Summaries are regenerated whenever any source moves, so a completion per entity per change would scale cost with exactly how much the conversation talks about its subjects. Reproducible output also means a change in the text always signals a change in the facts. Summary nodes keep their own EXTRACTED_FROM edges, rebuilt wholesale on regeneration -- a summary rebuilt from fewer sources must not keep claiming provenance its content can no longer support. 4,329 unit (+17) and 352 integration (+7) green. Release 0 warnings. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01PgDgctPpbTziBNE2RT8VdE --- docs/architecture.md | 2 +- .../Domain/LongTerm/EntitySummary.cs | 107 ++++++++ .../Domain/LongTerm/Fact.cs | 18 ++ .../Repositories/IEntitySummaryRepository.cs | 33 +++ .../Services/IEntitySummaryService.cs | 37 +++ .../Services/IEntitySummarySynthesizer.cs | 34 +++ .../DeterministicEntitySummarySynthesizer.cs | 87 +++++++ .../Memory/EntitySummaryService.cs | 134 ++++++++++ .../ServiceCollectionExtensions.cs | 8 + .../ServiceCollectionExtensions.cs | 4 + .../Queries/EntitySummaryQueries.cs | 76 ++++++ .../Neo4jEntitySummaryRepository.cs | 107 ++++++++ .../Repositories/Neo4jFactRepository.cs | 3 + .../EntitySummaryIntegrationTests.cs | 171 +++++++++++++ .../AbstractionsContractGuardTests.cs | 6 +- .../Memory/EntitySummaryTests.cs | 238 ++++++++++++++++++ .../Queries/CypherQuerySnapshot.snap | 26 +- .../Queries/CypherQuerySnapshotTests.cs | 5 +- 18 files changed, 1089 insertions(+), 7 deletions(-) create mode 100644 src/AgentMemory.Abstractions/Domain/LongTerm/EntitySummary.cs create mode 100644 src/AgentMemory.Abstractions/Repositories/IEntitySummaryRepository.cs create mode 100644 src/AgentMemory.Abstractions/Services/IEntitySummaryService.cs create mode 100644 src/AgentMemory.Abstractions/Services/IEntitySummarySynthesizer.cs create mode 100644 src/AgentMemory.Core/Memory/DeterministicEntitySummarySynthesizer.cs create mode 100644 src/AgentMemory.Core/Memory/EntitySummaryService.cs create mode 100644 src/AgentMemory.Neo4j/Queries/EntitySummaryQueries.cs create mode 100644 src/AgentMemory.Neo4j/Repositories/Neo4jEntitySummaryRepository.cs create mode 100644 tests/AgentMemory.Tests.Integration/Repositories/EntitySummaryIntegrationTests.cs create mode 100644 tests/AgentMemory.Tests.Unit/Memory/EntitySummaryTests.cs diff --git a/docs/architecture.md b/docs/architecture.md index aafb83f2a..6db669163 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -142,7 +142,7 @@ graph TD | **Purpose** | Domain contracts — all models, interfaces, and configuration types shared across the system | | **Dependencies** | **Microsoft.Extensions.AI.Abstractions** 10.8.0 (approved, D-AR2-1) — .NET BCL otherwise (multi-targets net8.0/net9.0/net10.0) | | **MUST NOT reference** | Neo4j.Driver, Microsoft.Agents.*, any GraphRAG SDK, any MCP SDK, any NuGet package **except** Microsoft.Extensions.AI.Abstractions | -| **Key types** | 53 domain records (Conversation, Message, Entity, Fact, Preference, Relationship, MemoryHistoryQuery, MemoryHistoryRecord, ReasoningTrace, ReasoningStep, ToolCall, ToolCallStats, IngestionItemOutcome, MemoryContextRankedItem, MemoryContextSectionDiagnostics, UnifiedExtractionResult, ExtractionWindow, etc.), 42 service interfaces (incl. `IMemoryIsolationPolicy`, `IUnifiedMemoryExtractor`, `IMultiSessionUnifiedMemoryExtractor`, and `IMemoryReranker`), 11 repository interfaces, 16 configuration types (incl. `MemoryRankingOptions`, `MemoryIsolationOptions`), 26 enums (incl. `MemoryProfile`, `RankingIntent`, `DuplicateStatus`, `EntityMatchType`, `MemoryNodeKind`, `MemoryOperationAccess`, `MemoryIsolationMode`, `IngestionStatus`, `IngestionStage`, `IngestionItemStatus`, `MemoryItemKind`, `IngestionFailureMode`, `MemoryTrustLevel`, `AssistantContentMode`, `TemporalValidityMode`, `TraceKind`, `ExtractionProvenanceMode`) | +| **Key types** | 54 domain records (Conversation, Message, Entity, Fact, Preference, Relationship, MemoryHistoryQuery, MemoryHistoryRecord, ReasoningTrace, ReasoningStep, ToolCall, ToolCallStats, IngestionItemOutcome, MemoryContextRankedItem, MemoryContextSectionDiagnostics, UnifiedExtractionResult, ExtractionWindow, EntitySummary, etc.), 44 service interfaces (incl. `IMemoryIsolationPolicy`, `IUnifiedMemoryExtractor`, `IMultiSessionUnifiedMemoryExtractor`, and `IMemoryReranker`, `IEntitySummaryService`), 12 repository interfaces, 16 configuration types (incl. `MemoryRankingOptions`, `MemoryIsolationOptions`), 26 enums (incl. `MemoryProfile`, `RankingIntent`, `DuplicateStatus`, `EntityMatchType`, `MemoryNodeKind`, `MemoryOperationAccess`, `MemoryIsolationMode`, `IngestionStatus`, `IngestionStage`, `IngestionItemStatus`, `MemoryItemKind`, `IngestionFailureMode`, `MemoryTrustLevel`, `AssistantContentMode`, `TemporalValidityMode`, `TraceKind`, `ExtractionProvenanceMode`) | **Namespace structure:** ``` diff --git a/src/AgentMemory.Abstractions/Domain/LongTerm/EntitySummary.cs b/src/AgentMemory.Abstractions/Domain/LongTerm/EntitySummary.cs new file mode 100644 index 000000000..910ad58c9 --- /dev/null +++ b/src/AgentMemory.Abstractions/Domain/LongTerm/EntitySummary.cs @@ -0,0 +1,107 @@ +using System.Globalization; +using System.Security.Cryptography; +using System.Text; + +namespace AgentMemory.Abstractions.Domain; + +/// +/// A synthesized description of everything memory holds about one entity (S1). +/// +/// +/// +/// Recall about a well-known entity returns twenty separate facts that each cost context and say one +/// thing. A summary says the same in one item — but a summary is a derived memory, and derived +/// memory is where stores quietly start lying: the sources change, the summary does not, and nothing +/// about it looks any different afterwards. +/// +/// +/// So staleness here is proved, never assumed. is computed +/// from the exact set of live facts the summary was written from. Before a summary is used, that +/// fingerprint is recomputed from the store; if it does not match — a source superseded, a new fact +/// added, a confidence moved — the summary is not used. It is not repaired on the read path +/// and it is not returned with a caveat, because a caveat is something a caller can ignore. +/// +/// +/// That check is also why this earns its place in a graph rather than a summary column in a +/// relational row: the fingerprint is derived from the very provenance edges the store already keeps, +/// so "is this still true?" is a query rather than a convention someone has to remember to follow. +/// +/// +public sealed record EntitySummary +{ + /// Unique identifier for the summary node. + public required string SummaryId { get; init; } + + /// The entity this summarises. + public required string EntityId { get; init; } + + /// The synthesized text. + public required string Content { get; init; } + + /// + /// The facts this summary was written from, in the order they were read. + /// + /// + /// Retained rather than merely hashed so a stale summary can say what changed, and so the + /// EXTRACTED_FROM edges to the underlying sources can be rebuilt without re-synthesising. + /// + public required IReadOnlyList SourceFactIds { get; init; } + + /// + /// Hash of the source facts as they stood when the summary was written. + /// + /// + /// The staleness proof. Recompute it from the store and compare: equal means every source is + /// still live and unchanged, anything else means this text describes a state of the world that + /// no longer exists. + /// + public required string SourceFingerprint { get; init; } + + /// Owner (R1). Null = shared/global. + public string? OwnerId { get; init; } + + /// When the summary was synthesized. + public required DateTimeOffset GeneratedAtUtc { get; init; } + + /// Embedding of , when one has been generated. + public IReadOnlyList? Embedding { get; init; } + + /// + /// Computes the fingerprint of a set of source facts. + /// + /// + /// + /// Covers each fact's id, its confidence and whether it has been invalidated — the three things + /// that change what a truthful summary would say. Confidence is included deliberately: + /// reinforcement (S2) moves it, and a summary asserting something the store has since grown + /// doubtful about is exactly the stale shadow this design exists to prevent. + /// + /// + /// Order-independent, because the fingerprint must answer "are these the same facts?" and not + /// "did they come back in the same order?" — a query plan change would otherwise invalidate every + /// summary in the store without a single fact having moved. + /// + /// + public static string ComputeFingerprint(IEnumerable sources) + { + ArgumentNullException.ThrowIfNull(sources); + + var canonical = sources + .Select(s => string.Create( + CultureInfo.InvariantCulture, + $"{s.FactId}|{s.Confidence:F6}|{(s.Invalidated ? 1 : 0)}")) + .OrderBy(s => s, StringComparer.Ordinal) + .ToList(); + + var joined = string.Join("\n", canonical); + return Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(joined))).ToLowerInvariant(); + } +} + +/// +/// One fact's contribution to a summary's fingerprint (S1). +/// +/// The fact. +/// Its confidence at the time of reading. +/// Whether it has been superseded or otherwise invalidated. +public readonly record struct EntitySummarySource(string FactId, double Confidence, bool Invalidated); diff --git a/src/AgentMemory.Abstractions/Domain/LongTerm/Fact.cs b/src/AgentMemory.Abstractions/Domain/LongTerm/Fact.cs index 7814e5440..4128323ed 100644 --- a/src/AgentMemory.Abstractions/Domain/LongTerm/Fact.cs +++ b/src/AgentMemory.Abstractions/Domain/LongTerm/Fact.cs @@ -53,6 +53,24 @@ public sealed record Fact /// public DateTimeOffset? ValidUntil { get; init; } + /// + /// When this fact was superseded or otherwise invalidated; while it is live. + /// + /// + /// + /// The transaction clock, distinct from 's real-world one: + /// ValidUntil says when a fact stopped being true, this says when the store stopped + /// believing it. A contradicted fact is invalidated without its validity period changing at all. + /// + /// + /// The store has always recorded this — supersession is non-destructive precisely so as-of recall + /// can still reach it — but it was never projected onto the domain record, so a caller reading + /// facts had no way to tell a superseded one from a live one. Anything deriving from a fact set + /// (S1's summaries first among them) needs exactly that distinction. + /// + /// + public DateTimeOffset? InvalidatedAtUtc { get; init; } + /// /// Optional embedding vector for semantic search. /// diff --git a/src/AgentMemory.Abstractions/Repositories/IEntitySummaryRepository.cs b/src/AgentMemory.Abstractions/Repositories/IEntitySummaryRepository.cs new file mode 100644 index 000000000..ef44c9db2 --- /dev/null +++ b/src/AgentMemory.Abstractions/Repositories/IEntitySummaryRepository.cs @@ -0,0 +1,33 @@ +using AgentMemory.Abstractions.Domain; +using AgentMemory.Abstractions.Options; + +namespace AgentMemory.Abstractions.Repositories; + +/// +/// Persistence for synthesized entity summaries (S1). +/// +/// +/// One summary per entity per owner. There is no history: a summary is derived, so an old one is not +/// a record of what was believed — it is a record of what a synthesizer produced from facts the store +/// still has, and those facts carry their own bitemporal history already. +/// +public interface IEntitySummaryRepository +{ + /// Stores a summary, replacing the entity's existing one. + Task UpsertAsync(EntitySummary summary, CancellationToken cancellationToken = default); + + /// + /// Reads the entity's summary, if any. Owner-confined (R1). + /// + /// + /// Returns whatever is stored, without judging freshness: staleness is decided against the + /// live facts by the caller that has them, and a repository silently withholding a stale row + /// would make "no summary" and "outdated summary" indistinguishable to a regeneration pass. + /// + Task GetByEntityAsync( + string entityId, MemoryScope scope, CancellationToken cancellationToken = default); + + /// Removes an entity's summary. + Task DeleteByEntityAsync( + string entityId, MemoryScope scope, CancellationToken cancellationToken = default); +} diff --git a/src/AgentMemory.Abstractions/Services/IEntitySummaryService.cs b/src/AgentMemory.Abstractions/Services/IEntitySummaryService.cs new file mode 100644 index 000000000..8255392b5 --- /dev/null +++ b/src/AgentMemory.Abstractions/Services/IEntitySummaryService.cs @@ -0,0 +1,37 @@ +using AgentMemory.Abstractions.Domain; +using AgentMemory.Abstractions.Options; + +namespace AgentMemory.Abstractions.Services; + +/// +/// Writes entity summaries and withholds any that can no longer prove it is current (S1). +/// +/// +/// A summary is derived memory, which is where a store quietly starts lying: the sources change, the +/// summary does not, and nothing about it looks any different afterwards. Every read here is checked +/// against the live facts before it is handed back. +/// +public interface IEntitySummaryService +{ + /// + /// Synthesizes and stores a summary for the entity, replacing any existing one. + /// + /// The stored summary, or when there was nothing to summarise. + Task RefreshAsync( + Entity entity, MemoryScope scope, CancellationToken cancellationToken = default); + + /// + /// Returns the entity's summary only if it still describes the current facts. + /// + /// + /// The summary when its fingerprint still matches the store; when there is + /// none or when the one on record has gone stale. + /// + /// + /// Null for stale rather than a summary carrying an IsStale flag. A flag puts the decision + /// in every caller's hands, and one caller forgetting to check it is indistinguishable from + /// correct memory. + /// + Task GetIfCurrentAsync( + Entity entity, MemoryScope scope, CancellationToken cancellationToken = default); +} diff --git a/src/AgentMemory.Abstractions/Services/IEntitySummarySynthesizer.cs b/src/AgentMemory.Abstractions/Services/IEntitySummarySynthesizer.cs new file mode 100644 index 000000000..e5e03c5a9 --- /dev/null +++ b/src/AgentMemory.Abstractions/Services/IEntitySummarySynthesizer.cs @@ -0,0 +1,34 @@ +using AgentMemory.Abstractions.Domain; + +namespace AgentMemory.Abstractions.Services; + +/// +/// Turns an entity's facts into one synthesized description (S1). +/// +/// +/// +/// A seam rather than a fixed implementation because the two reasonable answers have opposite cost +/// profiles. The shipped default composes the text deterministically — no completion, reproducible +/// across runs, and free to regenerate whenever sources move, which matters because sources move +/// constantly. A host that wants prose can supply an LLM implementation and pay for it. +/// +/// +/// Whatever synthesises, it may only use the facts it is given. The summary's fingerprint is +/// computed from exactly those facts, so a synthesizer that reached for anything else would produce +/// text whose staleness could never be detected — the one failure this design exists to rule out. +/// +/// +public interface IEntitySummarySynthesizer +{ + /// + /// Composes a description of from . + /// + /// + /// The summary text, or when there is nothing worth summarising — which is + /// a real answer, not a failure, and must not be stored as an empty summary. + /// + Task SynthesizeAsync( + Entity entity, + IReadOnlyList facts, + CancellationToken cancellationToken = default); +} diff --git a/src/AgentMemory.Core/Memory/DeterministicEntitySummarySynthesizer.cs b/src/AgentMemory.Core/Memory/DeterministicEntitySummarySynthesizer.cs new file mode 100644 index 000000000..3c4781ee4 --- /dev/null +++ b/src/AgentMemory.Core/Memory/DeterministicEntitySummarySynthesizer.cs @@ -0,0 +1,87 @@ +using System.Globalization; +using System.Text; +using AgentMemory.Abstractions.Domain; +using AgentMemory.Abstractions.Services; + +namespace AgentMemory.Core.Memory; + +/// +/// Composes an entity summary from its facts without a model call (S1). +/// +/// +/// +/// The shipped default, and deliberately not an LLM. A summary is invalidated whenever any of its +/// sources moves, so it is regenerated often; a completion per entity per change would make the +/// feature cost scale with how much the conversation talks about its subjects — which is exactly the +/// entities that get summarised most. +/// +/// +/// It is also reproducible. Two runs over the same facts produce the same bytes, so a summary can be +/// compared, fingerprinted and diffed, and a change in the text always means a change in the facts +/// rather than a change in a sampler's mood. +/// +/// +/// The output is a grouped rendering, not prose: predicates gathered with their objects, ordered +/// deterministically, most-confident first within each predicate. Honest about what it is — a +/// compact projection of the graph, which is what a caller needs when the alternative is twenty +/// separate facts each spending context to say one thing. +/// +/// +internal sealed class DeterministicEntitySummarySynthesizer : IEntitySummarySynthesizer +{ + /// Facts below this confidence are left out of the summary. + /// + /// A summary states things flatly, with none of the per-fact confidence a caller would otherwise + /// see. Folding a 0.2-confidence guess into that sentence would present it with the same + /// authority as everything else and remove the only signal that it was doubtful. + /// + public double MinimumConfidence { get; init; } = 0.5; + + /// + public Task SynthesizeAsync( + Entity entity, + IReadOnlyList facts, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(entity); + ArgumentNullException.ThrowIfNull(facts); + + var usable = facts + .Where(f => f.Confidence >= MinimumConfidence) + .Where(f => !string.IsNullOrWhiteSpace(f.Predicate) && !string.IsNullOrWhiteSpace(f.Object)) + .ToList(); + + // Null, not an empty summary. "Nothing is known about this entity above the confidence floor" + // is a real answer, and storing it as an empty string would create a node that satisfies every + // has-a-summary check while saying nothing. + if (usable.Count == 0) return Task.FromResult(null); + + var builder = new StringBuilder(); + builder.Append(entity.Name); + if (!string.IsNullOrWhiteSpace(entity.Type)) + builder.Append(" (").Append(entity.Type).Append(')'); + builder.Append(':'); + + var grouped = usable + .GroupBy(f => f.Predicate, StringComparer.OrdinalIgnoreCase) + .OrderBy(g => g.Key, StringComparer.OrdinalIgnoreCase); + + foreach (var group in grouped) + { + var objects = group + // Most-confident first, then alphabetical: confidence alone leaves ties resolved by + // whatever order the store happened to return, which would make the text -- and so its + // fingerprint -- vary between identical runs. + .OrderByDescending(f => f.Confidence) + .ThenBy(f => f.Object, StringComparer.OrdinalIgnoreCase) + .Select(f => f.Object) + .Distinct(StringComparer.OrdinalIgnoreCase); + + builder.Append(string.Create( + CultureInfo.InvariantCulture, + $"\n- {group.Key}: {string.Join("; ", objects)}")); + } + + return Task.FromResult(builder.ToString()); + } +} diff --git a/src/AgentMemory.Core/Memory/EntitySummaryService.cs b/src/AgentMemory.Core/Memory/EntitySummaryService.cs new file mode 100644 index 000000000..6fe1c53ea --- /dev/null +++ b/src/AgentMemory.Core/Memory/EntitySummaryService.cs @@ -0,0 +1,134 @@ +using AgentMemory.Abstractions.Domain; +using AgentMemory.Abstractions.Options; +using AgentMemory.Abstractions.Repositories; +using AgentMemory.Abstractions.Services; +using Microsoft.Extensions.Logging; + +namespace AgentMemory.Core.Memory; + +/// +/// Writes entity summaries, and refuses to hand back one that can no longer prove it is current (S1). +/// +/// +/// +/// The synthesis is the easy half. The half that decides whether this feature is an asset or a +/// liability is what happens after a source fact is superseded — and the answer here is that +/// the summary stops being used, immediately, without anything having to remember to delete it. +/// +/// +/// Detection rather than invalidation-on-write. A supersession could instead go and mark every +/// affected summary, and that is the design that fails quietly: it has to find them all, it has to run +/// in the same transaction, and any path that writes a fact without going through it leaves a summary +/// that looks current and is not. Recomputing the fingerprint on read cannot be bypassed by a writer +/// that did not know summaries existed. +/// +/// +/// The cost is one extra read of the entity's facts before a summary is used. That is the same read +/// the summary is saving the caller, so a stale summary costs what not having one would have cost — +/// and a fresh summary still saves the context, which is what it was for. +/// +/// +internal sealed class EntitySummaryService : IEntitySummaryService +{ + private readonly IFactRepository _facts; + private readonly IEntitySummaryRepository _summaries; + private readonly IEntitySummarySynthesizer _synthesizer; + private readonly IClock _clock; + private readonly IIdGenerator _ids; + private readonly ILogger _logger; + + /// Public ctor: the type is internal, and DI can only activate it through one. + public EntitySummaryService( + IFactRepository facts, + IEntitySummaryRepository summaries, + IEntitySummarySynthesizer synthesizer, + IClock clock, + IIdGenerator ids, + ILogger logger) + { + _facts = facts; + _summaries = summaries; + _synthesizer = synthesizer; + _clock = clock; + _ids = ids; + _logger = logger; + } + + /// + /// Synthesizes and stores a summary for , replacing any existing one. + /// + /// The stored summary, or when there was nothing to summarise. + public async Task RefreshAsync( + Entity entity, + MemoryScope scope, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(entity); + + var facts = await _facts.GetBySubjectAsync(entity.Name, scope, cancellationToken).ConfigureAwait(false); + var live = facts.Where(f => f.InvalidatedAtUtc is null).ToList(); + + var content = await _synthesizer + .SynthesizeAsync(entity, live, cancellationToken).ConfigureAwait(false); + if (string.IsNullOrWhiteSpace(content)) + { + _logger.LogDebug("Nothing to summarise for entity {EntityId}.", entity.EntityId); + return null; + } + + // Fingerprinted from the SAME list handed to the synthesizer. Reading the facts twice, or + // filtering differently here, would produce a fingerprint describing a set the text was never + // written from -- a summary that could be stale from the moment it was stored. + var summary = new EntitySummary + { + SummaryId = _ids.GenerateId(), + EntityId = entity.EntityId, + Content = content, + SourceFactIds = live.Select(f => f.FactId).ToList(), + SourceFingerprint = EntitySummary.ComputeFingerprint(Sources(live)), + OwnerId = entity.OwnerId, + GeneratedAtUtc = _clock.UtcNow, + }; + + await _summaries.UpsertAsync(summary, cancellationToken).ConfigureAwait(false); + return summary; + } + + /// + /// Returns the entity's summary only if it still describes the current facts. + /// + /// + /// The summary when its fingerprint still matches the store; when there is + /// none or when the one on record has gone stale. + /// + /// + /// Null for stale is deliberate and is the whole safety property. Returning it with an + /// IsStale flag would put the decision in every caller's hands, and the failure of any one + /// of them to check it is indistinguishable from correct memory. + /// + public async Task GetIfCurrentAsync( + Entity entity, + MemoryScope scope, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(entity); + + var stored = await _summaries + .GetByEntityAsync(entity.EntityId, scope, cancellationToken).ConfigureAwait(false); + if (stored is null) return null; + + var facts = await _facts.GetBySubjectAsync(entity.Name, scope, cancellationToken).ConfigureAwait(false); + var live = facts.Where(f => f.InvalidatedAtUtc is null).ToList(); + var current = EntitySummary.ComputeFingerprint(Sources(live)); + + if (string.Equals(current, stored.SourceFingerprint, StringComparison.Ordinal)) return stored; + + _logger.LogDebug( + "Entity summary for {EntityId} is stale ({Stored} != {Current}); not used.", + entity.EntityId, stored.SourceFingerprint, current); + return null; + } + + private static IEnumerable Sources(IEnumerable facts) => + facts.Select(f => new EntitySummarySource(f.FactId, f.Confidence, Invalidated: false)); +} diff --git a/src/AgentMemory.Core/ServiceCollectionExtensions.cs b/src/AgentMemory.Core/ServiceCollectionExtensions.cs index 62f51cbe0..0d38d0f7f 100644 --- a/src/AgentMemory.Core/ServiceCollectionExtensions.cs +++ b/src/AgentMemory.Core/ServiceCollectionExtensions.cs @@ -5,6 +5,7 @@ using AgentMemory.Abstractions.Options; using AgentMemory.Abstractions.Services; using AgentMemory.Core.Extraction; +using AgentMemory.Core.Memory; using AgentMemory.Core.Resolution; using AgentMemory.Core.Services; using AgentMemory.Core.Services.Budgeting; @@ -228,6 +229,13 @@ public static IServiceCollection AddAgentMemoryCore( // Callers may override by registering their own IEntityResolver before calling this method. services.TryAddScoped(); + // S1. The deterministic synthesizer is the default because a summary is regenerated every + // time any of its sources moves; a completion per entity per change would scale cost with + // exactly how much the conversation talks about its subjects. TryAdd, so a host wanting + // prose registers an LLM synthesizer before calling this and keeps it. + services.TryAddScoped(); + services.TryAddScoped(); + // Keep StubEntityResolver available for explicit fallback use. services.TryAddScoped(); diff --git a/src/AgentMemory.Neo4j/Infrastructure/ServiceCollectionExtensions.cs b/src/AgentMemory.Neo4j/Infrastructure/ServiceCollectionExtensions.cs index 5408555fe..c99638d03 100644 --- a/src/AgentMemory.Neo4j/Infrastructure/ServiceCollectionExtensions.cs +++ b/src/AgentMemory.Neo4j/Infrastructure/ServiceCollectionExtensions.cs @@ -72,6 +72,10 @@ public static IServiceCollection AddNeo4jAgentMemory( services.TryAddTransient(); services.TryAddTransient(); services.TryAddTransient(); + // S1. Registered beside the other repositories rather than behind a feature flag: an + // unregistered store would make EntitySummaryService unresolvable, so the option that + // enables summaries would fail at host start rather than simply doing nothing. + services.TryAddTransient(); services.TryAddTransient(); // Provenance diff --git a/src/AgentMemory.Neo4j/Queries/EntitySummaryQueries.cs b/src/AgentMemory.Neo4j/Queries/EntitySummaryQueries.cs new file mode 100644 index 000000000..6636f18e0 --- /dev/null +++ b/src/AgentMemory.Neo4j/Queries/EntitySummaryQueries.cs @@ -0,0 +1,76 @@ +namespace AgentMemory.Neo4j.Queries; + +/// +/// Cypher for synthesized entity summaries (S1). +/// +/// +/// +/// One summary per entity, held on a :EntitySummary node joined to its entity by +/// :SUMMARISES. A separate node rather than a property on :Entity because it carries +/// its own provenance edges and its own fingerprint, and because a derived value living beside +/// extracted ones invites exactly the confusion this design is trying to avoid. +/// +/// +/// No history. A superseded summary is not a record of what was once believed — the facts it +/// was built from carry their own bitemporal history already, and keeping stale derivations around +/// would mean two answers to "what did we think about Alice in March", only one of them authoritative. +/// +/// +internal static class EntitySummaryQueries +{ + /// + /// Writes the entity's summary, replacing whatever was there. + /// + /// + /// MERGE on entity_id + owner_key, so two tenants summarising the same shared entity + /// keep separate summaries — each was synthesised from the facts that tenant can see, and sharing + /// one node would leak one owner's knowledge into another's context. + /// + public const string Upsert = """ + MERGE (s:EntitySummary {entity_id: $entityId, owner_key: $ownerKey}) + SET s.id = $summaryId, + s.content = $content, + s.source_fact_ids = $sourceFactIds, + s.source_fingerprint = $sourceFingerprint, + s.owner_id = $ownerId, + s.generated_at = datetime($generatedAt) + WITH s + OPTIONAL MATCH (e:Entity {id: $entityId}) + FOREACH (_ IN CASE WHEN e IS NULL THEN [] ELSE [1] END | + MERGE (s)-[:SUMMARISES]->(e)) + WITH s + // Provenance to the facts the text was written from. Rebuilt wholesale rather than merged, + // because a summary regenerated from a smaller fact set must not keep edges to sources it no + // longer draws on -- that would claim provenance the content cannot support. + OPTIONAL MATCH (s)-[old:EXTRACTED_FROM]->(:Fact) + DELETE old + WITH s + UNWIND $sourceFactIds AS factId + MATCH (f:Fact {id: factId}) + MERGE (s)-[:EXTRACTED_FROM]->(f) + RETURN s + """; + + /// Reads an entity's summary, owner-confined (R1). + public static string GetByEntity(bool hasOwnerFilter, bool includeShared) + { + var owner = !hasOwnerFilter ? string.Empty + : includeShared ? " AND (s.owner_id = $ownerId OR s.owner_id IS NULL)" + : " AND s.owner_id = $ownerId"; + return $"MATCH (s:EntitySummary) WHERE s.entity_id = $entityId{owner} RETURN s"; + } + + /// Removes an entity's summary and its edges. + public static string DeleteByEntity(bool hasOwnerFilter, bool includeShared) + { + var owner = !hasOwnerFilter ? string.Empty + : includeShared ? " AND (s.owner_id = $ownerId OR s.owner_id IS NULL)" + : " AND s.owner_id = $ownerId"; + return $""" + MATCH (s:EntitySummary) WHERE s.entity_id = $entityId{owner} + WITH s, count(s) AS found + DETACH DELETE s + RETURN found + """; + } +} diff --git a/src/AgentMemory.Neo4j/Repositories/Neo4jEntitySummaryRepository.cs b/src/AgentMemory.Neo4j/Repositories/Neo4jEntitySummaryRepository.cs new file mode 100644 index 000000000..8435d4846 --- /dev/null +++ b/src/AgentMemory.Neo4j/Repositories/Neo4jEntitySummaryRepository.cs @@ -0,0 +1,107 @@ +using AgentMemory.Abstractions.Domain; +using AgentMemory.Abstractions.Options; +using AgentMemory.Abstractions.Repositories; +using AgentMemory.Neo4j.Infrastructure; +using AgentMemory.Neo4j.Queries; +using Microsoft.Extensions.Logging; +using Neo4j.Driver; + +namespace AgentMemory.Neo4j.Repositories; + +/// +/// Neo4j persistence for synthesized entity summaries (S1). +/// +/// +/// The summary node keeps its own :EXTRACTED_FROM edges to the facts it was written from. That +/// is the part a summary column in a relational row cannot do, and it is what makes "which facts is +/// this claim standing on?" a query rather than a comment. +/// +internal sealed class Neo4jEntitySummaryRepository : IEntitySummaryRepository +{ + private readonly INeo4jTransactionRunner _tx; + private readonly ILogger _logger; + + /// Public ctor: the type is internal, and DI can only activate it through one. + public Neo4jEntitySummaryRepository( + INeo4jTransactionRunner tx, + ILogger logger) + { + _tx = tx; + _logger = logger; + } + + /// + public async Task UpsertAsync( + EntitySummary summary, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(summary); + + await _tx.WriteAsync(async runner => + { + var cursor = await runner.RunAsync(EntitySummaryQueries.Upsert, new + { + summaryId = summary.SummaryId, + entityId = summary.EntityId, + content = summary.Content, + sourceFactIds = summary.SourceFactIds.ToArray(), + sourceFingerprint = summary.SourceFingerprint, + ownerId = summary.OwnerId, + ownerKey = summary.OwnerId ?? Neo4jFactRepository.OwnerKeyShared, + generatedAt = summary.GeneratedAtUtc.ToString("o"), + }).ConfigureAwait(false); + await cursor.ConsumeAsync().ConfigureAwait(false); + return true; + }, cancellationToken).ConfigureAwait(false); + + _logger.LogDebug( + "Stored summary for entity {EntityId} over {SourceCount} fact(s).", + summary.EntityId, summary.SourceFactIds.Count); + return summary; + } + + /// + public Task GetByEntityAsync( + string entityId, MemoryScope scope, CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(entityId); + + return _tx.ReadAsync(async runner => + { + var cursor = await runner.RunAsync( + EntitySummaryQueries.GetByEntity(scope.OwnerId is not null, scope.IncludeShared), + new { entityId, ownerId = scope.OwnerId }).ConfigureAwait(false); + var records = await cursor.ToListAsync().ConfigureAwait(false); + return records.Count == 0 ? null : Map(records[0]["s"].As()); + }, cancellationToken); + } + + /// + public Task DeleteByEntityAsync( + string entityId, MemoryScope scope, CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(entityId); + + return _tx.WriteAsync(async runner => + { + var cursor = await runner.RunAsync( + EntitySummaryQueries.DeleteByEntity(scope.OwnerId is not null, scope.IncludeShared), + new { entityId, ownerId = scope.OwnerId }).ConfigureAwait(false); + var records = await cursor.ToListAsync().ConfigureAwait(false); + return records.Count > 0; + }, cancellationToken); + } + + private static EntitySummary Map(INode node) => new() + { + SummaryId = node.Properties.TryGetValue("id", out var id) ? id.As() : string.Empty, + EntityId = node["entity_id"].As(), + Content = node["content"].As(), + SourceFactIds = node.Properties.TryGetValue("source_fact_ids", out var ids) + ? ids.As>().Select(v => v.ToString()!).ToList() + : [], + SourceFingerprint = node["source_fingerprint"].As(), + OwnerId = node.Properties.TryGetValue("owner_id", out var owner) ? owner.As() : null, + GeneratedAtUtc = Neo4jDateTimeHelper.ReadNullableDateTimeOffset(node["generated_at"]) + ?? DateTimeOffset.MinValue, + }; +} diff --git a/src/AgentMemory.Neo4j/Repositories/Neo4jFactRepository.cs b/src/AgentMemory.Neo4j/Repositories/Neo4jFactRepository.cs index 1f8dbc54a..7665d208c 100644 --- a/src/AgentMemory.Neo4j/Repositories/Neo4jFactRepository.cs +++ b/src/AgentMemory.Neo4j/Repositories/Neo4jFactRepository.cs @@ -594,6 +594,9 @@ private static Fact MapToFact(INode node, float[]? embedding) => ValidUntil = node.Properties.TryGetValue("valid_until", out var vu) ? Neo4jDateTimeHelper.ReadNullableDateTimeOffset(vu) : null, + InvalidatedAtUtc = node.Properties.TryGetValue("invalidated_at", out var iat) + ? Neo4jDateTimeHelper.ReadNullableDateTimeOffset(iat) + : null, Embedding = embedding, SourceMessageIds = node.Properties.TryGetValue("source_message_ids", out var sm) ? sm.As>().Select(v => v.ToString()!).ToList() diff --git a/tests/AgentMemory.Tests.Integration/Repositories/EntitySummaryIntegrationTests.cs b/tests/AgentMemory.Tests.Integration/Repositories/EntitySummaryIntegrationTests.cs new file mode 100644 index 000000000..b90d145b1 --- /dev/null +++ b/tests/AgentMemory.Tests.Integration/Repositories/EntitySummaryIntegrationTests.cs @@ -0,0 +1,171 @@ +using AgentMemory.Abstractions.Domain; +using AgentMemory.Abstractions.Options; +using AgentMemory.Neo4j.Repositories; +using AgentMemory.Tests.Integration.Fixtures; +using FluentAssertions; +using Microsoft.Extensions.Logging.Abstractions; +using Neo4j.Driver; + +namespace AgentMemory.Tests.Integration.Repositories; + +/// +/// Entity summaries in the database (S1), where the provenance edges either exist or do not. +/// +/// +/// +/// The staleness arithmetic is unit-tested. What only a live database can show is that a summary +/// actually attaches to its sources, that regenerating it does not accumulate edges to facts it no +/// longer draws on, and that one tenant's summary is invisible to another. +/// +/// +/// The provenance edges are the reason this is a graph node rather than a column. If they silently +/// failed to attach, every unit test would still pass and "which facts is this claim standing on?" +/// would quietly have no answer. +/// +/// +[Collection("Neo4j Integration")] +[Trait("Category", "Integration")] +public class EntitySummaryIntegrationTests : IAsyncLifetime +{ + private readonly Neo4jIntegrationFixture _fixture; + private readonly Neo4jEntitySummaryRepository _summaries; + private readonly Neo4jFactRepository _facts; + + public EntitySummaryIntegrationTests(Neo4jIntegrationFixture fixture) + { + _fixture = fixture; + _summaries = new Neo4jEntitySummaryRepository( + fixture.TransactionRunner, NullLogger.Instance); + _facts = new Neo4jFactRepository( + fixture.TransactionRunner, NullLogger.Instance); + } + + public Task InitializeAsync() => _fixture.CleanDatabaseAsync(); + public Task DisposeAsync() => Task.CompletedTask; + + private static readonly MemoryScope Alice = MemoryScope.For("alice", includeShared: false); + + private Task FactAsync(string id, string @object, string owner = "alice") => + _facts.UpsertAsync(new Fact + { + FactId = id, + Subject = "Alice", + Predicate = "lives in", + Object = @object, + Confidence = 0.9, + OwnerId = owner, + CreatedAtUtc = DateTimeOffset.UtcNow, + }); + + private static EntitySummary Summary( + string content, IReadOnlyList sourceIds, string owner = "alice", string id = "sum-1") => new() + { + SummaryId = id, + EntityId = "e-alice", + Content = content, + SourceFactIds = sourceIds, + SourceFingerprint = $"fp-{content.GetHashCode():X}", + OwnerId = owner, + GeneratedAtUtc = DateTimeOffset.UtcNow, + }; + + private Task ProvenanceEdgeCountAsync() => + _fixture.TransactionRunner.ReadAsync(async runner => + { + var cursor = await runner.RunAsync( + "MATCH (:EntitySummary {entity_id: 'e-alice'})-[r:EXTRACTED_FROM]->(:Fact) RETURN count(r) AS c"); + var records = await cursor.ToListAsync(); + return records.Count == 0 ? 0 : global::Neo4j.Driver.ValueExtensions.As(records[0]["c"]); + }); + + [Fact] + public async Task ASummaryRoundTrips() + { + await FactAsync("f-1", "Zurich"); + var stored = Summary("Alice lives in Zurich", ["f-1"]); + + await _summaries.UpsertAsync(stored); + var read = await _summaries.GetByEntityAsync("e-alice", Alice); + + read.Should().NotBeNull(); + read!.Content.Should().Be("Alice lives in Zurich"); + read.SourceFingerprint.Should().Be(stored.SourceFingerprint); + read.SourceFactIds.Should().Equal("f-1"); + } + + [Fact] + public async Task ProvenanceEdgesAttachToEverySourceFact() + { + // The reason this is a node in a graph. If the MERGE silently matched nothing, every unit + // test would still pass and the summary would stand on no recorded evidence at all. + await FactAsync("f-1", "Zurich"); + await FactAsync("f-2", "Basel"); + + await _summaries.UpsertAsync(Summary("two sources", ["f-1", "f-2"])); + + (await ProvenanceEdgeCountAsync()).Should().Be(2); + } + + [Fact] + public async Task RegeneratingFromFewerSourcesDropsTheStaleEdges() + { + // THE edge case, literally. Edges are rebuilt rather than merged: a summary regenerated from a + // smaller fact set must not keep pointing at sources its text no longer draws on, which would + // claim provenance the content cannot support. + await FactAsync("f-1", "Zurich"); + await FactAsync("f-2", "Basel"); + await _summaries.UpsertAsync(Summary("two sources", ["f-1", "f-2"])); + + await _summaries.UpsertAsync(Summary("one source", ["f-1"], id: "sum-2")); + + (await ProvenanceEdgeCountAsync()).Should().Be(1); + } + + [Fact] + public async Task ASecondUpsertReplacesRatherThanDuplicates() + { + await FactAsync("f-1", "Zurich"); + await _summaries.UpsertAsync(Summary("first", ["f-1"])); + await _summaries.UpsertAsync(Summary("second", ["f-1"], id: "sum-2")); + + var read = await _summaries.GetByEntityAsync("e-alice", Alice); + + read!.Content.Should().Be("second", "one summary per entity per owner"); + } + + [Fact] + public async Task AnotherOwnersSummaryIsInvisible() + { + // R1. Two tenants summarising the same entity each did so from the facts they can see, so a + // shared node would leak one owner's knowledge straight into another's context. + await FactAsync("f-1", "Zurich", owner: "bob"); + await _summaries.UpsertAsync(Summary("bob's view", ["f-1"], owner: "bob")); + + (await _summaries.GetByEntityAsync("e-alice", Alice)).Should().BeNull(); + } + + [Fact] + public async Task DeletingRemovesTheSummaryAndItsEdges() + { + await FactAsync("f-1", "Zurich"); + await _summaries.UpsertAsync(Summary("to be removed", ["f-1"])); + + var deleted = await _summaries.DeleteByEntityAsync("e-alice", Alice); + + deleted.Should().BeTrue(); + (await _summaries.GetByEntityAsync("e-alice", Alice)).Should().BeNull(); + (await ProvenanceEdgeCountAsync()).Should().Be(0); + } + + [Fact] + public async Task ASummaryOverAMissingFactStillStores() + { + // A source fact removed between synthesis and write must not fail the write. The summary is + // then short an edge, which is exactly what the fingerprint check will catch on read -- the + // failure is detected where it can be acted on rather than thrown where it cannot. + var act = async () => await _summaries.UpsertAsync(Summary("orphaned", ["f-missing"])); + + await act.Should().NotThrowAsync(); + (await _summaries.GetByEntityAsync("e-alice", Alice)).Should().NotBeNull(); + } +} diff --git a/tests/AgentMemory.Tests.Unit/Infrastructure/AbstractionsContractGuardTests.cs b/tests/AgentMemory.Tests.Unit/Infrastructure/AbstractionsContractGuardTests.cs index eeed387bc..8545dc95e 100644 --- a/tests/AgentMemory.Tests.Unit/Infrastructure/AbstractionsContractGuardTests.cs +++ b/tests/AgentMemory.Tests.Unit/Infrastructure/AbstractionsContractGuardTests.cs @@ -16,9 +16,9 @@ public sealed class AbstractionsContractGuardTests private static readonly Assembly Abstractions = typeof(IMemoryService).Assembly; // Counts mirrored in docs/architecture.md §3.1 and docs/design.md §5/§6. - private const int DocumentedServiceInterfaces = 42; // +IMemoryReranker (PLAN 10.1, seam only) - private const int DocumentedRepositoryInterfaces = 11; - private const int DocumentedDomainRecords = 53; // +ExtractionWindow (PLAN 12.6 / E2) + private const int DocumentedServiceInterfaces = 44; // +IEntitySummarySynthesizer, +IEntitySummaryService (PLAN 12.7 / S1) + private const int DocumentedRepositoryInterfaces = 12; // +IEntitySummaryRepository (PLAN 12.7 / S1) + private const int DocumentedDomainRecords = 54; // +EntitySummary (PLAN 12.7 / S1; EntitySummarySource is a record STRUCT and the guard counts reference records only) private const int DocumentedEnums = 29; // +TraceKind (7.1), +ExtractionProvenanceMode (9.3) private static IEnumerable PublicTypes() => diff --git a/tests/AgentMemory.Tests.Unit/Memory/EntitySummaryTests.cs b/tests/AgentMemory.Tests.Unit/Memory/EntitySummaryTests.cs new file mode 100644 index 000000000..0cf6a8886 --- /dev/null +++ b/tests/AgentMemory.Tests.Unit/Memory/EntitySummaryTests.cs @@ -0,0 +1,238 @@ +using AgentMemory.Abstractions.Domain; +using AgentMemory.Abstractions.Options; +using AgentMemory.Abstractions.Repositories; +using AgentMemory.Abstractions.Services; +using AgentMemory.Core.Memory; +using FluentAssertions; +using Microsoft.Extensions.Logging.Abstractions; +using NSubstitute; +using Xunit; + +namespace AgentMemory.Tests.Unit.Memory; + +/// +/// Synthesized entity summaries (S1) — and the one property that decides whether they are an asset +/// or a liability: a summary that cannot prove it is current is never used. +/// +/// +/// +/// Recall about a well-known entity returns twenty facts that each cost context to say one thing. A +/// summary says the same in one item. But a summary is derived memory, and derived memory is +/// where a store quietly starts lying: the sources change, the summary does not, and afterwards +/// nothing about it looks any different. +/// +/// +/// So the tests here are mostly about a summary being withheld. The synthesis is the easy half. +/// +/// +public sealed class EntitySummaryTests +{ + private readonly IFactRepository _facts = Substitute.For(); + private readonly IEntitySummaryRepository _summaries = Substitute.For(); + private readonly IClock _clock = Substitute.For(); + + private static readonly Entity Alice = new() + { + EntityId = "e-alice", + Name = "Alice", + Type = "Person", + Confidence = 1.0, + OwnerId = "owner-1", + CreatedAtUtc = DateTimeOffset.UnixEpoch, + }; + + private static readonly MemoryScope Scope = MemoryScope.For("owner-1"); + + private static Fact F( + string id, string predicate, string @object, + double confidence = 0.9, DateTimeOffset? invalidatedAt = null) => new() + { + FactId = id, + Subject = "Alice", + Predicate = predicate, + Object = @object, + Confidence = confidence, + OwnerId = "owner-1", + CreatedAtUtc = DateTimeOffset.UnixEpoch, + InvalidatedAtUtc = invalidatedAt, + }; + + public EntitySummaryTests() => _clock.UtcNow.Returns(DateTimeOffset.UnixEpoch); + + private EntitySummaryService CreateSut() + { + var ids = Substitute.For(); + ids.GenerateId().Returns("sum-1"); + return new EntitySummaryService( + _facts, _summaries, new DeterministicEntitySummarySynthesizer(), + _clock, ids, NullLogger.Instance); + } + + private void StoreHolds(params Fact[] facts) => + _facts.GetBySubjectAsync("Alice", Arg.Any(), Arg.Any()) + .Returns(Task.FromResult>(facts)); + + private void SummaryOnRecord(EntitySummary summary) => + _summaries.GetByEntityAsync("e-alice", Arg.Any(), Arg.Any()) + .Returns(Task.FromResult(summary)); + + // ── staleness: the point of the whole design ────────────────────────── + + [Fact] + public async Task ASummaryIsUsedWhileItsSourcesAreUnchanged() + { + StoreHolds(F("f-1", "lives in", "Zurich")); + var written = await CreateSut().RefreshAsync(Alice, Scope); + SummaryOnRecord(written!); + + (await CreateSut().GetIfCurrentAsync(Alice, Scope)).Should().NotBeNull(); + } + + [Fact] + public async Task ASupersededSourceMakesTheSummaryUnusable() + { + // THE test. A fact was contradicted; the stored text still asserts it. Nothing about that + // summary looks different, so the only thing standing between it and a confident wrong answer + // is this check. + StoreHolds(F("f-1", "lives in", "Zurich")); + var written = await CreateSut().RefreshAsync(Alice, Scope); + SummaryOnRecord(written!); + + StoreHolds(F("f-1", "lives in", "Zurich", invalidatedAt: DateTimeOffset.UnixEpoch)); + + (await CreateSut().GetIfCurrentAsync(Alice, Scope)).Should().BeNull(); + } + + [Fact] + public async Task ANewFactMakesTheSummaryUnusable() + { + // Incompleteness is staleness too. A summary that omits something the store now knows is + // exactly as misleading as one asserting something it no longer believes. + StoreHolds(F("f-1", "lives in", "Zurich")); + var written = await CreateSut().RefreshAsync(Alice, Scope); + SummaryOnRecord(written!); + + StoreHolds(F("f-1", "lives in", "Zurich"), F("f-2", "works at", "Acme")); + + (await CreateSut().GetIfCurrentAsync(Alice, Scope)).Should().BeNull(); + } + + [Fact] + public async Task AMovedConfidenceMakesTheSummaryUnusable() + { + // Confidence is in the fingerprint deliberately. S2 reinforcement moves it, and a summary + // stating flatly what the store has since grown doubtful about is the stale shadow this + // design exists to prevent. + StoreHolds(F("f-1", "lives in", "Zurich", confidence: 0.9)); + var written = await CreateSut().RefreshAsync(Alice, Scope); + SummaryOnRecord(written!); + + StoreHolds(F("f-1", "lives in", "Zurich", confidence: 0.6)); + + (await CreateSut().GetIfCurrentAsync(Alice, Scope)).Should().BeNull(); + } + + [Fact] + public async Task NoSummaryOnRecordIsNotAnError() + { + _summaries.GetByEntityAsync("e-alice", Arg.Any(), Arg.Any()) + .Returns(Task.FromResult(null)); + + (await CreateSut().GetIfCurrentAsync(Alice, Scope)).Should().BeNull(); + } + + // ── the fingerprint ─────────────────────────────────────────────────── + + [Fact] + public void TheFingerprintIsOrderIndependent() + { + // It must answer "are these the same facts?", not "did they come back in the same order?". + // Otherwise a query-plan change invalidates every summary in the store without a single fact + // having moved. + var a = new EntitySummarySource("f-1", 0.9, false); + var b = new EntitySummarySource("f-2", 0.7, false); + + EntitySummary.ComputeFingerprint([a, b]) + .Should().Be(EntitySummary.ComputeFingerprint([b, a])); + } + + [Fact] + public void InvalidationChangesTheFingerprintOnItsOwn() + { + // Even with the id and confidence untouched: a fact the store has stopped believing is not + // the same source it was. + EntitySummary.ComputeFingerprint([new EntitySummarySource("f-1", 0.9, false)]) + .Should().NotBe(EntitySummary.ComputeFingerprint([new EntitySummarySource("f-1", 0.9, true)])); + } + + [Fact] + public void AnEmptySourceSetStillHasAFingerprint() => + EntitySummary.ComputeFingerprint([]).Should().NotBeNullOrWhiteSpace(); + + // ── synthesis ───────────────────────────────────────────────────────── + + [Fact] + public async Task SynthesisIsReproducible() + { + // Two runs over the same facts produce the same bytes, so a change in the text always means a + // change in the facts rather than a change in a sampler's mood. That is also what makes the + // summary safe to fingerprint at all. + var synth = new DeterministicEntitySummarySynthesizer(); + var facts = new[] { F("f-1", "lives in", "Zurich"), F("f-2", "works at", "Acme") }; + + var first = await synth.SynthesizeAsync(Alice, facts); + // AsEnumerable() first: on an array, Reverse() binds to Array.Reverse, which returns void. + var second = await synth.SynthesizeAsync(Alice, facts.AsEnumerable().Reverse().ToArray()); + + first.Should().Be(second); + } + + [Fact] + public async Task LowConfidenceFactsAreLeftOut() + { + // A summary states things flatly, with none of the per-fact confidence a caller would see. + // Folding a guess in gives it the same authority as everything else. + var summary = await new DeterministicEntitySummarySynthesizer() + .SynthesizeAsync(Alice, [F("f-1", "lives in", "Zurich"), F("f-2", "owns", "a yacht", confidence: 0.2)]); + + summary.Should().Contain("Zurich"); + summary.Should().NotContain("yacht"); + } + + [Fact] + public async Task NothingWorthSummarisingReturnsNullNotAnEmptySummary() + { + // An empty summary would satisfy every has-a-summary check while saying nothing. + var summary = await new DeterministicEntitySummarySynthesizer() + .SynthesizeAsync(Alice, [F("f-1", "owns", "a yacht", confidence: 0.1)]); + + summary.Should().BeNull(); + } + + [Fact] + public async Task NothingWorthSummarisingIsNotStored() + { + StoreHolds(F("f-1", "owns", "a yacht", confidence: 0.1)); + + var result = await CreateSut().RefreshAsync(Alice, Scope); + + result.Should().BeNull(); + await _summaries.DidNotReceiveWithAnyArgs().UpsertAsync(default!, default); + } + + [Fact] + public async Task SupersededFactsAreNotSummarised() + { + // Correctness, not merely staleness: the summary must not assert something the store has + // already stopped believing at the moment it is written. + StoreHolds( + F("f-1", "lives in", "Basel", invalidatedAt: DateTimeOffset.UnixEpoch), + F("f-2", "lives in", "Zurich")); + + var summary = await CreateSut().RefreshAsync(Alice, Scope); + + summary!.Content.Should().Contain("Zurich"); + summary.Content.Should().NotContain("Basel"); + summary.SourceFactIds.Should().Equal("f-2"); + } +} diff --git a/tests/AgentMemory.Tests.Unit/Queries/CypherQuerySnapshot.snap b/tests/AgentMemory.Tests.Unit/Queries/CypherQuerySnapshot.snap index 9e6504bed..df6ad5eea 100644 --- a/tests/AgentMemory.Tests.Unit/Queries/CypherQuerySnapshot.snap +++ b/tests/AgentMemory.Tests.Unit/Queries/CypherQuerySnapshot.snap @@ -1,4 +1,4 @@ -# Cypher Query Snapshot — 159 queries +# Cypher Query Snapshot — 160 queries # Auto-generated by CypherQuerySnapshotTests. # To regenerate: set UPDATE_CYPHER_SNAPSHOTS=1 and re-run the test. @@ -237,6 +237,30 @@ UNWIND $items AS item e.updated_at = datetime() RETURN e +## EntitySummaryQueries.Upsert +MERGE (s:EntitySummary {entity_id: $entityId, owner_key: $ownerKey}) +SET s.id = $summaryId, + s.content = $content, + s.source_fact_ids = $sourceFactIds, + s.source_fingerprint = $sourceFingerprint, + s.owner_id = $ownerId, + s.generated_at = datetime($generatedAt) +WITH s +OPTIONAL MATCH (e:Entity {id: $entityId}) +FOREACH (_ IN CASE WHEN e IS NULL THEN [] ELSE [1] END | + MERGE (s)-[:SUMMARISES]->(e)) +WITH s +// Provenance to the facts the text was written from. Rebuilt wholesale rather than merged, +// because a summary regenerated from a smaller fact set must not keep edges to sources it no +// longer draws on -- that would claim provenance the content cannot support. +OPTIONAL MATCH (s)-[old:EXTRACTED_FROM]->(:Fact) +DELETE old +WITH s +UNWIND $sourceFactIds AS factId +MATCH (f:Fact {id: factId}) +MERGE (s)-[:EXTRACTED_FROM]->(f) +RETURN s + ## ExtractorQueries.CreateExtractedByRelationship MATCH (e:Entity {id: $entity_id}) MATCH (ex:Extractor {name: $extractor_name}) diff --git a/tests/AgentMemory.Tests.Unit/Queries/CypherQuerySnapshotTests.cs b/tests/AgentMemory.Tests.Unit/Queries/CypherQuerySnapshotTests.cs index acc076371..7da2a37b3 100644 --- a/tests/AgentMemory.Tests.Unit/Queries/CypherQuerySnapshotTests.cs +++ b/tests/AgentMemory.Tests.Unit/Queries/CypherQuerySnapshotTests.cs @@ -29,7 +29,8 @@ private static string ResolveSnapshotPath([CallerFilePath] string? sourceFile = private static readonly HashSet KnownNodeLabels = new(StringComparer.Ordinal) { - "ConsolidationRun", "Conversation", "Entity", "Extractor", "Fact", "MemoryReadAudit", "Message", + "ConsolidationRun", "Conversation", "Entity", "EntitySummary", "Extractor", "Fact", + "MemoryReadAudit", "Message", "Migration", "Preference", "ReasoningStep", "ReasoningTrace", "Schema", "Tool", "ToolCall" }; @@ -37,7 +38,7 @@ private static string ResolveSnapshotPath([CallerFilePath] string? sourceFile = // ── Expected query inventory count ──────────────────────────────────────── // Update this constant whenever queries are deliberately added or removed. - private const int ExpectedQueryCount = 156; // +FactQueries.MentionCounts (PLAN 10.3: reads the INGESTION-side salience counter for a whole candidate set in one query -- deliberately not the :MemoryReadAudit trail, which counts our own retrievals and would make ranking self-reinforcing). // +SchemaQueries.TraceKindIndex (PLAN 7.2: the promotion marker separating an episode from a reusable procedure -- seekable so a procedures-only search is a seek rather than a post-filter over the whole label, and mirroring trace_success_idx). // +SchemaQueries.FactOwnerKeyIndex (measured: FindDuplicate runs on every fact write and had NO index entry point, planning a full 20,000-row scan; this seeks 100). // +SchemaQueries.MessageSessionIndex (measured: Neo4j will not seek a composite from a leading-column predicate alone, so 0007's (session_id, timestamp) gave GetRecentBySession nothing). // +SchemaQueries.MessageSessionTimestampIndex (Message.session_id is the predicate of the PRIMARY short-term recall path -- MessageQueries.cs:201, run on essentially every turn -- and nothing indexed it, so the plan was proportional to every message in the store rather than to the session; composite because TemporalQueries.cs:99-103 adds a trailing timestamp range to the same equality). // +SchemaQueries.FactPredicateKeyIndex (the same shape one column over: predicate_key sits at column 3 of fact_merge_key_idx and Neo4j serves a composite only on a matching PREFIX, so FactQueries.cs:88 had no index entry point at all -- a full :Fact scan across all owners whenever relation-completeness retrieval fires). // +SchemaQueries.MemoryReadAuditMemoryIdIndex (BUG-A2: HistoryQueries OPTIONAL MATCHes MemoryReadAudit on memory_id, which nothing indexed, so every history row scanned the whole label -- and the label grows ~25 rows per recall, so it degrades with TIME rather than data size). // +SchemaQueries.FactMergeKeyIndex (L11: every fact MERGE was an all-:Fact label scan; the composite {subject_key, object_key, predicate_key, owner_key} index is what makes the range-index key cap real for facts, which is why IndexKeyBudget.EnsureCompositeIndexable lands with it). // -1: SearchByCanonicalPredicates became an owner-conditional *method* (excluded, like GetBySubject) when the audit found it ignored IncludeShared and coerced a null-owner scope to the shared bucket. // +FactQueries.SelectFactsMissingCanonicalKeys/ApplyCanonicalKeys (Phase 1.1: canonical identity needs a C#-driven backfill, since Cypher's toLower diverges from ToLowerInvariant on U+0130). // +FactQueries.SearchByCanonicalPredicates (G3B.13: top-K is a relevance cutoff and cannot answer "how many", so a relation is retrieved whole via predicate_key). // +SchemaQueries.ShowIndexStates (BUG-S1: only vector-index dimensions were validated at bootstrap, so a FAILED range index degraded silently into full scans). base + ConsolidationQueries/SchemaQueries/TOUCHED/ConflictQueries consts. +MessageQueries.GetAllBySession (cycle-3); +SchemaQueries.ShowConstraintNames/ShowIndexNames (schema-check CLI); +SchemaPersistenceQueries.Save/DeactivateByName/LoadActiveByName/LoadByNameVersion/List/Exists/DeleteById (G4 schema-node CRUD, 7); +MemoryReadAudit constraint/index. Owner-conditional queries are *methods* (excluded): EntityQueries.ApplyConfidenceDelta/Delete/MergeEntities/SearchByLocation/SearchInBoundingBox/GetByType/FindSimilarByEmbedding, FactQueries.Delete/FindByTriple, PreferenceQueries.Delete, DecayQueries.PruneEntities/PruneFacts/PrunePreferences, ExtractorQueries.GetEntityProvenance, ReasoningQueries.ListTracesBySession (R2 owner-scoped 2026-06-13), ReasoningQueries.DeleteBySession (R6-C owner-scoped 2026-06-20: const→method, −1); +PreferenceQueries.UpsertBatch/RelationshipQueries.UpsertBatch (feat-04, +2). + private const int ExpectedQueryCount = 157; // +EntitySummaryQueries.Upsert (PLAN 12.7 / S1: GetByEntity and DeleteByEntity are owner-conditional METHODS and so excluded, like GetBySubject). // +FactQueries.MentionCounts (PLAN 10.3: reads the INGESTION-side salience counter for a whole candidate set in one query -- deliberately not the :MemoryReadAudit trail, which counts our own retrievals and would make ranking self-reinforcing). // +SchemaQueries.TraceKindIndex (PLAN 7.2: the promotion marker separating an episode from a reusable procedure -- seekable so a procedures-only search is a seek rather than a post-filter over the whole label, and mirroring trace_success_idx). // +SchemaQueries.FactOwnerKeyIndex (measured: FindDuplicate runs on every fact write and had NO index entry point, planning a full 20,000-row scan; this seeks 100). // +SchemaQueries.MessageSessionIndex (measured: Neo4j will not seek a composite from a leading-column predicate alone, so 0007's (session_id, timestamp) gave GetRecentBySession nothing). // +SchemaQueries.MessageSessionTimestampIndex (Message.session_id is the predicate of the PRIMARY short-term recall path -- MessageQueries.cs:201, run on essentially every turn -- and nothing indexed it, so the plan was proportional to every message in the store rather than to the session; composite because TemporalQueries.cs:99-103 adds a trailing timestamp range to the same equality). // +SchemaQueries.FactPredicateKeyIndex (the same shape one column over: predicate_key sits at column 3 of fact_merge_key_idx and Neo4j serves a composite only on a matching PREFIX, so FactQueries.cs:88 had no index entry point at all -- a full :Fact scan across all owners whenever relation-completeness retrieval fires). // +SchemaQueries.MemoryReadAuditMemoryIdIndex (BUG-A2: HistoryQueries OPTIONAL MATCHes MemoryReadAudit on memory_id, which nothing indexed, so every history row scanned the whole label -- and the label grows ~25 rows per recall, so it degrades with TIME rather than data size). // +SchemaQueries.FactMergeKeyIndex (L11: every fact MERGE was an all-:Fact label scan; the composite {subject_key, object_key, predicate_key, owner_key} index is what makes the range-index key cap real for facts, which is why IndexKeyBudget.EnsureCompositeIndexable lands with it). // -1: SearchByCanonicalPredicates became an owner-conditional *method* (excluded, like GetBySubject) when the audit found it ignored IncludeShared and coerced a null-owner scope to the shared bucket. // +FactQueries.SelectFactsMissingCanonicalKeys/ApplyCanonicalKeys (Phase 1.1: canonical identity needs a C#-driven backfill, since Cypher's toLower diverges from ToLowerInvariant on U+0130). // +FactQueries.SearchByCanonicalPredicates (G3B.13: top-K is a relevance cutoff and cannot answer "how many", so a relation is retrieved whole via predicate_key). // +SchemaQueries.ShowIndexStates (BUG-S1: only vector-index dimensions were validated at bootstrap, so a FAILED range index degraded silently into full scans). base + ConsolidationQueries/SchemaQueries/TOUCHED/ConflictQueries consts. +MessageQueries.GetAllBySession (cycle-3); +SchemaQueries.ShowConstraintNames/ShowIndexNames (schema-check CLI); +SchemaPersistenceQueries.Save/DeactivateByName/LoadActiveByName/LoadByNameVersion/List/Exists/DeleteById (G4 schema-node CRUD, 7); +MemoryReadAudit constraint/index. Owner-conditional queries are *methods* (excluded): EntityQueries.ApplyConfidenceDelta/Delete/MergeEntities/SearchByLocation/SearchInBoundingBox/GetByType/FindSimilarByEmbedding, FactQueries.Delete/FindByTriple, PreferenceQueries.Delete, DecayQueries.PruneEntities/PruneFacts/PrunePreferences, ExtractorQueries.GetEntityProvenance, ReasoningQueries.ListTracesBySession (R2 owner-scoped 2026-06-13), ReasoningQueries.DeleteBySession (R6-C owner-scoped 2026-06-20: const→method, −1); +PreferenceQueries.UpsertBatch/RelationshipQueries.UpsertBatch (feat-04, +2). // ── MemberData source ───────────────────────────────────────────────────── From d034d73f86eef5dcfe63d43a043151e8ab50ccdc Mon Sep 17 00:00:00 2001 From: Jose Luis Latorre Millas Date: Wed, 12 Aug 2026 22:58:25 +0200 Subject: [PATCH 27/31] Make memory legible without making the block the store (S4) The honest assessment of this store was "capable but opaque": memory could be queried but not seen. `agentmemory block` renders what an owner holds -- entities, facts, preferences -- as one readable page. The design decision that matters is what was left out. Block-memory systems elsewhere let the agent edit its own block and hand it back, and at that point the block is the store: every provenance edge, trust level and supersession record in the graph describes a shadow of what the system actually believes. So there is no `block --write` and no parser turning a block back into memories. The absence is the feature. Legibility stays actionable another way: every line prints its memory id, so a human who spots something wrong acts on that exact memory through invalidate or supersede -- audited, attributable, reversible -- instead of rewriting prose and hoping something parses it. Two failure modes get explicit guards, because both look exactly like a correct block to whoever is reading it. Superseded memories are dropped twice, at the query and again in the renderer, since a caller forgetting IncludeInvalidated would otherwise get retracted claims presented as current. And truncation is counted into OmittedCount *and* stated in the rendered text -- a block that quietly stops short reads as "this is everything", which invites the conclusion that a missing memory was never stored. Built on the existing audited IMemoryHistoryService read path rather than new queries, which is what makes calling it a projection true rather than aspirational. Rendering is deterministic, so diffing two blocks shows what changed in memory rather than in the renderer. 4,343 unit tests green (+14). Release build 0 warnings. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01PgDgctPpbTziBNE2RT8VdE --- docs/architecture.md | 2 +- .../Domain/MemoryBlock.cs | 59 +++++ .../Memory/MemoryBlockRenderer.cs | 188 ++++++++++++++ .../AbstractionsContractGuardTests.cs | 2 +- .../Memory/MemoryBlockRendererTests.cs | 245 ++++++++++++++++++ .../Commands/MemoryCommands.cs | 55 ++++ tools/AgentMemory.Cli/Program.cs | 9 +- 7 files changed, 557 insertions(+), 3 deletions(-) create mode 100644 src/AgentMemory.Abstractions/Domain/MemoryBlock.cs create mode 100644 src/AgentMemory.Core/Memory/MemoryBlockRenderer.cs create mode 100644 tests/AgentMemory.Tests.Unit/Memory/MemoryBlockRendererTests.cs diff --git a/docs/architecture.md b/docs/architecture.md index 6db669163..5fcc6c31f 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -142,7 +142,7 @@ graph TD | **Purpose** | Domain contracts — all models, interfaces, and configuration types shared across the system | | **Dependencies** | **Microsoft.Extensions.AI.Abstractions** 10.8.0 (approved, D-AR2-1) — .NET BCL otherwise (multi-targets net8.0/net9.0/net10.0) | | **MUST NOT reference** | Neo4j.Driver, Microsoft.Agents.*, any GraphRAG SDK, any MCP SDK, any NuGet package **except** Microsoft.Extensions.AI.Abstractions | -| **Key types** | 54 domain records (Conversation, Message, Entity, Fact, Preference, Relationship, MemoryHistoryQuery, MemoryHistoryRecord, ReasoningTrace, ReasoningStep, ToolCall, ToolCallStats, IngestionItemOutcome, MemoryContextRankedItem, MemoryContextSectionDiagnostics, UnifiedExtractionResult, ExtractionWindow, EntitySummary, etc.), 44 service interfaces (incl. `IMemoryIsolationPolicy`, `IUnifiedMemoryExtractor`, `IMultiSessionUnifiedMemoryExtractor`, and `IMemoryReranker`, `IEntitySummaryService`), 12 repository interfaces, 16 configuration types (incl. `MemoryRankingOptions`, `MemoryIsolationOptions`), 26 enums (incl. `MemoryProfile`, `RankingIntent`, `DuplicateStatus`, `EntityMatchType`, `MemoryNodeKind`, `MemoryOperationAccess`, `MemoryIsolationMode`, `IngestionStatus`, `IngestionStage`, `IngestionItemStatus`, `MemoryItemKind`, `IngestionFailureMode`, `MemoryTrustLevel`, `AssistantContentMode`, `TemporalValidityMode`, `TraceKind`, `ExtractionProvenanceMode`) | +| **Key types** | 56 domain records (Conversation, Message, Entity, Fact, Preference, Relationship, MemoryHistoryQuery, MemoryHistoryRecord, ReasoningTrace, ReasoningStep, ToolCall, ToolCallStats, IngestionItemOutcome, MemoryContextRankedItem, MemoryContextSectionDiagnostics, UnifiedExtractionResult, ExtractionWindow, EntitySummary, MemoryBlock, etc.), 44 service interfaces (incl. `IMemoryIsolationPolicy`, `IUnifiedMemoryExtractor`, `IMultiSessionUnifiedMemoryExtractor`, and `IMemoryReranker`, `IEntitySummaryService`), 12 repository interfaces, 16 configuration types (incl. `MemoryRankingOptions`, `MemoryIsolationOptions`), 26 enums (incl. `MemoryProfile`, `RankingIntent`, `DuplicateStatus`, `EntityMatchType`, `MemoryNodeKind`, `MemoryOperationAccess`, `MemoryIsolationMode`, `IngestionStatus`, `IngestionStage`, `IngestionItemStatus`, `MemoryItemKind`, `IngestionFailureMode`, `MemoryTrustLevel`, `AssistantContentMode`, `TemporalValidityMode`, `TraceKind`, `ExtractionProvenanceMode`) | **Namespace structure:** ``` diff --git a/src/AgentMemory.Abstractions/Domain/MemoryBlock.cs b/src/AgentMemory.Abstractions/Domain/MemoryBlock.cs new file mode 100644 index 000000000..41c61930d --- /dev/null +++ b/src/AgentMemory.Abstractions/Domain/MemoryBlock.cs @@ -0,0 +1,59 @@ +namespace AgentMemory.Abstractions.Domain; + +/// +/// One addressable line of a memory block (S4). +/// +/// The id of the underlying memory — what makes this line actionable. +/// Entity, fact or preference. +/// The rendered line. +public sealed record MemoryBlockLine(string MemoryId, MemoryItemKind Kind, string Text); + +/// +/// A small, human-readable projection of what memory holds (S4). +/// +/// +/// +/// The honest assessment of this store was "capable but opaque". A developer could not look at +/// an owner's memory and see it; they could query it, which is not the same thing. This is the view +/// that closes that gap. +/// +/// +/// Rendered on demand and never stored. That is the deliberate divergence from the +/// block-memory designs this borrows legibility from. A stored block is a second copy of the truth +/// that drifts from the graph the moment anything is written, and once an agent is allowed to edit +/// the block directly, the block — not the audited store — becomes where memory actually lives. Every +/// provenance edge, every trust level and every supersession record then describes a shadow of what +/// the system believes. +/// +/// +/// So this is a read surface. Changing memory goes through the ordinary write path, which +/// stamps trust, records provenance and leaves a history. To make that practical rather than +/// obstructive, each line carries its : a human who spots +/// something wrong can act on that exact item instead of rewriting prose and hoping a parser agrees. +/// +/// +public sealed record MemoryBlock +{ + /// The owner this block describes; null = shared/global. + public string? OwnerId { get; init; } + + /// When the block was rendered. It is a snapshot, and it says so. + public required DateTimeOffset RenderedAtUtc { get; init; } + + /// The lines, in rendering order. + public required IReadOnlyList Lines { get; init; } + + /// + /// How many memories were left out because the block was full. + /// + /// + /// Counted and surfaced rather than silently dropped. A block is a view a human trusts to + /// show them what memory holds; one that quietly stops short reads as "this is everything" and is + /// worse than no view at all, because it invites the conclusion that a missing memory was never + /// stored. + /// + public int OmittedCount { get; init; } + + /// Whether anything was left out. + public bool IsTruncated => OmittedCount > 0; +} diff --git a/src/AgentMemory.Core/Memory/MemoryBlockRenderer.cs b/src/AgentMemory.Core/Memory/MemoryBlockRenderer.cs new file mode 100644 index 000000000..a3638cf1e --- /dev/null +++ b/src/AgentMemory.Core/Memory/MemoryBlockRenderer.cs @@ -0,0 +1,188 @@ +using System.Globalization; +using System.Text; +using AgentMemory.Abstractions.Domain; + +namespace AgentMemory.Core.Memory; + +/// +/// Renders a memory block from what the graph currently holds (S4). +/// +/// +/// +/// Deterministic and projection-only. Two renders over the same memory produce the same bytes, so a +/// developer diffing two blocks sees what changed in memory rather than what changed in a sampler. +/// +/// +/// There is no inverse. Nothing here parses a block back into memories, and that absence is the +/// design: a round-trippable block invites an agent to edit it and hand it back, at which point the +/// block becomes the real store and the graph's provenance, trust and supersession records describe +/// something else. +/// +/// +internal static class MemoryBlockRenderer +{ + /// + /// Builds a block from the supplied memories, newest and most-confident first within each kind. + /// + /// Entities to include. + /// Facts to include. Superseded facts are excluded. + /// Preferences to include. + /// Render timestamp. + /// Owner the block describes. + /// + /// Hard cap. Anything beyond it is counted into rather + /// than dropped quietly. + /// + public static MemoryBlock Render( + IReadOnlyList entities, + IReadOnlyList facts, + IReadOnlyList preferences, + DateTimeOffset renderedAtUtc, + string? ownerId = null, + int maxLines = 50) + { + ArgumentNullException.ThrowIfNull(entities); + ArgumentNullException.ThrowIfNull(facts); + ArgumentNullException.ThrowIfNull(preferences); + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(maxLines); + + var all = new List(); + + all.AddRange(entities + .OrderByDescending(e => e.Confidence) + .ThenBy(e => e.Name, StringComparer.OrdinalIgnoreCase) + .Select(e => new MemoryBlockLine( + e.EntityId, + MemoryItemKind.Entity, + string.IsNullOrWhiteSpace(e.Type) ? e.Name : $"{e.Name} — {e.Type}"))); + + all.AddRange(facts + // A block is what memory believes NOW. Showing a superseded fact beside a live one, with + // no per-item history in view, presents a retracted claim as current. + .Where(f => f.InvalidatedAtUtc is null) + .OrderByDescending(f => f.Confidence) + .ThenBy(f => f.Subject, StringComparer.OrdinalIgnoreCase) + .ThenBy(f => f.Predicate, StringComparer.OrdinalIgnoreCase) + .Select(f => new MemoryBlockLine( + f.FactId, + MemoryItemKind.Fact, + $"{f.Subject} {f.Predicate} {f.Object}"))); + + all.AddRange(preferences + .OrderByDescending(p => p.Confidence) + .ThenBy(p => p.Category, StringComparer.OrdinalIgnoreCase) + .Select(p => new MemoryBlockLine( + p.PreferenceId, + MemoryItemKind.Preference, + string.IsNullOrWhiteSpace(p.Category) ? p.PreferenceText : $"{p.Category}: {p.PreferenceText}"))); + + return new MemoryBlock + { + OwnerId = ownerId, + RenderedAtUtc = renderedAtUtc, + Lines = all.Take(maxLines).ToList(), + OmittedCount = Math.Max(0, all.Count - maxLines), + }; + } + + /// + /// Builds a block from memory-history rows (S4). + /// + /// + /// + /// The practical entry point. IMemoryHistoryService already answers "what does memory hold + /// for this owner" across all three kinds through an audited read path, so the block is a + /// rendering of an existing query rather than a new way to reach into the store — which is the + /// point of calling it a projection. + /// + /// + /// Invalidated rows are dropped here as well as being excludable at the query. Belt and braces on + /// purpose: a caller that forgets IncludeInvalidated = false would otherwise get a block + /// presenting retracted claims as current, and nothing about it would look wrong. + /// + /// + public static MemoryBlock Render( + IReadOnlyList records, + DateTimeOffset renderedAtUtc, + string? ownerId = null, + int maxLines = 50) + { + ArgumentNullException.ThrowIfNull(records); + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(maxLines); + + var lines = records + .Where(r => r.Status != MemoryHistoryStatus.Invalidated && r.InvalidatedAtUtc is null) + .OrderBy(r => KindOrder(r.Kind)) + .ThenBy(r => r.Summary, StringComparer.OrdinalIgnoreCase) + .Select(r => new MemoryBlockLine(r.Id, ToItemKind(r.Kind), r.Summary)) + .ToList(); + + return new MemoryBlock + { + OwnerId = ownerId, + RenderedAtUtc = renderedAtUtc, + Lines = lines.Take(maxLines).ToList(), + OmittedCount = Math.Max(0, lines.Count - maxLines), + }; + } + + private static int KindOrder(MemoryHistoryKind kind) => kind switch + { + MemoryHistoryKind.Entity => 0, + MemoryHistoryKind.Fact => 1, + _ => 2, + }; + + private static MemoryItemKind ToItemKind(MemoryHistoryKind kind) => kind switch + { + MemoryHistoryKind.Entity => MemoryItemKind.Entity, + MemoryHistoryKind.Fact => MemoryItemKind.Fact, + _ => MemoryItemKind.Preference, + }; + + /// + /// Renders a block as the text a human reads. + /// + /// + /// Ids are shown, not hidden behind the prose. They are what turns "that fact is wrong" into an + /// action against a specific memory, which is the whole reason this stayed a read surface: the + /// correction goes through the audited write path rather than by editing this text. + /// + public static string ToText(MemoryBlock block) + { + ArgumentNullException.ThrowIfNull(block); + + var builder = new StringBuilder(); + builder.Append("# Memory"); + if (!string.IsNullOrWhiteSpace(block.OwnerId)) + builder.Append(" — ").Append(block.OwnerId); + builder.Append(string.Create( + CultureInfo.InvariantCulture, + $"\n_rendered {block.RenderedAtUtc:u} — a snapshot, not a document to edit_\n")); + + foreach (var kind in new[] { MemoryItemKind.Entity, MemoryItemKind.Fact, MemoryItemKind.Preference }) + { + var lines = block.Lines.Where(l => l.Kind == kind).ToList(); + if (lines.Count == 0) continue; + + builder.Append('\n').Append("## ").Append(kind).Append("\n"); + foreach (var line in lines) + builder.Append(string.Create(CultureInfo.InvariantCulture, $"- {line.Text} `{line.MemoryId}`\n")); + } + + if (block.IsTruncated) + { + // Stated in the text itself, not only on the object. A reader who sees a block ending + // cleanly assumes they have seen everything, and the whole value of this view is that it + // can be trusted to show what memory holds. + builder.Append(string.Create( + CultureInfo.InvariantCulture, + $"\n_… {block.OmittedCount} more not shown (block limit reached)_\n")); + } + + if (block.Lines.Count == 0) + builder.Append("\n_Memory holds nothing for this owner yet._\n"); + + return builder.ToString(); + } +} diff --git a/tests/AgentMemory.Tests.Unit/Infrastructure/AbstractionsContractGuardTests.cs b/tests/AgentMemory.Tests.Unit/Infrastructure/AbstractionsContractGuardTests.cs index 8545dc95e..b870126d4 100644 --- a/tests/AgentMemory.Tests.Unit/Infrastructure/AbstractionsContractGuardTests.cs +++ b/tests/AgentMemory.Tests.Unit/Infrastructure/AbstractionsContractGuardTests.cs @@ -18,7 +18,7 @@ public sealed class AbstractionsContractGuardTests // Counts mirrored in docs/architecture.md §3.1 and docs/design.md §5/§6. private const int DocumentedServiceInterfaces = 44; // +IEntitySummarySynthesizer, +IEntitySummaryService (PLAN 12.7 / S1) private const int DocumentedRepositoryInterfaces = 12; // +IEntitySummaryRepository (PLAN 12.7 / S1) - private const int DocumentedDomainRecords = 54; // +EntitySummary (PLAN 12.7 / S1; EntitySummarySource is a record STRUCT and the guard counts reference records only) + private const int DocumentedDomainRecords = 56; // +EntitySummary (PLAN 12.7 / S1; EntitySummarySource is a record STRUCT and the guard counts reference records only); +MemoryBlock, +MemoryBlockLine (PLAN 12.9 / S4) private const int DocumentedEnums = 29; // +TraceKind (7.1), +ExtractionProvenanceMode (9.3) private static IEnumerable PublicTypes() => diff --git a/tests/AgentMemory.Tests.Unit/Memory/MemoryBlockRendererTests.cs b/tests/AgentMemory.Tests.Unit/Memory/MemoryBlockRendererTests.cs new file mode 100644 index 000000000..8cdce5cd0 --- /dev/null +++ b/tests/AgentMemory.Tests.Unit/Memory/MemoryBlockRendererTests.cs @@ -0,0 +1,245 @@ +using AgentMemory.Abstractions.Domain; +using AgentMemory.Core.Memory; +using FluentAssertions; +using Xunit; + +namespace AgentMemory.Tests.Unit.Memory; + +/// +/// A memory block a human can actually read (S4), and the two things it must never become. +/// +/// +/// +/// The honest assessment of this store was "capable but opaque": memory could be queried but +/// not seen. This closes that gap — while refusing the part of the block-memory design that +/// makes the block itself the store. +/// +/// +/// So the two properties under test beyond formatting are: a block never presents a retracted claim +/// as current, and it never ends silently when there was more to show. Both failures look exactly +/// like a correct block to the person reading it. +/// +/// +public sealed class MemoryBlockRendererTests +{ + private static readonly DateTimeOffset At = new(2026, 8, 12, 9, 0, 0, TimeSpan.Zero); + + private static Entity E(string id, string name, string type = "Person", double confidence = 0.9) => new() + { + EntityId = id, Name = name, Type = type, Confidence = confidence, + CreatedAtUtc = At, OwnerId = "alice", + }; + + private static Fact F( + string id, string subject, string predicate, string @object, + double confidence = 0.9, DateTimeOffset? invalidatedAt = null) => new() + { + FactId = id, Subject = subject, Predicate = predicate, Object = @object, + Confidence = confidence, CreatedAtUtc = At, OwnerId = "alice", + InvalidatedAtUtc = invalidatedAt, + }; + + private static Preference P(string id, string category, string text, double confidence = 0.9) => new() + { + PreferenceId = id, Category = category, PreferenceText = text, + Confidence = confidence, CreatedAtUtc = At, OwnerId = "alice", + }; + + private static MemoryBlock Render( + Entity[]? entities = null, Fact[]? facts = null, Preference[]? preferences = null, int maxLines = 50) => + MemoryBlockRenderer.Render( + entities ?? [], facts ?? [], preferences ?? [], At, "alice", maxLines); + + // ── what a block must never do ──────────────────────────────────────── + + [Fact] + public void ASupersededFactIsNotShown() + { + // A block says what memory believes NOW, and shows no per-item history. A retracted claim + // sitting beside a live one is simply presented as true. + var block = Render(facts: + [ + F("f-1", "Alice", "lives in", "Basel", invalidatedAt: At), + F("f-2", "Alice", "lives in", "Zurich"), + ]); + + block.Lines.Should().ContainSingle().Which.MemoryId.Should().Be("f-2"); + } + + [Fact] + public void TruncationIsCountedRatherThanSilent() + { + // THE failure mode of any bounded view. A block that quietly stops short reads as "this is + // everything", and invites the conclusion that a missing memory was never stored at all. + var facts = Enumerable.Range(1, 10) + .Select(i => F($"f-{i}", "Alice", "likes", $"thing-{i}")).ToArray(); + + var block = Render(facts: facts, maxLines: 4); + + block.Lines.Should().HaveCount(4); + block.OmittedCount.Should().Be(6); + block.IsTruncated.Should().BeTrue(); + } + + [Fact] + public void TruncationIsVisibleInTheTextToo() + { + // The object carries OmittedCount; the person reading the rendered block sees only the text. + var facts = Enumerable.Range(1, 10) + .Select(i => F($"f-{i}", "Alice", "likes", $"thing-{i}")).ToArray(); + + MemoryBlockRenderer.ToText(Render(facts: facts, maxLines: 4)) + .Should().Contain("6 more not shown"); + } + + // ── what makes it actionable ────────────────────────────────────────── + + [Fact] + public void EveryLineCarriesItsMemoryId() + { + // The reason this could stay a read surface. A human who spots something wrong acts on that + // exact memory through the audited write path, instead of editing prose and hoping something + // parses it back. + var block = Render( + entities: [E("e-1", "Alice")], + facts: [F("f-1", "Alice", "lives in", "Zurich")], + preferences: [P("p-1", "food", "vegetarian")]); + + block.Lines.Select(l => l.MemoryId).Should().Equal("e-1", "f-1", "p-1"); + MemoryBlockRenderer.ToText(block).Should().Contain("`f-1`"); + } + + [Fact] + public void KindsAreGroupedUnderHeadings() + { + var text = MemoryBlockRenderer.ToText(Render( + entities: [E("e-1", "Alice")], + facts: [F("f-1", "Alice", "lives in", "Zurich")])); + + text.Should().Contain("## Entity").And.Contain("## Fact"); + } + + [Fact] + public void RenderingIsDeterministic() + { + // Two renders of the same memory produce the same bytes, so a developer diffing blocks sees + // what changed in memory rather than what changed in the renderer. + Fact[] facts = [F("f-2", "Alice", "works at", "Acme"), F("f-1", "Alice", "lives in", "Zurich")]; + + MemoryBlockRenderer.ToText(Render(facts: facts)) + .Should().Be(MemoryBlockRenderer.ToText(Render(facts: facts.AsEnumerable().Reverse().ToArray()))); + } + + [Fact] + public void MoreConfidentMemoriesComeFirst() + { + var block = Render(facts: + [ + F("f-low", "Alice", "maybe likes", "sailing", confidence: 0.55), + F("f-high", "Alice", "lives in", "Zurich", confidence: 0.99), + ]); + + block.Lines.First().MemoryId.Should().Be("f-high"); + } + + // ── shape ───────────────────────────────────────────────────────────── + + [Fact] + public void AnEmptyMemorySaysSoRatherThanRenderingNothing() + { + // A blank block is indistinguishable from a broken renderer. + var text = MemoryBlockRenderer.ToText(Render()); + + text.Should().Contain("holds nothing"); + } + + [Fact] + public void TheBlockDeclaresItIsASnapshot() + { + // It is rendered on demand and never stored. Saying so in the text is what stops a reader + // treating it as a document to edit -- which is exactly how a block quietly becomes the store. + MemoryBlockRenderer.ToText(Render(entities: [E("e-1", "Alice")])) + .Should().Contain("snapshot, not a document to edit"); + } + + [Fact] + public void AZeroLineBudgetIsRejected() + { + // Not silently treated as unlimited or as empty: both produce a block that misrepresents + // memory, in opposite directions. + var act = () => Render(maxLines: 0); + + act.Should().Throw(); + } +} + +/// +/// Rendering a block from the memory-history rows the CLI actually feeds it (S4). +/// +/// +/// The typed overload is precise; this is the one a person reaches through agentmemory block, +/// and it is where a mistake would be seen rather than merely possible. +/// +public sealed class MemoryBlockFromHistoryTests +{ + private static readonly DateTimeOffset At = new(2026, 8, 12, 9, 0, 0, TimeSpan.Zero); + + private static MemoryHistoryRecord R( + string id, MemoryHistoryKind kind, string summary, + MemoryHistoryStatus status = MemoryHistoryStatus.Live, + DateTimeOffset? invalidatedAt = null) => new() + { + Id = id, + Kind = kind, + Summary = summary, + Status = status, + CreatedAtUtc = At, + InvalidatedAtUtc = invalidatedAt, + OwnerId = "alice", + }; + + [Fact] + public void InvalidatedRowsAreDroppedEvenIfTheQueryAskedForThem() + { + // Belt and braces, deliberately. A caller forgetting IncludeInvalidated = false would get a + // block presenting retracted claims as current, and nothing about it would look wrong. + var block = MemoryBlockRenderer.Render( + [ + R("f-1", MemoryHistoryKind.Fact, "Alice lives in Basel", + status: MemoryHistoryStatus.Invalidated, invalidatedAt: At), + R("f-2", MemoryHistoryKind.Fact, "Alice lives in Zurich"), + ], At, "alice"); + + block.Lines.Should().ContainSingle().Which.MemoryId.Should().Be("f-2"); + } + + [Fact] + public void KindsAreGroupedEntitiesThenFactsThenPreferences() + { + var block = MemoryBlockRenderer.Render( + [ + R("p-1", MemoryHistoryKind.Preference, "food: vegetarian"), + R("f-1", MemoryHistoryKind.Fact, "Alice lives in Zurich"), + R("e-1", MemoryHistoryKind.Entity, "Alice"), + ], At, "alice"); + + block.Lines.Select(l => l.MemoryId).Should().Equal("e-1", "f-1", "p-1"); + } + + [Fact] + public void TruncationIsStillCounted() + { + var rows = Enumerable.Range(1, 8) + .Select(i => R($"f-{i}", MemoryHistoryKind.Fact, $"fact {i}")).ToArray(); + + var block = MemoryBlockRenderer.Render(rows, At, "alice", maxLines: 3); + + block.Lines.Should().HaveCount(3); + block.OmittedCount.Should().Be(5); + } + + [Fact] + public void AnOwnerWithNoMemoriesRendersAnHonestEmptyBlock() => + MemoryBlockRenderer.ToText(MemoryBlockRenderer.Render([], At, "alice")) + .Should().Contain("holds nothing"); +} diff --git a/tools/AgentMemory.Cli/Commands/MemoryCommands.cs b/tools/AgentMemory.Cli/Commands/MemoryCommands.cs index 2308d54a7..b65604104 100644 --- a/tools/AgentMemory.Cli/Commands/MemoryCommands.cs +++ b/tools/AgentMemory.Cli/Commands/MemoryCommands.cs @@ -4,6 +4,7 @@ using AgentMemory.Abstractions.Options; using AgentMemory.Abstractions.Repositories; using AgentMemory.Abstractions.Services; +using AgentMemory.Core.Memory; using AgentMemory.Neo4j.Infrastructure; using AgentMemory.Neo4j.Queries; using AgentMemory.Neo4j.Schema.Parity; @@ -492,3 +493,57 @@ private static string DescribeQuery(MemoryHistoryQuery query) private static string FormatOptional(DateTimeOffset? value) => value is null ? "open" : Format(value.Value); } + +/// +/// Renders a human-readable block of what memory holds for an owner (S4). +/// +/// +/// +/// The gap this closes was described honestly in the analysis as "capable but opaque": memory +/// could be queried but not seen. history answers questions about one memory; +/// this shows the shape of the whole thing at a glance. +/// +/// +/// There is no write counterpart, deliberately. Block-memory designs elsewhere let the agent +/// edit its own block, and at that point the block is the store: every provenance edge, trust level +/// and supersession record in the graph then describes a shadow of what the system actually believes. +/// Each line therefore prints its memory id, so a correction is made against that exact memory +/// through invalidate or supersede — audited, attributable, and reversible. +/// +/// +public sealed class BlockCommand(IMemoryHistoryService service, TextWriter output) +{ + public async Task ExecuteAsync( + string? owner, + string? limitValue, + CancellationToken cancellationToken = default) + { + var limit = 50; + if (!string.IsNullOrWhiteSpace(limitValue) + && (!int.TryParse(limitValue, NumberStyles.Integer, CultureInfo.InvariantCulture, out limit) + || limit <= 0)) + { + output.WriteLine("error: block --limit must be a positive integer."); + return 1; + } + + var records = await service.GetHistoryAsync( + new MemoryHistoryQuery + { + OwnerId = string.IsNullOrWhiteSpace(owner) ? null : owner, + // The block is what memory believes NOW; retracted claims shown beside live ones, + // with no per-item history in view, simply read as true. + IncludeInvalidated = false, + // Over-fetch so the omitted count is real. Asking for exactly `limit` would make a + // full block indistinguishable from an exactly-full one. + Limit = limit + 1, + }, + cancellationToken).ConfigureAwait(false); + + var block = MemoryBlockRenderer.Render( + records, DateTimeOffset.UtcNow, owner, limit); + + output.Write(MemoryBlockRenderer.ToText(block)); + return 0; + } +} diff --git a/tools/AgentMemory.Cli/Program.cs b/tools/AgentMemory.Cli/Program.cs index bff1dca38..d68ac6ee4 100644 --- a/tools/AgentMemory.Cli/Program.cs +++ b/tools/AgentMemory.Cli/Program.cs @@ -21,7 +21,7 @@ return cli.Command is null ? 1 : 0; } -var known = new[] { "migrate", "bootstrap", "consolidate", "decay", "conflicts", "schema-parity", "schema-check", "invalidate", "supersede", "history", "evaluate", "perf" }; +var known = new[] { "migrate", "bootstrap", "consolidate", "decay", "conflicts", "schema-parity", "schema-check", "invalidate", "supersede", "history", "evaluate", "perf", "block" }; if (!known.Contains(cli.Command, StringComparer.OrdinalIgnoreCase)) { Console.Error.WriteLine($"error: unknown command '{cli.Command}'."); @@ -225,6 +225,13 @@ string Resolve(string option, string defaultValue, string cfgKey, string envKey) liveOnly: cli.HasFlag("live-only"), ownOnly: cli.HasFlag("own-only"), limitValue: cli.Get("limit")), + // S4. "ours is capable but opaque" -- memory could be queried but not SEEN. Read-only by + // design: there is deliberately no `block --write`, because a block an agent can hand back + // becomes the store, and the graph's provenance and supersession records then describe a + // shadow of what the system believes. + "block" => await new BlockCommand( + sp.GetRequiredService(), output) + .ExecuteAsync(cli.Get("owner"), cli.Get("limit")), "evaluate" => await new EvaluationCommand( sp.GetRequiredService(), sp.GetRequiredService(), From 72c29cfe868351537321761c8bb40137c649923b Mon Sep 17 00:00:00 2001 From: Jose Luis Latorre Millas Date: Wed, 12 Aug 2026 23:03:34 +0200 Subject: [PATCH 28/31] Document the bulk-ingestion path and bound its concurrency (rank 27) Loading a backlog was always possible, and both obvious ways are wrong in opposite directions. A serial loop over ExtractAndPersistAsync is correct and takes hours. An unbounded Parallel.ForEachAsync saturates the provider quota and the connection pool, and the damage lands on everything else sharing the process: median latency barely moves, which is what makes it hard to notice, while p99 degrades 20-70% under saturation. IngestBulkAsync is a default interface method on IMemoryIngestion that paces calls the host could have made itself. Deliberately composed rather than new machinery: a separate bulk pipeline would be a second place for trust stamping, provenance and owner scoping to drift out of agreement with the per-request path, and that drift would only surface in the corpus months later. Failures come back per request with their index into the submitted list. A bulk API returning "8,412 of 10,000 succeeded" tells the caller they have a problem and nothing about which inputs to retry, so the realistic response is to re-run all ten thousand -- more expensive than the failure was. NotAttemptedCount is kept separate from FailedCount for the same reason: re-running a request that was never attempted is always safe, re-running one that failed part-way may not be. A stop-on-error run completes normally and returns its report; a cancellation the caller requested still throws. Discarding the record of what did succeed because something failed is the opposite of useful. Its own test found a real bug: the semaphore wait sat outside the try block, so tasks queued behind the gate when a stop-on-error run halted threw OperationCanceledException straight through Task.WhenAll instead of being recorded as unattempted. Scoped honestly, per the plan: this is a public surface plus documentation. It is not the 10.70x throughput figure, which is ten-owner concurrency and has always been available to any caller willing to parallelise -- what is documented is the safe way to do that. 4,352 unit tests green (+9). Release build 0 warnings. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01PgDgctPpbTziBNE2RT8VdE --- docs/architecture.md | 2 +- docs/performance/README.md | 1 + docs/performance/bulk-ingestion.md | 75 +++++++ .../Domain/Extraction/BulkIngestion.cs | 77 +++++++ .../Services/IMemoryIngestion.cs | 96 ++++++++ .../AbstractionsContractGuardTests.cs | 2 +- .../Services/BulkIngestionTests.cs | 205 ++++++++++++++++++ 7 files changed, 456 insertions(+), 2 deletions(-) create mode 100644 docs/performance/bulk-ingestion.md create mode 100644 src/AgentMemory.Abstractions/Domain/Extraction/BulkIngestion.cs create mode 100644 tests/AgentMemory.Tests.Unit/Services/BulkIngestionTests.cs diff --git a/docs/architecture.md b/docs/architecture.md index 5fcc6c31f..81d0aa5ae 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -142,7 +142,7 @@ graph TD | **Purpose** | Domain contracts — all models, interfaces, and configuration types shared across the system | | **Dependencies** | **Microsoft.Extensions.AI.Abstractions** 10.8.0 (approved, D-AR2-1) — .NET BCL otherwise (multi-targets net8.0/net9.0/net10.0) | | **MUST NOT reference** | Neo4j.Driver, Microsoft.Agents.*, any GraphRAG SDK, any MCP SDK, any NuGet package **except** Microsoft.Extensions.AI.Abstractions | -| **Key types** | 56 domain records (Conversation, Message, Entity, Fact, Preference, Relationship, MemoryHistoryQuery, MemoryHistoryRecord, ReasoningTrace, ReasoningStep, ToolCall, ToolCallStats, IngestionItemOutcome, MemoryContextRankedItem, MemoryContextSectionDiagnostics, UnifiedExtractionResult, ExtractionWindow, EntitySummary, MemoryBlock, etc.), 44 service interfaces (incl. `IMemoryIsolationPolicy`, `IUnifiedMemoryExtractor`, `IMultiSessionUnifiedMemoryExtractor`, and `IMemoryReranker`, `IEntitySummaryService`), 12 repository interfaces, 16 configuration types (incl. `MemoryRankingOptions`, `MemoryIsolationOptions`), 26 enums (incl. `MemoryProfile`, `RankingIntent`, `DuplicateStatus`, `EntityMatchType`, `MemoryNodeKind`, `MemoryOperationAccess`, `MemoryIsolationMode`, `IngestionStatus`, `IngestionStage`, `IngestionItemStatus`, `MemoryItemKind`, `IngestionFailureMode`, `MemoryTrustLevel`, `AssistantContentMode`, `TemporalValidityMode`, `TraceKind`, `ExtractionProvenanceMode`) | +| **Key types** | 59 domain records (Conversation, Message, Entity, Fact, Preference, Relationship, MemoryHistoryQuery, MemoryHistoryRecord, ReasoningTrace, ReasoningStep, ToolCall, ToolCallStats, IngestionItemOutcome, MemoryContextRankedItem, MemoryContextSectionDiagnostics, UnifiedExtractionResult, ExtractionWindow, EntitySummary, MemoryBlock, BulkIngestionResult, etc.), 44 service interfaces (incl. `IMemoryIsolationPolicy`, `IUnifiedMemoryExtractor`, `IMultiSessionUnifiedMemoryExtractor`, and `IMemoryReranker`, `IEntitySummaryService`), 12 repository interfaces, 16 configuration types (incl. `MemoryRankingOptions`, `MemoryIsolationOptions`), 26 enums (incl. `MemoryProfile`, `RankingIntent`, `DuplicateStatus`, `EntityMatchType`, `MemoryNodeKind`, `MemoryOperationAccess`, `MemoryIsolationMode`, `IngestionStatus`, `IngestionStage`, `IngestionItemStatus`, `MemoryItemKind`, `IngestionFailureMode`, `MemoryTrustLevel`, `AssistantContentMode`, `TemporalValidityMode`, `TraceKind`, `ExtractionProvenanceMode`) | **Namespace structure:** ``` diff --git a/docs/performance/README.md b/docs/performance/README.md index c9002eef1..6420c6204 100644 --- a/docs/performance/README.md +++ b/docs/performance/README.md @@ -7,6 +7,7 @@ What AgentMemory costs you per agent turn, how that cost is measured, and how to | **README.md** (this file) | The two phases, what is and isn't measured, how to run it yourself | | [`hermetic-S.json`](../../eng/perf/baselines/hermetic-S.json) | Machine-readable counter + quality baseline enforced on pull requests | | [baseline-1.3.0.md](baseline-1.3.0.md) | The measured cost model for release 1.3.0 | +| [bulk-ingestion.md](bulk-ingestion.md) | Loading a backlog without degrading p99 for everyone else in the process | --- diff --git a/docs/performance/bulk-ingestion.md b/docs/performance/bulk-ingestion.md new file mode 100644 index 000000000..f734738e3 --- /dev/null +++ b/docs/performance/bulk-ingestion.md @@ -0,0 +1,75 @@ +# Bulk ingestion + +Loading a backlog — a year of transcripts, an export from another system, a migration — is a +different problem from ingesting a live turn, and the two obvious ways to do it are both wrong. + +A serial loop over `ExtractAndPersistAsync` is correct and takes hours: extraction is a model call, +and one at a time means one round trip at a time. + +An unbounded `Parallel.ForEachAsync` is fast until it isn't. It saturates the provider quota and the +Neo4j connection pool, and the damage is not to the bulk load — it is to **everything else sharing +the process**. Median latency barely moves, which is what makes it hard to notice; p99 degrades by +20–70% under saturation, so the tenants who suffer are the ones already having the worst time. + +## The documented path + +```csharp +var result = await memory.IngestBulkAsync( + requests, + new BulkIngestionOptions { MaxConcurrency = 4 }, + cancellationToken); + +Console.WriteLine($"{result.SucceededCount} ingested, {result.FailedCount} failed, " + + $"{result.NotAttemptedCount} not attempted"); + +foreach (var failure in result.Outcomes.Where(o => !o.Succeeded)) + Console.WriteLine($" request {failure.Index}: {failure.Error!.Message}"); +``` + +`IngestBulkAsync` is a **default interface method on `IMemoryIngestion`**. It paces calls the host +could have made itself; it is not a second ingestion pipeline. That distinction matters more than it +sounds: a separate bulk path would be a second place for trust stamping, provenance and owner scoping +to drift out of agreement with the per-request path, and the drift would only show up in the corpus +months later. + +## What the options actually buy + +| Option | Default | Why | +|---|---|---| +| `MaxConcurrency` | `4` | High enough to beat a serial loop, low enough that a caller who never tunes it does not saturate a shared quota by accident. Bounding this does not make the bulk load faster — it stops the bulk load making everything else slower. | +| `ContinueOnError` | `true` | In a ten-thousand-conversation load, one malformed transcript aborting the other 9,999 is rarely what was wanted. Continuing is not ignoring: every failure comes back in `Outcomes`. | + +## Failures are reported per request, not counted + +A bulk API that returns *"8,412 of 10,000 succeeded"* tells you that you have a problem and nothing +about which inputs to retry — so the realistic response is to re-run all ten thousand, which costs +more than the failure did. Every outcome carries its `Index` into the submitted list and the +`Exception` that stopped it. + +`NotAttemptedCount` is kept separate from `FailedCount` on purpose. **"Tried and failed" and "never +tried" call for different actions**: re-running a request that was never attempted is always safe, +re-running one that failed part-way may not be. Collapsing them makes a stopped run look like a +partially broken corpus. + +## Cancellation + +A run stopped by `ContinueOnError = false` **completes normally** and returns its report — the +failure that stopped it is already in `Outcomes`, and throwing would discard the record of what did +succeed. + +A cancellation *you* requested throws `OperationCanceledException`. That is a different event and is +not swallowed into a tidy report. + +## What this is not + +This is not the 10.70× figure from the throughput analysis. That number is ten-owner concurrent +throughput, and it has always been available to any caller willing to parallelise across owners — +it is a property of the store, not of this API. What is documented here is the safe way to do the +parallelising, which is a smaller and more honest claim. + +## Related + +- [Performance baselines](README.md) +- Extraction batching (`MaxConcurrentExtractionBatches`) bounds concurrency *inside* one extraction; + this bounds concurrency *across* requests. They compose, and multiply — a `MaxConcurrency` of 4 + over batches of 6 is 24 in-flight provider calls. diff --git a/src/AgentMemory.Abstractions/Domain/Extraction/BulkIngestion.cs b/src/AgentMemory.Abstractions/Domain/Extraction/BulkIngestion.cs new file mode 100644 index 000000000..553abd4d4 --- /dev/null +++ b/src/AgentMemory.Abstractions/Domain/Extraction/BulkIngestion.cs @@ -0,0 +1,77 @@ +namespace AgentMemory.Abstractions.Domain; + +/// +/// How a bulk ingestion run is paced (rank 27). +/// +/// +/// +/// Backpressure is the whole point. A caller with ten thousand conversations to load will otherwise +/// issue them all at once, and the failure is not a clean rejection — it is provider throttling, +/// connection-pool exhaustion and a p99 that degrades for every other tenant sharing the +/// process while median latency looks untouched. +/// +/// +public sealed record BulkIngestionOptions +{ + /// + /// How many requests may be in flight at once. + /// + /// + /// Defaults to 4: high enough to be worth using over a serial loop, low enough that a caller who + /// never tunes it does not saturate a shared provider quota by accident. The measured effect of + /// bounding this is small at the median and 20–70% at p99 under saturation — which is to + /// say it does not make bulk loading faster, it stops bulk loading making everything else slower. + /// + public int MaxConcurrency { get; init; } = 4; + + /// + /// Whether a failed request stops the run. + /// + /// + /// Defaults to : in a ten-thousand-conversation load, one malformed + /// transcript aborting the other 9,999 is rarely what the caller wanted. Failures are returned + /// per request rather than thrown, so continuing is not the same as ignoring. + /// + public bool ContinueOnError { get; init; } = true; +} + +/// What happened to one request in a bulk run. +/// Position in the submitted list, so a failure can be traced to its input. +/// The extraction result, or if it failed. +/// The failure, or if it succeeded. +public sealed record BulkIngestionOutcome(int Index, ExtractionResult? Result, Exception? Error) +{ + /// Whether this request was ingested. + public bool Succeeded => Error is null; +} + +/// +/// The result of a bulk ingestion run (rank 27). +/// +/// +/// Per-request outcomes, not an aggregate count. A bulk API that returns "8,412 of 10,000 +/// succeeded" tells the caller they have a problem and nothing about which inputs to retry — so the +/// realistic response is to re-run everything, which is worse than the failure. Every outcome carries +/// its index and its exception. +/// +public sealed record BulkIngestionResult +{ + /// One entry per submitted request, in submission order. + public required IReadOnlyList Outcomes { get; init; } + + /// How many requests were ingested. + public int SucceededCount => Outcomes.Count(o => o.Succeeded); + + /// How many requests failed. + public int FailedCount => Outcomes.Count(o => !o.Succeeded); + + /// + /// Requests that were never attempted because the run stopped early. + /// + /// + /// Distinct from a failure, and reported separately for the same reason 6.5's trace counts are + /// nullable: "tried and failed" and "never tried" call for different actions, and collapsing them + /// makes a stopped run look like a partially broken corpus. + /// + public int NotAttemptedCount { get; init; } +} diff --git a/src/AgentMemory.Abstractions/Services/IMemoryIngestion.cs b/src/AgentMemory.Abstractions/Services/IMemoryIngestion.cs index 47ed4cf4b..235ac609b 100644 --- a/src/AgentMemory.Abstractions/Services/IMemoryIngestion.cs +++ b/src/AgentMemory.Abstractions/Services/IMemoryIngestion.cs @@ -86,4 +86,100 @@ Task ExtractFromConversationAsync( string conversationId, string? userId = null, CancellationToken cancellationToken = default); + + /// + /// Ingests many extraction requests with bounded concurrency (rank 27). + /// + /// + /// + /// The documented bulk path. Loading a backlog was always possible — call + /// in a loop, or in a Parallel.ForEachAsync — and both + /// obvious ways are wrong in opposite directions: the serial loop wastes hours, and the unbounded + /// parallel one saturates the provider quota and the connection pool, degrading p99 for every + /// other tenant in the process while median latency looks fine. + /// + /// + /// Composed, not new machinery. This paces calls the host could have made itself; it does + /// not add a second ingestion path. That matters because a separate bulk pipeline would be a + /// second place for trust stamping, provenance and owner scoping to drift out of agreement with + /// the per-request one. + /// + /// + /// A default implementation is supplied so the interface stays SemVer-compatible: an existing + /// implementation keeps compiling and gets a correct, if + /// unoptimised, bulk path for free. + /// + /// + async Task IngestBulkAsync( + IReadOnlyList requests, + BulkIngestionOptions? options = null, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(requests); + options ??= new BulkIngestionOptions(); + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(options.MaxConcurrency); + + var outcomes = new BulkIngestionOutcome?[requests.Count]; + using var gate = new SemaphoreSlim(options.MaxConcurrency, options.MaxConcurrency); + // Cancels the remaining work when ContinueOnError is false. Linked rather than replacing the + // caller's token, so a stop-on-error run still honours an outer cancellation. + using var stop = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + + var running = new List(requests.Count); + for (var i = 0; i < requests.Count; i++) + { + if (stop.IsCancellationRequested) break; + + var index = i; + running.Add(Task.Run(async () => + { + try + { + await gate.WaitAsync(stop.Token).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + // Queued behind the gate when the run stopped. Left unattempted -- and caught + // HERE rather than by the block below, whose finally would Release a slot this + // task never acquired. + return; + } + + try + { + var result = await ExtractAndPersistAsync(requests[index], stop.Token) + .ConfigureAwait(false); + outcomes[index] = new BulkIngestionOutcome(index, result, null); + } + catch (OperationCanceledException) when (stop.IsCancellationRequested) + { + // Left null: never attempted, or abandoned mid-flight. Recorded as + // NotAttemptedCount rather than as a failure, because re-running a cancelled + // request is correct and re-running a failed one may not be. + } + catch (Exception ex) + { + outcomes[index] = new BulkIngestionOutcome(index, null, ex); + if (!options.ContinueOnError) await stop.CancelAsync().ConfigureAwait(false); + } + finally + { + gate.Release(); + } + }, CancellationToken.None)); + } + + await Task.WhenAll(running).ConfigureAwait(false); + + // An outer cancellation is still an error for the caller; a stop-on-first-failure is not, + // because the failure it stopped for is already reported in the outcomes. + cancellationToken.ThrowIfCancellationRequested(); + + var completed = outcomes.Where(o => o is not null).Select(o => o!).ToList(); + return new BulkIngestionResult + { + Outcomes = completed, + NotAttemptedCount = requests.Count - completed.Count, + }; + } } diff --git a/tests/AgentMemory.Tests.Unit/Infrastructure/AbstractionsContractGuardTests.cs b/tests/AgentMemory.Tests.Unit/Infrastructure/AbstractionsContractGuardTests.cs index b870126d4..a6e9b60b3 100644 --- a/tests/AgentMemory.Tests.Unit/Infrastructure/AbstractionsContractGuardTests.cs +++ b/tests/AgentMemory.Tests.Unit/Infrastructure/AbstractionsContractGuardTests.cs @@ -18,7 +18,7 @@ public sealed class AbstractionsContractGuardTests // Counts mirrored in docs/architecture.md §3.1 and docs/design.md §5/§6. private const int DocumentedServiceInterfaces = 44; // +IEntitySummarySynthesizer, +IEntitySummaryService (PLAN 12.7 / S1) private const int DocumentedRepositoryInterfaces = 12; // +IEntitySummaryRepository (PLAN 12.7 / S1) - private const int DocumentedDomainRecords = 56; // +EntitySummary (PLAN 12.7 / S1; EntitySummarySource is a record STRUCT and the guard counts reference records only); +MemoryBlock, +MemoryBlockLine (PLAN 12.9 / S4) + private const int DocumentedDomainRecords = 59; // +EntitySummary (PLAN 12.7 / S1; EntitySummarySource is a record STRUCT and the guard counts reference records only); +MemoryBlock, +MemoryBlockLine (PLAN 12.9 / S4); +BulkIngestionOptions, +BulkIngestionOutcome, +BulkIngestionResult (PLAN 2.9 / rank 27) private const int DocumentedEnums = 29; // +TraceKind (7.1), +ExtractionProvenanceMode (9.3) private static IEnumerable PublicTypes() => diff --git a/tests/AgentMemory.Tests.Unit/Services/BulkIngestionTests.cs b/tests/AgentMemory.Tests.Unit/Services/BulkIngestionTests.cs new file mode 100644 index 000000000..7b97205da --- /dev/null +++ b/tests/AgentMemory.Tests.Unit/Services/BulkIngestionTests.cs @@ -0,0 +1,205 @@ +using AgentMemory.Abstractions.Domain; +using AgentMemory.Abstractions.Services; +using FluentAssertions; +using Xunit; + +namespace AgentMemory.Tests.Unit.Services; + +/// +/// The documented bulk path and its backpressure (rank 27). +/// +/// +/// +/// Loading a backlog was always possible, and both obvious ways are wrong in opposite directions: a +/// serial loop wastes hours, and an unbounded Parallel.ForEachAsync saturates the provider +/// quota and the connection pool — degrading p99 for every other tenant in the process while median +/// latency looks untouched. +/// +/// +/// So the tests are about the two things a bulk API is usually careless with: how many calls it makes +/// at once, and what it tells you when some of them fail. +/// +/// +public sealed class BulkIngestionTests +{ + /// Records concurrency and fails on demand; nothing else about ingestion is exercised. + private sealed class RecordingIngestion : IMemoryIngestion + { + private int _inFlight; + private readonly Func? _failOn; + private readonly TimeSpan _delay; + + public RecordingIngestion(Func? failOn = null, TimeSpan? delay = null) + { + _failOn = failOn; + _delay = delay ?? TimeSpan.FromMilliseconds(20); + } + + public int PeakConcurrency; + public readonly List Seen = []; + private readonly Lock _sync = new(); + + public async Task ExtractAndPersistAsync( + ExtractionRequest request, CancellationToken cancellationToken = default) + { + var now = Interlocked.Increment(ref _inFlight); + InterlockedMax(ref PeakConcurrency, now); + try + { + await Task.Delay(_delay, cancellationToken).ConfigureAwait(false); + lock (_sync) Seen.Add(request.SessionId); + + var index = int.Parse(request.SessionId.Split('-')[1], System.Globalization.CultureInfo.InvariantCulture); + if (_failOn?.Invoke(index) == true) throw new InvalidOperationException($"boom {index}"); + return new ExtractionResult(); + } + finally + { + Interlocked.Decrement(ref _inFlight); + } + } + + private static void InterlockedMax(ref int target, int value) + { + int seen; + while ((seen = Volatile.Read(ref target)) < value + && Interlocked.CompareExchange(ref target, value, seen) != seen) { } + } + + public Task AddMessageAsync( + string sessionId, string conversationId, string role, string content, + IReadOnlyDictionary? metadata = null, CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task AddMessageWithIdAsync( + string messageId, string sessionId, string conversationId, string role, string content, + IReadOnlyDictionary? metadata = null, CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task> AddMessagesAsync( + IEnumerable messages, CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task ExtractFromSessionAsync( + string sessionId, string? userId = null, CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task ExtractFromConversationAsync( + string conversationId, string? userId = null, CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + } + + private static IReadOnlyList Requests(int count) => + Enumerable.Range(0, count) + .Select(i => new ExtractionRequest { Messages = [], SessionId = $"s-{i}" }) + .ToList(); + + [Fact] + public async Task ConcurrencyIsBounded() + { + // THE test. Everything else here is about reporting; this is the property the feature exists + // for, and the one an unbounded Parallel.ForEachAsync gets wrong. + var ingestion = new RecordingIngestion(); + + // Through the interface: a default interface method is not inherited into the implementing type. + await ((IMemoryIngestion)ingestion).IngestBulkAsync(Requests(20), new BulkIngestionOptions { MaxConcurrency = 3 }); + + ingestion.PeakConcurrency.Should().BeLessThanOrEqualTo(3); + } + + [Fact] + public async Task EveryRequestIsIngested() + { + var ingestion = new RecordingIngestion(); + + var result = await ((IMemoryIngestion)ingestion).IngestBulkAsync(Requests(12), new BulkIngestionOptions { MaxConcurrency = 4 }); + + result.SucceededCount.Should().Be(12); + ingestion.Seen.Should().HaveCount(12); + } + + [Fact] + public async Task AFailureIsReportedAgainstItsInputRatherThanThrown() + { + // A bulk API that returns "8,412 of 10,000 succeeded" tells the caller they have a problem and + // nothing about which inputs to retry, so the realistic response is to re-run everything -- + // worse than the failure itself. + var ingestion = new RecordingIngestion(failOn: i => i == 3); + + var result = await ((IMemoryIngestion)ingestion).IngestBulkAsync(Requests(6)); + + result.FailedCount.Should().Be(1); + var failure = result.Outcomes.Single(o => !o.Succeeded); + failure.Index.Should().Be(3); + failure.Error.Should().BeOfType(); + } + + [Fact] + public async Task OneBadInputDoesNotAbortTheRunByDefault() + { + // In a ten-thousand-conversation load, one malformed transcript stopping the other 9,999 is + // rarely what the caller wanted -- and continuing is not ignoring, because the failure is + // returned. + var ingestion = new RecordingIngestion(failOn: i => i == 0); + + var result = await ((IMemoryIngestion)ingestion).IngestBulkAsync(Requests(8), new BulkIngestionOptions { MaxConcurrency = 2 }); + + result.SucceededCount.Should().Be(7); + result.FailedCount.Should().Be(1); + } + + [Fact] + public async Task StopOnErrorLeavesTheRestUnattemptedRatherThanFailed() + { + // "Tried and failed" and "never tried" call for different actions: re-running a request that + // was never attempted is always correct, re-running a failed one may not be. Collapsing them + // makes a stopped run look like a partially broken corpus. + var ingestion = new RecordingIngestion(failOn: i => i == 0, delay: TimeSpan.FromMilliseconds(50)); + + var result = await ((IMemoryIngestion)ingestion).IngestBulkAsync( + Requests(20), new BulkIngestionOptions { MaxConcurrency = 1, ContinueOnError = false }); + + result.FailedCount.Should().Be(1); + result.NotAttemptedCount.Should().BeGreaterThan(0); + (result.SucceededCount + result.FailedCount + result.NotAttemptedCount).Should().Be(20); + } + + [Fact] + public async Task AnOuterCancellationStillThrows() + { + // A stop-on-error run completes normally -- the failure is in the outcomes. A cancellation + // the CALLER requested is a different thing and must not be swallowed into a tidy report. + using var cts = new CancellationTokenSource(); + var ingestion = new RecordingIngestion(delay: TimeSpan.FromMilliseconds(200)); + + var run = ((IMemoryIngestion)ingestion).IngestBulkAsync( + Requests(20), new BulkIngestionOptions { MaxConcurrency = 2 }, cts.Token); + await cts.CancelAsync(); + + await FluentActions.Awaiting(() => run).Should().ThrowAsync(); + } + + [Fact] + public async Task AnEmptyRunIsNotAnError() => + (await ((IMemoryIngestion)new RecordingIngestion()).IngestBulkAsync([])).Outcomes.Should().BeEmpty(); + + [Fact] + public async Task ZeroConcurrencyIsRejected() + { + // Not silently treated as unbounded, which is precisely the behaviour this exists to prevent. + var act = async () => await ((IMemoryIngestion)new RecordingIngestion()) + .IngestBulkAsync(Requests(1), new BulkIngestionOptions { MaxConcurrency = 0 }); + + await act.Should().ThrowAsync(); + } + + [Fact] + public void TheDefaultsAreConservative() + { + // A caller who never tunes this must not saturate a shared provider quota by accident. + var options = new BulkIngestionOptions(); + + options.MaxConcurrency.Should().Be(4); + options.ContinueOnError.Should().BeTrue(); + } +} From 580241108b08e10a2c9f86b440012151b1472043 Mon Sep 17 00:00:00 2001 From: Jose Luis Latorre Millas Date: Wed, 12 Aug 2026 23:17:45 +0200 Subject: [PATCH 29/31] Degrade recall under a latency budget, visibly (rank 13) RecallOptions.LatencyBudget bounds how long context assembly may take. Sections still running when it expires are dropped and the context comes back without them, rather than the caller waiting on the slowest one. The marking is the feature, not the timing. A section cut short reports Searched = true and Returned = 0, which is byte-identical to a section that ran to completion and genuinely found nothing -- so the obvious implementation degrades completely invisibly, and the caller answers confidently from memory that was never consulted. Every dropped section now carries TimedOut in its diagnostics, and the context carries LatencyBudgetExceeded so a caller can notice without inspecting each section. LatencyBudgetExceeded is deliberately a separate flag from Truncated. Truncation is the context budget trimming memories that were retrieved; this is a retrieval that never came back. A caller told only "truncated" would reasonably conclude the memories exist and were dropped to fit, when they may never have been looked at. Abandoned queries are not cancelled. They are already in the driver and tearing them down buys nothing the caller is waiting for -- it turns a wasted result into a wasted result plus a cancellation storm. Their faults are observed so a slow section that also fails does not surface later as an unobserved task exception. Off by default, and with no budget the wait is the original unconditional Task.WhenAll, byte for byte. Wiring verified red-before-fix: 2 of 7 tests fail with the branch disabled, and the red run takes 1m30s against 0.8s green -- the runtime difference is itself the proof that the budget short-circuits rather than merely relabelling a completed wait. 4,359 unit (+7) and 352 integration green. Release 0 warnings. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01PgDgctPpbTziBNE2RT8VdE --- .../Domain/Context/MemoryContext.cs | 18 ++ .../Domain/Context/MemoryContextSection.cs | 19 +- .../Options/RecallOptions.cs | 23 +++ .../Services/MemoryContextAssembler.cs | 111 +++++++++- .../Services/RecallLatencyBudgetTests.cs | 190 ++++++++++++++++++ 5 files changed, 349 insertions(+), 12 deletions(-) create mode 100644 tests/AgentMemory.Tests.Unit/Services/RecallLatencyBudgetTests.cs diff --git a/src/AgentMemory.Abstractions/Domain/Context/MemoryContext.cs b/src/AgentMemory.Abstractions/Domain/Context/MemoryContext.cs index d628d151f..c9351f8f8 100644 --- a/src/AgentMemory.Abstractions/Domain/Context/MemoryContext.cs +++ b/src/AgentMemory.Abstractions/Domain/Context/MemoryContext.cs @@ -106,6 +106,24 @@ public sealed record MemoryContext /// public bool Truncated { get; init; } + /// + /// At least one section was abandoned because the recall latency budget expired (rank 13). + /// + /// + /// + /// Separate from , which means something else. Truncation is the + /// context budget trimming items that were retrieved; this is a retrieval that never came back. + /// A caller told only "truncated" would reasonably conclude the memories exist and were dropped + /// to fit — here they may exist and were never looked at. + /// + /// + /// Which sections were lost is on their own diagnostics. This flag exists so a caller can notice + /// at all without inspecting every section, because a partial context that looks complete is + /// worse than a slow one: the answer is given confidently from memory that was never consulted. + /// + /// + public bool LatencyBudgetExceeded { get; init; } + /// /// Additional metadata. /// diff --git a/src/AgentMemory.Abstractions/Domain/Context/MemoryContextSection.cs b/src/AgentMemory.Abstractions/Domain/Context/MemoryContextSection.cs index 80e05caa8..5c662c96c 100644 --- a/src/AgentMemory.Abstractions/Domain/Context/MemoryContextSection.cs +++ b/src/AgentMemory.Abstractions/Domain/Context/MemoryContextSection.cs @@ -74,14 +74,31 @@ public sealed record MemoryContextRankedItem( /// /// Lowest returned score, or under the same rules. /// The similarity floor in force, so "all below threshold" is checkable. +/// +/// Whether this section was abandoned because the recall latency budget expired (rank 13). Without +/// it, a section cut short is byte-identical to one that searched and found nothing. +/// public sealed record MemoryContextSectionDiagnostics( bool Searched, int RequestedLimit, int Returned, double? TopScore, double? LowestScore, - double MinimumScore) + double MinimumScore, + bool TimedOut = false) { + /// + /// This section was abandoned because the recall latency budget expired (rank 13). + /// + /// + /// The reason a latency budget needs a flag at all. A section cut short reports + /// Searched = true and Returned = 0, which is byte-identical to a section that ran + /// to completion and genuinely found nothing. Without this, degrading under load would look + /// exactly like an owner having no memories — and a partial recall that looks complete is worse + /// than a slow one, because the caller answers confidently from a context they believe is whole. + /// + public bool AbandonedToLatencyBudget => TimedOut; + /// /// The retrieval ran and came back with nothing — as opposed to never having run. /// diff --git a/src/AgentMemory.Abstractions/Options/RecallOptions.cs b/src/AgentMemory.Abstractions/Options/RecallOptions.cs index 281e5a572..6fec728dd 100644 --- a/src/AgentMemory.Abstractions/Options/RecallOptions.cs +++ b/src/AgentMemory.Abstractions/Options/RecallOptions.cs @@ -5,6 +5,29 @@ /// public sealed record RecallOptions { + /// + /// How long recall may spend assembling context before returning what it has (rank 13). + /// + /// + /// + /// — the default — waits for every section, which is exactly today's + /// behaviour. When set, sections still running when the budget expires are dropped and the + /// context comes back without them. + /// + /// + /// Degradation is marked, never silent. Every dropped section reports + /// TimedOut in its diagnostics, and RecallResult.Metadata["latencyBudgetExceeded"] + /// is set. A section cut short otherwise looks identical to one that searched and found nothing, + /// and a caller answering from a context it believes is complete is worse off than one that + /// waited. + /// + /// + /// Abandoned work is not cancelled mid-flight — the queries are already in the driver, and + /// tearing them down buys nothing the caller is waiting for. Their results are simply not used. + /// + /// + public TimeSpan? LatencyBudget { get; init; } + /// Maximum recent messages to include. public int MaxRecentMessages { get; init; } = 10; diff --git a/src/AgentMemory.Core/Services/MemoryContextAssembler.cs b/src/AgentMemory.Core/Services/MemoryContextAssembler.cs index 5649003e8..b8bd22433 100644 --- a/src/AgentMemory.Core/Services/MemoryContextAssembler.cs +++ b/src/AgentMemory.Core/Services/MemoryContextAssembler.cs @@ -113,7 +113,8 @@ private static MemoryContextSectionDiagnostics Diagnose( int requestedLimit, IReadOnlyList items, IReadOnlyList ranked, - double minimumScore) + double minimumScore, + bool timedOut = false) { // Scores come from the ranked items, which exist only when the provider could supply them. // An unscoreable section reports null rather than a placeholder: zero is a real score. @@ -125,7 +126,7 @@ private static MemoryContextSectionDiagnostics Diagnose( } return new MemoryContextSectionDiagnostics( - searched, requestedLimit, items.Count, top, low, minimumScore); + searched, requestedLimit, items.Count, top, low, minimumScore, timedOut); } /// @@ -221,6 +222,13 @@ public async Task AssembleContextAsync( IReadOnlyList<(Fact Fact, double Score)> factScores = Array.Empty<(Fact, double)>(); IReadOnlyList<(ReasoningTrace Trace, double Score)> traceScores = Array.Empty<(ReasoningTrace, double)>(); + // Rank 13 bookkeeping, at METHOD scope: the fan-out lives in a nested block and these are read + // by the diagnostics built after it closes. A section that searched and found nothing and one + // that never came back produce an identical section; only one means the answer is incomplete. + var budgetExceeded = false; + bool droppedRecent = false, droppedRelevant = false, droppedEntities = false, + droppedPreferences = false, droppedFacts = false, droppedTraces = false; + // J2.2 resolution, computed once at method scope so it can be both used by the fact search // and reported on the context. Which relations a question resolved to is the difference // between "expansion had nothing to expand" and "expansion ran and did not help", and those @@ -399,12 +407,42 @@ public async Task AssembleContextAsync( // One handle per section, so a fault in any of them is still observed here rather than // surfacing later as an unobserved task exception. - await Task.WhenAll( + Task[] sections = + [ recentTask, relevantTask, entitiesScoredTask ?? (Task)entitiesTask, preferencesScoredTask ?? (Task)preferencesTask, factsScoredTask ?? (Task)factsTask, - tracesScoredTask ?? (Task)tracesTask).ConfigureAwait(false); + tracesScoredTask ?? (Task)tracesTask, + ]; + + // Rank 13. With no budget this is the original unconditional wait, byte for byte. + if (recallOpts.LatencyBudget is { } latencyBudget + && await WaitWithinBudgetAsync(sections, latencyBudget, cancellationToken).ConfigureAwait(false)) + { + // Each unfinished section is replaced by an empty result AND recorded, because the two + // have to be told apart downstream: "searched and found nothing" and "never came back" + // produce an identical section, and only one of them means the answer is incomplete. + budgetExceeded = true; + if (DropIfUnfinished>(ref recentTask, [])) droppedRecent = true; + if (DropIfUnfinished(ref relevantTask, RelevantMessageSearchResult.Empty)) droppedRelevant = true; + if (entitiesScoredTask is not null) + droppedEntities = DropIfUnfinished>(ref entitiesScoredTask, []); + else droppedEntities = DropIfUnfinished>(ref entitiesTask, []); + if (preferencesScoredTask is not null) + droppedPreferences = DropIfUnfinished>(ref preferencesScoredTask, []); + else droppedPreferences = DropIfUnfinished>(ref preferencesTask, []); + if (factsScoredTask is not null) + droppedFacts = DropIfUnfinished(ref factsScoredTask, new ScoredFactSearchResult([], [])); + else droppedFacts = DropIfUnfinished>(ref factsTask, []); + if (tracesScoredTask is not null) + droppedTraces = DropIfUnfinished>(ref tracesScoredTask, []); + else droppedTraces = DropIfUnfinished>(ref tracesTask, []); + + _logger.LogWarning( + "Recall latency budget of {Budget}ms expired; returning partial context.", + latencyBudget.TotalMilliseconds); + } recentMessages = await recentTask.ConfigureAwait(false); var relevantResult = await relevantTask.ConfigureAwait(false); @@ -523,7 +561,7 @@ await Task.WhenAll( // No vector involved: session-scoped and time-ordered, so the limit alone decides. Diagnostics = recallOpts.IncludeDiagnostics ? Diagnose(recallOpts.MaxRecentMessages > 0, recallOpts.MaxRecentMessages, - recentMessages, Array.Empty(), minScore) + recentMessages, Array.Empty(), minScore, droppedRecent) : null }, RelevantMessages = new MemoryContextSection @@ -532,7 +570,7 @@ await Task.WhenAll( RankedItems = relevantRanked, Diagnostics = recallOpts.IncludeDiagnostics ? Diagnose(vectorSearchRan && recallOpts.MaxRelevantMessages > 0, - recallOpts.MaxRelevantMessages, relevantMessages, relevantRanked, minScore) + recallOpts.MaxRelevantMessages, relevantMessages, relevantRanked, minScore, droppedRelevant) : null }, RelevantEntities = new MemoryContextSection @@ -541,7 +579,7 @@ await Task.WhenAll( RankedItems = entityRanked, Diagnostics = recallOpts.IncludeDiagnostics ? Diagnose(vectorSearchRan && recallOpts.MaxEntities > 0, - recallOpts.MaxEntities, entities, entityRanked, minScore) + recallOpts.MaxEntities, entities, entityRanked, minScore, droppedEntities) : null }, RelevantPreferences = new MemoryContextSection @@ -550,7 +588,7 @@ await Task.WhenAll( RankedItems = preferenceRanked, Diagnostics = recallOpts.IncludeDiagnostics ? Diagnose(vectorSearchRan && recallOpts.MaxPreferences > 0, - recallOpts.MaxPreferences, preferences, preferenceRanked, minScore) + recallOpts.MaxPreferences, preferences, preferenceRanked, minScore, droppedPreferences) : null }, RelevantFacts = new MemoryContextSection @@ -559,7 +597,7 @@ await Task.WhenAll( RankedItems = factRanked, Diagnostics = recallOpts.IncludeDiagnostics ? Diagnose(vectorSearchRan && recallOpts.MaxFacts > 0, - recallOpts.MaxFacts, facts, factRanked, minScore) + recallOpts.MaxFacts, facts, factRanked, minScore, droppedFacts) : null }, SimilarTraces = new MemoryContextSection @@ -568,14 +606,15 @@ await Task.WhenAll( RankedItems = traceRanked, Diagnostics = recallOpts.IncludeDiagnostics ? Diagnose(vectorSearchRan && recallOpts.MaxTraces > 0, - recallOpts.MaxTraces, traces, traceRanked, minScore) + recallOpts.MaxTraces, traces, traceRanked, minScore, droppedTraces) : null }, GraphRagContext = graphRagContext, GraphRagItems = graphRagItems, ResolvedQueryRelations = resolvedQueryRelations, BlendMode = blendMode, - Truncated = truncated + Truncated = truncated, + LatencyBudgetExceeded = budgetExceeded }; _logger.LogDebug( @@ -1040,4 +1079,54 @@ private ITruncationStrategy ResolveStrategy(TruncationStrategy strategy) => _truncationStrategies.TryGetValue(strategy, out var resolved) ? resolved : _truncationStrategies[TruncationStrategy.OldestFirst]; + + /// + /// Waits for the section tasks, giving up after (rank 13). + /// + /// when the budget expired with work still running. + /// + /// + /// The still-running queries are not cancelled. They are already in the driver, and tearing + /// them down buys nothing the caller is waiting for — it only turns a wasted result into a wasted + /// result plus a cancellation storm. Their faults are observed so a slow section that also fails + /// does not surface later as an unobserved task exception. + /// + /// + private static async Task WaitWithinBudgetAsync( + Task[] sections, TimeSpan budget, CancellationToken cancellationToken) + { + var all = Task.WhenAll(sections); + using var expiry = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + var timer = Task.Delay(budget, expiry.Token); + + var finished = await Task.WhenAny(all, timer).ConfigureAwait(false); + if (finished == all) + { + await expiry.CancelAsync().ConfigureAwait(false); + await all.ConfigureAwait(false); // rethrow section faults exactly as before + return false; + } + + foreach (var section in sections) + { + _ = section.ContinueWith( + faulted => _ = faulted.Exception, + CancellationToken.None, + TaskContinuationOptions.OnlyOnFaulted, + TaskScheduler.Default); + } + + return true; + } + + /// + /// Replaces a section that did not finish in time with an empty result (rank 13). + /// + /// when a substitution happened, i.e. this section was dropped. + private static bool DropIfUnfinished(ref Task section, T empty) + { + if (section.IsCompletedSuccessfully) return false; + section = Task.FromResult(empty); + return true; + } } diff --git a/tests/AgentMemory.Tests.Unit/Services/RecallLatencyBudgetTests.cs b/tests/AgentMemory.Tests.Unit/Services/RecallLatencyBudgetTests.cs new file mode 100644 index 000000000..0648df358 --- /dev/null +++ b/tests/AgentMemory.Tests.Unit/Services/RecallLatencyBudgetTests.cs @@ -0,0 +1,190 @@ +using AgentMemory.Abstractions.Domain; +using AgentMemory.Abstractions.Options; +using AgentMemory.Abstractions.Services; +using AgentMemory.Core.Services; +using FluentAssertions; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using NSubstitute; +using Xunit; + +namespace AgentMemory.Tests.Unit.Services; + +/// +/// Degrading under load without pretending otherwise (rank 13). +/// +/// +/// +/// A recall latency budget is easy to add and easy to get dangerously wrong. The dangerous version +/// returns whatever arrived in time and says nothing: a section cut short reports Searched and +/// Returned = 0, which is byte-identical to a section that ran to completion and genuinely +/// found nothing. +/// +/// +/// A partial recall that looks complete is worse than a slow one. The caller answers +/// confidently from memory that was never consulted, and nothing anywhere records that it happened. +/// So most of what follows is about the marking, not the timing. +/// +/// +public sealed class RecallLatencyBudgetTests +{ + private readonly IShortTermMemoryService _shortTerm = Substitute.For(); + private readonly ILongTermMemoryService _longTerm = Substitute.For(); + private readonly IReasoningMemoryService _reasoning = Substitute.For(); + private readonly IEmbeddingOrchestrator _embeddings = Substitute.For(); + + private static readonly Message AMessage = new() + { + MessageId = "m-1", ConversationId = "c-1", SessionId = "s-1", + Role = "user", Content = "hello", TimestampUtc = DateTimeOffset.UnixEpoch, + }; + + public RecallLatencyBudgetTests() + { + _embeddings.EmbedQueryAsync(Arg.Any(), Arg.Any()) + .Returns(Task.FromResult([0.1f, 0.2f])); + + _shortTerm.GetRecentMessagesAsync(Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(Task.FromResult>([AMessage])); + _shortTerm.SearchMessagesAsync( + Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any(), + Arg.Any()) + .Returns(Task.FromResult>([])); + + _longTerm.SearchPreferencesAsync( + Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(Task.FromResult>([])); + _reasoning.SearchSimilarTracesAsync( + Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any(), + Arg.Any(), Arg.Any()) + .Returns(Task.FromResult>([])); + } + + /// Entities return promptly; facts hang past any sane budget. + private void FactsAreSlow(TimeSpan delay) + { + _longTerm.SearchEntitiesAsync( + Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(Task.FromResult>( + [ + new Entity + { + EntityId = "e-1", Name = "Alice", Type = "Person", + Confidence = 0.9, CreatedAtUtc = DateTimeOffset.UnixEpoch, + }, + ])); + + _longTerm.SearchFactsAsync( + Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(async _ => + { + await Task.Delay(delay).ConfigureAwait(false); + return (IReadOnlyList)[]; + }); + } + + private MemoryContextAssembler CreateSut() => + new(_shortTerm, _longTerm, _reasoning, + graphRag: null, + _embeddings, + Substitute.For(), + Options.Create(new MemoryOptions()), + NullLogger.Instance, + Substitute.For()); + + private Task AssembleAsync(TimeSpan? budget) => + CreateSut().AssembleContextAsync(new RecallRequest + { + SessionId = "s-1", + Query = "what do I know", + Options = new RecallOptions + { + IncludeDiagnostics = true, + LatencyBudget = budget, + }, + }); + + [Fact] + public async Task ASlowSectionIsDroppedRatherThanWaitedFor() + { + FactsAreSlow(TimeSpan.FromSeconds(30)); + + var context = await AssembleAsync(TimeSpan.FromMilliseconds(150)); + + context.LatencyBudgetExceeded.Should().BeTrue(); + } + + [Fact] + public async Task TheDroppedSectionSaysSoAndTheOthersDoNot() + { + // THE test. Without this, "cut short" and "searched and found nothing" are the same section, + // and degradation under load is invisible in every artifact and every log. + FactsAreSlow(TimeSpan.FromSeconds(30)); + + var context = await AssembleAsync(TimeSpan.FromMilliseconds(150)); + + context.RelevantFacts.Diagnostics!.TimedOut.Should().BeTrue(); + context.RelevantFacts.Diagnostics.AbandonedToLatencyBudget.Should().BeTrue(); + context.RelevantEntities.Diagnostics!.TimedOut.Should().BeFalse( + "a section that finished in time must not be tarred with the budget failure"); + } + + [Fact] + public async Task WhatArrivedInTimeIsStillReturned() + { + // The point of degrading rather than failing: a partial answer beats no answer, provided the + // caller can tell it is partial. + FactsAreSlow(TimeSpan.FromSeconds(30)); + + var context = await AssembleAsync(TimeSpan.FromMilliseconds(150)); + + context.RelevantEntities.Items.Should().ContainSingle(); + context.RecentMessages.Items.Should().ContainSingle(); + } + + [Fact] + public async Task NoBudgetWaitsForEverything() + { + // The off state, and it is the default. Every measurement in this track was taken without a + // budget, and a slow section must still be waited for rather than quietly dropped. + FactsAreSlow(TimeSpan.FromMilliseconds(200)); + + var context = await AssembleAsync(budget: null); + + context.LatencyBudgetExceeded.Should().BeFalse(); + context.RelevantFacts.Diagnostics!.TimedOut.Should().BeFalse(); + } + + [Fact] + public async Task AGenerousBudgetChangesNothing() + { + // A budget nobody exceeds must leave the context identical to having no budget at all. + FactsAreSlow(TimeSpan.FromMilliseconds(10)); + + var context = await AssembleAsync(TimeSpan.FromSeconds(30)); + + context.LatencyBudgetExceeded.Should().BeFalse(); + context.RelevantEntities.Items.Should().ContainSingle(); + } + + [Fact] + public void TheBudgetIsOffByDefault() => + new RecallOptions().LatencyBudget.Should().BeNull(); + + [Fact] + public void PartialIsNotTheSameFlagAsTruncated() + { + // Truncation is the context budget trimming memories that WERE retrieved. This is a retrieval + // that never came back. A caller told only "truncated" would conclude the memories exist and + // were dropped to fit, when they may never have been looked at. + var context = new MemoryContext + { + SessionId = "s-1", + AssembledAtUtc = DateTimeOffset.UnixEpoch, + LatencyBudgetExceeded = true, + }; + + context.LatencyBudgetExceeded.Should().BeTrue(); + context.Truncated.Should().BeFalse(); + } +} From 369fb97ed68cdd7c8f660175ff8413e5a75d717f Mon Sep 17 00:00:00 2001 From: Jose Luis Latorre Millas Date: Wed, 12 Aug 2026 23:20:50 +0200 Subject: [PATCH 30/31] Measure what memory saves in tokens, honestly (B3) "1.6k context tokens instead of 115k" is the most persuasive number in this category precisely because it is a claim about architecture. It cannot be inflated by a better answer model, cannot be tuned with prompt engineering, and does not move when the judge changes its mind -- it is either true of the assembled context or it is not. Which is exactly why it had to wait for C4. Until real tokenization landed, this codebase converted tokens to characters by multiplying by four, and a compression ratio derived from that would have been a claim about arithmetic wearing the clothes of a claim about design. ContextTokenBreakdown reports per-section token cost of an assembled context against the full transcript. The baseline is deliberately the whole conversation -- what a memoryless agent would have to send to answer the same question -- rather than a truncated window, which would flatter the result by measuring against a system that has already given up on remembering. Three choices worth naming. The counting method travels with the number, so a fallback estimate can never be read as a measurement. Empty sections are still listed, because an omitted section reads as "this kind of memory does not exist" rather than "it contributed nothing here". And an empty history yields a null ratio rather than infinity: an absent denominator is an absent measurement, and reporting a number there would be the single most flattering way to be wrong. Provider-free, so the instrument is asserted in unit tests rather than bought. Per-stage latency already existed via LongMemEvalStage; what remains for publication is one measured run. 416 LongMemEval tests green (+6). Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01PgDgctPpbTziBNE2RT8VdE --- .../ContextTokenBreakdownTests.cs | 126 ++++++++++++++++++ .../ContextTokenBreakdown.cs | 107 +++++++++++++++ 2 files changed, 233 insertions(+) create mode 100644 tests/AgentMemory.Tests.Unit.LongMemEval/ContextTokenBreakdownTests.cs create mode 100644 tools/AgentMemory.LongMemEval/ContextTokenBreakdown.cs diff --git a/tests/AgentMemory.Tests.Unit.LongMemEval/ContextTokenBreakdownTests.cs b/tests/AgentMemory.Tests.Unit.LongMemEval/ContextTokenBreakdownTests.cs new file mode 100644 index 000000000..42f45180d --- /dev/null +++ b/tests/AgentMemory.Tests.Unit.LongMemEval/ContextTokenBreakdownTests.cs @@ -0,0 +1,126 @@ +using AgentMemory.Abstractions.Domain; +using AgentMemory.LongMemEval; +using FluentAssertions; +using Xunit; + +namespace AgentMemory.Tests.Unit.LongMemEval; + +/// +/// The token half of B3: what answering from memory costs against answering from the whole +/// conversation. +/// +/// +/// +/// This is the most persuasive number in the category precisely because it is a claim about +/// architecture. "1.6k context tokens instead of 115k" cannot be inflated by a better answer +/// model, cannot be tuned with prompt engineering, and does not move when the judge changes its mind. +/// It is either true of the assembled context or it is not. +/// +/// +/// Which is why it is counted rather than estimated, and why the counting method travels with the +/// number: a ratio derived from a chars/4 heuristic would be a claim about arithmetic wearing the +/// clothes of a claim about design. +/// +/// +public sealed class ContextTokenBreakdownTests +{ + private static readonly LongMemEvalTokenCounter Counter = new("gpt-4o"); + + private static Message M(string content) => new() + { + MessageId = $"m-{content.GetHashCode():X}", + ConversationId = "c-1", + SessionId = "s-1", + Role = "user", + Content = content, + TimestampUtc = DateTimeOffset.UnixEpoch, + }; + + private static MemoryContext Context(params Fact[] facts) => new() + { + SessionId = "s-1", + AssembledAtUtc = DateTimeOffset.UnixEpoch, + RelevantFacts = new MemoryContextSection { Items = facts }, + }; + + private static Fact F(string subject, string predicate, string @object) => new() + { + FactId = $"f-{@object}", + Subject = subject, + Predicate = predicate, + Object = @object, + Confidence = 0.9, + CreatedAtUtc = DateTimeOffset.UnixEpoch, + }; + + [Fact] + public void MemoryCostsFarLessThanTheConversationItWasBuiltFrom() + { + // THE claim. A long conversation distils to a handful of triples, and the ratio is what a + // reader is actually being asked to believe. + var history = Enumerable.Range(0, 200) + .Select(i => M($"turn {i}: a fairly ordinary sentence about the day and the weather")) + .ToList(); + + var breakdown = ContextTokenBreakdown.Measure( + Context(F("user", "lives in", "Zurich"), F("user", "works at", "Acme")), + history, Counter); + + breakdown.ContextTokens.Should().BeLessThan(breakdown.FullHistoryTokens); + breakdown.CompressionRatio.Should().BeGreaterThan(10); + } + + [Fact] + public void TheCountIsRealRatherThanEstimated() + { + // The whole reason C4 had to land before this could be published. + var breakdown = ContextTokenBreakdown.Measure( + Context(F("user", "lives in", "Zurich")), [M("hello")], Counter); + + breakdown.CountMethod.Should().NotBe(TokenCountMethodNameForEstimate); + breakdown.Encoding.Should().NotBeNullOrWhiteSpace(); + } + + private const string TokenCountMethodNameForEstimate = "Estimated"; + + [Fact] + public void SectionsAreOrderedByCost() + { + // A breakdown exists to show where the budget goes; a fixed section order buries that. + var breakdown = ContextTokenBreakdown.Measure( + Context(F("user", "lives in", "a rather long place name indeed for testing purposes")), + [M("hello")], Counter); + + breakdown.Sections.Should().BeInDescendingOrder(s => s.Tokens); + breakdown.Sections.First().Section.Should().Be("RelevantFacts"); + } + + [Fact] + public void EverySectionIsAccountedForEvenWhenEmpty() + { + // An omitted empty section reads as "this kind of memory does not exist" rather than "it + // contributed nothing here", and the two invite different conclusions about the design. + var breakdown = ContextTokenBreakdown.Measure(Context(), [M("hello")], Counter); + + breakdown.Sections.Should().HaveCount(6); + breakdown.ContextTokens.Should().Be(0); + } + + [Fact] + public void NoHistoryMeansNoRatioRatherThanAFlatteringOne() + { + // An empty denominator is not a ratio of infinity, it is an absent measurement -- and it would + // be the single most flattering way to be wrong. + ContextTokenBreakdown.Measure(Context(F("user", "lives in", "Zurich")), [], Counter) + .CompressionRatio.Should().BeNull(); + } + + [Fact] + public void AnEmptyContextAgainstRealHistoryDoesNotDivideByZero() + { + var breakdown = ContextTokenBreakdown.Measure(Context(), [M("a real turn of conversation")], Counter); + + breakdown.CompressionRatio.Should().NotBeNull(); + breakdown.CompressionRatio.Should().BePositive(); + } +} diff --git a/tools/AgentMemory.LongMemEval/ContextTokenBreakdown.cs b/tools/AgentMemory.LongMemEval/ContextTokenBreakdown.cs new file mode 100644 index 000000000..400ed312b --- /dev/null +++ b/tools/AgentMemory.LongMemEval/ContextTokenBreakdown.cs @@ -0,0 +1,107 @@ +using AgentMemory.Abstractions.Domain; + +namespace AgentMemory.LongMemEval; + +/// +/// What one section of an assembled context costs in tokens (B3). +/// +/// Section name, as it appears on . +/// How many memories the section contributed. +/// Their cost, counted with the real tokenizer. +internal sealed record ContextSectionTokens(string Section, int Items, int Tokens); + +/// +/// The token cost of answering from memory versus from the whole conversation (B3). +/// +/// +/// +/// The most persuasive number in this category is a claim about architecture, not model quality. +/// "1.6k context tokens instead of 115k" cannot be inflated by a better answer model, cannot be +/// tuned by prompt engineering, and does not move when the judge changes its mind. It is either true +/// of the context that was assembled or it is not. +/// +/// +/// Which is exactly why it has to be counted rather than estimated. Until C4 this codebase converted +/// tokens to characters by multiplying by four; publishing a compression ratio derived from that +/// would have been a claim about arithmetic dressed up as a claim about design. +/// +/// +/// The baseline is deliberately the full transcript — what a memoryless agent would have to +/// send to answer the same question. Comparing against a truncated window instead would flatter the +/// result by measuring against a system that has already given up on remembering. +/// +/// +internal sealed record ContextTokenBreakdown +{ + /// Per-section costs, largest first. + internal required IReadOnlyList Sections { get; init; } + + /// Total tokens in the assembled memory context. + internal int ContextTokens => Sections.Sum(s => s.Tokens); + + /// Tokens in the full conversation the memory was built from. + internal required int FullHistoryTokens { get; init; } + + /// How the counting was done, so a reader can tell a real count from a fallback. + /// + /// Recorded rather than assumed. falls back to an estimate + /// when it cannot resolve an encoding for the model, and a compression ratio computed from an + /// estimate must not be readable as a measurement. + /// + internal required string CountMethod { get; init; } + + /// The encoding used, when one was resolved. + internal string? Encoding { get; init; } + + /// + /// How many times smaller the memory context is than the full history. + /// + /// + /// Null when there is no history to compare against — an empty denominator is not a ratio of + /// infinity, it is an absent measurement, and reporting a number there would be the single most + /// flattering way to be wrong. + /// + internal double? CompressionRatio => + FullHistoryTokens == 0 ? null : (double)FullHistoryTokens / Math.Max(1, ContextTokens); + + /// + /// Measures an assembled context against the conversation it was distilled from. + /// + internal static ContextTokenBreakdown Measure( + MemoryContext context, + IReadOnlyList fullHistory, + LongMemEvalTokenCounter counter) + { + ArgumentNullException.ThrowIfNull(context); + ArgumentNullException.ThrowIfNull(fullHistory); + ArgumentNullException.ThrowIfNull(counter); + + var sections = new List + { + Section("RecentMessages", context.RecentMessages.Items, m => m.Content, counter), + Section("RelevantMessages", context.RelevantMessages.Items, m => m.Content, counter), + Section("RelevantEntities", context.RelevantEntities.Items, + e => $"{e.Name} {e.Type} {e.Description}", counter), + Section("RelevantFacts", context.RelevantFacts.Items, + f => $"{f.Subject} {f.Predicate} {f.Object}", counter), + Section("RelevantPreferences", context.RelevantPreferences.Items, + p => $"{p.Category} {p.PreferenceText}", counter), + Section("SimilarTraces", context.SimilarTraces.Items, + t => $"{t.Task} {t.Outcome}", counter), + }; + + return new ContextTokenBreakdown + { + // Largest first: the point of a breakdown is to show where the budget actually goes, and + // a fixed section order buries that behind whichever section happens to be listed first. + Sections = sections.OrderByDescending(s => s.Tokens).ToList(), + FullHistoryTokens = fullHistory.Sum(m => counter.Count(m.Content)), + CountMethod = counter.Method.ToString(), + Encoding = counter.Encoding, + }; + } + + private static ContextSectionTokens Section( + string name, IReadOnlyList items, Func render, LongMemEvalTokenCounter counter) => + new(name, items.Count, items.Sum(item => counter.Count(render(item)))); +} From acdc8c3b7aa7eeeba4b7a21ef21c9bb3d03bb773 Mon Sep 17 00:00:00 2001 From: Jose Luis Latorre Millas Date: Wed, 12 Aug 2026 23:25:37 +0200 Subject: [PATCH 31/31] Build the harness that can tell procedural memory it failed (7.6) LongMemEval cannot answer whether procedural memory helps. It scores answers about a transcript; a promoted procedure changes how an agent works across repeated attempts at a multi-step task. With no trace nodes and -- until 6.5 -- a probe blind to them, a promoted procedure was invisible to every instrument this project had. The feature could not fail. That is the defect shape this harness exists to prevent, not merely a gap in coverage. Two comparisons, and both are needed. On versus off answers "does it help at all". First run versus last answers "did it help because it learned" -- an arm that is already cheap on attempt one learned nothing during the run, and whatever separates the arms is not a procedure. Every efficiency number is gated on completion rate, without tolerance. An agent that abandons a task sooner takes fewer steps and makes fewer tool calls, and both read as improvements in every measure a harness would naturally report; a system that scored giving up as its biggest win would be worse than no measurement. Any drop in completion disqualifies the efficiency claim outright, because an agent with the wrong procedure executes confidently and the steps it saved are not a saving if the task is not done. Step and tool-call means are taken over completed runs only. Mixing "solved it in three steps" with "gave up after three" treats abandonment as a cheap success. Arms run sequentially. The hypothesis is that attempt N benefits from what attempt N-1 stored; running them concurrently would race the write that the next read depends on, and measure a feature that had not happened yet. Driven through an IAgentTaskRunner seam, so the part that can be quietly wrong is unit-tested against a scripted agent while the part that costs money is deferred to a measured run. 4,359 unit + 425 LongMemEval tests green (+9). Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01PgDgctPpbTziBNE2RT8VdE --- .../ProceduralBenefitHarnessTests.cs | 153 ++++++++++++++++ .../ProceduralBenefitHarness.cs | 165 ++++++++++++++++++ 2 files changed, 318 insertions(+) create mode 100644 tests/AgentMemory.Tests.Unit.LongMemEval/ProceduralBenefitHarnessTests.cs create mode 100644 tools/AgentMemory.LongMemEval/ProceduralBenefitHarness.cs diff --git a/tests/AgentMemory.Tests.Unit.LongMemEval/ProceduralBenefitHarnessTests.cs b/tests/AgentMemory.Tests.Unit.LongMemEval/ProceduralBenefitHarnessTests.cs new file mode 100644 index 000000000..2803dbb75 --- /dev/null +++ b/tests/AgentMemory.Tests.Unit.LongMemEval/ProceduralBenefitHarnessTests.cs @@ -0,0 +1,153 @@ +using AgentMemory.LongMemEval; +using FluentAssertions; +using Xunit; + +namespace AgentMemory.Tests.Unit.LongMemEval; + +/// +/// Whether procedural memory helps an agent repeat a task — and refusing to say yes for the wrong +/// reason (7.6). +/// +/// +/// +/// LongMemEval cannot answer this. It scores answers about a transcript; a promoted procedure +/// changes how an agent works across repeated attempts. With no trace nodes and a probe that +/// was label-blind to them, a promoted procedure was invisible to every instrument here — the feature +/// could not fail, which is the defect shape this harness exists to prevent. +/// +/// +/// The harness is driven by a scripted runner, so the thing that can be quietly wrong is tested +/// without buying the thing that costs money. +/// +/// +public sealed class ProceduralBenefitHarnessTests +{ + /// A scripted agent: outcomes are decided by the test, not by a model. + private sealed class ScriptedRunner(Func script) : IAgentTaskRunner + { + public readonly List<(bool Enabled, int Attempt)> Calls = []; + + public Task RunAsync( + string taskId, bool procedureMemoryEnabled, int attempt, CancellationToken cancellationToken = default) + { + Calls.Add((procedureMemoryEnabled, attempt)); + return Task.FromResult(script(procedureMemoryEnabled, attempt)); + } + } + + private static Task MeasureAsync( + Func script, int attempts = 4) => + ProceduralBenefitResult.MeasureAsync(new ScriptedRunner(script), "book-a-trip", attempts); + + [Fact] + public async Task AGenuineBenefitIsReported() + { + // The happy path: procedures cut the work and the task still gets done every time. + var result = await MeasureAsync((enabled, attempt) => + enabled && attempt > 1 + ? new AgentTaskRun(true, Steps: 3, ToolCalls: 2) + : new AgentTaskRun(true, Steps: 9, ToolCalls: 7)); + + result.ShowsBenefit.Should().BeTrue(); + result.StepReduction.Should().BeGreaterThan(0); + result.ImprovedWithRepetition.Should().BeTrue(); + } + + [Fact] + public async Task GivingUpSoonerIsNotABenefit() + { + // THE test. An agent that abandons the task takes fewer steps and makes fewer tool calls, and + // both read as an improvement in every efficiency measure. Completion rate is what stops that + // being reported as a win. + var result = await MeasureAsync((enabled, _) => + enabled + ? new AgentTaskRun(Completed: false, Steps: 2, ToolCalls: 1) + : new AgentTaskRun(Completed: true, Steps: 9, ToolCalls: 7)); + + result.CompletionRateDelta.Should().BeLessThan(0); + result.ShowsBenefit.Should().BeFalse("fewer steps is not a saving if the task is not done"); + } + + [Fact] + public async Task EvenASmallCorrectnessLossDisqualifiesTheClaim() + { + // Not a hair's tolerance. An agent with the wrong procedure executes confidently, so a + // completion rate that slipped is a safety finding, not a rounding error to trade away. + var result = await MeasureAsync((enabled, attempt) => + enabled && attempt == 4 + ? new AgentTaskRun(false, 2, 1) + : new AgentTaskRun(true, enabled ? 3 : 9, enabled ? 2 : 7)); + + result.StepReduction.Should().BeGreaterThan(0); + result.ShowsBenefit.Should().BeFalse(); + } + + [Fact] + public async Task StartingCheaperIsNotLearning() + { + // An arm that is fast from attempt one did not learn a procedure during the run -- something + // else separates the arms, and reporting it as a memory benefit would be wrong. + var result = await MeasureAsync((enabled, _) => + new AgentTaskRun(true, enabled ? 3 : 9, enabled ? 2 : 7)); + + result.StepReduction.Should().BeGreaterThan(0); + result.ImprovedWithRepetition.Should().BeFalse(); + } + + [Fact] + public async Task FailedAttemptsDoNotDragDownTheStepAverage() + { + // Averaged over completed runs only: mixing "solved it in three steps" with "gave up after + // three" treats abandonment as a cheap success. + var result = await MeasureAsync((enabled, attempt) => + enabled && attempt == 1 + ? new AgentTaskRun(false, Steps: 1, ToolCalls: 0) + : new AgentTaskRun(true, Steps: 4, ToolCalls: 3)); + + result.WithProcedures.MeanStepsWhenCompleted.Should().Be(4); + } + + [Fact] + public async Task AttemptsRunInOrderAndSequentially() + { + // The hypothesis is that attempt N benefits from what attempt N-1 stored. Running them + // concurrently would race the write the next read depends on, measuring a feature that had + // not happened yet. + var runner = new ScriptedRunner((_, _) => new AgentTaskRun(true, 5, 4)); + + await ProceduralBenefitResult.MeasureAsync(runner, "book-a-trip", attempts: 3); + + runner.Calls.Where(c => c.Enabled).Select(c => c.Attempt).Should().Equal(1, 2, 3); + runner.Calls.Where(c => !c.Enabled).Select(c => c.Attempt).Should().Equal(1, 2, 3); + } + + [Fact] + public async Task BothArmsAreRun() + { + var runner = new ScriptedRunner((_, _) => new AgentTaskRun(true, 5, 4)); + + var result = await ProceduralBenefitResult.MeasureAsync(runner, "book-a-trip", attempts: 2); + + result.WithProcedures.Runs.Should().HaveCount(2); + result.WithoutProcedures.Runs.Should().HaveCount(2); + } + + [Fact] + public async Task ASingleAttemptIsRejected() + { + // One attempt per arm cannot show learning at all -- there is no run 1 versus run N. + var act = async () => await ProceduralBenefitResult.MeasureAsync( + new ScriptedRunner((_, _) => new AgentTaskRun(true, 1, 1)), "t", attempts: 1); + + await act.Should().ThrowAsync(); + } + + [Fact] + public async Task AnArmThatNeverCompletesReportsZeroRatherThanDividingByZero() + { + var result = await MeasureAsync((_, _) => new AgentTaskRun(false, 3, 2)); + + result.WithProcedures.MeanStepsWhenCompleted.Should().Be(0); + result.ShowsBenefit.Should().BeFalse(); + } +} diff --git a/tools/AgentMemory.LongMemEval/ProceduralBenefitHarness.cs b/tools/AgentMemory.LongMemEval/ProceduralBenefitHarness.cs new file mode 100644 index 000000000..242b15df8 --- /dev/null +++ b/tools/AgentMemory.LongMemEval/ProceduralBenefitHarness.cs @@ -0,0 +1,165 @@ +namespace AgentMemory.LongMemEval; + +/// What one attempt at a task cost, and whether it worked (7.6). +/// Whether the task was actually finished. The gate on every other number. +/// Agent turns taken. +/// Tool invocations made. +public sealed record AgentTaskRun(bool Completed, int Steps, int ToolCalls); + +/// +/// Runs a task once and reports what it cost (7.6). +/// +/// +/// A seam so the harness can be exercised against a scripted agent in a unit test and against a real +/// Agent Framework agent in a measured run. The harness is the part that can be wrong in a way nobody +/// notices; the agent is the part that costs money. +/// +public interface IAgentTaskRunner +{ + /// Attempts the task once, with procedural memory on or off. + Task RunAsync( + string taskId, bool procedureMemoryEnabled, int attempt, CancellationToken cancellationToken = default); +} + +/// One arm of the comparison, aggregated (7.6). +public sealed record ProceduralBenefitArm(bool ProcedureMemoryEnabled, IReadOnlyList Runs) +{ + /// Share of attempts that finished the task. + /// + /// The gate on every efficiency number beside it. An agent that gives up sooner takes fewer + /// steps and makes fewer tool calls, and both of those read as an improvement. + /// + public double CompletionRate => Runs.Count == 0 ? 0d : (double)Runs.Count(r => r.Completed) / Runs.Count; + + /// Mean steps over the runs that completed. + /// + /// Averaged over completed runs only. Including failures mixes "solved it in three steps" + /// with "gave up after three", and the second is not a cheaper success. + /// + public double MeanStepsWhenCompleted => Mean(Runs.Where(r => r.Completed).Select(r => (double)r.Steps)); + + /// Mean tool calls over the runs that completed. + public double MeanToolCallsWhenCompleted => + Mean(Runs.Where(r => r.Completed).Select(r => (double)r.ToolCalls)); + + /// The first attempt, before any procedure could have been learned. + public AgentTaskRun? FirstRun => Runs.Count > 0 ? Runs[0] : null; + + /// The last attempt, by which point a procedure should exist. + public AgentTaskRun? LastRun => Runs.Count > 0 ? Runs[^1] : null; + + private static double Mean(IEnumerable values) + { + var list = values.ToList(); + return list.Count == 0 ? 0d : list.Average(); + } +} + +/// +/// Whether procedural memory actually helps an agent repeat a task (7.6). +/// +/// +/// +/// LongMemEval cannot answer this. It scores answers to questions about a transcript; a +/// promoted procedure changes how an agent works, over repeated attempts at a multi-step task. +/// With no trace nodes and — until 6.5 — a probe that was label-blind to them, a promoted procedure +/// was invisible to every instrument this project had, which is to say the feature could not +/// fail. That is the defect shape this harness exists to prevent, not merely a gap in coverage. +/// +/// +/// Two comparisons, and both are needed. On versus off answers "does the feature help at all". +/// First run versus last answers "does it help because it learned, or because the two arms +/// differ for some other reason" — an arm that is faster from attempt one learned nothing. +/// +/// +/// Every efficiency number is gated on completion rate. An agent that abandons a task sooner takes +/// fewer steps and makes fewer tool calls, and a harness that reported those alone would score giving +/// up as the biggest improvement available. +/// +/// +public sealed record ProceduralBenefitResult +{ + /// The arm with procedural memory enabled. + public required ProceduralBenefitArm WithProcedures { get; init; } + + /// The control arm. + public required ProceduralBenefitArm WithoutProcedures { get; init; } + + /// Reduction in mean steps, as a fraction. Negative means it got worse. + public double StepReduction => Reduction( + WithoutProcedures.MeanStepsWhenCompleted, WithProcedures.MeanStepsWhenCompleted); + + /// Reduction in mean tool calls, as a fraction. + public double ToolCallReduction => Reduction( + WithoutProcedures.MeanToolCallsWhenCompleted, WithProcedures.MeanToolCallsWhenCompleted); + + /// Change in completion rate; negative means procedures cost correctness. + public double CompletionRateDelta => + WithProcedures.CompletionRate - WithoutProcedures.CompletionRate; + + /// + /// Whether the enabled arm got cheaper across attempts rather than starting cheaper. + /// + /// + /// The learning signal. If the first attempt is already as cheap as the last, whatever separates + /// the arms is not a procedure that was learned during the run. + /// + public bool ImprovedWithRepetition => + WithProcedures.FirstRun is { } first && WithProcedures.LastRun is { } last + && last.Completed && first.Completed && last.Steps < first.Steps; + + /// + /// Whether the result may be reported as a benefit at all. + /// + /// + /// Correctness first, and not by a hair. Any drop in completion rate disqualifies the + /// efficiency claim outright: an agent with the wrong procedure executes confidently, and the + /// steps it saves are not a saving if the task is not done. Cheapness is only a benefit on top of + /// finishing at least as often. + /// + public bool ShowsBenefit => + CompletionRateDelta >= 0 && (StepReduction > 0 || ToolCallReduction > 0); + + private static double Reduction(double baseline, double measured) => + baseline <= 0 ? 0d : (baseline - measured) / baseline; + + /// + /// Runs both arms and compares them (7.6). + /// + /// Executes one attempt. + /// The repeated task. + /// How many times to repeat it per arm. + public static async Task MeasureAsync( + IAgentTaskRunner runner, + string taskId, + int attempts, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(runner); + ArgumentException.ThrowIfNullOrWhiteSpace(taskId); + ArgumentOutOfRangeException.ThrowIfLessThan(attempts, 2); + + return new ProceduralBenefitResult + { + WithProcedures = await RunArmAsync(runner, taskId, true, attempts, cancellationToken) + .ConfigureAwait(false), + WithoutProcedures = await RunArmAsync(runner, taskId, false, attempts, cancellationToken) + .ConfigureAwait(false), + }; + } + + private static async Task RunArmAsync( + IAgentTaskRunner runner, string taskId, bool enabled, int attempts, CancellationToken cancellationToken) + { + var runs = new List(attempts); + // Sequential, deliberately. The whole hypothesis is that attempt N benefits from what attempt + // N-1 stored, and running them concurrently would race the write that the next read depends on + // -- measuring a feature that had not happened yet. + for (var attempt = 1; attempt <= attempts; attempt++) + { + runs.Add(await runner.RunAsync(taskId, enabled, attempt, cancellationToken).ConfigureAwait(false)); + } + + return new ProceduralBenefitArm(enabled, runs); + } +}