diff --git a/CHANGELOG.md b/CHANGELOG.md index 7310c9a61..d601b299f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,42 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- **Valid-time recall (opt-in).** `RecallOptions.ValidTime = ValidTimeMode.Current` filters facts on + their real-world window (`valid_from`/`valid_until`) rather than only on the transaction clock. + Default `Ignore`, which is byte-for-byte today's behaviour, and `MemoryProfile.Parity` resolves to + `Ignore` — parity means parity. **TCK exposure is zero**: the bridge exposes neither `/search_facts` + nor `/get_facts_about`, so no conformance surface can observe this. + + Two things write validity bounds, and only two: temporal extraction, when + `TemporalValidityMode.Extract` asks the model for a window, and **supersession**, which stamps + `valid_until` as it closes a fact. The second is worth knowing about because it changes nothing + here — supersession also stamps `invalidated_at`, and the transaction-clock filter already removes + the fact from live recall whatever this option says. **So the gate is redundant for superseded + facts and load-bearing only for windows the conversation actually stated.** Reaching for it to hide + superseded data solves a problem that is already solved. + +- **Reasoning traces carry a trust level.** `ReasoningMemoryOptions.DefaultTraceTrustLevel`, default + `ModelGenerated` — a trace is the agent's own record of what it did. Traces were the one recall + category with no trust signal: caller-supplied `trust_level` is stripped (a caller must never + self-assign a level that bypasses admission checks) and nothing then stamped one, so every trace + read back as `Untrusted` — indistinguishable from "no signal recorded". Safe at shipped defaults: + `MinimumTrustForAdmissionBypass` is `ApplicationTrusted`, which `ModelGenerated` does not reach. + +- **Owner-starvation rescue for short results (opt-in).** `MemoryOptions.RescueShortOwnerResults`. + Neo4j's vector index is global, so an owner filter is a post-filter on a top-K drawn from every + tenant — measured, a mean of **7 of 60** candidates reached the querying owner. Previously only a + *totally empty* result triggered a rescue, on the argument that a short result still answers the + question. It does for a small tenant and does not for a crowded one: one measured question returned + **2 facts from a 710-fact graph** with the answer present, and was answered wrongly. + + The rescue is the owner-bounded **scan**, not a wider index query — widening is another draw on the + same global index, and a tenant losing to 50 neighbours at top-60 usually loses again at top-480. + The scan's cost scales with one owner's rows, so the small tenant the original argument protected + pays *less* for it than for a wider index query. Off by default, and it takes whichever result is + larger, so a genuinely sparse owner never loses indexed rows. + ### Changed - **A greeting no longer costs a full recall.** Measured on the hermetic `PERF-R-01` scenario — a diff --git a/docs/architecture.md b/docs/architecture.md index c9407fb97..81d0aa5ae 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -142,7 +142,7 @@ graph TD | **Purpose** | Domain contracts — all models, interfaces, and configuration types shared across the system | | **Dependencies** | **Microsoft.Extensions.AI.Abstractions** 10.8.0 (approved, D-AR2-1) — .NET BCL otherwise (multi-targets net8.0/net9.0/net10.0) | | **MUST NOT reference** | Neo4j.Driver, Microsoft.Agents.*, any GraphRAG SDK, any MCP SDK, any NuGet package **except** Microsoft.Extensions.AI.Abstractions | -| **Key types** | 52 domain records (Conversation, Message, Entity, Fact, Preference, Relationship, MemoryHistoryQuery, MemoryHistoryRecord, ReasoningTrace, ReasoningStep, ToolCall, ToolCallStats, IngestionItemOutcome, MemoryContextRankedItem, MemoryContextSectionDiagnostics, UnifiedExtractionResult, etc.), 41 service interfaces (incl. `IMemoryIsolationPolicy`, `IUnifiedMemoryExtractor`, and `IMultiSessionUnifiedMemoryExtractor`), 11 repository interfaces, 16 configuration types (incl. `MemoryRankingOptions`, `MemoryIsolationOptions`), 26 enums (incl. `MemoryProfile`, `RankingIntent`, `DuplicateStatus`, `EntityMatchType`, `MemoryNodeKind`, `MemoryOperationAccess`, `MemoryIsolationMode`, `IngestionStatus`, `IngestionStage`, `IngestionItemStatus`, `MemoryItemKind`, `IngestionFailureMode`, `MemoryTrustLevel`, `AssistantContentMode`, `TemporalValidityMode`, `TraceKind`, `ExtractionProvenanceMode`) | +| **Key types** | 59 domain records (Conversation, Message, Entity, Fact, Preference, Relationship, MemoryHistoryQuery, MemoryHistoryRecord, ReasoningTrace, ReasoningStep, ToolCall, ToolCallStats, IngestionItemOutcome, MemoryContextRankedItem, MemoryContextSectionDiagnostics, UnifiedExtractionResult, ExtractionWindow, EntitySummary, MemoryBlock, BulkIngestionResult, etc.), 44 service interfaces (incl. `IMemoryIsolationPolicy`, `IUnifiedMemoryExtractor`, `IMultiSessionUnifiedMemoryExtractor`, and `IMemoryReranker`, `IEntitySummaryService`), 12 repository interfaces, 16 configuration types (incl. `MemoryRankingOptions`, `MemoryIsolationOptions`), 26 enums (incl. `MemoryProfile`, `RankingIntent`, `DuplicateStatus`, `EntityMatchType`, `MemoryNodeKind`, `MemoryOperationAccess`, `MemoryIsolationMode`, `IngestionStatus`, `IngestionStage`, `IngestionItemStatus`, `MemoryItemKind`, `IngestionFailureMode`, `MemoryTrustLevel`, `AssistantContentMode`, `TemporalValidityMode`, `TraceKind`, `ExtractionProvenanceMode`) | **Namespace structure:** ``` diff --git a/docs/performance/README.md b/docs/performance/README.md index c9002eef1..6420c6204 100644 --- a/docs/performance/README.md +++ b/docs/performance/README.md @@ -7,6 +7,7 @@ What AgentMemory costs you per agent turn, how that cost is measured, and how to | **README.md** (this file) | The two phases, what is and isn't measured, how to run it yourself | | [`hermetic-S.json`](../../eng/perf/baselines/hermetic-S.json) | Machine-readable counter + quality baseline enforced on pull requests | | [baseline-1.3.0.md](baseline-1.3.0.md) | The measured cost model for release 1.3.0 | +| [bulk-ingestion.md](bulk-ingestion.md) | Loading a backlog without degrading p99 for everyone else in the process | --- diff --git a/docs/performance/bulk-ingestion.md b/docs/performance/bulk-ingestion.md new file mode 100644 index 000000000..f734738e3 --- /dev/null +++ b/docs/performance/bulk-ingestion.md @@ -0,0 +1,75 @@ +# Bulk ingestion + +Loading a backlog — a year of transcripts, an export from another system, a migration — is a +different problem from ingesting a live turn, and the two obvious ways to do it are both wrong. + +A serial loop over `ExtractAndPersistAsync` is correct and takes hours: extraction is a model call, +and one at a time means one round trip at a time. + +An unbounded `Parallel.ForEachAsync` is fast until it isn't. It saturates the provider quota and the +Neo4j connection pool, and the damage is not to the bulk load — it is to **everything else sharing +the process**. Median latency barely moves, which is what makes it hard to notice; p99 degrades by +20–70% under saturation, so the tenants who suffer are the ones already having the worst time. + +## The documented path + +```csharp +var result = await memory.IngestBulkAsync( + requests, + new BulkIngestionOptions { MaxConcurrency = 4 }, + cancellationToken); + +Console.WriteLine($"{result.SucceededCount} ingested, {result.FailedCount} failed, " + + $"{result.NotAttemptedCount} not attempted"); + +foreach (var failure in result.Outcomes.Where(o => !o.Succeeded)) + Console.WriteLine($" request {failure.Index}: {failure.Error!.Message}"); +``` + +`IngestBulkAsync` is a **default interface method on `IMemoryIngestion`**. It paces calls the host +could have made itself; it is not a second ingestion pipeline. That distinction matters more than it +sounds: a separate bulk path would be a second place for trust stamping, provenance and owner scoping +to drift out of agreement with the per-request path, and the drift would only show up in the corpus +months later. + +## What the options actually buy + +| Option | Default | Why | +|---|---|---| +| `MaxConcurrency` | `4` | High enough to beat a serial loop, low enough that a caller who never tunes it does not saturate a shared quota by accident. Bounding this does not make the bulk load faster — it stops the bulk load making everything else slower. | +| `ContinueOnError` | `true` | In a ten-thousand-conversation load, one malformed transcript aborting the other 9,999 is rarely what was wanted. Continuing is not ignoring: every failure comes back in `Outcomes`. | + +## Failures are reported per request, not counted + +A bulk API that returns *"8,412 of 10,000 succeeded"* tells you that you have a problem and nothing +about which inputs to retry — so the realistic response is to re-run all ten thousand, which costs +more than the failure did. Every outcome carries its `Index` into the submitted list and the +`Exception` that stopped it. + +`NotAttemptedCount` is kept separate from `FailedCount` on purpose. **"Tried and failed" and "never +tried" call for different actions**: re-running a request that was never attempted is always safe, +re-running one that failed part-way may not be. Collapsing them makes a stopped run look like a +partially broken corpus. + +## Cancellation + +A run stopped by `ContinueOnError = false` **completes normally** and returns its report — the +failure that stopped it is already in `Outcomes`, and throwing would discard the record of what did +succeed. + +A cancellation *you* requested throws `OperationCanceledException`. That is a different event and is +not swallowed into a tidy report. + +## What this is not + +This is not the 10.70× figure from the throughput analysis. That number is ten-owner concurrent +throughput, and it has always been available to any caller willing to parallelise across owners — +it is a property of the store, not of this API. What is documented here is the safe way to do the +parallelising, which is a smaller and more honest claim. + +## Related + +- [Performance baselines](README.md) +- Extraction batching (`MaxConcurrentExtractionBatches`) bounds concurrency *inside* one extraction; + this bounds concurrency *across* requests. They compose, and multiply — a `MaxConcurrency` of 4 + over batches of 6 is 24 in-flight provider calls. diff --git a/src/AgentMemory.Abstractions/Domain/Context/MemoryContext.cs b/src/AgentMemory.Abstractions/Domain/Context/MemoryContext.cs index d628d151f..c9351f8f8 100644 --- a/src/AgentMemory.Abstractions/Domain/Context/MemoryContext.cs +++ b/src/AgentMemory.Abstractions/Domain/Context/MemoryContext.cs @@ -106,6 +106,24 @@ public sealed record MemoryContext /// public bool Truncated { get; init; } + /// + /// At least one section was abandoned because the recall latency budget expired (rank 13). + /// + /// + /// + /// Separate from , which means something else. Truncation is the + /// context budget trimming items that were retrieved; this is a retrieval that never came back. + /// A caller told only "truncated" would reasonably conclude the memories exist and were dropped + /// to fit — here they may exist and were never looked at. + /// + /// + /// Which sections were lost is on their own diagnostics. This flag exists so a caller can notice + /// at all without inspecting every section, because a partial context that looks complete is + /// worse than a slow one: the answer is given confidently from memory that was never consulted. + /// + /// + public bool LatencyBudgetExceeded { get; init; } + /// /// Additional metadata. /// diff --git a/src/AgentMemory.Abstractions/Domain/Context/MemoryContextSection.cs b/src/AgentMemory.Abstractions/Domain/Context/MemoryContextSection.cs index 80e05caa8..5c662c96c 100644 --- a/src/AgentMemory.Abstractions/Domain/Context/MemoryContextSection.cs +++ b/src/AgentMemory.Abstractions/Domain/Context/MemoryContextSection.cs @@ -74,14 +74,31 @@ public sealed record MemoryContextRankedItem( /// /// Lowest returned score, or under the same rules. /// The similarity floor in force, so "all below threshold" is checkable. +/// +/// Whether this section was abandoned because the recall latency budget expired (rank 13). Without +/// it, a section cut short is byte-identical to one that searched and found nothing. +/// public sealed record MemoryContextSectionDiagnostics( bool Searched, int RequestedLimit, int Returned, double? TopScore, double? LowestScore, - double MinimumScore) + double MinimumScore, + bool TimedOut = false) { + /// + /// This section was abandoned because the recall latency budget expired (rank 13). + /// + /// + /// The reason a latency budget needs a flag at all. A section cut short reports + /// Searched = true and Returned = 0, which is byte-identical to a section that ran + /// to completion and genuinely found nothing. Without this, degrading under load would look + /// exactly like an owner having no memories — and a partial recall that looks complete is worse + /// than a slow one, because the caller answers confidently from a context they believe is whole. + /// + public bool AbandonedToLatencyBudget => TimedOut; + /// /// The retrieval ran and came back with nothing — as opposed to never having run. /// diff --git a/src/AgentMemory.Abstractions/Domain/Extraction/BulkIngestion.cs b/src/AgentMemory.Abstractions/Domain/Extraction/BulkIngestion.cs new file mode 100644 index 000000000..553abd4d4 --- /dev/null +++ b/src/AgentMemory.Abstractions/Domain/Extraction/BulkIngestion.cs @@ -0,0 +1,77 @@ +namespace AgentMemory.Abstractions.Domain; + +/// +/// How a bulk ingestion run is paced (rank 27). +/// +/// +/// +/// Backpressure is the whole point. A caller with ten thousand conversations to load will otherwise +/// issue them all at once, and the failure is not a clean rejection — it is provider throttling, +/// connection-pool exhaustion and a p99 that degrades for every other tenant sharing the +/// process while median latency looks untouched. +/// +/// +public sealed record BulkIngestionOptions +{ + /// + /// How many requests may be in flight at once. + /// + /// + /// Defaults to 4: high enough to be worth using over a serial loop, low enough that a caller who + /// never tunes it does not saturate a shared provider quota by accident. The measured effect of + /// bounding this is small at the median and 20–70% at p99 under saturation — which is to + /// say it does not make bulk loading faster, it stops bulk loading making everything else slower. + /// + public int MaxConcurrency { get; init; } = 4; + + /// + /// Whether a failed request stops the run. + /// + /// + /// Defaults to : in a ten-thousand-conversation load, one malformed + /// transcript aborting the other 9,999 is rarely what the caller wanted. Failures are returned + /// per request rather than thrown, so continuing is not the same as ignoring. + /// + public bool ContinueOnError { get; init; } = true; +} + +/// What happened to one request in a bulk run. +/// Position in the submitted list, so a failure can be traced to its input. +/// The extraction result, or if it failed. +/// The failure, or if it succeeded. +public sealed record BulkIngestionOutcome(int Index, ExtractionResult? Result, Exception? Error) +{ + /// Whether this request was ingested. + public bool Succeeded => Error is null; +} + +/// +/// The result of a bulk ingestion run (rank 27). +/// +/// +/// Per-request outcomes, not an aggregate count. A bulk API that returns "8,412 of 10,000 +/// succeeded" tells the caller they have a problem and nothing about which inputs to retry — so the +/// realistic response is to re-run everything, which is worse than the failure. Every outcome carries +/// its index and its exception. +/// +public sealed record BulkIngestionResult +{ + /// One entry per submitted request, in submission order. + public required IReadOnlyList Outcomes { get; init; } + + /// How many requests were ingested. + public int SucceededCount => Outcomes.Count(o => o.Succeeded); + + /// How many requests failed. + public int FailedCount => Outcomes.Count(o => !o.Succeeded); + + /// + /// Requests that were never attempted because the run stopped early. + /// + /// + /// Distinct from a failure, and reported separately for the same reason 6.5's trace counts are + /// nullable: "tried and failed" and "never tried" call for different actions, and collapsing them + /// makes a stopped run look like a partially broken corpus. + /// + public int NotAttemptedCount { get; init; } +} diff --git a/src/AgentMemory.Abstractions/Domain/Extraction/ExtractionRequest.cs b/src/AgentMemory.Abstractions/Domain/Extraction/ExtractionRequest.cs index 0b0460151..ced298846 100644 --- a/src/AgentMemory.Abstractions/Domain/Extraction/ExtractionRequest.cs +++ b/src/AgentMemory.Abstractions/Domain/Extraction/ExtractionRequest.cs @@ -10,6 +10,25 @@ public sealed record ExtractionRequest /// public required IReadOnlyList Messages { get; init; } + /// + /// Earlier turns supplied only so can be understood (E2). + /// + /// + /// + /// Nothing is extracted from these and nothing is attributed to them — they exist so + /// "I moved there last year" can resolve the place named a few turns earlier. Empty by default, + /// which is exactly the pre-E2 behaviour. + /// + /// + /// Kept separate from rather than prepended to it, because + /// extracting from them would re-assert stored facts — and a re-assertion now earns confidence + /// (S2) and increments the mention_count the salience reranker reads (R7). A fact would + /// gain both every time it happened to sit inside a sliding window, so corroboration would + /// quietly become recency. + /// + /// + public IReadOnlyList ContextMessages { get; init; } = []; + /// /// Session context for the extraction. /// diff --git a/src/AgentMemory.Abstractions/Domain/Extraction/ExtractionWindow.cs b/src/AgentMemory.Abstractions/Domain/Extraction/ExtractionWindow.cs new file mode 100644 index 000000000..a876a0e30 --- /dev/null +++ b/src/AgentMemory.Abstractions/Domain/Extraction/ExtractionWindow.cs @@ -0,0 +1,50 @@ +namespace AgentMemory.Abstractions.Domain; + +/// +/// The turns an extractor should extract from, plus the earlier turns it may read to understand them +/// (E2). +/// +/// +/// +/// Extraction has always seen one batch — typically a user turn and the reply — which is often not +/// enough to resolve what was said. "I moved there last year" needs the turn that named the place; +/// "she recommended it" needs the turn that named her. Widening the batch fixes the reference and +/// creates a worse problem, which is why this type exists rather than a bigger messages list. +/// +/// +/// Context is not a target, and the distinction is now a correctness property rather than a +/// question of token efficiency. Re-extracting the preceding turns re-asserts facts already +/// stored. That used to be merely wasteful — the exact triple MERGEs, so nothing was corrupted — but +/// with confidence reinforcement (S2) a re-assertion earns α, and the salience reranker (R7) reads a +/// mention_count the MERGE increments. If context turns were extracted as targets, a fact +/// would gain confidence and mentions every time it happened to sit inside a sliding window, so both +/// signals would measure how recently something was said rather than how often the world +/// asserted it. Corroboration would quietly become recency. +/// +/// +/// So the contract is: read to disambiguate, extract only from +/// , and attribute provenance only to . +/// +/// +public sealed record ExtractionWindow +{ + /// The turns to extract memories from. Provenance is attributed here and nowhere else. + public required IReadOnlyList Targets { get; init; } + + /// + /// Earlier turns, in order, supplied purely so the targets can be understood. + /// + /// + /// Nothing may be extracted from these. They carry no provenance and must never appear in + /// SourceMessageIds: an EXTRACTED_FROM edge to a context turn would claim the fact + /// was stated there, which is exactly the turn the extractor was told not to extract from. + /// + public IReadOnlyList Context { get; init; } = []; + + /// A window with no context — identical in effect to the pre-E2 ExtractAsync(messages). + public static ExtractionWindow ForTargets(IReadOnlyList targets) => + new() { Targets = targets }; + + /// when there is context to render at all. + public bool HasContext => Context.Count > 0; +} diff --git a/src/AgentMemory.Abstractions/Domain/LongTerm/EntitySummary.cs b/src/AgentMemory.Abstractions/Domain/LongTerm/EntitySummary.cs new file mode 100644 index 000000000..910ad58c9 --- /dev/null +++ b/src/AgentMemory.Abstractions/Domain/LongTerm/EntitySummary.cs @@ -0,0 +1,107 @@ +using System.Globalization; +using System.Security.Cryptography; +using System.Text; + +namespace AgentMemory.Abstractions.Domain; + +/// +/// A synthesized description of everything memory holds about one entity (S1). +/// +/// +/// +/// Recall about a well-known entity returns twenty separate facts that each cost context and say one +/// thing. A summary says the same in one item — but a summary is a derived memory, and derived +/// memory is where stores quietly start lying: the sources change, the summary does not, and nothing +/// about it looks any different afterwards. +/// +/// +/// So staleness here is proved, never assumed. is computed +/// from the exact set of live facts the summary was written from. Before a summary is used, that +/// fingerprint is recomputed from the store; if it does not match — a source superseded, a new fact +/// added, a confidence moved — the summary is not used. It is not repaired on the read path +/// and it is not returned with a caveat, because a caveat is something a caller can ignore. +/// +/// +/// That check is also why this earns its place in a graph rather than a summary column in a +/// relational row: the fingerprint is derived from the very provenance edges the store already keeps, +/// so "is this still true?" is a query rather than a convention someone has to remember to follow. +/// +/// +public sealed record EntitySummary +{ + /// Unique identifier for the summary node. + public required string SummaryId { get; init; } + + /// The entity this summarises. + public required string EntityId { get; init; } + + /// The synthesized text. + public required string Content { get; init; } + + /// + /// The facts this summary was written from, in the order they were read. + /// + /// + /// Retained rather than merely hashed so a stale summary can say what changed, and so the + /// EXTRACTED_FROM edges to the underlying sources can be rebuilt without re-synthesising. + /// + public required IReadOnlyList SourceFactIds { get; init; } + + /// + /// Hash of the source facts as they stood when the summary was written. + /// + /// + /// The staleness proof. Recompute it from the store and compare: equal means every source is + /// still live and unchanged, anything else means this text describes a state of the world that + /// no longer exists. + /// + public required string SourceFingerprint { get; init; } + + /// Owner (R1). Null = shared/global. + public string? OwnerId { get; init; } + + /// When the summary was synthesized. + public required DateTimeOffset GeneratedAtUtc { get; init; } + + /// Embedding of , when one has been generated. + public IReadOnlyList? Embedding { get; init; } + + /// + /// Computes the fingerprint of a set of source facts. + /// + /// + /// + /// Covers each fact's id, its confidence and whether it has been invalidated — the three things + /// that change what a truthful summary would say. Confidence is included deliberately: + /// reinforcement (S2) moves it, and a summary asserting something the store has since grown + /// doubtful about is exactly the stale shadow this design exists to prevent. + /// + /// + /// Order-independent, because the fingerprint must answer "are these the same facts?" and not + /// "did they come back in the same order?" — a query plan change would otherwise invalidate every + /// summary in the store without a single fact having moved. + /// + /// + public static string ComputeFingerprint(IEnumerable sources) + { + ArgumentNullException.ThrowIfNull(sources); + + var canonical = sources + .Select(s => string.Create( + CultureInfo.InvariantCulture, + $"{s.FactId}|{s.Confidence:F6}|{(s.Invalidated ? 1 : 0)}")) + .OrderBy(s => s, StringComparer.Ordinal) + .ToList(); + + var joined = string.Join("\n", canonical); + return Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(joined))).ToLowerInvariant(); + } +} + +/// +/// One fact's contribution to a summary's fingerprint (S1). +/// +/// The fact. +/// Its confidence at the time of reading. +/// Whether it has been superseded or otherwise invalidated. +public readonly record struct EntitySummarySource(string FactId, double Confidence, bool Invalidated); diff --git a/src/AgentMemory.Abstractions/Domain/LongTerm/Fact.cs b/src/AgentMemory.Abstractions/Domain/LongTerm/Fact.cs index 7814e5440..4128323ed 100644 --- a/src/AgentMemory.Abstractions/Domain/LongTerm/Fact.cs +++ b/src/AgentMemory.Abstractions/Domain/LongTerm/Fact.cs @@ -53,6 +53,24 @@ public sealed record Fact /// public DateTimeOffset? ValidUntil { get; init; } + /// + /// When this fact was superseded or otherwise invalidated; while it is live. + /// + /// + /// + /// The transaction clock, distinct from 's real-world one: + /// ValidUntil says when a fact stopped being true, this says when the store stopped + /// believing it. A contradicted fact is invalidated without its validity period changing at all. + /// + /// + /// The store has always recorded this — supersession is non-destructive precisely so as-of recall + /// can still reach it — but it was never projected onto the domain record, so a caller reading + /// facts had no way to tell a superseded one from a live one. Anything deriving from a fact set + /// (S1's summaries first among them) needs exactly that distinction. + /// + /// + public DateTimeOffset? InvalidatedAtUtc { get; init; } + /// /// Optional embedding vector for semantic search. /// diff --git a/src/AgentMemory.Abstractions/Domain/MemoryBlock.cs b/src/AgentMemory.Abstractions/Domain/MemoryBlock.cs new file mode 100644 index 000000000..41c61930d --- /dev/null +++ b/src/AgentMemory.Abstractions/Domain/MemoryBlock.cs @@ -0,0 +1,59 @@ +namespace AgentMemory.Abstractions.Domain; + +/// +/// One addressable line of a memory block (S4). +/// +/// The id of the underlying memory — what makes this line actionable. +/// Entity, fact or preference. +/// The rendered line. +public sealed record MemoryBlockLine(string MemoryId, MemoryItemKind Kind, string Text); + +/// +/// A small, human-readable projection of what memory holds (S4). +/// +/// +/// +/// The honest assessment of this store was "capable but opaque". A developer could not look at +/// an owner's memory and see it; they could query it, which is not the same thing. This is the view +/// that closes that gap. +/// +/// +/// Rendered on demand and never stored. That is the deliberate divergence from the +/// block-memory designs this borrows legibility from. A stored block is a second copy of the truth +/// that drifts from the graph the moment anything is written, and once an agent is allowed to edit +/// the block directly, the block — not the audited store — becomes where memory actually lives. Every +/// provenance edge, every trust level and every supersession record then describes a shadow of what +/// the system believes. +/// +/// +/// So this is a read surface. Changing memory goes through the ordinary write path, which +/// stamps trust, records provenance and leaves a history. To make that practical rather than +/// obstructive, each line carries its : a human who spots +/// something wrong can act on that exact item instead of rewriting prose and hoping a parser agrees. +/// +/// +public sealed record MemoryBlock +{ + /// The owner this block describes; null = shared/global. + public string? OwnerId { get; init; } + + /// When the block was rendered. It is a snapshot, and it says so. + public required DateTimeOffset RenderedAtUtc { get; init; } + + /// The lines, in rendering order. + public required IReadOnlyList Lines { get; init; } + + /// + /// How many memories were left out because the block was full. + /// + /// + /// Counted and surfaced rather than silently dropped. A block is a view a human trusts to + /// show them what memory holds; one that quietly stops short reads as "this is everything" and is + /// worse than no view at all, because it invites the conclusion that a missing memory was never + /// stored. + /// + public int OmittedCount { get; init; } + + /// Whether anything was left out. + public bool IsTruncated => OmittedCount > 0; +} diff --git a/src/AgentMemory.Abstractions/Options/ExtractionOptions.cs b/src/AgentMemory.Abstractions/Options/ExtractionOptions.cs index 535dc33a0..f4fb37145 100644 --- a/src/AgentMemory.Abstractions/Options/ExtractionOptions.cs +++ b/src/AgentMemory.Abstractions/Options/ExtractionOptions.cs @@ -95,6 +95,60 @@ public sealed class ExtractionOptions /// /// public bool SupersedeReplacedFacts { get; set; } + + /// + /// Skips the extraction call entirely for turns that cannot carry a fact — greetings, thanks, + /// bare acknowledgement (E4). + /// + /// + /// + /// The extraction call is where the money is: the write is one round trip to Neo4j, while + /// extraction is a model call over the rendered transcript and on a corpus build it is + /// essentially the whole bill. An "ok, thanks!" turn pays it for nothing. + /// + /// + /// Deliberately not a write-side gate. Skipping the persistence of a triple already in the + /// store looks like the more obvious saving and would silently disable two features that depend + /// on the duplicate write happening: , + /// where a re-asserted fact earns α, and the mention_count the salience reranker reads. + /// Corroboration is the repeated write. + /// + /// + /// Detection is a closed vocabulary, deterministic, and biased hard toward extracting: a missed + /// skip costs one call, while gating a turn that did carry a fact means the memory is never + /// formed and nothing downstream can recover it. "yes", "no" and "sure" are therefore treated as + /// contentful — each is a complete answer to a question. + /// + /// + /// Off by default: it changes what gets extracted, and prompt-and-transcript bytes are + /// fingerprinted into every measured run. + /// + /// + public bool SkipUninformativeTurns { get; set; } + + /// + /// How many earlier turns to hand the extractors as read-only context (E2). 0 — the + /// default — is the pre-E2 behaviour of extracting from the batch alone. + /// + /// + /// + /// One batch is often not enough to understand itself: "I moved there last year" needs the turn + /// that named the place, and "she recommended it" needs the turn that named her. Without context + /// those either extract nothing or extract an unresolved pronoun as an entity. + /// + /// + /// Context turns are read, never extracted from, and never attributed to. Extracting them + /// would re-assert facts already stored — which now earns confidence + /// () and increments the + /// mention_count the salience reranker reads, so a fact would gain both merely by sitting + /// inside a sliding window. Corroboration would become recency. + /// + /// + /// Costs input tokens on every extraction, which is why it is off rather than defaulted to a + /// nominal value. Zep uses 4. + /// + /// + public int ExtractionContextTurns { get; set; } } /// Controls which matching strategies are used for entity resolution. @@ -125,4 +179,5 @@ public sealed class EntityValidationOptions public bool RejectPunctuationOnly { get; set; } = true; /// Reject entities whose names are common stop-words. public bool UseStopwordFilter { get; set; } = true; + } diff --git a/src/AgentMemory.Abstractions/Options/MemoryOptions.cs b/src/AgentMemory.Abstractions/Options/MemoryOptions.cs index 68fc61138..da84abcc7 100644 --- a/src/AgentMemory.Abstractions/Options/MemoryOptions.cs +++ b/src/AgentMemory.Abstractions/Options/MemoryOptions.cs @@ -23,6 +23,157 @@ public sealed record MemoryOptions /// Whether to enable GraphRAG integration. public bool EnableGraphRag { get; init; } + /// + /// Falls back to an owner-bounded similarity scan when an owner-scoped vector search returns + /// fewer rows than were asked for. + /// + /// + /// + /// Neo4j's vector index is global, so an owner filter is a POST-filter on a top-K drawn from every + /// tenant. Measured on a 50-owner corpus, a mean of 7 of 60 candidates reached the querying + /// owner. Today only a totally empty result triggers a rescue; a search returning 2 rows of + /// a requested 10 is accepted as the answer. + /// + /// + /// That is sometimes right and sometimes badly wrong: question 5d3d2817 returned 2 facts + /// from a 710-fact graph with the answer present, and was answered incorrectly in both arms. + /// + /// + /// Off by default. It costs one extra query per short result — bounded by the owner's own rows, + /// not the corpus — and every recorded measurement was taken without it, so turning it on is a + /// stated decision rather than an inherited one. + /// + /// + public bool RescueShortOwnerResults { get; init; } + + /// + /// Boosts recalled facts that sit close, in the graph, to the entity the query is about (R6). + /// + /// + /// + /// Vector similarity asks whether a row looks like the query; this asks whether it is + /// about what the query is about. A plainly-worded fact hanging directly off the named + /// entity is often the answer, while a fact that paraphrases the query beautifully may concern + /// someone else entirely. + /// + /// + /// Reuses MemoryRankingOptions.StructuralDecayGamma as its decay constant rather than + /// introducing a second one, and does nothing unless that decay is also enabled -- at gamma = 1 + /// every boost is 1.0, so the two extra queries could not change an ordering. + /// + /// + /// Off by default: it changes recall ordering, costs two bounded queries per fact section, and + /// every recorded measurement was taken without it. + /// + /// + public bool NodeDistanceReranking { get; init; } + + /// + /// Boosts recalled facts the conversation keeps returning to (R7). + /// + /// + /// + /// A fact the world asserted five times is usually more central than one mentioned once, and + /// similarity cannot see that: two facts phrased alike score alike however often either was said. + /// + /// + /// Ingestion-side. It counts how often the WORLD re-asserted a fact, not how often we + /// retrieved it. Ranking on our own retrievals -- the :MemoryReadAudit trail -- would be a + /// rich-get-richer loop: whatever ranks highly gets retrieved, which raises its count, which + /// raises its rank. That looks like learning and is self-reinforcement. + /// + /// + /// Logarithmic, so the gap between one mention and three counts while the gap between thirty and + /// thirty-two does not. Off by default; every recorded measurement was taken without it. + /// + /// + public bool MentionFrequencyReranking { get; init; } + + /// + /// Starts the post-recall access-tracking write without waiting for it. + /// + /// + /// + /// Access tracking is bookkeeping — it feeds decay and retention, and nothing in the recall the + /// caller is waiting for depends on it. Awaiting it puts a write burst on the pre-model read path: + /// at shipped defaults, up to 25 nodes updated before the model is invoked. + /// + /// + /// Off by default, and the reason is a hazard rather than caution. The write runs on scoped + /// services — a driver session, a repository — and a host that disposes its DI scope when the + /// response is returned will dispose them out from under a deferred write. That surfaces as an + /// ObjectDisposedException in a log nobody reads, and access tracking silently stops. Hosts + /// whose scope outlives the response (a long-lived agent, a hosted service) can turn it on; a + /// short-lived request scope should not. + /// + /// + /// Deferred work is detached from the request's cancellation token deliberately: that token is + /// cancelled as soon as the response completes, so passing it through would cancel the very write + /// that was deferred — a feature that looks enabled and does nothing. + /// + /// + public bool DeferAccessTracking { get; init; } + + /// + /// How much a fact's confidence moves when the world corroborates or contradicts it (S2). + /// + /// + /// + /// Today confidence is set once by extraction and never earns or loses anything: a fact stated + /// five times and one stated in passing carry whatever number the extractor happened to report. + /// With α > 0 a re-asserted triple gains α, and a fact superseded by a contradiction loses 2α. + /// + /// + /// Asymmetric on purpose. Being contradicted is stronger evidence against a fact than one + /// more restatement is for it — a claim that is simply repeated may just be a habit of phrasing, + /// while a claim the world has replaced is one the world stopped believing. + /// + /// + /// Clamped to [0,1] at both ends. Confidence is read by ranking, dedup and decay, and a value + /// outside that range would propagate into computations where it means nothing. + /// + /// + /// 0 (off) by default, and it is the off switch in Cypher too: at α = 0 the upsert's + /// confidence assignment is byte-for-byte what it was, so no sealed measurement moves. A sensible + /// starting value is 0.02–0.05 — large enough to separate a well-attested fact over a + /// conversation, small enough that a single restatement does not overwhelm what extraction judged. + /// + /// + public double ConfidenceReinforcementAlpha { get; init; } + + /// + /// Routes a turn that names a past time to bitemporal recall at that time (R4). + /// + /// + /// + /// RecallAsOfAsync has existed since the bitemporal work, and until now nothing in an + /// ordinary conversation could reach it: "what did I think back in March?" recalled against now, + /// exactly like every other question. Without this, the bitemporal read path is a capability + /// nothing can ask for. + /// + /// + /// Detection is deterministic, needs no model call, and errs heavily toward not matching: + /// a missed expression costs nothing (the turn recalls against now, today's behaviour), while a + /// false positive silently narrows recall to a window the user never asked about and returns an + /// answer that looks entirely ordinary. So "in March" resolves and a bare "March" does not; + /// "last week" resolves and "the last item" does not. + /// + /// + /// Off by default: it changes which memories a temporal question sees. + /// + /// + /// Scope. Honoured by IMemoryService.RecallAsync, and therefore by everything that + /// recalls through it — the MCP memory tools, the Agent Framework and Semantic Kernel adapters. + /// The memory://context/{session_id} MCP resource is the one read surface that does + /// not: it composes IMemoryContextAssembler directly, and the MCP server references only + /// AgentMemory.Abstractions by design. Reaching the resolver from there would mean a project + /// reference to AgentMemory.Core purely for a secondary surface, so the divergence is recorded + /// here rather than papered over — a temporal question asked through that resource recalls against + /// now. + /// + /// + public bool ResolveTemporalQueries { get; init; } + // NOTE: extraction at the Core layer is explicit (call ExtractAndPersistAsync / // ExtractFromSessionAsync). Automatic extraction on message persist is an adapter concern, configured // by AgentFrameworkOptions.AutoExtractOnPersist. The former EnableAutoExtraction flag here was read diff --git a/src/AgentMemory.Abstractions/Options/ReasoningMemoryOptions.cs b/src/AgentMemory.Abstractions/Options/ReasoningMemoryOptions.cs index e31ce4a13..07183c322 100644 --- a/src/AgentMemory.Abstractions/Options/ReasoningMemoryOptions.cs +++ b/src/AgentMemory.Abstractions/Options/ReasoningMemoryOptions.cs @@ -1,3 +1,5 @@ +using AgentMemory.Abstractions.Domain; + namespace AgentMemory.Abstractions.Options; /// @@ -13,4 +15,31 @@ public sealed record ReasoningMemoryOptions /// Maximum number of traces to retain per session. public int? MaxTracesPerSession { get; init; } + + /// + /// The trust level stamped on every reasoning trace. + /// + /// + /// + /// Traces were the one recall category with no trust level at all: StartTraceAsync strips + /// any caller-supplied trust_level — correctly, since a caller must never self-assign a + /// level that bypasses the admission policy — and then nothing stamped one, so every trace read + /// back as . That is indistinguishable from "no signal + /// recorded", which is exactly what the enum exists to avoid. + /// + /// + /// is the accurate label: a trace is the agent's own + /// record of what it did, not something a user asserted. + /// + /// + /// Safe at shipped defaults, and worth stating why. MinimumTrustForAdmissionBypass + /// defaults to , which + /// does not reach, so no trace gains an admission + /// bypass. MinimumTrustForSystemRole defaults to , + /// which everything already met. A host that has lowered the bypass threshold to + /// or below is the one case where this changes + /// behaviour, and it can set this back to . + /// + /// + public MemoryTrustLevel DefaultTraceTrustLevel { get; init; } = MemoryTrustLevel.ModelGenerated; } diff --git a/src/AgentMemory.Abstractions/Options/RecallOptions.cs b/src/AgentMemory.Abstractions/Options/RecallOptions.cs index 281e5a572..6fec728dd 100644 --- a/src/AgentMemory.Abstractions/Options/RecallOptions.cs +++ b/src/AgentMemory.Abstractions/Options/RecallOptions.cs @@ -5,6 +5,29 @@ /// public sealed record RecallOptions { + /// + /// How long recall may spend assembling context before returning what it has (rank 13). + /// + /// + /// + /// — the default — waits for every section, which is exactly today's + /// behaviour. When set, sections still running when the budget expires are dropped and the + /// context comes back without them. + /// + /// + /// Degradation is marked, never silent. Every dropped section reports + /// TimedOut in its diagnostics, and RecallResult.Metadata["latencyBudgetExceeded"] + /// is set. A section cut short otherwise looks identical to one that searched and found nothing, + /// and a caller answering from a context it believes is complete is worse off than one that + /// waited. + /// + /// + /// Abandoned work is not cancelled mid-flight — the queries are already in the driver, and + /// tearing them down buys nothing the caller is waiting for. Their results are simply not used. + /// + /// + public TimeSpan? LatencyBudget { get; init; } + /// Maximum recent messages to include. public int MaxRecentMessages { get; init; } = 10; diff --git a/src/AgentMemory.Abstractions/Repositories/IEntitySummaryRepository.cs b/src/AgentMemory.Abstractions/Repositories/IEntitySummaryRepository.cs new file mode 100644 index 000000000..ef44c9db2 --- /dev/null +++ b/src/AgentMemory.Abstractions/Repositories/IEntitySummaryRepository.cs @@ -0,0 +1,33 @@ +using AgentMemory.Abstractions.Domain; +using AgentMemory.Abstractions.Options; + +namespace AgentMemory.Abstractions.Repositories; + +/// +/// Persistence for synthesized entity summaries (S1). +/// +/// +/// One summary per entity per owner. There is no history: a summary is derived, so an old one is not +/// a record of what was believed — it is a record of what a synthesizer produced from facts the store +/// still has, and those facts carry their own bitemporal history already. +/// +public interface IEntitySummaryRepository +{ + /// Stores a summary, replacing the entity's existing one. + Task UpsertAsync(EntitySummary summary, CancellationToken cancellationToken = default); + + /// + /// Reads the entity's summary, if any. Owner-confined (R1). + /// + /// + /// Returns whatever is stored, without judging freshness: staleness is decided against the + /// live facts by the caller that has them, and a repository silently withholding a stale row + /// would make "no summary" and "outdated summary" indistinguishable to a regeneration pass. + /// + Task GetByEntityAsync( + string entityId, MemoryScope scope, CancellationToken cancellationToken = default); + + /// Removes an entity's summary. + Task DeleteByEntityAsync( + string entityId, MemoryScope scope, CancellationToken cancellationToken = default); +} diff --git a/src/AgentMemory.Abstractions/Services/IEntityExtractor.cs b/src/AgentMemory.Abstractions/Services/IEntityExtractor.cs index 3c815b99a..7917430a6 100644 --- a/src/AgentMemory.Abstractions/Services/IEntityExtractor.cs +++ b/src/AgentMemory.Abstractions/Services/IEntityExtractor.cs @@ -13,4 +13,29 @@ public interface IEntityExtractor Task> ExtractAsync( IReadOnlyList messages, CancellationToken cancellationToken = default); + + /// + /// Extracts from , reading + /// only to understand them (E2). + /// + /// + /// + /// The default implementation ignores the context entirely and calls the existing overload + /// with the targets — byte-identical to pre-E2 behaviour. An extractor that cannot use context is + /// therefore correct by doing nothing, rather than being broken by a widened window. + /// + /// + /// A default interface method rather than a new required member: the extractor interfaces are + /// public and locked under SemVer, and a third-party extractor must keep compiling. + /// + /// + /// Implementers must not extract from the context turns. Doing so re-asserts facts already stored, + /// which now inflates confidence (S2) and mention_count (R7) purely because a fact sat + /// inside a sliding window — turning corroboration into recency. + /// + /// + Task> ExtractWithContextAsync( + ExtractionWindow window, + CancellationToken cancellationToken = default) + => ExtractAsync(window.Targets, cancellationToken); } diff --git a/src/AgentMemory.Abstractions/Services/IEntitySummaryService.cs b/src/AgentMemory.Abstractions/Services/IEntitySummaryService.cs new file mode 100644 index 000000000..8255392b5 --- /dev/null +++ b/src/AgentMemory.Abstractions/Services/IEntitySummaryService.cs @@ -0,0 +1,37 @@ +using AgentMemory.Abstractions.Domain; +using AgentMemory.Abstractions.Options; + +namespace AgentMemory.Abstractions.Services; + +/// +/// Writes entity summaries and withholds any that can no longer prove it is current (S1). +/// +/// +/// A summary is derived memory, which is where a store quietly starts lying: the sources change, the +/// summary does not, and nothing about it looks any different afterwards. Every read here is checked +/// against the live facts before it is handed back. +/// +public interface IEntitySummaryService +{ + /// + /// Synthesizes and stores a summary for the entity, replacing any existing one. + /// + /// The stored summary, or when there was nothing to summarise. + Task RefreshAsync( + Entity entity, MemoryScope scope, CancellationToken cancellationToken = default); + + /// + /// Returns the entity's summary only if it still describes the current facts. + /// + /// + /// The summary when its fingerprint still matches the store; when there is + /// none or when the one on record has gone stale. + /// + /// + /// Null for stale rather than a summary carrying an IsStale flag. A flag puts the decision + /// in every caller's hands, and one caller forgetting to check it is indistinguishable from + /// correct memory. + /// + Task GetIfCurrentAsync( + Entity entity, MemoryScope scope, CancellationToken cancellationToken = default); +} diff --git a/src/AgentMemory.Abstractions/Services/IEntitySummarySynthesizer.cs b/src/AgentMemory.Abstractions/Services/IEntitySummarySynthesizer.cs new file mode 100644 index 000000000..e5e03c5a9 --- /dev/null +++ b/src/AgentMemory.Abstractions/Services/IEntitySummarySynthesizer.cs @@ -0,0 +1,34 @@ +using AgentMemory.Abstractions.Domain; + +namespace AgentMemory.Abstractions.Services; + +/// +/// Turns an entity's facts into one synthesized description (S1). +/// +/// +/// +/// A seam rather than a fixed implementation because the two reasonable answers have opposite cost +/// profiles. The shipped default composes the text deterministically — no completion, reproducible +/// across runs, and free to regenerate whenever sources move, which matters because sources move +/// constantly. A host that wants prose can supply an LLM implementation and pay for it. +/// +/// +/// Whatever synthesises, it may only use the facts it is given. The summary's fingerprint is +/// computed from exactly those facts, so a synthesizer that reached for anything else would produce +/// text whose staleness could never be detected — the one failure this design exists to rule out. +/// +/// +public interface IEntitySummarySynthesizer +{ + /// + /// Composes a description of from . + /// + /// + /// The summary text, or when there is nothing worth summarising — which is + /// a real answer, not a failure, and must not be stored as an empty summary. + /// + Task SynthesizeAsync( + Entity entity, + IReadOnlyList facts, + CancellationToken cancellationToken = default); +} diff --git a/src/AgentMemory.Abstractions/Services/IFactExtractor.cs b/src/AgentMemory.Abstractions/Services/IFactExtractor.cs index 2ebaa4533..ff26a9b40 100644 --- a/src/AgentMemory.Abstractions/Services/IFactExtractor.cs +++ b/src/AgentMemory.Abstractions/Services/IFactExtractor.cs @@ -13,4 +13,29 @@ public interface IFactExtractor Task> ExtractAsync( IReadOnlyList messages, CancellationToken cancellationToken = default); + + /// + /// Extracts from , reading + /// only to understand them (E2). + /// + /// + /// + /// The default implementation ignores the context entirely and calls the existing overload + /// with the targets — byte-identical to pre-E2 behaviour. An extractor that cannot use context is + /// therefore correct by doing nothing, rather than being broken by a widened window. + /// + /// + /// A default interface method rather than a new required member: the extractor interfaces are + /// public and locked under SemVer, and a third-party extractor must keep compiling. + /// + /// + /// Implementers must not extract from the context turns. Doing so re-asserts facts already stored, + /// which now inflates confidence (S2) and mention_count (R7) purely because a fact sat + /// inside a sliding window — turning corroboration into recency. + /// + /// + Task> ExtractWithContextAsync( + ExtractionWindow window, + CancellationToken cancellationToken = default) + => ExtractAsync(window.Targets, cancellationToken); } diff --git a/src/AgentMemory.Abstractions/Services/IMemoryIngestion.cs b/src/AgentMemory.Abstractions/Services/IMemoryIngestion.cs index 47ed4cf4b..235ac609b 100644 --- a/src/AgentMemory.Abstractions/Services/IMemoryIngestion.cs +++ b/src/AgentMemory.Abstractions/Services/IMemoryIngestion.cs @@ -86,4 +86,100 @@ Task ExtractFromConversationAsync( string conversationId, string? userId = null, CancellationToken cancellationToken = default); + + /// + /// Ingests many extraction requests with bounded concurrency (rank 27). + /// + /// + /// + /// The documented bulk path. Loading a backlog was always possible — call + /// in a loop, or in a Parallel.ForEachAsync — and both + /// obvious ways are wrong in opposite directions: the serial loop wastes hours, and the unbounded + /// parallel one saturates the provider quota and the connection pool, degrading p99 for every + /// other tenant in the process while median latency looks fine. + /// + /// + /// Composed, not new machinery. This paces calls the host could have made itself; it does + /// not add a second ingestion path. That matters because a separate bulk pipeline would be a + /// second place for trust stamping, provenance and owner scoping to drift out of agreement with + /// the per-request one. + /// + /// + /// A default implementation is supplied so the interface stays SemVer-compatible: an existing + /// implementation keeps compiling and gets a correct, if + /// unoptimised, bulk path for free. + /// + /// + async Task IngestBulkAsync( + IReadOnlyList requests, + BulkIngestionOptions? options = null, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(requests); + options ??= new BulkIngestionOptions(); + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(options.MaxConcurrency); + + var outcomes = new BulkIngestionOutcome?[requests.Count]; + using var gate = new SemaphoreSlim(options.MaxConcurrency, options.MaxConcurrency); + // Cancels the remaining work when ContinueOnError is false. Linked rather than replacing the + // caller's token, so a stop-on-error run still honours an outer cancellation. + using var stop = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + + var running = new List(requests.Count); + for (var i = 0; i < requests.Count; i++) + { + if (stop.IsCancellationRequested) break; + + var index = i; + running.Add(Task.Run(async () => + { + try + { + await gate.WaitAsync(stop.Token).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + // Queued behind the gate when the run stopped. Left unattempted -- and caught + // HERE rather than by the block below, whose finally would Release a slot this + // task never acquired. + return; + } + + try + { + var result = await ExtractAndPersistAsync(requests[index], stop.Token) + .ConfigureAwait(false); + outcomes[index] = new BulkIngestionOutcome(index, result, null); + } + catch (OperationCanceledException) when (stop.IsCancellationRequested) + { + // Left null: never attempted, or abandoned mid-flight. Recorded as + // NotAttemptedCount rather than as a failure, because re-running a cancelled + // request is correct and re-running a failed one may not be. + } + catch (Exception ex) + { + outcomes[index] = new BulkIngestionOutcome(index, null, ex); + if (!options.ContinueOnError) await stop.CancelAsync().ConfigureAwait(false); + } + finally + { + gate.Release(); + } + }, CancellationToken.None)); + } + + await Task.WhenAll(running).ConfigureAwait(false); + + // An outer cancellation is still an error for the caller; a stop-on-first-failure is not, + // because the failure it stopped for is already reported in the outcomes. + cancellationToken.ThrowIfCancellationRequested(); + + var completed = outcomes.Where(o => o is not null).Select(o => o!).ToList(); + return new BulkIngestionResult + { + Outcomes = completed, + NotAttemptedCount = requests.Count - completed.Count, + }; + } } diff --git a/src/AgentMemory.Abstractions/Services/IMemoryReranker.cs b/src/AgentMemory.Abstractions/Services/IMemoryReranker.cs new file mode 100644 index 000000000..b9634b83d --- /dev/null +++ b/src/AgentMemory.Abstractions/Services/IMemoryReranker.cs @@ -0,0 +1,93 @@ +using AgentMemory.Abstractions.Domain; +using AgentMemory.Abstractions.Options; + +namespace AgentMemory.Abstractions.Services; + +/// +/// Reorders the candidates a retrieval returned, before context budgeting truncates them. +/// +/// +/// +/// This seam is deliberately narrower than the task that proposed it. R2 asked for a reranker +/// in order to fuse GraphRAG as a distinct retrieval channel, and that premise was killed on +/// measurement: of 132 GraphRAG items, 132 were duplicates of what the structured surface had +/// already retrieved and 0 verdicts changed. Fusing a channel that returns the same rows in a +/// different order cannot help, and the trigger for revisiting it — a corpus with a genuinely separate +/// knowledge graph — has not fired. +/// +/// +/// What survives is the extension point, which R6 (node-distance) and R7 (mention frequency) +/// both need and neither of which is about fusion. Keeping the seam and dropping the fusion is the +/// honest reading of that measurement: the mechanism was worth building, the use case was not. +/// +/// +/// Reranking reorders survivors; it cannot rescue what retrieval never returned. That is why +/// this ships behind the retrieval-sufficiency work rather than before it — a reranker over a starved +/// candidate set reorders seven of an owner's 504 facts and reports success. The signal now measures +/// AUC 0.709–0.768, which is what made this safe to build. +/// +/// +/// Implementations must be pure and fast: this runs inside recall, on the blocking path, once +/// per section. Anything needing a database round trip should issue one bounded query, never +/// one per candidate. +/// +/// +public interface IMemoryReranker +{ + /// + /// Whether this reranker should run at all for the current request. + /// + /// + /// Checked before the candidates are materialised, so a disabled reranker costs nothing — not even + /// the allocation of the list it would have reordered. + /// + bool IsEnabled { get; } + + /// + /// Returns reordered, or the same sequence when nothing changed. + /// + /// + /// The retrieval's own ranking, best first. Scores are the provider's, comparable within this + /// section and meaningless across sections. + /// + /// What the request knows: the query, its embedding, and the owner scope. + /// Cancels the reordering; a cancelled rerank leaves the provider order. + /// + /// + /// Returning a different set — adding or dropping candidates — is not supported and not + /// checked for cheaply, so it would corrupt the section's diagnostics silently. A reranker + /// reorders; a filter belongs in retrieval, where its effect on recall is measurable. + /// + /// + /// A reranker that throws must not fail the recall: callers treat an exception as "no reordering" + /// and continue with the provider's own ranking, because a degraded order is recoverable and a + /// failed recall is not. + /// + /// + Task> RerankAsync( + IReadOnlyList candidates, + MemoryRerankContext context, + CancellationToken cancellationToken = default); +} + +/// +/// What a reranker is told about the request it is reordering for. +/// +/// The user's query text, for a reranker that reads language. +/// +/// The query vector, when one was generated. Null on a turn narrowed to categories that need no +/// vector — a reranker depending on it must handle that rather than assume. +/// +/// +/// The owner scope the retrieval ran under. A reranker issuing its own query must carry this: +/// a reordering pass that reads another owner's graph is an isolation breach wearing a ranking hat. +/// +/// +/// Which recall category is being reordered, so one reranker can serve several and behave differently +/// per kind rather than pretending a fact and a trace rank alike. +/// +public sealed record MemoryRerankContext( + string? Query, + float[]? QueryEmbedding, + MemoryScope? Scope, + MemoryItemKind SectionKind); diff --git a/src/AgentMemory.Abstractions/Services/IPreferenceExtractor.cs b/src/AgentMemory.Abstractions/Services/IPreferenceExtractor.cs index 348262991..c5d6f29a4 100644 --- a/src/AgentMemory.Abstractions/Services/IPreferenceExtractor.cs +++ b/src/AgentMemory.Abstractions/Services/IPreferenceExtractor.cs @@ -13,4 +13,29 @@ public interface IPreferenceExtractor Task> ExtractAsync( IReadOnlyList messages, CancellationToken cancellationToken = default); + + /// + /// Extracts from , reading + /// only to understand them (E2). + /// + /// + /// + /// The default implementation ignores the context entirely and calls the existing overload + /// with the targets — byte-identical to pre-E2 behaviour. An extractor that cannot use context is + /// therefore correct by doing nothing, rather than being broken by a widened window. + /// + /// + /// A default interface method rather than a new required member: the extractor interfaces are + /// public and locked under SemVer, and a third-party extractor must keep compiling. + /// + /// + /// Implementers must not extract from the context turns. Doing so re-asserts facts already stored, + /// which now inflates confidence (S2) and mention_count (R7) purely because a fact sat + /// inside a sliding window — turning corroboration into recency. + /// + /// + Task> ExtractWithContextAsync( + ExtractionWindow window, + CancellationToken cancellationToken = default) + => ExtractAsync(window.Targets, cancellationToken); } diff --git a/src/AgentMemory.Abstractions/Services/IRelationshipExtractor.cs b/src/AgentMemory.Abstractions/Services/IRelationshipExtractor.cs index 32592025d..2b1a77613 100644 --- a/src/AgentMemory.Abstractions/Services/IRelationshipExtractor.cs +++ b/src/AgentMemory.Abstractions/Services/IRelationshipExtractor.cs @@ -13,4 +13,29 @@ public interface IRelationshipExtractor Task> ExtractAsync( IReadOnlyList messages, CancellationToken cancellationToken = default); + + /// + /// Extracts from , reading + /// only to understand them (E2). + /// + /// + /// + /// The default implementation ignores the context entirely and calls the existing overload + /// with the targets — byte-identical to pre-E2 behaviour. An extractor that cannot use context is + /// therefore correct by doing nothing, rather than being broken by a widened window. + /// + /// + /// A default interface method rather than a new required member: the extractor interfaces are + /// public and locked under SemVer, and a third-party extractor must keep compiling. + /// + /// + /// Implementers must not extract from the context turns. Doing so re-asserts facts already stored, + /// which now inflates confidence (S2) and mention_count (R7) purely because a fact sat + /// inside a sliding window — turning corroboration into recency. + /// + /// + Task> ExtractWithContextAsync( + ExtractionWindow window, + CancellationToken cancellationToken = default) + => ExtractAsync(window.Targets, cancellationToken); } diff --git a/src/AgentMemory.Abstractions/Services/IUnifiedMemoryExtractor.cs b/src/AgentMemory.Abstractions/Services/IUnifiedMemoryExtractor.cs index fc3420f71..47d3e50aa 100644 --- a/src/AgentMemory.Abstractions/Services/IUnifiedMemoryExtractor.cs +++ b/src/AgentMemory.Abstractions/Services/IUnifiedMemoryExtractor.cs @@ -12,4 +12,18 @@ public interface IUnifiedMemoryExtractor Task ExtractAsync( IReadOnlyList messages, CancellationToken cancellationToken = default); + + /// + /// Extracts every supported category from , reading + /// only to understand them (E2). + /// + /// + /// The default implementation ignores the context and calls the existing overload — byte-identical + /// to pre-E2 behaviour. Implementers must not extract from context turns: doing so re-asserts + /// stored facts and inflates confidence (S2) and mention_count (R7) on nothing but recency. + /// + Task ExtractWithContextAsync( + ExtractionWindow window, + CancellationToken cancellationToken = default) + => ExtractAsync(window.Targets, cancellationToken); } diff --git a/src/AgentMemory.AgentFramework/AgentFrameworkOptions.cs b/src/AgentMemory.AgentFramework/AgentFrameworkOptions.cs index d74aa781b..03e243468 100644 --- a/src/AgentMemory.AgentFramework/AgentFrameworkOptions.cs +++ b/src/AgentMemory.AgentFramework/AgentFrameworkOptions.cs @@ -11,6 +11,33 @@ public sealed class AgentFrameworkOptions /// public ContextFormatOptions ContextFormat { get; set; } = new(); + /// + /// Drops recalled chat history the host is already sending in the live thread. + /// + /// + /// + /// The provider sees the full live thread and, until now, discarded it — so recall's + /// RecentMessages re-sent turns the model was already being given, and the host paid for + /// both copies. + /// + /// + /// Filtered before MaxChatHistoryMessages applies, so this is a quality change as + /// much as a cost one: the same budget then carries that many genuinely new messages instead of + /// duplicates of the current turn. + /// + /// + /// Matching is on content only, never role. RecalledMessageRoleGate rewrites a + /// recalled message's role — privileged down to user below the trust threshold — while leaving its + /// content identical, so a role-keyed comparison would miss every match on exactly the hosts that + /// raised MinimumTrustForSystemRole. + /// + /// + /// On by default: sending the model two copies of the same turn has no upside, and the comparison + /// is a hash set over the thread the provider already holds. + /// + /// + public bool DeduplicateRecalledHistory { get; set; } = true; + /// /// When , the memory service runs extraction (entity/fact/preference) /// automatically each time a message is persisted. Set to to extract diff --git a/src/AgentMemory.AgentFramework/Mapping/MafTypeMapper.cs b/src/AgentMemory.AgentFramework/Mapping/MafTypeMapper.cs index b95beb03b..58f45d79b 100644 --- a/src/AgentMemory.AgentFramework/Mapping/MafTypeMapper.cs +++ b/src/AgentMemory.AgentFramework/Mapping/MafTypeMapper.cs @@ -59,7 +59,8 @@ public static IReadOnlyList ToContextMessages( MemoryContext context, ContextFormatOptions? formatOptions = null, IMemoryContextAdmissionPolicy? admissionPolicy = null, - ILogger? logger = null) + ILogger? logger = null, + IReadOnlyList? liveThread = null) { var options = formatOptions ?? new ContextFormatOptions(); var admission = admissionPolicy ?? new DefaultMemoryContextAdmissionPolicy(); @@ -199,6 +200,38 @@ List CategoryMessages( // newest `chatBudget` items are the FRONT of the list — Take(chatBudget). (R6-D: the previous // Skip(count - chatBudget) kept the TAIL, i.e. the OLDEST messages, dropping the newest turns // first — the opposite of "most recent".) Order is preserved (lead → chat → memory). + // The host already sends the live thread, and recall returns the same recent turns from + // storage, so the model sees them twice. Filtered BEFORE the budget on purpose: dropping + // duplicates first turns the same `chatBudget` slots into that many genuinely new messages + // rather than merely shortening the block -- a quality gain, not only a token saving. + // + // Fingerprinted on CONTENT ONLY, never role. RecalledMessageRoleGate rewrites a recalled + // message's role -- privileged to "user" below the trust threshold -- while leaving its + // content untouched, so a role-keyed fingerprint would fail to match precisely for the hosts + // that hardened MinimumTrustForSystemRole: the feature would silently no-op for the + // security-conscious configuration and work everywhere else. + if (liveThread is { Count: > 0 }) + { + var live = liveThread + .Select(message => NormalizeForDedup(message.Text)) + .Where(text => text.Length > 0) + .ToHashSet(StringComparer.Ordinal); + + if (live.Count > 0) + { + var deduped = chatMessages + .Where(message => !live.Contains(NormalizeForDedup(message.Text))) + .ToList(); + if (deduped.Count != chatMessages.Count) + { + logger?.LogDebug( + "Dropped {Dropped} recalled message(s) already present in the live thread.", + chatMessages.Count - deduped.Count); + } + chatMessages = deduped; + } + } + int chatBudget = Math.Max(0, options.MaxChatHistoryMessages); var keptChat = chatMessages.Count > chatBudget ? chatMessages.Take(chatBudget).ToList() @@ -211,6 +244,28 @@ List CategoryMessages( return result; } + /// + /// The dedup key for a chat message: its content, normalised for whitespace and case. + /// + /// + /// + /// Content only. Role is excluded because RecalledMessageRoleGate rewrites a recalled + /// message's role while leaving its content identical, so keying on role would miss every match on + /// exactly the hosts that raised MinimumTrustForSystemRole. + /// + /// + /// Whitespace-collapsed and case-folded because the live thread and the stored copy travel + /// different paths — one through the host's own formatting, one through persistence and back — and + /// a trailing newline is not a different message. Ordinal comparison after folding, so this stays + /// culture-independent. + /// + /// + internal static string NormalizeForDedup(string? text) => + string.IsNullOrWhiteSpace(text) + ? string.Empty + : string.Join(' ', text.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries)) + .ToUpperInvariant(); + // Extracted from ToContextMessages' local Admit closure (stabilization fix) so it can be shared with // ToGatedChatMessages below, instead of the two call sites re-deriving the same admission decision and // logging independently. Behavior is unchanged from the closure this replaces. diff --git a/src/AgentMemory.AgentFramework/Neo4jMemoryContextProvider.cs b/src/AgentMemory.AgentFramework/Neo4jMemoryContextProvider.cs index f06fdce70..63e7f756c 100644 --- a/src/AgentMemory.AgentFramework/Neo4jMemoryContextProvider.cs +++ b/src/AgentMemory.AgentFramework/Neo4jMemoryContextProvider.cs @@ -103,7 +103,12 @@ internal async Task BuildContextAsync( using var ownerScope = _ownerContext?.BeginOwnerScope(userId); try { - var userMessages = messages + // Materialised once: the thread is enumerated for the query below AND handed to the + // mapper for dedup, and `messages` is an IEnumerable that a caller may well have built + // lazily. Enumerating it twice would be a silent correctness bug for a generator source. + var liveThread = messages as IReadOnlyList ?? messages.ToList(); + + var userMessages = liveThread .Where(m => m.Role == ChatRole.User && !string.IsNullOrWhiteSpace(m.Text)) .ToList(); @@ -194,7 +199,13 @@ internal async Task BuildContextAsync( return BuildResult(); } - var contextMessages = MafTypeMapper.ToContextMessages(recallResult.Context, _formatOptions, _admissionPolicy, _logger); + // 2.5. The host is already sending the live thread; recall returns the same recent turns + // from storage, so without this the model sees them twice and pays for both. Passed here + // rather than filtered afterwards because the mapper drops duplicates BEFORE applying + // MaxChatHistoryMessages -- so the same budget carries that many genuinely new messages. + var contextMessages = MafTypeMapper.ToContextMessages( + recallResult.Context, _formatOptions, _admissionPolicy, _logger, + _agentOptions.DeduplicateRecalledHistory ? liveThread : null); if (contextMessages.Count == 0) return BuildResult(); diff --git a/src/AgentMemory.Core/Extraction/ConversationTextBuilder.cs b/src/AgentMemory.Core/Extraction/ConversationTextBuilder.cs index 6d43797d5..966cd217f 100644 --- a/src/AgentMemory.Core/Extraction/ConversationTextBuilder.cs +++ b/src/AgentMemory.Core/Extraction/ConversationTextBuilder.cs @@ -50,4 +50,42 @@ public static string BuildNumbered(IReadOnlyList messages) } return builder.ToString(); } + + /// + /// Renders a window as a fenced, unnumbered context block followed by the turns to extract from + /// (E2). Falls back to the plain rendering when the window carries no context. + /// + /// + /// + /// The context block is deliberately not numbered. Per-item provenance (L3c) numbers turns + /// from 1 and resolves turn N positionally to Targets[N-1]. Numbering the context + /// into the same sequence would shift every target index by the context length, and the failure + /// would not look like a crash — each fact would simply be attributed to a turn a few places + /// earlier, which afterwards is indistinguishable from correct attribution. + /// + /// + /// The fence is explicit text rather than a formatting convention because the model has to act on + /// it: everything inside is to be read and nothing inside is to be extracted. + /// + /// + public static string BuildWindow(ExtractionWindow window, bool numbered) + { + var targets = numbered ? BuildNumbered(window.Targets) : Build(window.Targets); + if (!window.HasContext) return targets; + + var builder = new StringBuilder(); + builder.Append(ContextOpen).Append('\n') + .Append(Build(window.Context)).Append('\n') + .Append(ContextClose).Append("\n\n") + .Append(targets); + return builder.ToString(); + } + + /// Opening fence of the read-only context block. + internal const string ContextOpen = + "--- EARLIER CONVERSATION (for reference only — do NOT extract anything from these turns) ---"; + + /// Closing fence of the read-only context block. + internal const string ContextClose = + "--- END OF EARLIER CONVERSATION. Extract ONLY from the turns below. ---"; } diff --git a/src/AgentMemory.Core/Extraction/ExtractionNoveltyGate.cs b/src/AgentMemory.Core/Extraction/ExtractionNoveltyGate.cs new file mode 100644 index 000000000..46e4490b5 --- /dev/null +++ b/src/AgentMemory.Core/Extraction/ExtractionNoveltyGate.cs @@ -0,0 +1,120 @@ +using AgentMemory.Abstractions.Domain; + +namespace AgentMemory.Core.Extraction; + +/// +/// Decides whether a batch of turns is worth spending an extraction call on (E4). +/// +/// +/// +/// This gate sits before the completion, not before the write, and that placement is the whole +/// design. A gate on the write path would skip persisting a triple already in the store — which +/// sounds like the obvious saving and would silently disable two things that depend on the duplicate +/// write actually happening: S2 confidence reinforcement, where a re-asserted fact earns α, and R7's +/// mention_count, which is incremented by the MERGE. Corroboration is the repeated write. +/// Gating it away would leave both features enabled and inert. +/// +/// +/// The completion is also where the money is. The write is one round trip to Neo4j; the extraction is +/// a model call over the rendered transcript, and on a corpus build it is essentially the entire bill. +/// +/// +/// Precision over recall, and the asymmetry is worse here than anywhere else in the system. +/// Declining to gate costs one extraction we might not have needed. Gating a turn that did carry a +/// fact loses that fact until the user happens to say it again — the memory is simply never formed, +/// and nothing downstream can recover it or even report that it is missing. So the gate fires only on +/// turns that cannot carry a proposition at all: greetings, thanks, acknowledgement. Every genuinely +/// ambiguous token is left OUT of the vocabulary on purpose — "yes", "no" and "sure" answer questions +/// and are therefore contentful, however short they look. +/// +/// +/// Deterministic and vocabulary-based rather than model-scored, for the same reason 9.1's resolver is: +/// a gate that cost a completion to decide whether to spend a completion saves nothing, and one that +/// varied between runs would make every measured build unreproducible. +/// +/// +internal static class ExtractionNoveltyGate +{ + /// + /// Tokens that cannot assert anything on their own — greetings, gratitude, acknowledgement, and + /// the filler words that glue them together ("thanks so much", "hi there"). + /// + /// + /// Deliberately absent: yes, no, yep, nope, sure, right, + /// correct. Each is a complete answer to a question, and a batch may hold the answer while + /// the question sat in the previous batch — so the question-mark check below cannot save them. + /// + private static readonly HashSet Uninformative = new(StringComparer.OrdinalIgnoreCase) + { + // acknowledgement + "ok", "okay", "k", "kk", "got", "it", "understood", "noted", "alright", "gotcha", + // gratitude + "thanks", "thank", "you", "thx", "ty", "cheers", "welcome", "youre", "pleasure", "my", + // appraisal with no object + "great", "cool", "nice", "perfect", "awesome", "excellent", "brilliant", "good", "fine", + // greeting and farewell + "hi", "hello", "hey", "there", "bye", "goodbye", "morning", "afternoon", "evening", "night", + // apology and politeness + "sorry", "apologies", "np", "problem", "worries", + // filler that binds the above + "a", "the", "so", "much", "very", "really", "that", "is", "was", "and", "then", "well", + "lol", "haha", "hah", "yay", + }; + + /// + /// when this batch is worth an extraction call. + /// + /// + /// Biased hard toward : anything the vocabulary does not fully explain is + /// treated as potentially contentful, including empty input, which is cheap to extract from and + /// whose handling belongs to the extractors rather than here. + /// + internal static bool IsWorthExtracting(IReadOnlyList messages) + { + if (messages is null || messages.Count == 0) return true; + + foreach (var message in messages) + { + var content = message.Content; + if (string.IsNullOrWhiteSpace(content)) continue; + + // A question mark means someone asked something, and the answer -- possibly a single word + // -- is exactly the kind of content this gate must never discard. + if (content.Contains('?', StringComparison.Ordinal)) return true; + + if (!IsPurelyUninformative(content)) return true; + } + + return false; + } + + private static bool IsPurelyUninformative(string content) + { + var tokens = content.Split( + (char[]?)null, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + + foreach (var token in tokens) + { + var trimmed = token.Trim(Punctuation); + if (trimmed.Length == 0) continue; + + // Any digit makes it contentful: "3" answers "how many?", and a bare number is data. + foreach (var c in trimmed) + { + if (char.IsDigit(c)) return false; + } + + if (!Uninformative.Contains(trimmed.ToLowerInvariant()) + && !Uninformative.Contains( + trimmed.Replace("'", string.Empty, StringComparison.Ordinal).ToLowerInvariant())) + { + return false; + } + } + + return true; + } + + private static readonly char[] Punctuation = + ['.', ',', '!', '?', ';', ':', '-', '—', '–', '"', '\'', '(', ')', '[', ']', '…']; +} diff --git a/src/AgentMemory.Core/Extraction/ExtractionStage.cs b/src/AgentMemory.Core/Extraction/ExtractionStage.cs index a2f5fb5a2..077190cbb 100644 --- a/src/AgentMemory.Core/Extraction/ExtractionStage.cs +++ b/src/AgentMemory.Core/Extraction/ExtractionStage.cs @@ -58,7 +58,15 @@ public Task ExtractAsync( ExtractionTypes typesToExtract, MemoryScope? scope = null, CancellationToken cancellationToken = default) => - ExtractCoreAsync(messages, typesToExtract, scope, preExtracted: null, cancellationToken); + ExtractCoreAsync( + ExtractionWindow.ForTargets(messages), typesToExtract, scope, preExtracted: null, cancellationToken); + + public Task ExtractWithContextAsync( + ExtractionWindow window, + ExtractionTypes typesToExtract, + MemoryScope? scope = null, + CancellationToken cancellationToken = default) => + ExtractCoreAsync(window, typesToExtract, scope, preExtracted: null, cancellationToken); public Task ProcessUnifiedAsync( IReadOnlyList messages, @@ -68,17 +76,39 @@ public Task ProcessUnifiedAsync( CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(extracted); - return ExtractCoreAsync(messages, typesToExtract, scope, extracted, cancellationToken); + return ExtractCoreAsync( + ExtractionWindow.ForTargets(messages), typesToExtract, scope, extracted, cancellationToken); } private async Task ExtractCoreAsync( - IReadOnlyList messages, + ExtractionWindow window, ExtractionTypes typesToExtract, MemoryScope? scope, UnifiedExtractionResult? preExtracted, CancellationToken cancellationToken) { + var messages = window.Targets; + + // Targets only, never the context (E2). A context turn in here would become an EXTRACTED_FROM + // edge asserting the memory was stated in a turn the extractors were told not to extract from + // -- and afterwards that is indistinguishable from correct attribution. var sourceMessageIds = messages.Select(m => m.MessageId).ToList(); + + // E4. Turns that cannot carry a fact -- "ok, thanks!" -- are not worth a completion. Only on + // the paying path: when preExtracted is non-null the caller has already spent the call, and + // gating there would discard a result that has been paid for rather than avoid the cost. + if (_options.SkipUninformativeTurns + && preExtracted is null + && !ExtractionNoveltyGate.IsWorthExtracting(messages)) + { + // Logged rather than silent: a gate that quietly drops turns is indistinguishable from an + // extractor that found nothing, and the saving it claims would be unverifiable. + _logger.LogDebug( + "Skipping extraction for {Count} turn(s) carrying no extractable content (E4).", + messages.Count); + return new ExtractionStageResult { SourceMessageIds = sourceMessageIds }; + } + var strategy = _options.MergeStrategy; var failFast = _options.FailureMode == IngestionFailureMode.FailFast; @@ -105,7 +135,7 @@ private async Task ExtractCoreAsync( else if (unifiedExtractor is not null) { var unifiedRun = await ExtractUnifiedSafeAsync( - unifiedExtractor, messages, typesToExtract, cancellationToken).ConfigureAwait(false); + unifiedExtractor, window, typesToExtract, cancellationToken).ConfigureAwait(false); var unified = unifiedRun.Result; unifiedOutcomes = unifiedRun.Outcomes; entityRun = CompletedRun(typesToExtract.HasFlag(ExtractionTypes.Entities) ? unified.Entities : []); @@ -116,22 +146,30 @@ private async Task ExtractCoreAsync( else { entityRun = typesToExtract.HasFlag(ExtractionTypes.Entities) - ? RunExtractorsAsync(_entityExtractors, e => e.ExtractAsync(messages, cancellationToken), + ? RunExtractorsAsync(_entityExtractors, e => window.HasContext + ? e.ExtractWithContextAsync(window, cancellationToken) + : e.ExtractAsync(messages, cancellationToken), strategy, MergeStrategyFactory.CreateEntityStrategy, "entity", MemoryItemKind.Entity, MemoryErrorCodes.EntityExtractionFailed, cancellationToken) : EmptyRun(); factRun = typesToExtract.HasFlag(ExtractionTypes.Facts) - ? RunExtractorsAsync(_factExtractors, f => f.ExtractAsync(messages, cancellationToken), + ? RunExtractorsAsync(_factExtractors, f => window.HasContext + ? f.ExtractWithContextAsync(window, cancellationToken) + : f.ExtractAsync(messages, cancellationToken), strategy, MergeStrategyFactory.CreateFactStrategy, "fact", MemoryItemKind.Fact, MemoryErrorCodes.FactExtractionFailed, cancellationToken) : EmptyRun(); prefRun = typesToExtract.HasFlag(ExtractionTypes.Preferences) - ? RunExtractorsAsync(_preferenceExtractors, p => p.ExtractAsync(messages, cancellationToken), + ? RunExtractorsAsync(_preferenceExtractors, p => window.HasContext + ? p.ExtractWithContextAsync(window, cancellationToken) + : p.ExtractAsync(messages, cancellationToken), strategy, MergeStrategyFactory.CreatePreferenceStrategy, "preference", MemoryItemKind.Preference, MemoryErrorCodes.PreferenceExtractionFailed, cancellationToken) : EmptyRun(); relRun = typesToExtract.HasFlag(ExtractionTypes.Relationships) - ? RunExtractorsAsync(_relationshipExtractors, r => r.ExtractAsync(messages, cancellationToken), + ? RunExtractorsAsync(_relationshipExtractors, r => window.HasContext + ? r.ExtractWithContextAsync(window, cancellationToken) + : r.ExtractAsync(messages, cancellationToken), strategy, MergeStrategyFactory.CreateRelationshipStrategy, "relationship", MemoryItemKind.Relationship, MemoryErrorCodes.RelationshipExtractionFailed, cancellationToken) : EmptyRun(); @@ -359,13 +397,19 @@ await batchResolver.PrepareCandidatesAsync(candidateTypes, scope, cancellationTo private async Task<(UnifiedExtractionResult Result, IReadOnlyList Outcomes)> ExtractUnifiedSafeAsync( IUnifiedMemoryExtractor extractor, - IReadOnlyList messages, + ExtractionWindow window, ExtractionTypes typesToExtract, CancellationToken cancellationToken) { try { - return (await extractor.ExtractAsync(messages, cancellationToken).ConfigureAwait(false), + // Without context, the ORIGINAL call. Not a shortcut: it keeps the off state identical at + // the call itself rather than only in the rendered transcript, so an extractor that never + // implemented the window overload -- including any third party's -- takes the exact path + // it always did. + return (await (window.HasContext + ? extractor.ExtractWithContextAsync(window, cancellationToken) + : extractor.ExtractAsync(window.Targets, cancellationToken)).ConfigureAwait(false), Array.Empty()); } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) diff --git a/src/AgentMemory.Core/Extraction/ExtractorBase.cs b/src/AgentMemory.Core/Extraction/ExtractorBase.cs index bb6dd8ece..fd747587c 100644 --- a/src/AgentMemory.Core/Extraction/ExtractorBase.cs +++ b/src/AgentMemory.Core/Extraction/ExtractorBase.cs @@ -35,6 +35,18 @@ protected ExtractorBase(ILogger logger) protected abstract Task> ExtractCoreAsync( IReadOnlyList messages, CancellationToken cancellationToken); + /// + /// The window-aware core (E2). Reads to resolve references + /// and extracts only from . + /// + /// + /// Defaults to the context-free core, so an extractor that has not opted in behaves byte-identically + /// to its pre-E2 self rather than silently widening what it extracts. + /// + protected virtual Task> ExtractCoreWithContextAsync( + ExtractionWindow window, CancellationToken cancellationToken) + => ExtractCoreAsync(window.Targets, cancellationToken); + /// /// Extracts items from the supplied messages, returning an empty list when no messages /// are provided or when the underlying extraction fails. @@ -45,13 +57,24 @@ protected abstract Task> ExtractCoreAsync( /// A task that resolves to the extracted items, or an empty list if there are no /// messages or an error occurs during extraction. /// - public async Task> ExtractAsync( + public Task> ExtractAsync( IReadOnlyList messages, CancellationToken cancellationToken = default) + => ExtractWithContextAsync(ExtractionWindow.ForTargets(messages), cancellationToken); + + /// + /// Extracts from the window's targets, reading its context only to understand them (E2), with the + /// same failure containment as the message overload. + /// + public async Task> ExtractWithContextAsync( + ExtractionWindow window, CancellationToken cancellationToken = default) { - if (messages.Count == 0) return Array.Empty(); + // Emptiness is judged on the TARGETS. A window carrying context but nothing to extract from + // has nothing to do -- calling out with only background would spend a completion on turns the + // extractor has been told not to extract from. + if (window.Targets.Count == 0) return Array.Empty(); try { - return await ExtractCoreAsync(messages, cancellationToken).ConfigureAwait(false); + return await ExtractCoreWithContextAsync(window, cancellationToken).ConfigureAwait(false); } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { diff --git a/src/AgentMemory.Core/Extraction/IExtractionStage.cs b/src/AgentMemory.Core/Extraction/IExtractionStage.cs index 171c153f6..166c286d0 100644 --- a/src/AgentMemory.Core/Extraction/IExtractionStage.cs +++ b/src/AgentMemory.Core/Extraction/IExtractionStage.cs @@ -23,6 +23,22 @@ Task ExtractAsync( MemoryScope? scope = null, CancellationToken cancellationToken = default); + /// + /// As , + /// but supplying earlier turns the extractors may read to resolve references (E2). + /// + /// + /// Provenance is attributed to only. A context turn must + /// never appear in SourceMessageIds: an EXTRACTED_FROM edge pointing at one would + /// claim the memory was stated in a turn the extractor was explicitly told not to extract from. + /// + Task ExtractWithContextAsync( + ExtractionWindow window, + ExtractionTypes typesToExtract, + MemoryScope? scope = null, + CancellationToken cancellationToken = default) + => ExtractAsync(window.Targets, typesToExtract, scope, cancellationToken); + /// /// Applies the normal validation, owner-scoped resolution, and filtering stages to a unified /// result that was already extracted by a validated multi-session batch. diff --git a/src/AgentMemory.Core/Memory/DeterministicEntitySummarySynthesizer.cs b/src/AgentMemory.Core/Memory/DeterministicEntitySummarySynthesizer.cs new file mode 100644 index 000000000..3c4781ee4 --- /dev/null +++ b/src/AgentMemory.Core/Memory/DeterministicEntitySummarySynthesizer.cs @@ -0,0 +1,87 @@ +using System.Globalization; +using System.Text; +using AgentMemory.Abstractions.Domain; +using AgentMemory.Abstractions.Services; + +namespace AgentMemory.Core.Memory; + +/// +/// Composes an entity summary from its facts without a model call (S1). +/// +/// +/// +/// The shipped default, and deliberately not an LLM. A summary is invalidated whenever any of its +/// sources moves, so it is regenerated often; a completion per entity per change would make the +/// feature cost scale with how much the conversation talks about its subjects — which is exactly the +/// entities that get summarised most. +/// +/// +/// It is also reproducible. Two runs over the same facts produce the same bytes, so a summary can be +/// compared, fingerprinted and diffed, and a change in the text always means a change in the facts +/// rather than a change in a sampler's mood. +/// +/// +/// The output is a grouped rendering, not prose: predicates gathered with their objects, ordered +/// deterministically, most-confident first within each predicate. Honest about what it is — a +/// compact projection of the graph, which is what a caller needs when the alternative is twenty +/// separate facts each spending context to say one thing. +/// +/// +internal sealed class DeterministicEntitySummarySynthesizer : IEntitySummarySynthesizer +{ + /// Facts below this confidence are left out of the summary. + /// + /// A summary states things flatly, with none of the per-fact confidence a caller would otherwise + /// see. Folding a 0.2-confidence guess into that sentence would present it with the same + /// authority as everything else and remove the only signal that it was doubtful. + /// + public double MinimumConfidence { get; init; } = 0.5; + + /// + public Task SynthesizeAsync( + Entity entity, + IReadOnlyList facts, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(entity); + ArgumentNullException.ThrowIfNull(facts); + + var usable = facts + .Where(f => f.Confidence >= MinimumConfidence) + .Where(f => !string.IsNullOrWhiteSpace(f.Predicate) && !string.IsNullOrWhiteSpace(f.Object)) + .ToList(); + + // Null, not an empty summary. "Nothing is known about this entity above the confidence floor" + // is a real answer, and storing it as an empty string would create a node that satisfies every + // has-a-summary check while saying nothing. + if (usable.Count == 0) return Task.FromResult(null); + + var builder = new StringBuilder(); + builder.Append(entity.Name); + if (!string.IsNullOrWhiteSpace(entity.Type)) + builder.Append(" (").Append(entity.Type).Append(')'); + builder.Append(':'); + + var grouped = usable + .GroupBy(f => f.Predicate, StringComparer.OrdinalIgnoreCase) + .OrderBy(g => g.Key, StringComparer.OrdinalIgnoreCase); + + foreach (var group in grouped) + { + var objects = group + // Most-confident first, then alphabetical: confidence alone leaves ties resolved by + // whatever order the store happened to return, which would make the text -- and so its + // fingerprint -- vary between identical runs. + .OrderByDescending(f => f.Confidence) + .ThenBy(f => f.Object, StringComparer.OrdinalIgnoreCase) + .Select(f => f.Object) + .Distinct(StringComparer.OrdinalIgnoreCase); + + builder.Append(string.Create( + CultureInfo.InvariantCulture, + $"\n- {group.Key}: {string.Join("; ", objects)}")); + } + + return Task.FromResult(builder.ToString()); + } +} diff --git a/src/AgentMemory.Core/Memory/EntitySummaryService.cs b/src/AgentMemory.Core/Memory/EntitySummaryService.cs new file mode 100644 index 000000000..6fe1c53ea --- /dev/null +++ b/src/AgentMemory.Core/Memory/EntitySummaryService.cs @@ -0,0 +1,134 @@ +using AgentMemory.Abstractions.Domain; +using AgentMemory.Abstractions.Options; +using AgentMemory.Abstractions.Repositories; +using AgentMemory.Abstractions.Services; +using Microsoft.Extensions.Logging; + +namespace AgentMemory.Core.Memory; + +/// +/// Writes entity summaries, and refuses to hand back one that can no longer prove it is current (S1). +/// +/// +/// +/// The synthesis is the easy half. The half that decides whether this feature is an asset or a +/// liability is what happens after a source fact is superseded — and the answer here is that +/// the summary stops being used, immediately, without anything having to remember to delete it. +/// +/// +/// Detection rather than invalidation-on-write. A supersession could instead go and mark every +/// affected summary, and that is the design that fails quietly: it has to find them all, it has to run +/// in the same transaction, and any path that writes a fact without going through it leaves a summary +/// that looks current and is not. Recomputing the fingerprint on read cannot be bypassed by a writer +/// that did not know summaries existed. +/// +/// +/// The cost is one extra read of the entity's facts before a summary is used. That is the same read +/// the summary is saving the caller, so a stale summary costs what not having one would have cost — +/// and a fresh summary still saves the context, which is what it was for. +/// +/// +internal sealed class EntitySummaryService : IEntitySummaryService +{ + private readonly IFactRepository _facts; + private readonly IEntitySummaryRepository _summaries; + private readonly IEntitySummarySynthesizer _synthesizer; + private readonly IClock _clock; + private readonly IIdGenerator _ids; + private readonly ILogger _logger; + + /// Public ctor: the type is internal, and DI can only activate it through one. + public EntitySummaryService( + IFactRepository facts, + IEntitySummaryRepository summaries, + IEntitySummarySynthesizer synthesizer, + IClock clock, + IIdGenerator ids, + ILogger logger) + { + _facts = facts; + _summaries = summaries; + _synthesizer = synthesizer; + _clock = clock; + _ids = ids; + _logger = logger; + } + + /// + /// Synthesizes and stores a summary for , replacing any existing one. + /// + /// The stored summary, or when there was nothing to summarise. + public async Task RefreshAsync( + Entity entity, + MemoryScope scope, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(entity); + + var facts = await _facts.GetBySubjectAsync(entity.Name, scope, cancellationToken).ConfigureAwait(false); + var live = facts.Where(f => f.InvalidatedAtUtc is null).ToList(); + + var content = await _synthesizer + .SynthesizeAsync(entity, live, cancellationToken).ConfigureAwait(false); + if (string.IsNullOrWhiteSpace(content)) + { + _logger.LogDebug("Nothing to summarise for entity {EntityId}.", entity.EntityId); + return null; + } + + // Fingerprinted from the SAME list handed to the synthesizer. Reading the facts twice, or + // filtering differently here, would produce a fingerprint describing a set the text was never + // written from -- a summary that could be stale from the moment it was stored. + var summary = new EntitySummary + { + SummaryId = _ids.GenerateId(), + EntityId = entity.EntityId, + Content = content, + SourceFactIds = live.Select(f => f.FactId).ToList(), + SourceFingerprint = EntitySummary.ComputeFingerprint(Sources(live)), + OwnerId = entity.OwnerId, + GeneratedAtUtc = _clock.UtcNow, + }; + + await _summaries.UpsertAsync(summary, cancellationToken).ConfigureAwait(false); + return summary; + } + + /// + /// Returns the entity's summary only if it still describes the current facts. + /// + /// + /// The summary when its fingerprint still matches the store; when there is + /// none or when the one on record has gone stale. + /// + /// + /// Null for stale is deliberate and is the whole safety property. Returning it with an + /// IsStale flag would put the decision in every caller's hands, and the failure of any one + /// of them to check it is indistinguishable from correct memory. + /// + public async Task GetIfCurrentAsync( + Entity entity, + MemoryScope scope, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(entity); + + var stored = await _summaries + .GetByEntityAsync(entity.EntityId, scope, cancellationToken).ConfigureAwait(false); + if (stored is null) return null; + + var facts = await _facts.GetBySubjectAsync(entity.Name, scope, cancellationToken).ConfigureAwait(false); + var live = facts.Where(f => f.InvalidatedAtUtc is null).ToList(); + var current = EntitySummary.ComputeFingerprint(Sources(live)); + + if (string.Equals(current, stored.SourceFingerprint, StringComparison.Ordinal)) return stored; + + _logger.LogDebug( + "Entity summary for {EntityId} is stale ({Stored} != {Current}); not used.", + entity.EntityId, stored.SourceFingerprint, current); + return null; + } + + private static IEnumerable Sources(IEnumerable facts) => + facts.Select(f => new EntitySummarySource(f.FactId, f.Confidence, Invalidated: false)); +} diff --git a/src/AgentMemory.Core/Memory/MemoryBlockRenderer.cs b/src/AgentMemory.Core/Memory/MemoryBlockRenderer.cs new file mode 100644 index 000000000..a3638cf1e --- /dev/null +++ b/src/AgentMemory.Core/Memory/MemoryBlockRenderer.cs @@ -0,0 +1,188 @@ +using System.Globalization; +using System.Text; +using AgentMemory.Abstractions.Domain; + +namespace AgentMemory.Core.Memory; + +/// +/// Renders a memory block from what the graph currently holds (S4). +/// +/// +/// +/// Deterministic and projection-only. Two renders over the same memory produce the same bytes, so a +/// developer diffing two blocks sees what changed in memory rather than what changed in a sampler. +/// +/// +/// There is no inverse. Nothing here parses a block back into memories, and that absence is the +/// design: a round-trippable block invites an agent to edit it and hand it back, at which point the +/// block becomes the real store and the graph's provenance, trust and supersession records describe +/// something else. +/// +/// +internal static class MemoryBlockRenderer +{ + /// + /// Builds a block from the supplied memories, newest and most-confident first within each kind. + /// + /// Entities to include. + /// Facts to include. Superseded facts are excluded. + /// Preferences to include. + /// Render timestamp. + /// Owner the block describes. + /// + /// Hard cap. Anything beyond it is counted into rather + /// than dropped quietly. + /// + public static MemoryBlock Render( + IReadOnlyList entities, + IReadOnlyList facts, + IReadOnlyList preferences, + DateTimeOffset renderedAtUtc, + string? ownerId = null, + int maxLines = 50) + { + ArgumentNullException.ThrowIfNull(entities); + ArgumentNullException.ThrowIfNull(facts); + ArgumentNullException.ThrowIfNull(preferences); + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(maxLines); + + var all = new List(); + + all.AddRange(entities + .OrderByDescending(e => e.Confidence) + .ThenBy(e => e.Name, StringComparer.OrdinalIgnoreCase) + .Select(e => new MemoryBlockLine( + e.EntityId, + MemoryItemKind.Entity, + string.IsNullOrWhiteSpace(e.Type) ? e.Name : $"{e.Name} — {e.Type}"))); + + all.AddRange(facts + // A block is what memory believes NOW. Showing a superseded fact beside a live one, with + // no per-item history in view, presents a retracted claim as current. + .Where(f => f.InvalidatedAtUtc is null) + .OrderByDescending(f => f.Confidence) + .ThenBy(f => f.Subject, StringComparer.OrdinalIgnoreCase) + .ThenBy(f => f.Predicate, StringComparer.OrdinalIgnoreCase) + .Select(f => new MemoryBlockLine( + f.FactId, + MemoryItemKind.Fact, + $"{f.Subject} {f.Predicate} {f.Object}"))); + + all.AddRange(preferences + .OrderByDescending(p => p.Confidence) + .ThenBy(p => p.Category, StringComparer.OrdinalIgnoreCase) + .Select(p => new MemoryBlockLine( + p.PreferenceId, + MemoryItemKind.Preference, + string.IsNullOrWhiteSpace(p.Category) ? p.PreferenceText : $"{p.Category}: {p.PreferenceText}"))); + + return new MemoryBlock + { + OwnerId = ownerId, + RenderedAtUtc = renderedAtUtc, + Lines = all.Take(maxLines).ToList(), + OmittedCount = Math.Max(0, all.Count - maxLines), + }; + } + + /// + /// Builds a block from memory-history rows (S4). + /// + /// + /// + /// The practical entry point. IMemoryHistoryService already answers "what does memory hold + /// for this owner" across all three kinds through an audited read path, so the block is a + /// rendering of an existing query rather than a new way to reach into the store — which is the + /// point of calling it a projection. + /// + /// + /// Invalidated rows are dropped here as well as being excludable at the query. Belt and braces on + /// purpose: a caller that forgets IncludeInvalidated = false would otherwise get a block + /// presenting retracted claims as current, and nothing about it would look wrong. + /// + /// + public static MemoryBlock Render( + IReadOnlyList records, + DateTimeOffset renderedAtUtc, + string? ownerId = null, + int maxLines = 50) + { + ArgumentNullException.ThrowIfNull(records); + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(maxLines); + + var lines = records + .Where(r => r.Status != MemoryHistoryStatus.Invalidated && r.InvalidatedAtUtc is null) + .OrderBy(r => KindOrder(r.Kind)) + .ThenBy(r => r.Summary, StringComparer.OrdinalIgnoreCase) + .Select(r => new MemoryBlockLine(r.Id, ToItemKind(r.Kind), r.Summary)) + .ToList(); + + return new MemoryBlock + { + OwnerId = ownerId, + RenderedAtUtc = renderedAtUtc, + Lines = lines.Take(maxLines).ToList(), + OmittedCount = Math.Max(0, lines.Count - maxLines), + }; + } + + private static int KindOrder(MemoryHistoryKind kind) => kind switch + { + MemoryHistoryKind.Entity => 0, + MemoryHistoryKind.Fact => 1, + _ => 2, + }; + + private static MemoryItemKind ToItemKind(MemoryHistoryKind kind) => kind switch + { + MemoryHistoryKind.Entity => MemoryItemKind.Entity, + MemoryHistoryKind.Fact => MemoryItemKind.Fact, + _ => MemoryItemKind.Preference, + }; + + /// + /// Renders a block as the text a human reads. + /// + /// + /// Ids are shown, not hidden behind the prose. They are what turns "that fact is wrong" into an + /// action against a specific memory, which is the whole reason this stayed a read surface: the + /// correction goes through the audited write path rather than by editing this text. + /// + public static string ToText(MemoryBlock block) + { + ArgumentNullException.ThrowIfNull(block); + + var builder = new StringBuilder(); + builder.Append("# Memory"); + if (!string.IsNullOrWhiteSpace(block.OwnerId)) + builder.Append(" — ").Append(block.OwnerId); + builder.Append(string.Create( + CultureInfo.InvariantCulture, + $"\n_rendered {block.RenderedAtUtc:u} — a snapshot, not a document to edit_\n")); + + foreach (var kind in new[] { MemoryItemKind.Entity, MemoryItemKind.Fact, MemoryItemKind.Preference }) + { + var lines = block.Lines.Where(l => l.Kind == kind).ToList(); + if (lines.Count == 0) continue; + + builder.Append('\n').Append("## ").Append(kind).Append("\n"); + foreach (var line in lines) + builder.Append(string.Create(CultureInfo.InvariantCulture, $"- {line.Text} `{line.MemoryId}`\n")); + } + + if (block.IsTruncated) + { + // Stated in the text itself, not only on the object. A reader who sees a block ending + // cleanly assumes they have seen everything, and the whole value of this view is that it + // can be trusted to show what memory holds. + builder.Append(string.Create( + CultureInfo.InvariantCulture, + $"\n_… {block.OmittedCount} more not shown (block limit reached)_\n")); + } + + if (block.Lines.Count == 0) + builder.Append("\n_Memory holds nothing for this owner yet._\n"); + + return builder.ToString(); + } +} diff --git a/src/AgentMemory.Core/Memory/TemporalQueryParser.cs b/src/AgentMemory.Core/Memory/TemporalQueryParser.cs new file mode 100644 index 000000000..f05afde21 --- /dev/null +++ b/src/AgentMemory.Core/Memory/TemporalQueryParser.cs @@ -0,0 +1,161 @@ +using System.Globalization; +using System.Text.RegularExpressions; + +namespace AgentMemory.Core.Memory; + +/// +/// Finds the point in time a question is asking about, so bitemporal recall can be reached from a +/// natural-language turn (R4). +/// +/// +/// +/// RecallAsOfAsync(validAsOf, systemAsOf) already exists and nothing in an ordinary conversation +/// can reach it: "what did I think back in March?" recalls against now, exactly like every other +/// question. Without this the bitemporal read path is a capability nothing can ask for. +/// +/// +/// Precision over recall, deliberately and asymmetrically. A missed expression costs nothing — +/// the turn recalls against now, which is today's behaviour. A false positive silently filters memory +/// to a window the user never asked about, and the answer that comes back looks perfectly ordinary. +/// Every pattern here therefore requires an explicit temporal marker rather than inferring one: +/// "in March" is temporal, a bare "March" may be a surname; "last week" is +/// temporal, "the last item" is not. +/// +/// +/// No model call and no culture-dependent parsing: this runs on every turn when enabled, and a +/// resolver that cost a completion would be more expensive than the recall it modifies. +/// +/// +internal static class TemporalQueryParser +{ + private const RegexOptions Options = + RegexOptions.IgnoreCase | RegexOptions.CultureInvariant | RegexOptions.ExplicitCapture; + + private static readonly TimeSpan Timeout = TimeSpan.FromMilliseconds(100); + + /// + /// last|past N unit(s) ago and the bare last week/month/quarter/year. + /// + /// + /// "last" must be followed by a time unit. Alone it is far more often ordinal — "the last item", + /// "last one wins" — and treating those as temporal would rewrite the recall window of ordinary + /// questions. + /// + private static readonly Regex RelativeUnits = new( + @"\b(?:(?\d{1,3})\s+)?(?day|days|week|weeks|month|months|quarter|quarters|year|years)\s+ago\b" + + @"|\blast\s+(?week|month|quarter|year)\b" + + @"|\bpast\s+(?\d{1,3})\s+(?day|days|week|weeks|month|months|year|years)\b", + Options, Timeout); + + /// A month name behind a preposition, optionally with a year. + /// + /// The preposition is required. "March" is a common surname and "may" is a verb in nearly every + /// sentence that contains it; "in May" is unambiguous where "may" is not. + /// + private static readonly Regex MonthName = new( + @"\b(?:in|back\s+in|during|around)\s+(?january|february|march|april|may|june|july|august|" + + @"september|october|november|december)(?:\s+(?(?:19|20)\d{2}))?\b", + Options, Timeout); + + /// A four-digit year behind a preposition. + /// + /// Bounded to 19xx/20xx and requiring a preposition, so "in 2024" resolves and "in 2024 units of + /// stock" — where the number is a quantity — does not accidentally become a date. + /// + private static readonly Regex Year = new( + @"\b(?:in|back\s+in|during)\s+(?(?:19|20)\d{2})\b(?!\s*(?:units?|items?|records?|rows?))", + Options, Timeout); + + private static readonly Regex Yesterday = new(@"\byesterday\b", Options, Timeout); + + /// + /// The instant this question is asking about, or when it asks about now. + /// + /// The user's turn. + /// The reference instant; relative expressions resolve against it. + /// + /// + /// Null is the overwhelmingly common answer and the safe one: it means "recall as the system + /// behaves today". Only an explicit past reference produces an instant. + /// + /// + /// A future reference returns null rather than a future instant. As-of recall reconstructs + /// what was known at a past moment; asking it about next month would return everything, which is + /// indistinguishable from ordinary recall except that it silently ignored the user's question. + /// + /// + internal static DateTimeOffset? Resolve(string? query, DateTimeOffset now) + { + if (string.IsNullOrWhiteSpace(query)) return null; + + var resolved = ResolveCore(query, now); + + // Never ahead of the reference instant. A parse that lands in the future is either a + // misreading or a question about the future, and neither should narrow a recall window. + return resolved is { } instant && instant < now ? instant : null; + } + + private static DateTimeOffset? ResolveCore(string query, DateTimeOffset now) + { + if (Yesterday.IsMatch(query)) return now.AddDays(-1); + + var relative = RelativeUnits.Match(query); + if (relative.Success) + { + var count = relative.Groups["n"].Success + ? int.Parse(relative.Groups["n"].Value, CultureInfo.InvariantCulture) + : relative.Groups["n2"].Success + ? int.Parse(relative.Groups["n2"].Value, CultureInfo.InvariantCulture) + : 1; + + var unit = relative.Groups["unit"].Success ? relative.Groups["unit"].Value + : relative.Groups["lunit"].Success ? relative.Groups["lunit"].Value + : relative.Groups["punit"].Value; + + return unit.ToLowerInvariant().TrimEnd('s') switch + { + "day" => now.AddDays(-count), + "week" => now.AddDays(-7 * count), + "month" => now.AddMonths(-count), + "quarter" => now.AddMonths(-3 * count), + "year" => now.AddYears(-count), + _ => null, + }; + } + + var month = MonthName.Match(query); + if (month.Success) + { + var monthNumber = MonthNumber(month.Groups["month"].Value); + if (monthNumber == 0) return null; + + var year = month.Groups["year"].Success + ? int.Parse(month.Groups["year"].Value, CultureInfo.InvariantCulture) + : now.Month >= monthNumber ? now.Year : now.Year - 1; + + // The END of the named month: "what did I think back in March" means anything known by + // the close of March, not by its first instant -- which would exclude the whole month. + return EndOfMonth(year, monthNumber, now.Offset); + } + + var year4 = Year.Match(query); + if (year4.Success) + { + var value = int.Parse(year4.Groups["year"].Value, CultureInfo.InvariantCulture); + return EndOfMonth(value, 12, now.Offset); + } + + return null; + } + + private static DateTimeOffset EndOfMonth(int year, int month, TimeSpan offset) => + new DateTimeOffset(year, month, DateTime.DaysInMonth(year, month), 23, 59, 59, offset); + + private static int MonthNumber(string name) => name.ToLowerInvariant() switch + { + "january" => 1, "february" => 2, "march" => 3, "april" => 4, + "may" => 5, "june" => 6, "july" => 7, "august" => 8, + "september" => 9, "october" => 10, "november" => 11, "december" => 12, + _ => 0, + }; +} diff --git a/src/AgentMemory.Core/ServiceCollectionExtensions.cs b/src/AgentMemory.Core/ServiceCollectionExtensions.cs index 62f51cbe0..0d38d0f7f 100644 --- a/src/AgentMemory.Core/ServiceCollectionExtensions.cs +++ b/src/AgentMemory.Core/ServiceCollectionExtensions.cs @@ -5,6 +5,7 @@ using AgentMemory.Abstractions.Options; using AgentMemory.Abstractions.Services; using AgentMemory.Core.Extraction; +using AgentMemory.Core.Memory; using AgentMemory.Core.Resolution; using AgentMemory.Core.Services; using AgentMemory.Core.Services.Budgeting; @@ -228,6 +229,13 @@ public static IServiceCollection AddAgentMemoryCore( // Callers may override by registering their own IEntityResolver before calling this method. services.TryAddScoped(); + // S1. The deterministic synthesizer is the default because a summary is regenerated every + // time any of its sources moves; a completion per entity per change would scale cost with + // exactly how much the conversation talks about its subjects. TryAdd, so a host wanting + // prose registers an LLM synthesizer before calling this and keeps it. + services.TryAddScoped(); + services.TryAddScoped(); + // Keep StubEntityResolver available for explicit fallback use. services.TryAddScoped(); diff --git a/src/AgentMemory.Core/Services/MemoryContextAssembler.cs b/src/AgentMemory.Core/Services/MemoryContextAssembler.cs index 5649003e8..b8bd22433 100644 --- a/src/AgentMemory.Core/Services/MemoryContextAssembler.cs +++ b/src/AgentMemory.Core/Services/MemoryContextAssembler.cs @@ -113,7 +113,8 @@ private static MemoryContextSectionDiagnostics Diagnose( int requestedLimit, IReadOnlyList items, IReadOnlyList ranked, - double minimumScore) + double minimumScore, + bool timedOut = false) { // Scores come from the ranked items, which exist only when the provider could supply them. // An unscoreable section reports null rather than a placeholder: zero is a real score. @@ -125,7 +126,7 @@ private static MemoryContextSectionDiagnostics Diagnose( } return new MemoryContextSectionDiagnostics( - searched, requestedLimit, items.Count, top, low, minimumScore); + searched, requestedLimit, items.Count, top, low, minimumScore, timedOut); } /// @@ -221,6 +222,13 @@ public async Task AssembleContextAsync( IReadOnlyList<(Fact Fact, double Score)> factScores = Array.Empty<(Fact, double)>(); IReadOnlyList<(ReasoningTrace Trace, double Score)> traceScores = Array.Empty<(ReasoningTrace, double)>(); + // Rank 13 bookkeeping, at METHOD scope: the fan-out lives in a nested block and these are read + // by the diagnostics built after it closes. A section that searched and found nothing and one + // that never came back produce an identical section; only one means the answer is incomplete. + var budgetExceeded = false; + bool droppedRecent = false, droppedRelevant = false, droppedEntities = false, + droppedPreferences = false, droppedFacts = false, droppedTraces = false; + // J2.2 resolution, computed once at method scope so it can be both used by the fact search // and reported on the context. Which relations a question resolved to is the difference // between "expansion had nothing to expand" and "expansion ran and did not help", and those @@ -399,12 +407,42 @@ public async Task AssembleContextAsync( // One handle per section, so a fault in any of them is still observed here rather than // surfacing later as an unobserved task exception. - await Task.WhenAll( + Task[] sections = + [ recentTask, relevantTask, entitiesScoredTask ?? (Task)entitiesTask, preferencesScoredTask ?? (Task)preferencesTask, factsScoredTask ?? (Task)factsTask, - tracesScoredTask ?? (Task)tracesTask).ConfigureAwait(false); + tracesScoredTask ?? (Task)tracesTask, + ]; + + // Rank 13. With no budget this is the original unconditional wait, byte for byte. + if (recallOpts.LatencyBudget is { } latencyBudget + && await WaitWithinBudgetAsync(sections, latencyBudget, cancellationToken).ConfigureAwait(false)) + { + // Each unfinished section is replaced by an empty result AND recorded, because the two + // have to be told apart downstream: "searched and found nothing" and "never came back" + // produce an identical section, and only one of them means the answer is incomplete. + budgetExceeded = true; + if (DropIfUnfinished>(ref recentTask, [])) droppedRecent = true; + if (DropIfUnfinished(ref relevantTask, RelevantMessageSearchResult.Empty)) droppedRelevant = true; + if (entitiesScoredTask is not null) + droppedEntities = DropIfUnfinished>(ref entitiesScoredTask, []); + else droppedEntities = DropIfUnfinished>(ref entitiesTask, []); + if (preferencesScoredTask is not null) + droppedPreferences = DropIfUnfinished>(ref preferencesScoredTask, []); + else droppedPreferences = DropIfUnfinished>(ref preferencesTask, []); + if (factsScoredTask is not null) + droppedFacts = DropIfUnfinished(ref factsScoredTask, new ScoredFactSearchResult([], [])); + else droppedFacts = DropIfUnfinished>(ref factsTask, []); + if (tracesScoredTask is not null) + droppedTraces = DropIfUnfinished>(ref tracesScoredTask, []); + else droppedTraces = DropIfUnfinished>(ref tracesTask, []); + + _logger.LogWarning( + "Recall latency budget of {Budget}ms expired; returning partial context.", + latencyBudget.TotalMilliseconds); + } recentMessages = await recentTask.ConfigureAwait(false); var relevantResult = await relevantTask.ConfigureAwait(false); @@ -523,7 +561,7 @@ await Task.WhenAll( // No vector involved: session-scoped and time-ordered, so the limit alone decides. Diagnostics = recallOpts.IncludeDiagnostics ? Diagnose(recallOpts.MaxRecentMessages > 0, recallOpts.MaxRecentMessages, - recentMessages, Array.Empty(), minScore) + recentMessages, Array.Empty(), minScore, droppedRecent) : null }, RelevantMessages = new MemoryContextSection @@ -532,7 +570,7 @@ await Task.WhenAll( RankedItems = relevantRanked, Diagnostics = recallOpts.IncludeDiagnostics ? Diagnose(vectorSearchRan && recallOpts.MaxRelevantMessages > 0, - recallOpts.MaxRelevantMessages, relevantMessages, relevantRanked, minScore) + recallOpts.MaxRelevantMessages, relevantMessages, relevantRanked, minScore, droppedRelevant) : null }, RelevantEntities = new MemoryContextSection @@ -541,7 +579,7 @@ await Task.WhenAll( RankedItems = entityRanked, Diagnostics = recallOpts.IncludeDiagnostics ? Diagnose(vectorSearchRan && recallOpts.MaxEntities > 0, - recallOpts.MaxEntities, entities, entityRanked, minScore) + recallOpts.MaxEntities, entities, entityRanked, minScore, droppedEntities) : null }, RelevantPreferences = new MemoryContextSection @@ -550,7 +588,7 @@ await Task.WhenAll( RankedItems = preferenceRanked, Diagnostics = recallOpts.IncludeDiagnostics ? Diagnose(vectorSearchRan && recallOpts.MaxPreferences > 0, - recallOpts.MaxPreferences, preferences, preferenceRanked, minScore) + recallOpts.MaxPreferences, preferences, preferenceRanked, minScore, droppedPreferences) : null }, RelevantFacts = new MemoryContextSection @@ -559,7 +597,7 @@ await Task.WhenAll( RankedItems = factRanked, Diagnostics = recallOpts.IncludeDiagnostics ? Diagnose(vectorSearchRan && recallOpts.MaxFacts > 0, - recallOpts.MaxFacts, facts, factRanked, minScore) + recallOpts.MaxFacts, facts, factRanked, minScore, droppedFacts) : null }, SimilarTraces = new MemoryContextSection @@ -568,14 +606,15 @@ await Task.WhenAll( RankedItems = traceRanked, Diagnostics = recallOpts.IncludeDiagnostics ? Diagnose(vectorSearchRan && recallOpts.MaxTraces > 0, - recallOpts.MaxTraces, traces, traceRanked, minScore) + recallOpts.MaxTraces, traces, traceRanked, minScore, droppedTraces) : null }, GraphRagContext = graphRagContext, GraphRagItems = graphRagItems, ResolvedQueryRelations = resolvedQueryRelations, BlendMode = blendMode, - Truncated = truncated + Truncated = truncated, + LatencyBudgetExceeded = budgetExceeded }; _logger.LogDebug( @@ -1040,4 +1079,54 @@ private ITruncationStrategy ResolveStrategy(TruncationStrategy strategy) => _truncationStrategies.TryGetValue(strategy, out var resolved) ? resolved : _truncationStrategies[TruncationStrategy.OldestFirst]; + + /// + /// Waits for the section tasks, giving up after (rank 13). + /// + /// when the budget expired with work still running. + /// + /// + /// The still-running queries are not cancelled. They are already in the driver, and tearing + /// them down buys nothing the caller is waiting for — it only turns a wasted result into a wasted + /// result plus a cancellation storm. Their faults are observed so a slow section that also fails + /// does not surface later as an unobserved task exception. + /// + /// + private static async Task WaitWithinBudgetAsync( + Task[] sections, TimeSpan budget, CancellationToken cancellationToken) + { + var all = Task.WhenAll(sections); + using var expiry = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + var timer = Task.Delay(budget, expiry.Token); + + var finished = await Task.WhenAny(all, timer).ConfigureAwait(false); + if (finished == all) + { + await expiry.CancelAsync().ConfigureAwait(false); + await all.ConfigureAwait(false); // rethrow section faults exactly as before + return false; + } + + foreach (var section in sections) + { + _ = section.ContinueWith( + faulted => _ = faulted.Exception, + CancellationToken.None, + TaskContinuationOptions.OnlyOnFaulted, + TaskScheduler.Default); + } + + return true; + } + + /// + /// Replaces a section that did not finish in time with an empty result (rank 13). + /// + /// when a substitution happened, i.e. this section was dropped. + private static bool DropIfUnfinished(ref Task section, T empty) + { + if (section.IsCompletedSuccessfully) return false; + section = Task.FromResult(empty); + return true; + } } diff --git a/src/AgentMemory.Core/Services/MemoryExtractionPipeline.cs b/src/AgentMemory.Core/Services/MemoryExtractionPipeline.cs index 6fab1abe2..2ea9b28c0 100644 --- a/src/AgentMemory.Core/Services/MemoryExtractionPipeline.cs +++ b/src/AgentMemory.Core/Services/MemoryExtractionPipeline.cs @@ -58,8 +58,17 @@ public async Task ExtractAsync( var scope = _isolationPolicy.ResolveReadScope( explicitScope: null, request.UserId, nameof(ExtractAsync), MemoryOperationAccess.Tenant); - var staged = await _extractionStage.ExtractAsync( - request.Messages, request.TypesToExtract, scope, cancellationToken).ConfigureAwait(false); + // E2. Context reaches the extractors as context; the window keeps it out of provenance and + // out of the extraction targets, which is what stops it inflating S2 confidence and R7 + // mention counts on facts that merely stayed inside the window. + // Without context, the original call — the off state stays identical at the call itself, not + // merely in what the extractors end up seeing. + var staged = request.ContextMessages.Count == 0 + ? await _extractionStage.ExtractAsync( + request.Messages, request.TypesToExtract, scope, cancellationToken).ConfigureAwait(false) + : await _extractionStage.ExtractWithContextAsync( + new ExtractionWindow { Targets = request.Messages, Context = request.ContextMessages }, + request.TypesToExtract, scope, cancellationToken).ConfigureAwait(false); var ownerId = _isolationPolicy.ResolveWriteOwner(request.UserId, nameof(ExtractAsync), MemoryOperationAccess.Tenant); // #92 Phase 3: a per-request TrustLevel override wins; otherwise fall back to the configured default. diff --git a/src/AgentMemory.Core/Services/MemoryService.cs b/src/AgentMemory.Core/Services/MemoryService.cs index 5e6ce04b1..8c0184c4d 100644 --- a/src/AgentMemory.Core/Services/MemoryService.cs +++ b/src/AgentMemory.Core/Services/MemoryService.cs @@ -5,6 +5,7 @@ using AgentMemory.Abstractions.Options; using AgentMemory.Abstractions.Repositories; using AgentMemory.Abstractions.Services; +using AgentMemory.Core.Memory; namespace AgentMemory.Core.Services; @@ -86,13 +87,55 @@ public async Task RecallAsync( // outside; both can be present without colliding. using var activity = AgentMemoryDiagnostics.Source.StartActivity("memory.recall.total"); + // R4. A turn that names a past time is asking a bitemporal question, and until now no + // conversational turn could reach RecallAsOfAsync at all. Resolution is deterministic and + // biased hard toward returning null -- see TemporalQueryParser -- so the ordinary turn takes + // exactly the path it always did. + if (_options.ResolveTemporalQueries + && TemporalQueryParser.Resolve(request.Query, _clock.UtcNow) is { } asOf) + { + activity?.SetTag("memory.recall.resolved_as_of", asOf.ToString("O")); + _logger.LogDebug( + "Query names a past time ({AsOf}); recalling bitemporally instead of against now.", asOf); + // Both clocks: the question is "what did I think then", which is what was true then AS + // known then. Passing only the valid clock would answer with today's corrections applied + // to the past -- a different question, and a subtly misleading one. + return await RecallAsOfCoreAsync(request, asOf, asOf, cancellationToken).ConfigureAwait(false); + } + var context = await _assembler.AssembleContextAsync(request, cancellationToken).ConfigureAwait(false); - // Update access timestamps for recalled long-term memories (awaited so failures and - // cancellation are observed; the method itself is resilient and logs internally). + // Update access timestamps for recalled long-term memories. Awaited by default so failures and + // cancellation are observed; the method itself is resilient and logs internally. if (_decayService is not null) { - await UpdateAccessTimestampsAsync(context, cancellationToken).ConfigureAwait(false); + if (_options.DeferAccessTracking) + { + // 2.4. Bookkeeping the caller is not waiting for: it feeds decay and retention, and + // nothing in the returned context depends on it. + // + // CancellationToken.None, NOT the request's token. That token is cancelled as soon as + // the response completes, so passing it through would cancel the very write being + // deferred -- an option that reads as enabled and does nothing. + // + // The task is deliberately not awaited and deliberately not discarded silently: the + // continuation is what keeps a deferred failure from being an unobserved exception. + _ = UpdateAccessTimestampsAsync(context, CancellationToken.None) + .ContinueWith( + task => _logger.LogWarning( + task.Exception, + "Deferred access tracking failed after recall returned. If this is an " + + "ObjectDisposedException, the host's DI scope was disposed before the " + + "write completed -- MemoryOptions.DeferAccessTracking is unsafe for a " + + "request-scoped host."), + CancellationToken.None, + TaskContinuationOptions.OnlyOnFaulted, + TaskScheduler.Default); + } + else + { + await UpdateAccessTimestampsAsync(context, cancellationToken).ConfigureAwait(false); + } } int totalItems = context.RecentMessages.Items.Count @@ -255,13 +298,66 @@ public Task> AddMessagesAsync( } /// - public Task ExtractAndPersistAsync( + public async Task ExtractAndPersistAsync( ExtractionRequest request, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(request); _logger.LogDebug("Extracting and persisting memory for session {SessionId}", request.SessionId); - return _extraction.ExtractAsync(request, cancellationToken); + request = await WithExtractionContextAsync(request, cancellationToken).ConfigureAwait(false); + return await _extraction.ExtractAsync(request, cancellationToken).ConfigureAwait(false); + } + + /// + /// Attaches the preceding turns an extractor may read to resolve references (E2). + /// + /// + /// + /// Filled in here rather than at each call site because this is the one chokepoint every + /// incremental extraction passes through — the Agent Framework provider, the Microsoft memory + /// facade and the MCP ingest tool all arrive at this method. Resolving it per caller would make + /// the window depend on which host was used, which is the failure this codebase has hit + /// repeatedly: a setting only some components respect is worse than no setting. + /// + /// + /// An explicitly supplied context always wins — a caller that knows the conversation better than + /// a recency query does should not be second-guessed. + /// + /// + private async Task WithExtractionContextAsync( + ExtractionRequest request, + CancellationToken cancellationToken) + { + if (_options.Extraction.ExtractionContextTurns <= 0 + || request.ContextMessages.Count > 0 + || request.Messages.Count == 0) + { + return request; + } + + var targetIds = request.Messages.Select(m => m.MessageId).ToHashSet(StringComparer.Ordinal); + + // Over-fetch by the batch size: the most recent stored messages usually ARE the batch, so + // asking for exactly N would return N turns of which most are excluded and leave the window + // short without reporting it. + var wanted = _options.Extraction.ExtractionContextTurns; + var recent = await _shortTerm.GetRecentMessagesAsync( + request.SessionId, wanted + request.Messages.Count, cancellationToken).ConfigureAwait(false); + + // The batch itself is never context. Its turns are the targets, and a message appearing in + // both would be handed to the model twice -- once as "do not extract from this". + var context = recent + .Where(m => !targetIds.Contains(m.MessageId)) + .TakeLast(wanted) + .ToList(); + + if (context.Count == 0) return request; + + _logger.LogDebug( + "Extraction window: {Targets} target turn(s) with {Context} turn(s) of read-only context (E2).", + request.Messages.Count, context.Count); + + return request with { ContextMessages = context }; } /// diff --git a/src/AgentMemory.Core/Services/ReasoningMemoryService.cs b/src/AgentMemory.Core/Services/ReasoningMemoryService.cs index c5ac8229b..70f2a3209 100644 --- a/src/AgentMemory.Core/Services/ReasoningMemoryService.cs +++ b/src/AgentMemory.Core/Services/ReasoningMemoryService.cs @@ -84,7 +84,13 @@ public async Task StartTraceAsync( // #92 Phase 3: trust_level is a framework-reserved metadata key -- a caller of this public // service method must never be able to self-assign a trust level that bypasses the admission // policy's instruction-like-content detection for this trace's Task text on recall. - Metadata = (metadata ?? new Dictionary()).WithoutCallerSuppliedTrustLevel() + // Stripped first, then stamped from CONFIGURATION -- never from the caller. The strip is + // the security property (#92 Phase 3); the stamp closes 6.3, where every trace read back + // as Untrusted because nothing ever assigned one, making "the agent generated this" and + // "no signal was recorded" the same value. + Metadata = (metadata ?? new Dictionary()) + .WithoutCallerSuppliedTrustLevel() + .WithTrustLevel(_options.DefaultTraceTrustLevel) }; _logger.LogDebug("Starting trace {TraceId} for session {SessionId}", trace.TraceId, sessionId); diff --git a/src/AgentMemory.Extraction.Llm/ExtractionPromptSemantics.cs b/src/AgentMemory.Extraction.Llm/ExtractionPromptSemantics.cs index 977f498c9..39c503305 100644 --- a/src/AgentMemory.Extraction.Llm/ExtractionPromptSemantics.cs +++ b/src/AgentMemory.Extraction.Llm/ExtractionPromptSemantics.cs @@ -134,4 +134,25 @@ internal static class ExtractionPromptSemantics _ => string.Empty, }; + + /// + /// Tells the model that the fenced earlier turns are background and not extraction targets (E2). + /// + /// + /// + /// Appended only when a window actually carries context, so a prompt without context is + /// byte-for-byte what it was before E2 existed. Prompt bytes are fingerprinted into every + /// measured run in this track; an instruction that appeared unconditionally would invalidate every + /// sealed base to say something about turns that were not supplied. + /// + /// + /// Authored here rather than per extractor for the usual reason: an instruction only some + /// extractors carry makes what a memory is depend on which performance flag is set. + /// + /// + internal const string ExtractionContextInstruction = + "\nThe transcript begins with earlier turns marked as EARLIER CONVERSATION. Read them only to " + + "resolve references — who \"she\" is, where \"there\" is, what \"it\" refers to. Do NOT extract " + + "any memory whose statement lives in those turns; extract only from the turns after the end " + + "marker, even if an earlier turn states something you would otherwise record."; } diff --git a/src/AgentMemory.Extraction.Llm/LlmEntityExtractor.cs b/src/AgentMemory.Extraction.Llm/LlmEntityExtractor.cs index dbed253ca..9c62e397a 100644 --- a/src/AgentMemory.Extraction.Llm/LlmEntityExtractor.cs +++ b/src/AgentMemory.Extraction.Llm/LlmEntityExtractor.cs @@ -48,12 +48,18 @@ public LlmEntityExtractor( _runner = new LlmExtractionRunner(chatClient, _options, logger); } - protected override async Task> ExtractCoreAsync( + protected override Task> ExtractCoreAsync( IReadOnlyList messages, CancellationToken cancellationToken) + => ExtractCoreWithContextAsync(ExtractionWindow.ForTargets(messages), cancellationToken); + + protected override async Task> ExtractCoreWithContextAsync( + ExtractionWindow window, CancellationToken cancellationToken) { - var conversationText = ConversationTextBuilder.Build(messages); + var conversationText = ConversationTextBuilder.BuildWindow(window, numbered: false); return await _runner.RunAsync( - _options.EntityExtractionPrompt ?? BuildSystemPrompt(_options.EntityTypes), + (_options.EntityExtractionPrompt ?? BuildSystemPrompt(_options.EntityTypes)) + // Only when context is present, so a context-free prompt stays byte-identical (E2). + + (window.HasContext ? ExtractionPromptSemantics.ExtractionContextInstruction : string.Empty), "Extract entities from this conversation:", conversationText, ProjectEntities, diff --git a/src/AgentMemory.Extraction.Llm/LlmExtractionBatchDiagnostics.cs b/src/AgentMemory.Extraction.Llm/LlmExtractionBatchDiagnostics.cs index 5ce9235ee..f3d5278f7 100644 --- a/src/AgentMemory.Extraction.Llm/LlmExtractionBatchDiagnostics.cs +++ b/src/AgentMemory.Extraction.Llm/LlmExtractionBatchDiagnostics.cs @@ -8,6 +8,9 @@ internal sealed class LlmExtractionBatchDiagnostics private readonly ConcurrentQueue _details = new(); private long _splits; private long _droppedDetails; + private long _contentRejections; + private long _sessionsRefused; + private readonly ConcurrentQueue _refusedSessionIds = new(); internal void RecordSplit(Exception exception, int sourceSessions) { @@ -22,11 +25,42 @@ internal void RecordSplit(Exception exception, int sourceSessions) Interlocked.Increment(ref _droppedDetails); } + /// + /// Records a batch the provider refused on content grounds, and the sessions lost with it. + /// + /// + /// Counted separately from splits because it is a different event with a different remedy: a + /// split is a recoverable shape problem, a refusal is terminal for that text. The session count + /// is what a reader needs — "2 refusals" says nothing about whether the corpus is usable, "2 + /// refusals costing 6 sessions of 2,418" does. + /// + internal void RecordContentRejection( + Exception exception, int sourceSessions, string? sessionId = null) + { + ArgumentNullException.ThrowIfNull(exception); + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(sourceSessions); + Interlocked.Increment(ref _contentRejections); + Interlocked.Add(ref _sessionsRefused, sourceSessions); + // The identifier, never the text. A refusal that leaves no trace of what caused it cannot be + // investigated or reported to the provider -- and this recurs -- but the diagnostics are + // deliberately free of dataset content, so the id is the handle and the dataset is the lookup. + if (!string.IsNullOrEmpty(sessionId)) _refusedSessionIds.Enqueue(sessionId); + _details.Enqueue(new LlmExtractionBatchSplitDetail( + "content-rejected", + sourceSessions, + exception.GetType().FullName ?? exception.GetType().Name)); + while (_details.Count > MaximumDetails && _details.TryDequeue(out _)) + Interlocked.Increment(ref _droppedDetails); + } + internal LlmExtractionBatchDiagnosticsSnapshot Snapshot() => new( Interlocked.Read(ref _splits), _details.ToArray(), - Interlocked.Read(ref _droppedDetails)); + Interlocked.Read(ref _droppedDetails), + Interlocked.Read(ref _contentRejections), + Interlocked.Read(ref _sessionsRefused), + _refusedSessionIds.ToArray()); private static string Classify(Exception exception) => exception.Message switch @@ -46,14 +80,20 @@ private static string Classify(Exception exception) => internal sealed record LlmExtractionBatchDiagnosticsSnapshot( long Splits, IReadOnlyList Details, - long DroppedDetails) + long DroppedDetails, + long ContentRejections = 0, + long SessionsRefused = 0, + IReadOnlyList? RefusedSessionIds = null) { internal LlmExtractionBatchDiagnosticsSnapshot Delta( LlmExtractionBatchDiagnosticsSnapshot baseline) => new( Splits - baseline.Splits, Details.Skip(Math.Min(Details.Count, baseline.Details.Count)).ToArray(), - DroppedDetails - baseline.DroppedDetails); + DroppedDetails - baseline.DroppedDetails, + ContentRejections - baseline.ContentRejections, + SessionsRefused - baseline.SessionsRefused, + (RefusedSessionIds ?? []).Skip((baseline.RefusedSessionIds ?? []).Count).ToArray()); } internal sealed record LlmExtractionBatchSplitDetail( diff --git a/src/AgentMemory.Extraction.Llm/LlmFactExtractor.cs b/src/AgentMemory.Extraction.Llm/LlmFactExtractor.cs index 7376919d7..46102e9bd 100644 --- a/src/AgentMemory.Extraction.Llm/LlmFactExtractor.cs +++ b/src/AgentMemory.Extraction.Llm/LlmFactExtractor.cs @@ -46,16 +46,21 @@ public LlmFactExtractor( _runner = new LlmExtractionRunner(chatClient, _options, logger); } - protected override async Task> ExtractCoreAsync( + protected override Task> ExtractCoreAsync( IReadOnlyList messages, CancellationToken cancellationToken) + => ExtractCoreWithContextAsync(ExtractionWindow.ForTargets(messages), cancellationToken); + + protected override async Task> ExtractCoreWithContextAsync( + ExtractionWindow window, CancellationToken cancellationToken) { - var conversationText = _options.Provenance == ExtractionProvenanceMode.PerItem - ? ConversationTextBuilder.BuildNumbered(messages) - : ConversationTextBuilder.Build(messages); + var conversationText = ConversationTextBuilder.BuildWindow( + window, numbered: _options.Provenance == ExtractionProvenanceMode.PerItem); return await _runner.RunAsync( - _options.FactExtractionPrompt + (_options.FactExtractionPrompt ?? BuildSystemPrompt( - _options.AssistantContent, _options.TemporalValidity, _options.Provenance), + _options.AssistantContent, _options.TemporalValidity, _options.Provenance)) + // Only when context is present: a context-free prompt must stay byte-identical (E2). + + (window.HasContext ? ExtractionPromptSemantics.ExtractionContextInstruction : string.Empty), "Extract facts from this conversation:", conversationText, ProjectFacts, diff --git a/src/AgentMemory.Extraction.Llm/LlmMultiSessionUnifiedMemoryExtractor.cs b/src/AgentMemory.Extraction.Llm/LlmMultiSessionUnifiedMemoryExtractor.cs index c20368894..778f06e94 100644 --- a/src/AgentMemory.Extraction.Llm/LlmMultiSessionUnifiedMemoryExtractor.cs +++ b/src/AgentMemory.Extraction.Llm/LlmMultiSessionUnifiedMemoryExtractor.cs @@ -247,6 +247,50 @@ private async Task> Extract // count. Removing it does not remove a working recovery; it removes a misleading one. A real // transport retry is tracked separately, because it must be reconciled with the harness's // exact-call-count invariant rather than quietly breaking it. + // A refusal on a MULTI-session batch is split first, exactly like a shape failure -- not to + // retry (the policy is deterministic for the same text) but to ISOLATE. Measured: the batch + // that killed a 616-call preparation went 4 -> 2 -> 1 and still failed at 1, so one session's + // content was responsible and its three batch-mates were innocent. Skipping the whole batch + // would have discarded all four. + catch (Exception ex) when (batch.Count > 1 && IsContentRejection(ex)) + { + _batchDiagnostics?.RecordSplit(ex, batch.Count); + _logger.LogWarning( + ex, + "Provider refused a batch of {Count} source sessions; splitting to isolate which " + + "one(s) it objects to, so the rest are not lost with it.", + batch.Count); + var midpoint = batch.Count / 2; + var isolatedLeft = await ExtractOrSplitAsync( + batch.Take(midpoint).ToArray(), maxInputTokens, cancellationToken).ConfigureAwait(false); + var isolatedRight = await ExtractOrSplitAsync( + batch.Skip(midpoint).ToArray(), maxInputTokens, cancellationToken).ConfigureAwait(false); + return isolatedLeft.Concat(isolatedRight) + .ToDictionary(pair => pair.Key, pair => pair.Value, StringComparer.Ordinal); + } + + // Isolated to one session, and the provider still refuses it: terminal. No retry and no + // further split changes the text or the policy. Skipping costs this session's extraction; + // propagating costs the ENTIRE preparation -- ~616 calls and over an hour for 50 questions. + // + // The SESSION ID is recorded, not the content. A refusal that leaves no trace of what caused + // it cannot be investigated or reported to the provider, and this will recur: the id is the + // handle an operator needs to look the conversation up in the dataset themselves, without the + // diagnostics carrying dataset text they are deliberately free of. + catch (Exception ex) when (IsContentRejection(ex)) + { + var refused = batch[0].SessionId; + _batchDiagnostics?.RecordContentRejection(ex, batch.Count, refused); + _logger.LogWarning( + ex, + "Provider refused source session '{SessionId}' on content grounds; its extraction is " + + "skipped and recorded. Whatever that conversation held is absent from this corpus.", + refused); + return batch.ToDictionary( + request => request.SessionId, + _ => new UnifiedExtractionResult(), + StringComparer.Ordinal); + } catch (Exception ex) when (batch.Count > 1 && IsBatchShapeFailure(ex)) { _batchDiagnostics?.RecordSplit(ex, batch.Count); @@ -508,9 +552,43 @@ internal static bool IsBatchShapeFailure(Exception exception) if (exception is FormatException) return true; + // A content-filter rejection is a 4xx, but it is NOT a shape failure: the request is + // well-formed and the provider is refusing the content. Splitting re-sends the same text to + // the same filter, so it can only fail again, one session at a time, until the batch reaches + // size 1 and the exception escapes. + if (IsContentRejection(exception)) return false; + var status = Internal.LlmExtractionRunner.TryGetStatus(exception); return status is >= 400 and < 500 and not 408 and not 429; } + /// + /// Whether the provider refused this content outright, rather than failing to process it. + /// + /// + /// + /// Terminal by nature. A retry sends the same text to the same policy, and a split sends smaller + /// pieces of the same text; neither can succeed. Measured cost of treating it as retryable: a + /// 50-question preparation died after 270 of 616 calls because two sessions of a public + /// research dataset tripped an Azure content policy. + /// + /// + /// Matched on the provider's own vocabulary rather than on status alone, because 400 covers both + /// "your request is malformed" (which splitting legitimately diagnoses) and "I will not process + /// this" (which it cannot). + /// + /// + internal static bool IsContentRejection(Exception exception) + { + var status = Internal.LlmExtractionRunner.TryGetStatus(exception); + if (status != 400) return false; + + var message = exception.Message; + return message.Contains("content_filter", StringComparison.OrdinalIgnoreCase) + || message.Contains("content filter", StringComparison.OrdinalIgnoreCase) + || message.Contains("cyber_policy", StringComparison.OrdinalIgnoreCase) + || message.Contains("ResponsibleAIPolicyViolation", StringComparison.OrdinalIgnoreCase); + } + private sealed class BatchValidationException(string message) : FormatException(message); } diff --git a/src/AgentMemory.Extraction.Llm/LlmPreferenceExtractor.cs b/src/AgentMemory.Extraction.Llm/LlmPreferenceExtractor.cs index bdabaa4fc..7d5a4ba7f 100644 --- a/src/AgentMemory.Extraction.Llm/LlmPreferenceExtractor.cs +++ b/src/AgentMemory.Extraction.Llm/LlmPreferenceExtractor.cs @@ -45,14 +45,21 @@ public LlmPreferenceExtractor( _runner = new LlmExtractionRunner(chatClient, _options, logger); } - protected override async Task> ExtractCoreAsync( + protected override Task> ExtractCoreAsync( IReadOnlyList messages, CancellationToken cancellationToken) + => ExtractCoreWithContextAsync(ExtractionWindow.ForTargets(messages), cancellationToken); + + protected override async Task> ExtractCoreWithContextAsync( + ExtractionWindow window, CancellationToken cancellationToken) { + var messages = window.Targets; var conversationText = _options.Provenance == ExtractionProvenanceMode.PerItem - ? ConversationTextBuilder.BuildNumbered(messages) - : ConversationTextBuilder.Build(messages); + ? ConversationTextBuilder.BuildWindow(window, numbered: true) + : ConversationTextBuilder.BuildWindow(window, numbered: false); return await _runner.RunAsync( - _options.PreferenceExtractionPrompt ?? DefaultSystemPrompt, + (_options.PreferenceExtractionPrompt ?? DefaultSystemPrompt) + // Only when context is present, so a context-free prompt stays byte-identical (E2). + + (window.HasContext ? ExtractionPromptSemantics.ExtractionContextInstruction : string.Empty), "Extract preferences from this conversation:", conversationText, ProjectPreferences, diff --git a/src/AgentMemory.Extraction.Llm/LlmRelationshipExtractor.cs b/src/AgentMemory.Extraction.Llm/LlmRelationshipExtractor.cs index 7ea3e4777..152c52780 100644 --- a/src/AgentMemory.Extraction.Llm/LlmRelationshipExtractor.cs +++ b/src/AgentMemory.Extraction.Llm/LlmRelationshipExtractor.cs @@ -44,12 +44,18 @@ public LlmRelationshipExtractor( _runner = new LlmExtractionRunner(chatClient, _options, logger); } - protected override async Task> ExtractCoreAsync( + protected override Task> ExtractCoreAsync( IReadOnlyList messages, CancellationToken cancellationToken) + => ExtractCoreWithContextAsync(ExtractionWindow.ForTargets(messages), cancellationToken); + + protected override async Task> ExtractCoreWithContextAsync( + ExtractionWindow window, CancellationToken cancellationToken) { - var conversationText = ConversationTextBuilder.Build(messages); + var conversationText = ConversationTextBuilder.BuildWindow(window, numbered: false); return await _runner.RunAsync( - _options.RelationshipExtractionPrompt ?? DefaultSystemPrompt, + (_options.RelationshipExtractionPrompt ?? DefaultSystemPrompt) + // Only when context is present, so a context-free prompt stays byte-identical (E2). + + (window.HasContext ? ExtractionPromptSemantics.ExtractionContextInstruction : string.Empty), "Extract relationships from this conversation:", conversationText, ProjectRelationships, diff --git a/src/AgentMemory.Extraction.Llm/LlmUnifiedMemoryExtractor.cs b/src/AgentMemory.Extraction.Llm/LlmUnifiedMemoryExtractor.cs index e101d4fa2..2241ca56d 100644 --- a/src/AgentMemory.Extraction.Llm/LlmUnifiedMemoryExtractor.cs +++ b/src/AgentMemory.Extraction.Llm/LlmUnifiedMemoryExtractor.cs @@ -44,22 +44,31 @@ public LlmUnifiedMemoryExtractor( public bool IsEnabled => _options.UseUnifiedExtraction; - public async Task ExtractAsync( + public Task ExtractAsync( IReadOnlyList messages, CancellationToken cancellationToken = default) + => ExtractWithContextAsync(ExtractionWindow.ForTargets(messages), cancellationToken); + + public async Task ExtractWithContextAsync( + ExtractionWindow window, + CancellationToken cancellationToken = default) { - if (messages.Count == 0) + // Judged on the targets: a window carrying only background has nothing to extract from, and + // calling out would spend a completion on turns the prompt forbids extracting from. + if (window.Targets.Count == 0) return new UnifiedExtractionResult(); using var activity = AgentMemoryDiagnostics.Source.StartActivity("memory.extract.unified"); var results = await _runner.RunAsync( BuildSystemPrompt( _options.AssistantContent, _options.EntityTypes, _options.TemporalValidity, - _options.Provenance), + _options.Provenance) + // Appended only when context is actually present, so a context-free prompt stays + // byte-for-byte what every sealed measurement was taken under (E2). + + (window.HasContext ? ExtractionPromptSemantics.ExtractionContextInstruction : string.Empty), "Extract all supported memory from this conversation:", - _options.Provenance == ExtractionProvenanceMode.PerItem - ? ConversationTextBuilder.BuildNumbered(messages) - : ConversationTextBuilder.Build(messages), + ConversationTextBuilder.BuildWindow( + window, numbered: _options.Provenance == ExtractionProvenanceMode.PerItem), response => new[] { Project(response) }, cancellationToken, failOnParseExhaustion: true).ConfigureAwait(false); diff --git a/src/AgentMemory.Neo4j/Infrastructure/ServiceCollectionExtensions.cs b/src/AgentMemory.Neo4j/Infrastructure/ServiceCollectionExtensions.cs index 5408555fe..c99638d03 100644 --- a/src/AgentMemory.Neo4j/Infrastructure/ServiceCollectionExtensions.cs +++ b/src/AgentMemory.Neo4j/Infrastructure/ServiceCollectionExtensions.cs @@ -72,6 +72,10 @@ public static IServiceCollection AddNeo4jAgentMemory( services.TryAddTransient(); services.TryAddTransient(); services.TryAddTransient(); + // S1. Registered beside the other repositories rather than behind a feature flag: an + // unregistered store would make EntitySummaryService unresolvable, so the option that + // enables summaries would fail at host start rather than simply doing nothing. + services.TryAddTransient(); services.TryAddTransient(); // Provenance diff --git a/src/AgentMemory.Neo4j/Queries/EntitySummaryQueries.cs b/src/AgentMemory.Neo4j/Queries/EntitySummaryQueries.cs new file mode 100644 index 000000000..6636f18e0 --- /dev/null +++ b/src/AgentMemory.Neo4j/Queries/EntitySummaryQueries.cs @@ -0,0 +1,76 @@ +namespace AgentMemory.Neo4j.Queries; + +/// +/// Cypher for synthesized entity summaries (S1). +/// +/// +/// +/// One summary per entity, held on a :EntitySummary node joined to its entity by +/// :SUMMARISES. A separate node rather than a property on :Entity because it carries +/// its own provenance edges and its own fingerprint, and because a derived value living beside +/// extracted ones invites exactly the confusion this design is trying to avoid. +/// +/// +/// No history. A superseded summary is not a record of what was once believed — the facts it +/// was built from carry their own bitemporal history already, and keeping stale derivations around +/// would mean two answers to "what did we think about Alice in March", only one of them authoritative. +/// +/// +internal static class EntitySummaryQueries +{ + /// + /// Writes the entity's summary, replacing whatever was there. + /// + /// + /// MERGE on entity_id + owner_key, so two tenants summarising the same shared entity + /// keep separate summaries — each was synthesised from the facts that tenant can see, and sharing + /// one node would leak one owner's knowledge into another's context. + /// + public const string Upsert = """ + MERGE (s:EntitySummary {entity_id: $entityId, owner_key: $ownerKey}) + SET s.id = $summaryId, + s.content = $content, + s.source_fact_ids = $sourceFactIds, + s.source_fingerprint = $sourceFingerprint, + s.owner_id = $ownerId, + s.generated_at = datetime($generatedAt) + WITH s + OPTIONAL MATCH (e:Entity {id: $entityId}) + FOREACH (_ IN CASE WHEN e IS NULL THEN [] ELSE [1] END | + MERGE (s)-[:SUMMARISES]->(e)) + WITH s + // Provenance to the facts the text was written from. Rebuilt wholesale rather than merged, + // because a summary regenerated from a smaller fact set must not keep edges to sources it no + // longer draws on -- that would claim provenance the content cannot support. + OPTIONAL MATCH (s)-[old:EXTRACTED_FROM]->(:Fact) + DELETE old + WITH s + UNWIND $sourceFactIds AS factId + MATCH (f:Fact {id: factId}) + MERGE (s)-[:EXTRACTED_FROM]->(f) + RETURN s + """; + + /// Reads an entity's summary, owner-confined (R1). + public static string GetByEntity(bool hasOwnerFilter, bool includeShared) + { + var owner = !hasOwnerFilter ? string.Empty + : includeShared ? " AND (s.owner_id = $ownerId OR s.owner_id IS NULL)" + : " AND s.owner_id = $ownerId"; + return $"MATCH (s:EntitySummary) WHERE s.entity_id = $entityId{owner} RETURN s"; + } + + /// Removes an entity's summary and its edges. + public static string DeleteByEntity(bool hasOwnerFilter, bool includeShared) + { + var owner = !hasOwnerFilter ? string.Empty + : includeShared ? " AND (s.owner_id = $ownerId OR s.owner_id IS NULL)" + : " AND s.owner_id = $ownerId"; + return $""" + MATCH (s:EntitySummary) WHERE s.entity_id = $entityId{owner} + WITH s, count(s) AS found + DETACH DELETE s + RETURN found + """; + } +} diff --git a/src/AgentMemory.Neo4j/Queries/FactQueries.cs b/src/AgentMemory.Neo4j/Queries/FactQueries.cs index a9f9c7b92..6f635e322 100644 --- a/src/AgentMemory.Neo4j/Queries/FactQueries.cs +++ b/src/AgentMemory.Neo4j/Queries/FactQueries.cs @@ -109,16 +109,31 @@ ON CREATE SET f.valid_until = CASE WHEN $validUntil IS NOT NULL THEN datetime($validUntil) ELSE null END, f.source_message_ids = $sourceMessageIds, f.created_at = datetime($createdAtUtc), - f.metadata = $metadata + f.metadata = $metadata, + // R7 salience. Counts how often THE WORLD asserted this fact, incremented only here + // and in ON MATCH -- i.e. once per ingestion that re-states the same triple. + // Deliberately NOT the :MemoryReadAudit trail, which counts how often WE surfaced it: + // ranking on our own retrievals is a rich-get-richer loop that reinforces whatever + // already ranks highly and calls it learning. + f.mention_count = 1 ON MATCH SET f.category = $category, - f.confidence = $confidence, + // S2 corroboration. A re-asserted triple is the world saying it again, so confidence + // EARNS rather than being replaced by whatever the latest extraction happened to + // report. Gated by the parameter itself -- at alpha 0 this is the original assignment, + // byte for byte, so nothing recorded before S2 moves. + f.confidence = CASE WHEN $reinforceAlpha > 0 + THEN CASE WHEN coalesce(f.confidence, $confidence) + $reinforceAlpha > 1.0 + THEN 1.0 + ELSE coalesce(f.confidence, $confidence) + $reinforceAlpha END + ELSE $confidence END, f.valid_from = CASE WHEN $validFrom IS NOT NULL THEN datetime($validFrom) ELSE f.valid_from END, f.valid_until = CASE WHEN $validUntil IS NOT NULL THEN datetime($validUntil) ELSE f.valid_until END, f.source_message_ids = $sourceMessageIds, f.updated_at = datetime($updatedAtUtc), f.metadata = $metadata, - f.invalidated_at = null + f.invalidated_at = null, + f.mention_count = coalesce(f.mention_count, 1) + 1 RETURN f"; // ── UpsertBatchAsync ─────────────────────────────────────────────── @@ -148,18 +163,54 @@ ON CREATE SET f.valid_until = CASE WHEN item.valid_until IS NOT NULL THEN datetime(item.valid_until) ELSE null END, f.source_message_ids = item.source_message_ids, f.created_at = datetime(item.created_at), - f.metadata = item.metadata + f.metadata = item.metadata, + f.mention_count = 1 ON MATCH SET f.category = item.category, - f.confidence = item.confidence, + // S2 corroboration, identical to the single-item path. A counter that depended on + // which write path ran would make reinforcement a property of batching rather than of + // the conversation. + f.confidence = CASE WHEN $reinforceAlpha > 0 + THEN CASE WHEN coalesce(f.confidence, item.confidence) + $reinforceAlpha > 1.0 + THEN 1.0 + ELSE coalesce(f.confidence, item.confidence) + $reinforceAlpha END + ELSE item.confidence END, f.valid_from = CASE WHEN item.valid_from IS NOT NULL THEN datetime(item.valid_from) ELSE f.valid_from END, f.valid_until = CASE WHEN item.valid_until IS NOT NULL THEN datetime(item.valid_until) ELSE f.valid_until END, f.source_message_ids = item.source_message_ids, f.updated_at = datetime(item.updated_at), f.metadata = item.metadata, - f.invalidated_at = null + f.invalidated_at = null, + f.mention_count = coalesce(f.mention_count, 1) + 1 RETURN f"; + // ── R7 mention frequency ─────────────────────────────────────────── + + /// + /// How often the world asserted each of these facts, for the whole candidate set at once. + /// + /// + /// + /// mention_count is incremented only by the upsert paths, i.e. once per ingestion that + /// re-states the same triple. It is a salience signal: a fact the conversation keeps + /// returning to matters more than one mentioned in passing. + /// + /// + /// Explicitly not the read audit. :MemoryReadAudit records how often we + /// surfaced a fact, and ranking on that is a rich-get-richer loop: whatever ranks highly gets + /// retrieved, which raises its count, which raises its rank. It would look like learning and be + /// self-reinforcement. + /// + /// + /// Facts written before this property existed report 1 via coalesce — asserted once, + /// which is what a single ingestion means and the only honest reading of an absent counter. + /// + /// + public const string MentionCounts = @" + MATCH (f:Fact) + WHERE f.id IN $candidateIds + RETURN f.id AS candidateId, coalesce(f.mention_count, 1) AS mentions"; + // ── GetByIdAsync ─────────────────────────────────────────────────── /// Get a single fact by id. @@ -406,7 +457,16 @@ WHERE true" + winnerOwner + @" AND coalesce(loser.owner_id, '*') = coalesce(winner.owner_id, '*') AND loser <> winner SET loser.invalidated_at = coalesce(loser.invalidated_at, datetime($now)), - loser.valid_until = coalesce(loser.valid_until, datetime($now)) + loser.valid_until = coalesce(loser.valid_until, datetime($now)), + // S2 contradiction. Twice the corroboration step, and downward: being contradicted is + // stronger evidence against a fact than one more restatement is for it. Floored at 0 + // rather than allowed negative -- confidence is a [0,1] quantity every ranking and + // dedup computation reads, and a negative would propagate somewhere it means nothing. + loser.confidence = CASE WHEN $reinforceAlpha > 0 + THEN CASE WHEN coalesce(loser.confidence, 0.0) - (2 * $reinforceAlpha) < 0.0 + THEN 0.0 + ELSE coalesce(loser.confidence, 0.0) - (2 * $reinforceAlpha) END + ELSE loser.confidence END MERGE (loser)-[:SUPERSEDED_BY]->(winner) RETURN count(loser) > 0 AS superseded"; } diff --git a/src/AgentMemory.Neo4j/Queries/FusedPersistenceQueries.cs b/src/AgentMemory.Neo4j/Queries/FusedPersistenceQueries.cs index fa9622ca4..f937ff3e7 100644 --- a/src/AgentMemory.Neo4j/Queries/FusedPersistenceQueries.cs +++ b/src/AgentMemory.Neo4j/Queries/FusedPersistenceQueries.cs @@ -65,14 +65,23 @@ ON CREATE SET f.confidence = item.confidence, f.valid_from = CASE WHEN item.valid_from IS NOT NULL THEN datetime(item.valid_from) ELSE null END, f.valid_until = CASE WHEN item.valid_until IS NOT NULL THEN datetime(item.valid_until) ELSE null END, + f.mention_count = 1, f.source_message_ids = item.source_message_ids, f.created_at = datetime(item.created_at), f.metadata = item.metadata ON MATCH SET f.category = item.category, - f.confidence = item.confidence, + // S2 corroboration, identical to the other two write paths. Reinforcement that + // depended on which path ran would be a property of batching rather than of the + // conversation. + f.confidence = CASE WHEN $reinforceAlpha > 0 + THEN CASE WHEN coalesce(f.confidence, item.confidence) + $reinforceAlpha > 1.0 + THEN 1.0 + ELSE coalesce(f.confidence, item.confidence) + $reinforceAlpha END + ELSE item.confidence END, f.valid_from = CASE WHEN item.valid_from IS NOT NULL THEN datetime(item.valid_from) ELSE f.valid_from END, f.valid_until = CASE WHEN item.valid_until IS NOT NULL THEN datetime(item.valid_until) ELSE f.valid_until END, + f.mention_count = coalesce(f.mention_count, 1) + 1, f.source_message_ids = item.source_message_ids, f.updated_at = datetime(item.updated_at), f.metadata = item.metadata, diff --git a/src/AgentMemory.Neo4j/Queries/NodeDistanceQueries.cs b/src/AgentMemory.Neo4j/Queries/NodeDistanceQueries.cs new file mode 100644 index 000000000..f131bc383 --- /dev/null +++ b/src/AgentMemory.Neo4j/Queries/NodeDistanceQueries.cs @@ -0,0 +1,68 @@ +namespace AgentMemory.Neo4j.Queries; + +/// +/// Graph distance from a query-matched centroid entity to a set of candidate facts (R6). +/// +/// +/// +/// Vector similarity asks "does this row look like the query?"; graph distance asks "is this row +/// about what the query is about?". They disagree usefully: a fact whose wording is unremarkable +/// but which hangs directly off the entity the question names is often the answer, and a fact that +/// merely paraphrases the query can be about someone else entirely. +/// +/// +/// One bounded query for the whole candidate set, never one per candidate. A per-candidate +/// traversal would multiply a recall's database round trips by the limit — and it runs on the blocking +/// path, so a reranker that costs more than the retrieval it reorders is not an improvement. +/// +/// +internal static class NodeDistanceQueries +{ + /// + /// The entity whose name or aliases best match the query, by vector similarity. + /// + /// + /// The centroid is chosen by the same embedding the retrieval used, so "near the centroid" + /// and "similar to the query" are commensurable rather than two unrelated notions of relevance. + /// Owner-scoped: a centroid drawn from another tenant's graph would silently reorder this tenant's + /// results around a node they cannot see. + /// + internal static string CentroidEntity(bool hasOwnerFilter, bool includeShared) + { + var owner = !hasOwnerFilter ? string.Empty + : includeShared ? " AND (node.owner_id = $ownerId OR node.owner_id IS NULL)" + : " AND node.owner_id = $ownerId"; + return $@" + CALL db.index.vector.queryNodes('entity_embedding_idx', $topK, $embedding) + YIELD node, score + WHERE score >= $minScore{owner} + RETURN node.id AS entityId, score AS score + ORDER BY score DESC + LIMIT 1"; + } + + /// + /// Hop distance from the centroid entity to each candidate fact, for the whole set at once. + /// + /// + /// + /// Bounded three ways, because an unbounded traversal on a dense graph is how a reranker becomes + /// the slowest thing in a recall: the candidate list is the starting set (not the corpus), the + /// path length is capped at $maxHops, and only RELATED_TO/ABOUT edges are + /// followed — the two that carry meaning between an entity and a fact. EXTRACTED_FROM is + /// deliberately excluded: it links to source messages, so following it would make every fact from + /// a shared conversation look adjacent regardless of subject. + /// + /// + /// Candidates with no path within the cap are simply absent from the result, which the caller + /// reads as "no boost" — the alternative, returning a sentinel distance, invites arithmetic on a + /// number that means "unreachable". + /// + /// + internal static string DistanceToCandidates => @" + MATCH (centroid:Entity {id: $centroidId}) + MATCH (f:Fact) WHERE f.id IN $candidateIds + MATCH path = shortestPath((centroid)-[:RELATED_TO|ABOUT*..4]-(f)) + WHERE length(path) <= $maxHops + RETURN f.id AS candidateId, length(path) AS hops"; +} diff --git a/src/AgentMemory.Neo4j/Repositories/Neo4jEntityRepository.cs b/src/AgentMemory.Neo4j/Repositories/Neo4jEntityRepository.cs index b7618b2b2..afb62fabb 100644 --- a/src/AgentMemory.Neo4j/Repositories/Neo4jEntityRepository.cs +++ b/src/AgentMemory.Neo4j/Repositories/Neo4jEntityRepository.cs @@ -19,9 +19,42 @@ internal sealed partial class Neo4jEntityRepository : IEntityRepository, IUpsert { private readonly INeo4jTransactionRunner _tx; + private readonly bool _rescueShortOwnerResults; private readonly ILogger _logger; private readonly MemoryRankingOptions _ranking; private readonly MemoryDecayOptions _decay; + /// Scores this owner's OWN entitys directly, bypassing the global vector index. + /// + /// Extracted because two conditions reach it -- an empty scoped result, and (opt-in) a short one + /// -- and two copies of a fallback drift. + /// + private async Task> OwnerScopedScanAsync( + float[] queryEmbedding, + int limit, + double minScore, + MemoryScope scope, + bool includeShared, + CancellationToken cancellationToken) => + await _tx.ReadAsync(async runner => + { + var cursor = await runner.RunAsync( + EntityQueries.SearchByVectorOwnerScopedFallback(includeShared), + new Dictionary + { + ["embedding"] = queryEmbedding.ToList(), + ["limit"] = limit, + ["minScore"] = minScore, + ["ownerId"] = scope.OwnerId, + }).ConfigureAwait(false); + var records = await cursor.ToListAsync().ConfigureAwait(false); + return records.Select(r => + { + var node = r["node"].As(); + var score = r["score"].As(); + return (MapToEntity(node, ReadEmbedding(node)), score); + }).ToList(); + }, cancellationToken).ConfigureAwait(false) ?? []; + private readonly IMemoryRankingContext? _rankingContext; public Neo4jEntityRepository( @@ -29,8 +62,10 @@ public Neo4jEntityRepository( ILogger logger, IOptions? ranking = null, IOptions? decay = null, - IMemoryRankingContext? rankingContext = null) + IMemoryRankingContext? rankingContext = null, + IOptions? memoryOptions = null) { + _rescueShortOwnerResults = memoryOptions?.Value.RescueShortOwnerResults ?? false; _tx = tx; _logger = logger; _ranking = ranking?.Value ?? MemoryRankingOptions.Default; @@ -234,27 +269,25 @@ public async Task> GetByNameAsync( _logger.LogDebug( "Owner-scoped entity vector search still empty after widening; falling back to a " + "scoped similarity scan."); - results = await _tx.ReadAsync(async runner => - { - var cursor = await runner.RunAsync( - EntityQueries.SearchByVectorOwnerScopedFallback(includeShared), - new Dictionary - { - ["embedding"] = queryEmbedding.ToList(), - ["limit"] = limit, - ["minScore"] = minScore, - ["ownerId"] = scope!.OwnerId, - }).ConfigureAwait(false); - var records = await cursor.ToListAsync().ConfigureAwait(false); - return records.Select(r => - { - var node = r["node"].As(); - var score = r["score"].As(); - return (MapToEntity(node, ReadEmbedding(node)), score); - }).ToList(); - }, cancellationToken).ConfigureAwait(false) ?? []; + results = await OwnerScopedScanAsync( + queryEmbedding, limit, minScore, scope!, includeShared, cancellationToken) + .ConfigureAwait(false); } } + else if (_rescueShortOwnerResults + && OwnerVectorOverFetch.ShouldRescueShortResult(results.Count, limit, hasOwner)) + { + // 2.12, the same rescue the fact path carries. Extended here because "a setting only some + // components respect" is the defect shape this project has found twice -- a host enabling + // RescueShortOwnerResults would otherwise get it on facts and silently not on entitys. + _logger.LogDebug( + "Owner-scoped entity vector search returned {Returned} of {Limit}; rescuing with a " + + "scoped similarity scan.", results.Count, limit); + var scanned = await OwnerScopedScanAsync( + queryEmbedding, limit, minScore, scope!, includeShared, cancellationToken) + .ConfigureAwait(false); + if (scanned.Count > results.Count) results = scanned; + } EmitVectorYield( activity, hasOwner, limit, escalatedTopK ?? topK, results.Count, diff --git a/src/AgentMemory.Neo4j/Repositories/Neo4jEntitySummaryRepository.cs b/src/AgentMemory.Neo4j/Repositories/Neo4jEntitySummaryRepository.cs new file mode 100644 index 000000000..8435d4846 --- /dev/null +++ b/src/AgentMemory.Neo4j/Repositories/Neo4jEntitySummaryRepository.cs @@ -0,0 +1,107 @@ +using AgentMemory.Abstractions.Domain; +using AgentMemory.Abstractions.Options; +using AgentMemory.Abstractions.Repositories; +using AgentMemory.Neo4j.Infrastructure; +using AgentMemory.Neo4j.Queries; +using Microsoft.Extensions.Logging; +using Neo4j.Driver; + +namespace AgentMemory.Neo4j.Repositories; + +/// +/// Neo4j persistence for synthesized entity summaries (S1). +/// +/// +/// The summary node keeps its own :EXTRACTED_FROM edges to the facts it was written from. That +/// is the part a summary column in a relational row cannot do, and it is what makes "which facts is +/// this claim standing on?" a query rather than a comment. +/// +internal sealed class Neo4jEntitySummaryRepository : IEntitySummaryRepository +{ + private readonly INeo4jTransactionRunner _tx; + private readonly ILogger _logger; + + /// Public ctor: the type is internal, and DI can only activate it through one. + public Neo4jEntitySummaryRepository( + INeo4jTransactionRunner tx, + ILogger logger) + { + _tx = tx; + _logger = logger; + } + + /// + public async Task UpsertAsync( + EntitySummary summary, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(summary); + + await _tx.WriteAsync(async runner => + { + var cursor = await runner.RunAsync(EntitySummaryQueries.Upsert, new + { + summaryId = summary.SummaryId, + entityId = summary.EntityId, + content = summary.Content, + sourceFactIds = summary.SourceFactIds.ToArray(), + sourceFingerprint = summary.SourceFingerprint, + ownerId = summary.OwnerId, + ownerKey = summary.OwnerId ?? Neo4jFactRepository.OwnerKeyShared, + generatedAt = summary.GeneratedAtUtc.ToString("o"), + }).ConfigureAwait(false); + await cursor.ConsumeAsync().ConfigureAwait(false); + return true; + }, cancellationToken).ConfigureAwait(false); + + _logger.LogDebug( + "Stored summary for entity {EntityId} over {SourceCount} fact(s).", + summary.EntityId, summary.SourceFactIds.Count); + return summary; + } + + /// + public Task GetByEntityAsync( + string entityId, MemoryScope scope, CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(entityId); + + return _tx.ReadAsync(async runner => + { + var cursor = await runner.RunAsync( + EntitySummaryQueries.GetByEntity(scope.OwnerId is not null, scope.IncludeShared), + new { entityId, ownerId = scope.OwnerId }).ConfigureAwait(false); + var records = await cursor.ToListAsync().ConfigureAwait(false); + return records.Count == 0 ? null : Map(records[0]["s"].As()); + }, cancellationToken); + } + + /// + public Task DeleteByEntityAsync( + string entityId, MemoryScope scope, CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(entityId); + + return _tx.WriteAsync(async runner => + { + var cursor = await runner.RunAsync( + EntitySummaryQueries.DeleteByEntity(scope.OwnerId is not null, scope.IncludeShared), + new { entityId, ownerId = scope.OwnerId }).ConfigureAwait(false); + var records = await cursor.ToListAsync().ConfigureAwait(false); + return records.Count > 0; + }, cancellationToken); + } + + private static EntitySummary Map(INode node) => new() + { + SummaryId = node.Properties.TryGetValue("id", out var id) ? id.As() : string.Empty, + EntityId = node["entity_id"].As(), + Content = node["content"].As(), + SourceFactIds = node.Properties.TryGetValue("source_fact_ids", out var ids) + ? ids.As>().Select(v => v.ToString()!).ToList() + : [], + SourceFingerprint = node["source_fingerprint"].As(), + OwnerId = node.Properties.TryGetValue("owner_id", out var owner) ? owner.As() : null, + GeneratedAtUtc = Neo4jDateTimeHelper.ReadNullableDateTimeOffset(node["generated_at"]) + ?? DateTimeOffset.MinValue, + }; +} diff --git a/src/AgentMemory.Neo4j/Repositories/Neo4jFactRepository.Fused.cs b/src/AgentMemory.Neo4j/Repositories/Neo4jFactRepository.Fused.cs index b273e52b7..63bbe0c23 100644 --- a/src/AgentMemory.Neo4j/Repositories/Neo4jFactRepository.Fused.cs +++ b/src/AgentMemory.Neo4j/Repositories/Neo4jFactRepository.Fused.cs @@ -68,7 +68,9 @@ public async Task> UpsertFusedBatchAsync( return await _tx.WriteAsync(async runner => { - var cursor = await runner.RunAsync(FusedPersistenceQueries.FactUpsertBatch, new { items }) + var cursor = await runner.RunAsync( + FusedPersistenceQueries.FactUpsertBatch, + new { items, reinforceAlpha = _reinforceAlpha }) .ConfigureAwait(false); var records = await cursor.ToListAsync().ConfigureAwait(false); var embeddingByTriple = deduped.ToDictionary( diff --git a/src/AgentMemory.Neo4j/Repositories/Neo4jFactRepository.cs b/src/AgentMemory.Neo4j/Repositories/Neo4jFactRepository.cs index 3b34245c8..7665d208c 100644 --- a/src/AgentMemory.Neo4j/Repositories/Neo4jFactRepository.cs +++ b/src/AgentMemory.Neo4j/Repositories/Neo4jFactRepository.cs @@ -23,6 +23,8 @@ internal sealed partial class Neo4jFactRepository : IFactRepository, IUpsertPers internal const string OwnerKeyShared = "*"; private readonly INeo4jTransactionRunner _tx; + private readonly bool _rescueShortOwnerResults; + private readonly double _reinforceAlpha; private readonly ILogger _logger; private readonly MemoryRankingOptions _ranking; private readonly MemoryDecayOptions _decay; @@ -33,8 +35,11 @@ public Neo4jFactRepository( ILogger logger, IOptions? ranking = null, IOptions? decay = null, - IMemoryRankingContext? rankingContext = null) + IMemoryRankingContext? rankingContext = null, + IOptions? memoryOptions = null) { + _rescueShortOwnerResults = memoryOptions?.Value.RescueShortOwnerResults ?? false; + _reinforceAlpha = memoryOptions?.Value.ConfidenceReinforcementAlpha ?? 0.0; _tx = tx; _logger = logger; _ranking = ranking?.Value ?? MemoryRankingOptions.Default; @@ -78,6 +83,7 @@ public async Task UpsertAsync(Fact fact, CancellationToken cancellationTok ["ownerKey"] = fact.OwnerId ?? OwnerKeyShared, ["category"] = fact.Category, ["confidence"] = fact.Confidence, + ["reinforceAlpha"] = _reinforceAlpha, ["validFrom"] = (object?)(fact.ValidFrom?.ToString("O")), ["validUntil"] = (object?)(fact.ValidUntil?.ToString("O")), ["sourceMessageIds"] = fact.SourceMessageIds.ToList(), @@ -174,7 +180,9 @@ public async Task> UpsertBatchAsync(IReadOnlyList fact return await _tx.WriteAsync(async runner => { - var cursor = await runner.RunAsync(FactQueries.UpsertBatch, new { items }).ConfigureAwait(false); + var cursor = await runner.RunAsync( + FactQueries.UpsertBatch, + new { items, reinforceAlpha = _reinforceAlpha }).ConfigureAwait(false); var records = await cursor.ToListAsync().ConfigureAwait(false); // The MERGE returns the SURVIVING node per triple; for a pre-existing triple that id is the @@ -293,6 +301,43 @@ public async Task> GetBySubjectAsync( }, cancellationToken).ConfigureAwait(false); } + /// + /// Scores this owner's OWN facts directly, bypassing the global vector index. + /// + /// + /// Extracted because two different conditions now reach it — a totally empty scoped result, and + /// (opt-in) a short one — and two copies of a fallback drift. Bounded by one owner's rows rather + /// than by the corpus, which is what makes it affordable for the small tenant and effective for + /// the crowded one. + /// + private async Task> OwnerScopedScanAsync( + float[] queryEmbedding, + int limit, + double minScore, + MemoryScope scope, + bool includeShared, + bool currentValidTime, + CancellationToken cancellationToken) => + await _tx.ReadAsync(async runner => + { + var cursor = await runner.RunAsync( + FactQueries.SearchByVectorOwnerScopedFallback(includeShared, currentValidTime), + new Dictionary + { + ["embedding"] = queryEmbedding.ToList(), + ["limit"] = limit, + ["minScore"] = minScore, + ["ownerId"] = scope.OwnerId, + }).ConfigureAwait(false); + var records = await cursor.ToListAsync().ConfigureAwait(false); + return records.Select(r => + { + var node = r["node"].As(); + var score = r["score"].As(); + return (MapToFact(node, ReadEmbedding(node)), score); + }).ToList(); + }, cancellationToken).ConfigureAwait(false) ?? []; + public Task> SearchByVectorAsync( float[] queryEmbedding, int limit = 10, @@ -387,27 +432,34 @@ public async Task> GetBySubjectAsync( _logger.LogDebug( "Owner-scoped fact vector search still empty after widening to {Widened}; " + "falling back to a scoped similarity scan.", escalatedTopK ?? topK); - results = await _tx.ReadAsync(async runner => - { - var cursor = await runner.RunAsync( - FactQueries.SearchByVectorOwnerScopedFallback(includeShared, currentValidTime), - new Dictionary - { - ["embedding"] = queryEmbedding.ToList(), - ["limit"] = limit, - ["minScore"] = minScore, - ["ownerId"] = scope!.OwnerId, - }).ConfigureAwait(false); - var records = await cursor.ToListAsync().ConfigureAwait(false); - return records.Select(r => - { - var node = r["node"].As(); - var score = r["score"].As(); - return (MapToFact(node, ReadEmbedding(node)), score); - }).ToList(); - }, cancellationToken).ConfigureAwait(false) ?? []; + results = await OwnerScopedScanAsync( + queryEmbedding, limit, minScore, scope!, includeShared, currentValidTime, + cancellationToken).ConfigureAwait(false); } } + else if (_rescueShortOwnerResults + && OwnerVectorOverFetch.ShouldRescueShortResult(results.Count, limit, hasOwner)) + { + // 2.11. "Short still answers the question" is true for a small tenant and false for a + // crowded one, and the returned count cannot tell them apart: question 5d3d2817 returned + // 2 facts from a 710-fact graph with the answer present, and was answered wrongly. + // + // The rescue is the SCAN, not another widening. Widening is a second draw on the same + // global index, and a tenant losing to 50 neighbours at top-60 usually loses again at + // top-480. The scan is bounded by one owner's rows, so the small tenant this heuristic + // was protecting pays less than it would for a wider index query. + // + // Takes whichever result is larger rather than replacing outright: the scan applies the + // same minScore and orders by the same similarity, so a shorter scan result means the + // owner genuinely holds no more, and discarding indexed rows for it would be a loss. + _logger.LogDebug( + "Owner-scoped fact vector search returned {Returned} of {Limit}; rescuing with a " + + "scoped similarity scan.", results.Count, limit); + var scanned = await OwnerScopedScanAsync( + queryEmbedding, limit, minScore, scope!, includeShared, currentValidTime, + cancellationToken).ConfigureAwait(false); + if (scanned.Count > results.Count) results = scanned; + } if (activity is not null) { @@ -542,6 +594,9 @@ private static Fact MapToFact(INode node, float[]? embedding) => ValidUntil = node.Properties.TryGetValue("valid_until", out var vu) ? Neo4jDateTimeHelper.ReadNullableDateTimeOffset(vu) : null, + InvalidatedAtUtc = node.Properties.TryGetValue("invalidated_at", out var iat) + ? Neo4jDateTimeHelper.ReadNullableDateTimeOffset(iat) + : null, Embedding = embedding, SourceMessageIds = node.Properties.TryGetValue("source_message_ids", out var sm) ? sm.As>().Select(v => v.ToString()!).ToList() @@ -643,7 +698,7 @@ public async Task SupersedeAsync(string loserFactId, string winnerFactId, return await _tx.WriteAsync(async runner => { - var parameters = new Dictionary { ["loserId"] = loserFactId, ["winnerId"] = winnerFactId, ["now"] = now }; + var parameters = new Dictionary { ["loserId"] = loserFactId, ["winnerId"] = winnerFactId, ["now"] = now, ["reinforceAlpha"] = _reinforceAlpha }; if (hasOwner) parameters["ownerId"] = scope!.OwnerId; var cursor = await runner.RunAsync(cypher, parameters).ConfigureAwait(false); var records = await cursor.ToListAsync().ConfigureAwait(false); diff --git a/src/AgentMemory.Neo4j/Repositories/Neo4jPreferenceRepository.cs b/src/AgentMemory.Neo4j/Repositories/Neo4jPreferenceRepository.cs index 430e847c5..502dd0d26 100644 --- a/src/AgentMemory.Neo4j/Repositories/Neo4jPreferenceRepository.cs +++ b/src/AgentMemory.Neo4j/Repositories/Neo4jPreferenceRepository.cs @@ -19,9 +19,42 @@ internal sealed partial class Neo4jPreferenceRepository : IPreferenceRepository, { private readonly INeo4jTransactionRunner _tx; + private readonly bool _rescueShortOwnerResults; private readonly ILogger _logger; private readonly MemoryRankingOptions _ranking; private readonly MemoryDecayOptions _decay; + /// Scores this owner's OWN preferences directly, bypassing the global vector index. + /// + /// Extracted because two conditions reach it -- an empty scoped result, and (opt-in) a short one + /// -- and two copies of a fallback drift. + /// + private async Task> OwnerScopedScanAsync( + float[] queryEmbedding, + int limit, + double minScore, + MemoryScope scope, + bool includeShared, + CancellationToken cancellationToken) => + await _tx.ReadAsync(async runner => + { + var cursor = await runner.RunAsync( + PreferenceQueries.SearchByVectorOwnerScopedFallback(includeShared), + new Dictionary + { + ["embedding"] = queryEmbedding.ToList(), + ["limit"] = limit, + ["minScore"] = minScore, + ["ownerId"] = scope.OwnerId, + }).ConfigureAwait(false); + var records = await cursor.ToListAsync().ConfigureAwait(false); + return records.Select(r => + { + var node = r["node"].As(); + var score = r["score"].As(); + return (MapToPreference(node, ReadEmbedding(node)), score); + }).ToList(); + }, cancellationToken).ConfigureAwait(false) ?? []; + private readonly IMemoryRankingContext? _rankingContext; public Neo4jPreferenceRepository( @@ -29,8 +62,10 @@ public Neo4jPreferenceRepository( ILogger logger, IOptions? ranking = null, IOptions? decay = null, - IMemoryRankingContext? rankingContext = null) + IMemoryRankingContext? rankingContext = null, + IOptions? memoryOptions = null) { + _rescueShortOwnerResults = memoryOptions?.Value.RescueShortOwnerResults ?? false; _tx = tx; _logger = logger; _ranking = ranking?.Value ?? MemoryRankingOptions.Default; @@ -248,27 +283,25 @@ public async Task> GetByCategoryAsync( _logger.LogDebug( "Owner-scoped preference vector search still empty after widening; falling back to a " + "scoped similarity scan."); - results = await _tx.ReadAsync(async runner => - { - var cursor = await runner.RunAsync( - PreferenceQueries.SearchByVectorOwnerScopedFallback(includeShared), - new Dictionary - { - ["embedding"] = queryEmbedding.ToList(), - ["limit"] = limit, - ["minScore"] = minScore, - ["ownerId"] = scope!.OwnerId, - }).ConfigureAwait(false); - var records = await cursor.ToListAsync().ConfigureAwait(false); - return records.Select(r => - { - var node = r["node"].As(); - var score = r["score"].As(); - return (MapToPreference(node, ReadEmbedding(node)), score); - }).ToList(); - }, cancellationToken).ConfigureAwait(false) ?? []; + results = await OwnerScopedScanAsync( + queryEmbedding, limit, minScore, scope!, includeShared, cancellationToken) + .ConfigureAwait(false); } } + else if (_rescueShortOwnerResults + && OwnerVectorOverFetch.ShouldRescueShortResult(results.Count, limit, hasOwner)) + { + // 2.12, the same rescue the fact path carries. A host enabling RescueShortOwnerResults + // would otherwise get it on facts and silently not on preferences -- the "setting only some + // components respect" shape this project has found twice. + _logger.LogDebug( + "Owner-scoped preference vector search returned {Returned} of {Limit}; rescuing with " + + "a scoped similarity scan.", results.Count, limit); + var scanned = await OwnerScopedScanAsync( + queryEmbedding, limit, minScore, scope!, includeShared, cancellationToken) + .ConfigureAwait(false); + if (scanned.Count > results.Count) results = scanned; + } // Explicit null check rather than `activity?.SetTag(...)` per AgentMemoryDiagnostics' remarks: the // tag block exists purely to produce telemetry, so it is skipped whole when nobody is listening. diff --git a/src/AgentMemory.Neo4j/Repositories/Neo4jReasoningTraceRepository.cs b/src/AgentMemory.Neo4j/Repositories/Neo4jReasoningTraceRepository.cs index b58d10136..dfe0475a6 100644 --- a/src/AgentMemory.Neo4j/Repositories/Neo4jReasoningTraceRepository.cs +++ b/src/AgentMemory.Neo4j/Repositories/Neo4jReasoningTraceRepository.cs @@ -1,5 +1,6 @@ using System.Diagnostics; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using AgentMemory.Abstractions.Diagnostics; using AgentMemory.Abstractions.Domain; using AgentMemory.Abstractions.Options; @@ -15,10 +16,50 @@ namespace AgentMemory.Neo4j.Repositories; internal sealed class Neo4jReasoningTraceRepository : IReasoningTraceRepository { private readonly INeo4jTransactionRunner _tx; + private readonly bool _rescueShortOwnerResults; + /// Scores this owner's OWN traces directly, bypassing the global vector index. + /// + /// Extracted because two conditions reach it -- an empty scoped result, and (opt-in) a short one + /// -- and two copies of a fallback drift. + /// + private async Task> OwnerScopedScanAsync( + float[] queryEmbedding, + int limit, + double minScore, + MemoryScope scope, + bool includeShared, + bool? successFilter, + bool? proceduresOnly, + CancellationToken cancellationToken) => + await _tx.ReadAsync(async runner => + { + var cursor = await runner.RunAsync( + ReasoningQueries.SearchByTaskVectorOwnerScopedFallback( + successFilter.HasValue, includeShared, proceduresOnly), + new Dictionary + { + ["embedding"] = queryEmbedding.ToList(), + ["limit"] = limit, + ["minScore"] = minScore, + ["ownerId"] = scope.OwnerId, + }).ConfigureAwait(false); + var records = await cursor.ToListAsync().ConfigureAwait(false); + return records.Select(r => + { + var node = r["node"].As(); + var score = r["score"].As(); + return (MapToTrace(node, ReadEmbedding(node)), score); + }).ToList(); + }, cancellationToken).ConfigureAwait(false) ?? []; + private readonly ILogger _logger; - public Neo4jReasoningTraceRepository(INeo4jTransactionRunner tx, ILogger logger) + public Neo4jReasoningTraceRepository( + INeo4jTransactionRunner tx, + ILogger logger, + IOptions? memoryOptions = null) { + _rescueShortOwnerResults = memoryOptions?.Value.RescueShortOwnerResults ?? false; _tx = tx; _logger = logger; } @@ -243,23 +284,25 @@ public async Task> ListAllAsync(int limit = 50, int ["ownerId"] = scope!.OwnerId!, }; if (successFilter.HasValue) fallbackParameters["successFilter"] = successFilter.Value; - - results = await _tx.ReadAsync(async runner => - { - var cursor = await runner.RunAsync( - ReasoningQueries.SearchByTaskVectorOwnerScopedFallback( - successFilter.HasValue, includeShared, proceduresOnly), - fallbackParameters).ConfigureAwait(false); - var records = await cursor.ToListAsync().ConfigureAwait(false); - return records.Select(r => - { - var node = r["node"].As(); - var score = r["score"].As(); - return (MapToTrace(node, ReadEmbedding(node)), score); - }).ToList(); - }, cancellationToken).ConfigureAwait(false) ?? []; + results = await OwnerScopedScanAsync( + taskEmbedding, limit, minScore, scope!, includeShared, successFilter, + proceduresOnly, cancellationToken).ConfigureAwait(false); } } + else if (_rescueShortOwnerResults + && OwnerVectorOverFetch.ShouldRescueShortResult(results.Count, limit, hasOwner)) + { + // 2.12, the same rescue the fact path carries. A host enabling RescueShortOwnerResults + // would otherwise get it on facts and silently not on traces -- the "setting only some + // components respect" shape this project has found twice. + _logger.LogDebug( + "Owner-scoped trace vector search returned {Returned} of {Limit}; rescuing with " + + "a scoped similarity scan.", results.Count, limit); + var scanned = await OwnerScopedScanAsync( + taskEmbedding, limit, minScore, scope!, includeShared, successFilter, + proceduresOnly, cancellationToken).ConfigureAwait(false); + if (scanned.Count > results.Count) results = scanned; + } // Explicit null check rather than `activity?.SetTag(...)` per AgentMemoryDiagnostics' remarks: the // tag block exists purely to produce telemetry, so it is skipped whole when nobody is listening. diff --git a/src/AgentMemory.Neo4j/Repositories/OwnerVectorOverFetch.cs b/src/AgentMemory.Neo4j/Repositories/OwnerVectorOverFetch.cs index 63257a270..4cb419a52 100644 --- a/src/AgentMemory.Neo4j/Repositories/OwnerVectorOverFetch.cs +++ b/src/AgentMemory.Neo4j/Repositories/OwnerVectorOverFetch.cs @@ -27,6 +27,20 @@ namespace AgentMemory.Neo4j.Repositories; /// with an extra query on every recall forever, because an owner holding three facts can never fill a /// ten-row limit. So: one extra query, only when the first pass found nothing, and bounded. /// +/// +/// That last argument has a measured counter-example. Question 5d3d2817 returned +/// 2 facts from a 710-fact graph with the gold answer present at coverage 1.00, and both arms +/// answered it wrongly. "Short still answers the question" is true for a small tenant and false for a +/// crowded one, and the returned count alone cannot tell them apart. +/// +/// +/// The rescue is therefore not more widening. Widening is another draw on the same global +/// index, and a tenant losing to 50 neighbours at top-60 usually loses again at top-480. The +/// owner-scoped similarity scan already used as the last resort is the right instrument: its cost +/// scales with one owner's data rather than with the corpus, so the small tenant this argument +/// was protecting pays almost nothing — scanning three facts is cheaper than a second index query — +/// while the crowded tenant gets its true top-K instead of whatever survived the neighbours. +/// /// internal static class OwnerVectorOverFetch { @@ -57,6 +71,30 @@ internal static int InitialTopK(int limit, bool hasOwner) => /// internal static bool ShouldEscalate(int returned, bool hasOwner) => hasOwner && returned == 0; + /// + /// Whether a scoped search that returned something, but less than asked for, should fall + /// back to the owner-bounded scan. + /// + /// + /// + /// Opt-in (MemoryOptions.RescueShortOwnerResults), because it trades latency for recall on + /// every short result and every recorded measurement was taken without it. + /// + /// + /// Deliberately not a fraction of the limit. A ratio invites a threshold nobody can + /// justify — is 4 of 10 starved? 6 of 10? — and the honest answer is that any shortfall might be + /// crowding, since the index gave the owner whatever the neighbours left. The scan is bounded by + /// the owner's own rows either way, so the gate is simply "short", and the cost question is + /// answered by the scan's shape rather than by guessing a cutoff. + /// + /// + /// Excludes the empty case, which already owns and which reaches the + /// same scan through a path that first tries one widened query. + /// + /// + internal static bool ShouldRescueShortResult(int returned, int limit, bool hasOwner) => + hasOwner && returned > 0 && returned < limit; + /// The widened width, capped. internal static int EscalatedTopK(int currentTopK) { diff --git a/src/AgentMemory.Neo4j/Services/MentionFrequencyReranker.cs b/src/AgentMemory.Neo4j/Services/MentionFrequencyReranker.cs new file mode 100644 index 000000000..74dad34d8 --- /dev/null +++ b/src/AgentMemory.Neo4j/Services/MentionFrequencyReranker.cs @@ -0,0 +1,119 @@ +using AgentMemory.Abstractions.Domain; +using AgentMemory.Abstractions.Options; +using AgentMemory.Abstractions.Services; +using AgentMemory.Neo4j.Infrastructure; +using AgentMemory.Neo4j.Queries; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Neo4j.Driver; + +namespace AgentMemory.Neo4j.Services; + +/// +/// Boosts facts the conversation keeps returning to (R7). +/// +/// +/// +/// A fact the world asserted five times is usually more central than one mentioned once. Similarity +/// cannot see that: two facts phrased alike score alike however often either was actually said. +/// +/// +/// The signal is ingestion-side, and that distinction is the whole task. It counts how often +/// the world re-asserted a fact — mention_count, incremented by the upsert paths when a +/// new ingestion re-states an existing triple. The tempting substitute, :MemoryReadAudit, +/// counts how often we surfaced it, and ranking on that is a rich-get-richer loop: whatever +/// ranks highly gets retrieved, retrieval raises the count, the count raises the rank. It would look +/// like learning and be self-reinforcement, so it is not used here and must not be. +/// +/// +/// Logarithmic, not linear. The gap between one mention and three is real; the gap between +/// thirty and thirty-two is noise, and a linear boost would let a chatty topic bury a precise answer. +/// +/// +/// One bounded query for the whole candidate set. Off by default: it changes recall ordering and +/// every recorded measurement was taken without it. +/// +/// +internal sealed class MentionFrequencyReranker : IMemoryReranker +{ + private readonly INeo4jTransactionRunner _tx; + private readonly MemoryOptions _memory; + private readonly ILogger _logger; + + public MentionFrequencyReranker( + INeo4jTransactionRunner tx, + ILogger logger, + IOptions? memory = null) + { + _tx = tx; + _logger = logger; + _memory = memory?.Value ?? new MemoryOptions(); + } + + /// + /// How much a well-attested fact may outrank a better-matching one. + /// + /// + /// At 0.15, a fact mentioned eight times gains ~1.31× while a once-mentioned one gains 1.0 — enough + /// to break a near-tie, not enough to promote a weak match over a strong one. Salience is a + /// tiebreaker among plausible answers, not a substitute for matching the question. + /// + private const double BoostWeight = 0.15; + + /// + public bool IsEnabled => _memory.MentionFrequencyReranking; + + /// + public async Task> RerankAsync( + IReadOnlyList candidates, + MemoryRerankContext context, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(candidates); + ArgumentNullException.ThrowIfNull(context); + + // mention_count is maintained on Fact only. Boosting another kind by a counter nothing + // increments would be a no-op dressed as a feature. + if (context.SectionKind != MemoryItemKind.Fact || candidates.Count < 2) return candidates; + + var mentions = await MentionCountsAsync(candidates, cancellationToken).ConfigureAwait(false); + if (mentions.Count == 0) return candidates; + + var reordered = candidates + .Select(item => new + { + Item = item, + Boosted = item.Score * (1.0 + (BoostWeight * Math.Log( + mentions.TryGetValue(item.ItemId, out var count) ? Math.Max(count, 1) : 1))), + }) + .OrderByDescending(scored => scored.Boosted) + // Retrieval rank breaks ties, so equal salience leaves the provider's order intact rather + // than an arbitrary one. + .ThenBy(scored => scored.Item.RetrievalRank) + .Select((scored, index) => scored.Item with { ContextRank = index + 1 }) + .ToList(); + + _logger.LogDebug( + "Mention-frequency rerank over {Count} candidates; most-mentioned {Max}.", + candidates.Count, mentions.Count == 0 ? 0 : mentions.Values.Max()); + + return reordered; + } + + private async Task> MentionCountsAsync( + IReadOnlyList candidates, CancellationToken cancellationToken) => + await _tx.ReadAsync(async runner => + { + var cursor = await runner.RunAsync( + FactQueries.MentionCounts, + new Dictionary + { + ["candidateIds"] = candidates.Select(item => item.ItemId).ToList(), + }).ConfigureAwait(false); + var records = await cursor.ToListAsync().ConfigureAwait(false); + return (IReadOnlyDictionary)records.ToDictionary( + r => r["candidateId"].As(), + r => r["mentions"].As(), + StringComparer.Ordinal); + }, cancellationToken).ConfigureAwait(false) ?? new Dictionary(StringComparer.Ordinal); +} diff --git a/src/AgentMemory.Neo4j/Services/Neo4jConflictDetectionService.cs b/src/AgentMemory.Neo4j/Services/Neo4jConflictDetectionService.cs index 09ee1dfe9..aa9db533d 100644 --- a/src/AgentMemory.Neo4j/Services/Neo4jConflictDetectionService.cs +++ b/src/AgentMemory.Neo4j/Services/Neo4jConflictDetectionService.cs @@ -1,9 +1,11 @@ using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using AgentMemory.Abstractions.Services; using AgentMemory.Neo4j.Infrastructure; using AgentMemory.Neo4j.Queries; using Neo4j.Driver; +using AgentMemory.Abstractions.Options; namespace AgentMemory.Neo4j.Services; /// @@ -19,13 +21,21 @@ internal sealed class Neo4jConflictDetectionService : IConflictDetectionService public Neo4jConflictDetectionService( INeo4jTransactionRunner tx, IClock clock, - ILogger logger) + ILogger logger, + IOptions? memoryOptions = null) { _tx = tx ?? throw new ArgumentNullException(nameof(tx)); _clock = clock ?? throw new ArgumentNullException(nameof(clock)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + // S2. This is the SECOND caller of the supersede query -- the offline hygiene pass, next to + // the write-time one -- and a contradiction resolved here is the same event either way. An + // integration test caught the omission; the unit tests could not, because the missing + // parameter is only rejected by the server. + _reinforceAlpha = memoryOptions?.Value.ConfidenceReinforcementAlpha ?? 0.0; } + private readonly double _reinforceAlpha; + /// public async Task DetectConflictsAsync( ConflictDetectionOptions? options = null, @@ -96,6 +106,7 @@ public async Task ResolveFactContradictionsAsync( ["loserId"] = loser.FactId, ["winnerId"] = winner.FactId, ["now"] = now, + ["reinforceAlpha"] = _reinforceAlpha, }; if (hasOwner) parameters["ownerId"] = conflict.OwnerId; var cursor = await runner.RunAsync(cypher, parameters).ConfigureAwait(false); diff --git a/src/AgentMemory.Neo4j/Services/NodeDistanceReranker.cs b/src/AgentMemory.Neo4j/Services/NodeDistanceReranker.cs new file mode 100644 index 000000000..2c7e10bdc --- /dev/null +++ b/src/AgentMemory.Neo4j/Services/NodeDistanceReranker.cs @@ -0,0 +1,170 @@ +using AgentMemory.Abstractions.Domain; +using AgentMemory.Abstractions.Options; +using AgentMemory.Abstractions.Services; +using AgentMemory.Neo4j.Infrastructure; +using AgentMemory.Neo4j.Queries; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Neo4j.Driver; + +namespace AgentMemory.Neo4j.Services; + +/// +/// Boosts candidates that sit close, in the graph, to the entity the query is about (R6). +/// +/// +/// +/// Vector similarity asks whether a row looks like the query. This asks whether it is +/// about what the query is about, and the two disagree in useful ways: a plainly-worded fact +/// hanging directly off the named entity is often the answer, while a fact that paraphrases the query +/// beautifully may concern someone else entirely. +/// +/// +/// One decay constant, not two. The boost is γ^hops using +/// — the same γ GraphRAG's hop decay +/// already uses. A second, independently-tuned constant for the same idea would drift from the first +/// and nobody would notice which one a given result came from. +/// +/// +/// Cost is bounded to two queries per section, regardless of candidate count: one to find the +/// centroid, one to measure distance to every candidate at once. This runs on the blocking recall +/// path, so a per-candidate traversal would make the reranker cost more than the retrieval it +/// reorders. +/// +/// +/// Off by default. It changes recall ordering, every recorded measurement was taken without it, and +/// the hermetic perf harness counts queries — so enabling it is a stated decision with a visible cost. +/// +/// +internal sealed class NodeDistanceReranker : IMemoryReranker +{ + private readonly INeo4jTransactionRunner _tx; + private readonly MemoryRankingOptions _ranking; + private readonly MemoryOptions _memory; + private readonly ILogger _logger; + + public NodeDistanceReranker( + INeo4jTransactionRunner tx, + ILogger logger, + IOptions? ranking = null, + IOptions? memory = null) + { + _tx = tx; + _logger = logger; + _ranking = ranking?.Value ?? MemoryRankingOptions.Default; + _memory = memory?.Value ?? new MemoryOptions(); + } + + /// Candidate ceiling for the centroid lookup, matching the ordinary over-fetch shape. + private const int CentroidTopK = 20; + + /// Hops beyond which the boost is indistinguishable from noise at any sane γ. + /// + /// At γ = 0.5 the fourth hop contributes 0.0625; the query caps at the same depth, so raising this + /// alone would silently do nothing. Kept next to the Cypher's *..4 for that reason. + /// + private const int MaxHops = 4; + + /// + /// + /// Requires the structural decay to be switched on as well as the reranker: with γ = 1 every boost + /// is 1.0, so running the two queries could not change an ordering and would be pure cost. + /// + public bool IsEnabled => _memory.NodeDistanceReranking && _ranking.StructuralDecayEnabled; + + /// + public async Task> RerankAsync( + IReadOnlyList candidates, + MemoryRerankContext context, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(candidates); + ArgumentNullException.ThrowIfNull(context); + + // Facts are the only kind this understands: the distance query walks ABOUT/RELATED_TO from an + // entity to a Fact. Silently reordering another kind by a measure that does not apply to it + // would be worse than not running. + if (context.SectionKind != MemoryItemKind.Fact) return candidates; + if (candidates.Count < 2 || context.QueryEmbedding is not { Length: > 0 }) return candidates; + + var centroidId = await FindCentroidAsync(context, cancellationToken).ConfigureAwait(false); + if (centroidId is null) return candidates; + + var hops = await MeasureHopsAsync(centroidId, candidates, cancellationToken).ConfigureAwait(false); + if (hops.Count == 0) return candidates; + + var gamma = _ranking.EffectiveStructuralDecayGamma; + + // Multiplicative on the provider score, so a candidate the retrieval already ranked badly is + // not promoted past a strong one by proximity alone -- graph closeness is evidence about + // relevance, not a replacement for it. Unreachable candidates keep their score exactly. + var reordered = candidates + .Select(item => new + { + Item = item, + Boosted = hops.TryGetValue(item.ItemId, out var h) + ? item.Score * (1.0 + Math.Pow(gamma, h)) + : item.Score, + }) + .OrderByDescending(scored => scored.Boosted) + // Provider rank breaks ties, so an unreachable candidate and a 4-hop one with equal + // boosted scores stay in the order retrieval put them, rather than an arbitrary one. + .ThenBy(scored => scored.Item.RetrievalRank) + .Select((scored, index) => scored.Item with { ContextRank = index + 1 }) + .ToList(); + + _logger.LogDebug( + "Node-distance rerank: {Reached} of {Total} candidates within {MaxHops} hops of centroid " + + "{Centroid} (gamma {Gamma}).", + hops.Count, candidates.Count, MaxHops, centroidId, gamma); + + return reordered; + } + + private async Task FindCentroidAsync( + MemoryRerankContext context, CancellationToken cancellationToken) + { + var hasOwner = context.Scope?.HasOwnerFilter == true; + var includeShared = context.Scope?.IncludeShared ?? true; + var parameters = new Dictionary + { + ["embedding"] = context.QueryEmbedding!.ToList(), + ["topK"] = CentroidTopK, + ["minScore"] = 0.0, + }; + if (hasOwner) parameters["ownerId"] = context.Scope!.OwnerId; + + return await _tx.ReadAsync(async runner => + { + var cursor = await runner.RunAsync( + NodeDistanceQueries.CentroidEntity(hasOwner, includeShared), parameters) + .ConfigureAwait(false); + var records = await cursor.ToListAsync().ConfigureAwait(false); + return records.Count == 0 ? null : records[0]["entityId"].As(); + }, cancellationToken).ConfigureAwait(false); + } + + private async Task> MeasureHopsAsync( + string centroidId, + IReadOnlyList candidates, + CancellationToken cancellationToken) + { + var parameters = new Dictionary + { + ["centroidId"] = centroidId, + ["candidateIds"] = candidates.Select(item => item.ItemId).ToList(), + ["maxHops"] = MaxHops, + }; + + return await _tx.ReadAsync(async runner => + { + var cursor = await runner.RunAsync( + NodeDistanceQueries.DistanceToCandidates, parameters).ConfigureAwait(false); + var records = await cursor.ToListAsync().ConfigureAwait(false); + return (IReadOnlyDictionary)records.ToDictionary( + r => r["candidateId"].As(), + r => r["hops"].As(), + StringComparer.Ordinal); + }, cancellationToken).ConfigureAwait(false) ?? new Dictionary(StringComparer.Ordinal); + } +} diff --git a/tests/AgentMemory.Tests.Integration/Repositories/ConfidenceReinforcementIntegrationTests.cs b/tests/AgentMemory.Tests.Integration/Repositories/ConfidenceReinforcementIntegrationTests.cs new file mode 100644 index 000000000..7f043d633 --- /dev/null +++ b/tests/AgentMemory.Tests.Integration/Repositories/ConfidenceReinforcementIntegrationTests.cs @@ -0,0 +1,166 @@ +using FluentAssertions; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using AgentMemory.Abstractions.Domain; +using AgentMemory.Abstractions.Options; +using AgentMemory.Neo4j.Repositories; +using AgentMemory.Tests.Integration.Fixtures; + +namespace AgentMemory.Tests.Integration.Repositories; + +/// +/// Confidence that earns and loses (S2), verified where the arithmetic actually runs. +/// +/// +/// +/// Confidence was set once by extraction and never moved: a fact stated five times and one stated in +/// passing carried whatever number the extractor happened to report. With α > 0 a re-asserted +/// triple gains α and a superseded one loses 2α. +/// +/// +/// Asymmetric deliberately. Being contradicted is stronger evidence against a fact than one +/// more restatement is for it — a repeated claim may just be a habit of phrasing, while a replaced +/// one is a claim the world stopped believing. +/// +/// +/// The clamping is the part that has to be checked in the database. Confidence is read by ranking, +/// dedup and decay; a value that escaped [0,1] would propagate into computations where it means +/// nothing, and Cypher arithmetic is where such an escape would happen. +/// +/// +[Collection("Neo4j Integration")] +[Trait("Category", "Integration")] +public class ConfidenceReinforcementIntegrationTests : IAsyncLifetime +{ + private readonly Neo4jIntegrationFixture _fixture; + + private const double Alpha = 0.05; + + public ConfidenceReinforcementIntegrationTests(Neo4jIntegrationFixture fixture) => _fixture = fixture; + + public Task InitializeAsync() => _fixture.CleanDatabaseAsync(); + public Task DisposeAsync() => Task.CompletedTask; + + private Neo4jFactRepository Repository(double alpha) => + new(_fixture.TransactionRunner, + NullLogger.Instance, + memoryOptions: Options.Create(new MemoryOptions { ConfidenceReinforcementAlpha = alpha })); + + private static Fact NewFact(string @object, double confidence) => new() + { + FactId = $"fact-{Guid.NewGuid():N}", + Subject = "user", + Predicate = "lives in", + Object = @object, + Confidence = confidence, + OwnerId = "alice", + CreatedAtUtc = DateTimeOffset.UtcNow, + }; + + [Fact] + public async Task ReAssertionRaisesConfidence() + { + // The world said it again, so the claim earns rather than being overwritten by whatever the + // latest extraction reported. + var repo = Repository(Alpha); + var stored = await repo.UpsertAsync(NewFact("Zurich", 0.50)); + await repo.UpsertAsync(NewFact("Zurich", 0.50)); + + var reinforced = await repo.GetByIdAsync(stored.FactId); + + reinforced!.Confidence.Should().BeApproximately(0.55, 1e-9); + } + + [Fact] + public async Task RepeatedReAssertionAccumulatesButNeverExceedsOne() + { + // The upper clamp, exercised past the boundary rather than near it. Confidence is a [0,1] + // quantity every ranking and dedup computation reads. + var repo = Repository(0.2); + var stored = await repo.UpsertAsync(NewFact("Zurich", 0.90)); + for (var i = 0; i < 5; i++) + await repo.UpsertAsync(NewFact("Zurich", 0.90)); + + var reinforced = await repo.GetByIdAsync(stored.FactId); + + reinforced!.Confidence.Should().Be(1.0); + } + + [Fact] + public async Task BeingSupersededCostsTwiceWhatCorroborationEarns() + { + // The asymmetry, measured: 0.60 - 2*0.05 = 0.50. Being contradicted is stronger evidence + // against a fact than one more restatement is for it. + var repo = Repository(Alpha); + var loser = await repo.UpsertAsync(NewFact("Basel", 0.60)); + var winner = await repo.UpsertAsync(NewFact("Zurich", 0.90)); + + await repo.SupersedeAsync(loser.FactId, winner.FactId, MemoryScope.For("alice", includeShared: false)); + + var demoted = await repo.GetByIdAsync(loser.FactId); + demoted!.Confidence.Should().BeApproximately(0.50, 1e-9); + } + + [Fact] + public async Task ConfidenceNeverGoesNegative() + { + // The lower clamp. A negative confidence would be read by ranking and decay as a number, not + // as an error, and would propagate somewhere it means nothing. + var repo = Repository(0.4); + var loser = await repo.UpsertAsync(NewFact("Basel", 0.10)); + var winner = await repo.UpsertAsync(NewFact("Zurich", 0.90)); + + await repo.SupersedeAsync(loser.FactId, winner.FactId, MemoryScope.For("alice", includeShared: false)); + + (await repo.GetByIdAsync(loser.FactId))!.Confidence.Should().Be(0.0); + } + + [Fact] + public async Task TheWinnerOfASupersessionIsUntouched() + { + // Only the loser is demoted. Moving both would make every contradiction erode the graph. + var repo = Repository(Alpha); + var loser = await repo.UpsertAsync(NewFact("Basel", 0.60)); + var winner = await repo.UpsertAsync(NewFact("Zurich", 0.90)); + + await repo.SupersedeAsync(loser.FactId, winner.FactId, MemoryScope.For("alice", includeShared: false)); + + (await repo.GetByIdAsync(winner.FactId))!.Confidence.Should().BeApproximately(0.90, 1e-9); + } + + [Fact] + public async Task AtAlphaZeroTheBehaviourIsExactlyWhatItWas() + { + // The byte-identical guarantee, and the reason the gate lives in Cypher rather than in C#: at + // alpha 0 the assignment is the original one, so every sealed measurement stands. + var repo = Repository(0.0); + var stored = await repo.UpsertAsync(NewFact("Zurich", 0.50)); + await repo.UpsertAsync(NewFact("Zurich", 0.70)); + + var after = await repo.GetByIdAsync(stored.FactId); + + after!.Confidence.Should().BeApproximately(0.70, 1e-9, + "with reinforcement off, the latest assertion's confidence replaces the stored one"); + } + + [Fact] + public async Task ADifferentFactDoesNotReinforce() + { + // Corroboration is per triple. If any write about the same subject reinforced, one chatty + // subject would drag every fact about it upward. + var repo = Repository(Alpha); + var lives = await repo.UpsertAsync(NewFact("Zurich", 0.50)); + await repo.UpsertAsync(new Fact + { + FactId = $"fact-{Guid.NewGuid():N}", + Subject = "user", + Predicate = "works at", + Object = "Acme", + Confidence = 0.5, + OwnerId = "alice", + CreatedAtUtc = DateTimeOffset.UtcNow, + }); + + (await repo.GetByIdAsync(lives.FactId))!.Confidence.Should().BeApproximately(0.50, 1e-9); + } +} diff --git a/tests/AgentMemory.Tests.Integration/Repositories/EntitySummaryIntegrationTests.cs b/tests/AgentMemory.Tests.Integration/Repositories/EntitySummaryIntegrationTests.cs new file mode 100644 index 000000000..b90d145b1 --- /dev/null +++ b/tests/AgentMemory.Tests.Integration/Repositories/EntitySummaryIntegrationTests.cs @@ -0,0 +1,171 @@ +using AgentMemory.Abstractions.Domain; +using AgentMemory.Abstractions.Options; +using AgentMemory.Neo4j.Repositories; +using AgentMemory.Tests.Integration.Fixtures; +using FluentAssertions; +using Microsoft.Extensions.Logging.Abstractions; +using Neo4j.Driver; + +namespace AgentMemory.Tests.Integration.Repositories; + +/// +/// Entity summaries in the database (S1), where the provenance edges either exist or do not. +/// +/// +/// +/// The staleness arithmetic is unit-tested. What only a live database can show is that a summary +/// actually attaches to its sources, that regenerating it does not accumulate edges to facts it no +/// longer draws on, and that one tenant's summary is invisible to another. +/// +/// +/// The provenance edges are the reason this is a graph node rather than a column. If they silently +/// failed to attach, every unit test would still pass and "which facts is this claim standing on?" +/// would quietly have no answer. +/// +/// +[Collection("Neo4j Integration")] +[Trait("Category", "Integration")] +public class EntitySummaryIntegrationTests : IAsyncLifetime +{ + private readonly Neo4jIntegrationFixture _fixture; + private readonly Neo4jEntitySummaryRepository _summaries; + private readonly Neo4jFactRepository _facts; + + public EntitySummaryIntegrationTests(Neo4jIntegrationFixture fixture) + { + _fixture = fixture; + _summaries = new Neo4jEntitySummaryRepository( + fixture.TransactionRunner, NullLogger.Instance); + _facts = new Neo4jFactRepository( + fixture.TransactionRunner, NullLogger.Instance); + } + + public Task InitializeAsync() => _fixture.CleanDatabaseAsync(); + public Task DisposeAsync() => Task.CompletedTask; + + private static readonly MemoryScope Alice = MemoryScope.For("alice", includeShared: false); + + private Task FactAsync(string id, string @object, string owner = "alice") => + _facts.UpsertAsync(new Fact + { + FactId = id, + Subject = "Alice", + Predicate = "lives in", + Object = @object, + Confidence = 0.9, + OwnerId = owner, + CreatedAtUtc = DateTimeOffset.UtcNow, + }); + + private static EntitySummary Summary( + string content, IReadOnlyList sourceIds, string owner = "alice", string id = "sum-1") => new() + { + SummaryId = id, + EntityId = "e-alice", + Content = content, + SourceFactIds = sourceIds, + SourceFingerprint = $"fp-{content.GetHashCode():X}", + OwnerId = owner, + GeneratedAtUtc = DateTimeOffset.UtcNow, + }; + + private Task ProvenanceEdgeCountAsync() => + _fixture.TransactionRunner.ReadAsync(async runner => + { + var cursor = await runner.RunAsync( + "MATCH (:EntitySummary {entity_id: 'e-alice'})-[r:EXTRACTED_FROM]->(:Fact) RETURN count(r) AS c"); + var records = await cursor.ToListAsync(); + return records.Count == 0 ? 0 : global::Neo4j.Driver.ValueExtensions.As(records[0]["c"]); + }); + + [Fact] + public async Task ASummaryRoundTrips() + { + await FactAsync("f-1", "Zurich"); + var stored = Summary("Alice lives in Zurich", ["f-1"]); + + await _summaries.UpsertAsync(stored); + var read = await _summaries.GetByEntityAsync("e-alice", Alice); + + read.Should().NotBeNull(); + read!.Content.Should().Be("Alice lives in Zurich"); + read.SourceFingerprint.Should().Be(stored.SourceFingerprint); + read.SourceFactIds.Should().Equal("f-1"); + } + + [Fact] + public async Task ProvenanceEdgesAttachToEverySourceFact() + { + // The reason this is a node in a graph. If the MERGE silently matched nothing, every unit + // test would still pass and the summary would stand on no recorded evidence at all. + await FactAsync("f-1", "Zurich"); + await FactAsync("f-2", "Basel"); + + await _summaries.UpsertAsync(Summary("two sources", ["f-1", "f-2"])); + + (await ProvenanceEdgeCountAsync()).Should().Be(2); + } + + [Fact] + public async Task RegeneratingFromFewerSourcesDropsTheStaleEdges() + { + // THE edge case, literally. Edges are rebuilt rather than merged: a summary regenerated from a + // smaller fact set must not keep pointing at sources its text no longer draws on, which would + // claim provenance the content cannot support. + await FactAsync("f-1", "Zurich"); + await FactAsync("f-2", "Basel"); + await _summaries.UpsertAsync(Summary("two sources", ["f-1", "f-2"])); + + await _summaries.UpsertAsync(Summary("one source", ["f-1"], id: "sum-2")); + + (await ProvenanceEdgeCountAsync()).Should().Be(1); + } + + [Fact] + public async Task ASecondUpsertReplacesRatherThanDuplicates() + { + await FactAsync("f-1", "Zurich"); + await _summaries.UpsertAsync(Summary("first", ["f-1"])); + await _summaries.UpsertAsync(Summary("second", ["f-1"], id: "sum-2")); + + var read = await _summaries.GetByEntityAsync("e-alice", Alice); + + read!.Content.Should().Be("second", "one summary per entity per owner"); + } + + [Fact] + public async Task AnotherOwnersSummaryIsInvisible() + { + // R1. Two tenants summarising the same entity each did so from the facts they can see, so a + // shared node would leak one owner's knowledge straight into another's context. + await FactAsync("f-1", "Zurich", owner: "bob"); + await _summaries.UpsertAsync(Summary("bob's view", ["f-1"], owner: "bob")); + + (await _summaries.GetByEntityAsync("e-alice", Alice)).Should().BeNull(); + } + + [Fact] + public async Task DeletingRemovesTheSummaryAndItsEdges() + { + await FactAsync("f-1", "Zurich"); + await _summaries.UpsertAsync(Summary("to be removed", ["f-1"])); + + var deleted = await _summaries.DeleteByEntityAsync("e-alice", Alice); + + deleted.Should().BeTrue(); + (await _summaries.GetByEntityAsync("e-alice", Alice)).Should().BeNull(); + (await ProvenanceEdgeCountAsync()).Should().Be(0); + } + + [Fact] + public async Task ASummaryOverAMissingFactStillStores() + { + // A source fact removed between synthesis and write must not fail the write. The summary is + // then short an edge, which is exactly what the fingerprint check will catch on read -- the + // failure is detected where it can be acted on rather than thrown where it cannot. + var act = async () => await _summaries.UpsertAsync(Summary("orphaned", ["f-missing"])); + + await act.Should().NotThrowAsync(); + (await _summaries.GetByEntityAsync("e-alice", Alice)).Should().NotBeNull(); + } +} diff --git a/tests/AgentMemory.Tests.Integration/Repositories/MentionCountIntegrationTests.cs b/tests/AgentMemory.Tests.Integration/Repositories/MentionCountIntegrationTests.cs new file mode 100644 index 000000000..51e34f26b --- /dev/null +++ b/tests/AgentMemory.Tests.Integration/Repositories/MentionCountIntegrationTests.cs @@ -0,0 +1,148 @@ +using FluentAssertions; +using Microsoft.Extensions.Logging.Abstractions; +using AgentMemory.Abstractions.Domain; +using AgentMemory.Abstractions.Options; +using AgentMemory.Neo4j.Queries; +using AgentMemory.Neo4j.Repositories; +using AgentMemory.Tests.Integration.Fixtures; +using Neo4j.Driver; + +namespace AgentMemory.Tests.Integration.Repositories; + +/// +/// The salience counter behind R7, verified in the database rather than in the Cypher text. +/// +/// +/// +/// mention_count must record how often the world asserted a fact. Unit tests can prove +/// the query says so; only a live database proves the MERGE actually increments on re-assertion and +/// not on, say, every write of any fact about the same subject. +/// +/// +/// The failure this guards against is silent and slow: a counter that never moves makes the reranker +/// a no-op that looks enabled, and one that moves on the wrong event makes it rank on noise. +/// +/// +[Collection("Neo4j Integration")] +[Trait("Category", "Integration")] +public class MentionCountIntegrationTests : IAsyncLifetime +{ + private readonly Neo4jIntegrationFixture _fixture; + private readonly Neo4jFactRepository _facts; + + public MentionCountIntegrationTests(Neo4jIntegrationFixture fixture) + { + _fixture = fixture; + _facts = new Neo4jFactRepository( + fixture.TransactionRunner, NullLogger.Instance); + } + + public Task InitializeAsync() => _fixture.CleanDatabaseAsync(); + public Task DisposeAsync() => Task.CompletedTask; + + private Task AssertAsync(string subject, string predicate, string @object) => + _facts.UpsertAsync(new Fact + { + FactId = $"fact-{Guid.NewGuid():N}", + Subject = subject, + Predicate = predicate, + Object = @object, + Confidence = 0.9, + OwnerId = "alice", + CreatedAtUtc = DateTimeOffset.UtcNow, + }); + + private Task MentionsOfAsync(string factId) => + _fixture.TransactionRunner.ReadAsync(async runner => + { + var cursor = await runner.RunAsync( + FactQueries.MentionCounts, new { candidateIds = new[] { factId } }); + var records = await cursor.ToListAsync(); + // Fully qualified: FluentAssertions and the Neo4j driver both offer As, and the + // ambiguity is a compile error rather than a silent wrong conversion (the precedent is + // ReasoningTracePruneIntegrationTests). + return records.Count == 0 + ? 0 + : global::Neo4j.Driver.ValueExtensions.As(records[0]["mentions"]); + }); + + [Fact] + public async Task AFirstAssertionCountsOnce() + { + var stored = await AssertAsync("user", "lives in", "Zurich"); + + (await MentionsOfAsync(stored.FactId)).Should().Be(1); + } + + [Fact] + public async Task ReAssertingTheSameFactIncrements() + { + // THE behaviour: the world said it again. The triple MERGEs to the same node, so this is the + // one moment that distinguishes a fact the conversation keeps returning to from one mentioned + // in passing. + var stored = await AssertAsync("user", "lives in", "Zurich"); + await AssertAsync("user", "lives in", "Zurich"); + await AssertAsync("user", "lives in", "Zurich"); + + (await MentionsOfAsync(stored.FactId)).Should().Be(3); + } + + [Fact] + public async Task ADifferentFactAboutTheSameSubjectDoesNotIncrement() + { + // Salience is per fact, not per subject. Counting every write about "user" would make one + // chatty subject's facts all look equally central. + var lives = await AssertAsync("user", "lives in", "Zurich"); + await AssertAsync("user", "works at", "Acme"); + await AssertAsync("user", "likes", "tea"); + + (await MentionsOfAsync(lives.FactId)).Should().Be(1); + } + + [Fact] + public async Task ADifferentObjectIsADifferentFact() + { + // "lives in Zurich" and "lives in Basel" are separate assertions, not one fact asserted twice + // -- the MERGE key includes the object precisely so contradiction stays visible. + var zurich = await AssertAsync("user", "lives in", "Zurich"); + var basel = await AssertAsync("user", "lives in", "Basel"); + + (await MentionsOfAsync(zurich.FactId)).Should().Be(1); + (await MentionsOfAsync(basel.FactId)).Should().Be(1); + } + + [Fact] + public async Task ReadingAFactNeverIncrementsIt() + { + // The loop this must not close. Retrieval must leave the counter alone, or ranking starts + // reinforcing whatever already ranks highly on evidence it generated itself. + var stored = await AssertAsync("user", "lives in", "Zurich"); + + for (var i = 0; i < 5; i++) + await _facts.GetByIdAsync(stored.FactId); + await _facts.GetBySubjectAsync("user", MemoryScope.For("alice")); + + (await MentionsOfAsync(stored.FactId)).Should().Be(1, + "reads are not mentions -- counting them would make the ranking self-reinforcing"); + } + + [Fact] + public async Task AnotherOwnersAssertionCountsSeparately() + { + // The MERGE key includes owner_key, so two tenants asserting the same sentence are two facts. + // A shared counter would leak one tenant's conversational emphasis into another's ranking. + var alice = await AssertAsync("user", "lives in", "Zurich"); + await _facts.UpsertAsync(new Fact + { + FactId = $"fact-{Guid.NewGuid():N}", + Subject = "user", + Predicate = "lives in", + Object = "Zurich", + Confidence = 0.9, + OwnerId = "bob", + CreatedAtUtc = DateTimeOffset.UtcNow, + }); + + (await MentionsOfAsync(alice.FactId)).Should().Be(1); + } +} diff --git a/tests/AgentMemory.Tests.Integration/Repositories/ValidTimeGateIntegrationTests.cs b/tests/AgentMemory.Tests.Integration/Repositories/ValidTimeGateIntegrationTests.cs new file mode 100644 index 000000000..d5869cffb --- /dev/null +++ b/tests/AgentMemory.Tests.Integration/Repositories/ValidTimeGateIntegrationTests.cs @@ -0,0 +1,157 @@ +using FluentAssertions; +using Microsoft.Extensions.Logging.Abstractions; +using AgentMemory.Abstractions.Domain; +using AgentMemory.Abstractions.Options; +using AgentMemory.Neo4j.Repositories; +using AgentMemory.Tests.Integration.Fixtures; + +namespace AgentMemory.Tests.Integration.Repositories; + +/// +/// The valid-time gate's falsifier (PLAN 1.5): with the gate off and on, results must be identical +/// unless something actually wrote a validity bound. +/// +/// +/// +/// The gate filters on valid_from/valid_until. If turning it on changes results over a +/// corpus that nobody set validity bounds on, then an unaudited writer of validity bounds exists +/// and must be found before the option ships. The audit is only as good as the inventory, so these +/// tests pin the inventory rather than assert a vague "nothing changed". +/// +/// +/// Two writers exist, both deliberate. Temporal extraction sets bounds when +/// TemporalValidityMode.Extract asks the model for them, and supersession stamps +/// valid_until when a fact is closed. The second is the one worth testing, because write-time +/// supersession (9.1) made it reachable on the ingestion path rather than only from an offline +/// hygiene pass. +/// +/// +/// The interesting result is that supersession changes nothing here: it stamps invalidated_at +/// too, and the existing transaction-clock filter already removes the fact from live recall. So the +/// valid-time gate is redundant for superseded facts and load-bearing only for facts whose +/// real-world window was stated. Worth knowing before anyone reaches for it to hide superseded data. +/// +/// +[Collection("Neo4j Integration")] +[Trait("Category", "Integration")] +public class ValidTimeGateIntegrationTests : IAsyncLifetime +{ + private readonly Neo4jIntegrationFixture _fixture; + private readonly Neo4jFactRepository _facts; + + private static readonly float[] Embedding = [0.2f, 0.4f, 0.4f, 0.2f]; + + public ValidTimeGateIntegrationTests(Neo4jIntegrationFixture fixture) + { + _fixture = fixture; + _facts = new Neo4jFactRepository( + fixture.TransactionRunner, NullLogger.Instance); + } + + public Task InitializeAsync() => _fixture.CleanDatabaseAsync(); + public Task DisposeAsync() => Task.CompletedTask; + + private Task StoreAsync( + string @object, DateTimeOffset? validFrom = null, DateTimeOffset? validUntil = null) => + _facts.UpsertAsync(new Fact + { + FactId = $"fact-{Guid.NewGuid():N}", + Subject = "user", + Predicate = "lives in", + Object = @object, + Confidence = 0.9, + OwnerId = "alice", + Embedding = Embedding, + ValidFrom = validFrom, + ValidUntil = validUntil, + CreatedAtUtc = DateTimeOffset.UtcNow, + }); + + private Task> SearchAsync(ValidTimeMode mode) => + _facts.SearchByVectorAsync( + Embedding, limit: 20, minScore: 0.0, + scope: MemoryScope.For("alice", includeShared: false), validTime: mode); + + private static string[] Objects(IReadOnlyList<(Fact Fact, double Score)> results) => + [.. results.Select(r => r.Fact.Object).OrderBy(o => o, StringComparer.Ordinal)]; + + [Fact] + public async Task WithNoValidityBoundsAnywhereBothModesReturnTheSameSet() + { + // THE falsifier. If these differ, something wrote a bound nobody audited. + await StoreAsync("Zurich"); + await StoreAsync("Basel"); + await StoreAsync("Bern"); + + Objects(await SearchAsync(ValidTimeMode.Ignore)) + .Should().Equal(Objects(await SearchAsync(ValidTimeMode.Current))); + } + + [Fact] + public async Task AnExpiredFactIsExcludedOnlyWhenTheGateIsOn() + { + // The gate doing its job, and the proof that the test above is not passing vacuously: if the + // gate never filtered anything, both assertions would hold for the wrong reason. + await StoreAsync("Zurich"); + await StoreAsync("Basel", validUntil: DateTimeOffset.UtcNow.AddDays(-1)); + + Objects(await SearchAsync(ValidTimeMode.Ignore)).Should().Equal("Basel", "Zurich"); + Objects(await SearchAsync(ValidTimeMode.Current)).Should().Equal("Zurich"); + } + + [Fact] + public async Task AFactNotYetValidIsExcludedOnlyWhenTheGateIsOn() + { + // Both bounds, not just the upper one: a valid_from in the future is the case an + // implementation that only checked valid_until would return as current. + await StoreAsync("Zurich"); + await StoreAsync("Geneva", validFrom: DateTimeOffset.UtcNow.AddDays(1)); + + Objects(await SearchAsync(ValidTimeMode.Ignore)).Should().Equal("Geneva", "Zurich"); + Objects(await SearchAsync(ValidTimeMode.Current)).Should().Equal("Zurich"); + } + + [Fact] + public async Task AnOpenEndedFactIsCurrentUnderBothModes() + { + // Null must mean "unbounded", never "not yet valid". An implementation comparing nulls with + // ordinary operators would drop every fact that never stated a window -- which is nearly all + // of them. + await StoreAsync("Zurich", validFrom: DateTimeOffset.UtcNow.AddYears(-1), validUntil: null); + + Objects(await SearchAsync(ValidTimeMode.Current)).Should().Equal("Zurich"); + } + + // ── the second writer: supersession ─────────────────────────────────── + + [Fact] + public async Task ASupersededFactIsAlreadyGoneBeforeTheGateIsConsulted() + { + // Supersession stamps valid_until AND invalidated_at, and the transaction-clock filter runs + // regardless of this option. So the gate is redundant for superseded facts and load-bearing + // only for stated real-world windows. Anyone reaching for valid-time to hide superseded data + // is solving a problem that is already solved. + var loser = await StoreAsync("Basel"); + var winner = await StoreAsync("Zurich"); + await _facts.SupersedeAsync( + loser.FactId, winner.FactId, MemoryScope.For("alice", includeShared: false)); + + Objects(await SearchAsync(ValidTimeMode.Ignore)).Should().Equal("Zurich"); + Objects(await SearchAsync(ValidTimeMode.Current)).Should().Equal("Zurich"); + } + + [Fact] + public async Task SupersessionIsTheOnlyWriterThatFiresWithoutTemporalExtraction() + { + // The inventory, asserted. Ordinary ingestion writes null bounds; only an explicitly supplied + // window (temporal extraction) or a supersession sets one. If a third writer appears, the + // first test in this class starts failing and this one says where to look. + var untouched = await StoreAsync("Zurich"); + + var stored = await _facts.GetByIdAsync(untouched.FactId); + + stored.Should().NotBeNull(); + stored!.ValidFrom.Should().BeNull("ordinary ingestion states no window"); + stored.ValidUntil.Should().BeNull(); + } +} diff --git a/tests/AgentMemory.Tests.Unit.LongMemEval/AbstentionAccuracyTests.cs b/tests/AgentMemory.Tests.Unit.LongMemEval/AbstentionAccuracyTests.cs new file mode 100644 index 000000000..b35b3a2ce --- /dev/null +++ b/tests/AgentMemory.Tests.Unit.LongMemEval/AbstentionAccuracyTests.cs @@ -0,0 +1,128 @@ +using AgentMemory.LongMemEval; +using FluentAssertions; +using Xunit; + +namespace AgentMemory.Tests.Unit.LongMemEval; + +/// +/// Meta-memory: how well the agent declines to answer what memory does not hold. +/// +/// +/// +/// This exists because two different things share the word "absent", and the difference is easy to +/// miss. The sufficiency AUC's absentCount is a ground-truth input — the label +/// retrieval confidence is ordered against — and it is identical across arms by construction. +/// Abstention accuracy is an outcome: whether the agent behaved correctly on those +/// questions. Reporting only the first invites reading a class balance as a result. +/// +/// +/// It is worth its own line because the project's taxonomy names abstention questions as the only +/// place this dataset scores meta-memory at all, and across 52 recorded runs before typed sampling +/// shipped, none had ever been drawn. +/// +/// +public sealed class AbstentionAccuracyTests +{ + private static LongMemEvalQuestionTelemetry Question(int number, string id, bool isAbstention) => + new(number, MessagesStored: 10, ItemsRetrieved: 5, RecallTruncated: false, Status: "ok") + { + QuestionId = id, + IsAbstention = isAbstention, + }; + + private static Func Verdicts(params (string Id, bool? Correct)[] verdicts) + { + var map = verdicts.ToDictionary(v => v.Id, v => v.Correct, StringComparer.Ordinal); + return id => id is not null && map.TryGetValue(id, out var v) ? v : null; + } + + [Fact] + public void TheTwoHalvesAreScoredSeparately() + { + // The whole point: an aggregate hides whether the system is good at knowing what it does not + // know, which is a different capability from retrieving what it does. + var telemetry = new[] + { + Question(1, "abs-1", isAbstention: true), + Question(2, "abs-2", isAbstention: true), + Question(3, "ord-1", isAbstention: false), + }; + + var scored = LongMemEvalAbstentionAccuracy.Score(telemetry, Verdicts( + ("abs-1", true), ("abs-2", false), ("ord-1", true))); + + scored.AbstentionCorrect.Should().Be(1); + scored.AbstentionTotal.Should().Be(2); + scored.OrdinaryCorrect.Should().Be(1); + scored.OrdinaryTotal.Should().Be(1); + } + + [Fact] + public void AnsweringWhenItShouldHaveAbstainedIsNamedForTheBehaviour() + { + // Not "incorrect": this specific wrong answer is the interesting one, because the agent + // asserted something memory did not hold. That is the failure a memory system is least able + // to detect in itself. + var telemetry = new[] + { + Question(1, "abs-1", isAbstention: true), + Question(2, "abs-2", isAbstention: true), + }; + + var scored = LongMemEvalAbstentionAccuracy.Score(telemetry, Verdicts( + ("abs-1", true), ("abs-2", false))); + + scored.AnsweredWhenItShouldHaveAbstained.Should().Be(1); + } + + [Fact] + public void NoAbstentionQuestionsMeansNotMeasuredRatherThanZero() + { + // Before typed sampling this was every run. A rate of 0 would read as "it never abstains + // correctly"; null reads as "it was never asked to", which is the truth. + var scored = LongMemEvalAbstentionAccuracy.Score( + [Question(1, "ord-1", isAbstention: false)], Verdicts(("ord-1", true))); + + scored.AbstentionAccuracy.Should().BeNull(); + scored.AbstentionTotal.Should().Be(0); + scored.OrdinaryAccuracy.Should().Be(1.0); + } + + [Fact] + public void AnUnjudgedQuestionLeavesBothDenominatorsAlone() + { + // Counting it wrong would understate every rate by however many the judge could not parse -- + // and judge-parse failures are a known, recurring class here. + var telemetry = new[] + { + Question(1, "abs-1", isAbstention: true), + Question(2, "abs-2", isAbstention: true), + }; + + var scored = LongMemEvalAbstentionAccuracy.Score(telemetry, Verdicts( + ("abs-1", true), ("abs-2", null))); + + scored.AbstentionTotal.Should().Be(1, "an unjudged question measured nothing"); + scored.AbstentionAccuracy.Should().Be(1.0); + } + + [Fact] + public void TheMeasuredRunReproduces() + { + // The 50q abstention-enriched run of 2026-08-12: 18 of 20 abstention questions correct in both + // arms, against 25/30 and 26/30 ordinary. Pinned as a shape check on the scorer -- if this + // ever computes 20/20 or 0/20 from the same inputs, the scorer changed, not the system. + var telemetry = Enumerable.Range(1, 20) + .Select(i => Question(i, $"abs-{i}", isAbstention: true)) + .Concat(Enumerable.Range(21, 30).Select(i => Question(i, $"ord-{i}", isAbstention: false))) + .ToArray(); + + var scored = LongMemEvalAbstentionAccuracy.Score(telemetry, id => + id is null ? null + : id.StartsWith("abs-", StringComparison.Ordinal) ? id is not ("abs-1" or "abs-2") + : id is not ("ord-21" or "ord-22" or "ord-23" or "ord-24" or "ord-25")); + + scored.AbstentionAccuracy.Should().BeApproximately(0.90, 1e-9); + scored.OrdinaryAccuracy.Should().BeApproximately(25.0 / 30.0, 1e-9); + } +} diff --git a/tests/AgentMemory.Tests.Unit.LongMemEval/ContextTokenBreakdownTests.cs b/tests/AgentMemory.Tests.Unit.LongMemEval/ContextTokenBreakdownTests.cs new file mode 100644 index 000000000..42f45180d --- /dev/null +++ b/tests/AgentMemory.Tests.Unit.LongMemEval/ContextTokenBreakdownTests.cs @@ -0,0 +1,126 @@ +using AgentMemory.Abstractions.Domain; +using AgentMemory.LongMemEval; +using FluentAssertions; +using Xunit; + +namespace AgentMemory.Tests.Unit.LongMemEval; + +/// +/// The token half of B3: what answering from memory costs against answering from the whole +/// conversation. +/// +/// +/// +/// This is the most persuasive number in the category precisely because it is a claim about +/// architecture. "1.6k context tokens instead of 115k" cannot be inflated by a better answer +/// model, cannot be tuned with prompt engineering, and does not move when the judge changes its mind. +/// It is either true of the assembled context or it is not. +/// +/// +/// Which is why it is counted rather than estimated, and why the counting method travels with the +/// number: a ratio derived from a chars/4 heuristic would be a claim about arithmetic wearing the +/// clothes of a claim about design. +/// +/// +public sealed class ContextTokenBreakdownTests +{ + private static readonly LongMemEvalTokenCounter Counter = new("gpt-4o"); + + private static Message M(string content) => new() + { + MessageId = $"m-{content.GetHashCode():X}", + ConversationId = "c-1", + SessionId = "s-1", + Role = "user", + Content = content, + TimestampUtc = DateTimeOffset.UnixEpoch, + }; + + private static MemoryContext Context(params Fact[] facts) => new() + { + SessionId = "s-1", + AssembledAtUtc = DateTimeOffset.UnixEpoch, + RelevantFacts = new MemoryContextSection { Items = facts }, + }; + + private static Fact F(string subject, string predicate, string @object) => new() + { + FactId = $"f-{@object}", + Subject = subject, + Predicate = predicate, + Object = @object, + Confidence = 0.9, + CreatedAtUtc = DateTimeOffset.UnixEpoch, + }; + + [Fact] + public void MemoryCostsFarLessThanTheConversationItWasBuiltFrom() + { + // THE claim. A long conversation distils to a handful of triples, and the ratio is what a + // reader is actually being asked to believe. + var history = Enumerable.Range(0, 200) + .Select(i => M($"turn {i}: a fairly ordinary sentence about the day and the weather")) + .ToList(); + + var breakdown = ContextTokenBreakdown.Measure( + Context(F("user", "lives in", "Zurich"), F("user", "works at", "Acme")), + history, Counter); + + breakdown.ContextTokens.Should().BeLessThan(breakdown.FullHistoryTokens); + breakdown.CompressionRatio.Should().BeGreaterThan(10); + } + + [Fact] + public void TheCountIsRealRatherThanEstimated() + { + // The whole reason C4 had to land before this could be published. + var breakdown = ContextTokenBreakdown.Measure( + Context(F("user", "lives in", "Zurich")), [M("hello")], Counter); + + breakdown.CountMethod.Should().NotBe(TokenCountMethodNameForEstimate); + breakdown.Encoding.Should().NotBeNullOrWhiteSpace(); + } + + private const string TokenCountMethodNameForEstimate = "Estimated"; + + [Fact] + public void SectionsAreOrderedByCost() + { + // A breakdown exists to show where the budget goes; a fixed section order buries that. + var breakdown = ContextTokenBreakdown.Measure( + Context(F("user", "lives in", "a rather long place name indeed for testing purposes")), + [M("hello")], Counter); + + breakdown.Sections.Should().BeInDescendingOrder(s => s.Tokens); + breakdown.Sections.First().Section.Should().Be("RelevantFacts"); + } + + [Fact] + public void EverySectionIsAccountedForEvenWhenEmpty() + { + // An omitted empty section reads as "this kind of memory does not exist" rather than "it + // contributed nothing here", and the two invite different conclusions about the design. + var breakdown = ContextTokenBreakdown.Measure(Context(), [M("hello")], Counter); + + breakdown.Sections.Should().HaveCount(6); + breakdown.ContextTokens.Should().Be(0); + } + + [Fact] + public void NoHistoryMeansNoRatioRatherThanAFlatteringOne() + { + // An empty denominator is not a ratio of infinity, it is an absent measurement -- and it would + // be the single most flattering way to be wrong. + ContextTokenBreakdown.Measure(Context(F("user", "lives in", "Zurich")), [], Counter) + .CompressionRatio.Should().BeNull(); + } + + [Fact] + public void AnEmptyContextAgainstRealHistoryDoesNotDivideByZero() + { + var breakdown = ContextTokenBreakdown.Measure(Context(), [M("a real turn of conversation")], Counter); + + breakdown.CompressionRatio.Should().NotBeNull(); + breakdown.CompressionRatio.Should().BePositive(); + } +} diff --git a/tests/AgentMemory.Tests.Unit.LongMemEval/JudgeVerdictProtocolWiringTests.cs b/tests/AgentMemory.Tests.Unit.LongMemEval/JudgeVerdictProtocolWiringTests.cs new file mode 100644 index 000000000..c33e1a9ba --- /dev/null +++ b/tests/AgentMemory.Tests.Unit.LongMemEval/JudgeVerdictProtocolWiringTests.cs @@ -0,0 +1,77 @@ +using AgentEval.Memory.External.LongMemEval; +using AgentEval.Memory.External.Models; +using AgentMemory.LongMemEval; +using FluentAssertions; +using Xunit; + +namespace AgentMemory.Tests.Unit.LongMemEval; + +/// +/// Making StructuredJson reachable — and impossible to use invisibly (PLAN 3.7). +/// +/// +/// +/// AgentEval's free-text verdict parser "vetoes a leading yes when the word no appears later in +/// the response", so a judge answering "yes — there is no discrepancy" is scored as a failure. +/// That is a systematic mis-scoring, and StructuredJson is the actual fix. +/// +/// +/// It was also unreachable. The protocol parameter existed on CreateOptions and every +/// call site took the default, so no run could ever select it — the dead-option shape. +/// +/// +/// The default must not move. The same AgentEval docs state results under StructuredJson are +/// not comparable with a free-text base, and every sealed base in this track is free-text. Flipping +/// the default would not produce a wrong number; it would produce a better one that silently +/// invalidates every comparison drawn against the existing runs. +/// +/// +public sealed class JudgeVerdictProtocolWiringTests +{ + private static JudgeVerdictProtocol Parse(string? value) => + LongMemEvalPreparedPairProgram.ParseJudgeProtocolForTests(value); + + [Fact] + public void TheDefaultIsFreeText() + { + // Every sealed base here was scored under FreeText. This is the assertion that stops a later + // change making the numbers better and the comparisons meaningless. + Parse(null).Should().Be(JudgeVerdictProtocol.FreeText); + Parse("").Should().Be(JudgeVerdictProtocol.FreeText); + } + + [Theory] + [InlineData("structured-json")] + [InlineData("structuredjson")] + [InlineData("json")] + [InlineData("STRUCTURED-JSON")] + public void StructuredJsonIsReachable(string value) => + Parse(value).Should().Be(JudgeVerdictProtocol.StructuredJson); + + [Fact] + public void AnUnknownProtocolIsRejectedRatherThanDefaulted() + { + // Silently falling back to FreeText would mean a run the operator believed was StructuredJson + // produced a free-text score under a StructuredJson label -- the worst of both, and invisible. + var act = () => Parse("strict"); + + act.Should().Throw().WithMessage("*free-text, structured-json*"); + } + + [Fact] + public void TheBenchmarkOptionsCarryTheChoiceThrough() + { + // The wiring, not the parsing: a protocol parsed and then dropped before CreateOptions would + // pass every test above while every run stayed free-text. + var options = LongMemEvalBenchmarkProtocol.CreateOptions( + datasetPath: "dataset.json", + questions: 10, + seed: 42, + judgeRetryAttempts: 2, + evidenceDetail: LongMemEvalEvidenceDetail.Identifiers, + maxRelevantMessages: 20, + verdictProtocol: JudgeVerdictProtocol.StructuredJson); + + options.JudgeVerdictProtocol.Should().Be(JudgeVerdictProtocol.StructuredJson); + } +} diff --git a/tests/AgentMemory.Tests.Unit.LongMemEval/LongMemEvalGraphSnapshotTraceVisibilityTests.cs b/tests/AgentMemory.Tests.Unit.LongMemEval/LongMemEvalGraphSnapshotTraceVisibilityTests.cs new file mode 100644 index 000000000..2ef28120a --- /dev/null +++ b/tests/AgentMemory.Tests.Unit.LongMemEval/LongMemEvalGraphSnapshotTraceVisibilityTests.cs @@ -0,0 +1,66 @@ +using AgentMemory.LongMemEval; +using FluentAssertions; +using Xunit; + +namespace AgentMemory.Tests.Unit.LongMemEval; + +/// +/// Making the probe able to see procedural memory at all (PLAN 6.5). +/// +/// +/// +/// LongMemEvalGraphProbe counted Entity, Fact, Preference and RELATED_TO. It was +/// label-blind to :ReasoningTrace, which meant "the corpus contains no traces" and "the +/// probe cannot see traces" produced byte-identical output — and Phase 7's procedural work was about +/// to be measured against a graph nobody had confirmed held anything to measure. +/// +/// +public sealed class LongMemEvalGraphSnapshotTraceVisibilityTests +{ + private static LongMemEvalGraphSnapshot Snapshot(int? traces = null, int? procedures = null) => + new(Entities: 10, Facts: 20, Preferences: 5, Relationships: 3, + RelationshipsWithProvenance: 3, LearnedItems: 35, LearnedItemsWithProvenance: 35, + ProvenanceEdges: 40, SourceMessages: 12, + ReasoningTraces: traces, Procedures: procedures); + + [Fact] + public void TotalLearnedStillExcludesTraces() + { + // Comparability. This number appears in every measurement sealed before the probe could see + // traces; widening its definition would make every prior build appear to have grown for a + // reason unrelated to what was extracted. + Snapshot(traces: 7).TotalLearned.Should().Be(38); + } + + [Fact] + public void TracesAreCountedSeparately() => + Snapshot(traces: 7).TotalIncludingTraces.Should().Be(45); + + [Fact] + public void NotMeasuredIsNotTheSameAsMeasuredZero() + { + // THE distinction, and the reason the counts are nullable. A manifest written before this + // change has no such field, and a non-nullable int would deserialize it to 0 -- reproducing in + // the recorded data the exact ambiguity 6.5 exists to remove. + var legacy = Snapshot(traces: null); + var measured = Snapshot(traces: 0); + + legacy.TracesMeasured.Should().BeFalse(); + legacy.HasNoTraces.Should().BeFalse("a probe that never looked has not found the corpus empty"); + + measured.TracesMeasured.Should().BeTrue(); + measured.HasNoTraces.Should().BeTrue(); + } + + [Fact] + public void AnUnmeasuredTraceCountDoesNotInflateTheTotal() => + Snapshot(traces: null).TotalIncludingTraces.Should().Be(38); + + [Fact] + public void ProceduresAreCountedApartFromTracesInGeneral() + { + // Phase 7 promotes procedures specifically; a total trace count that lumped episodes in with + // them would report a corpus as procedure-bearing when it holds none. + Snapshot(traces: 7, procedures: 2).Procedures.Should().Be(2); + } +} diff --git a/tests/AgentMemory.Tests.Unit.LongMemEval/PreparedCorpusDriftTests.cs b/tests/AgentMemory.Tests.Unit.LongMemEval/PreparedCorpusDriftTests.cs index c0456a129..c6230e0bd 100644 --- a/tests/AgentMemory.Tests.Unit.LongMemEval/PreparedCorpusDriftTests.cs +++ b/tests/AgentMemory.Tests.Unit.LongMemEval/PreparedCorpusDriftTests.cs @@ -33,7 +33,8 @@ private static LongMemEvalPreparationManifest Prepared( string datasetSha = "dataset-sha", int seed = 42, int questions = 2, - IReadOnlyList? memoryTypes = null) => + IReadOnlyList? memoryTypes = null, + string abstention = "AsSampled") => LongMemEvalPreparationManifest.Create( preparationId: "prep-1", datasetSha256: datasetSha, @@ -57,6 +58,7 @@ private static LongMemEvalPreparationManifest Prepared( preparedAtUtc: "2026-08-10T09:26:14Z", description: "the stratified 50", memoryTypes: memoryTypes ?? [], + abstentionPolicy: abstention, questionSeed: seed); private static PreparedCorpusIdentity Current( @@ -65,7 +67,8 @@ private static PreparedCorpusIdentity Current( string datasetSha = "dataset-sha", int seed = 42, int questions = 2, - IReadOnlyList? memoryTypes = null) => new() + IReadOnlyList? memoryTypes = null, + string abstention = "AsSampled") => new() { DatasetSha256 = datasetSha, ExtractionModelId = extractionModel, @@ -79,6 +82,7 @@ private static PreparedCorpusIdentity Current( QuestionSeed = seed, QuestionCount = questions, MemoryTypes = memoryTypes ?? [], + AbstentionPolicy = abstention, }; [Fact] @@ -142,15 +146,45 @@ public void AnEpisodicSampleCannotSilentlyReuseAStratifiedCorpus() [Fact] public void AnOlderManifestThatRecordedNothingIsDriftNotAgreement() { - // Schema 5 and earlier did not record assistantContent at all, so the field comes back empty. - // "We do not know how this was built" must never read as "it matches" -- treating unknown as - // equal is precisely how a check stops being able to fail. - var legacy = Prepared() with { AssistantContent = "", ExtractionVocabularySha256 = "" }; + // Schema 5 and earlier carried none of the ingestion-identity fields, so deserialising one + // fills them from the record's DEFAULTS -- "Ignore", "Batch", false. Those are plausible + // values, and comparing against them reported a corpus built with Utterance as MATCHING a run + // configured for Ignore. Unknown silently reading as agreement is the one thing this check + // exists to prevent, and it survived the first version because the two defaults coincided. + var legacy = Prepared(assistantContent: "Ignore") with { SchemaVersion = 5 }; - var drift = LongMemEvalPreparedCorpusDrift.Compare(legacy, Current()); + var drift = LongMemEvalPreparedCorpusDrift.Compare(legacy, Current(assistantContent: "Ignore")); - drift.Select(d => d.Field) - .Should().Contain("assistantContent").And.Contain("extractionVocabularySha256"); + drift.Select(d => d.Field).Should() + .Contain("assistantContent", "the default must not stand in for a value never recorded") + .And.Contain("extractionProvenance") + .And.Contain("usePredicateVocabulary") + .And.Contain("questionSeed"); + + // memoryTypes is the exception, and deliberately so: empty means "every type", and a corpus + // built before typed sampling existed genuinely IS an all-types corpus. Empty-against-empty + // is a real match here rather than an unrecorded value, which is why an episodic run against + // the same legacy corpus still drifts (asserted separately). + drift.Select(d => d.Field).Should().NotContain("memoryTypes"); + } + + [Fact] + public void ALegacyCorpusStillDriftsAgainstATypedRun() + { + // The other half of the memoryTypes exception: "all types" matching "all types" must not also + // mean it matches an episodic sample, whose questions it never ingested. + var legacy = Prepared() with { SchemaVersion = 5 }; + + LongMemEvalPreparedCorpusDrift.Compare(legacy, Current(memoryTypes: ["episodic"])) + .Select(d => d.Field).Should().Contain("memoryTypes"); + } + + [Fact] + public void ACurrentManifestIsStillComparedOnItsRealValues() + { + // The other side of that: schema 6 records these for real, so a matching corpus must still + // report clean rather than being blanketed as unrecorded. + LongMemEvalPreparedCorpusDrift.Compare(Prepared(), Current()).Should().BeEmpty(); } [Fact] @@ -208,4 +242,86 @@ public void TheDescriptionDoesNotChangeTheFingerprint() LongMemEvalPreparationManifest.ComputeFingerprint(b).Should().Be(a.Fingerprint); } + + // ── older corpora must stay readable ────────────────────────────────── + + [Fact] + public void AManifestSealedUnderAnOlderSchemaStillVerifies() + { + // THE regression this caught on real data. Bumping CurrentSchemaVersion to 6 made + // VerifyIntegrity reject every corpus sealed at schema 5 -- each one a 7-9 hour build -- + // which is the precise opposite of what recording ingestion identity is for. Older schemas + // are read with the field set they were written under; only NEWER ones are refused. + var current = Prepared(); + var legacy = current with { SchemaVersion = 5 }; + var legacySealed = legacy with + { + Fingerprint = LongMemEvalPreparationManifest.ComputeFingerprint(legacy), + }; + + var verify = () => legacySealed.VerifyIntegrity(); + + verify.Should().NotThrow(); + } + + [Fact] + public void ALegacyManifestHashesDifferentlyFromACurrentOne() + { + // The two field sets must stay genuinely separate: if the legacy path silently included the + // schema-6 fields, an old manifest would fail to verify for a reason nobody could see. + var current = Prepared(); + var legacy = current with { SchemaVersion = 5 }; + + LongMemEvalPreparationManifest.ComputeFingerprint(legacy) + .Should().NotBe(LongMemEvalPreparationManifest.ComputeFingerprint(current)); + } + + [Fact] + public void AManifestFromANewerBuildIsRefused() + { + // It was written by a build that knew things this one does not, so its fingerprint cannot be + // recomputed here and its extra fields would be dropped in silence. + var future = Prepared() with { SchemaVersion = 99 }; + + var verify = () => future.VerifyIntegrity(); + + verify.Should().Throw().WithMessage("*newer build*"); + } + + [Fact] + public void ATamperedManifestStillFailsAtEverySchema() + { + // Version tolerance must not become "any fingerprint will do". The point of the check is that + // a manifest matches its own contents. + var tampered = Prepared() with { Fingerprint = "not-the-real-hash" }; + + ((Action)(() => tampered.VerifyIntegrity())).Should().Throw(); + ((Action)(() => (tampered with { SchemaVersion = 5 }).VerifyIntegrity())) + .Should().Throw(); + } + + // ── abstention: the class the AUC has to order against ──────────────── + + [Fact] + public void AbstentionPolicyIsIngestionAffectingAndCompared() + { + // Not merely evaluation-affecting: an abstention question still carries a conversation + // history, so including one changes which histories were ingested. A corpus built without + // them cannot serve a run that expects them. + LongMemEvalPreparedCorpusDrift.Compare( + Prepared(abstention: "AsSampled"), Current(abstention: "TargetProportion")) + .Select(d => d.Field).Should().Contain("abstention"); + } + + [Fact] + public void ALegacyCorpusReportsAbstentionAsUnrecordedRatherThanAsSampled() + { + // "AsSampled" is the record default, so a schema-5 manifest would otherwise claim to have + // been built with it -- unknown reading as agreement, the failure this whole check exists to + // prevent, reappearing through the newest field. + var legacy = Prepared() with { SchemaVersion = 5 }; + + LongMemEvalPreparedCorpusDrift.Compare(legacy, Current(abstention: "AsSampled")) + .Select(d => d.Field).Should().Contain("abstention"); + } } diff --git a/tests/AgentMemory.Tests.Unit.LongMemEval/ProceduralBenefitHarnessTests.cs b/tests/AgentMemory.Tests.Unit.LongMemEval/ProceduralBenefitHarnessTests.cs new file mode 100644 index 000000000..2803dbb75 --- /dev/null +++ b/tests/AgentMemory.Tests.Unit.LongMemEval/ProceduralBenefitHarnessTests.cs @@ -0,0 +1,153 @@ +using AgentMemory.LongMemEval; +using FluentAssertions; +using Xunit; + +namespace AgentMemory.Tests.Unit.LongMemEval; + +/// +/// Whether procedural memory helps an agent repeat a task — and refusing to say yes for the wrong +/// reason (7.6). +/// +/// +/// +/// LongMemEval cannot answer this. It scores answers about a transcript; a promoted procedure +/// changes how an agent works across repeated attempts. With no trace nodes and a probe that +/// was label-blind to them, a promoted procedure was invisible to every instrument here — the feature +/// could not fail, which is the defect shape this harness exists to prevent. +/// +/// +/// The harness is driven by a scripted runner, so the thing that can be quietly wrong is tested +/// without buying the thing that costs money. +/// +/// +public sealed class ProceduralBenefitHarnessTests +{ + /// A scripted agent: outcomes are decided by the test, not by a model. + private sealed class ScriptedRunner(Func script) : IAgentTaskRunner + { + public readonly List<(bool Enabled, int Attempt)> Calls = []; + + public Task RunAsync( + string taskId, bool procedureMemoryEnabled, int attempt, CancellationToken cancellationToken = default) + { + Calls.Add((procedureMemoryEnabled, attempt)); + return Task.FromResult(script(procedureMemoryEnabled, attempt)); + } + } + + private static Task MeasureAsync( + Func script, int attempts = 4) => + ProceduralBenefitResult.MeasureAsync(new ScriptedRunner(script), "book-a-trip", attempts); + + [Fact] + public async Task AGenuineBenefitIsReported() + { + // The happy path: procedures cut the work and the task still gets done every time. + var result = await MeasureAsync((enabled, attempt) => + enabled && attempt > 1 + ? new AgentTaskRun(true, Steps: 3, ToolCalls: 2) + : new AgentTaskRun(true, Steps: 9, ToolCalls: 7)); + + result.ShowsBenefit.Should().BeTrue(); + result.StepReduction.Should().BeGreaterThan(0); + result.ImprovedWithRepetition.Should().BeTrue(); + } + + [Fact] + public async Task GivingUpSoonerIsNotABenefit() + { + // THE test. An agent that abandons the task takes fewer steps and makes fewer tool calls, and + // both read as an improvement in every efficiency measure. Completion rate is what stops that + // being reported as a win. + var result = await MeasureAsync((enabled, _) => + enabled + ? new AgentTaskRun(Completed: false, Steps: 2, ToolCalls: 1) + : new AgentTaskRun(Completed: true, Steps: 9, ToolCalls: 7)); + + result.CompletionRateDelta.Should().BeLessThan(0); + result.ShowsBenefit.Should().BeFalse("fewer steps is not a saving if the task is not done"); + } + + [Fact] + public async Task EvenASmallCorrectnessLossDisqualifiesTheClaim() + { + // Not a hair's tolerance. An agent with the wrong procedure executes confidently, so a + // completion rate that slipped is a safety finding, not a rounding error to trade away. + var result = await MeasureAsync((enabled, attempt) => + enabled && attempt == 4 + ? new AgentTaskRun(false, 2, 1) + : new AgentTaskRun(true, enabled ? 3 : 9, enabled ? 2 : 7)); + + result.StepReduction.Should().BeGreaterThan(0); + result.ShowsBenefit.Should().BeFalse(); + } + + [Fact] + public async Task StartingCheaperIsNotLearning() + { + // An arm that is fast from attempt one did not learn a procedure during the run -- something + // else separates the arms, and reporting it as a memory benefit would be wrong. + var result = await MeasureAsync((enabled, _) => + new AgentTaskRun(true, enabled ? 3 : 9, enabled ? 2 : 7)); + + result.StepReduction.Should().BeGreaterThan(0); + result.ImprovedWithRepetition.Should().BeFalse(); + } + + [Fact] + public async Task FailedAttemptsDoNotDragDownTheStepAverage() + { + // Averaged over completed runs only: mixing "solved it in three steps" with "gave up after + // three" treats abandonment as a cheap success. + var result = await MeasureAsync((enabled, attempt) => + enabled && attempt == 1 + ? new AgentTaskRun(false, Steps: 1, ToolCalls: 0) + : new AgentTaskRun(true, Steps: 4, ToolCalls: 3)); + + result.WithProcedures.MeanStepsWhenCompleted.Should().Be(4); + } + + [Fact] + public async Task AttemptsRunInOrderAndSequentially() + { + // The hypothesis is that attempt N benefits from what attempt N-1 stored. Running them + // concurrently would race the write the next read depends on, measuring a feature that had + // not happened yet. + var runner = new ScriptedRunner((_, _) => new AgentTaskRun(true, 5, 4)); + + await ProceduralBenefitResult.MeasureAsync(runner, "book-a-trip", attempts: 3); + + runner.Calls.Where(c => c.Enabled).Select(c => c.Attempt).Should().Equal(1, 2, 3); + runner.Calls.Where(c => !c.Enabled).Select(c => c.Attempt).Should().Equal(1, 2, 3); + } + + [Fact] + public async Task BothArmsAreRun() + { + var runner = new ScriptedRunner((_, _) => new AgentTaskRun(true, 5, 4)); + + var result = await ProceduralBenefitResult.MeasureAsync(runner, "book-a-trip", attempts: 2); + + result.WithProcedures.Runs.Should().HaveCount(2); + result.WithoutProcedures.Runs.Should().HaveCount(2); + } + + [Fact] + public async Task ASingleAttemptIsRejected() + { + // One attempt per arm cannot show learning at all -- there is no run 1 versus run N. + var act = async () => await ProceduralBenefitResult.MeasureAsync( + new ScriptedRunner((_, _) => new AgentTaskRun(true, 1, 1)), "t", attempts: 1); + + await act.Should().ThrowAsync(); + } + + [Fact] + public async Task AnArmThatNeverCompletesReportsZeroRatherThanDividingByZero() + { + var result = await MeasureAsync((_, _) => new AgentTaskRun(false, 3, 2)); + + result.WithProcedures.MeanStepsWhenCompleted.Should().Be(0); + result.ShowsBenefit.Should().BeFalse(); + } +} diff --git a/tests/AgentMemory.Tests.Unit.LongMemEval/ProcedureRetrievalPrecisionTests.cs b/tests/AgentMemory.Tests.Unit.LongMemEval/ProcedureRetrievalPrecisionTests.cs new file mode 100644 index 000000000..593cdf3a9 --- /dev/null +++ b/tests/AgentMemory.Tests.Unit.LongMemEval/ProcedureRetrievalPrecisionTests.cs @@ -0,0 +1,113 @@ +using AgentMemory.LongMemEval; +using FluentAssertions; +using Xunit; + +namespace AgentMemory.Tests.Unit.LongMemEval; + +/// +/// Whether the retrieved procedure was the right one, not merely a fast one (PLAN 7.7). +/// +/// +/// +/// An agent with no procedural memory investigates; an agent with the wrong one executes. The +/// second is slower to detect and more expensive to undo, and every efficiency measure a promotion +/// feature has — hit rate, steps, latency — improves when the retriever becomes more willing to +/// answer, including when it becomes more willing to answer wrongly. +/// +/// +public sealed class ProcedureRetrievalPrecisionTests +{ + private static ProcedureRetrievalCase Case( + string taskId, string[] retrieved, params string[] correct) => + new(taskId, retrieved, correct); + + [Fact] + public void TheTopResultIsWhatCounts() + { + // An agent acts on the first procedure it is handed. A correct one at rank 3 is a ranking + // signal, not a success -- the agent already executed the wrong plan. + var scored = ProcedureRetrievalPrecision.Score( + [Case("t-1", ["wrong-a", "wrong-b", "right"], "right")]); + + scored.CorrectAtOne.Should().Be(0); + scored.WrongAtOne.Should().Be(1); + scored.MeanReciprocalRank.Should().BeApproximately(1d / 3, 1e-9); + } + + [Fact] + public void AbstentionIsNotAFailure() + { + // THE distinction. Returning nothing makes the agent investigate: slower, and safe. Counting + // it with wrong answers would make a cautious retriever indistinguishable from a reckless one. + var scored = ProcedureRetrievalPrecision.Score([Case("t-1", [], "right")]); + + scored.Abstained.Should().Be(1); + scored.WrongAtOne.Should().Be(0); + scored.WrongProcedureRate.Should().Be(0); + } + + [Fact] + public void AnsweringWhenNothingAppliesIsWrong() + { + // No stored procedure fits this task, so any confident retrieval is the error -- this is the + // case where an agent executes a plan built for a different problem entirely. + var scored = ProcedureRetrievalPrecision.Score([Case("t-1", ["some-procedure"])]); + + scored.WrongAtOne.Should().Be(1); + scored.MeanReciprocalRank.Should().Be(0); + } + + [Fact] + public void PrecisionAndPrecisionWhenAnsweringMoveApart() + { + // Why both are reported. A retriever that answers 2 of 4 tasks and gets both right has a + // precision@1 of 0.5 and is perfectly accurate whenever it commits. Reporting either number + // alone hides the trade the caution is buying. + var scored = ProcedureRetrievalPrecision.Score( + [ + Case("t-1", ["right-1"], "right-1"), + Case("t-2", ["right-2"], "right-2"), + Case("t-3", []), + Case("t-4", [], "right-4"), + ]); + + scored.PrecisionAtOne.Should().Be(0.5); + scored.PrecisionWhenAnswering.Should().Be(1.0); + scored.AbstentionRate.Should().Be(0.5); + } + + [Fact] + public void MoreThanOneProcedureCanBeCorrect() + { + // Tasks legitimately have several valid approaches; scoring against a single gold id would + // count a correct alternative as a safety failure. + ProcedureRetrievalPrecision.Score([Case("t-1", ["b"], "a", "b")]) + .CorrectAtOne.Should().Be(1); + } + + [Fact] + public void AnEmptyRunScoresZeroRatherThanDividingByZero() + { + var scored = ProcedureRetrievalPrecision.Score([]); + + scored.Total.Should().Be(0); + scored.PrecisionAtOne.Should().Be(0); + scored.WrongProcedureRate.Should().Be(0); + scored.PrecisionWhenAnswering.Should().Be(0); + } + + [Fact] + public void TheThreeOutcomesAccountForEveryTask() + { + // No task may fall through the classification: a total that does not reconcile would let a + // silently dropped case look like an improved wrong-procedure rate. + var scored = ProcedureRetrievalPrecision.Score( + [ + Case("t-1", ["right"], "right"), + Case("t-2", ["wrong"], "right"), + Case("t-3", []), + ]); + + (scored.CorrectAtOne + scored.WrongAtOne + scored.Abstained).Should().Be(scored.Total); + } +} diff --git a/tests/AgentMemory.Tests.Unit.LongMemEval/RefusedEvidenceTests.cs b/tests/AgentMemory.Tests.Unit.LongMemEval/RefusedEvidenceTests.cs new file mode 100644 index 000000000..85f8ca2f1 --- /dev/null +++ b/tests/AgentMemory.Tests.Unit.LongMemEval/RefusedEvidenceTests.cs @@ -0,0 +1,189 @@ +using AgentMemory.LongMemEval; +using FluentAssertions; +using Xunit; + +namespace AgentMemory.Tests.Unit.LongMemEval; + +/// +/// Whether a content refusal cost a question its answer, or merely some context. +/// +/// +/// +/// A refused source session is skipped so the build survives — 3 of 2,386 sessions is well inside +/// tolerance. But tolerance is a statement about volume, not about consequence. If the +/// refused session held a question's gold evidence, that question is unanswerable from memory for a +/// reason unrelated to retrieval, and counting it wrong attributes a content-policy decision to +/// recall quality. +/// +/// +/// Recording that sessions were refused is not enough. What matters is whether they mattered. +/// +/// +public sealed class RefusedEvidenceTests +{ + private static LongMemEvalMessageOrigin Message(int sourceSession, bool hasAnswer) => + new( + MessageOrdinal: 0, + SourceSessionId: $"s{sourceSession}", + SourceSessionOrdinal: sourceSession, + SourceTurnOrdinal: 0, + SourceTimestamp: "2026-01-01", + Role: "user", + FormattedContent: "", + IsSyntheticBoundary: false, + IsSyntheticFormatterPadding: false, + HasAnswer: hasAnswer); + + private static LongMemEvalEvidenceQuestion Question( + string id, params LongMemEvalMessageOrigin[] messages) => + new( + QuestionId: id, + QuestionType: "single-session-user", + Question: "q", + InvocationPrompt: "q", + GoldAnswer: "a", + QuestionDate: "2026-01-01", + IsAbstention: false, + AnswerSessionIds: new HashSet(StringComparer.Ordinal), + AnnotatedGoldTurnCount: 1, + Messages: messages); + + private static string SessionId(int question, int source) => + $"longmemeval-prepared-20260812T140253Z-session-{question:D4}-source-{source:D4}"; + + [Fact] + public void ARefusedSessionHoldingGoldEvidenceIsFlagged() + { + // THE case. This question can no longer be answered from memory, and nothing about that is a + // retrieval failure. + var questions = new[] { Question("q-1", Message(25, hasAnswer: true)) }; + + var analysed = LongMemEvalRefusedEvidence.Analyse([SessionId(1, 25)], questions); + + analysed.Should().ContainSingle(); + analysed[0].HeldGoldEvidence.Should().BeTrue(); + analysed[0].QuestionId.Should().Be("q-1"); + } + + [Fact] + public void ARefusedSessionHoldingOnlyContextIsNotFlagged() + { + // The common and genuinely harmless case: a conversation the question never depended on. It + // is still reported by id, because "harmless" is a judgement the reader should be able to + // check rather than one buried in a filter. + var questions = new[] { Question("q-1", Message(25, hasAnswer: false)) }; + + var analysed = LongMemEvalRefusedEvidence.Analyse([SessionId(1, 25)], questions); + + analysed[0].HeldGoldEvidence.Should().BeFalse(); + analysed[0].QuestionId.Should().Be("q-1"); + } + + [Fact] + public void TheGoldCheckIsScopedToTheRefusedSourceSession() + { + // A question whose answer lives in session 30 is untouched by a refusal of session 25. Without + // the ordinal comparison every refusal would look catastrophic for its whole question. + var questions = new[] + { + Question("q-1", Message(25, hasAnswer: false), Message(30, hasAnswer: true)), + }; + + LongMemEvalRefusedEvidence.Analyse([SessionId(1, 25)], questions)[0] + .HeldGoldEvidence.Should().BeFalse(); + } + + [Fact] + public void TheQuestionIndexInTheIdIsOneBased() + { + // Preparation numbers questions from 1 and the id carries that. An off-by-one here would + // attribute every refusal to the neighbouring question -- and still produce a plausible + // report. + var questions = new[] + { + Question("q-1", Message(3, hasAnswer: true)), + Question("q-2", Message(3, hasAnswer: true)), + }; + + LongMemEvalRefusedEvidence.Analyse([SessionId(2, 3)], questions)[0] + .QuestionId.Should().Be("q-2"); + } + + [Fact] + public void SyntheticFormatterMessagesNeverCountAsGoldEvidence() + { + // Session boundaries and padding are the harness's own text. Treating one as gold would report + // a refusal as costing an answer that never lived there. + var padding = Message(25, hasAnswer: true) with { IsSyntheticFormatterPadding = true }; + var boundary = Message(26, hasAnswer: true) with { IsSyntheticBoundary = true }; + var questions = new[] { Question("q-1", padding, boundary) }; + + var analysed = LongMemEvalRefusedEvidence.Analyse( + [SessionId(1, 25), SessionId(1, 26)], questions); + + analysed.Should().OnlyContain(r => !r.HeldGoldEvidence); + } + + [Fact] + public void AnUnparseableIdIsUnknownRatherThanHarmless() + { + // The id shape is a convention, not a contract. If it changes, the analysis must degrade to + // "we cannot tell" -- reported, with no question attached -- rather than silently answering + // "nothing was lost". + var analysed = LongMemEvalRefusedEvidence.Analyse(["some-other-session-naming"], []); + + analysed.Should().ContainSingle(); + analysed[0].QuestionId.Should().BeNull(); + analysed[0].SessionId.Should().Be("some-other-session-naming"); + } + + [Fact] + public void AQuestionIndexOutsideTheSampleIsUnknownToo() + { + var analysed = LongMemEvalRefusedEvidence.Analyse([SessionId(99, 3)], [Question("q-1")]); + + analysed[0].QuestionId.Should().BeNull(); + analysed[0].HeldGoldEvidence.Should().BeFalse(); + } + + [Fact] + public void NoCompromisedQuestionsProducesNoValidationIssue() + { + // A clean run must stay clean. An issue raised on every refusal would train the reader to + // ignore it, which is how the one that matters gets missed. + var questions = new[] { Question("q-1", Message(25, hasAnswer: false)) }; + var analysed = LongMemEvalRefusedEvidence.Analyse([SessionId(1, 25)], questions); + + LongMemEvalRefusedEvidence.DescribeCompromisedQuestions(analysed).Should().BeNull(); + } + + [Fact] + public void ACompromisedQuestionNamesItselfAndTheConsequence() + { + // The message has to say what changes about reading the number, not just that something + // happened -- the count is already reported elsewhere. + var questions = new[] { Question("q-7", Message(25, hasAnswer: true)) }; + var analysed = LongMemEvalRefusedEvidence.Analyse([SessionId(1, 25)], questions); + + var issue = LongMemEvalRefusedEvidence.DescribeCompromisedQuestions(analysed); + + issue.Should().NotBeNull(); + issue.Should().Contain("q-7").And.Contain("unrelated to retrieval"); + } + + [Fact] + public void EachCompromisedQuestionIsNamedOnce() + { + // Two refused sessions of the same question is one compromised question, not two. A count + // that double-reported would overstate the damage. + var questions = new[] + { + Question("q-1", Message(3, hasAnswer: true), Message(4, hasAnswer: true)), + }; + var analysed = LongMemEvalRefusedEvidence.Analyse( + [SessionId(1, 3), SessionId(1, 4)], questions); + + LongMemEvalRefusedEvidence.DescribeCompromisedQuestions(analysed) + .Should().Contain("1 question(s)"); + } +} diff --git a/tests/AgentMemory.Tests.Unit.LongMemEval/SufficiencyLabellingTests.cs b/tests/AgentMemory.Tests.Unit.LongMemEval/SufficiencyLabellingTests.cs new file mode 100644 index 000000000..84b52d415 --- /dev/null +++ b/tests/AgentMemory.Tests.Unit.LongMemEval/SufficiencyLabellingTests.cs @@ -0,0 +1,127 @@ +using AgentMemory.LongMemEval; +using FluentAssertions; +using Xunit; + +namespace AgentMemory.Tests.Unit.LongMemEval; + +/// +/// Where "was this question answerable?" comes from — and why it is not the presence gate. +/// +/// +/// +/// The sufficiency AUC asks whether retrieval confidence ranks answerable questions above +/// unanswerable ones, so it needs a ground truth for answerability. The first version used the +/// answer-presence gate, which asks whether the gold answer's distinctive tokens appear anywhere in +/// memory — a deliberately cheap floor for spotting extraction failure. +/// +/// +/// Measured on an abstention-enriched sample, that gate reported 3 of 4 abstention questions as +/// "present". Their topic is discussed in the conversation even though the specific fact is +/// not, so the tokens are there. A heuristic floor for extraction failure is not a ground truth for +/// answerability, and using it as one put most of the unanswerable class on the wrong side of every +/// comparison. +/// +/// +/// The dataset already knows: an _abs question is unanswerable by definition. That label is +/// free, exact, and needs no gate. +/// +/// +public sealed class SufficiencyLabellingTests +{ + private static LongMemEvalQuestionTelemetry Question( + int number, + double? signal, + bool isAbstention = false, + bool checkable = true, + bool present = true) => + new(number, MessagesStored: 10, ItemsRetrieved: 5, RecallTruncated: false, Status: "ok") + { + SufficiencySignal = signal, + IsAbstention = isAbstention, + // (Checkable, Present) -- in that order. Getting it backwards here made a checkable + // missing answer read as uncheckable, and the test caught it. + AnswerPresence = new LongMemEvalAnswerPresenceResult(checkable, present, [], 0), + }; + + private static dynamic Report(params LongMemEvalQuestionTelemetry[] questions) => + LongMemEvalSufficiencyReport.From(questions); + + [Fact] + public void AnAbstentionQuestionCountsAsUnanswerableEvenWhenTheGateSaysPresent() + { + // THE case, and it is the common one: 3 of 4 abstention questions in a real run had their + // gold answer's tokens present in memory. Deferring to the gate put them on the answerable + // side and destroyed the class the AUC exists to order against. + var report = Report( + Question(1, 0.5, isAbstention: true, present: true), + Question(2, 0.9, present: true)); + + ((int)report.presentCount).Should().Be(1); + ((int)report.absentCount).Should().Be(1); + ((double?)report.auc).Should().Be(1.0, "the answerable question scored higher"); + } + + [Fact] + public void AnAbstentionQuestionNeedsNoCheckableGoldAnswer() + { + // Its unanswerability is a dataset fact, not a measurement. Requiring the gate to confirm it + // would drop exactly the questions that make the metric computable -- an abstention answer is + // often phrased in common words with no distinctive tokens to find. + var report = Report( + Question(1, 0.5, isAbstention: true, checkable: false, present: false), + Question(2, 0.9, present: true)); + + ((int)report.absentCount).Should().Be(1); + ((int)report.abstentionQuestions).Should().Be(1); + } + + [Fact] + public void AnOrdinaryQuestionStillNeedsTheGate() + { + // The other half. For a normal question the dataset asserts nothing about whether OUR + // extraction stored the answer, so the gate remains the only evidence -- and an unmeasurable + // one is excluded rather than guessed. + var report = Report( + Question(1, 0.9, present: true), + Question(2, 0.5, checkable: false, present: false), + Question(3, 0.4, present: false)); + + ((int)report.presentCount).Should().Be(1); + ((int)report.absentCount).Should().Be(1); + ((int)report.excludedNotCheckable).Should().Be(1); + } + + [Fact] + public void AnAbstentionQuestionIsNeverCountedAsUncheckable() + { + // It was not excluded, so reporting it under an exclusion count would make the denominators + // stop adding up -- and the denominators are the only way to judge whether the AUC means + // anything. + var report = Report(Question(1, 0.5, isAbstention: true, checkable: false)); + + ((int)report.excludedNotCheckable).Should().Be(0); + } + + [Fact] + public void TheAbstentionCountIsReportedSoAZeroIsVisible() + { + // When it is 0 the unanswerable class came entirely from accidental extraction misses, which + // is how a 50-question run ended up with ONE absent observation and two arms reporting 0.969 + // and 0.094 from the same corpus. A reader must be able to see that without recounting. + ((int)Report(Question(1, 0.9), Question(2, 0.4, present: false)).abstentionQuestions) + .Should().Be(0); + } + + [Fact] + public void AQuestionWithNoSignalIsExcludedWhateverItsLabel() + { + // Diagnostics off means the signal was never collected. An abstention label cannot substitute + // for the measurement the comparison is actually about. + var report = Report( + Question(1, null, isAbstention: true), + Question(2, 0.9, present: true)); + + ((int)report.absentCount).Should().Be(0); + ((int)report.excludedNoSignal).Should().Be(1); + } +} diff --git a/tests/AgentMemory.Tests.Unit/AgentFramework/HistoryThreadDedupTests.cs b/tests/AgentMemory.Tests.Unit/AgentFramework/HistoryThreadDedupTests.cs new file mode 100644 index 000000000..28dfb3e41 --- /dev/null +++ b/tests/AgentMemory.Tests.Unit/AgentFramework/HistoryThreadDedupTests.cs @@ -0,0 +1,177 @@ +using AgentMemory.Abstractions.Domain; +using AgentMemory.AgentFramework; +using AgentMemory.AgentFramework.Mapping; +using FluentAssertions; +using Microsoft.Extensions.AI; +using Xunit; + +namespace AgentMemory.Tests.Unit.AgentFramework; + +/// +/// Recalled chat history the host is already sending must not be sent twice (PLAN 2.5). +/// +/// +/// +/// The provider receives the full live thread and, until now, discarded it — so recall's +/// RecentMessages re-sent turns the model was already being given, and the host paid for both +/// copies. +/// +/// +/// The role trap is the interesting part. RecalledMessageRoleGate rewrites a recalled +/// message's role — privileged down to user below the trust threshold — while leaving its +/// content identical. A role-keyed fingerprint therefore matches nothing on precisely the hosts that +/// hardened MinimumTrustForSystemRole: the feature would work everywhere except the +/// security-conscious configuration, and look correct in every test that did not set that option. +/// +/// +public sealed class HistoryThreadDedupTests +{ + private static Message Recalled(string content, string role = "user") => new() + { + MessageId = $"m-{content.GetHashCode():X}", + ConversationId = "c-1", + SessionId = "s-1", + Role = role, + Content = content, + TimestampUtc = DateTimeOffset.UnixEpoch, + }; + + private static MemoryContext Context(params Message[] recent) => new() + { + SessionId = "s-1", + AssembledAtUtc = DateTimeOffset.UnixEpoch, + RecentMessages = new MemoryContextSection { Items = recent }, + }; + + private static IReadOnlyList Map( + MemoryContext context, + IReadOnlyList? liveThread, + int chatBudget = 10, + MemoryTrustLevel systemRoleThreshold = MemoryTrustLevel.Untrusted) => + MafTypeMapper.ToContextMessages( + context, + new ContextFormatOptions + { + MaxChatHistoryMessages = chatBudget, + MinimumTrustForSystemRole = systemRoleThreshold, + }, + admissionPolicy: null, + logger: null, + liveThread: liveThread); + + [Fact] + public void ARecalledMessageAlreadyInTheLiveThreadIsDropped() + { + var mapped = Map( + Context(Recalled("what is my deploy command"), Recalled("an older turn")), + [new ChatMessage(ChatRole.User, "what is my deploy command")]); + + mapped.Select(m => m.Text).Should().NotContain("what is my deploy command"); + mapped.Select(m => m.Text).Should().Contain(t => t.Contains("an older turn")); + } + + [Fact] + public void MatchingIsOnContentEvenWhenTheRoleWasRewritten() + { + // THE case. The live thread carries an assistant turn; recall returns the same content, and + // the role gate rewrites it to user because its trust is below the threshold. Content is + // identical, roles are not -- a role-keyed fingerprint would miss it. + var recalled = Recalled("I recommended the 14:05 train", role: "assistant"); + + var mapped = Map( + Context(recalled), + [new ChatMessage(ChatRole.Assistant, "I recommended the 14:05 train")], + systemRoleThreshold: MemoryTrustLevel.ApplicationTrusted); + + mapped.Select(m => m.Text).Should().NotContain(t => t.Contains("14:05 train")); + } + + [Fact] + public void TheDedupHappensBeforeTheChatBudget() + { + // The quality half, not just the cost half. With a budget of 2 and the two newest turns + // already in the thread, filtering first lets the budget carry two OLDER messages the model + // has not seen; filtering afterwards would have spent both slots on duplicates and delivered + // nothing. + var mapped = Map( + Context(Recalled("newest"), Recalled("second"), Recalled("third"), Recalled("fourth")), + [new ChatMessage(ChatRole.User, "newest"), new ChatMessage(ChatRole.User, "second")], + chatBudget: 2); + + var texts = mapped.Select(m => m.Text).ToList(); + texts.Should().Contain(t => t.Contains("third")); + texts.Should().Contain(t => t.Contains("fourth")); + texts.Should().NotContain(t => t.Contains("newest")); + } + + [Fact] + public void WhitespaceAndCasingDoNotDefeatTheMatch() + { + // The live thread and the stored copy travel different paths -- one through the host's own + // formatting, one through persistence and back. A trailing newline is not a different message. + var mapped = Map( + Context(Recalled("Deploy the service\n")), + [new ChatMessage(ChatRole.User, "deploy the service")]); + + mapped.Select(m => m.Text).Should().NotContain(t => t.Contains("Deploy")); + } + + [Fact] + public void ANonMatchingLiveThreadChangesNothing() + { + var mapped = Map( + Context(Recalled("stored turn")), + [new ChatMessage(ChatRole.User, "a completely different question")]); + + mapped.Select(m => m.Text).Should().Contain(t => t.Contains("stored turn")); + } + + [Fact] + public void NoLiveThreadLeavesTheOutputByteIdentical() + { + // The off switch and the byte-identical guarantee: passing null must reproduce exactly the + // pre-2.5 behaviour, which is what every recorded measurement was taken under. + var context = Context(Recalled("stored turn"), Recalled("another")); + + var withoutThread = Map(context, liveThread: null); + var withEmptyThread = Map(context, liveThread: []); + + withoutThread.Select(m => m.Text).Should().Equal(withEmptyThread.Select(m => m.Text)); + // Both recalled turns survive. Counted by content rather than by list length, because the + // mapper also emits the security prefix -- asserting a raw count would break whenever that + // preamble changed, for a reason unrelated to dedup. + withoutThread.Select(m => m.Text).Should() + .Contain(t => t.Contains("stored turn")) + .And.Contain(t => t.Contains("another")); + } + + [Fact] + public void AnEmptyOrBlankLiveMessageMatchesNothing() + { + // A blank entry normalises to the empty string; if that were a valid key it would match every + // recalled message whose content was also blank, and worse, invite an empty-vs-empty match + // that silently drops content. + var mapped = Map( + Context(Recalled("stored turn")), + [new ChatMessage(ChatRole.User, " "), new ChatMessage(ChatRole.User, "")]); + + mapped.Select(m => m.Text).Should().Contain(t => t.Contains("stored turn")); + } + + [Fact] + public void TheNormalizerIsCultureIndependent() + { + // ToUpperInvariant, not ToUpper: a Turkish locale folds 'i' to 'İ', so a culture-sensitive + // normaliser would make dedup depend on the host's locale. + MafTypeMapper.NormalizeForDedup("Istanbul file") + .Should().Be(MafTypeMapper.NormalizeForDedup("ISTANBUL FILE")); + } + + [Fact] + public void DedupIsOnByDefault() + { + // Sending the model two copies of the same turn has no upside, and the comparison is a hash + // set over a thread the provider already holds. + new AgentFrameworkOptions().DeduplicateRecalledHistory.Should().BeTrue(); + } +} diff --git a/tests/AgentMemory.Tests.Unit/Extraction/ContentRefusalResilienceTests.cs b/tests/AgentMemory.Tests.Unit/Extraction/ContentRefusalResilienceTests.cs new file mode 100644 index 000000000..17b577422 --- /dev/null +++ b/tests/AgentMemory.Tests.Unit/Extraction/ContentRefusalResilienceTests.cs @@ -0,0 +1,171 @@ +using System.Net; +using AgentMemory.Extraction.Llm; +using FluentAssertions; +using Xunit; + +namespace AgentMemory.Tests.Unit.Extraction; + +/// +/// A provider that refuses some content must not cost the whole preparation. +/// +/// +/// +/// Measured, 2026-08-12. A 50-question cold build died after 270 of 616 extraction calls +/// because two sessions of a public research dataset tripped an Azure content policy. The batch-split +/// path treated the 400 as a shape failure, halved the batch, re-sent the same text to the same +/// filter, and propagated once the batch reached size 1 — losing an hour of work to content that was +/// never going to be accepted. +/// +/// +/// A refusal is terminal: neither a retry nor a split changes the text or the policy. The only +/// useful responses are to skip the affected sessions or to abandon the run, and skipping is +/// overwhelmingly better provided the loss is recorded — a corpus with gaps that looks complete +/// would attribute those gaps to recall. +/// +/// +public sealed class ContentRefusalResilienceTests +{ + // HttpRequestException rather than ClientResultException: TryGetStatus reads the status + // reflectively from either, and the test should not take a provider package dependency to + // classify an error the production code deliberately classifies without one. + private static Exception Refusal(string body) => + new HttpRequestException($"HTTP 400 ({body})", null, HttpStatusCode.BadRequest); + + private static Exception Status(int code) => + new HttpRequestException($"HTTP {code} (something else)", null, (HttpStatusCode)code); + + [Theory] + [InlineData("content_filter")] + [InlineData("invalid_request_error: cyber_policy")] + [InlineData("ResponsibleAIPolicyViolation")] + [InlineData("The response was filtered due to the content filter")] + public void TheProvidersRefusalVocabularyIsRecognised(string body) + { + // Matched on the provider's own words, not on status alone: 400 covers both "your request is + // malformed" -- which splitting legitimately diagnoses -- and "I will not process this", + // which it cannot. + LlmMultiSessionUnifiedMemoryExtractor.IsContentRejection(Refusal(body)).Should().BeTrue(); + } + + [Fact] + public void AMultiSessionRefusalIsSplitToIsolateTheOffender() + { + // Not a retry -- the policy is deterministic for the same text -- but isolation. Measured: the + // batch that killed a 616-call preparation went 4 -> 2 -> 1 and still failed at 1, so ONE + // session was responsible and its three batch-mates were innocent. Skipping the whole batch + // would have discarded all four. + // + // Asserted through the classification the split path keys on: a refusal must not be a shape + // failure (or it would split forever and then propagate), while the extractor splits it for + // its own reason at batch > 1. + var refusal = Refusal("content_filter"); + + LlmMultiSessionUnifiedMemoryExtractor.IsContentRejection(refusal).Should().BeTrue(); + LlmMultiSessionUnifiedMemoryExtractor.IsBatchShapeFailure(refusal).Should().BeFalse(); + } + + [Fact] + public void ARefusalIsNotTreatedAsABatchShapeFailure() + { + // THE fix. Classifying it as a shape failure is what made the extractor split, re-send the + // same text to the same policy, and eventually propagate. + var refusal = Refusal("content_filter"); + + LlmMultiSessionUnifiedMemoryExtractor.IsContentRejection(refusal).Should().BeTrue(); + LlmMultiSessionUnifiedMemoryExtractor.IsBatchShapeFailure(refusal).Should().BeFalse(); + } + + [Fact] + public void AnOrdinaryBadRequestIsStillASplittableShapeFailure() + { + // The other direction, and the one that must not regress: a genuine 400 -- an over-long batch, + // a malformed schema -- is exactly what halving the batch diagnoses. + var malformed = Status(400); + + LlmMultiSessionUnifiedMemoryExtractor.IsContentRejection(malformed).Should().BeFalse(); + LlmMultiSessionUnifiedMemoryExtractor.IsBatchShapeFailure(malformed).Should().BeTrue(); + } + + [Theory] + [InlineData(408)] + [InlineData(429)] + [InlineData(500)] + [InlineData(503)] + public void TransientAndServerFailuresAreNeitherRefusalsNorShapeFailures(int status) + { + // A timeout, a throttle or a server error may succeed on the same text later, so neither + // skipping the content nor splitting the batch is the right response. + LlmMultiSessionUnifiedMemoryExtractor.IsContentRejection(Status(status)).Should().BeFalse(); + LlmMultiSessionUnifiedMemoryExtractor.IsBatchShapeFailure(Status(status)).Should().BeFalse(); + } + + [Fact] + public void AFormatFailureRemainsASplittableShapeFailure() + { + LlmMultiSessionUnifiedMemoryExtractor.IsBatchShapeFailure(new FormatException("bad json")) + .Should().BeTrue(); + LlmMultiSessionUnifiedMemoryExtractor.IsContentRejection(new FormatException("bad json")) + .Should().BeFalse(); + } + + [Fact] + public void TheRefusedSessionIdIsRecordedSoItCanBeInvestigated() + { + // "Something was refused" is not a report anyone can act on, and content filters recur. The + // ID is the handle: an operator looks the conversation up in the dataset themselves, which + // keeps the diagnostics free of dataset text while still being actionable. + var diagnostics = new LlmExtractionBatchDiagnostics(); + + diagnostics.RecordContentRejection(Refusal("content_filter"), 1, sessionId: "session-42"); + + diagnostics.Snapshot().RefusedSessionIds.Should().Equal("session-42"); + } + + [Fact] + public void ARefusalIsRecordedWithTheSessionsItCost() + { + // "2 refusals" says nothing about whether the corpus is usable. "2 refusals costing 6 of + // 2,418 sessions" does, and it is the session count that the tolerance and the manifest need. + var diagnostics = new LlmExtractionBatchDiagnostics(); + + diagnostics.RecordContentRejection(Refusal("content_filter"), sourceSessions: 4); + diagnostics.RecordContentRejection(Refusal("content_filter"), sourceSessions: 2); + + var snapshot = diagnostics.Snapshot(); + snapshot.ContentRejections.Should().Be(2); + snapshot.SessionsRefused.Should().Be(6); + } + + [Fact] + public void RefusalsAreCountedSeparatelyFromSplits() + { + // Different events with different remedies: a split is a recoverable shape problem, a refusal + // is terminal for that text. Folding them together would hide a corpus losing content behind + // a counter that looks like ordinary batching noise. + var diagnostics = new LlmExtractionBatchDiagnostics(); + + diagnostics.RecordSplit(new FormatException("bad json"), sourceSessions: 4); + diagnostics.RecordContentRejection(Refusal("content_filter"), sourceSessions: 1); + + var snapshot = diagnostics.Snapshot(); + snapshot.Splits.Should().Be(1); + snapshot.ContentRejections.Should().Be(1); + snapshot.Details.Should().Contain(d => d.Reason == "content-rejected"); + } + + [Fact] + public void ADeltaSubtractsRefusalsToo() + { + // The harness reconciles per-question deltas against a baseline. A field that did not subtract + // would report every earlier refusal again on every subsequent question. + var diagnostics = new LlmExtractionBatchDiagnostics(); + diagnostics.RecordContentRejection(Refusal("content_filter"), sourceSessions: 3); + var baseline = diagnostics.Snapshot(); + + diagnostics.RecordContentRejection(Refusal("content_filter"), sourceSessions: 2); + var delta = diagnostics.Snapshot().Delta(baseline); + + delta.ContentRejections.Should().Be(1); + delta.SessionsRefused.Should().Be(2); + } +} diff --git a/tests/AgentMemory.Tests.Unit/Extraction/ExtractionNoveltyGateTests.cs b/tests/AgentMemory.Tests.Unit/Extraction/ExtractionNoveltyGateTests.cs new file mode 100644 index 000000000..3aa5cc449 --- /dev/null +++ b/tests/AgentMemory.Tests.Unit/Extraction/ExtractionNoveltyGateTests.cs @@ -0,0 +1,144 @@ +using AgentMemory.Abstractions.Domain; +using AgentMemory.Abstractions.Options; +using AgentMemory.Core.Extraction; +using FluentAssertions; +using Xunit; + +namespace AgentMemory.Tests.Unit.Extraction; + +/// +/// Not paying for an extraction call on "ok, thanks!" (E4) — and, far more importantly, paying for it +/// whenever there is the slightest chance the turn said something. +/// +/// +/// +/// The asymmetry is worse here than anywhere else in the system, so the tests are lopsided. +/// Declining to gate costs one extraction call. Gating a turn that did carry a fact means the memory +/// is never formed — nothing downstream can recover it, and nothing can even report it missing. The +/// bulk of these cases are therefore phrases that must still be extracted. +/// +/// +public sealed class ExtractionNoveltyGateTests +{ + private static Message M(string content, string role = "user") => new() + { + MessageId = $"m-{content.GetHashCode():X}", + ConversationId = "c-1", + SessionId = "s-1", + Role = role, + Content = content, + TimestampUtc = DateTimeOffset.UnixEpoch, + }; + + private static bool Worth(params string[] contents) => + ExtractionNoveltyGate.IsWorthExtracting(contents.Select(c => M(c)).ToList()); + + // ── must still be extracted ─────────────────────────────────────────── + + [Theory] + [InlineData("yes")] + [InlineData("no")] + [InlineData("sure")] + [InlineData("nope")] + [InlineData("correct")] + [InlineData("right")] + public void ABareAnswerIsContentful(string answer) + { + // THE dangerous class. Each of these is a complete answer to a question -- "do you eat meat?" + // "no" -- and the question may have been in the PREVIOUS batch, so nothing in this one reveals + // that a fact was just asserted. They are kept out of the vocabulary deliberately. + Worth(answer).Should().BeTrue(); + } + + [Theory] + [InlineData("Basel")] + [InlineData("tea")] + [InlineData("3")] + [InlineData("42 Rue Lafayette")] + public void AShortContentfulTurnIsNotAPleasantry(string content) + { + // Short is not the same as empty. A one-word answer naming a place, a preference or a quantity + // is exactly the fact worth remembering. + Worth(content).Should().BeTrue(); + } + + [Fact] + public void AQuestionMarkAlwaysWinsExtraction() + { + // Someone asked something, so an answer is in play -- possibly a single word in this very + // batch. Cheap insurance against the whole ambiguous class. + Worth("ok thanks?").Should().BeTrue(); + } + + [Fact] + public void OnePleasantryBesideOneRealTurnStillExtracts() + { + // The batch is the unit. One contentful message makes the whole call worth paying for. + Worth("thanks!", "I moved to Zurich last month").Should().BeTrue(); + } + + [Fact] + public void AnEmptyBatchIsNotGated() + { + // Biased toward extracting even here: an empty batch is cheap, and what to do with it belongs + // to the extractors rather than to a gate guessing on their behalf. + ExtractionNoveltyGate.IsWorthExtracting([]).Should().BeTrue(); + } + + [Fact] + public void AGratitudeWordInsideARealSentenceDoesNotGate() + { + // "thanks" appears constantly inside contentful turns. The whole message must be explained by + // the vocabulary, token by token -- not merely contain one of its words. + Worth("thanks, my address is 12 Bahnhofstrasse").Should().BeTrue(); + } + + // ── may be skipped ──────────────────────────────────────────────────── + + [Theory] + [InlineData("ok")] + [InlineData("Thanks!")] + [InlineData("thank you so much")] + [InlineData("ok, got it")] + [InlineData("great, thanks")] + [InlineData("hi there")] + [InlineData("good morning")] + [InlineData("you're welcome")] + [InlineData("perfect")] + [InlineData("lol")] + public void PureAcknowledgementIsNotWorthACompletion(string content) => + Worth(content).Should().BeFalse(); + + [Theory] + [InlineData("no problem")] + [InlineData("no worries")] + public void APleasantryStartingWithAnAmbiguousTokenIsNotSkipped(string content) + { + // A missed skip, and the correct trade. "no problem" is plainly a pleasantry, but gating it + // means putting "no" in the vocabulary -- and then a bare "no", the complete answer to "do you + // eat meat?", stops being extracted. One wasted call is the cheaper mistake by a wide margin. + Worth(content).Should().BeTrue(); + } + + [Fact] + public void AWholeBatchOfPleasantriesIsSkipped() => + Worth("thanks!", "You're welcome!").Should().BeFalse(); + + [Fact] + public void BlankMessagesDoNotByThemselvesForceExtraction() => + Worth("ok", " ").Should().BeFalse(); + + [Fact] + public void PunctuationAndCasingDoNotDefeatTheVocabulary() => + Worth("OK... Thanks!!!").Should().BeFalse(); + + // ── shape ───────────────────────────────────────────────────────────── + + [Fact] + public void TheGateIsOffByDefault() + { + // It changes what gets extracted, and transcript bytes are fingerprinted into every measured + // run in this track. + new ExtractionOptions().SkipUninformativeTurns.Should().BeFalse(); + } +} diff --git a/tests/AgentMemory.Tests.Unit/Extraction/ExtractionStageNoveltyGatingTests.cs b/tests/AgentMemory.Tests.Unit/Extraction/ExtractionStageNoveltyGatingTests.cs new file mode 100644 index 000000000..e7ffae516 --- /dev/null +++ b/tests/AgentMemory.Tests.Unit/Extraction/ExtractionStageNoveltyGatingTests.cs @@ -0,0 +1,133 @@ +using AgentMemory.Abstractions.Domain; +using AgentMemory.Abstractions.Options; +using AgentMemory.Abstractions.Services; +using AgentMemory.Core.Extraction; +using FluentAssertions; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using NSubstitute; +using Xunit; + +namespace AgentMemory.Tests.Unit.Extraction; + +/// +/// The wiring half of E4: the gate has to prevent the call, not merely be able to judge it. +/// +/// +/// +/// 's own tests prove a batch of pleasantries scores as +/// uninformative. None of them prove a single completion was avoided. A gate nothing consults +/// passes all of them while every turn still costs exactly what it did before — and since the saving +/// is invisible from the outside, nothing else in the system would report the feature as inert. +/// +/// +/// So these assert on the extractor: was it invoked, and on which of the two paths. +/// +/// +public sealed class ExtractionStageNoveltyGatingTests +{ + private readonly IFactExtractor _extractor = Substitute.For(); + private readonly IUnifiedMemoryExtractor _unified = Substitute.For(); + + public ExtractionStageNoveltyGatingTests() + { + _extractor.ExtractAsync(Arg.Any>(), Arg.Any()) + .Returns(Task.FromResult>([])); + + _unified.IsEnabled.Returns(true); + _unified.ExtractAsync(Arg.Any>(), Arg.Any()) + .Returns(Task.FromResult(new UnifiedExtractionResult())); + } + + private static Message M(string content) => new() + { + MessageId = $"m-{content.GetHashCode():X}", + ConversationId = "c-1", + SessionId = "s-1", + Role = "user", + Content = content, + TimestampUtc = DateTimeOffset.UnixEpoch, + }; + + private ExtractionStage CreateSut(bool skipUninformativeTurns, bool withUnified = false) => + new([], [_extractor], [], [], + withUnified ? [_unified] : [], + Substitute.For(), + Options.Create(new ExtractionOptions { SkipUninformativeTurns = skipUninformativeTurns }), + NullLogger.Instance); + + [Fact] + public async Task AnUninformativeBatchNeverReachesTheExtractor() + { + // THE test. Everything else in E4 is arrangement around this call not being made. + await CreateSut(skipUninformativeTurns: true) + .ExtractAsync([M("ok, thanks!")], ExtractionTypes.All); + + await _extractor.DidNotReceive().ExtractAsync( + Arg.Any>(), Arg.Any()); + } + + [Fact] + public async Task TheUnifiedExtractorIsSkippedToo() + { + // The unified extractor is a separate dispatch path and is the one a real deployment uses -- + // gating only the per-type extractors would save nothing where it counts. + await CreateSut(skipUninformativeTurns: true, withUnified: true) + .ExtractAsync([M("thanks!")], ExtractionTypes.All); + + await _unified.DidNotReceive().ExtractAsync( + Arg.Any>(), Arg.Any()); + } + + [Fact] + public async Task AContentfulBatchStillReachesTheExtractor() + { + await CreateSut(skipUninformativeTurns: true) + .ExtractAsync([M("I moved to Zurich")], ExtractionTypes.All); + + await _extractor.Received(1).ExtractAsync( + Arg.Any>(), Arg.Any()); + } + + [Fact] + public async Task DisabledMeansEveryTurnStillCosts() + { + // The off switch, and the byte-identical guarantee: with the gate off, a pleasantry is + // extracted from exactly as it was before E4 existed. + await CreateSut(skipUninformativeTurns: false) + .ExtractAsync([M("ok, thanks!")], ExtractionTypes.All); + + await _extractor.Received(1).ExtractAsync( + Arg.Any>(), Arg.Any()); + } + + [Fact] + public async Task AlreadyExtractedResultsAreNeverGatedAway() + { + // The path that must NOT be gated. When the caller supplies a pre-extracted result the + // completion has already been paid for; discarding it there would throw away work that cost + // money instead of avoiding the cost -- the gate's exact inverse. + var preExtracted = new UnifiedExtractionResult + { + Facts = [new ExtractedFact { Subject = "user", Predicate = "lives in", Object = "Zurich", Confidence = 0.9 }], + }; + + var result = await CreateSut(skipUninformativeTurns: true) + .ProcessUnifiedAsync([M("ok, thanks!")], preExtracted, ExtractionTypes.All); + + result.RawFacts.Should().HaveCount(1, + "the caller already paid for this extraction; gating it discards work rather than avoiding it"); + } + + [Fact] + public async Task ASkippedBatchStillReportsItsSourceMessages() + { + // Skipping the call must not skip the bookkeeping: provenance is what lets a later audit tell + // "these turns were considered and carried nothing" apart from "these turns were never seen". + var result = await CreateSut(skipUninformativeTurns: true) + .ExtractAsync([M("ok"), M("thanks!")], ExtractionTypes.All); + + result.SourceMessageIds.Should().HaveCount(2); + result.RawFacts.Should().BeEmpty(); + } +} diff --git a/tests/AgentMemory.Tests.Unit/Extraction/ExtractionWindowTests.cs b/tests/AgentMemory.Tests.Unit/Extraction/ExtractionWindowTests.cs new file mode 100644 index 000000000..005c1a593 --- /dev/null +++ b/tests/AgentMemory.Tests.Unit/Extraction/ExtractionWindowTests.cs @@ -0,0 +1,157 @@ +using AgentMemory.Abstractions.Domain; +using AgentMemory.Abstractions.Options; +using AgentMemory.Abstractions.Services; +using AgentMemory.Core.Extraction; +using FluentAssertions; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using NSubstitute; +using Xunit; + +namespace AgentMemory.Tests.Unit.Extraction; + +/// +/// Extracting with earlier turns as context (E2) — and never extracting from them. +/// +/// +/// +/// One batch is often not enough to understand itself: "I moved there last year" needs the turn that +/// named the place. Widening the batch fixes the reference and breaks something worse. +/// +/// +/// Context must not be a target, and that stopped being a token-efficiency question this week. +/// Re-extracting the preceding turns re-asserts stored facts. That used to be merely wasteful — the +/// exact triple MERGEs — but a re-assertion now earns confidence (S2) and increments the +/// mention_count the salience reranker reads (R7). Extracting context would make a fact gain +/// both every time it sat inside a sliding window, so two signals that mean "the world keeps saying +/// this" would quietly come to mean "this was said recently". Nothing would look broken. +/// +/// +public sealed class ExtractionWindowTests +{ + private static Message M(string id, string content, string role = "user") => new() + { + MessageId = id, + ConversationId = "c-1", + SessionId = "s-1", + Role = role, + Content = content, + TimestampUtc = DateTimeOffset.UnixEpoch, + }; + + // ── rendering ───────────────────────────────────────────────────────── + + [Fact] + public void WithoutContextTheTranscriptIsByteIdentical() + { + // The off state. Every sealed measurement in this track was taken on this rendering, and + // transcript bytes are fingerprinted into each of them. + var targets = new[] { M("m-1", "I live in Zurich"), M("m-2", "Noted.", "assistant") }; + + ConversationTextBuilder.BuildWindow(ExtractionWindow.ForTargets(targets), numbered: false) + .Should().Be(ConversationTextBuilder.Build(targets)); + ConversationTextBuilder.BuildWindow(ExtractionWindow.ForTargets(targets), numbered: true) + .Should().Be(ConversationTextBuilder.BuildNumbered(targets)); + } + + [Fact] + public void ContextIsFencedAheadOfTheTargets() + { + var rendered = ConversationTextBuilder.BuildWindow( + new ExtractionWindow { Targets = [M("m-2", "I moved there last year")], Context = [M("m-1", "Zurich is lovely")] }, + numbered: false); + + rendered.Should().Contain(ConversationTextBuilder.ContextOpen) + .And.Contain("Zurich is lovely") + .And.Contain(ConversationTextBuilder.ContextClose); + rendered.IndexOf(ConversationTextBuilder.ContextClose, StringComparison.Ordinal) + .Should().BeLessThan(rendered.IndexOf("I moved there last year", StringComparison.Ordinal), + "the fence has to close before the turns the model may extract from"); + } + + [Fact] + public void ContextIsNeverNumbered() + { + // THE provenance trap. L3c numbers turns from 1 and resolves turn N positionally to + // Targets[N-1]. If context joined the numbering, every target index would shift by the context + // length -- and the result is not a crash, it is each fact attributed to a turn a few places + // earlier, which afterwards is indistinguishable from precise attribution. + var rendered = ConversationTextBuilder.BuildWindow( + new ExtractionWindow + { + Targets = [M("m-3", "the third")], + Context = [M("m-1", "the first"), M("m-2", "the second")], + }, + numbered: true); + + rendered.Should().Contain("[1] user: the third"); + rendered.Should().NotContain("[1] user: the first"); + rendered.Should().NotContain("[2]"); + } + + // ── the contract ────────────────────────────────────────────────────── + + [Fact] + public void AWindowWithNoContextIsTheDefault() => + ExtractionWindow.ForTargets([M("m-1", "hi")]).HasContext.Should().BeFalse(); + + [Fact] + public void TheOptionIsOffByDefault() => + new ExtractionOptions().ExtractionContextTurns.Should().Be(0); + + [Fact] + public async Task ContextIsKeptOutOfProvenance() + { + // The invariant the whole design exists to protect. An EXTRACTED_FROM edge to a context turn + // would assert the memory was stated in the very turn the extractor was told not to extract + // from. + var stage = new ExtractionStage( + [], [], [], [], [], Substitute.For(), + Options.Create(new ExtractionOptions()), + NullLogger.Instance); + + var result = await stage.ExtractWithContextAsync( + new ExtractionWindow + { + Targets = [M("m-2", "and I moved there")], + Context = [M("m-1", "Zurich is lovely")], + }, + ExtractionTypes.All); + + result.SourceMessageIds.Should().Equal(["m-2"]); + } + + [Fact] + public async Task AnExtractorThatIgnoresContextStillSeesOnlyItsTargets() + { + // The default interface method, exercised on a REAL implementation rather than a substitute: + // a mocking proxy intercepts the default body too, so asserting through one would only prove + // the mock forwards what it was told to. + // + // An extractor written before E2 must keep working, and must receive exactly the turns it + // always received -- not a widened batch it would extract from, which is the failure the whole + // Targets/Context split exists to prevent. + // Through the INTERFACE, not the class: a default interface method is not inherited into the + // implementing type, so this is also how a caller must reach it in production. + IFactExtractor extractor = new LegacyFactExtractor(); + + await extractor.ExtractWithContextAsync( + new ExtractionWindow { Targets = [M("m-2", "target")], Context = [M("m-1", "context")] }); + + ((LegacyFactExtractor)extractor).Seen.Should().ContainSingle() + .Which.MessageId.Should().Be("m-2"); + } + + /// An extractor written before E2 existed: it implements only the original overload. + private sealed class LegacyFactExtractor : IFactExtractor + { + public List Seen { get; } = []; + + public Task> ExtractAsync( + IReadOnlyList messages, CancellationToken cancellationToken = default) + { + Seen.AddRange(messages); + return Task.FromResult>([]); + } + } +} diff --git a/tests/AgentMemory.Tests.Unit/Infrastructure/AbstractionsContractGuardTests.cs b/tests/AgentMemory.Tests.Unit/Infrastructure/AbstractionsContractGuardTests.cs index 6104145c6..a6e9b60b3 100644 --- a/tests/AgentMemory.Tests.Unit/Infrastructure/AbstractionsContractGuardTests.cs +++ b/tests/AgentMemory.Tests.Unit/Infrastructure/AbstractionsContractGuardTests.cs @@ -16,9 +16,9 @@ public sealed class AbstractionsContractGuardTests private static readonly Assembly Abstractions = typeof(IMemoryService).Assembly; // Counts mirrored in docs/architecture.md §3.1 and docs/design.md §5/§6. - private const int DocumentedServiceInterfaces = 41; // +IMultiSessionUnifiedMemoryExtractor (M-27-V2 LAB-B1) - private const int DocumentedRepositoryInterfaces = 11; - private const int DocumentedDomainRecords = 52; // +MemoryContextSectionDiagnostics (PLAN 4.1) + private const int DocumentedServiceInterfaces = 44; // +IEntitySummarySynthesizer, +IEntitySummaryService (PLAN 12.7 / S1) + private const int DocumentedRepositoryInterfaces = 12; // +IEntitySummaryRepository (PLAN 12.7 / S1) + private const int DocumentedDomainRecords = 59; // +EntitySummary (PLAN 12.7 / S1; EntitySummarySource is a record STRUCT and the guard counts reference records only); +MemoryBlock, +MemoryBlockLine (PLAN 12.9 / S4); +BulkIngestionOptions, +BulkIngestionOutcome, +BulkIngestionResult (PLAN 2.9 / rank 27) private const int DocumentedEnums = 29; // +TraceKind (7.1), +ExtractionProvenanceMode (9.3) private static IEnumerable PublicTypes() => diff --git a/tests/AgentMemory.Tests.Unit/Memory/EntitySummaryTests.cs b/tests/AgentMemory.Tests.Unit/Memory/EntitySummaryTests.cs new file mode 100644 index 000000000..0cf6a8886 --- /dev/null +++ b/tests/AgentMemory.Tests.Unit/Memory/EntitySummaryTests.cs @@ -0,0 +1,238 @@ +using AgentMemory.Abstractions.Domain; +using AgentMemory.Abstractions.Options; +using AgentMemory.Abstractions.Repositories; +using AgentMemory.Abstractions.Services; +using AgentMemory.Core.Memory; +using FluentAssertions; +using Microsoft.Extensions.Logging.Abstractions; +using NSubstitute; +using Xunit; + +namespace AgentMemory.Tests.Unit.Memory; + +/// +/// Synthesized entity summaries (S1) — and the one property that decides whether they are an asset +/// or a liability: a summary that cannot prove it is current is never used. +/// +/// +/// +/// Recall about a well-known entity returns twenty facts that each cost context to say one thing. A +/// summary says the same in one item. But a summary is derived memory, and derived memory is +/// where a store quietly starts lying: the sources change, the summary does not, and afterwards +/// nothing about it looks any different. +/// +/// +/// So the tests here are mostly about a summary being withheld. The synthesis is the easy half. +/// +/// +public sealed class EntitySummaryTests +{ + private readonly IFactRepository _facts = Substitute.For(); + private readonly IEntitySummaryRepository _summaries = Substitute.For(); + private readonly IClock _clock = Substitute.For(); + + private static readonly Entity Alice = new() + { + EntityId = "e-alice", + Name = "Alice", + Type = "Person", + Confidence = 1.0, + OwnerId = "owner-1", + CreatedAtUtc = DateTimeOffset.UnixEpoch, + }; + + private static readonly MemoryScope Scope = MemoryScope.For("owner-1"); + + private static Fact F( + string id, string predicate, string @object, + double confidence = 0.9, DateTimeOffset? invalidatedAt = null) => new() + { + FactId = id, + Subject = "Alice", + Predicate = predicate, + Object = @object, + Confidence = confidence, + OwnerId = "owner-1", + CreatedAtUtc = DateTimeOffset.UnixEpoch, + InvalidatedAtUtc = invalidatedAt, + }; + + public EntitySummaryTests() => _clock.UtcNow.Returns(DateTimeOffset.UnixEpoch); + + private EntitySummaryService CreateSut() + { + var ids = Substitute.For(); + ids.GenerateId().Returns("sum-1"); + return new EntitySummaryService( + _facts, _summaries, new DeterministicEntitySummarySynthesizer(), + _clock, ids, NullLogger.Instance); + } + + private void StoreHolds(params Fact[] facts) => + _facts.GetBySubjectAsync("Alice", Arg.Any(), Arg.Any()) + .Returns(Task.FromResult>(facts)); + + private void SummaryOnRecord(EntitySummary summary) => + _summaries.GetByEntityAsync("e-alice", Arg.Any(), Arg.Any()) + .Returns(Task.FromResult(summary)); + + // ── staleness: the point of the whole design ────────────────────────── + + [Fact] + public async Task ASummaryIsUsedWhileItsSourcesAreUnchanged() + { + StoreHolds(F("f-1", "lives in", "Zurich")); + var written = await CreateSut().RefreshAsync(Alice, Scope); + SummaryOnRecord(written!); + + (await CreateSut().GetIfCurrentAsync(Alice, Scope)).Should().NotBeNull(); + } + + [Fact] + public async Task ASupersededSourceMakesTheSummaryUnusable() + { + // THE test. A fact was contradicted; the stored text still asserts it. Nothing about that + // summary looks different, so the only thing standing between it and a confident wrong answer + // is this check. + StoreHolds(F("f-1", "lives in", "Zurich")); + var written = await CreateSut().RefreshAsync(Alice, Scope); + SummaryOnRecord(written!); + + StoreHolds(F("f-1", "lives in", "Zurich", invalidatedAt: DateTimeOffset.UnixEpoch)); + + (await CreateSut().GetIfCurrentAsync(Alice, Scope)).Should().BeNull(); + } + + [Fact] + public async Task ANewFactMakesTheSummaryUnusable() + { + // Incompleteness is staleness too. A summary that omits something the store now knows is + // exactly as misleading as one asserting something it no longer believes. + StoreHolds(F("f-1", "lives in", "Zurich")); + var written = await CreateSut().RefreshAsync(Alice, Scope); + SummaryOnRecord(written!); + + StoreHolds(F("f-1", "lives in", "Zurich"), F("f-2", "works at", "Acme")); + + (await CreateSut().GetIfCurrentAsync(Alice, Scope)).Should().BeNull(); + } + + [Fact] + public async Task AMovedConfidenceMakesTheSummaryUnusable() + { + // Confidence is in the fingerprint deliberately. S2 reinforcement moves it, and a summary + // stating flatly what the store has since grown doubtful about is the stale shadow this + // design exists to prevent. + StoreHolds(F("f-1", "lives in", "Zurich", confidence: 0.9)); + var written = await CreateSut().RefreshAsync(Alice, Scope); + SummaryOnRecord(written!); + + StoreHolds(F("f-1", "lives in", "Zurich", confidence: 0.6)); + + (await CreateSut().GetIfCurrentAsync(Alice, Scope)).Should().BeNull(); + } + + [Fact] + public async Task NoSummaryOnRecordIsNotAnError() + { + _summaries.GetByEntityAsync("e-alice", Arg.Any(), Arg.Any()) + .Returns(Task.FromResult(null)); + + (await CreateSut().GetIfCurrentAsync(Alice, Scope)).Should().BeNull(); + } + + // ── the fingerprint ─────────────────────────────────────────────────── + + [Fact] + public void TheFingerprintIsOrderIndependent() + { + // It must answer "are these the same facts?", not "did they come back in the same order?". + // Otherwise a query-plan change invalidates every summary in the store without a single fact + // having moved. + var a = new EntitySummarySource("f-1", 0.9, false); + var b = new EntitySummarySource("f-2", 0.7, false); + + EntitySummary.ComputeFingerprint([a, b]) + .Should().Be(EntitySummary.ComputeFingerprint([b, a])); + } + + [Fact] + public void InvalidationChangesTheFingerprintOnItsOwn() + { + // Even with the id and confidence untouched: a fact the store has stopped believing is not + // the same source it was. + EntitySummary.ComputeFingerprint([new EntitySummarySource("f-1", 0.9, false)]) + .Should().NotBe(EntitySummary.ComputeFingerprint([new EntitySummarySource("f-1", 0.9, true)])); + } + + [Fact] + public void AnEmptySourceSetStillHasAFingerprint() => + EntitySummary.ComputeFingerprint([]).Should().NotBeNullOrWhiteSpace(); + + // ── synthesis ───────────────────────────────────────────────────────── + + [Fact] + public async Task SynthesisIsReproducible() + { + // Two runs over the same facts produce the same bytes, so a change in the text always means a + // change in the facts rather than a change in a sampler's mood. That is also what makes the + // summary safe to fingerprint at all. + var synth = new DeterministicEntitySummarySynthesizer(); + var facts = new[] { F("f-1", "lives in", "Zurich"), F("f-2", "works at", "Acme") }; + + var first = await synth.SynthesizeAsync(Alice, facts); + // AsEnumerable() first: on an array, Reverse() binds to Array.Reverse, which returns void. + var second = await synth.SynthesizeAsync(Alice, facts.AsEnumerable().Reverse().ToArray()); + + first.Should().Be(second); + } + + [Fact] + public async Task LowConfidenceFactsAreLeftOut() + { + // A summary states things flatly, with none of the per-fact confidence a caller would see. + // Folding a guess in gives it the same authority as everything else. + var summary = await new DeterministicEntitySummarySynthesizer() + .SynthesizeAsync(Alice, [F("f-1", "lives in", "Zurich"), F("f-2", "owns", "a yacht", confidence: 0.2)]); + + summary.Should().Contain("Zurich"); + summary.Should().NotContain("yacht"); + } + + [Fact] + public async Task NothingWorthSummarisingReturnsNullNotAnEmptySummary() + { + // An empty summary would satisfy every has-a-summary check while saying nothing. + var summary = await new DeterministicEntitySummarySynthesizer() + .SynthesizeAsync(Alice, [F("f-1", "owns", "a yacht", confidence: 0.1)]); + + summary.Should().BeNull(); + } + + [Fact] + public async Task NothingWorthSummarisingIsNotStored() + { + StoreHolds(F("f-1", "owns", "a yacht", confidence: 0.1)); + + var result = await CreateSut().RefreshAsync(Alice, Scope); + + result.Should().BeNull(); + await _summaries.DidNotReceiveWithAnyArgs().UpsertAsync(default!, default); + } + + [Fact] + public async Task SupersededFactsAreNotSummarised() + { + // Correctness, not merely staleness: the summary must not assert something the store has + // already stopped believing at the moment it is written. + StoreHolds( + F("f-1", "lives in", "Basel", invalidatedAt: DateTimeOffset.UnixEpoch), + F("f-2", "lives in", "Zurich")); + + var summary = await CreateSut().RefreshAsync(Alice, Scope); + + summary!.Content.Should().Contain("Zurich"); + summary.Content.Should().NotContain("Basel"); + summary.SourceFactIds.Should().Equal("f-2"); + } +} diff --git a/tests/AgentMemory.Tests.Unit/Memory/MemoryBlockRendererTests.cs b/tests/AgentMemory.Tests.Unit/Memory/MemoryBlockRendererTests.cs new file mode 100644 index 000000000..8cdce5cd0 --- /dev/null +++ b/tests/AgentMemory.Tests.Unit/Memory/MemoryBlockRendererTests.cs @@ -0,0 +1,245 @@ +using AgentMemory.Abstractions.Domain; +using AgentMemory.Core.Memory; +using FluentAssertions; +using Xunit; + +namespace AgentMemory.Tests.Unit.Memory; + +/// +/// A memory block a human can actually read (S4), and the two things it must never become. +/// +/// +/// +/// The honest assessment of this store was "capable but opaque": memory could be queried but +/// not seen. This closes that gap — while refusing the part of the block-memory design that +/// makes the block itself the store. +/// +/// +/// So the two properties under test beyond formatting are: a block never presents a retracted claim +/// as current, and it never ends silently when there was more to show. Both failures look exactly +/// like a correct block to the person reading it. +/// +/// +public sealed class MemoryBlockRendererTests +{ + private static readonly DateTimeOffset At = new(2026, 8, 12, 9, 0, 0, TimeSpan.Zero); + + private static Entity E(string id, string name, string type = "Person", double confidence = 0.9) => new() + { + EntityId = id, Name = name, Type = type, Confidence = confidence, + CreatedAtUtc = At, OwnerId = "alice", + }; + + private static Fact F( + string id, string subject, string predicate, string @object, + double confidence = 0.9, DateTimeOffset? invalidatedAt = null) => new() + { + FactId = id, Subject = subject, Predicate = predicate, Object = @object, + Confidence = confidence, CreatedAtUtc = At, OwnerId = "alice", + InvalidatedAtUtc = invalidatedAt, + }; + + private static Preference P(string id, string category, string text, double confidence = 0.9) => new() + { + PreferenceId = id, Category = category, PreferenceText = text, + Confidence = confidence, CreatedAtUtc = At, OwnerId = "alice", + }; + + private static MemoryBlock Render( + Entity[]? entities = null, Fact[]? facts = null, Preference[]? preferences = null, int maxLines = 50) => + MemoryBlockRenderer.Render( + entities ?? [], facts ?? [], preferences ?? [], At, "alice", maxLines); + + // ── what a block must never do ──────────────────────────────────────── + + [Fact] + public void ASupersededFactIsNotShown() + { + // A block says what memory believes NOW, and shows no per-item history. A retracted claim + // sitting beside a live one is simply presented as true. + var block = Render(facts: + [ + F("f-1", "Alice", "lives in", "Basel", invalidatedAt: At), + F("f-2", "Alice", "lives in", "Zurich"), + ]); + + block.Lines.Should().ContainSingle().Which.MemoryId.Should().Be("f-2"); + } + + [Fact] + public void TruncationIsCountedRatherThanSilent() + { + // THE failure mode of any bounded view. A block that quietly stops short reads as "this is + // everything", and invites the conclusion that a missing memory was never stored at all. + var facts = Enumerable.Range(1, 10) + .Select(i => F($"f-{i}", "Alice", "likes", $"thing-{i}")).ToArray(); + + var block = Render(facts: facts, maxLines: 4); + + block.Lines.Should().HaveCount(4); + block.OmittedCount.Should().Be(6); + block.IsTruncated.Should().BeTrue(); + } + + [Fact] + public void TruncationIsVisibleInTheTextToo() + { + // The object carries OmittedCount; the person reading the rendered block sees only the text. + var facts = Enumerable.Range(1, 10) + .Select(i => F($"f-{i}", "Alice", "likes", $"thing-{i}")).ToArray(); + + MemoryBlockRenderer.ToText(Render(facts: facts, maxLines: 4)) + .Should().Contain("6 more not shown"); + } + + // ── what makes it actionable ────────────────────────────────────────── + + [Fact] + public void EveryLineCarriesItsMemoryId() + { + // The reason this could stay a read surface. A human who spots something wrong acts on that + // exact memory through the audited write path, instead of editing prose and hoping something + // parses it back. + var block = Render( + entities: [E("e-1", "Alice")], + facts: [F("f-1", "Alice", "lives in", "Zurich")], + preferences: [P("p-1", "food", "vegetarian")]); + + block.Lines.Select(l => l.MemoryId).Should().Equal("e-1", "f-1", "p-1"); + MemoryBlockRenderer.ToText(block).Should().Contain("`f-1`"); + } + + [Fact] + public void KindsAreGroupedUnderHeadings() + { + var text = MemoryBlockRenderer.ToText(Render( + entities: [E("e-1", "Alice")], + facts: [F("f-1", "Alice", "lives in", "Zurich")])); + + text.Should().Contain("## Entity").And.Contain("## Fact"); + } + + [Fact] + public void RenderingIsDeterministic() + { + // Two renders of the same memory produce the same bytes, so a developer diffing blocks sees + // what changed in memory rather than what changed in the renderer. + Fact[] facts = [F("f-2", "Alice", "works at", "Acme"), F("f-1", "Alice", "lives in", "Zurich")]; + + MemoryBlockRenderer.ToText(Render(facts: facts)) + .Should().Be(MemoryBlockRenderer.ToText(Render(facts: facts.AsEnumerable().Reverse().ToArray()))); + } + + [Fact] + public void MoreConfidentMemoriesComeFirst() + { + var block = Render(facts: + [ + F("f-low", "Alice", "maybe likes", "sailing", confidence: 0.55), + F("f-high", "Alice", "lives in", "Zurich", confidence: 0.99), + ]); + + block.Lines.First().MemoryId.Should().Be("f-high"); + } + + // ── shape ───────────────────────────────────────────────────────────── + + [Fact] + public void AnEmptyMemorySaysSoRatherThanRenderingNothing() + { + // A blank block is indistinguishable from a broken renderer. + var text = MemoryBlockRenderer.ToText(Render()); + + text.Should().Contain("holds nothing"); + } + + [Fact] + public void TheBlockDeclaresItIsASnapshot() + { + // It is rendered on demand and never stored. Saying so in the text is what stops a reader + // treating it as a document to edit -- which is exactly how a block quietly becomes the store. + MemoryBlockRenderer.ToText(Render(entities: [E("e-1", "Alice")])) + .Should().Contain("snapshot, not a document to edit"); + } + + [Fact] + public void AZeroLineBudgetIsRejected() + { + // Not silently treated as unlimited or as empty: both produce a block that misrepresents + // memory, in opposite directions. + var act = () => Render(maxLines: 0); + + act.Should().Throw(); + } +} + +/// +/// Rendering a block from the memory-history rows the CLI actually feeds it (S4). +/// +/// +/// The typed overload is precise; this is the one a person reaches through agentmemory block, +/// and it is where a mistake would be seen rather than merely possible. +/// +public sealed class MemoryBlockFromHistoryTests +{ + private static readonly DateTimeOffset At = new(2026, 8, 12, 9, 0, 0, TimeSpan.Zero); + + private static MemoryHistoryRecord R( + string id, MemoryHistoryKind kind, string summary, + MemoryHistoryStatus status = MemoryHistoryStatus.Live, + DateTimeOffset? invalidatedAt = null) => new() + { + Id = id, + Kind = kind, + Summary = summary, + Status = status, + CreatedAtUtc = At, + InvalidatedAtUtc = invalidatedAt, + OwnerId = "alice", + }; + + [Fact] + public void InvalidatedRowsAreDroppedEvenIfTheQueryAskedForThem() + { + // Belt and braces, deliberately. A caller forgetting IncludeInvalidated = false would get a + // block presenting retracted claims as current, and nothing about it would look wrong. + var block = MemoryBlockRenderer.Render( + [ + R("f-1", MemoryHistoryKind.Fact, "Alice lives in Basel", + status: MemoryHistoryStatus.Invalidated, invalidatedAt: At), + R("f-2", MemoryHistoryKind.Fact, "Alice lives in Zurich"), + ], At, "alice"); + + block.Lines.Should().ContainSingle().Which.MemoryId.Should().Be("f-2"); + } + + [Fact] + public void KindsAreGroupedEntitiesThenFactsThenPreferences() + { + var block = MemoryBlockRenderer.Render( + [ + R("p-1", MemoryHistoryKind.Preference, "food: vegetarian"), + R("f-1", MemoryHistoryKind.Fact, "Alice lives in Zurich"), + R("e-1", MemoryHistoryKind.Entity, "Alice"), + ], At, "alice"); + + block.Lines.Select(l => l.MemoryId).Should().Equal("e-1", "f-1", "p-1"); + } + + [Fact] + public void TruncationIsStillCounted() + { + var rows = Enumerable.Range(1, 8) + .Select(i => R($"f-{i}", MemoryHistoryKind.Fact, $"fact {i}")).ToArray(); + + var block = MemoryBlockRenderer.Render(rows, At, "alice", maxLines: 3); + + block.Lines.Should().HaveCount(3); + block.OmittedCount.Should().Be(5); + } + + [Fact] + public void AnOwnerWithNoMemoriesRendersAnHonestEmptyBlock() => + MemoryBlockRenderer.ToText(MemoryBlockRenderer.Render([], At, "alice")) + .Should().Contain("holds nothing"); +} diff --git a/tests/AgentMemory.Tests.Unit/Memory/TemporalQueryParserTests.cs b/tests/AgentMemory.Tests.Unit/Memory/TemporalQueryParserTests.cs new file mode 100644 index 000000000..e05129266 --- /dev/null +++ b/tests/AgentMemory.Tests.Unit/Memory/TemporalQueryParserTests.cs @@ -0,0 +1,176 @@ +using AgentMemory.Core.Memory; +using FluentAssertions; +using Xunit; + +namespace AgentMemory.Tests.Unit.Memory; + +/// +/// Finding the point in time a question asks about (R4) — and, far more importantly, not finding one. +/// +/// +/// +/// RecallAsOfAsync has existed since the bitemporal work and nothing in an ordinary conversation +/// could reach it: "what did I think back in March?" recalled against now, exactly like every other +/// question. +/// +/// +/// The failure modes are wildly asymmetric, so the tests are too. A missed expression costs +/// nothing — the turn recalls against now, which is today's behaviour. A false positive silently +/// narrows recall to a window the user never asked about, and the answer that comes back looks +/// completely ordinary. Most of what follows is therefore about the phrases that must NOT parse. +/// +/// +public sealed class TemporalQueryParserTests +{ + private static readonly DateTimeOffset Now = new(2026, 8, 12, 12, 0, 0, TimeSpan.Zero); + + private static DateTimeOffset? Resolve(string query) => TemporalQueryParser.Resolve(query, Now); + + // ── what must NOT parse ─────────────────────────────────────────────── + + [Theory] + [InlineData("what was the last item I bought")] + [InlineData("give me the last one")] + [InlineData("who spoke last")] + [InlineData("the last thing you said")] + public void OrdinalLastIsNotTemporal(string query) + { + // "last" alone is far more often ordinal than temporal. Treating these as dates would rewrite + // the recall window of perfectly ordinary questions, and the answer would look normal. + Resolve(query).Should().BeNull(); + } + + [Theory] + [InlineData("what did March say about the contract")] + [InlineData("is April coming to the meeting")] + [InlineData("that may be the problem")] + [InlineData("May I ask about the budget")] + public void ABareMonthWordIsNotADate(string query) + { + // March and April are common names; "may" is a verb in nearly every sentence containing it. + // The preposition is what separates a date from a word that looks like one. + Resolve(query).Should().BeNull(); + } + + [Theory] + [InlineData("we shipped in 2024 units of stock")] + [InlineData("the order was for 2019 items")] + public void ANumberThatIsAQuantityIsNotAYear(string query) + { + Resolve(query).Should().BeNull(); + } + + [Theory] + [InlineData("what do I like")] + [InlineData("who is my manager")] + [InlineData("")] + [InlineData(" ")] + public void AnOrdinaryQuestionResolvesToNow(string query) + { + // Null means "recall as the system behaves today" -- the overwhelmingly common answer and the + // safe one. + Resolve(query).Should().BeNull(); + } + + [Fact] + public void AFutureDateIsNotAnAsOfInstant() + { + // As-of recall reconstructs what was known at a PAST moment. Pointed at the future it returns + // everything, which is indistinguishable from ordinary recall except that it silently ignored + // the question. + Resolve("what is planned in 2030").Should().BeNull(); + } + + [Fact] + public void FutureTenseIsNotDetected_AndThatIsTheAcceptedLimit() + { + // "what will change in December", asked in January, resolves to LAST December. The parser + // reads dates, not tense, and a bare month behind a preposition is genuinely ambiguous -- it + // most often does mean the most recent one. + // + // Recorded rather than fixed: tense detection is a substantially harder problem, and the cost + // of getting THIS wrong is asymmetric in the tolerable direction. The question is answered + // against last December's memory instead of all of it, which is a narrower answer to a + // question about the future -- not a wrong answer to a question about the past. + TemporalQueryParser.Resolve( + "what will change in December", new DateTimeOffset(2026, 1, 5, 0, 0, 0, TimeSpan.Zero)) + .Should().Be(new DateTimeOffset(2025, 12, 31, 23, 59, 59, TimeSpan.Zero)); + } + + // ── what must parse ─────────────────────────────────────────────────── + + [Fact] + public void Yesterday() => + Resolve("what did I say yesterday").Should().Be(Now.AddDays(-1)); + + [Theory] + [InlineData("what did I think last week", -7)] + [InlineData("what was true 3 days ago", -3)] + [InlineData("in the past 10 days what changed", -10)] + public void RelativeDayExpressions(string query, int days) => + Resolve(query).Should().Be(Now.AddDays(days)); + + [Fact] + public void LastQuarterIsThreeMonths() => + Resolve("what were my priorities last quarter").Should().Be(Now.AddMonths(-3)); + + [Fact] + public void LastYear() => + Resolve("where did I live last year").Should().Be(Now.AddYears(-1)); + + [Fact] + public void AMonthNameBehindAPrepositionResolvesToTheEndOfThatMonth() + { + // The END of the month, not its first instant: "back in March" means anything known by the + // close of March, and an as-of at 1 March would exclude the entire month being asked about. + Resolve("what did I think back in March") + .Should().Be(new DateTimeOffset(2026, 3, 31, 23, 59, 59, TimeSpan.Zero)); + } + + [Fact] + public void AMonthLaterInTheYearResolvesToLastYear() + { + // Asked in August, "in December" cannot mean this year's December -- that has not happened. + Resolve("what was the plan in December") + .Should().Be(new DateTimeOffset(2025, 12, 31, 23, 59, 59, TimeSpan.Zero)); + } + + [Fact] + public void AnExplicitMonthAndYearBeatsTheInference() => + Resolve("what did we decide in March 2024") + .Should().Be(new DateTimeOffset(2024, 3, 31, 23, 59, 59, TimeSpan.Zero)); + + [Fact] + public void AYearBehindAPrepositionResolvesToItsEnd() => + Resolve("what was true in 2023") + .Should().Be(new DateTimeOffset(2023, 12, 31, 23, 59, 59, TimeSpan.Zero)); + + // ── shape ───────────────────────────────────────────────────────────── + + [Fact] + public void TheReferenceInstantsOffsetIsPreserved() + { + // A host in a non-UTC offset must not have its temporal questions silently shifted by hours. + var offset = TimeSpan.FromHours(2); + var now = new DateTimeOffset(2026, 8, 12, 12, 0, 0, offset); + + TemporalQueryParser.Resolve("what did we decide in March 2024", now)! + .Value.Offset.Should().Be(offset); + } + + [Fact] + public void CasingDoesNotMatter() => + Resolve("What Did I Think Back In March").Should().NotBeNull(); + + [Fact] + public void ALongQueryDoesNotHangTheParser() + { + // Every pattern carries a 100 ms timeout: these run on the recall path, and a pathological + // input must degrade to "no temporal reference" rather than stall a turn. + var haystack = string.Join(' ', Enumerable.Repeat("last night we discussed many things", 500)); + + var act = () => Resolve(haystack); + + act.Should().NotThrow(); + } +} diff --git a/tests/AgentMemory.Tests.Unit/Queries/CypherQuerySnapshot.snap b/tests/AgentMemory.Tests.Unit/Queries/CypherQuerySnapshot.snap index bf3fc27fe..df6ad5eea 100644 --- a/tests/AgentMemory.Tests.Unit/Queries/CypherQuerySnapshot.snap +++ b/tests/AgentMemory.Tests.Unit/Queries/CypherQuerySnapshot.snap @@ -1,4 +1,4 @@ -# Cypher Query Snapshot — 158 queries +# Cypher Query Snapshot — 160 queries # Auto-generated by CypherQuerySnapshotTests. # To regenerate: set UPDATE_CYPHER_SNAPSHOTS=1 and re-run the test. @@ -237,6 +237,30 @@ UNWIND $items AS item e.updated_at = datetime() RETURN e +## EntitySummaryQueries.Upsert +MERGE (s:EntitySummary {entity_id: $entityId, owner_key: $ownerKey}) +SET s.id = $summaryId, + s.content = $content, + s.source_fact_ids = $sourceFactIds, + s.source_fingerprint = $sourceFingerprint, + s.owner_id = $ownerId, + s.generated_at = datetime($generatedAt) +WITH s +OPTIONAL MATCH (e:Entity {id: $entityId}) +FOREACH (_ IN CASE WHEN e IS NULL THEN [] ELSE [1] END | + MERGE (s)-[:SUMMARISES]->(e)) +WITH s +// Provenance to the facts the text was written from. Rebuilt wholesale rather than merged, +// because a summary regenerated from a smaller fact set must not keep edges to sources it no +// longer draws on -- that would claim provenance the content cannot support. +OPTIONAL MATCH (s)-[old:EXTRACTED_FROM]->(:Fact) +DELETE old +WITH s +UNWIND $sourceFactIds AS factId +MATCH (f:Fact {id: factId}) +MERGE (s)-[:EXTRACTED_FROM]->(f) +RETURN s + ## ExtractorQueries.CreateExtractedByRelationship MATCH (e:Entity {id: $entity_id}) MATCH (ex:Extractor {name: $extractor_name}) @@ -311,6 +335,11 @@ MATCH (f:Fact) WHERE f.embedding IS NULL RETURN f LIMIT $limit ## FactQueries.MarkDeduplicated MATCH (f:Fact {id: $id}) SET f.confidence = $confidence RETURN f +## FactQueries.MentionCounts +MATCH (f:Fact) + WHERE f.id IN $candidateIds + RETURN f.id AS candidateId, coalesce(f.mention_count, 1) AS mentions + ## FactQueries.SelectFactsMissingCanonicalKeys MATCH (f:Fact) WHERE f.predicate_key IS NULL @@ -334,16 +363,31 @@ MERGE (f:Fact {subject_key: $subjectKey, predicate_key: $predicateKey, object_ke f.valid_until = CASE WHEN $validUntil IS NOT NULL THEN datetime($validUntil) ELSE null END, f.source_message_ids = $sourceMessageIds, f.created_at = datetime($createdAtUtc), - f.metadata = $metadata + f.metadata = $metadata, + // R7 salience. Counts how often THE WORLD asserted this fact, incremented only here + // and in ON MATCH -- i.e. once per ingestion that re-states the same triple. + // Deliberately NOT the :MemoryReadAudit trail, which counts how often WE surfaced it: + // ranking on our own retrievals is a rich-get-richer loop that reinforces whatever + // already ranks highly and calls it learning. + f.mention_count = 1 ON MATCH SET f.category = $category, - f.confidence = $confidence, + // S2 corroboration. A re-asserted triple is the world saying it again, so confidence + // EARNS rather than being replaced by whatever the latest extraction happened to + // report. Gated by the parameter itself -- at alpha 0 this is the original assignment, + // byte for byte, so nothing recorded before S2 moves. + f.confidence = CASE WHEN $reinforceAlpha > 0 + THEN CASE WHEN coalesce(f.confidence, $confidence) + $reinforceAlpha > 1.0 + THEN 1.0 + ELSE coalesce(f.confidence, $confidence) + $reinforceAlpha END + ELSE $confidence END, f.valid_from = CASE WHEN $validFrom IS NOT NULL THEN datetime($validFrom) ELSE f.valid_from END, f.valid_until = CASE WHEN $validUntil IS NOT NULL THEN datetime($validUntil) ELSE f.valid_until END, f.source_message_ids = $sourceMessageIds, f.updated_at = datetime($updatedAtUtc), f.metadata = $metadata, - f.invalidated_at = null + f.invalidated_at = null, + f.mention_count = coalesce(f.mention_count, 1) + 1 RETURN f ## FactQueries.UpsertBatch @@ -361,16 +405,25 @@ UNWIND $items AS item f.valid_until = CASE WHEN item.valid_until IS NOT NULL THEN datetime(item.valid_until) ELSE null END, f.source_message_ids = item.source_message_ids, f.created_at = datetime(item.created_at), - f.metadata = item.metadata + f.metadata = item.metadata, + f.mention_count = 1 ON MATCH SET f.category = item.category, - f.confidence = item.confidence, + // S2 corroboration, identical to the single-item path. A counter that depended on + // which write path ran would make reinforcement a property of batching rather than of + // the conversation. + f.confidence = CASE WHEN $reinforceAlpha > 0 + THEN CASE WHEN coalesce(f.confidence, item.confidence) + $reinforceAlpha > 1.0 + THEN 1.0 + ELSE coalesce(f.confidence, item.confidence) + $reinforceAlpha END + ELSE item.confidence END, f.valid_from = CASE WHEN item.valid_from IS NOT NULL THEN datetime(item.valid_from) ELSE f.valid_from END, f.valid_until = CASE WHEN item.valid_until IS NOT NULL THEN datetime(item.valid_until) ELSE f.valid_until END, f.source_message_ids = item.source_message_ids, f.updated_at = datetime(item.updated_at), f.metadata = item.metadata, - f.invalidated_at = null + f.invalidated_at = null, + f.mention_count = coalesce(f.mention_count, 1) + 1 RETURN f ## FusedPersistenceQueries.EntityUpsertBatch @@ -431,14 +484,23 @@ UNWIND $items AS item f.confidence = item.confidence, f.valid_from = CASE WHEN item.valid_from IS NOT NULL THEN datetime(item.valid_from) ELSE null END, f.valid_until = CASE WHEN item.valid_until IS NOT NULL THEN datetime(item.valid_until) ELSE null END, + f.mention_count = 1, f.source_message_ids = item.source_message_ids, f.created_at = datetime(item.created_at), f.metadata = item.metadata ON MATCH SET f.category = item.category, - f.confidence = item.confidence, + // S2 corroboration, identical to the other two write paths. Reinforcement that + // depended on which path ran would be a property of batching rather than of the + // conversation. + f.confidence = CASE WHEN $reinforceAlpha > 0 + THEN CASE WHEN coalesce(f.confidence, item.confidence) + $reinforceAlpha > 1.0 + THEN 1.0 + ELSE coalesce(f.confidence, item.confidence) + $reinforceAlpha END + ELSE item.confidence END, f.valid_from = CASE WHEN item.valid_from IS NOT NULL THEN datetime(item.valid_from) ELSE f.valid_from END, f.valid_until = CASE WHEN item.valid_until IS NOT NULL THEN datetime(item.valid_until) ELSE f.valid_until END, + f.mention_count = coalesce(f.mention_count, 1) + 1, f.source_message_ids = item.source_message_ids, f.updated_at = datetime(item.updated_at), f.metadata = item.metadata, diff --git a/tests/AgentMemory.Tests.Unit/Queries/CypherQuerySnapshotTests.cs b/tests/AgentMemory.Tests.Unit/Queries/CypherQuerySnapshotTests.cs index 776f5e053..7da2a37b3 100644 --- a/tests/AgentMemory.Tests.Unit/Queries/CypherQuerySnapshotTests.cs +++ b/tests/AgentMemory.Tests.Unit/Queries/CypherQuerySnapshotTests.cs @@ -29,7 +29,8 @@ private static string ResolveSnapshotPath([CallerFilePath] string? sourceFile = private static readonly HashSet KnownNodeLabels = new(StringComparer.Ordinal) { - "ConsolidationRun", "Conversation", "Entity", "Extractor", "Fact", "MemoryReadAudit", "Message", + "ConsolidationRun", "Conversation", "Entity", "EntitySummary", "Extractor", "Fact", + "MemoryReadAudit", "Message", "Migration", "Preference", "ReasoningStep", "ReasoningTrace", "Schema", "Tool", "ToolCall" }; @@ -37,7 +38,7 @@ private static string ResolveSnapshotPath([CallerFilePath] string? sourceFile = // ── Expected query inventory count ──────────────────────────────────────── // Update this constant whenever queries are deliberately added or removed. - private const int ExpectedQueryCount = 155; // +SchemaQueries.TraceKindIndex (PLAN 7.2: the promotion marker separating an episode from a reusable procedure -- seekable so a procedures-only search is a seek rather than a post-filter over the whole label, and mirroring trace_success_idx). // +SchemaQueries.FactOwnerKeyIndex (measured: FindDuplicate runs on every fact write and had NO index entry point, planning a full 20,000-row scan; this seeks 100). // +SchemaQueries.MessageSessionIndex (measured: Neo4j will not seek a composite from a leading-column predicate alone, so 0007's (session_id, timestamp) gave GetRecentBySession nothing). // +SchemaQueries.MessageSessionTimestampIndex (Message.session_id is the predicate of the PRIMARY short-term recall path -- MessageQueries.cs:201, run on essentially every turn -- and nothing indexed it, so the plan was proportional to every message in the store rather than to the session; composite because TemporalQueries.cs:99-103 adds a trailing timestamp range to the same equality). // +SchemaQueries.FactPredicateKeyIndex (the same shape one column over: predicate_key sits at column 3 of fact_merge_key_idx and Neo4j serves a composite only on a matching PREFIX, so FactQueries.cs:88 had no index entry point at all -- a full :Fact scan across all owners whenever relation-completeness retrieval fires). // +SchemaQueries.MemoryReadAuditMemoryIdIndex (BUG-A2: HistoryQueries OPTIONAL MATCHes MemoryReadAudit on memory_id, which nothing indexed, so every history row scanned the whole label -- and the label grows ~25 rows per recall, so it degrades with TIME rather than data size). // +SchemaQueries.FactMergeKeyIndex (L11: every fact MERGE was an all-:Fact label scan; the composite {subject_key, object_key, predicate_key, owner_key} index is what makes the range-index key cap real for facts, which is why IndexKeyBudget.EnsureCompositeIndexable lands with it). // -1: SearchByCanonicalPredicates became an owner-conditional *method* (excluded, like GetBySubject) when the audit found it ignored IncludeShared and coerced a null-owner scope to the shared bucket. // +FactQueries.SelectFactsMissingCanonicalKeys/ApplyCanonicalKeys (Phase 1.1: canonical identity needs a C#-driven backfill, since Cypher's toLower diverges from ToLowerInvariant on U+0130). // +FactQueries.SearchByCanonicalPredicates (G3B.13: top-K is a relevance cutoff and cannot answer "how many", so a relation is retrieved whole via predicate_key). // +SchemaQueries.ShowIndexStates (BUG-S1: only vector-index dimensions were validated at bootstrap, so a FAILED range index degraded silently into full scans). base + ConsolidationQueries/SchemaQueries/TOUCHED/ConflictQueries consts. +MessageQueries.GetAllBySession (cycle-3); +SchemaQueries.ShowConstraintNames/ShowIndexNames (schema-check CLI); +SchemaPersistenceQueries.Save/DeactivateByName/LoadActiveByName/LoadByNameVersion/List/Exists/DeleteById (G4 schema-node CRUD, 7); +MemoryReadAudit constraint/index. Owner-conditional queries are *methods* (excluded): EntityQueries.ApplyConfidenceDelta/Delete/MergeEntities/SearchByLocation/SearchInBoundingBox/GetByType/FindSimilarByEmbedding, FactQueries.Delete/FindByTriple, PreferenceQueries.Delete, DecayQueries.PruneEntities/PruneFacts/PrunePreferences, ExtractorQueries.GetEntityProvenance, ReasoningQueries.ListTracesBySession (R2 owner-scoped 2026-06-13), ReasoningQueries.DeleteBySession (R6-C owner-scoped 2026-06-20: const→method, −1); +PreferenceQueries.UpsertBatch/RelationshipQueries.UpsertBatch (feat-04, +2). + private const int ExpectedQueryCount = 157; // +EntitySummaryQueries.Upsert (PLAN 12.7 / S1: GetByEntity and DeleteByEntity are owner-conditional METHODS and so excluded, like GetBySubject). // +FactQueries.MentionCounts (PLAN 10.3: reads the INGESTION-side salience counter for a whole candidate set in one query -- deliberately not the :MemoryReadAudit trail, which counts our own retrievals and would make ranking self-reinforcing). // +SchemaQueries.TraceKindIndex (PLAN 7.2: the promotion marker separating an episode from a reusable procedure -- seekable so a procedures-only search is a seek rather than a post-filter over the whole label, and mirroring trace_success_idx). // +SchemaQueries.FactOwnerKeyIndex (measured: FindDuplicate runs on every fact write and had NO index entry point, planning a full 20,000-row scan; this seeks 100). // +SchemaQueries.MessageSessionIndex (measured: Neo4j will not seek a composite from a leading-column predicate alone, so 0007's (session_id, timestamp) gave GetRecentBySession nothing). // +SchemaQueries.MessageSessionTimestampIndex (Message.session_id is the predicate of the PRIMARY short-term recall path -- MessageQueries.cs:201, run on essentially every turn -- and nothing indexed it, so the plan was proportional to every message in the store rather than to the session; composite because TemporalQueries.cs:99-103 adds a trailing timestamp range to the same equality). // +SchemaQueries.FactPredicateKeyIndex (the same shape one column over: predicate_key sits at column 3 of fact_merge_key_idx and Neo4j serves a composite only on a matching PREFIX, so FactQueries.cs:88 had no index entry point at all -- a full :Fact scan across all owners whenever relation-completeness retrieval fires). // +SchemaQueries.MemoryReadAuditMemoryIdIndex (BUG-A2: HistoryQueries OPTIONAL MATCHes MemoryReadAudit on memory_id, which nothing indexed, so every history row scanned the whole label -- and the label grows ~25 rows per recall, so it degrades with TIME rather than data size). // +SchemaQueries.FactMergeKeyIndex (L11: every fact MERGE was an all-:Fact label scan; the composite {subject_key, object_key, predicate_key, owner_key} index is what makes the range-index key cap real for facts, which is why IndexKeyBudget.EnsureCompositeIndexable lands with it). // -1: SearchByCanonicalPredicates became an owner-conditional *method* (excluded, like GetBySubject) when the audit found it ignored IncludeShared and coerced a null-owner scope to the shared bucket. // +FactQueries.SelectFactsMissingCanonicalKeys/ApplyCanonicalKeys (Phase 1.1: canonical identity needs a C#-driven backfill, since Cypher's toLower diverges from ToLowerInvariant on U+0130). // +FactQueries.SearchByCanonicalPredicates (G3B.13: top-K is a relevance cutoff and cannot answer "how many", so a relation is retrieved whole via predicate_key). // +SchemaQueries.ShowIndexStates (BUG-S1: only vector-index dimensions were validated at bootstrap, so a FAILED range index degraded silently into full scans). base + ConsolidationQueries/SchemaQueries/TOUCHED/ConflictQueries consts. +MessageQueries.GetAllBySession (cycle-3); +SchemaQueries.ShowConstraintNames/ShowIndexNames (schema-check CLI); +SchemaPersistenceQueries.Save/DeactivateByName/LoadActiveByName/LoadByNameVersion/List/Exists/DeleteById (G4 schema-node CRUD, 7); +MemoryReadAudit constraint/index. Owner-conditional queries are *methods* (excluded): EntityQueries.ApplyConfidenceDelta/Delete/MergeEntities/SearchByLocation/SearchInBoundingBox/GetByType/FindSimilarByEmbedding, FactQueries.Delete/FindByTriple, PreferenceQueries.Delete, DecayQueries.PruneEntities/PruneFacts/PrunePreferences, ExtractorQueries.GetEntityProvenance, ReasoningQueries.ListTracesBySession (R2 owner-scoped 2026-06-13), ReasoningQueries.DeleteBySession (R6-C owner-scoped 2026-06-20: const→method, −1); +PreferenceQueries.UpsertBatch/RelationshipQueries.UpsertBatch (feat-04, +2). // ── MemberData source ───────────────────────────────────────────────────── diff --git a/tests/AgentMemory.Tests.Unit/Repositories/ShortOwnerResultRescueTests.cs b/tests/AgentMemory.Tests.Unit/Repositories/ShortOwnerResultRescueTests.cs new file mode 100644 index 000000000..3c5a4984a --- /dev/null +++ b/tests/AgentMemory.Tests.Unit/Repositories/ShortOwnerResultRescueTests.cs @@ -0,0 +1,113 @@ +using AgentMemory.Neo4j.Repositories; +using FluentAssertions; +using Xunit; + +namespace AgentMemory.Tests.Unit.Repositories; + +/// +/// When an owner-scoped vector search comes back short but not empty (PLAN 2.11). +/// +/// +/// +/// Neo4j's vector index is global, so an owner filter is a POST-filter on a top-K drawn from every +/// tenant. Measured on a 50-owner corpus, a mean of 7 of 60 candidates reached the querying +/// owner. Only a totally empty result triggered a rescue, on the argument that "a short-but-non-empty +/// result still answers the question". +/// +/// +/// That argument has a measured counter-example. Question 5d3d2817 returned 2 facts from +/// a 710-fact graph with the gold answer present at coverage 1.00, and both arms answered it wrongly. +/// The claim is true for a small tenant and false for a crowded one, and the returned count alone +/// cannot separate them. +/// +/// +/// The rescue is the owner-bounded scan, not another widening — widening is a second draw on +/// the same global index, and a tenant losing to 50 neighbours at top-60 usually loses again at +/// top-480. The scan's cost scales with one owner's rows, so the small tenant the original argument +/// protected pays less for it than for a wider index query. +/// +/// +public sealed class ShortOwnerResultRescueTests +{ + [Fact] + public void AShortScopedResultIsRescued() + { + // The measured case: 2 rows of a requested 10. + OwnerVectorOverFetch.ShouldRescueShortResult(returned: 2, limit: 10, hasOwner: true) + .Should().BeTrue(); + } + + [Fact] + public void AFullResultIsNotRescued() + { + // Nothing was crowded out: the limit was met, so there is no shortfall to explain. + OwnerVectorOverFetch.ShouldRescueShortResult(returned: 10, limit: 10, hasOwner: true) + .Should().BeFalse(); + } + + [Fact] + public void AnEmptyResultIsLeftToTheEscalationPath() + { + // Zero already has an owner: ShouldEscalate tries one widened index query first and only then + // falls back to the scan. Claiming it here too would run the scan twice on the same search. + OwnerVectorOverFetch.ShouldRescueShortResult(returned: 0, limit: 10, hasOwner: true) + .Should().BeFalse(); + OwnerVectorOverFetch.ShouldEscalate(returned: 0, hasOwner: true).Should().BeTrue(); + } + + [Fact] + public void AnUnscopedSearchIsNeverRescued() + { + // With no owner filter there is no post-filter and therefore no crowding: a short result means + // the corpus genuinely held that little above the floor, and a scan returns the same rows more + // slowly. + OwnerVectorOverFetch.ShouldRescueShortResult(returned: 2, limit: 10, hasOwner: false) + .Should().BeFalse(); + } + + [Fact] + public void TheTwoRescuePathsAreMutuallyExclusiveAtEveryCount() + { + // A count that satisfied both would run the widening AND the scan for one search. Asserted + // across the whole range rather than at the boundary, because the two predicates are written + // independently and nothing else forces them apart. + for (var returned = 0; returned <= 12; returned++) + { + var escalate = OwnerVectorOverFetch.ShouldEscalate(returned, hasOwner: true); + var rescue = OwnerVectorOverFetch.ShouldRescueShortResult(returned, limit: 10, hasOwner: true); + + (escalate && rescue).Should().BeFalse($"returned={returned} must not trigger both paths"); + } + } + + [Fact] + public void EveryShortfallCounts_ThereIsNoRatioThreshold() + { + // A fraction of the limit would need a cutoff nobody can justify -- is 4 of 10 starved? 6? -- + // and the honest answer is that any shortfall might be crowding, because the index gave the + // owner whatever the neighbours left. The scan is bounded by the owner's rows either way, so + // the cost question is answered by its shape rather than by guessing a threshold. + for (var returned = 1; returned < 10; returned++) + { + OwnerVectorOverFetch.ShouldRescueShortResult(returned, limit: 10, hasOwner: true) + .Should().BeTrue($"returned={returned} of 10 is short"); + } + } + + [Fact] + public void TheRescueIsOffByDefault() + { + // It trades latency for recall on every short result, and every recorded measurement was taken + // without it, so enabling it is a stated decision rather than an inherited one. + new Abstractions.Options.MemoryOptions().RescueShortOwnerResults.Should().BeFalse(); + } + + [Fact] + public void EscalationStillOnlyFiresOnEmpty() + { + // The byte-identical guarantee: 2.11 adds a path, it does not widen the existing one. A host + // that leaves the option off gets exactly today's behaviour. + OwnerVectorOverFetch.ShouldEscalate(returned: 1, hasOwner: true).Should().BeFalse(); + OwnerVectorOverFetch.ShouldEscalate(returned: 0, hasOwner: false).Should().BeFalse(); + } +} diff --git a/tests/AgentMemory.Tests.Unit/Services/BulkIngestionTests.cs b/tests/AgentMemory.Tests.Unit/Services/BulkIngestionTests.cs new file mode 100644 index 000000000..7b97205da --- /dev/null +++ b/tests/AgentMemory.Tests.Unit/Services/BulkIngestionTests.cs @@ -0,0 +1,205 @@ +using AgentMemory.Abstractions.Domain; +using AgentMemory.Abstractions.Services; +using FluentAssertions; +using Xunit; + +namespace AgentMemory.Tests.Unit.Services; + +/// +/// The documented bulk path and its backpressure (rank 27). +/// +/// +/// +/// Loading a backlog was always possible, and both obvious ways are wrong in opposite directions: a +/// serial loop wastes hours, and an unbounded Parallel.ForEachAsync saturates the provider +/// quota and the connection pool — degrading p99 for every other tenant in the process while median +/// latency looks untouched. +/// +/// +/// So the tests are about the two things a bulk API is usually careless with: how many calls it makes +/// at once, and what it tells you when some of them fail. +/// +/// +public sealed class BulkIngestionTests +{ + /// Records concurrency and fails on demand; nothing else about ingestion is exercised. + private sealed class RecordingIngestion : IMemoryIngestion + { + private int _inFlight; + private readonly Func? _failOn; + private readonly TimeSpan _delay; + + public RecordingIngestion(Func? failOn = null, TimeSpan? delay = null) + { + _failOn = failOn; + _delay = delay ?? TimeSpan.FromMilliseconds(20); + } + + public int PeakConcurrency; + public readonly List Seen = []; + private readonly Lock _sync = new(); + + public async Task ExtractAndPersistAsync( + ExtractionRequest request, CancellationToken cancellationToken = default) + { + var now = Interlocked.Increment(ref _inFlight); + InterlockedMax(ref PeakConcurrency, now); + try + { + await Task.Delay(_delay, cancellationToken).ConfigureAwait(false); + lock (_sync) Seen.Add(request.SessionId); + + var index = int.Parse(request.SessionId.Split('-')[1], System.Globalization.CultureInfo.InvariantCulture); + if (_failOn?.Invoke(index) == true) throw new InvalidOperationException($"boom {index}"); + return new ExtractionResult(); + } + finally + { + Interlocked.Decrement(ref _inFlight); + } + } + + private static void InterlockedMax(ref int target, int value) + { + int seen; + while ((seen = Volatile.Read(ref target)) < value + && Interlocked.CompareExchange(ref target, value, seen) != seen) { } + } + + public Task AddMessageAsync( + string sessionId, string conversationId, string role, string content, + IReadOnlyDictionary? metadata = null, CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task AddMessageWithIdAsync( + string messageId, string sessionId, string conversationId, string role, string content, + IReadOnlyDictionary? metadata = null, CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task> AddMessagesAsync( + IEnumerable messages, CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task ExtractFromSessionAsync( + string sessionId, string? userId = null, CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + public Task ExtractFromConversationAsync( + string conversationId, string? userId = null, CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + } + + private static IReadOnlyList Requests(int count) => + Enumerable.Range(0, count) + .Select(i => new ExtractionRequest { Messages = [], SessionId = $"s-{i}" }) + .ToList(); + + [Fact] + public async Task ConcurrencyIsBounded() + { + // THE test. Everything else here is about reporting; this is the property the feature exists + // for, and the one an unbounded Parallel.ForEachAsync gets wrong. + var ingestion = new RecordingIngestion(); + + // Through the interface: a default interface method is not inherited into the implementing type. + await ((IMemoryIngestion)ingestion).IngestBulkAsync(Requests(20), new BulkIngestionOptions { MaxConcurrency = 3 }); + + ingestion.PeakConcurrency.Should().BeLessThanOrEqualTo(3); + } + + [Fact] + public async Task EveryRequestIsIngested() + { + var ingestion = new RecordingIngestion(); + + var result = await ((IMemoryIngestion)ingestion).IngestBulkAsync(Requests(12), new BulkIngestionOptions { MaxConcurrency = 4 }); + + result.SucceededCount.Should().Be(12); + ingestion.Seen.Should().HaveCount(12); + } + + [Fact] + public async Task AFailureIsReportedAgainstItsInputRatherThanThrown() + { + // A bulk API that returns "8,412 of 10,000 succeeded" tells the caller they have a problem and + // nothing about which inputs to retry, so the realistic response is to re-run everything -- + // worse than the failure itself. + var ingestion = new RecordingIngestion(failOn: i => i == 3); + + var result = await ((IMemoryIngestion)ingestion).IngestBulkAsync(Requests(6)); + + result.FailedCount.Should().Be(1); + var failure = result.Outcomes.Single(o => !o.Succeeded); + failure.Index.Should().Be(3); + failure.Error.Should().BeOfType(); + } + + [Fact] + public async Task OneBadInputDoesNotAbortTheRunByDefault() + { + // In a ten-thousand-conversation load, one malformed transcript stopping the other 9,999 is + // rarely what the caller wanted -- and continuing is not ignoring, because the failure is + // returned. + var ingestion = new RecordingIngestion(failOn: i => i == 0); + + var result = await ((IMemoryIngestion)ingestion).IngestBulkAsync(Requests(8), new BulkIngestionOptions { MaxConcurrency = 2 }); + + result.SucceededCount.Should().Be(7); + result.FailedCount.Should().Be(1); + } + + [Fact] + public async Task StopOnErrorLeavesTheRestUnattemptedRatherThanFailed() + { + // "Tried and failed" and "never tried" call for different actions: re-running a request that + // was never attempted is always correct, re-running a failed one may not be. Collapsing them + // makes a stopped run look like a partially broken corpus. + var ingestion = new RecordingIngestion(failOn: i => i == 0, delay: TimeSpan.FromMilliseconds(50)); + + var result = await ((IMemoryIngestion)ingestion).IngestBulkAsync( + Requests(20), new BulkIngestionOptions { MaxConcurrency = 1, ContinueOnError = false }); + + result.FailedCount.Should().Be(1); + result.NotAttemptedCount.Should().BeGreaterThan(0); + (result.SucceededCount + result.FailedCount + result.NotAttemptedCount).Should().Be(20); + } + + [Fact] + public async Task AnOuterCancellationStillThrows() + { + // A stop-on-error run completes normally -- the failure is in the outcomes. A cancellation + // the CALLER requested is a different thing and must not be swallowed into a tidy report. + using var cts = new CancellationTokenSource(); + var ingestion = new RecordingIngestion(delay: TimeSpan.FromMilliseconds(200)); + + var run = ((IMemoryIngestion)ingestion).IngestBulkAsync( + Requests(20), new BulkIngestionOptions { MaxConcurrency = 2 }, cts.Token); + await cts.CancelAsync(); + + await FluentActions.Awaiting(() => run).Should().ThrowAsync(); + } + + [Fact] + public async Task AnEmptyRunIsNotAnError() => + (await ((IMemoryIngestion)new RecordingIngestion()).IngestBulkAsync([])).Outcomes.Should().BeEmpty(); + + [Fact] + public async Task ZeroConcurrencyIsRejected() + { + // Not silently treated as unbounded, which is precisely the behaviour this exists to prevent. + var act = async () => await ((IMemoryIngestion)new RecordingIngestion()) + .IngestBulkAsync(Requests(1), new BulkIngestionOptions { MaxConcurrency = 0 }); + + await act.Should().ThrowAsync(); + } + + [Fact] + public void TheDefaultsAreConservative() + { + // A caller who never tunes this must not saturate a shared provider quota by accident. + var options = new BulkIngestionOptions(); + + options.MaxConcurrency.Should().Be(4); + options.ContinueOnError.Should().BeTrue(); + } +} diff --git a/tests/AgentMemory.Tests.Unit/Services/MemoryServiceAccessTrackingTests.cs b/tests/AgentMemory.Tests.Unit/Services/MemoryServiceAccessTrackingTests.cs index 3302b5973..a0530f75c 100644 --- a/tests/AgentMemory.Tests.Unit/Services/MemoryServiceAccessTrackingTests.cs +++ b/tests/AgentMemory.Tests.Unit/Services/MemoryServiceAccessTrackingTests.cs @@ -72,14 +72,132 @@ public MemoryServiceAccessTrackingTests() return (IReadOnlyCollection<(string NodeId, MemoryNodeKind NodeKind)>)call.GetArguments()[0]!; } - private MemoryService CreateSut(IMemoryDecayService? decay = null) => + private MemoryService CreateSut(IMemoryDecayService? decay = null, bool deferred = false) => new(_shortTerm, _assembler, _extractionPipeline, _entityRepository, _factRepository, _preferenceRepository, _embeddingOrchestrator, - Options.Create(new MemoryOptions()), + Options.Create(new MemoryOptions { DeferAccessTracking = deferred }), _clock, _idGenerator, NullLogger.Instance, decay); + // ── 2.4: taking the bookkeeping write off the blocking path ────────── + + /// A context with one recalled fact, enough to trigger access tracking. + private MemoryContext OneFactContext() => new() + { + SessionId = "s1", + AssembledAtUtc = _fixedTime, + RelevantFacts = new MemoryContextSection + { + Items = new[] + { + new Fact + { + FactId = "fact-1", Subject = "s", Predicate = "p", Object = "o", + Confidence = 0.9, CreatedAtUtc = _fixedTime, + }, + }, + }, + }; + + [Fact] + public async Task DeferredAccessTrackingDoesNotBlockTheRecall() + { + // THE point. Access tracking feeds decay and retention; nothing in the returned context + // depends on it, so the caller should not wait for a write burst before the model is invoked. + // The substitute never completes, so a recall that returns proves it was not awaited. + var pending = new TaskCompletionSource(); + var decay = Substitute.For(); + decay.UpdateAccessTimestampsAsync( + Arg.Any>(), Arg.Any()) + .Returns(pending.Task); + _assembler.AssembleContextAsync(Arg.Any(), Arg.Any()) + .Returns(OneFactContext()); + + var recall = CreateSut(decay, deferred: true) + .RecallAsync(new RecallRequest { SessionId = "s1", Query = "q" }); + + (await Task.WhenAny(recall, Task.Delay(TimeSpan.FromSeconds(5)))).Should().Be(recall, + "the recall must return without waiting for the bookkeeping write"); + pending.SetResult(); + } + + [Fact] + public async Task TheDefaultStillAwaitsTheWrite() + { + // The byte-identical guarantee: leaving the option off keeps failures and cancellation + // observable on the calling path, which is why it was awaited in the first place. + var completed = false; + var decay = Substitute.For(); + decay.UpdateAccessTimestampsAsync( + Arg.Any>(), Arg.Any()) + .Returns(async _ => + { + await Task.Delay(20); + completed = true; + }); + _assembler.AssembleContextAsync(Arg.Any(), Arg.Any()) + .Returns(OneFactContext()); + + await CreateSut(decay).RecallAsync(new RecallRequest { SessionId = "s1", Query = "q" }); + + completed.Should().BeTrue("the default path awaits the write"); + } + + [Fact] + public async Task ADeferredWriteIsDetachedFromTheRequestCancellationToken() + { + // The trap that would make this a no-op wearing a feature's name. The request token is + // cancelled as soon as the response completes, so a deferred write that still carried it + // would be cancelled by the very act of returning. + CancellationToken captured = default; + var decay = Substitute.For(); + decay.UpdateAccessTimestampsAsync( + Arg.Any>(), Arg.Any()) + .Returns(call => + { + captured = call.ArgAt(1); + return Task.CompletedTask; + }); + _assembler.AssembleContextAsync(Arg.Any(), Arg.Any()) + .Returns(OneFactContext()); + + using var requestCts = new CancellationTokenSource(); + await CreateSut(decay, deferred: true) + .RecallAsync(new RecallRequest { SessionId = "s1", Query = "q" }, requestCts.Token); + await Task.Delay(50); + requestCts.Cancel(); + + captured.Should().Be(CancellationToken.None, + "a deferred write must outlive the request that scheduled it"); + } + + [Fact] + public async Task ADeferredFailureDoesNotFaultTheRecall() + { + // A bookkeeping write that throws must not surface as a failed recall -- and must not become + // an unobserved task exception either, which is why the continuation logs it. + var decay = Substitute.For(); + decay.UpdateAccessTimestampsAsync( + Arg.Any>(), Arg.Any()) + .Returns(_ => throw new InvalidOperationException("driver disposed")); + _assembler.AssembleContextAsync(Arg.Any(), Arg.Any()) + .Returns(OneFactContext()); + + var act = async () => await CreateSut(decay, deferred: true) + .RecallAsync(new RecallRequest { SessionId = "s1", Query = "q" }); + + await act.Should().NotThrowAsync(); + } + + [Fact] + public void DeferralIsOffByDefault() + { + // It is unsafe for a request-scoped host: the write runs on scoped services, and a scope + // disposed when the response returns takes the driver session with it. + new MemoryOptions().DeferAccessTracking.Should().BeFalse(); + } + [Fact] public async Task RecallAsync_WithDecayService_UpdatesEntityAccess() { diff --git a/tests/AgentMemory.Tests.Unit/Services/MemoryServiceExtractionContextTests.cs b/tests/AgentMemory.Tests.Unit/Services/MemoryServiceExtractionContextTests.cs new file mode 100644 index 000000000..bd8024913 --- /dev/null +++ b/tests/AgentMemory.Tests.Unit/Services/MemoryServiceExtractionContextTests.cs @@ -0,0 +1,157 @@ +using AgentMemory.Abstractions.Domain; +using AgentMemory.Abstractions.Options; +using AgentMemory.Abstractions.Repositories; +using AgentMemory.Abstractions.Services; +using AgentMemory.Core.Services; +using FluentAssertions; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using NSubstitute; +using Xunit; + +namespace AgentMemory.Tests.Unit.Services; + +/// +/// The wiring half of E2: the extraction window has to be filled in, once, where every host passes. +/// +/// +/// +/// and its rendering have their own +/// tests, and none of them prove a single context turn ever reaches an extractor. The window would be +/// correct, the prompt instruction would be authored, and every extraction would still see one batch. +/// +/// +/// It is filled at ExtractAndPersistAsync rather than per caller because the Agent Framework +/// provider, the Microsoft memory facade and the MCP ingest tool all arrive there. Resolving it per +/// caller would make the window depend on the host — a setting only some components respect, which is +/// the failure this codebase has hit repeatedly. +/// +/// +public sealed class MemoryServiceExtractionContextTests +{ + private readonly IShortTermMemoryService _shortTerm = Substitute.For(); + private readonly IMemoryExtractionPipeline _pipeline = Substitute.For(); + + private static Message M(string id, string content) => new() + { + MessageId = id, + ConversationId = "c-1", + SessionId = "s-1", + Role = "user", + Content = content, + TimestampUtc = DateTimeOffset.UnixEpoch, + }; + + public MemoryServiceExtractionContextTests() + { + _pipeline.ExtractAsync(Arg.Any(), Arg.Any()) + .Returns(Task.FromResult(new ExtractionResult())); + } + + private MemoryService CreateSut(int contextTurns) => + new(_shortTerm, + Substitute.For(), + _pipeline, + Substitute.For(), + Substitute.For(), + Substitute.For(), + Substitute.For(), + Options.Create(new MemoryOptions + { + Extraction = new ExtractionOptions { ExtractionContextTurns = contextTurns }, + }), + Substitute.For(), + Substitute.For(), + NullLogger.Instance); + + private void SessionHolds(params Message[] messages) => + _shortTerm.GetRecentMessagesAsync(Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(Task.FromResult>(messages)); + + private ExtractionRequest CapturedRequest() => + (ExtractionRequest)_pipeline.ReceivedCalls() + .Single(c => c.GetMethodInfo().Name == nameof(IMemoryExtractionPipeline.ExtractAsync)) + .GetArguments()[0]!; + + private Task ExtractAsync(int contextTurns, params Message[] targets) => + CreateSut(contextTurns).ExtractAndPersistAsync( + new ExtractionRequest { Messages = targets, SessionId = "s-1" }); + + [Fact] + public async Task PrecedingTurnsAreAttachedAsContext() + { + // THE test. Everything else in E2 is arrangement around these turns reaching the extractor. + SessionHolds(M("m-1", "Zurich is lovely"), M("m-2", "Agreed."), M("m-3", "I moved there")); + + await ExtractAsync(2, M("m-3", "I moved there")); + + CapturedRequest().ContextMessages.Select(m => m.MessageId).Should().Equal("m-1", "m-2"); + } + + [Fact] + public async Task TheBatchItselfIsNeverItsOwnContext() + { + // The recency query returns the batch too -- it was just persisted. Leaving it in would hand + // the model the same turn twice, the second copy labelled "do not extract from this". + SessionHolds(M("m-1", "earlier"), M("m-2", "target")); + + await ExtractAsync(4, M("m-2", "target")); + + CapturedRequest().ContextMessages.Select(m => m.MessageId).Should().Equal("m-1"); + } + + [Fact] + public async Task OnlyTheMostRecentTurnsAreKept() + { + SessionHolds(M("m-1", "oldest"), M("m-2", "older"), M("m-3", "old"), M("m-4", "target")); + + await ExtractAsync(2, M("m-4", "target")); + + // Note the collection overload: Equal("m-2", "m-3", "because...") would fold the reason into + // the expected array as a third element. + CapturedRequest().ContextMessages.Select(m => m.MessageId).Should().Equal( + ["m-2", "m-3"], + "context is the turns immediately before the batch, not the start of the session"); + } + + [Fact] + public async Task ZeroTurnsQueriesNothingAtAll() + { + // The off state, and it must be off at the source: a fetch whose result is then discarded + // still costs a round trip on every single extraction. + await ExtractAsync(0, M("m-2", "target")); + + await _shortTerm.DidNotReceiveWithAnyArgs() + .GetRecentMessagesAsync(default!, default, default); + CapturedRequest().ContextMessages.Should().BeEmpty(); + } + + [Fact] + public async Task AnExplicitContextIsNotOverwritten() + { + // A caller that knows the conversation better than a recency query does should not be + // second-guessed. + SessionHolds(M("m-1", "from the store"), M("m-9", "target")); + + await CreateSut(4).ExtractAndPersistAsync(new ExtractionRequest + { + Messages = [M("m-9", "target")], + SessionId = "s-1", + ContextMessages = [M("m-7", "chosen by the caller")], + }); + + CapturedRequest().ContextMessages.Select(m => m.MessageId).Should().Equal("m-7"); + } + + [Fact] + public async Task NoEarlierTurnsLeavesTheRequestUntouched() + { + // First turn of a session: nothing precedes it, and an empty context must stay empty rather + // than becoming a fenced block with nothing inside it. + SessionHolds(M("m-1", "target")); + + await ExtractAsync(4, M("m-1", "target")); + + CapturedRequest().ContextMessages.Should().BeEmpty(); + } +} diff --git a/tests/AgentMemory.Tests.Unit/Services/MemoryServiceTemporalRoutingTests.cs b/tests/AgentMemory.Tests.Unit/Services/MemoryServiceTemporalRoutingTests.cs new file mode 100644 index 000000000..d1496aa02 --- /dev/null +++ b/tests/AgentMemory.Tests.Unit/Services/MemoryServiceTemporalRoutingTests.cs @@ -0,0 +1,158 @@ +using FluentAssertions; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using AgentMemory.Abstractions.Domain; +using AgentMemory.Abstractions.Options; +using AgentMemory.Abstractions.Repositories; +using AgentMemory.Abstractions.Services; +using AgentMemory.Core.Services; +using NSubstitute; +using Xunit; + +namespace AgentMemory.Tests.Unit.Services; + +/// +/// The wiring half of R4: a turn that names a past time must actually reach bitemporal recall. +/// +/// +/// +/// has its own tests and they prove only that +/// a string becomes an instant. That is the half that cannot fail quietly. A parser nothing calls +/// passes every one of its own tests while RecallAsOfAsync stays exactly as unreachable as it was +/// before — which is the shape this track has now hit four separate times. +/// +/// +/// So these tests assert on the seam: which assembler method was invoked, and what the recall reported +/// about the clocks it used. +/// +/// +public sealed class MemoryServiceTemporalRoutingTests +{ + private readonly IMemoryContextAssembler _assembler = Substitute.For(); + private readonly IClock _clock = Substitute.For(); + + private static readonly DateTimeOffset Now = new(2026, 8, 12, 12, 0, 0, TimeSpan.Zero); + + public MemoryServiceTemporalRoutingTests() + { + _clock.UtcNow.Returns(Now); + + var empty = new MemoryContext { SessionId = "s-1", AssembledAtUtc = Now }; + _assembler.AssembleContextAsync(Arg.Any(), Arg.Any()) + .Returns(Task.FromResult(empty)); + _assembler.AssembleContextAsOfAsync( + Arg.Any(), Arg.Any(), Arg.Any(), + Arg.Any()) + .Returns(Task.FromResult(empty)); + } + + private MemoryService CreateSut(bool resolveTemporalQueries) => + new(Substitute.For(), + _assembler, + Substitute.For(), + Substitute.For(), + Substitute.For(), + Substitute.For(), + Substitute.For(), + Options.Create(new MemoryOptions { ResolveTemporalQueries = resolveTemporalQueries }), + _clock, + Substitute.For(), + NullLogger.Instance); + + private Task RecallAsync(string query, bool enabled = true) => + CreateSut(enabled).RecallAsync(new RecallRequest { SessionId = "s-1", Query = query }); + + [Fact] + public async Task ATemporalQueryReachesBitemporalRecall() + { + // THE test. Everything else in R4 is arrangement around this one call actually being made. + var result = await RecallAsync("what did I think back in March"); + + await _assembler.Received(1).AssembleContextAsOfAsync( + Arg.Any(), Arg.Any(), Arg.Any(), + Arg.Any()); + await _assembler.DidNotReceive().AssembleContextAsync( + Arg.Any(), Arg.Any()); + + result.Metadata["validAsOf"].Should() + .Be(new DateTimeOffset(2026, 3, 31, 23, 59, 59, TimeSpan.Zero)); + } + + [Fact] + public async Task BothClocksMoveTogether() + { + // "What did I think in March" asks what was true then AS KNOWN THEN. Moving only the valid + // clock would answer with today's corrections applied to the past -- a different question, and + // a subtly misleading one, because the answer would look like a faithful reconstruction. + await RecallAsync("what did I think back in March"); + + await _assembler.Received(1).AssembleContextAsOfAsync( + Arg.Any(), + Arg.Is(v => v == new DateTimeOffset(2026, 3, 31, 23, 59, 59, TimeSpan.Zero)), + Arg.Is(s => s == new DateTimeOffset(2026, 3, 31, 23, 59, 59, TimeSpan.Zero)), + Arg.Any()); + } + + [Fact] + public async Task AnOrdinaryQueryTakesTheOrdinaryPath() + { + // The common case, and the one that must not regress: no temporal marker means the live path, + // byte for byte as before. + await RecallAsync("what is my deploy command"); + + await _assembler.Received(1).AssembleContextAsync( + Arg.Any(), Arg.Any()); + await _assembler.DidNotReceive().AssembleContextAsOfAsync( + Arg.Any(), Arg.Any(), Arg.Any(), + Arg.Any()); + } + + [Fact] + public async Task TheOptionIsOffByDefault() + { + // It changes which memories a temporal question sees, so it is opt-in -- and every sealed + // measurement in this track was taken with it off. + new MemoryOptions().ResolveTemporalQueries.Should().BeFalse(); + } + + [Fact] + public async Task DisabledMeansTheParserNeverRuns() + { + // The off switch has to be genuinely off, not merely default-off: a temporal query with the + // option disabled must take exactly the path it took before R4 existed. + await RecallAsync("what did I think back in March", enabled: false); + + await _assembler.Received(1).AssembleContextAsync( + Arg.Any(), Arg.Any()); + await _assembler.DidNotReceive().AssembleContextAsOfAsync( + Arg.Any(), Arg.Any(), Arg.Any(), + Arg.Any()); + } + + [Fact] + public async Task TheParserResolvesAgainstTheInjectedClock() + { + // Not DateTimeOffset.UtcNow. "Last week" is meaningless without a reference instant, and a + // parser reading the wall clock while the rest of the service reads IClock would drift in + // exactly the tests written to pin temporal behaviour down. + _clock.UtcNow.Returns(new DateTimeOffset(2020, 6, 15, 0, 0, 0, TimeSpan.Zero)); + + var result = await RecallAsync("what did I think last week"); + + result.Metadata["validAsOf"].Should() + .Be(new DateTimeOffset(2020, 6, 8, 0, 0, 0, TimeSpan.Zero)); + } + + [Theory] + [InlineData("")] + [InlineData(" ")] + public async Task ABlankQueryTakesTheOrdinaryPath(string query) + { + // Recall with a blank query is legal -- it assembles recent context. The temporal branch must + // neither throw on it nor treat "no query" as some resolvable instant. + await RecallAsync(query); + + await _assembler.Received(1).AssembleContextAsync( + Arg.Any(), Arg.Any()); + } +} diff --git a/tests/AgentMemory.Tests.Unit/Services/MentionFrequencyRerankerTests.cs b/tests/AgentMemory.Tests.Unit/Services/MentionFrequencyRerankerTests.cs new file mode 100644 index 000000000..683aabfda --- /dev/null +++ b/tests/AgentMemory.Tests.Unit/Services/MentionFrequencyRerankerTests.cs @@ -0,0 +1,199 @@ +using AgentMemory.Abstractions.Domain; +using AgentMemory.Abstractions.Options; +using AgentMemory.Abstractions.Services; +using AgentMemory.Neo4j.Infrastructure; +using AgentMemory.Neo4j.Queries; +using AgentMemory.Neo4j.Services; +using FluentAssertions; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Neo4j.Driver; +using NSubstitute; +using Xunit; + +namespace AgentMemory.Tests.Unit.Services; + +/// +/// Mention-frequency reranking (R7), and the loop it must not become. +/// +/// +/// +/// A fact the world asserted five times is usually more central than one mentioned once, and +/// similarity cannot see that: two facts phrased alike score alike however often either was said. +/// +/// +/// The signal's provenance is the task. The counter must record how often the world +/// re-asserted a fact, never how often we surfaced it. Ranking on our own retrievals is a +/// rich-get-richer loop — whatever ranks highly gets retrieved, retrieval raises the count, the count +/// raises the rank — which looks like learning and is self-reinforcement. The tests below pin the +/// source of the number, not only its effect. +/// +/// +public sealed class MentionFrequencyRerankerTests +{ + private readonly INeo4jTransactionRunner _tx = Substitute.For(); + + private MentionFrequencyReranker CreateSut(bool enabled = true) => + new(_tx, + NullLogger.Instance, + Options.Create(new MemoryOptions { MentionFrequencyReranking = enabled })); + + private static MemoryRerankContext Context(MemoryItemKind kind = MemoryItemKind.Fact) => + new("where does alice live", [0.1f, 0.2f], MemoryScope.For("alice"), kind); + + private void Mentions(params (string Id, int Count)[] counts) => + _tx.ReadAsync( + Arg.Any>>>(), + Arg.Any()) + .Returns((IReadOnlyDictionary)counts.ToDictionary( + c => c.Id, c => c.Count, StringComparer.Ordinal)); + + // ── the signal's provenance ─────────────────────────────────────────── + + [Fact] + public void TheCounterIsIncrementedByIngestionAndNothingElse() + { + // THE constraint. mention_count appears in the upsert paths -- where a new ingestion re-states + // an existing triple -- and is read by exactly one query. If it ever appears in a retrieval + // path, the loop is closed and the ranking starts reinforcing itself. + var upsert = FactQueries.Upsert + FactQueries.UpsertBatch; + + upsert.Should().Contain("f.mention_count = 1", "a first assertion counts once"); + upsert.Should().Contain("coalesce(f.mention_count, 1) + 1", "a re-assertion counts again"); + FactQueries.MentionCounts.Should().Contain("coalesce(f.mention_count, 1)"); + } + + [Fact] + public void TheReadAuditIsNeverConsulted() + { + // The substitution this task exists to refuse. :MemoryReadAudit records our own retrievals; + // ranking on it would promote whatever already ranks highly, on evidence we generated. + FactQueries.MentionCounts.Should().NotContain("MemoryReadAudit"); + FactQueries.MentionCounts.Should().NotContain("read_count"); + } + + [Fact] + public void APreExistingFactCountsAsMentionedOnce() + { + // Facts written before the property existed have no counter. One is the honest reading of an + // absent count -- it was asserted at least once -- and it leaves their ranking unchanged, + // since log(1) is 0. + FactQueries.MentionCounts.Should().Contain("coalesce(f.mention_count, 1)"); + } + + // ── the boost ───────────────────────────────────────────────────────── + + [Fact] + public async Task AWellAttestedFactOvertakesANearTie() + { + // Salience breaks a near-tie: 0.80*(1+0.15*ln 20) = 1.16 beats 0.82*(1+0) = 0.82. + Mentions(("often", 20), ("once", 1)); + var candidates = new[] + { + new MemoryContextRankedItem("once", 0.82, 1, 1), + new MemoryContextRankedItem("often", 0.80, 2, 2), + }; + + var result = await CreateSut().RerankAsync(candidates, Context()); + + result.Select(r => r.ItemId).Should().Equal("often", "once"); + } + + [Fact] + public async Task ChatterDoesNotBuryAPreciseAnswer() + { + // The failure a linear boost would cause. A fact mentioned 50 times must not outrank a much + // better match: 0.30*(1+0.15*ln 50) = 0.48 stays below 0.95*(1+0) = 0.95. Salience is a + // tiebreaker among plausible answers, not a substitute for answering the question. + Mentions(("chatter", 50), ("precise", 1)); + var candidates = new[] + { + new MemoryContextRankedItem("precise", 0.95, 1, 1), + new MemoryContextRankedItem("chatter", 0.30, 2, 2), + }; + + var result = await CreateSut().RerankAsync(candidates, Context()); + + result.Select(r => r.ItemId).Should().Equal("precise", "chatter"); + } + + [Fact] + public async Task TheBoostIsLogarithmicNotLinear() + { + // The gap between 1 and 3 mentions is real; between 30 and 32 it is noise. Under a linear + // boost the 32-mention fact would win here; under a log boost the stronger match holds. + Mentions(("many", 32), ("fewer", 30)); + var candidates = new[] + { + new MemoryContextRankedItem("fewer", 0.90, 1, 1), + new MemoryContextRankedItem("many", 0.89, 2, 2), + }; + + var result = await CreateSut().RerankAsync(candidates, Context()); + + result.Select(r => r.ItemId).Should().Equal("fewer", "many"); + } + + [Fact] + public async Task ScoresAreNeverRewritten() + { + // A reranker reorders. Rewriting the provider's score would corrupt the section diagnostics, + // which report TopScore from these same items. + Mentions(("a", 10), ("b", 1)); + var candidates = new[] + { + new MemoryContextRankedItem("a", 0.70, 1, 1), + new MemoryContextRankedItem("b", 0.90, 2, 2), + }; + + var result = await CreateSut().RerankAsync(candidates, Context()); + + result.Should().OnlyContain(r => r.Score == 0.70 || r.Score == 0.90); + result.Select(r => r.ItemId).Should().BeEquivalentTo(["a", "b"]); + } + + // ── the gates ───────────────────────────────────────────────────────── + + [Fact] + public void ItIsOffByDefault() + { + new MemoryOptions().MentionFrequencyReranking.Should().BeFalse(); + CreateSut(enabled: false).IsEnabled.Should().BeFalse(); + } + + [Fact] + public async Task ANonFactSectionIsLeftAloneWithoutQuerying() + { + // mention_count is maintained on Fact only. Boosting another kind by a counter nothing + // increments would be a no-op dressed as a feature -- and would still cost a query. + var candidates = new[] + { + new MemoryContextRankedItem("a", 0.9, 1, 1), + new MemoryContextRankedItem("b", 0.8, 2, 2), + }; + + var result = await CreateSut().RerankAsync(candidates, Context(MemoryItemKind.Entity)); + + result.Select(r => r.ItemId).Should().Equal("a", "b"); + await _tx.DidNotReceive().ReadAsync( + Arg.Any>>>(), + Arg.Any()); + } + + [Fact] + public async Task EqualSalienceLeavesTheProviderOrderIntact() + { + // Ties break on retrieval rank rather than arbitrarily, so enabling the reranker on a corpus + // where every fact was asserted once changes nothing at all. + Mentions(("a", 1), ("b", 1)); + var candidates = new[] + { + new MemoryContextRankedItem("a", 0.9, 1, 1), + new MemoryContextRankedItem("b", 0.9, 2, 2), + }; + + var result = await CreateSut().RerankAsync(candidates, Context()); + + result.Select(r => r.ItemId).Should().Equal("a", "b"); + } +} diff --git a/tests/AgentMemory.Tests.Unit/Services/NodeDistanceRerankerTests.cs b/tests/AgentMemory.Tests.Unit/Services/NodeDistanceRerankerTests.cs new file mode 100644 index 000000000..5cb5da287 --- /dev/null +++ b/tests/AgentMemory.Tests.Unit/Services/NodeDistanceRerankerTests.cs @@ -0,0 +1,235 @@ +using AgentMemory.Abstractions.Domain; +using AgentMemory.Abstractions.Options; +using AgentMemory.Abstractions.Services; +using AgentMemory.Neo4j.Infrastructure; +using AgentMemory.Neo4j.Services; +using FluentAssertions; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Neo4j.Driver; +using NSubstitute; +using Xunit; + +namespace AgentMemory.Tests.Unit.Services; + +/// +/// Node-distance reranking (R6): boost what the query is about, not only what it looks like. +/// +/// +/// +/// The gates matter as much as the boost. This runs on the blocking recall path and issues two +/// queries per fact section, so every condition under which it declines to run is a cost avoided — +/// and a reranker that fires when it cannot change anything is pure overhead wearing a feature's name. +/// +/// +/// Shipped behind the retrieval-sufficiency work deliberately: reranking reorders survivors, so over a +/// starved candidate set it reorders seven of an owner's 504 facts and reports success. The signal +/// measured AUC 0.709–0.768 before this was built. +/// +/// +public sealed class NodeDistanceRerankerTests +{ + private readonly INeo4jTransactionRunner _tx = Substitute.For(); + + private NodeDistanceReranker CreateSut(bool enabled = true, double gamma = 0.5) => + new(_tx, + NullLogger.Instance, + Options.Create(new MemoryRankingOptions { StructuralDecayGamma = gamma }), + Options.Create(new MemoryOptions { NodeDistanceReranking = enabled })); + + private static MemoryRerankContext Context( + MemoryItemKind kind = MemoryItemKind.Fact, float[]? embedding = null) => + new("who does alice work for", embedding ?? [0.1f, 0.2f], MemoryScope.For("alice"), kind); + + private static IReadOnlyList Candidates() => + [ + new("fact-a", Score: 0.90, RetrievalRank: 1, ContextRank: 1), + new("fact-b", Score: 0.80, RetrievalRank: 2, ContextRank: 2), + ]; + + // ── the gates ───────────────────────────────────────────────────────── + + [Fact] + public void ItIsOffByDefault() + { + // It changes recall ordering and costs two queries per section; every recorded measurement was + // taken without it. + new MemoryOptions().NodeDistanceReranking.Should().BeFalse(); + CreateSut(enabled: false).IsEnabled.Should().BeFalse(); + } + + [Fact] + public void ItStaysOffWhenTheDecayConstantIsOne() + { + // At gamma = 1 every boost is 1.0, so the two queries could not change an ordering. Running + // them anyway would be pure cost -- and it is the same constant GraphRAG hop-decay uses, so + // "structural decay off" means off for both rather than for one of them. + CreateSut(enabled: true, gamma: 1.0).IsEnabled.Should().BeFalse(); + CreateSut(enabled: true, gamma: 0.5).IsEnabled.Should().BeTrue(); + } + + [Fact] + public async Task ANonFactSectionIsLeftAlone() + { + // The distance query walks ABOUT/RELATED_TO from an entity to a Fact. Reordering traces or + // preferences by a measure that does not apply to them would be worse than not running. + var result = await CreateSut().RerankAsync( + Candidates(), Context(kind: MemoryItemKind.Preference)); + + result.Should().BeSameAs(await Task.FromResult(result)); + result.Select(r => r.ItemId).Should().Equal("fact-a", "fact-b"); + await _tx.DidNotReceive().ReadAsync( + Arg.Any>>(), Arg.Any()); + } + + [Fact] + public async Task AMissingQueryEmbeddingCostsNothing() + { + // A turn narrowed to categories needing no vector has no embedding, and the centroid lookup is + // a vector search. Asking for one anyway would reintroduce exactly the round trip the trivial + // turn policy exists to elide. + var result = await CreateSut().RerankAsync(Candidates(), Context(embedding: [])); + + result.Select(r => r.ItemId).Should().Equal("fact-a", "fact-b"); + await _tx.DidNotReceive().ReadAsync( + Arg.Any>>(), Arg.Any()); + } + + [Fact] + public async Task ASingleCandidateIsNotWorthTwoQueries() + { + // Nothing to reorder. The cheapest correct answer is the one already held. + var single = new[] { new MemoryContextRankedItem("fact-a", 0.9, 1, 1) }; + + var result = await CreateSut().RerankAsync(single, Context()); + + result.Should().BeEquivalentTo(single); + await _tx.DidNotReceive().ReadAsync( + Arg.Any>>(), Arg.Any()); + } + + // ── the boost ───────────────────────────────────────────────────────── + + [Fact] + public async Task ACloserCandidateOvertakesAHigherScoringDistantOne() + { + // The whole point. fact-b scores lower on similarity but sits one hop from the entity the + // query names, while fact-a is four hops away: 0.80*(1+0.5^1)=1.20 beats 0.90*(1+0.5^4)=0.956. + _tx.ReadAsync(Arg.Any>>(), Arg.Any()) + .Returns("entity-1"); + _tx.ReadAsync( + Arg.Any>>>(), + Arg.Any()) + .Returns((IReadOnlyDictionary)new Dictionary(StringComparer.Ordinal) + { + ["fact-a"] = 4, + ["fact-b"] = 1, + }); + + var result = await CreateSut().RerankAsync(Candidates(), Context()); + + result.Select(r => r.ItemId).Should().Equal("fact-b", "fact-a"); + result[0].ContextRank.Should().Be(1); + result[1].ContextRank.Should().Be(2); + } + + [Fact] + public async Task ProximityDoesNotOverrideAStrongSimilarityGap() + { + // Graph closeness is evidence about relevance, not a replacement for it. The boost is + // multiplicative so a candidate retrieval ranked badly is not promoted past a strong one by + // adjacency alone: 0.90*(1+0.5^4)=0.956 still beats 0.20*(1+0.5)=0.30. + _tx.ReadAsync(Arg.Any>>(), Arg.Any()) + .Returns("entity-1"); + _tx.ReadAsync( + Arg.Any>>>(), + Arg.Any()) + .Returns((IReadOnlyDictionary)new Dictionary(StringComparer.Ordinal) + { + ["fact-a"] = 4, + ["weak"] = 1, + }); + + var candidates = new[] + { + new MemoryContextRankedItem("fact-a", 0.90, 1, 1), + new MemoryContextRankedItem("weak", 0.20, 2, 2), + }; + + var result = await CreateSut().RerankAsync(candidates, Context()); + + result.Select(r => r.ItemId).Should().Equal("fact-a", "weak"); + } + + [Fact] + public async Task AnUnreachableCandidateKeepsItsScoreExactly() + { + // Absent from the distance result means "no path within the cap", which the caller reads as + // no boost. A sentinel distance would invite arithmetic on a number meaning "unreachable". + _tx.ReadAsync(Arg.Any>>(), Arg.Any()) + .Returns("entity-1"); + _tx.ReadAsync( + Arg.Any>>>(), + Arg.Any()) + .Returns((IReadOnlyDictionary)new Dictionary(StringComparer.Ordinal) + { + ["fact-b"] = 1, + }); + + var result = await CreateSut().RerankAsync(Candidates(), Context()); + + // fact-b: 0.80*1.5 = 1.20; fact-a unreachable, keeps 0.90. + result.Select(r => r.ItemId).Should().Equal("fact-b", "fact-a"); + result.Should().OnlyContain(r => r.Score == 0.90 || r.Score == 0.80, + "the reranker reorders; it must not rewrite the provider's scores"); + } + + [Fact] + public async Task NoCentroidMeansNoReordering() + { + // An owner whose graph holds no entity resembling the query. Returning the provider order is + // correct and costs one query rather than two. + _tx.ReadAsync(Arg.Any>>(), Arg.Any()) + .Returns((string?)null); + + var result = await CreateSut().RerankAsync(Candidates(), Context()); + + result.Select(r => r.ItemId).Should().Equal("fact-a", "fact-b"); + } + + [Fact] + public async Task NoCandidateWithinRangeLeavesTheOrderUntouched() + { + _tx.ReadAsync(Arg.Any>>(), Arg.Any()) + .Returns("entity-1"); + _tx.ReadAsync( + Arg.Any>>>(), + Arg.Any()) + .Returns((IReadOnlyDictionary)new Dictionary(StringComparer.Ordinal)); + + var result = await CreateSut().RerankAsync(Candidates(), Context()); + + result.Select(r => r.ItemId).Should().Equal("fact-a", "fact-b"); + } + + [Fact] + public async Task TheCandidateSetIsNeverChanged() + { + // A reranker reorders. Adding or dropping a candidate would corrupt the section's diagnostics + // silently, because Returned is counted before this runs. + _tx.ReadAsync(Arg.Any>>(), Arg.Any()) + .Returns("entity-1"); + _tx.ReadAsync( + Arg.Any>>>(), + Arg.Any()) + .Returns((IReadOnlyDictionary)new Dictionary(StringComparer.Ordinal) + { + ["fact-b"] = 2, + }); + + var result = await CreateSut().RerankAsync(Candidates(), Context()); + + result.Select(r => r.ItemId).Should().BeEquivalentTo(["fact-a", "fact-b"]); + result.Should().HaveCount(2); + } +} diff --git a/tests/AgentMemory.Tests.Unit/Services/ReasoningMemoryServiceTests.cs b/tests/AgentMemory.Tests.Unit/Services/ReasoningMemoryServiceTests.cs index 9037714d9..1bd295b0e 100644 --- a/tests/AgentMemory.Tests.Unit/Services/ReasoningMemoryServiceTests.cs +++ b/tests/AgentMemory.Tests.Unit/Services/ReasoningMemoryServiceTests.cs @@ -89,7 +89,14 @@ public async Task StartTraceAsync_CallerSuppliedTrustLevel_IsStripped() var result = await sut.StartTraceAsync("session-1", "Test task", metadata: callerMetadata); - result.Metadata.GetTrustLevel().Should().Be(MemoryTrustLevel.Untrusted); + // Asserts the caller's value was discarded, NOT that the result is Untrusted. Those were the + // same thing until 6.3, when traces began carrying a configured trust level -- so the old + // assertion was checking a proxy that has since stopped tracking the property. What must hold + // is that ApplicationTrusted, which the caller asked for and which would bypass the admission + // policy, is not what got stored. + result.Metadata.GetTrustLevel().Should().NotBe(MemoryTrustLevel.ApplicationTrusted); + result.Metadata.GetTrustLevel().Should() + .Be(new ReasoningMemoryOptions().DefaultTraceTrustLevel); } [Fact] diff --git a/tests/AgentMemory.Tests.Unit/Services/RecallLatencyBudgetTests.cs b/tests/AgentMemory.Tests.Unit/Services/RecallLatencyBudgetTests.cs new file mode 100644 index 000000000..0648df358 --- /dev/null +++ b/tests/AgentMemory.Tests.Unit/Services/RecallLatencyBudgetTests.cs @@ -0,0 +1,190 @@ +using AgentMemory.Abstractions.Domain; +using AgentMemory.Abstractions.Options; +using AgentMemory.Abstractions.Services; +using AgentMemory.Core.Services; +using FluentAssertions; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using NSubstitute; +using Xunit; + +namespace AgentMemory.Tests.Unit.Services; + +/// +/// Degrading under load without pretending otherwise (rank 13). +/// +/// +/// +/// A recall latency budget is easy to add and easy to get dangerously wrong. The dangerous version +/// returns whatever arrived in time and says nothing: a section cut short reports Searched and +/// Returned = 0, which is byte-identical to a section that ran to completion and genuinely +/// found nothing. +/// +/// +/// A partial recall that looks complete is worse than a slow one. The caller answers +/// confidently from memory that was never consulted, and nothing anywhere records that it happened. +/// So most of what follows is about the marking, not the timing. +/// +/// +public sealed class RecallLatencyBudgetTests +{ + private readonly IShortTermMemoryService _shortTerm = Substitute.For(); + private readonly ILongTermMemoryService _longTerm = Substitute.For(); + private readonly IReasoningMemoryService _reasoning = Substitute.For(); + private readonly IEmbeddingOrchestrator _embeddings = Substitute.For(); + + private static readonly Message AMessage = new() + { + MessageId = "m-1", ConversationId = "c-1", SessionId = "s-1", + Role = "user", Content = "hello", TimestampUtc = DateTimeOffset.UnixEpoch, + }; + + public RecallLatencyBudgetTests() + { + _embeddings.EmbedQueryAsync(Arg.Any(), Arg.Any()) + .Returns(Task.FromResult([0.1f, 0.2f])); + + _shortTerm.GetRecentMessagesAsync(Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(Task.FromResult>([AMessage])); + _shortTerm.SearchMessagesAsync( + Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any(), + Arg.Any()) + .Returns(Task.FromResult>([])); + + _longTerm.SearchPreferencesAsync( + Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(Task.FromResult>([])); + _reasoning.SearchSimilarTracesAsync( + Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any(), + Arg.Any(), Arg.Any()) + .Returns(Task.FromResult>([])); + } + + /// Entities return promptly; facts hang past any sane budget. + private void FactsAreSlow(TimeSpan delay) + { + _longTerm.SearchEntitiesAsync( + Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(Task.FromResult>( + [ + new Entity + { + EntityId = "e-1", Name = "Alice", Type = "Person", + Confidence = 0.9, CreatedAtUtc = DateTimeOffset.UnixEpoch, + }, + ])); + + _longTerm.SearchFactsAsync( + Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(async _ => + { + await Task.Delay(delay).ConfigureAwait(false); + return (IReadOnlyList)[]; + }); + } + + private MemoryContextAssembler CreateSut() => + new(_shortTerm, _longTerm, _reasoning, + graphRag: null, + _embeddings, + Substitute.For(), + Options.Create(new MemoryOptions()), + NullLogger.Instance, + Substitute.For()); + + private Task AssembleAsync(TimeSpan? budget) => + CreateSut().AssembleContextAsync(new RecallRequest + { + SessionId = "s-1", + Query = "what do I know", + Options = new RecallOptions + { + IncludeDiagnostics = true, + LatencyBudget = budget, + }, + }); + + [Fact] + public async Task ASlowSectionIsDroppedRatherThanWaitedFor() + { + FactsAreSlow(TimeSpan.FromSeconds(30)); + + var context = await AssembleAsync(TimeSpan.FromMilliseconds(150)); + + context.LatencyBudgetExceeded.Should().BeTrue(); + } + + [Fact] + public async Task TheDroppedSectionSaysSoAndTheOthersDoNot() + { + // THE test. Without this, "cut short" and "searched and found nothing" are the same section, + // and degradation under load is invisible in every artifact and every log. + FactsAreSlow(TimeSpan.FromSeconds(30)); + + var context = await AssembleAsync(TimeSpan.FromMilliseconds(150)); + + context.RelevantFacts.Diagnostics!.TimedOut.Should().BeTrue(); + context.RelevantFacts.Diagnostics.AbandonedToLatencyBudget.Should().BeTrue(); + context.RelevantEntities.Diagnostics!.TimedOut.Should().BeFalse( + "a section that finished in time must not be tarred with the budget failure"); + } + + [Fact] + public async Task WhatArrivedInTimeIsStillReturned() + { + // The point of degrading rather than failing: a partial answer beats no answer, provided the + // caller can tell it is partial. + FactsAreSlow(TimeSpan.FromSeconds(30)); + + var context = await AssembleAsync(TimeSpan.FromMilliseconds(150)); + + context.RelevantEntities.Items.Should().ContainSingle(); + context.RecentMessages.Items.Should().ContainSingle(); + } + + [Fact] + public async Task NoBudgetWaitsForEverything() + { + // The off state, and it is the default. Every measurement in this track was taken without a + // budget, and a slow section must still be waited for rather than quietly dropped. + FactsAreSlow(TimeSpan.FromMilliseconds(200)); + + var context = await AssembleAsync(budget: null); + + context.LatencyBudgetExceeded.Should().BeFalse(); + context.RelevantFacts.Diagnostics!.TimedOut.Should().BeFalse(); + } + + [Fact] + public async Task AGenerousBudgetChangesNothing() + { + // A budget nobody exceeds must leave the context identical to having no budget at all. + FactsAreSlow(TimeSpan.FromMilliseconds(10)); + + var context = await AssembleAsync(TimeSpan.FromSeconds(30)); + + context.LatencyBudgetExceeded.Should().BeFalse(); + context.RelevantEntities.Items.Should().ContainSingle(); + } + + [Fact] + public void TheBudgetIsOffByDefault() => + new RecallOptions().LatencyBudget.Should().BeNull(); + + [Fact] + public void PartialIsNotTheSameFlagAsTruncated() + { + // Truncation is the context budget trimming memories that WERE retrieved. This is a retrieval + // that never came back. A caller told only "truncated" would conclude the memories exist and + // were dropped to fit, when they may never have been looked at. + var context = new MemoryContext + { + SessionId = "s-1", + AssembledAtUtc = DateTimeOffset.UnixEpoch, + LatencyBudgetExceeded = true, + }; + + context.LatencyBudgetExceeded.Should().BeTrue(); + context.Truncated.Should().BeFalse(); + } +} diff --git a/tests/AgentMemory.Tests.Unit/Services/TraceTrustStampingTests.cs b/tests/AgentMemory.Tests.Unit/Services/TraceTrustStampingTests.cs new file mode 100644 index 000000000..8b915738c --- /dev/null +++ b/tests/AgentMemory.Tests.Unit/Services/TraceTrustStampingTests.cs @@ -0,0 +1,127 @@ +using AgentMemory.Abstractions.Domain; +using AgentMemory.Abstractions.Options; +using AgentMemory.Abstractions.Repositories; +using AgentMemory.Abstractions.Services; +using AgentMemory.Core.Services; +using FluentAssertions; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using NSubstitute; +using Xunit; + +namespace AgentMemory.Tests.Unit.Services; + +/// +/// Reasoning traces carry a trust level (PLAN 6.3) — stamped from configuration, never from the caller. +/// +/// +/// +/// Traces were the one recall category with no trust signal at all. StartTraceAsync strips any +/// caller-supplied trust_level — correctly, since a caller must never self-assign a level that +/// bypasses the admission policy — and then nothing stamped one, so every trace read back as +/// . "The agent generated this" and "no signal was recorded" +/// were the same value, which is precisely what the enum exists to prevent. +/// +/// +/// Both halves have to hold at once, and they pull in opposite directions: the caller must not be able +/// to set it, and it must not stay unset. +/// +/// +public sealed class TraceTrustStampingTests +{ + private readonly IReasoningTraceRepository _traceRepo = Substitute.For(); + private readonly IReasoningStepRepository _stepRepo = Substitute.For(); + private readonly IToolCallRepository _toolRepo = Substitute.For(); + private readonly IEmbeddingOrchestrator _embeddings = Substitute.For(); + private readonly IClock _clock = Substitute.For(); + private readonly IIdGenerator _idGen = Substitute.For(); + + private readonly List _written = []; + + public TraceTrustStampingTests() + { + _clock.UtcNow.Returns(DateTimeOffset.UtcNow); + _idGen.GenerateId().Returns(_ => Guid.NewGuid().ToString("N")); + _embeddings.EmbedAsync(Arg.Any(), Arg.Any()).Returns(new float[8]); + _traceRepo.AddAsync(Arg.Any(), Arg.Any()) + .Returns(ci => + { + _written.Add(ci.Arg()); + return Task.FromResult(ci.Arg()); + }); + } + + private ReasoningMemoryService CreateSut(MemoryTrustLevel? level = null) => + new(_traceRepo, _stepRepo, _toolRepo, _embeddings, _clock, _idGen, + Options.Create(level is { } value + ? new ReasoningMemoryOptions { DefaultTraceTrustLevel = value } + : new ReasoningMemoryOptions()), + NullLogger.Instance, + new DefaultMemoryIsolationPolicy( + Options.Create(new MemoryIsolationOptions()), + NullLogger.Instance)); + + [Fact] + public async Task ATraceIsStampedRatherThanLeftUntrusted() + { + await CreateSut().StartTraceAsync("s-1", "deploy the service"); + + _written.Should().ContainSingle() + .Which.Metadata.GetTrustLevel().Should().Be(MemoryTrustLevel.ModelGenerated, + "a trace is the agent's own record of what it did, and Untrusted is indistinguishable " + + "from no signal at all"); + } + + [Fact] + public async Task ACallerStillCannotSetItsOwnTrustLevel() + { + // The security half, and it must survive the stamping. A caller that could self-assign + // ApplicationTrusted would bypass the admission policy's instruction-like-content detection + // for this trace's own Task text on recall. + await CreateSut().StartTraceAsync("s-1", "deploy the service", + metadata: new Dictionary + { + ["trust_level"] = nameof(MemoryTrustLevel.ApplicationTrusted), + }); + + _written.Should().ContainSingle() + .Which.Metadata.GetTrustLevel().Should().Be(MemoryTrustLevel.ModelGenerated, + "the caller's value is stripped before the configured one is stamped"); + } + + [Fact] + public async Task TheLevelIsConfigurable() + { + // The escape hatch for a host that lowered MinimumTrustForAdmissionBypass to ModelGenerated + // or below -- the one configuration where this stamp changes behaviour. + await CreateSut(MemoryTrustLevel.Untrusted).StartTraceAsync("s-1", "deploy the service"); + + _written.Should().ContainSingle() + .Which.Metadata.GetTrustLevel().Should().Be(MemoryTrustLevel.Untrusted); + } + + [Fact] + public async Task OtherCallerMetadataSurvives() + { + // Only the reserved key is removed. Stripping the whole dictionary would silently discard + // application data that has nothing to do with trust. + await CreateSut().StartTraceAsync("s-1", "deploy the service", + metadata: new Dictionary { ["ticket"] = "OPS-42" }); + + var stored = _written.Should().ContainSingle().Subject; + stored.Metadata.Should().ContainKey("ticket"); + stored.Metadata["ticket"].Should().Be("OPS-42"); + } + + [Fact] + public void TheDefaultDoesNotReachTheAdmissionBypassThreshold() + { + // Why stamping by default is safe: the shipped bypass threshold is ApplicationTrusted, which + // ModelGenerated does not reach. If this ever inverts, a trace would start bypassing the + // instruction-detection it is currently subject to. + var shipped = new ReasoningMemoryOptions().DefaultTraceTrustLevel; + var bypass = new AgentMemory.AgentFramework.ContextFormatOptions().MinimumTrustForAdmissionBypass; + + ((int)shipped).Should().BeLessThan((int)bypass); + } +} diff --git a/tools/AgentMemory.Cli/Commands/MemoryCommands.cs b/tools/AgentMemory.Cli/Commands/MemoryCommands.cs index 2308d54a7..b65604104 100644 --- a/tools/AgentMemory.Cli/Commands/MemoryCommands.cs +++ b/tools/AgentMemory.Cli/Commands/MemoryCommands.cs @@ -4,6 +4,7 @@ using AgentMemory.Abstractions.Options; using AgentMemory.Abstractions.Repositories; using AgentMemory.Abstractions.Services; +using AgentMemory.Core.Memory; using AgentMemory.Neo4j.Infrastructure; using AgentMemory.Neo4j.Queries; using AgentMemory.Neo4j.Schema.Parity; @@ -492,3 +493,57 @@ private static string DescribeQuery(MemoryHistoryQuery query) private static string FormatOptional(DateTimeOffset? value) => value is null ? "open" : Format(value.Value); } + +/// +/// Renders a human-readable block of what memory holds for an owner (S4). +/// +/// +/// +/// The gap this closes was described honestly in the analysis as "capable but opaque": memory +/// could be queried but not seen. history answers questions about one memory; +/// this shows the shape of the whole thing at a glance. +/// +/// +/// There is no write counterpart, deliberately. Block-memory designs elsewhere let the agent +/// edit its own block, and at that point the block is the store: every provenance edge, trust level +/// and supersession record in the graph then describes a shadow of what the system actually believes. +/// Each line therefore prints its memory id, so a correction is made against that exact memory +/// through invalidate or supersede — audited, attributable, and reversible. +/// +/// +public sealed class BlockCommand(IMemoryHistoryService service, TextWriter output) +{ + public async Task ExecuteAsync( + string? owner, + string? limitValue, + CancellationToken cancellationToken = default) + { + var limit = 50; + if (!string.IsNullOrWhiteSpace(limitValue) + && (!int.TryParse(limitValue, NumberStyles.Integer, CultureInfo.InvariantCulture, out limit) + || limit <= 0)) + { + output.WriteLine("error: block --limit must be a positive integer."); + return 1; + } + + var records = await service.GetHistoryAsync( + new MemoryHistoryQuery + { + OwnerId = string.IsNullOrWhiteSpace(owner) ? null : owner, + // The block is what memory believes NOW; retracted claims shown beside live ones, + // with no per-item history in view, simply read as true. + IncludeInvalidated = false, + // Over-fetch so the omitted count is real. Asking for exactly `limit` would make a + // full block indistinguishable from an exactly-full one. + Limit = limit + 1, + }, + cancellationToken).ConfigureAwait(false); + + var block = MemoryBlockRenderer.Render( + records, DateTimeOffset.UtcNow, owner, limit); + + output.Write(MemoryBlockRenderer.ToText(block)); + return 0; + } +} diff --git a/tools/AgentMemory.Cli/Program.cs b/tools/AgentMemory.Cli/Program.cs index bff1dca38..d68ac6ee4 100644 --- a/tools/AgentMemory.Cli/Program.cs +++ b/tools/AgentMemory.Cli/Program.cs @@ -21,7 +21,7 @@ return cli.Command is null ? 1 : 0; } -var known = new[] { "migrate", "bootstrap", "consolidate", "decay", "conflicts", "schema-parity", "schema-check", "invalidate", "supersede", "history", "evaluate", "perf" }; +var known = new[] { "migrate", "bootstrap", "consolidate", "decay", "conflicts", "schema-parity", "schema-check", "invalidate", "supersede", "history", "evaluate", "perf", "block" }; if (!known.Contains(cli.Command, StringComparer.OrdinalIgnoreCase)) { Console.Error.WriteLine($"error: unknown command '{cli.Command}'."); @@ -225,6 +225,13 @@ string Resolve(string option, string defaultValue, string cfgKey, string envKey) liveOnly: cli.HasFlag("live-only"), ownOnly: cli.HasFlag("own-only"), limitValue: cli.Get("limit")), + // S4. "ours is capable but opaque" -- memory could be queried but not SEEN. Read-only by + // design: there is deliberately no `block --write`, because a block an agent can hand back + // becomes the store, and the graph's provenance and supersession records then describe a + // shadow of what the system believes. + "block" => await new BlockCommand( + sp.GetRequiredService(), output) + .ExecuteAsync(cli.Get("owner"), cli.Get("limit")), "evaluate" => await new EvaluationCommand( sp.GetRequiredService(), sp.GetRequiredService(), diff --git a/tools/AgentMemory.LongMemEval/AgentMemoryLongMemEvalAdapter.cs b/tools/AgentMemory.LongMemEval/AgentMemoryLongMemEvalAdapter.cs index dbacd8e01..3db9b0a98 100644 --- a/tools/AgentMemory.LongMemEval/AgentMemoryLongMemEvalAdapter.cs +++ b/tools/AgentMemory.LongMemEval/AgentMemoryLongMemEvalAdapter.cs @@ -868,6 +868,7 @@ _chatClient is LongMemEvalChatCallMeter callMeter unionGraphTotal: relationUnionGraphTotal), answerPresence: answerPresence, questionType: evidenceQuestion?.QuestionType, + isAbstention: evidenceQuestion?.IsAbstention ?? false, retrievedGoldCoverage: RetrievedGoldCoverage( recall.Context.RelevantFacts.Items, originsByMessageId @@ -984,6 +985,7 @@ private void RecordTelemetry( LongMemEvalRelationCompleteness? relationCompleteness = null, LongMemEvalAnswerPresenceResult? answerPresence = null, string? questionType = null, + bool isAbstention = false, string? answerPromptText = null) { lock (_stateLock) @@ -1020,6 +1022,7 @@ private void RecordTelemetry( // work can skip and none of it has ever asked. SufficiencySignal = SufficiencySignalOf(context), QuestionType = questionType, + IsAbstention = isAbstention, // Makes "expansion had nothing to expand" visible per question, instead of // requiring the lexicon to be consulted by hand after a run. ResolvedQueryRelations = context?.ResolvedQueryRelations ?? [], @@ -1725,6 +1728,20 @@ public sealed record LongMemEvalQuestionTelemetry( /// public double? SufficiencySignal { get; init; } + /// + /// Whether the dataset declares this question unanswerable (an _abs question). + /// + /// + /// Ground truth, and the reason it is carried separately from + /// . The presence gate asks whether the gold answer's distinctive + /// tokens appear anywhere in memory -- a deliberately cheap floor for spotting extraction + /// failure. Measured on an abstention-enriched sample, it reported 3 of 4 abstention + /// questions as "present": their topic is discussed in the conversation even though the specific + /// fact is not, so the tokens are there. For "was this answerable?" the dataset's own label is + /// correct and the heuristic is not. + /// + public bool IsAbstention { get; init; } + /// /// The benchmark's own question type, carried so the gate can be read per type. /// diff --git a/tools/AgentMemory.LongMemEval/ContextTokenBreakdown.cs b/tools/AgentMemory.LongMemEval/ContextTokenBreakdown.cs new file mode 100644 index 000000000..400ed312b --- /dev/null +++ b/tools/AgentMemory.LongMemEval/ContextTokenBreakdown.cs @@ -0,0 +1,107 @@ +using AgentMemory.Abstractions.Domain; + +namespace AgentMemory.LongMemEval; + +/// +/// What one section of an assembled context costs in tokens (B3). +/// +/// Section name, as it appears on . +/// How many memories the section contributed. +/// Their cost, counted with the real tokenizer. +internal sealed record ContextSectionTokens(string Section, int Items, int Tokens); + +/// +/// The token cost of answering from memory versus from the whole conversation (B3). +/// +/// +/// +/// The most persuasive number in this category is a claim about architecture, not model quality. +/// "1.6k context tokens instead of 115k" cannot be inflated by a better answer model, cannot be +/// tuned by prompt engineering, and does not move when the judge changes its mind. It is either true +/// of the context that was assembled or it is not. +/// +/// +/// Which is exactly why it has to be counted rather than estimated. Until C4 this codebase converted +/// tokens to characters by multiplying by four; publishing a compression ratio derived from that +/// would have been a claim about arithmetic dressed up as a claim about design. +/// +/// +/// The baseline is deliberately the full transcript — what a memoryless agent would have to +/// send to answer the same question. Comparing against a truncated window instead would flatter the +/// result by measuring against a system that has already given up on remembering. +/// +/// +internal sealed record ContextTokenBreakdown +{ + /// Per-section costs, largest first. + internal required IReadOnlyList Sections { get; init; } + + /// Total tokens in the assembled memory context. + internal int ContextTokens => Sections.Sum(s => s.Tokens); + + /// Tokens in the full conversation the memory was built from. + internal required int FullHistoryTokens { get; init; } + + /// How the counting was done, so a reader can tell a real count from a fallback. + /// + /// Recorded rather than assumed. falls back to an estimate + /// when it cannot resolve an encoding for the model, and a compression ratio computed from an + /// estimate must not be readable as a measurement. + /// + internal required string CountMethod { get; init; } + + /// The encoding used, when one was resolved. + internal string? Encoding { get; init; } + + /// + /// How many times smaller the memory context is than the full history. + /// + /// + /// Null when there is no history to compare against — an empty denominator is not a ratio of + /// infinity, it is an absent measurement, and reporting a number there would be the single most + /// flattering way to be wrong. + /// + internal double? CompressionRatio => + FullHistoryTokens == 0 ? null : (double)FullHistoryTokens / Math.Max(1, ContextTokens); + + /// + /// Measures an assembled context against the conversation it was distilled from. + /// + internal static ContextTokenBreakdown Measure( + MemoryContext context, + IReadOnlyList fullHistory, + LongMemEvalTokenCounter counter) + { + ArgumentNullException.ThrowIfNull(context); + ArgumentNullException.ThrowIfNull(fullHistory); + ArgumentNullException.ThrowIfNull(counter); + + var sections = new List + { + Section("RecentMessages", context.RecentMessages.Items, m => m.Content, counter), + Section("RelevantMessages", context.RelevantMessages.Items, m => m.Content, counter), + Section("RelevantEntities", context.RelevantEntities.Items, + e => $"{e.Name} {e.Type} {e.Description}", counter), + Section("RelevantFacts", context.RelevantFacts.Items, + f => $"{f.Subject} {f.Predicate} {f.Object}", counter), + Section("RelevantPreferences", context.RelevantPreferences.Items, + p => $"{p.Category} {p.PreferenceText}", counter), + Section("SimilarTraces", context.SimilarTraces.Items, + t => $"{t.Task} {t.Outcome}", counter), + }; + + return new ContextTokenBreakdown + { + // Largest first: the point of a breakdown is to show where the budget actually goes, and + // a fixed section order buries that behind whichever section happens to be listed first. + Sections = sections.OrderByDescending(s => s.Tokens).ToList(), + FullHistoryTokens = fullHistory.Sum(m => counter.Count(m.Content)), + CountMethod = counter.Method.ToString(), + Encoding = counter.Encoding, + }; + } + + private static ContextSectionTokens Section( + string name, IReadOnlyList items, Func render, LongMemEvalTokenCounter counter) => + new(name, items.Count, items.Sum(item => counter.Count(render(item)))); +} diff --git a/tools/AgentMemory.LongMemEval/LongMemEvalAbstentionAccuracy.cs b/tools/AgentMemory.LongMemEval/LongMemEvalAbstentionAccuracy.cs new file mode 100644 index 000000000..87be58d09 --- /dev/null +++ b/tools/AgentMemory.LongMemEval/LongMemEvalAbstentionAccuracy.cs @@ -0,0 +1,89 @@ +namespace AgentMemory.LongMemEval; + +/// +/// How well the agent says "I don't know" — the only meta-memory score this dataset offers. +/// +/// +/// +/// An abstention question is one the dataset declares unanswerable. Getting it right means declining +/// to answer; getting it wrong means producing an answer anyway, which is hallucination under absence +/// — the failure mode a memory system is least able to detect in itself. +/// +/// +/// Not the same thing as the AUC's "absent" count, though both concern unanswerable questions, +/// and conflating them is easy. The AUC's absent count is an input: the ground-truth label +/// against which retrieval confidence is ordered, and it is identical across arms by construction. +/// This is an outcome: whether the agent behaved correctly on those questions. Reporting only +/// the first invites reading a class balance as a result. +/// +/// +/// Worth reporting on its own line because the project's taxonomy names abstention as the only place +/// LongMemEval scores meta-memory at all — and across 52 recorded runs before typed sampling shipped, +/// not one abstention question had ever been drawn, so this had never been measured. +/// +/// +internal static class LongMemEvalAbstentionAccuracy +{ + /// Correct-answer rates for the abstention and ordinary halves of a run. + /// Abstention questions the judge scored correct. + /// Abstention questions judged at all. + /// Ordinary questions the judge scored correct. + /// Ordinary questions judged at all. + internal readonly record struct Result( + int AbstentionCorrect, int AbstentionTotal, int OrdinaryCorrect, int OrdinaryTotal) + { + /// Abstention accuracy, or null when no abstention question was drawn. + /// + /// Null rather than 0 or 1: a sample containing no abstention question has not measured + /// meta-memory badly, it has not measured it. Before typed sampling shipped that was every run. + /// + internal double? AbstentionAccuracy => + AbstentionTotal == 0 ? null : (double)AbstentionCorrect / AbstentionTotal; + + internal double? OrdinaryAccuracy => + OrdinaryTotal == 0 ? null : (double)OrdinaryCorrect / OrdinaryTotal; + + /// + /// Abstention questions answered when they should have been declined. + /// + /// + /// Named for the behaviour rather than as "incorrect", because this specific wrong answer is + /// the interesting one: the agent asserted something memory did not hold. + /// + internal int AnsweredWhenItShouldHaveAbstained => AbstentionTotal - AbstentionCorrect; + } + + /// + /// Scores the two halves from per-question telemetry and the judge's own verdicts. + /// + /// + /// A question with no judge verdict is excluded from both denominators rather than counted wrong: + /// an unjudged question measured nothing, and folding it in as a failure would understate every + /// rate by however many the judge could not parse. + /// + internal static Result Score( + IReadOnlyList telemetry, + Func verdictFor) + { + ArgumentNullException.ThrowIfNull(telemetry); + ArgumentNullException.ThrowIfNull(verdictFor); + + int abstentionCorrect = 0, abstentionTotal = 0, ordinaryCorrect = 0, ordinaryTotal = 0; + foreach (var item in telemetry) + { + if (verdictFor(item.QuestionId) is not { } correct) continue; + if (item.IsAbstention) + { + abstentionTotal++; + if (correct) abstentionCorrect++; + } + else + { + ordinaryTotal++; + if (correct) ordinaryCorrect++; + } + } + + return new Result(abstentionCorrect, abstentionTotal, ordinaryCorrect, ordinaryTotal); + } +} diff --git a/tools/AgentMemory.LongMemEval/LongMemEvalDatasetLocator.cs b/tools/AgentMemory.LongMemEval/LongMemEvalDatasetLocator.cs new file mode 100644 index 000000000..12fa9a5d7 --- /dev/null +++ b/tools/AgentMemory.LongMemEval/LongMemEvalDatasetLocator.cs @@ -0,0 +1,112 @@ +using System.Security.Cryptography; + +namespace AgentMemory.LongMemEval; + +/// +/// Finds the LongMemEval dataset, and says whether it is the one every recorded result used. +/// +/// +/// +/// The dataset is a 277 MB file that is gitignored in this repository and in AgentEval, and tracked +/// by neither. It exists on exactly one disk, and its path has only ever lived in shell history — +/// which is how a session was lost to hunting for it. Worse, it is a -cleaned variant, so +/// re-downloading the public LongMemEval-S does not necessarily reproduce it. +/// +/// +/// The sha is the part worth keeping in git. Sixty-four characters, versioned forever, and they +/// answer the only question that matters about a recovered or re-downloaded file: is this the dataset +/// every sealed corpus and every recorded number was built from? Without it, a subtly different +/// dataset produces results that look normal and are not comparable to anything — and on a cold build +/// nothing downstream would catch it, because there is no earlier corpus to drift against. +/// +/// +/// A mismatch warns and records rather than failing. Evaluating a deliberately different +/// dataset — the oracle variant, a newer cleaned release — is legitimate work, and the run fingerprint +/// already carries the actual sha, so comparability stays mechanically checkable downstream. What must +/// not happen is that it passes silently. +/// +/// +internal static class LongMemEvalDatasetLocator +{ + /// + /// The sha256 of the dataset behind every sealed corpus and every recorded measurement. + /// + /// + /// Pinned here on 2026-08-12, verified against the datasetSha256 recorded in the prepared + /// corpora from 2026-08-08 through 2026-08-10. + /// + internal const string KnownGoodSha256 = + "d6f21ea9d60a0d56f34a05b609c79c88a451d2ae03597821ea3d5a9678c3a442"; + + /// The environment variable naming the dataset, so a path need not be typed each run. + internal const string PathVariable = "LONGMEMEVAL_DATASET"; + + /// + /// The dataset path: the explicit argument, else , else the known + /// checkout locations. + /// + /// + /// The fallbacks are a convenience for the machine this was developed on, deliberately last and + /// deliberately few: guessing widely would make it possible to run against a file nobody chose. + /// Returns null rather than throwing, so the caller keeps its own "dataset is required" message. + /// + internal static string? Resolve(string? explicitPath, Func environment) + { + ArgumentNullException.ThrowIfNull(environment); + + if (!string.IsNullOrWhiteSpace(explicitPath)) return explicitPath; + if (environment(PathVariable) is { Length: > 0 } fromEnvironment) return fromEnvironment; + + foreach (var candidate in KnownLocations) + if (File.Exists(candidate)) return candidate; + + return null; + } + + private static IEnumerable KnownLocations + { + get + { + // AgentEval vendors it beside its own memory benchmark; that is where it actually lives. + yield return Path.Combine( + "..", "AgentEval", "src", "AgentEval.Memory", "Data", "longmemeval", + "longmemeval_s_cleaned.json"); + yield return Path.Combine("data", "longmemeval_s_cleaned.json"); + } + } + + /// + /// Compares a dataset against the pinned sha and returns the line to print, or null when it matches. + /// + /// + /// Hashing 277 MB costs about a second, which is nothing beside a 7–9 hour cold build and is the + /// only moment the question can still be answered cheaply. + /// + internal static string? DescribeMismatch(string datasetPath, string actualSha256) + { + ArgumentException.ThrowIfNullOrWhiteSpace(datasetPath); + ArgumentException.ThrowIfNullOrWhiteSpace(actualSha256); + + if (string.Equals(actualSha256, KnownGoodSha256, StringComparison.OrdinalIgnoreCase)) + return null; + + return + $"longmemeval: WARNING - '{Path.GetFileName(datasetPath)}' is not the dataset every " + + "recorded result was produced from." + Environment.NewLine + + $" expected sha256 {KnownGoodSha256}" + Environment.NewLine + + $" actual sha256 {actualSha256}" + Environment.NewLine + + " Results from it are internally valid but NOT comparable to any sealed corpus or " + + "published number, and reuse of an existing corpus will be refused on dataset drift. " + + "Continuing, because evaluating a different dataset variant is legitimate work - the " + + "actual sha travels in this run's fingerprint."; + } + + /// The sha256 of a file on disk, lowercase hex. + internal static async Task ComputeSha256Async( + string path, CancellationToken cancellationToken = default) + { + await using var stream = File.OpenRead(path); + var hash = await SHA256.HashDataAsync(stream, cancellationToken).ConfigureAwait(false); + return Convert.ToHexStringLower(hash); + } +} diff --git a/tools/AgentMemory.LongMemEval/LongMemEvalGraphProbe.cs b/tools/AgentMemory.LongMemEval/LongMemEvalGraphProbe.cs index e226115d9..bcd3c09b2 100644 --- a/tools/AgentMemory.LongMemEval/LongMemEvalGraphProbe.cs +++ b/tools/AgentMemory.LongMemEval/LongMemEvalGraphProbe.cs @@ -140,6 +140,11 @@ RETURN count(f) AS facts MATCH (p:Preference {owner_id: $ownerId}) RETURN count(p) AS preferences } + CALL { + MATCH (t:ReasoningTrace {owner_id: $ownerId}) + RETURN count(t) AS reasoningTraces, + count(CASE WHEN t.trace_kind = 'Procedure' THEN 1 END) AS procedures + } CALL { MATCH ()-[r:RELATED_TO]->() WHERE r.owner_id = $ownerId @@ -158,7 +163,7 @@ RETURN count(DISTINCT n) AS learnedItems, } RETURN entities, facts, preferences, relationships, relationshipsWithProvenance, learnedItems, learnedItemsWithProvenance, - provenanceEdges, sourceMessages + provenanceEdges, sourceMessages, reasoningTraces, procedures """; private const string GoldCoverageQuery = @@ -252,7 +257,9 @@ public async Task ReadAsync( record["learnedItems"].As(), record["learnedItemsWithProvenance"].As(), record["provenanceEdges"].As(), - record["sourceMessages"].As()); + record["sourceMessages"].As(), + record["reasoningTraces"].As(), + record["procedures"].As()); }).ConfigureAwait(false); } } @@ -286,10 +293,46 @@ public sealed record LongMemEvalGraphSnapshot( int LearnedItems, int LearnedItemsWithProvenance, int ProvenanceEdges, - int SourceMessages) + int SourceMessages, + int? ReasoningTraces = null, + int? Procedures = null) { + /// + /// Entity + Fact + Preference + relationship count, deliberately excluding traces. + /// + /// + /// Traces are counted separately rather than folded in here. This number appears in every sealed + /// measurement taken before the probe could see :ReasoningTrace at all, and quietly + /// widening its definition would move recorded totals for a reason that has nothing to do with + /// what was extracted — every prior build would appear to have grown. + /// public int TotalLearned => Entities + Facts + Preferences + Relationships; + /// Everything the probe can see, traces included. + public int TotalIncludingTraces => TotalLearned + (ReasoningTraces ?? 0); + + /// + /// when this snapshot actually looked for traces. + /// + /// + /// Nullable rather than defaulted to 0, deliberately. A manifest written before the probe + /// could see :ReasoningTrace has no such field, and a non-nullable int would + /// deserialize it to zero — reproducing, in the recorded data, the exact ambiguity 6.5 exists to + /// remove: "measured, and there are none" would be indistinguishable from "never looked". + /// + public bool TracesMeasured => ReasoningTraces.HasValue; + + /// + /// when the corpus holds no procedural/episodic memory at all. + /// + /// + /// The finding 6.5 exists to make visible. The probe was label-blind to :ReasoningTrace, so + /// "the corpus contains no traces" and "the probe cannot see traces" produced identical output — + /// and Phase 7's procedural work would have been measured against a graph nobody had confirmed + /// contained anything to measure. + /// + public bool HasNoTraces => ReasoningTraces == 0; + public bool CompleteProvenance => LearnedItemsWithProvenance == LearnedItems && RelationshipsWithProvenance == Relationships; diff --git a/tools/AgentMemory.LongMemEval/LongMemEvalPreparationManifest.cs b/tools/AgentMemory.LongMemEval/LongMemEvalPreparationManifest.cs index 001570a34..520d84816 100644 --- a/tools/AgentMemory.LongMemEval/LongMemEvalPreparationManifest.cs +++ b/tools/AgentMemory.LongMemEval/LongMemEvalPreparationManifest.cs @@ -50,6 +50,10 @@ internal sealed record LongMemEvalPreparationManifest( string ExtractionVocabularySha256 = "", string QueryRelationLexiconSha256 = "", string ExtractionProvenance = "Batch", + string AbstentionPolicy = "AsSampled", + // Sessions whose content the provider refused, so their material is absent from this corpus. + // Part of the fingerprint: a corpus with gaps is not the same corpus as one without. + int RefusedSourceSessions = 0, // ── Catalog metadata (schema 6) ────────────────────────────────────── // Not part of the fingerprint: these describe the build for a human, and two corpora that differ // only in their description are the same corpus. @@ -93,6 +97,8 @@ internal static LongMemEvalPreparationManifest Create( string extractionVocabularySha256 = "", string queryRelationLexiconSha256 = "", string extractionProvenance = "Batch", + string abstentionPolicy = "AsSampled", + int refusedSourceSessions = 0, string preparedAtUtc = "", string description = "", IReadOnlyList? memoryTypes = null, @@ -165,6 +171,8 @@ internal static LongMemEvalPreparationManifest Create( ExtractionVocabularySha256: extractionVocabularySha256, QueryRelationLexiconSha256: queryRelationLexiconSha256, ExtractionProvenance: extractionProvenance, + AbstentionPolicy: abstentionPolicy, + RefusedSourceSessions: refusedSourceSessions, PreparedAtUtc: preparedAtUtc, Description: description, MemoryTypes: memoryTypes ?? [], @@ -172,9 +180,32 @@ internal static LongMemEvalPreparationManifest Create( return manifest with { Fingerprint = ComputeFingerprint(manifest) }; } + /// + /// Checks a manifest against itself: that its recorded fingerprint matches its recorded contents. + /// + /// + /// + /// Older schemas are read, not rejected. Requiring an exact schema match orphaned every + /// corpus sealed before the current version -- each one a 7-9 hour build -- which is the precise + /// opposite of what recording ingestion identity is for. A manifest written by an older build is + /// still a valid description of a real corpus; what it lacks is fields, and the drift comparison + /// already treats an unrecorded field as drift rather than as agreement. + /// + /// + /// A newer schema is still refused. It was written by a build that knew things this one + /// does not, so its fingerprint cannot be recomputed here and its extra fields would be silently + /// dropped. + /// + /// internal void VerifyIntegrity() { - if (SchemaVersion != CurrentSchemaVersion) + if (SchemaVersion > CurrentSchemaVersion) + { + throw new InvalidOperationException( + $"LongMemEval preparation manifest schema {SchemaVersion} was written by a newer " + + $"build than this one (schema {CurrentSchemaVersion}); it cannot be verified here."); + } + if (SchemaVersion < 1) { throw new InvalidOperationException( $"Unsupported LongMemEval preparation manifest schema {SchemaVersion}."); @@ -185,9 +216,67 @@ internal void VerifyIntegrity() throw new InvalidOperationException("LongMemEval preparation manifest fingerprint mismatch."); } + /// + /// The fingerprint of a manifest, computed with the field set its own schema wrote. + /// + /// + /// Schema 6 added five ingestion-identity fields to the hash. Hashing a schema-5 manifest with + /// them would never reproduce its recorded fingerprint, so every older corpus would read as + /// corrupt rather than as older -- and a corpus that took nine hours to build would be discarded + /// over a field it was never asked to record. + /// internal static string ComputeFingerprint(LongMemEvalPreparationManifest manifest) { ArgumentNullException.ThrowIfNull(manifest); + return manifest.SchemaVersion >= 6 + ? ComputeCurrentFingerprint(manifest) + : ComputeLegacyFingerprint(manifest); + } + + /// The pre-schema-6 field set, preserved verbatim so older manifests still verify. + private static string ComputeLegacyFingerprint(LongMemEvalPreparationManifest manifest) + { + var canonical = new + { + manifest.SchemaVersion, + manifest.PreparationId, + manifest.DatasetSha256, + manifest.AgentEvalRevision, + manifest.ScopeRunIdSha256, + manifest.AnswerModelId, + manifest.JudgeModelId, + manifest.ExtractionModelId, + manifest.EmbeddingModelId, + manifest.EmbeddingDimensions, + manifest.MaxRelevantMessages, + manifest.ExtractionSourceTime, + manifest.UseJsonResponseFormat, + manifest.ExtractionResponseContract, + manifest.UseUnifiedExtraction, + manifest.UseMultiSessionBatchExtraction, + manifest.PreparationWorkers, + manifest.MaxSessionsPerBatch, + manifest.MaxInputTokens, + manifest.MaxConcurrentBatchesPerExtraction, + manifest.MaxConcurrentExtractionBatches, + Questions = manifest.Questions.Select(question => new + { + question.QuestionNumber, + question.QuestionId, + question.HistorySha256, + question.ScopeSha256, + question.MessagesPrepared, + question.SourceSessions, + question.ExtractionUnitsPrepared, + question.GraphSnapshot + }), + manifest.InitialExtractionCalls + }; + return Hash(JsonSerializer.Serialize(canonical, JsonOptions)); + } + + private static string ComputeCurrentFingerprint(LongMemEvalPreparationManifest manifest) + { var canonical = new { manifest.SchemaVersion, @@ -210,6 +299,8 @@ internal static string ComputeFingerprint(LongMemEvalPreparationManifest manifes manifest.ExtractionVocabularySha256, manifest.QueryRelationLexiconSha256, manifest.ExtractionProvenance, + manifest.AbstentionPolicy, + manifest.RefusedSourceSessions, manifest.QuestionSeed, manifest.UseJsonResponseFormat, manifest.ExtractionResponseContract, @@ -268,27 +359,51 @@ internal sealed record LongMemEvalPreparationExpectation( internal void Validate(LongMemEvalPreparationManifest manifest) { ArgumentNullException.ThrowIfNull(manifest); - if (!string.Equals(manifest.DatasetSha256, DatasetSha256, StringComparison.Ordinal) || - !string.Equals(manifest.AgentEvalRevision, AgentEvalRevision, StringComparison.Ordinal) || - !string.Equals(manifest.AnswerModelId, AnswerModelId, StringComparison.Ordinal) || - !string.Equals(manifest.JudgeModelId, JudgeModelId, StringComparison.Ordinal) || - !string.Equals(manifest.ExtractionModelId, ExtractionModelId, StringComparison.Ordinal) || - !string.Equals(manifest.EmbeddingModelId, EmbeddingModelId, StringComparison.Ordinal) || - manifest.EmbeddingDimensions != EmbeddingDimensions || - manifest.MaxRelevantMessages != MaxRelevantMessages || - manifest.UseJsonResponseFormat != UseJsonResponseFormat || - !string.Equals(manifest.ExtractionResponseContract, ExtractionResponseContract, StringComparison.Ordinal) || - !string.Equals(manifest.ExtractionSourceTime, ExtractionSourceTime, StringComparison.Ordinal) || - manifest.UseUnifiedExtraction != UseUnifiedExtraction || - manifest.UseMultiSessionBatchExtraction != UseMultiSessionBatchExtraction || - manifest.PreparationWorkers != PreparationWorkers || - manifest.MaxSessionsPerBatch != MaxSessionsPerBatch || - manifest.MaxInputTokens != MaxInputTokens || - manifest.MaxConcurrentBatchesPerExtraction != MaxConcurrentBatchesPerExtraction || - manifest.MaxConcurrentExtractionBatches != MaxConcurrentExtractionBatches) + var mismatches = new List(); + void Check(string field, object? expected, object? actual) + { + if (!Equals(expected?.ToString(), actual?.ToString())) + mismatches.Add($"{field}: corpus={actual} run={expected}"); + } + + Check("datasetSha256", DatasetSha256, manifest.DatasetSha256); + Check("answerModelId", AnswerModelId, manifest.AnswerModelId); + Check("judgeModelId", JudgeModelId, manifest.JudgeModelId); + Check("extractionModelId", ExtractionModelId, manifest.ExtractionModelId); + Check("embeddingModelId", EmbeddingModelId, manifest.EmbeddingModelId); + Check("embeddingDimensions", EmbeddingDimensions, manifest.EmbeddingDimensions); + Check("maxRelevantMessages", MaxRelevantMessages, manifest.MaxRelevantMessages); + Check("useJsonResponseFormat", UseJsonResponseFormat, manifest.UseJsonResponseFormat); + Check("extractionResponseContract", ExtractionResponseContract, manifest.ExtractionResponseContract); + Check("extractionSourceTime", ExtractionSourceTime, manifest.ExtractionSourceTime); + Check("useUnifiedExtraction", UseUnifiedExtraction, manifest.UseUnifiedExtraction); + Check("useMultiSessionBatchExtraction", UseMultiSessionBatchExtraction, manifest.UseMultiSessionBatchExtraction); + Check("preparationWorkers", PreparationWorkers, manifest.PreparationWorkers); + Check("maxSessionsPerBatch", MaxSessionsPerBatch, manifest.MaxSessionsPerBatch); + Check("maxInputTokens", MaxInputTokens, manifest.MaxInputTokens); + Check("maxConcurrentBatchesPerExtraction", MaxConcurrentBatchesPerExtraction, manifest.MaxConcurrentBatchesPerExtraction); + Check("maxConcurrentExtractionBatches", MaxConcurrentExtractionBatches, manifest.MaxConcurrentExtractionBatches); + + // agentEvalRevision is DELIBERATELY not compared here, though it is still recorded. + // + // AgentEval's involvement in a corpus is real but bounded: it decides which questions are + // sampled and how each conversation history is rendered to text before AgentMemory extracts + // from it. Everything else it does -- the judge, the answer prompt, the verdict protocol -- + // happens at evaluation time and touches no stored fact. + // + // Both of those bounded effects are already checked EXACTLY, per question, by + // LongMemEvalPreparedState: QuestionId proves the same question was ingested, and + // HistorySha256 proves the same rendered text was ingested. Comparing a version string on top + // of that rejects corpora whose content is provably identical -- a 7-9 hour build discarded + // because a package moved 0.16 to 0.20 while producing byte-identical histories. If a future + // release DOES change the formatter or the sampler, the per-question hashes catch it and name + // the question; a version string only ever said "something, somewhere, might differ". + if (mismatches.Count > 0) { throw new InvalidOperationException( - "Prepared LongMemEval configuration does not match the sealed manifest."); + "Prepared LongMemEval configuration does not match the sealed manifest. " + + $"{mismatches.Count} field(s) differ:" + Environment.NewLine + + string.Join(Environment.NewLine, mismatches.Select(m => $" - {m}"))); } } } diff --git a/tools/AgentMemory.LongMemEval/LongMemEvalPreparedCorpusDrift.cs b/tools/AgentMemory.LongMemEval/LongMemEvalPreparedCorpusDrift.cs index b645476b9..192040fb2 100644 --- a/tools/AgentMemory.LongMemEval/LongMemEvalPreparedCorpusDrift.cs +++ b/tools/AgentMemory.LongMemEval/LongMemEvalPreparedCorpusDrift.cs @@ -57,6 +57,15 @@ internal static IReadOnlyList Compare( var differences = new List(); + // Schema 5 and earlier did not carry the ingestion-identity fields at all, so deserialising + // one fills them from the record's DEFAULTS -- "Ignore", "Batch", false. Those are plausible + // values, and comparing against them would report a corpus built with Utterance as MATCHING a + // run configured for Ignore: unknown silently reading as agreement, which is the one thing + // this check exists to prevent. Older manifests therefore report every such field as + // unrecorded, whatever the default happened to be. + var recordsIngestionIdentity = prepared.SchemaVersion >= 6; + string Recorded(string value) => recordsIngestionIdentity ? value : string.Empty; + void Check(string field, string preparedValue, string currentValue) { if (!string.Equals(preparedValue, currentValue, StringComparison.Ordinal)) @@ -69,22 +78,24 @@ void Check(string field, string preparedValue, string currentValue) Check("embeddingDimensions", prepared.EmbeddingDimensions.ToString(CultureInfo.InvariantCulture), current.EmbeddingDimensions.ToString(CultureInfo.InvariantCulture)); - Check("assistantContent", prepared.AssistantContent, current.AssistantContent); - Check("extractionProvenance", prepared.ExtractionProvenance, current.ExtractionProvenance); + Check("assistantContent", Recorded(prepared.AssistantContent), current.AssistantContent); + Check("extractionProvenance", Recorded(prepared.ExtractionProvenance), current.ExtractionProvenance); Check("usePredicateVocabulary", - prepared.UsePredicateVocabulary.ToString(), current.UsePredicateVocabulary.ToString()); + Recorded(prepared.UsePredicateVocabulary.ToString()), current.UsePredicateVocabulary.ToString()); Check("extractionVocabularySha256", prepared.ExtractionVocabularySha256, current.ExtractionVocabularySha256); Check("queryRelationLexiconSha256", prepared.QueryRelationLexiconSha256, current.QueryRelationLexiconSha256); Check("questionSeed", - prepared.QuestionSeed.ToString(CultureInfo.InvariantCulture), + Recorded(prepared.QuestionSeed.ToString(CultureInfo.InvariantCulture)), current.QuestionSeed.ToString(CultureInfo.InvariantCulture)); Check("questionCount", prepared.Questions.Count.ToString(CultureInfo.InvariantCulture), current.QuestionCount.ToString(CultureInfo.InvariantCulture)); + Check("abstention", + Recorded(prepared.AbstentionPolicy), current.AbstentionPolicy); Check("memoryTypes", - string.Join(",", (prepared.MemoryTypes ?? []).OrderBy(t => t, StringComparer.Ordinal)), + Recorded(string.Join(",", (prepared.MemoryTypes ?? []).OrderBy(t => t, StringComparer.Ordinal))), string.Join(",", current.MemoryTypes.OrderBy(t => t, StringComparer.Ordinal))); return differences; @@ -138,4 +149,14 @@ internal sealed record PreparedCorpusIdentity internal required int QuestionSeed { get; init; } internal required int QuestionCount { get; init; } internal IReadOnlyList MemoryTypes { get; init; } = []; + + /// + /// The abstention sampling policy, which decides whether unanswerable questions are in the sample. + /// + /// + /// Ingestion-affecting, not merely evaluation-affecting: an abstention question still carries a + /// conversation history, so including one changes which histories were ingested. A corpus built + /// without them cannot answer a run that expects them. + /// + internal string AbstentionPolicy { get; init; } = "AsSampled"; } diff --git a/tools/AgentMemory.LongMemEval/LongMemEvalPreparedCorpusRegistry.cs b/tools/AgentMemory.LongMemEval/LongMemEvalPreparedCorpusRegistry.cs index 0352366b8..9d5e1f3e4 100644 --- a/tools/AgentMemory.LongMemEval/LongMemEvalPreparedCorpusRegistry.cs +++ b/tools/AgentMemory.LongMemEval/LongMemEvalPreparedCorpusRegistry.cs @@ -1,4 +1,4 @@ -using System.Globalization; +using System.Globalization; using System.Text.Json; using System.Text.Json.Serialization; @@ -75,6 +75,7 @@ internal static void Record( EmbeddingDimensions = manifest.EmbeddingDimensions, AssistantContent = manifest.AssistantContent, ExtractionProvenance = manifest.ExtractionProvenance, + AbstentionPolicy = manifest.AbstentionPolicy, ExtractionVocabularySha256 = manifest.ExtractionVocabularySha256, AgentEvalRevision = manifest.AgentEvalRevision, }); @@ -135,12 +136,18 @@ internal static string Describe(IReadOnlyList entries, Date ? $"{(now - when).TotalDays:0} day(s) old" : "age unknown"; var types = entry.MemoryTypes is { Count: > 0 } t ? string.Join("+", t) : "all"; + // Shown because it is the difference between a corpus the sufficiency AUC can be computed + // on and one it cannot: without abstention questions the unanswerable class comes only + // from accidental extraction misses, which on a 50-question run meant one observation. + var abstention = entry.AbstentionPolicy is { Length: > 0 } policy && policy != "AsSampled" + ? $" abstention={policy}" + : string.Empty; var schema = entry.ManifestSchemaVersion < LongMemEvalPreparationManifest.CurrentSchemaVersion ? $" [manifest schema {entry.ManifestSchemaVersion} < {LongMemEvalPreparationManifest.CurrentSchemaVersion}" + " - some ingestion settings were not recorded, so reuse will report them as drift]" : string.Empty; return $" {entry.Volume}\n" - + $" {entry.Questions}q seed={entry.Seed} types={types} | {age} ({entry.PreparedAtUtc})\n" + + $" {entry.Questions}q seed={entry.Seed} types={types}{abstention} | {age} ({entry.PreparedAtUtc})\n" + $" extraction={entry.ExtractionModel} embedding={entry.EmbeddingModel}/{entry.EmbeddingDimensions}" + $" assistantContent={entry.AssistantContent} provenance={entry.ExtractionProvenance}\n" + $" {(string.IsNullOrWhiteSpace(entry.Description) ? "(no description)" : entry.Description)}{schema}"; @@ -172,6 +179,9 @@ internal sealed record PreparedCorpusEntry public int EmbeddingDimensions { get; init; } public string AssistantContent { get; init; } = ""; public string ExtractionProvenance { get; init; } = ""; + + /// Whether unanswerable questions were drawn into this corpus, and how. + public string AbstentionPolicy { get; init; } = ""; public string ExtractionVocabularySha256 { get; init; } = ""; public string AgentEvalRevision { get; init; } = ""; } diff --git a/tools/AgentMemory.LongMemEval/LongMemEvalPreparedPairProgram.cs b/tools/AgentMemory.LongMemEval/LongMemEvalPreparedPairProgram.cs index 303b512d4..5189ce99f 100644 --- a/tools/AgentMemory.LongMemEval/LongMemEvalPreparedPairProgram.cs +++ b/tools/AgentMemory.LongMemEval/LongMemEvalPreparedPairProgram.cs @@ -32,6 +32,16 @@ internal static class LongMemEvalPreparedPairProgram private const double ColdBuildSpeedTargetMilliseconds = 900_000d; private const int FixedTenExpectedSourceSessions = 474; + + /// + /// How much of a corpus may be missing to provider content refusals before it stops being usable. + /// + /// + /// A handful of sessions out of thousands is noise; hundreds is a different corpus wearing the + /// same name. 2% is deliberately low — the observed rate on this dataset is a fraction of one + /// percent, so a run approaching this limit has something new happening and should stop. + /// + private const double MaximumRefusedSessionShare = 0.02; internal static async Task RunAsync(string[] args) { try @@ -69,14 +79,29 @@ internal static async Task RunAsync(string[] args) options.JudgeRetryAttempts, options.EvidenceDetail, options.MaxRelevantMessages, + options.JudgeProtocol, // Typed sampling reaches the PREPARATION, not only the evaluation: an episodic-only // sample selects different questions, whose conversation histories have to be ingested // for the corpus to answer them at all. This is why an episodic arm cannot reuse a // stratified corpus and needs its own cold build. - includeQuestionTypes: LongMemEvalMemoryTypeSelection.TaskTypesFor(options.MemoryTypes)); + includeQuestionTypes: LongMemEvalMemoryTypeSelection.TaskTypesFor(options.MemoryTypes), + // The unanswerable class. Without it the sufficiency AUC has nothing to order + // against: a stratified 50 gave 32 present and 1 absent, so the metric rested on one + // observation and the two arms reported 0.969 and 0.094 from the same corpus. + abstentionPolicy: options.AbstentionPolicy, + abstentionTargetProportion: options.AbstentionProportion); var datasetSha256 = Convert.ToHexStringLower( SHA256.HashData( await File.ReadAllBytesAsync(options.DatasetPath).ConfigureAwait(false))); + // Checked here because this is the last cheap moment: a cold build is 7-9 hours, and a + // dataset that is not the one every sealed corpus used produces numbers comparable to + // nothing. Warns rather than fails -- a different variant is legitimate work, and the + // actual sha travels in the fingerprint -- but it must never pass silently. + if (LongMemEvalDatasetLocator.DescribeMismatch(options.DatasetPath, datasetSha256) + is { } datasetWarning) + { + Console.Error.WriteLine(datasetWarning); + } var agentEvalRevision = AgentEvalRevision(); var expectation = LongMemEvalPreparationFingerprint.Expect( datasetSha256, @@ -143,6 +168,7 @@ await LongMemEvalOrphanSweep new ProviderCompatibleExtractionChatClient( azureClient.GetChatClient(extractionDeployment).AsIChatClient())); LongMemEvalPreparationManifest manifest; + IReadOnlyList refusedEvidence = []; IReadOnlyList reuseDrift = []; IReadOnlyList preparationTelemetry; LongMemEvalPreparedBatchExecution? batchExecution = null; @@ -221,6 +247,7 @@ await LongMemEvalOrphanSweep QuestionSeed = options.Seed, QuestionCount = options.Questions, MemoryTypes = options.MemoryTypes, + AbstentionPolicy = options.AbstentionPolicy.ToString(), }); if (reuseDrift.Count > 0 && !options.AllowStalePrepared) @@ -318,14 +345,43 @@ await LongMemEvalOrphanSweep plans.Sum(plan => plan.SourceSessionCount); var plannedInputTokens = plans.Sum(plan => plan.TotalEstimatedInputTokens); - if (options.Questions == DefaultQuestions && - options.Seed == DefaultSeed && - plannedSourceSessions != FixedTenExpectedSourceSessions) + // The canonical guard pins the STRATIFIED ten to a recorded plan, so a change in + // sampling or batching cannot pass unnoticed. A typed sample is a different plan by + // construction -- --memory-types episodic selects different questions, with a + // different number of source sessions -- so the canonical number does not describe + // it. Skipped rather than adjusted, and said out loud: silently dropping a guard that + // exists to catch sampling drift would be worse than not having it. + // Canonical means "the plan every recorded run used": default size and seed, no type + // filter, and abstention left as-sampled. Anything that changes WHICH questions are + // drawn produces a different plan, so the recorded source-session count cannot + // describe it. Enumerated rather than inferred, so adding a third sampling knob later + // fails this comparison loudly instead of silently inheriting the canonical count. + var canonicalSampling = options.MemoryTypes.Count == 0 + && options.AbstentionPolicy == AbstentionSamplingPolicy.AsSampled; + var canonicalPlan = options.Questions == DefaultQuestions + && options.Seed == DefaultSeed + && canonicalSampling; + if (canonicalPlan && plannedSourceSessions != FixedTenExpectedSourceSessions) { throw new InvalidOperationException( $"Canonical fixed-ten preflight produced {plannedSourceSessions} " + $"source sessions; expected exactly {FixedTenExpectedSourceSessions}."); } + if (!canonicalSampling) + { + var how = options.MemoryTypes.Count > 0 + ? $"types={string.Join("+", options.MemoryTypes)}" + : string.Empty; + var abstention = options.AbstentionPolicy == AbstentionSamplingPolicy.AsSampled + ? string.Empty + : $"abstention={options.AbstentionPolicy}"; + Console.WriteLine( + "longmemeval: non-canonical sample (" + + string.Join(", ", new[] { how, abstention }.Where(s => s.Length > 0)) + + $") -- the canonical fixed-ten source-session guard " + + $"({FixedTenExpectedSourceSessions}) does not apply, because this sample " + + "selects different questions by construction."); + } Console.WriteLine( $"longmemeval: frozen preparation preflight {plannedCalls} calls for " + $"{plannedSourceSessions} source sessions and " + @@ -502,8 +558,55 @@ await LongMemEvalOrphanSweep // retries 0. Split sub-calls run under their own Activity, so the activity-based // retry counter cannot see them - a split appears as bare extra calls that neither // counter accounts for. One guard accepted that shape and the other rejected it. - var recordedSplits = baseProfile.Services - .GetRequiredService().Snapshot().Splits; + var batchDiagnostics = baseProfile.Services + .GetRequiredService().Snapshot(); + var recordedSplits = batchDiagnostics.Splits; + + // A corpus with gaps must never look complete. The provider refuses some content + // outright -- two sessions of this public research dataset tripped an Azure policy -- + // and skipping them is far better than losing a 616-call preparation to it. But the + // loss is real: those sessions contributed nothing to the graph, so any question + // depending on them is unanswerable for a reason that has nothing to do with recall. + // + // Tolerated up to a small share and REFUSED beyond it, because the failure changes + // character with scale: a handful of sessions out of 2,418 is noise, and hundreds is a + // different corpus wearing the same name. + if (batchDiagnostics.SessionsRefused > 0) + { + var refusedShare = (double)batchDiagnostics.SessionsRefused / Math.Max(1, plannedSourceSessions); + Console.Error.WriteLine( + $"longmemeval: WARNING - the provider refused {batchDiagnostics.ContentRejections} " + + $"source session(s) on content grounds, {batchDiagnostics.SessionsRefused} of " + + $"{plannedSourceSessions} ({refusedShare:P1}). Their content is absent from this " + + "corpus, and it is recorded in the manifest so a later reuse cannot mistake it " + + "for complete."); + // The ids, so the refusal is investigable. Content filters recur, and "something + // was refused" is not a report anyone can act on or raise with a provider. + // + // And, more important than the count: did it COST anything? A refused session + // holding a question's gold evidence makes that question unanswerable from memory + // for a reason unrelated to retrieval, so scoring it wrong attributes a + // content-policy decision to recall quality. + refusedEvidence = LongMemEvalRefusedEvidence.Analyse( + batchDiagnostics.RefusedSessionIds ?? [], questions); + foreach (var refused in refusedEvidence) + { + var cost = refused.HeldGoldEvidence + ? $"HELD GOLD EVIDENCE for question {refused.QuestionId}" + : refused.QuestionId is null + ? "unresolved question - treat as unknown, not harmless" + : $"context only for question {refused.QuestionId}"; + Console.Error.WriteLine($" - refused session: {refused.SessionId} ({cost})"); + } + if (refusedShare > MaximumRefusedSessionShare) + { + throw new InvalidOperationException( + $"Provider content refusals cost {batchDiagnostics.SessionsRefused} of " + + $"{plannedSourceSessions} source sessions ({refusedShare:P1}), above the " + + $"{MaximumRefusedSessionShare:P0} tolerance. A corpus missing that much is a " + + "different corpus, and measuring against it would attribute the loss to recall."); + } + } var successfulCalls = extractionSnapshot.Calls - extractionSnapshot.Failures; var accountingAcceptable = AgentMemoryLongMemEvalAdapter.IsBatchAccountingAcceptable( @@ -587,6 +690,8 @@ await LongMemEvalOrphanSweep usePredicateVocabulary: options.UsePredicateVocabulary, extractionVocabularySha256: MemoryPredicateSeedVocabulary.Fingerprint, queryRelationLexiconSha256: MemoryRelationSeedTable.Fingerprint, + abstentionPolicy: options.AbstentionPolicy.ToString(), + refusedSourceSessions: checked((int)batchDiagnostics.SessionsRefused), preparedAtUtc: DateTimeOffset.UtcNow.ToString("O"), description: options.Description ?? string.Empty, memoryTypes: options.MemoryTypes, @@ -763,11 +868,26 @@ await LongMemEvalOrphanSweep // --allow-stale-prepared was used, so a result taken over a drifted corpus carries // that fact with it instead of looking like any other run. preparedCorpusDrift = reuseDrift.Select(d => d.ToString()).ToArray(), + // Empty on a clean build. Non-empty means questions in THIS sample lost evidence + // to a content policy, which is the difference between a number that is low and a + // number that is measuring the wrong thing. + refusedSessionsHoldingGoldEvidence = refusedEvidence + .Where(r => r.HeldGoldEvidence) + .Select(r => new { r.SessionId, r.QuestionId }) + .ToArray(), preparedCorpusPreparedAtUtc = reusing ? manifest.PreparedAtUtc : null, preparedCorpusDescription = reusing ? manifest.Description : options.Description, evidenceDetail = options.EvidenceDetail.ToString().ToLowerInvariant(), oracleMode = options.OracleMode.ToString().ToLowerInvariant(), judgeRetryAttempts = options.JudgeRetryAttempts, + // 3.7. A COMPARABILITY BREAK, recorded like one. StructuredJson fixes a + // systematic free-text mis-scoring -- the parser vetoes a leading "yes" when the + // word "no" appears later, so "there is no discrepancy" scores as a failure -- but + // AgentEval's own docs say results under it are not comparable with a free-text + // base, and every sealed base here is free-text. Emitted unconditionally so a + // StructuredJson score can never be read beside a FreeText one without the + // difference being visible in the artifact itself. + judgeVerdictProtocol = options.JudgeProtocol.ToString(), neo4jImage = "neo4j:5.26", agentEval = agentEvalRevision, agentEvalDependency = "source-project:AgentEval.Memory" @@ -1012,6 +1132,30 @@ private static async Task RunArmAsync( LongMemEvalVectorYieldSummary.From(vectorYield.Samples)); } + /// The meta-memory half of an arm's result, or nulls when no abstention question ran. + private static object ProjectAbstentionAccuracy(PreparedArmExecution arm) + { + var verdicts = arm.Result.QuestionResults + .Where(q => q.QuestionId is not null) + .GroupBy(q => q.QuestionId!, StringComparer.Ordinal) + .ToDictionary(g => g.Key, g => (bool?)g.First().Correct, StringComparer.Ordinal); + + var scored = LongMemEvalAbstentionAccuracy.Score( + arm.Telemetry, + questionId => questionId is not null && verdicts.TryGetValue(questionId, out var v) ? v : null); + + return new + { + abstentionCorrect = scored.AbstentionCorrect, + abstentionTotal = scored.AbstentionTotal, + abstentionAccuracy = scored.AbstentionAccuracy, + answeredWhenItShouldHaveAbstained = scored.AnsweredWhenItShouldHaveAbstained, + ordinaryCorrect = scored.OrdinaryCorrect, + ordinaryTotal = scored.OrdinaryTotal, + ordinaryAccuracy = scored.OrdinaryAccuracy, + }; + } + private static object ProjectArm( PreparedArmExecution arm, LongMemEvalEvidenceDetail evidenceDetail) => @@ -1060,6 +1204,13 @@ private static object ProjectArm( // holds the two prices, never their difference). Reported as a group so a derived-answer // type's absences are not read as extraction failures. answerPresenceByType = LongMemEvalAnswerPresence.SummariseByType(arm.Telemetry), + // Meta-memory: how well the agent declines to answer what memory does not hold. + // Reported separately from the AUC's "absent" count, which is a ground-truth INPUT + // identical across arms -- reading a class balance as a result is the easy mistake, + // and only one of these two numbers is an outcome. The dataset's abstention questions + // are the only place it scores meta-memory, and until typed sampling shipped none had + // ever been drawn. + abstentionAccuracy = ProjectAbstentionAccuracy(arm), // PLAN 4.2. Emitted here as well as on the plain verb, because THIS is where the // 50-question runs happen: the plain verb's cold build costs ~684 extraction calls and // 7-9 hours, so every real measurement goes through the prepared pair. An instrument that @@ -1278,7 +1429,8 @@ private static string AgentEvalRevision() "--questions", "--resolve-query-relations", "--retain-prepared-volumes", "--reuse-prepared-volumes", "--seed", "--single-session-unified", "--description", "--memory-types", "--allow-stale-prepared", - "--use-predicate-vocabulary", + "--abstention", "--abstention-proportion", + "--use-predicate-vocabulary", "--judge-protocol", ]; private static PreparedPairOptions Parse(string[] args) @@ -1297,7 +1449,11 @@ bool Has(string name) => Array.IndexOf(args, name) >= 0; return new PreparedPairOptions( - Value("--dataset") ?? string.Empty, + // Explicit --dataset wins; LONGMEMEVAL_DATASET is the standing setting; the known + // checkout locations are the last resort. The path used to live only in shell history, + // which is how it went missing. + LongMemEvalDatasetLocator.Resolve( + Value("--dataset"), Environment.GetEnvironmentVariable) ?? string.Empty, ParsePositive(Value("--questions"), DefaultQuestions, "--questions"), ParsePositive(Value("--seed"), DefaultSeed, "--seed"), ParsePositive(Value("--max-relevant"), DefaultMaxRelevant, "--max-relevant"), @@ -1341,7 +1497,10 @@ bool Has(string name) => ParseNonNegative(Value("--graphrag-items"), 0, "--graphrag-items"), Value("--description"), ParseMemoryTypes(Value("--memory-types")), - Has("--allow-stale-prepared")); + Has("--allow-stale-prepared"), + ParseAbstention(Value("--abstention")), + ParseAbstentionProportion(Value("--abstention-proportion")), + ParseJudgeProtocol(Value("--judge-protocol"))); } /// @@ -1362,6 +1521,77 @@ private static IReadOnlyList ParseMemoryTypes(string? value) return types; } + /// + /// Parses --abstention exclude|as-sampled|only|target. + /// + /// + /// + /// Abstention questions are the only unanswerable questions in this dataset, and they are + /// what the retrieval-sufficiency AUC has to order against. Measured on a stratified 50: 32 of 33 + /// checkable questions had their answer present in memory, so the AUC rested on a single + /// observation and the two arms landed at 0.969 and 0.094 from the same data. Without an absent + /// class the metric is not noisy -- it is undefined. + /// + /// + /// Default as-sampled, which is what every recorded run used: LongMemEval-S contains few + /// _abs questions and stratified sampling almost never draws one, so across 52 recorded runs not + /// one ever ran. That default is preserved exactly rather than improved, because changing what a + /// sample contains changes every number computed from it. + /// + /// + /// + /// Parses --judge-protocol. Defaults to FreeText — the protocol every sealed base + /// here was scored under. + /// + /// + /// Never flipped silently. StructuredJson is the real fix for a systematic free-text + /// mis-scoring, and AgentEval's own documentation says results under it are not comparable with a + /// free-text base. Changing the default would not produce a wrong number; it would produce a + /// better one that silently invalidates every comparison anybody makes against the existing runs. + /// + private static JudgeVerdictProtocol ParseJudgeProtocol(string? value) => value?.ToLowerInvariant() switch + { + null or "" or "free-text" or "freetext" => JudgeVerdictProtocol.FreeText, + "structured-json" or "structuredjson" or "json" => JudgeVerdictProtocol.StructuredJson, + _ => throw new ArgumentException( + "--judge-protocol must be one of: free-text, structured-json."), + }; + + /// Test seam for the parser above; the method itself stays private. + internal static JudgeVerdictProtocol ParseJudgeProtocolForTests(string? value) => + ParseJudgeProtocol(value); + + private static AbstentionSamplingPolicy ParseAbstention(string? value) => + value?.ToLowerInvariant() switch + { + null or "" or "as-sampled" => AbstentionSamplingPolicy.AsSampled, + "exclude" => AbstentionSamplingPolicy.Exclude, + "only" => AbstentionSamplingPolicy.Only, + "target" => AbstentionSamplingPolicy.TargetProportion, + _ => throw new ArgumentException( + "--abstention must be one of: as-sampled, exclude, only, target."), + }; + + /// Parses --abstention-proportion, the share of the sample that must abstain. + /// + /// Only meaningful with --abstention target. Rejected outside (0,1): a proportion of 0 + /// silently means "exclude" and 1 means "only", and expressing either by accident would produce a + /// sample nobody chose. + /// + private static double? ParseAbstentionProportion(string? value) + { + if (string.IsNullOrWhiteSpace(value)) return null; + if (!double.TryParse(value, System.Globalization.NumberStyles.Float, + System.Globalization.CultureInfo.InvariantCulture, out var parsed) + || parsed is <= 0 or >= 1) + { + throw new ArgumentException( + "--abstention-proportion must be strictly between 0 and 1; use --abstention exclude " + + "or --abstention only for the endpoints."); + } + return parsed; + } + private static void Validate(PreparedPairOptions options) { if (!string.IsNullOrWhiteSpace(options.ReusePreparedVolume) && @@ -1381,7 +1611,11 @@ private static void Validate(PreparedPairOptions options) } if (string.IsNullOrWhiteSpace(options.DatasetPath)) - throw new ArgumentException("--dataset is required."); + throw new ArgumentException( + "--dataset is required, or set " + + $"{LongMemEvalDatasetLocator.PathVariable}. The file is gitignored here AND in " + + "AgentEval and is tracked by neither, so it exists on exactly one disk; the sha256 " + + $"of the one every recorded result used is {LongMemEvalDatasetLocator.KnownGoodSha256}."); if (!File.Exists(options.DatasetPath)) throw new FileNotFoundException("LongMemEval dataset not found.", options.DatasetPath); if ((options.DiagnosticQuestionPosition is null) != @@ -1607,7 +1841,10 @@ internal sealed record PreparedPairOptions( // reuse later, when nobody remembers what it was built for. string? Description = null, IReadOnlyList? MemoryTypesRequested = null, - bool AllowStalePrepared = false) + bool AllowStalePrepared = false, + AbstentionSamplingPolicy AbstentionPolicy = AbstentionSamplingPolicy.AsSampled, + double? AbstentionProportion = null, + JudgeVerdictProtocol JudgeProtocol = JudgeVerdictProtocol.FreeText) { /// The memory types this corpus was sampled for; empty means every type. internal IReadOnlyList MemoryTypes => MemoryTypesRequested ?? []; diff --git a/tools/AgentMemory.LongMemEval/LongMemEvalRefusedEvidence.cs b/tools/AgentMemory.LongMemEval/LongMemEvalRefusedEvidence.cs new file mode 100644 index 000000000..ee66234e9 --- /dev/null +++ b/tools/AgentMemory.LongMemEval/LongMemEvalRefusedEvidence.cs @@ -0,0 +1,121 @@ +using System.Globalization; +using System.Text.RegularExpressions; + +namespace AgentMemory.LongMemEval; + +/// +/// Whether a provider's content refusal cost a question its answer, or merely some context. +/// +/// +/// +/// A refused source session is skipped so the build survives, and 3 of 2,386 sessions is well inside +/// tolerance. But "inside tolerance" is a statement about volume, not about consequence: if the +/// refused session is the one holding a question's gold evidence, that question becomes +/// unanswerable from memory and will score as wrong — attributing an Azure content-policy +/// decision to our recall quality. +/// +/// +/// That is the exact failure this track exists to prevent: a number that is internally consistent, +/// reproducible, and measuring something other than what it claims. Recording that sessions +/// were refused is not enough; what matters is whether they mattered. +/// +/// +/// This does not change the score. Excluding a question from accuracy would require reaching into +/// AgentEval's scoring, and silently dropping questions is its own way to make a metric lie. It +/// raises a validation issue instead, so an affected run is flagged rather than quietly accepted, and +/// the reader can decide. +/// +/// +internal static class LongMemEvalRefusedEvidence +{ + // {preparationId}-session-{questionIndex:D4}-source-{sourceSessionOrdinal:D4} + private static readonly Regex SessionIdPattern = new( + @"-session-(?\d{4})-source-(?\d{4})$", + RegexOptions.CultureInvariant | RegexOptions.ExplicitCapture, TimeSpan.FromSeconds(1)); + + /// One refused session, and what it cost. + /// The scoped session id the provider refused. + /// The dataset question whose history it belonged to, when resolvable. + /// + /// Whether that source session contained a message the dataset marks as answer-bearing. True means + /// the question can no longer be answered from memory for a reason unrelated to retrieval. + /// + internal readonly record struct RefusedSession( + string SessionId, string? QuestionId, bool HeldGoldEvidence); + + /// + /// Resolves each refused session against the sampled questions' gold evidence. + /// + /// + /// A session id that does not parse, or names a question outside the sample, yields + /// HeldGoldEvidence = false with a null question — unknown rather than assumed harmless, + /// and visible as such because the id is still reported. + /// + internal static IReadOnlyList Analyse( + IReadOnlyList refusedSessionIds, + IReadOnlyList questions) + { + ArgumentNullException.ThrowIfNull(refusedSessionIds); + ArgumentNullException.ThrowIfNull(questions); + + var analysed = new List(refusedSessionIds.Count); + foreach (var sessionId in refusedSessionIds) + { + var match = SessionIdPattern.Match(sessionId); + if (!match.Success) + { + analysed.Add(new RefusedSession(sessionId, null, false)); + continue; + } + + // The question index is 1-based in the id, matching how preparation numbers them. + var questionIndex = int.Parse( + match.Groups["question"].Value, CultureInfo.InvariantCulture) - 1; + var sourceOrdinal = int.Parse( + match.Groups["source"].Value, CultureInfo.InvariantCulture); + + if (questionIndex < 0 || questionIndex >= questions.Count) + { + analysed.Add(new RefusedSession(sessionId, null, false)); + continue; + } + + var question = questions[questionIndex]; + var heldGold = question.Messages.Any(message => + message.SourceSessionOrdinal == sourceOrdinal + && message.HasAnswer + && !message.IsSyntheticBoundary + && !message.IsSyntheticFormatterPadding); + + analysed.Add(new RefusedSession(sessionId, question.QuestionId, heldGold)); + } + + return analysed; + } + + /// + /// The validation issue for a run whose refusals cost gold evidence, or null when none did. + /// + /// + /// Phrased around the consequence rather than the count, because the count is already reported and + /// the consequence is what changes how the accuracy number should be read. + /// + internal static string? DescribeCompromisedQuestions(IReadOnlyList analysed) + { + ArgumentNullException.ThrowIfNull(analysed); + + var compromised = analysed + .Where(session => session.HeldGoldEvidence && session.QuestionId is not null) + .Select(session => session.QuestionId!) + .Distinct(StringComparer.Ordinal) + .OrderBy(id => id, StringComparer.Ordinal) + .ToArray(); + + return compromised.Length == 0 + ? null + : $"The provider refused source sessions holding the gold evidence for " + + $"{compromised.Length} question(s): {string.Join(", ", compromised)}. Those questions " + + "are unanswerable from this corpus for a reason unrelated to retrieval, so counting " + + "them as wrong attributes a content-policy decision to recall quality."; + } +} diff --git a/tools/AgentMemory.LongMemEval/LongMemEvalSufficiencyReport.cs b/tools/AgentMemory.LongMemEval/LongMemEvalSufficiencyReport.cs index e32891e7c..a5d4b8812 100644 --- a/tools/AgentMemory.LongMemEval/LongMemEvalSufficiencyReport.cs +++ b/tools/AgentMemory.LongMemEval/LongMemEvalSufficiencyReport.cs @@ -23,13 +23,24 @@ internal static object From(IReadOnlyList telemetr { ArgumentNullException.ThrowIfNull(telemetry); + // An abstention question is unanswerable BY DEFINITION -- the dataset says so -- and needs no + // gate to establish it. Everything else is labelled by the presence gate, which requires a + // checkable gold answer. + // + // The order matters and was wrong at first. Deferring to the gate for abstention questions + // labelled 3 of 4 of them "answerable", because their topic IS discussed in the conversation + // even though the specific fact is not, so the gold answer's tokens are present. A heuristic + // floor for extraction failure is not a ground truth for answerability, and using it as one + // put most of the unanswerable class on the wrong side of the comparison. var usable = telemetry - .Where(item => item.SufficiencySignal is not null && item.AnswerPresence is { Checkable: true }) + .Where(item => item.SufficiencySignal is not null + && (item.IsAbstention || item.AnswerPresence is { Checkable: true })) .ToArray(); var result = LongMemEvalSufficiencyAuc.Compute( [.. usable.Select(item => new LongMemEvalSufficiencyAuc.Observation( - item.SufficiencySignal!.Value, item.AnswerPresence!.Present))]); + item.SufficiencySignal!.Value, + AnswerPresent: !item.IsAbstention && item.AnswerPresence!.Present))]); return new { @@ -42,9 +53,14 @@ internal static object From(IReadOnlyList telemetr // Every question that could not contribute, and why. Without these the AUC's denominator // is unauditable -- and it is the denominator, not the number, that decides whether the // result means anything. + // How much of the unanswerable class came from the dataset's own label rather than from + // an extraction failure. When this is 0 the AUC is resting on accidental misses, which is + // how a 50-question run ended up with a single absent observation. + abstentionQuestions = usable.Count(item => item.IsAbstention), excludedNoSignal = telemetry.Count(item => item.SufficiencySignal is null), excludedNotCheckable = telemetry.Count( item => item.SufficiencySignal is not null + && !item.IsAbstention && item.AnswerPresence is null or { Checkable: false }), questionsConsidered = telemetry.Count, }; diff --git a/tools/AgentMemory.LongMemEval/ProceduralBenefitHarness.cs b/tools/AgentMemory.LongMemEval/ProceduralBenefitHarness.cs new file mode 100644 index 000000000..242b15df8 --- /dev/null +++ b/tools/AgentMemory.LongMemEval/ProceduralBenefitHarness.cs @@ -0,0 +1,165 @@ +namespace AgentMemory.LongMemEval; + +/// What one attempt at a task cost, and whether it worked (7.6). +/// Whether the task was actually finished. The gate on every other number. +/// Agent turns taken. +/// Tool invocations made. +public sealed record AgentTaskRun(bool Completed, int Steps, int ToolCalls); + +/// +/// Runs a task once and reports what it cost (7.6). +/// +/// +/// A seam so the harness can be exercised against a scripted agent in a unit test and against a real +/// Agent Framework agent in a measured run. The harness is the part that can be wrong in a way nobody +/// notices; the agent is the part that costs money. +/// +public interface IAgentTaskRunner +{ + /// Attempts the task once, with procedural memory on or off. + Task RunAsync( + string taskId, bool procedureMemoryEnabled, int attempt, CancellationToken cancellationToken = default); +} + +/// One arm of the comparison, aggregated (7.6). +public sealed record ProceduralBenefitArm(bool ProcedureMemoryEnabled, IReadOnlyList Runs) +{ + /// Share of attempts that finished the task. + /// + /// The gate on every efficiency number beside it. An agent that gives up sooner takes fewer + /// steps and makes fewer tool calls, and both of those read as an improvement. + /// + public double CompletionRate => Runs.Count == 0 ? 0d : (double)Runs.Count(r => r.Completed) / Runs.Count; + + /// Mean steps over the runs that completed. + /// + /// Averaged over completed runs only. Including failures mixes "solved it in three steps" + /// with "gave up after three", and the second is not a cheaper success. + /// + public double MeanStepsWhenCompleted => Mean(Runs.Where(r => r.Completed).Select(r => (double)r.Steps)); + + /// Mean tool calls over the runs that completed. + public double MeanToolCallsWhenCompleted => + Mean(Runs.Where(r => r.Completed).Select(r => (double)r.ToolCalls)); + + /// The first attempt, before any procedure could have been learned. + public AgentTaskRun? FirstRun => Runs.Count > 0 ? Runs[0] : null; + + /// The last attempt, by which point a procedure should exist. + public AgentTaskRun? LastRun => Runs.Count > 0 ? Runs[^1] : null; + + private static double Mean(IEnumerable values) + { + var list = values.ToList(); + return list.Count == 0 ? 0d : list.Average(); + } +} + +/// +/// Whether procedural memory actually helps an agent repeat a task (7.6). +/// +/// +/// +/// LongMemEval cannot answer this. It scores answers to questions about a transcript; a +/// promoted procedure changes how an agent works, over repeated attempts at a multi-step task. +/// With no trace nodes and — until 6.5 — a probe that was label-blind to them, a promoted procedure +/// was invisible to every instrument this project had, which is to say the feature could not +/// fail. That is the defect shape this harness exists to prevent, not merely a gap in coverage. +/// +/// +/// Two comparisons, and both are needed. On versus off answers "does the feature help at all". +/// First run versus last answers "does it help because it learned, or because the two arms +/// differ for some other reason" — an arm that is faster from attempt one learned nothing. +/// +/// +/// Every efficiency number is gated on completion rate. An agent that abandons a task sooner takes +/// fewer steps and makes fewer tool calls, and a harness that reported those alone would score giving +/// up as the biggest improvement available. +/// +/// +public sealed record ProceduralBenefitResult +{ + /// The arm with procedural memory enabled. + public required ProceduralBenefitArm WithProcedures { get; init; } + + /// The control arm. + public required ProceduralBenefitArm WithoutProcedures { get; init; } + + /// Reduction in mean steps, as a fraction. Negative means it got worse. + public double StepReduction => Reduction( + WithoutProcedures.MeanStepsWhenCompleted, WithProcedures.MeanStepsWhenCompleted); + + /// Reduction in mean tool calls, as a fraction. + public double ToolCallReduction => Reduction( + WithoutProcedures.MeanToolCallsWhenCompleted, WithProcedures.MeanToolCallsWhenCompleted); + + /// Change in completion rate; negative means procedures cost correctness. + public double CompletionRateDelta => + WithProcedures.CompletionRate - WithoutProcedures.CompletionRate; + + /// + /// Whether the enabled arm got cheaper across attempts rather than starting cheaper. + /// + /// + /// The learning signal. If the first attempt is already as cheap as the last, whatever separates + /// the arms is not a procedure that was learned during the run. + /// + public bool ImprovedWithRepetition => + WithProcedures.FirstRun is { } first && WithProcedures.LastRun is { } last + && last.Completed && first.Completed && last.Steps < first.Steps; + + /// + /// Whether the result may be reported as a benefit at all. + /// + /// + /// Correctness first, and not by a hair. Any drop in completion rate disqualifies the + /// efficiency claim outright: an agent with the wrong procedure executes confidently, and the + /// steps it saves are not a saving if the task is not done. Cheapness is only a benefit on top of + /// finishing at least as often. + /// + public bool ShowsBenefit => + CompletionRateDelta >= 0 && (StepReduction > 0 || ToolCallReduction > 0); + + private static double Reduction(double baseline, double measured) => + baseline <= 0 ? 0d : (baseline - measured) / baseline; + + /// + /// Runs both arms and compares them (7.6). + /// + /// Executes one attempt. + /// The repeated task. + /// How many times to repeat it per arm. + public static async Task MeasureAsync( + IAgentTaskRunner runner, + string taskId, + int attempts, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(runner); + ArgumentException.ThrowIfNullOrWhiteSpace(taskId); + ArgumentOutOfRangeException.ThrowIfLessThan(attempts, 2); + + return new ProceduralBenefitResult + { + WithProcedures = await RunArmAsync(runner, taskId, true, attempts, cancellationToken) + .ConfigureAwait(false), + WithoutProcedures = await RunArmAsync(runner, taskId, false, attempts, cancellationToken) + .ConfigureAwait(false), + }; + } + + private static async Task RunArmAsync( + IAgentTaskRunner runner, string taskId, bool enabled, int attempts, CancellationToken cancellationToken) + { + var runs = new List(attempts); + // Sequential, deliberately. The whole hypothesis is that attempt N benefits from what attempt + // N-1 stored, and running them concurrently would race the write that the next read depends on + // -- measuring a feature that had not happened yet. + for (var attempt = 1; attempt <= attempts; attempt++) + { + runs.Add(await runner.RunAsync(taskId, enabled, attempt, cancellationToken).ConfigureAwait(false)); + } + + return new ProceduralBenefitArm(enabled, runs); + } +} diff --git a/tools/AgentMemory.LongMemEval/ProcedureRetrievalPrecision.cs b/tools/AgentMemory.LongMemEval/ProcedureRetrievalPrecision.cs new file mode 100644 index 000000000..9892f3728 --- /dev/null +++ b/tools/AgentMemory.LongMemEval/ProcedureRetrievalPrecision.cs @@ -0,0 +1,118 @@ +namespace AgentMemory.LongMemEval; + +/// +/// What a single task asked for and what procedural memory returned (PLAN 7.7). +/// +/// The task the agent was given. +/// Procedures returned, best-ranked first. Empty means none. +/// +/// Procedures that genuinely apply to this task. Empty means no stored procedure applies, so +/// any retrieval is wrong however confident it looked. +/// +public sealed record ProcedureRetrievalCase( + string TaskId, + IReadOnlyList RetrievedProcedureIds, + IReadOnlyList CorrectProcedureIds); + +/// +/// How often the retrieved procedure is the right one — not merely how fast it arrived +/// (PLAN 7.7). +/// +/// +/// +/// The two failure modes are not equally bad, and a latency number cannot tell them apart. An +/// agent with no procedural memory investigates: it is slower, and it is safe. An agent with the +/// wrong procedure executes — confidently, on a plan built for a different task. A promotion +/// feature that raises hit-rate while also raising the wrong-procedure rate can look like an +/// improvement in every efficiency measure it has. +/// +/// +/// So this reports three outcomes rather than an accuracy: correct, wrong, and abstained. +/// Abstention is deliberately not counted as a failure — it is the safe outcome, and folding it in +/// with wrong answers would make a cautious retriever look identical to a reckless one. That is the +/// same distinction Phase 5 drew for meta-memory, and it matters more here, because the cost of +/// acting on a wrong procedure is paid in tool calls rather than in a sentence. +/// +/// +/// Deterministic and provider-free: it scores id lists, so it can be asserted in a unit test rather +/// than measured against a model. +/// +/// +public sealed record ProcedureRetrievalPrecision( + int Total, + int CorrectAtOne, + int WrongAtOne, + int Abstained, + double MeanReciprocalRank) +{ + /// Share of tasks whose best-ranked procedure was a correct one. + public double PrecisionAtOne => Total == 0 ? 0d : (double)CorrectAtOne / Total; + + /// + /// Share of tasks where a procedure was returned and it was the wrong one — the safety + /// number. + /// + /// + /// This is the figure a promotion change has to be judged against. Hit-rate and latency both + /// improve when the retriever becomes more willing to answer; only this one gets worse. + /// + public double WrongProcedureRate => Total == 0 ? 0d : (double)WrongAtOne / Total; + + /// Share of tasks where nothing was returned — slower, and safe. + public double AbstentionRate => Total == 0 ? 0d : (double)Abstained / Total; + + /// + /// Share of the tasks it chose to answer that it answered correctly. + /// + /// + /// Separated from because the two move in opposite directions as a + /// retriever gets more cautious, and reporting only one of them would make the trade invisible. + /// + public double PrecisionWhenAnswering => + CorrectAtOne + WrongAtOne == 0 ? 0d : (double)CorrectAtOne / (CorrectAtOne + WrongAtOne); + + /// + /// Scores a set of tasks. A case whose + /// is empty counts any retrieval as wrong: nothing stored applies, so answering is the error. + /// + public static ProcedureRetrievalPrecision Score(IReadOnlyList cases) + { + ArgumentNullException.ThrowIfNull(cases); + + var correct = 0; + var wrong = 0; + var abstained = 0; + var reciprocalRankTotal = 0d; + + foreach (var item in cases) + { + if (item.RetrievedProcedureIds.Count == 0) + { + abstained++; + continue; + } + + var expected = new HashSet(item.CorrectProcedureIds, StringComparer.Ordinal); + + if (expected.Contains(item.RetrievedProcedureIds[0])) correct++; + else wrong++; + + // Rank of the first correct procedure. Reported alongside precision@1 rather than instead + // of it: an agent acts on the top result, so a correct procedure sitting at rank 3 is a + // ranking signal, not a success. + for (var rank = 0; rank < item.RetrievedProcedureIds.Count; rank++) + { + if (!expected.Contains(item.RetrievedProcedureIds[rank])) continue; + reciprocalRankTotal += 1d / (rank + 1); + break; + } + } + + return new ProcedureRetrievalPrecision( + cases.Count, + correct, + wrong, + abstained, + cases.Count == 0 ? 0d : reciprocalRankTotal / cases.Count); + } +} diff --git a/tools/AgentMemory.LongMemEval/Program.cs b/tools/AgentMemory.LongMemEval/Program.cs index a41ba1d8a..a75a0bf61 100644 --- a/tools/AgentMemory.LongMemEval/Program.cs +++ b/tools/AgentMemory.LongMemEval/Program.cs @@ -400,7 +400,11 @@ private static Options Parse(string[] args) } return new Options( - Value("--dataset") ?? string.Empty, + // Explicit --dataset wins; LONGMEMEVAL_DATASET is the standing setting; the known + // checkout locations are the last resort. The path used to live only in shell history, + // which is how it went missing. + LongMemEvalDatasetLocator.Resolve( + Value("--dataset"), Environment.GetEnvironmentVariable) ?? string.Empty, ParsePositive(Value("--questions"), DefaultQuestions, "--questions"), ParsePositive(Value("--seed"), DefaultSeed, "--seed"), ParsePositive(Value("--max-relevant"), DefaultMaxRelevant, "--max-relevant"), @@ -495,7 +499,9 @@ private static int ParseNonNegative(string? value, int defaultValue, string opti private static void ValidateInputs(Options options) { if (string.IsNullOrWhiteSpace(options.DatasetPath)) - throw new ArgumentException("--dataset is required."); + throw new ArgumentException( + "--dataset is required, or set " + + $"{LongMemEvalDatasetLocator.PathVariable}."); if (!File.Exists(options.DatasetPath)) throw new FileNotFoundException("LongMemEval dataset not found.", options.DatasetPath); }