diff --git a/AgentMemory.slnx b/AgentMemory.slnx
index 5142c838..afe8c063 100644
--- a/AgentMemory.slnx
+++ b/AgentMemory.slnx
@@ -32,6 +32,7 @@
+
@@ -39,6 +40,7 @@
+
diff --git a/CHANGELOG.md b/CHANGELOG.md
index ab6303f4..4c4dc6ac 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,6 +6,23 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
+### Changed
+
+- **Facts are now identified by canonical keys.** `Fact` nodes carry `subject_key`, `predicate_key`,
+ `object_key` and `owner_key`, and upserts MERGE on those rather than on the raw triple. This is what
+ makes one relation reachable under all of its stored phrasings.
+
+ **Upgrading an existing database:** call `ISchemaBootstrapper.BootstrapAsync()` before writing, as
+ the getting-started guide and every sample already do. It backfills the keys onto existing facts
+ idempotently. **If you skip it, an upsert of a fact that already exists will not match it and will
+ create a duplicate** — the pre-1.4 rows have no keys to match on. `agentmemory schema-check` now
+ reports this state explicitly so it is visible before it causes damage.
+
+- `INeo4jTransactionRunner` implementations that do not also implement `INeo4jAtomicTransactionRunner`
+ no longer throw at construction. Persistence degrades to pass-through and reports
+ `SupportsAtomicRollback = false` instead of refusing to start.
+
+
## [1.3.0] - 2026-07-19
### Added
diff --git a/Directory.Build.props b/Directory.Build.props
index ec9b28dc..7c5c0d2e 100644
--- a/Directory.Build.props
+++ b/Directory.Build.props
@@ -20,9 +20,9 @@
which builds and runs cleanly under net10.0). net8.0 costs nothing to add and lets consumers on the
still-widely-deployed .NET 8 LTS use the library without adopting a newer runtime; net10.0 keeps
pace with the newest release. Verified with real builds and executed tests on all three TFMs, not
- just compiled. Scoped the same way as the packaging metadata below, plus excluding the three
- non-packable tools/ console apps (Cli, TckBridge, TckBridge.Nams), which stay single-targeted. -->
-
+ just compiled. Scoped the same way as the packaging metadata below, plus excluding the four
+ non-packable tools/ console apps (Cli, LongMemEval, TckBridge, TckBridge.Nams), which stay single-targeted. -->
+
net10.0;net9.0;net8.0
diff --git a/docs/architecture.md b/docs/architecture.md
index fd476f03..180df9fb 100644
--- a/docs/architecture.md
+++ b/docs/architecture.md
@@ -142,7 +142,7 @@ graph TD
| **Purpose** | Domain contracts — all models, interfaces, and configuration types shared across the system |
| **Dependencies** | **Microsoft.Extensions.AI.Abstractions** 10.8.0 (approved, D-AR2-1) — .NET BCL otherwise (multi-targets net8.0/net9.0/net10.0) |
| **MUST NOT reference** | Neo4j.Driver, Microsoft.Agents.*, any GraphRAG SDK, any MCP SDK, any NuGet package **except** Microsoft.Extensions.AI.Abstractions |
-| **Key types** | 49 domain records (Conversation, Message, Entity, Fact, Preference, Relationship, MemoryHistoryQuery, MemoryHistoryRecord, ReasoningTrace, ReasoningStep, ToolCall, ToolCallStats, IngestionItemOutcome, etc.), 39 service interfaces (incl. `IMemoryIsolationPolicy`, #100), 11 repository interfaces, 16 configuration types (incl. `MemoryRankingOptions`, `MemoryIsolationOptions`), 24 enums (incl. `MemoryProfile`, `RankingIntent`, `DuplicateStatus`, `EntityMatchType`, `MemoryNodeKind`, `MemoryOperationAccess`, `MemoryIsolationMode`, `IngestionStatus`, `IngestionStage`, `IngestionItemStatus`, `MemoryItemKind`, `IngestionFailureMode`, `MemoryTrustLevel`) (see the catalogs in `design.md §5/§6` for the authoritative, per-type list) |
+| **Key types** | 51 domain records (Conversation, Message, Entity, Fact, Preference, Relationship, MemoryHistoryQuery, MemoryHistoryRecord, ReasoningTrace, ReasoningStep, ToolCall, ToolCallStats, IngestionItemOutcome, MemoryContextRankedItem, UnifiedExtractionResult, etc.), 41 service interfaces (incl. `IMemoryIsolationPolicy`, `IUnifiedMemoryExtractor`, and `IMultiSessionUnifiedMemoryExtractor`), 11 repository interfaces, 16 configuration types (incl. `MemoryRankingOptions`, `MemoryIsolationOptions`), 24 enums (incl. `MemoryProfile`, `RankingIntent`, `DuplicateStatus`, `EntityMatchType`, `MemoryNodeKind`, `MemoryOperationAccess`, `MemoryIsolationMode`, `IngestionStatus`, `IngestionStage`, `IngestionItemStatus`, `MemoryItemKind`, `IngestionFailureMode`, `MemoryTrustLevel`) |
**Namespace structure:**
```
diff --git a/docs/performance/README.md b/docs/performance/README.md
index fc37fa17..c9002eef 100644
--- a/docs/performance/README.md
+++ b/docs/performance/README.md
@@ -75,8 +75,15 @@ the per-fingerprint totals exactly matched `neo4j.queries`.
### Quality guards — deterministic and enforced
Every performance run also executes 19 judged retrieval cases and 20 judged extraction cases. Retrieval
-is scored with Recall@K, MRR, and forbidden-result checks; extraction is scored with precision and
-recall per memory kind plus false positives on six turns that should teach the system nothing.
+is scored with **deterministic-plumbing Recall@K/MRR** and forbidden-result checks; extraction is
+scored with precision and recall per memory kind plus false positives on six turns that should teach
+the system nothing.
+
+That label is permanent, like `bytes_est`. The fixture uses the deterministic FNV-1a test embedder and
+deliberately disjoint vocabulary, so 1.000 / 1.000 proves that retrieval wiring, ranking, scoping and
+guard enforcement still behave exactly—not that a production embedding model has perfect semantic
+quality. Sampled real-embedding/real-model quality belongs to M-27 (LongMemEval), with its model,
+dataset, seed and retrieval configuration fingerprinted.
Five fresh-container runs produced identical values for every guarded metric, so the committed
tolerance is the observed variance: **zero**. The gate is on by default and returns a non-zero exit when
@@ -109,13 +116,58 @@ connection pool, plus explicit Neo4j query-plan-cache clearing after scenario se
claim to reset the Neo4j page cache or host filesystem cache; fixture setup may touch both. These local
hermetic milliseconds are useful as an in-run ratio, not as deployment latency.
-### Not yet measured
+### Concurrent correctness and local saturation
+
+`perf concurrency` is an opt-in reliability characterization against one fixed, fingerprinted product
+driver pool (16 connections by default). It self-asserts owner-isolated reads, concurrent fact
+dedup-on-create, and non-destructive owner-scoped supersession at 1, 10, and 100 logical sessions.
+
+The first red probe proved the command was capable of finding a real defect: 10 concurrent same-owner
+near-duplicate fact creates left 10 live facts. After serializing that process-local dedup decision and
+scoping exact cosine comparison before ranking, the unchanged test left exactly 1 live fact. Every
+other correctness guard stayed exact:
+
+| Sessions | Errors | Owner leaks / misses | Live near-duplicates | Losers present / closed | Edges / live winners | Cross-owner edges |
+|---:|---:|---:|---:|---:|---:|---:|
+| 1 | 0 | 0 / 0 | 1 | 1 / 1 | 1 / 1 | 0 |
+| 10 | 0 | 0 / 0 | 1 | 10 / 10 | 10 / 10 | 0 |
+| 100 | 0 | 0 / 0 | 1 | 100 / 100 | 100 / 100 | 0 |
+
+The same accepted local run reported request p50/p99 and throughput as follows. These numbers describe
+that one hermetic run only; they are not deployment latency:
+
+| Workload | Sessions | p50 ms | p99 ms | operations/s |
+|---|---:|---:|---:|---:|
+| owner-isolation read | 10 | 14.342 | 14.613 | 662.17 |
+| dedup-on-create race | 10 | 202.582 | 255.409 | 38.92 |
+| owner-scoped supersession | 10 | 22.076 | 22.167 | 442.39 |
+| owner-isolation read | 100 | 1,537.608 | 3,060.237 | 32.65 |
+| dedup-on-create race | 100 | 527.468 | 1,295.569 | 76.23 |
+| owner-scoped supersession | 100 | 1,533.084 | 3,067.458 | 32.59 |
+
+The artifact also reports `transaction_entry_ms_est` percentiles. This is permanently labelled an
+upper-bound estimate: it includes connection acquisition, routing, and transaction begin, not exact
+pool queue time. The correctness claim covers concurrent sessions inside one application process;
+distributed dedup coordination across multiple application instances is not yet measured.
+
+### Fail-fast torn-write rollback
+
+Fail-fast extraction persistence prepares all external embeddings before opening one explicit Neo4j
+transaction. Entity, fact, preference, relationship, provenance, temporal, and supersession repository
+operations then join that transaction. Default best-effort mode retains its independent-write behavior.
+
+A dedicated live-Neo4j integration test kills the Neo4j JVM after the first repository write returns
+inside the transaction, verifies from a fresh driver that the database is unreachable, restarts the
+same container, and compares an isolated-owner graph snapshot with its pre-turn state. The red-first
+run without the atomic boundary left 1 entity and 1 provenance edge. With the boundary enabled, the
+post-failure snapshot was empty; one exact retry produced 2 entities, 1 fact, 1 preference,
+1 relationship, and 4 provenance edges, with no duplicates, invalidation, valid-time closure, or
+supersession artifacts. The test also self-asserts that model/embedding calls finish before the
+transaction opens and that the Neo4j coordinator and repositories share the same runner instance.
-Stated plainly rather than left for you to discover:
+### Not yet measured
- **Managed/hosted deployments** — no Aura or NAMS figures yet.
-- **Concurrency** — single-session only; no saturation or p99-under-load numbers.
-
### Scale-M validation
`--scale M` adds exactly 250,000 foreign-scope distractor memories: 50,000 each of entities, facts,
@@ -125,7 +177,7 @@ counts after restore.
On `PERF-R-04`, Scale S and Scale M performed the same structural work: 43 retrieved items, 25
access-tracked items, 9 queries, 6 read transactions, 1 write transaction, and 43 materialized records.
-Recall@K, MRR, and every extraction-quality score remained 1.000. Estimated payload changed from
+Deterministic-plumbing Recall@K, MRR, and every extraction-quality score remained 1.000. Estimated payload changed from
144,591 to 144,555 bytes (−36; −0.025%) and context length from 3,906 to 3,886 characters because the
approximate vector index selected a different equally relevant near-tied fixture item. A second
independent Scale-M restore reproduced 144,555 bytes and 3,886 characters exactly.
@@ -135,6 +187,114 @@ second Docker volume clone. This is a harness-usability result, **not deployment
---
+## Matched `feat-01` before/after characterization
+
+The exact pre-`feat-01` harness commit (`b1d924e9929b`) and post-`feat-01` commit (`0455c584ce`) were
+rerun back-to-back on the same machine with zero provider latency, 10 measured iterations, and 3
+warm-ups. “Full phase” is the elapsed time for the complete recall or ingestion harness phase.
+
+| Full phase | Before p50 / p95 | After p50 / p95 | Movement | Interpretation |
+|---|---:|---:|---:|---|
+| Recall | **313.03 / 641.45 ms** | **50.59 / 113.11 ms** | **−262.44 ms (−83.8%) p50; −528.34 ms (−82.4%) p95** | Attributable to batching 25 access-tracking write transactions into 1; 43 retrieved and 25 tracked items held |
+| Ingestion | **336.83 / 2,859.29 ms** | **221.92 / 352.90 ms** | −114.91 ms (−34.1%) p50 | Control variance only: `feat-01` did not change ingestion |
+
+These are local hermetic characterization timings, not deployment latency. The portable causal result
+is recall write transactions **25 → 1**, queries **31 → 9**, and total database round trips **31 → 7**,
+with retrieved and access-tracked item guards unchanged.
+
+---
+
+## Measured improvements after the 1.3.0 baseline
+
+| Improvement | Scenario | Portable counter | Before | After | Change |
+|---|---|---|---:|---:|---:|
+| Combined single-message Neo4j persistence | `PERF-W-02` | queries per turn | 43 | **40** | **−3 (−7.0%)** |
+| Combined single-message Neo4j persistence | `PERF-W-03` | queries per turn | 88 | **70** | **−18 (−20.5%)** |
+| Skip redundant provenance re-writes | `PERF-W-02` | write transactions per turn | 18 | **8** | **−10 (−55.6%)** |
+| Skip redundant provenance re-writes | `PERF-W-02` | queries per turn | 40 | **30** | **−10 (−25.0%)** |
+| Skip redundant provenance re-writes | `PERF-W-03` | write transactions per turn | 48 | **13** | **−35 (−72.9%)** |
+| Skip redundant provenance re-writes | `PERF-W-03` | queries per turn | 70 | **35** | **−35 (−50.0%)** |
+| Batch memory upserts | `PERF-W-02` | write transactions per turn | 8 | **6** | **−2 (−25.0%)** |
+| Batch memory upserts | `PERF-W-02` | queries per turn | 30 | **28** | **−2 (−6.7%)** |
+| Batch memory upserts | `PERF-W-03` | write transactions per turn | 13 | **11** | **−2 (−15.4%)** |
+| Batch memory upserts | `PERF-W-03` | queries per turn | 35 | **33** | **−2 (−5.7%)** |
+| Batch memory upserts | `PERF-W-05` | write transactions per extraction | 7 | **5** | **−2 (−28.6%)** |
+| Batch memory upserts | `PERF-W-05` | queries per extraction | 28 | **26** | **−2 (−7.1%)** |
+| Batch entity-resolution snapshots | `PERF-W-12-X01` | entity candidate reads per 40 sessions | 80 | **20** | **−60 (−75.0%)** |
+| Batch entity-resolution snapshots | `PERF-W-12-X01` | total read transactions per 40 sessions | 120 | **60** | **−60 (−50.0%)** |
+| Batch entity-resolution snapshots | `PERF-W-12-X01` | queries per 40 sessions | 930 | **870** | **−60 (−6.5%)** |
+| Batch entity-resolution snapshots | `PERF-W-12-X01` | estimated payload bytes per 40 sessions | 2,583,298 | **2,053,922** | **−529,376 (−20.5%)** |
+| Fused ordered source-session persistence | `PERF-W-12-X01` | queries per 40 sessions | 870 | **230** | **−640 (−73.6%)** |
+| Fused ordered source-session persistence | `PERF-W-12-X01` | read transactions per 40 sessions | 60 | **20** | **−40 (−66.7%)** |
+| Fused ordered source-session persistence | `PERF-W-12-X01` | write transactions per 40 sessions | 250 | **50** | **−200 (−80.0%)** |
+
+Message creation, optional embedding persistence, `HAS_MESSAGE`, `FIRST_MESSAGE`, and `NEXT_MESSAGE`
+maintenance now execute as one parameterized Cypher operation. Write transactions remain 18 / 48,
+message counts remain 1 / 6, and estimated payload remains 102,960 / 108,964 bytes. Deterministic
+retrieval and extraction quality guards remain unchanged at 1.000, with a 0% extraction false-positive
+rate. Local-container milliseconds are intentionally omitted because they are not deployment timings.
+
+Neo4j entity, fact, and preference upserts already create every `EXTRACTED_FROM` edge from the
+memory's source-message IDs. The core persistence stage now recognizes that internal capability and
+does not issue the same `MERGE` again in a separate transaction per memory/message pair. Repositories
+without the capability retain the existing explicit provenance behavior. The 50-message whole-session
+guard still reads back exactly 250 provenance edges (5 learned memories × 50 source messages), while
+payload, records, learned items, and deterministic quality stay unchanged.
+
+The remaining entity and fact writes now use one atomic `UNWIND` upsert per memory kind when the
+repository advertises batch support. The same opt-in capability also covers preferences and graph
+relationships when a turn contains more than one; live Neo4j tests verify their owner, temporal,
+embedding, metadata, and provenance fields. `ExtractionOptions.EnableBatchMemoryUpserts` can disable
+the optimization. Default best-effort mode rolls a failed atomic batch back and replays the existing
+item path so per-item outcomes are preserved; fail-fast mode intentionally keeps item writes inside
+its whole-turn transaction so an error still identifies the exact failing item. Two fresh-container
+runs reproduced every counter above exactly. Records, estimated bytes, learned items, and both
+zero-tolerance quality guards were unchanged.
+
+Multi-session extraction now fetches each owner/type entity candidate set once, prefetches independent
+types concurrently, and updates that request-local snapshot as chronological sessions are resolved.
+`ExtractionOptions.UseBatchEntityResolutionSnapshots` can disable the default-on optimization. A
+remote-latency-shaped, fresh-container control/candidate characterization moved the X01 extraction-wave
+p50 from **47,341.00 to 32,600.96 ms (−31.1%)** and X10 from **7,568.38 to 3,621.08 ms
+(−52.2%)**. Writes remained 250; model calls 10; embedding work 130 requests / 720 items; the learned
+80/40/40/40 entity/fact/preference/relationship graph, provenance, source order, owner isolation, and
+both zero-tolerance quality gates were unchanged. These milliseconds include injected provider delay
+and local Docker orchestration; they are controlled-host causal evidence, not deployment latency.
+The related five-worker scaling gate reached **2.991×** rather than the locked 3.000×, so the broader
+cold-build phase remains fail-closed pending the separate persistence candidate.
+
+That persistence candidate is now accepted. The pipeline keeps independent owners parallel and
+same-owner source sessions chronological, prepares embeddings outside the transaction, defers the
+resolver's duplicate entity writes, and commits each source session atomically. Neo4j entity, fact,
+and preference `UNWIND` queries now include embedding, message provenance, optional point data, and
+dynamic POLE+O labels; relationships were already one bounded query. A transaction-only intermediate
+was rejected because it regressed X05/X10. The accepted fused design reduced the query chain inside
+each commit and moved paired remote-shape p50 from **39,159.14 → 28,813.92 ms at X01 (−26.42%)**,
+**6,774.52 → 5,851.43 ms at X05 (−13.63%)**, and **3,807.44 → 3,788.83 ms at X10
+(−0.49%)**. Candidate X05/X10 scaling reached **4.924× / 7.605×**. Exact model, embedding,
+graph, provenance, ordering, isolation, and both quality guards held. These milliseconds include
+injected provider delay and local Docker; the portable causal result is the exact counter movement
+above.
+
+### Cold structured-memory build laboratory
+
+These opt-in laboratory arms measure preparation-workflow candidates; they are not yet shipped
+AgentMemory defaults and their controlled-host milliseconds are not deployment latency.
+
+| Candidate | Controlled comparison | Before p50 / p95 | After p50 / p95 | Movement | Correctness guards |
+|---|---|---:|---:|---:|---|
+| Batch 50 raw-message embeddings + writes | `PERF-W-06` control/candidate | 167.84 / 323.08 ms | 60.24 / 86.03 ms | **−64.1% / −73.4%** | 50 messages/vectors; requests 50 → 1; queries 102 → 1; quality 1.000 |
+| One typed extraction response | `PERF-W-07` → `PERF-W-09` | 903.66 / 909.96 ms | 908.79 / 916.96 ms | +0.6% / +0.8% wall; calls **4 → 1**; total tokens **979 → 353** | Exact 2/2/1/1 output; zero retries/failures; quality 1.000 |
+| Bounded independent-owner cold build | `PERF-W-10-C01` → `PERF-W-10-C10` | 34,202.82 / 47,516.61 ms | 3,195.68 / 4,732.44 ms | **10.70× / 10.04× faster** | Exact 10 calls, 10 messages, 20/20/10/10 learned graph, 80 embeddings, 40/70/270 reads/writes/queries, provenance/isolation, quality 1.000 |
+| Fused ordered source-session persistence | `PERF-W-12` feature off/on | X01 39,159.14 / 50,946.60 ms | X01 28,813.92 / 32,074.05 ms | **−26.42% / −37.04%**; queries 870 → 230; reads 60 → 20; writes 250 → 50 | Exact 10 calls, 130/720 embeddings, 80/40/40/40 graph, provenance/order/isolation, quality 1.000; X05/X10 scaling 4.924×/7.605× |
+
+The unified response reduces provider capacity and token cost, but not one-unit wall time because the
+four original category calls already overlap. The wall-time lever is bounded concurrency across
+independent owners. The next gate integrates that evidence into the prepared LongMemEval cold-build
+path and must project the fixed ten-question build below 15 minutes before another full build is run.
+
+---
+
## Reproduce it yourself
Requires Docker. The harness provisions its own pinned Neo4j, so it does not touch your database.
@@ -158,6 +318,20 @@ dotnet run --project tools/AgentMemory.Cli -- perf --label graphrag \
dotnet run --project tools/AgentMemory.Cli -- perf --label session-extraction \
--scenarios PERF-W-05 --iterations 3
+
+# Isolates resolution, learned-memory embeddings, persistence, provenance, and owner isolation
+dotnet run --project tools/AgentMemory.Cli -- perf --label frozen-persistence \
+ --scenarios PERF-W-08 --iterations 10
+
+# Compares the shipped four-call extractor with one typed unified extraction call
+dotnet run --project tools/AgentMemory.Cli -- perf --label unified-extraction \
+ --scenarios PERF-W-07,PERF-W-09 --latency remote --iterations 10
+
+# Measures ten complete owner-isolated cold-build units at 1, 5, and 10 workers
+dotnet run --project tools/AgentMemory.Cli -- perf --label cold-build-concurrency \
+ --scenarios PERF-W-10-C01,PERF-W-10-C05,PERF-W-10-C10 \
+ --latency remote --iterations 3
+
# Restores the reusable 250k-node Scale-M dataset, then runs the same guarded scenario
dotnet run --project tools/AgentMemory.Cli -- perf --label scale-m \
--scale M --scenarios PERF-R-04 --iterations 1
@@ -166,6 +340,10 @@ dotnet run --project tools/AgentMemory.Cli -- perf --label scale-m \
dotnet run --project tools/AgentMemory.Cli -- perf cold --label cold-r04 \
--scenarios PERF-R-04 --samples 5 --warmup 3
+# Opt-in concurrent correctness + local saturation (fixed 16-connection product pool)
+dotnet run --project tools/AgentMemory.Cli -- perf concurrency --label concurrency \
+ --levels 1,10,100 --pool-size 16
+
# Compare two in-process recall configurations, with quality in the same report
dotnet run --project tools/AgentMemory.Cli -- perf ab \
--control default \
@@ -222,8 +400,16 @@ the complete memory result. The greeting scenario locks its current default-poli
the per-turn ingestion scenarios verify message persistence and extraction outcomes. Whole-session
extraction additionally requires exactly 50 source messages and reads the graph back after the measured
turn to prove that two entities, two facts, one preference, and 250 provenance relationships were
-actually stored. Fixture setup and graph verification are outside the measured scope. Those failures
-are otherwise silent and would produce a confident, wrong number.
+actually stored. Fixture setup and graph verification are outside the measured scope.
+`PERF-W-08` separately bypasses model extraction for one harness-only marker, then exercises the real
+resolution-to-persistence product path. It requires zero model/storage/recall work inside the measured
+turn, exact 2/2/1/1 learned graph output, all supported source provenance, and zero cross-owner edges.
+Its deterministic embedding request count includes both semantic entity-resolution probes and
+learned-memory embeddings.
+`PERF-W-09` exercises the typed unified extractor directly over the same 2/2/1/1 shape as
+`PERF-W-07`, requires exactly one purpose-attributed model call with zero retries, and rejects any
+storage, resolution, embedding, persistence, or recall work. These self-assertions catch failures
+that would otherwise be silent and produce a confident, wrong number.
### Pull-request regression gate
diff --git a/docs/performance/baseline-1.3.0.md b/docs/performance/baseline-1.3.0.md
index 96bc1c84..4a1ab450 100644
--- a/docs/performance/baseline-1.3.0.md
+++ b/docs/performance/baseline-1.3.0.md
@@ -9,6 +9,8 @@ What one agent turn costs at shipped defaults, measured per phase.
> Read [README.md](README.md) first if you have not. In particular: the counters below are portable and
> reproducible; the timings are proportions from a local container, **not** deployment performance.
+> This file remains the immutable 1.3.0 reference. Measured post-baseline changes are listed in
+> [README.md](README.md#measured-improvements-after-the-130-baseline).
---
@@ -66,7 +68,7 @@ entity recall map projection omitted the stored vectors:
| Complete 43-item recall turn | 144,591 bytes | 113,871 bytes | **−21.2%** |
The 30,720-byte difference is exactly 10 returned entities × 384 vector values × 8 estimated bytes.
-Retrieved items, access tracking, queries, transactions, Recall@K, and MRR were unchanged. The
+Retrieved items, access tracking, queries, transactions, deterministic-plumbing Recall@K, and MRR were unchanged. The
projection was then reverted; it is the future rank-6 optimization, not part of this measurement change.
### Round trips by query
@@ -263,7 +265,7 @@ small-graph 1.3.0 baseline, not a replacement deployment-performance baseline.
| Materialized records | 43 | 43 | 0 |
| Estimated payload bytes | 144,591 | 144,555 | −36 (−0.025%) |
| Context characters | 3,906 | 3,886 | −20 (−0.512%) |
-| Retrieval Recall@K / MRR | 1.000 / 1.000 | 1.000 / 1.000 | 0 / 0 |
+| Deterministic-plumbing Recall@K / MRR | 1.000 / 1.000 | 1.000 / 1.000 | 0 / 0 |
| Extraction quality | 1.000 | 1.000 | 0 |
The small payload/context difference repeated exactly across independent Scale-M restores. Neo4j's
@@ -274,17 +276,23 @@ figures establish that the tier is practical to run; they are not deployment lat
### Quality guard applied beside this cost baseline
-The cost counters are only accepted when deterministic quality remains at this committed baseline:
+The cost counters are only accepted when deterministic regression guards remain at this committed baseline:
| Guard | Baseline |
|---|---:|
-| Retrieval Recall@K / MRR | 1.000 / 1.000 |
+| Deterministic-plumbing Recall@K / MRR | 1.000 / 1.000 |
| Retrieval cases with forbidden results | 0 of 19 |
| Entity precision / recall | 1.000 / 1.000 |
| Fact precision / recall | 1.000 / 1.000 |
| Preference precision / recall | 1.000 / 1.000 |
| Extraction false positives on learn-nothing turns | 0 of 6 (20 total cases) |
+“Deterministic-plumbing” is a permanent scope label, not a footnote. The FNV-1a test embedder and
+deliberately disjoint fixture vocabulary make expected neighbors construction-stable; these scores
+prove retrieval wiring, ranking, scoping and forbidden-result enforcement. They do **not** claim
+perfect semantic quality from a production embedding model. Sampled real-embedding/real-model quality
+belongs to M-27 (LongMemEval).
+
Every value above was identical across five fresh-container runs: maximum observed variance **0.000**.
The derived tolerance is therefore **zero**, recorded in
`eng/perf/baselines/quality.json`. The combined reviewable counter + quality snapshot used by pull
diff --git a/eng/perf/baselines/hermetic-S.json b/eng/perf/baselines/hermetic-S.json
index 837c2b28..3b15f0c7 100644
--- a/eng/perf/baselines/hermetic-S.json
+++ b/eng/perf/baselines/hermetic-S.json
@@ -41,12 +41,12 @@
"items.relevant": 5,
"items.retrieved": 43,
"items.traces": 3,
- "neo4j.bytes_est": 144591,
+ "neo4j.bytes_est": 145625,
"neo4j.queries": 9,
"neo4j.records": 43,
"neo4j.tx.read": 6,
"neo4j.tx.write": 1,
- "recall.chars": 3823
+ "recall.chars": 3828
}
},
"PERF-R-07": {
@@ -68,12 +68,12 @@
"items.relevant": 5,
"items.retrieved": 43,
"items.traces": 3,
- "neo4j.bytes_est": 144591,
+ "neo4j.bytes_est": 145625,
"neo4j.queries": 9,
"neo4j.records": 43,
"neo4j.tx.read": 6,
"neo4j.tx.write": 1,
- "recall.chars": 3823
+ "recall.chars": 3828
}
},
"PERF-R-08": {
@@ -95,29 +95,29 @@
"items.relevant": 5,
"items.retrieved": 43,
"items.traces": 3,
- "neo4j.bytes_est": 144591,
+ "neo4j.bytes_est": 145625,
"neo4j.queries": 9,
"neo4j.records": 43,
"neo4j.tx.read": 6,
"neo4j.tx.write": 1,
- "recall.chars": 3934
+ "recall.chars": 3939
}
},
"PERF-W-02": {
"counters": {
"embed.chars": 201,
"embed.items": 4,
- "embed.requests": 4,
+ "embed.requests": 2,
"extract.candidate_entities": 2,
"extract.source_messages": 2,
"llm.calls": 4,
"llm.tokens_in": 947,
"llm.tokens_out": 668,
- "neo4j.bytes_est": 102960,
- "neo4j.queries": 43,
- "neo4j.records": 32,
- "neo4j.tx.read": 4,
- "neo4j.tx.write": 18,
+ "neo4j.bytes_est": 99132,
+ "neo4j.queries": 8,
+ "neo4j.records": 30,
+ "neo4j.tx.read": 2,
+ "neo4j.tx.write": 2,
"persist.entities": 2,
"persist.facts": 2,
"persist.preferences": 1,
@@ -129,17 +129,17 @@
"counters": {
"embed.chars": 378,
"embed.items": 9,
- "embed.requests": 9,
+ "embed.requests": 7,
"extract.candidate_entities": 2,
"extract.source_messages": 7,
"llm.calls": 4,
"llm.tokens_in": 1170,
"llm.tokens_out": 668,
- "neo4j.bytes_est": 108964,
- "neo4j.queries": 88,
- "neo4j.records": 37,
- "neo4j.tx.read": 4,
- "neo4j.tx.write": 48,
+ "neo4j.bytes_est": 103856,
+ "neo4j.queries": 13,
+ "neo4j.records": 35,
+ "neo4j.tx.read": 2,
+ "neo4j.tx.write": 7,
"persist.entities": 2,
"persist.facts": 2,
"persist.preferences": 1,
@@ -151,22 +151,96 @@
"counters": {
"embed.chars": 159,
"embed.items": 7,
- "embed.requests": 7,
+ "embed.requests": 3,
"extract.candidate_entities": 2,
"extract.source_messages": 50,
"llm.calls": 4,
"llm.tokens_in": 7774,
"llm.tokens_out": 668,
- "neo4j.bytes_est": 43218,
- "neo4j.queries": 278,
- "neo4j.records": 57,
- "neo4j.tx.read": 5,
- "neo4j.tx.write": 257,
+ "neo4j.bytes_est": 53274,
+ "neo4j.queries": 8,
+ "neo4j.records": 55,
+ "neo4j.tx.read": 3,
+ "neo4j.tx.write": 1,
"persist.entities": 2,
"persist.facts": 2,
"persist.preferences": 1,
"persist.relationships": 0
}
+ },
+ "PERF-W-06": {
+ "counters": {
+ "embed.chars": 6250,
+ "embed.items": 50,
+ "embed.requests": 1,
+ "neo4j.bytes_est": 173950,
+ "neo4j.queries": 1,
+ "neo4j.records": 50,
+ "neo4j.tx.write": 1,
+ "store.messages": 50
+ }
+ },
+ "PERF-W-07": {
+ "counters": {
+ "extract.entities": 2,
+ "extract.facts": 2,
+ "extract.input_messages": 1,
+ "extract.preferences": 1,
+ "extract.relationships": 1,
+ "llm.calls": 4,
+ "llm.entity.calls": 1,
+ "llm.entity.retries": 0,
+ "llm.entity.tokens_in": 234,
+ "llm.entity.tokens_out": 35,
+ "llm.fact.calls": 1,
+ "llm.fact.retries": 0,
+ "llm.fact.tokens_in": 198,
+ "llm.fact.tokens_out": 49,
+ "llm.preference.calls": 1,
+ "llm.preference.retries": 0,
+ "llm.preference.tokens_in": 204,
+ "llm.preference.tokens_out": 28,
+ "llm.relationship.calls": 1,
+ "llm.relationship.retries": 0,
+ "llm.relationship.tokens_in": 203,
+ "llm.relationship.tokens_out": 28,
+ "llm.tokens_in": 839,
+ "llm.tokens_out": 140
+ }
+ },
+ "PERF-W-08": {
+ "counters": {
+ "embed.chars": 157,
+ "embed.items": 7,
+ "embed.requests": 3,
+ "extract.candidate_entities": 2,
+ "extract.source_messages": 1,
+ "neo4j.bytes_est": 17520,
+ "neo4j.queries": 8,
+ "neo4j.records": 6,
+ "neo4j.tx.read": 2,
+ "neo4j.tx.write": 1,
+ "persist.entities": 2,
+ "persist.facts": 2,
+ "persist.preferences": 1,
+ "persist.relationships": 1
+ }
+ },
+ "PERF-W-09": {
+ "counters": {
+ "extract.entities": 2,
+ "extract.facts": 2,
+ "extract.input_messages": 1,
+ "extract.preferences": 1,
+ "extract.relationships": 1,
+ "llm.calls": 1,
+ "llm.tokens_in": 188,
+ "llm.tokens_out": 165,
+ "llm.unified.calls": 1,
+ "llm.unified.retries": 0,
+ "llm.unified.tokens_in": 188,
+ "llm.unified.tokens_out": 165
+ }
}
},
"quality": {
@@ -179,6 +253,8 @@
"factRecall": 1,
"preferencePrecision": 1,
"preferenceRecall": 1,
- "extractionFalsePositiveRate": 0
+ "extractionFalsePositiveRate": 0,
+ "retrievalMeasurement": "deterministic-plumbing",
+ "semanticQualityClaim": false
}
}
diff --git a/eng/perf/baselines/quality.json b/eng/perf/baselines/quality.json
index 981477b6..659f3577 100644
--- a/eng/perf/baselines/quality.json
+++ b/eng/perf/baselines/quality.json
@@ -11,6 +11,8 @@
"toleranceDerivation": "Every guarded metric was identical across five fresh containers, so tolerance equals observed variance: zero."
},
"retrieval": {
+ "measurement": "deterministic-plumbing",
+ "semanticQualityClaim": false,
"recallAtK": 1.0,
"mrr": 1.0,
"cases": 19,
diff --git a/samples/AgentMemory.Sample.BlendedAgent/Program.cs b/samples/AgentMemory.Sample.BlendedAgent/Program.cs
index 192f6ed2..28cbd80a 100644
--- a/samples/AgentMemory.Sample.BlendedAgent/Program.cs
+++ b/samples/AgentMemory.Sample.BlendedAgent/Program.cs
@@ -55,19 +55,20 @@
});
// ── 2. Core memory services with GraphRAG enabled ─────────────────────────────
-builder.Services.AddAgentMemoryCore(options =>
+// Pass the options instance, not a configure lambda. MemoryOptions is a record with init-only
+// properties, so a lambda can neither assign them nor keep the result of a `with` expression — the
+// `options = options with { ... }` form this sample used to show rebinds the parameter local and is
+// thrown away on return. It compiled, it ran, and it left GraphRAG switched off.
+builder.Services.AddAgentMemoryCore(new MemoryOptions
{
- options = options with
+ EnableGraphRag = true,
+ Recall = new RecallOptions
{
- EnableGraphRag = true,
- Recall = new RecallOptions
- {
- BlendMode = RetrievalBlendMode.Blended,
- MaxGraphRagItems = 5,
- MaxEntities = 10,
- MaxFacts = 10,
- }
- };
+ BlendMode = RetrievalBlendMode.Blended,
+ MaxGraphRagItems = 5,
+ MaxEntities = 10,
+ MaxFacts = 10,
+ }
});
builder.Services.AddSingleton();
diff --git a/samples/AgentMemory.Sample.BlendedAgent/README.md b/samples/AgentMemory.Sample.BlendedAgent/README.md
index f329b539..878ac5a1 100644
--- a/samples/AgentMemory.Sample.BlendedAgent/README.md
+++ b/samples/AgentMemory.Sample.BlendedAgent/README.md
@@ -134,7 +134,10 @@ The blend mode is configured in `Program.cs` via `RecallOptions.BlendMode`. Swit
services.AddNeo4jAgentMemory(options => { ... });
// 2. Core memory services (short-term, long-term, reasoning, context assembly)
-services.AddAgentMemoryCore(options => { options = options with { EnableGraphRag = true, ... }; });
+// Pass the instance. MemoryOptions has init-only properties, so a configure lambda cannot set
+// them: `options = options with { ... }` compiles, rebinds a local, and is discarded — leaving
+// every default in place, GraphRAG included.
+services.AddAgentMemoryCore(new MemoryOptions { EnableGraphRag = true, ... });
services.AddSingleton();
services.AddSingleton();
services.AddSingleton>>(azureClient.GetEmbeddingClient(embeddingDeployment).AsIEmbeddingGenerator());
diff --git a/src/AgentMemory.Abstractions/Domain/Context/MemoryContext.cs b/src/AgentMemory.Abstractions/Domain/Context/MemoryContext.cs
index 3faf9d0d..d628d151 100644
--- a/src/AgentMemory.Abstractions/Domain/Context/MemoryContext.cs
+++ b/src/AgentMemory.Abstractions/Domain/Context/MemoryContext.cs
@@ -53,6 +53,40 @@ public sealed record MemoryContext
///
public string? GraphRagContext { get; init; }
+
+ ///
+ /// The canonical relations this turn's question resolved to, empty when resolution was off or
+ /// matched nothing.
+ ///
+ ///
+ /// Distinguishes "predicate expansion had nothing to expand" from "expansion ran and did not
+ /// help", which need opposite responses — a missing vocabulary entry versus a retrieval or
+ /// reading problem. The resolution was previously computed inline and discarded, so no report
+ /// could tell the two apart: a question failing because its verb is absent from the table looked
+ /// identical to one failing for any other reason. Verified by hand on the n=50 losses, where
+ /// service/serviced turned out to be absent entirely and has is a
+ /// deliberate query stop form.
+ ///
+ public IReadOnlyList ResolvedQueryRelations { get; init; } = Array.Empty();
+
+ ///
+ /// The GraphRAG passages behind , with their scores and source ids.
+ ///
+ ///
+ /// Every other memory surface contributes typed items that evidence accounting can attribute;
+ /// GraphRAG contributed one opaque string, so it could not be scored, attributed, or shown to
+ /// have helped or harmed - which is the most plausible reason its recall budget was set to zero
+ /// and left there.
+ ///
+ /// The information was never missing. GraphRagContextItem already carried
+ /// SourceNodeIds, Score and Metadata; the assembler joined the text and
+ /// discarded the rest. This retains them. is unchanged, so nothing
+ /// the reader sees moves - the addition is instrumentation, not a retrieval change.
+ ///
+ ///
+ public IReadOnlyList GraphRagItems { get; init; } =
+ Array.Empty();
+
///
/// The blend mode that produced this context. Determines which sources were retrieved
/// (see ) and the order in which memory and GraphRAG-derived
diff --git a/src/AgentMemory.Abstractions/Domain/Context/MemoryContextSection.cs b/src/AgentMemory.Abstractions/Domain/Context/MemoryContextSection.cs
index 90b6191b..7c8007e4 100644
--- a/src/AgentMemory.Abstractions/Domain/Context/MemoryContextSection.cs
+++ b/src/AgentMemory.Abstractions/Domain/Context/MemoryContextSection.cs
@@ -11,6 +11,13 @@ public sealed record MemoryContextSection
///
public IReadOnlyList Items { get; init; } = Array.Empty();
+ ///
+ /// Ranked retrieval diagnostics for . Empty unless the recall explicitly requested
+ /// diagnostics and the selected provider can return scores without issuing another retrieval query.
+ ///
+ public IReadOnlyList RankedItems { get; init; } =
+ Array.Empty();
+
///
/// Section-level metadata (e.g., retrieval method, scores).
///
@@ -22,3 +29,16 @@ public sealed record MemoryContextSection
///
public static MemoryContextSection Empty { get; } = new();
}
+
+///
+/// Identifies an item in a ranked retrieval result without duplicating the item payload.
+///
+/// Stable identifier of the retrieved item.
+/// Provider similarity score used for the retrieval ordering.
+/// One-based rank returned by the provider before context budgeting.
+/// One-based position among items that survived context budgeting.
+public sealed record MemoryContextRankedItem(
+ string ItemId,
+ double Score,
+ int RetrievalRank,
+ int ContextRank);
diff --git a/src/AgentMemory.Abstractions/Domain/Extraction/UnifiedExtractionResult.cs b/src/AgentMemory.Abstractions/Domain/Extraction/UnifiedExtractionResult.cs
new file mode 100644
index 00000000..2b7454fe
--- /dev/null
+++ b/src/AgentMemory.Abstractions/Domain/Extraction/UnifiedExtractionResult.cs
@@ -0,0 +1,14 @@
+namespace AgentMemory.Abstractions.Domain;
+
+/// Typed result of one model call that extracts every supported memory category.
+public sealed record UnifiedExtractionResult
+{
+ /// Extracted entities.
+ public IReadOnlyList Entities { get; init; } = Array.Empty();
+ /// Extracted facts.
+ public IReadOnlyList Facts { get; init; } = Array.Empty();
+ /// Extracted preferences.
+ public IReadOnlyList Preferences { get; init; } = Array.Empty();
+ /// Extracted entity relationships.
+ public IReadOnlyList Relationships { get; init; } = Array.Empty();
+}
diff --git a/src/AgentMemory.Abstractions/Domain/LongTerm/Fact.cs b/src/AgentMemory.Abstractions/Domain/LongTerm/Fact.cs
index 3dc2f7c3..7814e544 100644
--- a/src/AgentMemory.Abstractions/Domain/LongTerm/Fact.cs
+++ b/src/AgentMemory.Abstractions/Domain/LongTerm/Fact.cs
@@ -5,6 +5,19 @@ namespace AgentMemory.Abstractions.Domain;
///
public sealed record Fact
{
+ /// Metadata key recording how a recalled fact entered the context.
+ ///
+ /// Facts reaching the context by canonical-predicate expansion may legitimately carry provenance
+ /// outside the current query's window, because expansion returns a relation across the whole
+ /// owner rather than only what the query itself matched. Consumers that resolve provenance must
+ /// be able to tell that apart from a fact whose source genuinely cannot be resolved, which is
+ /// corruption. Marking the former keeps the latter detectable.
+ ///
+ public const string RetrievalSourceMetadataKey = "agentMemory.retrievalSource";
+
+ /// Value of for predicate-expanded facts.
+ public const string RetrievalSourcePredicateExpansion = "predicate-expansion";
+
///
/// Unique identifier for the fact.
///
diff --git a/src/AgentMemory.Abstractions/Options/ExtractionOptions.cs b/src/AgentMemory.Abstractions/Options/ExtractionOptions.cs
index 286d9c9e..0529cbc3 100644
--- a/src/AgentMemory.Abstractions/Options/ExtractionOptions.cs
+++ b/src/AgentMemory.Abstractions/Options/ExtractionOptions.cs
@@ -29,6 +29,40 @@ public sealed class ExtractionOptions
///
public IngestionFailureMode FailureMode { get; set; } = IngestionFailureMode.BestEffort;
+ ///
+ /// Uses atomic repository batch upserts when the configured repository explicitly advertises
+ /// support. Best-effort mode falls back to the existing item path if a batch fails, preserving
+ /// per-item outcomes. Fail-fast mode retains item upserts inside its outer atomic transaction so
+ /// the exception can still identify the exact failing item. Defaults to .
+ ///
+ public bool EnableBatchMemoryUpserts { get; set; } = true;
+
+ ///
+ /// Generates missing entity, fact, and preference embeddings in one aligned provider batch
+ /// before persistence. Result-count mismatches and unavailable vectors replay safely through
+ /// the existing single-item path. Defaults to ; disable to preserve the
+ /// legacy one-request-per-learned-item behavior.
+ ///
+ public bool UseBatchEmbeddingRequests { get; set; } = true;
+
+ ///
+ /// Reuses owner/type entity-resolution candidates within one multi-session extraction batch.
+ /// Candidate types are prefetched concurrently, while identity decisions remain chronological
+ /// and update the request-local snapshot after each resolution. Defaults to .
+ /// Disable to retain one repository candidate lookup per extracted entity.
+ ///
+ public bool UseBatchEntityResolutionSnapshots { get; set; } = true;
+
+ ///
+ /// Coalesces the successful repository work for one logical best-effort persistence operation
+ /// into one atomic store transaction when the provider can prove rollback. External embedding
+ /// work completes before the transaction opens. If any item fails, the coalesced attempt rolls
+ /// back and replays through the legacy item-isolated path; providers without atomic rollback
+ /// retain that path directly. Defaults to ; disable to preserve one
+ /// transaction per repository operation.
+ ///
+ public bool UseCoalescedPersistenceTransactions { get; set; } = true;
+
///
/// The trust level stamped on every entity/fact/preference persisted, unless a specific
/// ExtractionRequest.TrustLevel overrides it for that call (#92 Phase 3). Defaults to
diff --git a/src/AgentMemory.Abstractions/Options/MemoryDecayOptions.cs b/src/AgentMemory.Abstractions/Options/MemoryDecayOptions.cs
index b1dc7fc8..dd209d09 100644
--- a/src/AgentMemory.Abstractions/Options/MemoryDecayOptions.cs
+++ b/src/AgentMemory.Abstractions/Options/MemoryDecayOptions.cs
@@ -24,10 +24,23 @@ public sealed record MemoryDecayOptions
// MaxMemoriesPerSession property here was read nowhere and could not be coherently enforced, so it was removed.
///
- /// Boost factor applied per access (recall hit) when computing the retention score.
+ /// Boost factor applied to the logarithm of the access count when computing the retention
+ /// score: AccessBoostFactor × ln(1 + accessCount), capped by .
///
+ ///
+ /// The boost was applied linearly and undamped until BUG-R7, which let the access count alone
+ /// decide retention — one recall was enough to hold a memory above
+ /// permanently, however stale. It is now damped, capped, and subject to the same time decay as
+ /// confidence, so frequent access slows forgetting rather than preventing it.
+ ///
public double AccessBoostFactor { get; init; } = 0.2;
+ ///
+ /// Ceiling on the access-boost contribution to the retention score, so a very frequently recalled
+ /// memory cannot outweigh every other signal in the blend.
+ ///
+ public double MaxAccessBoost { get; init; } = 0.5;
+
// NOTE: a documented-but-dead `EnableAutoPrune` option was removed (R6 cleanup). It promised
// "automatically prune during extraction" but was read nowhere — auto-prune-on-extraction would wire
// the decay service into the extraction pipeline, which belongs with the (currently held) decay/forget
diff --git a/src/AgentMemory.Abstractions/Options/RecallOptions.cs b/src/AgentMemory.Abstractions/Options/RecallOptions.cs
index cac82cdd..e1144304 100644
--- a/src/AgentMemory.Abstractions/Options/RecallOptions.cs
+++ b/src/AgentMemory.Abstractions/Options/RecallOptions.cs
@@ -46,6 +46,58 @@ public sealed record RecallOptions
///
public RankingIntent Intent { get; init; } = RankingIntent.Default;
+ ///
+ /// Includes ranked retrieval diagnostics in returned memory-context sections when the selected
+ /// provider supports them. Disabled by default so ordinary recalls retain their current payload
+ /// and allocation profile.
+ ///
+ public bool IncludeDiagnostics { get; init; }
+
/// Default singleton instance.
public static RecallOptions Default { get; } = new();
+
+ ///
+ /// G5 "hard" tier. After the similarity-ranked facts are chosen, also returns every fact sharing
+ /// their canonical predicates, so a relation arrives whole. Default off.
+ ///
+ ///
+ /// Top-K is a relevance cutoff and gives no completeness guarantee, so aggregation questions
+ /// ("how many...", "list all...") cannot be answered from it: missing one of five matching facts
+ /// silently yields four. Enable this when the question is an aggregation; it widens the context,
+ /// so it is not the default.
+ ///
+ public bool ExpandFactsByPredicate { get; init; }
+
+ /// Cap on facts returned by predicate expansion. Unbounded completeness would exhaust the budget.
+ public int MaxExpandedFacts { get; init; } = 100;
+
+ ///
+ /// Restricts recalled reasoning traces by outcome: true successful only, false
+ /// failed only, null (default) no filter.
+ ///
+ ///
+ /// The repository and its Cypher have always supported this, and automatic recall passed a
+ /// hardcoded null, so nothing could ever reach it — a built, plumbed, unreachable option.
+ ///
+ /// It matters because a recalled trace is presented to the reader as precedent with nothing
+ /// marking it as a failure, so imitating reasoning that did not work is worse than recalling
+ /// nothing. Upstream neo4j-labs/agent-memory treats this as correctness rather than tuning
+ /// and defaults its equivalent to successful-only. The default here stays at today's behaviour
+ /// because nothing becomes a default before it is measured, and the trace surface has never been
+ /// measured at all — it has carried a recall budget of zero in every quality run to date.
+ ///
+ ///
+ public bool? SuccessfulTracesOnly { get; init; }
+
+ ///
+ /// Also expand on the relations the query text itself names, not only those the top-K surfaced.
+ ///
+ ///
+ /// Requires . Expansion makes one relation complete, but it can
+ /// only widen predicates similarity already nominated, so a question naming several relations
+ /// ("did I buy, assemble, sell, or fix...") reaches only whichever of them retrieval happened to
+ /// surface. This resolves the question's own verbs instead. Off by default: it widens the context,
+ /// and nothing is a default here until it has been measured.
+ ///
+ public bool ResolveQueryRelations { get; init; }
}
diff --git a/src/AgentMemory.Abstractions/Options/ShortTermMemoryOptions.cs b/src/AgentMemory.Abstractions/Options/ShortTermMemoryOptions.cs
index 6c026af3..61f86ecb 100644
--- a/src/AgentMemory.Abstractions/Options/ShortTermMemoryOptions.cs
+++ b/src/AgentMemory.Abstractions/Options/ShortTermMemoryOptions.cs
@@ -8,6 +8,13 @@ public sealed record ShortTermMemoryOptions
/// Whether to generate embeddings for messages automatically.
public bool GenerateEmbeddings { get; init; } = true;
+ ///
+ /// Whether AddMessagesAsync generates missing message embeddings in one provider batch.
+ /// Enabled by default; disable only for provider compatibility or controlled A/B measurement.
+ /// Positional alignment and already-provided embeddings are preserved in either mode.
+ ///
+ public bool UseBatchEmbeddingRequests { get; init; } = true;
+
/// Default number of recent messages to retrieve.
public int DefaultRecentMessageLimit { get; init; } = 10;
diff --git a/src/AgentMemory.Abstractions/Repositories/IFactRepository.cs b/src/AgentMemory.Abstractions/Repositories/IFactRepository.cs
index 26eeb7b7..d243398b 100644
--- a/src/AgentMemory.Abstractions/Repositories/IFactRepository.cs
+++ b/src/AgentMemory.Abstractions/Repositories/IFactRepository.cs
@@ -119,4 +119,23 @@ public interface IFactRepository
MemoryScope? scope = null,
DateTimeOffset? systemAsOf = null,
CancellationToken cancellationToken = default);
+
+ ///
+ /// Every fact under the given canonical predicates, bounded — a relation retrieved whole.
+ ///
+ ///
+ /// Top-K vector search is a relevance cutoff and gives no completeness guarantee, so it cannot
+ /// answer "how many": miss one of five births and the count is four. This composes with top-K
+ /// rather than replacing it — similarity finds which relation matters, this returns all of it.
+ ///
+ /// Defaults to empty so existing implementations remain source-compatible; a store that cannot
+ /// retrieve by relation simply contributes nothing rather than failing.
+ ///
+ ///
+ Task> SearchByCanonicalPredicatesAsync(
+ IReadOnlyList canonicalPredicates,
+ int limit,
+ MemoryScope scope,
+ CancellationToken cancellationToken = default) =>
+ Task.FromResult>(Array.Empty());
}
diff --git a/src/AgentMemory.Abstractions/Services/ILongTermMemoryService.cs b/src/AgentMemory.Abstractions/Services/ILongTermMemoryService.cs
index 4dc62253..d4a3e073 100644
--- a/src/AgentMemory.Abstractions/Services/ILongTermMemoryService.cs
+++ b/src/AgentMemory.Abstractions/Services/ILongTermMemoryService.cs
@@ -204,4 +204,50 @@ Task> SearchPreferencesAsOfAsync(
/// Supersedes the loser preference with the winner (D7). See .
Task SupersedePreferenceAsync(string loserPreferenceId, string winnerPreferenceId, MemoryScope? scope = null, CancellationToken cancellationToken = default);
+
+ ///
+ /// Fact recall with optional canonical-predicate expansion — a relation returned whole.
+ ///
+ ///
+ /// A default interface method, not extra optional parameters on the method above: adding optional
+ /// parameters to a published interface breaks every implementor. The default ignores expansion,
+ /// so a store that cannot retrieve by relation behaves exactly as before.
+ ///
+ Task> SearchFactsAsync(
+ float[] queryEmbedding,
+ int limit,
+ double minScore,
+ MemoryScope? scope,
+ bool expandByPredicate,
+ int expansionLimit,
+ CancellationToken cancellationToken) =>
+ SearchFactsAsync(queryEmbedding, limit, minScore, scope, cancellationToken);
+
+ ///
+ /// Fact recall with expansion driven by the relations the question itself names.
+ ///
+ ///
+ /// Expansion alone can only widen predicates that similarity already surfaced in the top-K, so a
+ /// question naming several relations reaches only whichever of them retrieval happened to nominate.
+ /// supplies them from the question instead. An empty list
+ /// reproduces the previous overload exactly, which is what keeps this from ever being worse than
+ /// the existing behaviour.
+ ///
+ /// A further default interface method for the same reason as the one above: adding optional
+ /// parameters to a published interface breaks every implementor, and the interface is locked
+ /// under SemVer.
+ ///
+ ///
+ Task> SearchFactsAsync(
+ float[] queryEmbedding,
+ int limit,
+ double minScore,
+ MemoryScope? scope,
+ bool expandByPredicate,
+ int expansionLimit,
+ IReadOnlyList questionRelations,
+ CancellationToken cancellationToken) =>
+ SearchFactsAsync(
+ queryEmbedding, limit, minScore, scope, expandByPredicate, expansionLimit,
+ cancellationToken);
}
diff --git a/src/AgentMemory.Abstractions/Services/IMemoryExtractionPipeline.cs b/src/AgentMemory.Abstractions/Services/IMemoryExtractionPipeline.cs
index c3807dc7..089d587f 100644
--- a/src/AgentMemory.Abstractions/Services/IMemoryExtractionPipeline.cs
+++ b/src/AgentMemory.Abstractions/Services/IMemoryExtractionPipeline.cs
@@ -18,4 +18,40 @@ public interface IMemoryExtractionPipeline
Task ExtractAsync(
ExtractionRequest request,
CancellationToken cancellationToken = default);
+
+ ///
+ /// Extracts several independent source sessions with token-bounded unified model calls, then
+ /// resolves and persists each session chronologically. The returned results follow commit order.
+ /// When no batch extractor is enabled, requests fall back to ordinary one-session extraction.
+ ///
+ async Task> ExtractBatchAsync(
+ IReadOnlyList requests,
+ int maxSessionsPerBatch,
+ int maxInputTokens,
+ CancellationToken cancellationToken = default)
+ {
+ ArgumentNullException.ThrowIfNull(requests);
+ ArgumentOutOfRangeException.ThrowIfNegativeOrZero(maxSessionsPerBatch);
+ ArgumentOutOfRangeException.ThrowIfNegativeOrZero(maxInputTokens);
+
+ var ordered = requests
+ .Select((request, index) => new
+ {
+ Request = request,
+ Index = index,
+ Timestamp = request.Messages
+ .Select(message => message.TimestampUtc)
+ .DefaultIfEmpty(DateTimeOffset.MinValue)
+ .Min(),
+ })
+ .OrderBy(item => item.Timestamp)
+ .ThenBy(item => item.Index)
+ .Select(item => item.Request)
+ .ToArray();
+
+ var results = new List(ordered.Length);
+ foreach (var request in ordered)
+ results.Add(await ExtractAsync(request, cancellationToken).ConfigureAwait(false));
+ return results;
+ }
}
diff --git a/src/AgentMemory.Abstractions/Services/IMultiSessionUnifiedMemoryExtractor.cs b/src/AgentMemory.Abstractions/Services/IMultiSessionUnifiedMemoryExtractor.cs
new file mode 100644
index 00000000..621b9826
--- /dev/null
+++ b/src/AgentMemory.Abstractions/Services/IMultiSessionUnifiedMemoryExtractor.cs
@@ -0,0 +1,56 @@
+using AgentMemory.Abstractions.Domain;
+
+namespace AgentMemory.Abstractions.Services;
+
+/// A deterministic, content-local batch boundary produced before provider work begins.
+public sealed record MultiSessionExtractionBatchPlan(
+ IReadOnlyList SourceSessionIds,
+ int EstimatedInputTokens);
+
+/// The complete provider-call plan for a multi-session extraction workload.
+public sealed record MultiSessionExtractionPlan(
+ IReadOnlyList Batches)
+{
+ /// Number of provider calls before validation retries or recursive splits.
+ public int BatchCount => Batches.Count;
+
+ /// Number of unique source sessions acknowledged by the plan.
+ public int SourceSessionCount => Batches.Sum(batch => batch.SourceSessionIds.Count);
+
+ /// Sum of the conservative per-batch input estimates.
+ public long TotalEstimatedInputTokens => Batches.Sum(batch => (long)batch.EstimatedInputTokens);
+}
+
+///
+/// Optionally extracts typed memory for several source sessions in token-bounded model requests.
+/// Every returned result is keyed to exactly one input session so provenance cannot bleed across
+/// session or owner boundaries.
+///
+public interface IMultiSessionUnifiedMemoryExtractor
+{
+ /// Whether the extractor is explicitly enabled.
+ bool IsEnabled { get; }
+
+ ///
+ /// Produces the exact stable partition that execution will use before any provider call.
+ /// Implementations that cannot expose a deterministic plan may retain the default failure;
+ /// callers that require preflight must fail closed rather than estimate.
+ ///
+ MultiSessionExtractionPlan Plan(
+ IReadOnlyList requests,
+ int maxSessionsPerBatch,
+ int maxInputTokens) =>
+ throw new NotSupportedException(
+ $"{GetType().Name} does not expose a deterministic multi-session extraction plan.");
+
+ ///
+ /// Extracts the supplied requests using contiguous batches no larger than
+ /// or .
+ /// Implementations must return exactly one result for every unique input session.
+ ///
+ Task> ExtractAsync(
+ IReadOnlyList requests,
+ int maxSessionsPerBatch,
+ int maxInputTokens,
+ CancellationToken cancellationToken = default);
+}
diff --git a/src/AgentMemory.Abstractions/Services/IUnifiedMemoryExtractor.cs b/src/AgentMemory.Abstractions/Services/IUnifiedMemoryExtractor.cs
new file mode 100644
index 00000000..fc3420f7
--- /dev/null
+++ b/src/AgentMemory.Abstractions/Services/IUnifiedMemoryExtractor.cs
@@ -0,0 +1,15 @@
+using AgentMemory.Abstractions.Domain;
+
+namespace AgentMemory.Abstractions.Services;
+
+/// Optionally extracts all memory categories in one provider call.
+public interface IUnifiedMemoryExtractor
+{
+ /// Whether this extractor should replace the category-specific fan-out.
+ bool IsEnabled { get; }
+
+ /// Extracts every supported category from the supplied messages.
+ Task ExtractAsync(
+ IReadOnlyList messages,
+ CancellationToken cancellationToken = default);
+}
diff --git a/src/AgentMemory.AgentFramework/Neo4jMemoryContextProvider.cs b/src/AgentMemory.AgentFramework/Neo4jMemoryContextProvider.cs
index bcf81b89..2ea5e4cb 100644
--- a/src/AgentMemory.AgentFramework/Neo4jMemoryContextProvider.cs
+++ b/src/AgentMemory.AgentFramework/Neo4jMemoryContextProvider.cs
@@ -228,6 +228,13 @@ private RecallOptions ResolveEffectiveOptions(AutomaticRecallDecision decision)
MaxEntities = decision.Categories.HasFlag(AutomaticRecallCategories.Entities) ? _recallOptions.MaxEntities : 0,
MaxFacts = decision.Categories.HasFlag(AutomaticRecallCategories.Facts) ? _recallOptions.MaxFacts : 0,
MaxPreferences = decision.Categories.HasFlag(AutomaticRecallCategories.Preferences) ? _recallOptions.MaxPreferences : 0,
+ // J4.1: the aggregation route. Top-K cannot answer "how many", so a routed decision
+ // turns on relation completeness for that turn only - it roughly triples the retrieved
+ // context, which is why it is routed rather than defaulted on.
+ ExpandFactsByPredicate =
+ _recallOptions.ExpandFactsByPredicate || decision.RequiresRelationCompleteness,
+ ResolveQueryRelations =
+ _recallOptions.ResolveQueryRelations || decision.RequiresRelationCompleteness,
MaxTraces = decision.Categories.HasFlag(AutomaticRecallCategories.ReasoningTraces) ? _recallOptions.MaxTraces : 0,
MaxGraphRagItems = decision.Categories.HasFlag(AutomaticRecallCategories.GraphRag) ? _recallOptions.MaxGraphRagItems : 0,
Intent = decision.Intent ?? _recallOptions.Intent
diff --git a/src/AgentMemory.AgentFramework/Recall/AutomaticRecallDecision.cs b/src/AgentMemory.AgentFramework/Recall/AutomaticRecallDecision.cs
index c046aad8..29d3817f 100644
--- a/src/AgentMemory.AgentFramework/Recall/AutomaticRecallDecision.cs
+++ b/src/AgentMemory.AgentFramework/Recall/AutomaticRecallDecision.cs
@@ -25,6 +25,23 @@ public sealed record AutomaticRecallDecision
///
public RankingIntent? Intent { get; init; }
+ ///
+ /// The question needs a relation returned whole, not merely its most similar members.
+ ///
+ ///
+ /// Top-K is a relevance cutoff and gives no completeness guarantee, so an aggregation question is
+ /// unanswerable from it: miss one of five matching facts and the count is four, with nothing
+ /// signalling the loss. Setting this turns on predicate expansion and query-relation resolution
+ /// for the turn.
+ ///
+ /// It is a routed decision rather than a global default because it is expensive — expansion
+ /// roughly triples the retrieved context — and because it only helps the questions that need
+ /// completeness. Measured: 73.3% to 90.0% on the questions it applies to, at a cost every other
+ /// turn would pay for nothing.
+ ///
+ ///
+ public bool RequiresRelationCompleteness { get; init; }
+
///
/// An explicit, complete override of the effective for
/// this turn. When set, and above are ignored -- this
diff --git a/src/AgentMemory.AgentFramework/Recall/HeuristicAutomaticRecallPolicy.cs b/src/AgentMemory.AgentFramework/Recall/HeuristicAutomaticRecallPolicy.cs
index 08de5680..e37397f7 100644
--- a/src/AgentMemory.AgentFramework/Recall/HeuristicAutomaticRecallPolicy.cs
+++ b/src/AgentMemory.AgentFramework/Recall/HeuristicAutomaticRecallPolicy.cs
@@ -48,6 +48,19 @@ public sealed class HeuristicAutomaticRecallPolicy : IAutomaticRecallPolicy
@"\b(debug|troubleshoot|workflow|steps to|how do i|walk me through|implement|error|bug|incident|root cause)\b",
RegexOptions.IgnoreCase | RegexOptions.Compiled);
+ ///
+ /// Questions that need a relation returned whole rather than its top-K most similar members.
+ ///
+ ///
+ /// Both of the failures this track spent the most effort on open with "how many", which is what
+ /// makes a lexical route sufficient here instead of a model call. Compiled and anchored on word
+ /// boundaries; no repeating group, so it cannot backtrack catastrophically the way an earlier
+ /// greeting pattern in this file did.
+ ///
+ private static readonly Regex AggregationOriented = new(
+ @"\b(how many|how much|count|total|list all|all the|every|number of)\b",
+ RegexOptions.IgnoreCase | RegexOptions.CultureInvariant | RegexOptions.Compiled);
+
///
public ValueTask DecideAsync(
AutomaticRecallContext context, CancellationToken cancellationToken = default)
@@ -77,7 +90,10 @@ public ValueTask DecideAsync(
{
ShouldRecall = true,
Categories = categories,
- Intent = intent
+ Intent = intent,
+ // G5's aggregation route. Deterministic and free, so it can run on every turn, and it
+ // catches the shape top-K structurally cannot answer.
+ RequiresRelationCompleteness = AggregationOriented.IsMatch(query)
});
}
diff --git a/src/AgentMemory.Core/AgentMemory.Core.csproj b/src/AgentMemory.Core/AgentMemory.Core.csproj
index d2015d98..0126e1a3 100644
--- a/src/AgentMemory.Core/AgentMemory.Core.csproj
+++ b/src/AgentMemory.Core/AgentMemory.Core.csproj
@@ -1,5 +1,12 @@
+
+
+
+
+
diff --git a/src/AgentMemory.Core/Extraction/ExtractionStage.cs b/src/AgentMemory.Core/Extraction/ExtractionStage.cs
index 8936c229..a2f5fb5a 100644
--- a/src/AgentMemory.Core/Extraction/ExtractionStage.cs
+++ b/src/AgentMemory.Core/Extraction/ExtractionStage.cs
@@ -6,6 +6,7 @@
using AgentMemory.Abstractions.Options;
using AgentMemory.Abstractions.Services;
using AgentMemory.Core.Extraction.MergeStrategies;
+using AgentMemory.Core.Resolution;
using AgentMemory.Core.Validation;
namespace AgentMemory.Core.Extraction;
@@ -21,6 +22,7 @@ internal sealed class ExtractionStage : IExtractionStage
private readonly IReadOnlyList _factExtractors;
private readonly IReadOnlyList _preferenceExtractors;
private readonly IReadOnlyList _relationshipExtractors;
+ private readonly IReadOnlyList _unifiedExtractors;
private readonly IEntityResolver _entityResolver;
private readonly ExtractionOptions _options;
private readonly ILogger _logger;
@@ -30,6 +32,7 @@ public ExtractionStage(
IEnumerable factExtractors,
IEnumerable preferenceExtractors,
IEnumerable relationshipExtractors,
+ IEnumerable unifiedExtractors,
IEntityResolver entityResolver,
IOptions extractionOptions,
ILogger logger)
@@ -38,16 +41,42 @@ public ExtractionStage(
_factExtractors = factExtractors.ToList().AsReadOnly();
_preferenceExtractors = preferenceExtractors.ToList().AsReadOnly();
_relationshipExtractors = relationshipExtractors.ToList().AsReadOnly();
+ _unifiedExtractors = unifiedExtractors.ToList().AsReadOnly();
_entityResolver = entityResolver;
_options = extractionOptions.Value;
_logger = logger;
}
- public async Task ExtractAsync(
+ public IDisposable? BeginResolutionBatch() =>
+ (_entityResolver as IExtractionEntityResolver)?.BeginBatch();
+
+ public void InvalidateResolutionBatch() =>
+ (_entityResolver as IExtractionEntityResolver)?.InvalidateBatch();
+
+ public Task ExtractAsync(
+ IReadOnlyList messages,
+ ExtractionTypes typesToExtract,
+ MemoryScope? scope = null,
+ CancellationToken cancellationToken = default) =>
+ ExtractCoreAsync(messages, typesToExtract, scope, preExtracted: null, cancellationToken);
+
+ public Task ProcessUnifiedAsync(
IReadOnlyList messages,
+ UnifiedExtractionResult extracted,
ExtractionTypes typesToExtract,
MemoryScope? scope = null,
CancellationToken cancellationToken = default)
+ {
+ ArgumentNullException.ThrowIfNull(extracted);
+ return ExtractCoreAsync(messages, typesToExtract, scope, extracted, cancellationToken);
+ }
+
+ private async Task ExtractCoreAsync(
+ IReadOnlyList messages,
+ ExtractionTypes typesToExtract,
+ MemoryScope? scope,
+ UnifiedExtractionResult? preExtracted,
+ CancellationToken cancellationToken)
{
var sourceMessageIds = messages.Select(m => m.MessageId).ToList();
var strategy = _options.MergeStrategy;
@@ -58,30 +87,55 @@ public async Task ExtractAsync(
_entityExtractors.Count, _factExtractors.Count,
_preferenceExtractors.Count, _relationshipExtractors.Count, strategy);
- // 1. Run all enabled extractor types in parallel.
- var entityRun = typesToExtract.HasFlag(ExtractionTypes.Entities)
- ? RunExtractorsAsync(_entityExtractors, e => e.ExtractAsync(messages, cancellationToken),
- strategy, MergeStrategyFactory.CreateEntityStrategy, "entity",
- MemoryItemKind.Entity, MemoryErrorCodes.EntityExtractionFailed, cancellationToken)
- : EmptyRun();
-
- var factRun = typesToExtract.HasFlag(ExtractionTypes.Facts)
- ? RunExtractorsAsync(_factExtractors, f => f.ExtractAsync(messages, cancellationToken),
- strategy, MergeStrategyFactory.CreateFactStrategy, "fact",
- MemoryItemKind.Fact, MemoryErrorCodes.FactExtractionFailed, cancellationToken)
- : EmptyRun();
-
- var prefRun = typesToExtract.HasFlag(ExtractionTypes.Preferences)
- ? RunExtractorsAsync(_preferenceExtractors, p => p.ExtractAsync(messages, cancellationToken),
- strategy, MergeStrategyFactory.CreatePreferenceStrategy, "preference",
- MemoryItemKind.Preference, MemoryErrorCodes.PreferenceExtractionFailed, cancellationToken)
- : EmptyRun();
-
- var relRun = typesToExtract.HasFlag(ExtractionTypes.Relationships)
- ? RunExtractorsAsync(_relationshipExtractors, r => r.ExtractAsync(messages, cancellationToken),
- strategy, MergeStrategyFactory.CreateRelationshipStrategy, "relationship",
- MemoryItemKind.Relationship, MemoryErrorCodes.RelationshipExtractionFailed, cancellationToken)
- : EmptyRun();
+ Task<(IReadOnlyList Items, IReadOnlyList Outcomes)> entityRun;
+ Task<(IReadOnlyList Items, IReadOnlyList Outcomes)> factRun;
+ Task<(IReadOnlyList Items, IReadOnlyList Outcomes)> prefRun;
+ Task<(IReadOnlyList Items, IReadOnlyList Outcomes)> relRun;
+ IReadOnlyList unifiedOutcomes = Array.Empty();
+ var unifiedExtractor = typesToExtract != ExtractionTypes.None
+ ? _unifiedExtractors.FirstOrDefault(extractor => extractor.IsEnabled)
+ : null;
+ if (preExtracted is not null)
+ {
+ entityRun = CompletedRun(typesToExtract.HasFlag(ExtractionTypes.Entities) ? preExtracted.Entities : []);
+ factRun = CompletedRun(typesToExtract.HasFlag(ExtractionTypes.Facts) ? preExtracted.Facts : []);
+ prefRun = CompletedRun(typesToExtract.HasFlag(ExtractionTypes.Preferences) ? preExtracted.Preferences : []);
+ relRun = CompletedRun(typesToExtract.HasFlag(ExtractionTypes.Relationships) ? preExtracted.Relationships : []);
+ }
+ else if (unifiedExtractor is not null)
+ {
+ var unifiedRun = await ExtractUnifiedSafeAsync(
+ unifiedExtractor, messages, typesToExtract, cancellationToken).ConfigureAwait(false);
+ var unified = unifiedRun.Result;
+ unifiedOutcomes = unifiedRun.Outcomes;
+ entityRun = CompletedRun(typesToExtract.HasFlag(ExtractionTypes.Entities) ? unified.Entities : []);
+ factRun = CompletedRun(typesToExtract.HasFlag(ExtractionTypes.Facts) ? unified.Facts : []);
+ prefRun = CompletedRun(typesToExtract.HasFlag(ExtractionTypes.Preferences) ? unified.Preferences : []);
+ relRun = CompletedRun(typesToExtract.HasFlag(ExtractionTypes.Relationships) ? unified.Relationships : []);
+ }
+ else
+ {
+ entityRun = typesToExtract.HasFlag(ExtractionTypes.Entities)
+ ? RunExtractorsAsync(_entityExtractors, e => e.ExtractAsync(messages, cancellationToken),
+ strategy, MergeStrategyFactory.CreateEntityStrategy, "entity",
+ MemoryItemKind.Entity, MemoryErrorCodes.EntityExtractionFailed, cancellationToken)
+ : EmptyRun();
+ factRun = typesToExtract.HasFlag(ExtractionTypes.Facts)
+ ? RunExtractorsAsync(_factExtractors, f => f.ExtractAsync(messages, cancellationToken),
+ strategy, MergeStrategyFactory.CreateFactStrategy, "fact",
+ MemoryItemKind.Fact, MemoryErrorCodes.FactExtractionFailed, cancellationToken)
+ : EmptyRun();
+ prefRun = typesToExtract.HasFlag(ExtractionTypes.Preferences)
+ ? RunExtractorsAsync(_preferenceExtractors, p => p.ExtractAsync(messages, cancellationToken),
+ strategy, MergeStrategyFactory.CreatePreferenceStrategy, "preference",
+ MemoryItemKind.Preference, MemoryErrorCodes.PreferenceExtractionFailed, cancellationToken)
+ : EmptyRun();
+ relRun = typesToExtract.HasFlag(ExtractionTypes.Relationships)
+ ? RunExtractorsAsync(_relationshipExtractors, r => r.ExtractAsync(messages, cancellationToken),
+ strategy, MergeStrategyFactory.CreateRelationshipStrategy, "relationship",
+ MemoryItemKind.Relationship, MemoryErrorCodes.RelationshipExtractionFailed, cancellationToken)
+ : EmptyRun();
+ }
await Task.WhenAll(entityRun, factRun, prefRun, relRun).ConfigureAwait(false);
@@ -91,6 +145,7 @@ public async Task ExtractAsync(
var (rawRelationships, relOutcomes) = await relRun.ConfigureAwait(false);
var outcomes = new List();
+ outcomes.AddRange(unifiedOutcomes);
outcomes.AddRange(entityOutcomes);
outcomes.AddRange(factOutcomes);
outcomes.AddRange(prefOutcomes);
@@ -106,6 +161,18 @@ public async Task ExtractAsync(
"Ingestion failed fast: one or more extractors threw.", outcomes);
}
+ if (_entityResolver is IExtractionEntityResolver batchResolver)
+ {
+ var candidateTypes = rawEntities
+ .Where(entity =>
+ entity.Confidence >= _options.MinConfidenceThreshold &&
+ EntityValidator.IsValid(entity, _options.Validation))
+ .Select(entity => entity.Type)
+ .ToArray();
+ await batchResolver.PrepareCandidatesAsync(candidateTypes, scope, cancellationToken)
+ .ConfigureAwait(false);
+ }
+
// 2. Filter + validate + resolve entities; build name→Entity map for relationship resolution.
// Spanned separately from extraction: resolution is a SEQUENTIAL per-entity loop, so unlike the
// concurrent extractor categories above its cost grows linearly with entity count.
@@ -144,8 +211,16 @@ public async Task ExtractAsync(
try
{
- var entity = await _entityResolver.ResolveEntityAsync(
- extracted, sourceMessageIds, scope, cancellationToken).ConfigureAwait(false);
+ // The extraction pipeline always hands this resolved entity to PersistenceStage. When
+ // that stage owns a coalesced transaction, an eager resolver upsert would write the
+ // same entity twice and sit outside the logical commit boundary. Direct resolver callers
+ // retain their historical persist-on-resolve behavior through ResolveEntityAsync.
+ var deferPersistence = failFast || _options.UseCoalescedPersistenceTransactions;
+ var entity = deferPersistence && _entityResolver is IExtractionEntityResolver deferredResolver
+ ? await deferredResolver.ResolveForPersistenceAsync(
+ extracted, sourceMessageIds, scope, cancellationToken).ConfigureAwait(false)
+ : await _entityResolver.ResolveEntityAsync(
+ extracted, sourceMessageIds, scope, cancellationToken).ConfigureAwait(false);
resolvedEntityMap[extracted.Name] = entity;
_logger.LogDebug("Resolved entity '{Name}' (id={Id}).", entity.Name, entity.EntityId);
}
@@ -278,6 +353,65 @@ public async Task ExtractAsync(
Task.FromResult<(IReadOnlyList, IReadOnlyList)>(
(Array.Empty(), Array.Empty()));
+ private static Task<(IReadOnlyList Items, IReadOnlyList Outcomes)> CompletedRun(
+ IReadOnlyList items) where T : class =>
+ Task.FromResult((items, (IReadOnlyList)Array.Empty()));
+
+ private async Task<(UnifiedExtractionResult Result, IReadOnlyList Outcomes)> ExtractUnifiedSafeAsync(
+ IUnifiedMemoryExtractor extractor,
+ IReadOnlyList messages,
+ ExtractionTypes typesToExtract,
+ CancellationToken cancellationToken)
+ {
+ try
+ {
+ return (await extractor.ExtractAsync(messages, cancellationToken).ConfigureAwait(false),
+ Array.Empty());
+ }
+ catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
+ {
+ throw;
+ }
+ catch (Exception ex)
+ {
+ _logger.LogError(ex, "Unified memory extraction threw — continuing with empty results.");
+ var outcomes = new List(4);
+ AddUnifiedFailure(outcomes, typesToExtract, ExtractionTypes.Entities,
+ MemoryItemKind.Entity, "entity", MemoryErrorCodes.EntityExtractionFailed, ex);
+ AddUnifiedFailure(outcomes, typesToExtract, ExtractionTypes.Facts,
+ MemoryItemKind.Fact, "fact", MemoryErrorCodes.FactExtractionFailed, ex);
+ AddUnifiedFailure(outcomes, typesToExtract, ExtractionTypes.Preferences,
+ MemoryItemKind.Preference, "preference", MemoryErrorCodes.PreferenceExtractionFailed, ex);
+ AddUnifiedFailure(outcomes, typesToExtract, ExtractionTypes.Relationships,
+ MemoryItemKind.Relationship, "relationship", MemoryErrorCodes.RelationshipExtractionFailed, ex);
+ return (new UnifiedExtractionResult(), outcomes);
+ }
+ }
+
+ private static void AddUnifiedFailure(
+ ICollection outcomes,
+ ExtractionTypes typesToExtract,
+ ExtractionTypes requiredType,
+ MemoryItemKind kind,
+ string sourceKey,
+ string errorCode,
+ Exception exception)
+ {
+ if (!typesToExtract.HasFlag(requiredType))
+ return;
+
+ outcomes.Add(new IngestionItemOutcome
+ {
+ Kind = kind,
+ Stage = IngestionStage.Extraction,
+ Status = IngestionItemStatus.Failed,
+ SourceKey = $"unified:{sourceKey}",
+ ErrorCode = errorCode,
+ ErrorMessage = exception.Message,
+ Retryable = true,
+ });
+ }
+
private async Task<(IReadOnlyList Items, IReadOnlyList Outcomes)> RunExtractorsAsync(
IReadOnlyList extractors,
Func>> extractFn,
diff --git a/src/AgentMemory.Core/Extraction/IBatchMemoryRepository.cs b/src/AgentMemory.Core/Extraction/IBatchMemoryRepository.cs
new file mode 100644
index 00000000..54d374ea
--- /dev/null
+++ b/src/AgentMemory.Core/Extraction/IBatchMemoryRepository.cs
@@ -0,0 +1,13 @@
+namespace AgentMemory.Core.Extraction;
+
+///
+/// Internal opt-in capability for repositories that can atomically upsert one memory kind as a batch.
+/// The public repository contracts remain unchanged, so third-party implementations keep their current
+/// item-at-a-time behavior unless the built-in pipeline can prove batch semantics are available.
+///
+internal interface IBatchMemoryRepository
+{
+ Task> UpsertBatchAsync(
+ IReadOnlyList items,
+ CancellationToken cancellationToken = default);
+}
diff --git a/src/AgentMemory.Core/Extraction/IExtractionStage.cs b/src/AgentMemory.Core/Extraction/IExtractionStage.cs
index a1ca268c..171c153f 100644
--- a/src/AgentMemory.Core/Extraction/IExtractionStage.cs
+++ b/src/AgentMemory.Core/Extraction/IExtractionStage.cs
@@ -9,6 +9,9 @@ namespace AgentMemory.Core.Extraction;
///
internal interface IExtractionStage
{
+ IDisposable? BeginResolutionBatch();
+ void InvalidateResolutionBatch();
+
///
/// Extracts, merges, filters, validates, and resolves items from the given messages. When
/// is supplied (R1) entity resolution is confined to the owner's own and
@@ -19,4 +22,15 @@ Task ExtractAsync(
ExtractionTypes typesToExtract,
MemoryScope? scope = null,
CancellationToken cancellationToken = default);
+
+ ///
+ /// Applies the normal validation, owner-scoped resolution, and filtering stages to a unified
+ /// result that was already extracted by a validated multi-session batch.
+ ///
+ Task ProcessUnifiedAsync(
+ IReadOnlyList messages,
+ UnifiedExtractionResult extracted,
+ ExtractionTypes typesToExtract,
+ MemoryScope? scope = null,
+ CancellationToken cancellationToken = default);
}
diff --git a/src/AgentMemory.Core/Extraction/IFusedBatchMemoryRepository.cs b/src/AgentMemory.Core/Extraction/IFusedBatchMemoryRepository.cs
new file mode 100644
index 00000000..bec26dfd
--- /dev/null
+++ b/src/AgentMemory.Core/Extraction/IFusedBatchMemoryRepository.cs
@@ -0,0 +1,13 @@
+namespace AgentMemory.Core.Extraction;
+
+///
+/// Internal opt-in capability for repositories that can fold an item's node mutation, embedding,
+/// provider-specific labels/location, and provenance edges into one bounded batch query. The public
+/// repository contracts and the legacy path remain unchanged.
+///
+internal interface IFusedBatchMemoryRepository
+{
+ Task> UpsertFusedBatchAsync(
+ IReadOnlyList items,
+ CancellationToken cancellationToken = default);
+}
diff --git a/src/AgentMemory.Core/Extraction/IMemoryPersistenceTransaction.cs b/src/AgentMemory.Core/Extraction/IMemoryPersistenceTransaction.cs
new file mode 100644
index 00000000..bbaeb340
--- /dev/null
+++ b/src/AgentMemory.Core/Extraction/IMemoryPersistenceTransaction.cs
@@ -0,0 +1,19 @@
+namespace AgentMemory.Core.Extraction;
+
+///
+/// Internal transaction boundary for one logical memory-persistence operation.
+/// Storage providers that support transactions commit all repository work atomically;
+/// providers without that capability execute the callback directly.
+///
+internal interface IMemoryPersistenceTransaction
+{
+ ///
+ /// Whether a failed callback is guaranteed not to commit and rollback failure is surfaced as a
+ /// different exception. The default keeps portable/pass-through providers on the legacy path.
+ ///
+ bool SupportsAtomicRollback => false;
+
+ Task ExecuteAsync(
+ Func> work,
+ CancellationToken cancellationToken = default);
+}
diff --git a/src/AgentMemory.Core/Extraction/IUpsertPersistsProvenance.cs b/src/AgentMemory.Core/Extraction/IUpsertPersistsProvenance.cs
new file mode 100644
index 00000000..1fd90aaf
--- /dev/null
+++ b/src/AgentMemory.Core/Extraction/IUpsertPersistsProvenance.cs
@@ -0,0 +1,7 @@
+namespace AgentMemory.Core.Extraction;
+
+///
+/// Internal capability marker for repositories whose upsert operation atomically persists every
+/// EXTRACTED_FROM edge named by the memory item's source-message IDs.
+///
+internal interface IUpsertPersistsProvenance;
diff --git a/src/AgentMemory.Core/Extraction/PersistenceStage.EmbeddingBatch.cs b/src/AgentMemory.Core/Extraction/PersistenceStage.EmbeddingBatch.cs
new file mode 100644
index 00000000..f9abb936
--- /dev/null
+++ b/src/AgentMemory.Core/Extraction/PersistenceStage.EmbeddingBatch.cs
@@ -0,0 +1,213 @@
+using AgentMemory.Abstractions.Diagnostics;
+using AgentMemory.Abstractions.Domain;
+using AgentMemory.Abstractions.Exceptions;
+using AgentMemory.Abstractions.Services;
+using Microsoft.Extensions.Logging;
+
+namespace AgentMemory.Core.Extraction;
+
+internal sealed partial class PersistenceStage
+{
+ private async Task PrepareEmbeddingsAsync(
+ ExtractionStageResult extraction,
+ CancellationToken cancellationToken)
+ {
+ if (!_options.UseBatchEmbeddingRequests)
+ return await PrepareEmbeddingsIndividuallyAsync(extraction, cancellationToken).ConfigureAwait(false);
+
+ var inputs = BuildLearnedEmbeddingInputs(extraction);
+ if (inputs.Count < 2)
+ return await PrepareEmbeddingsIndividuallyAsync(extraction, cancellationToken).ConfigureAwait(false);
+
+ var failFast = _options.FailureMode == Abstractions.Options.IngestionFailureMode.FailFast;
+ var outcomes = new List();
+ var entities = new Dictionary(StringComparer.OrdinalIgnoreCase);
+ var facts = new List(extraction.FilteredFacts.Count);
+ var preferences = new List(extraction.FilteredPreferences.Count);
+
+ foreach (var (name, entity) in extraction.ResolvedEntityMap)
+ {
+ if (entity.Embedding is not null)
+ entities[name] = entity;
+ }
+
+ IReadOnlyList? batchResults = null;
+ var replayWholeBatch = false;
+ try
+ {
+ batchResults = await _embeddingOrchestrator
+ .EmbedBatchAsync(inputs.Select(input => input.Text).ToArray(), cancellationToken)
+ .ConfigureAwait(false);
+
+ if (batchResults is null || batchResults.Count != inputs.Count)
+ {
+ replayWholeBatch = true;
+ _logger.LogWarning(
+ "Learned-memory embedding batch returned {Returned} vectors for {Requested} inputs; replaying the batch through the item path.",
+ batchResults?.Count ?? 0,
+ inputs.Count);
+ }
+ }
+ catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
+ {
+ throw;
+ }
+ catch (Exception ex)
+ {
+ replayWholeBatch = true;
+ _logger.LogWarning(
+ ex,
+ "Learned-memory embedding batch failed for {Count} inputs; replaying through the item path.",
+ inputs.Count);
+ }
+
+ for (var index = 0; index < inputs.Count; index++)
+ {
+ var input = inputs[index];
+ float[]? embedding = !replayWholeBatch &&
+ batchResults![index] is { Length: > 0 } available
+ ? available
+ : await EmbedSingleLearnedInputAsync(
+ input,
+ outcomes,
+ failFast,
+ cancellationToken).ConfigureAwait(false);
+
+ if (embedding is null)
+ continue;
+
+ switch (input.Kind)
+ {
+ case MemoryItemKind.Entity:
+ entities[input.SourceKey] = input.Entity! with { Embedding = embedding };
+ break;
+ case MemoryItemKind.Fact:
+ facts.Add(new PreparedFact(input.Fact!, embedding));
+ break;
+ case MemoryItemKind.Preference:
+ preferences.Add(new PreparedPreference(input.Preference!, embedding));
+ break;
+ default:
+ throw new InvalidOperationException(
+ $"Unsupported learned-memory embedding kind '{input.Kind}'.");
+ }
+ }
+
+ return new PreparedEmbeddings(entities, facts, preferences, outcomes);
+ }
+
+ private static List BuildLearnedEmbeddingInputs(
+ ExtractionStageResult extraction)
+ {
+ var inputs = new List(
+ extraction.ResolvedEntityMap.Count +
+ extraction.FilteredFacts.Count +
+ extraction.FilteredPreferences.Count);
+
+ foreach (var (name, entity) in extraction.ResolvedEntityMap)
+ {
+ if (entity.Embedding is null)
+ {
+ inputs.Add(new LearnedEmbeddingInput(
+ MemoryItemKind.Entity,
+ name,
+ entity.Name,
+ Entity: entity));
+ }
+ }
+
+ foreach (var fact in extraction.FilteredFacts)
+ {
+ var text = $"{fact.Subject} {fact.Predicate} {fact.Object}";
+ inputs.Add(new LearnedEmbeddingInput(
+ MemoryItemKind.Fact,
+ text,
+ text,
+ Fact: fact));
+ }
+
+ foreach (var preference in extraction.FilteredPreferences)
+ {
+ inputs.Add(new LearnedEmbeddingInput(
+ MemoryItemKind.Preference,
+ preference.PreferenceText,
+ preference.PreferenceText,
+ Preference: preference));
+ }
+
+ return inputs;
+ }
+
+ private async Task EmbedSingleLearnedInputAsync(
+ LearnedEmbeddingInput input,
+ List outcomes,
+ bool failFast,
+ CancellationToken cancellationToken)
+ {
+ try
+ {
+ return input.Kind switch
+ {
+ MemoryItemKind.Entity => await _embeddingOrchestrator
+ .EmbedEntityAsync(input.Entity!.Name, cancellationToken)
+ .ConfigureAwait(false),
+ MemoryItemKind.Fact => await _embeddingOrchestrator
+ .EmbedFactAsync(
+ input.Fact!.Subject,
+ input.Fact.Predicate,
+ input.Fact.Object,
+ cancellationToken)
+ .ConfigureAwait(false),
+ MemoryItemKind.Preference => await _embeddingOrchestrator
+ .EmbedPreferenceAsync(input.Preference!.PreferenceText, cancellationToken)
+ .ConfigureAwait(false),
+ _ => throw new InvalidOperationException(
+ $"Unsupported learned-memory embedding kind '{input.Kind}'.")
+ };
+ }
+ catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
+ {
+ throw;
+ }
+ catch (Exception ex)
+ {
+ _logger.LogError(
+ ex,
+ "Error generating learned-memory embedding for {Kind} '{SourceKey}'.",
+ input.Kind,
+ input.SourceKey);
+ RecordFailureAndMaybeThrow(
+ outcomes,
+ failFast,
+ input.Kind,
+ IngestionStage.Embedding,
+ MemoryErrorCodes.EmbeddingGenerationFailed,
+ input.SourceKey,
+ null,
+ ex,
+ FailFastEmbeddingMessage(input));
+ return null;
+ }
+ }
+
+ private static string FailFastEmbeddingMessage(LearnedEmbeddingInput input) =>
+ input.Kind switch
+ {
+ MemoryItemKind.Entity =>
+ $"Ingestion failed fast: embedding generation failed for entity '{input.SourceKey}'.",
+ MemoryItemKind.Fact =>
+ $"Ingestion failed fast: embedding generation failed for fact '{input.SourceKey}'.",
+ MemoryItemKind.Preference =>
+ "Ingestion failed fast: embedding generation failed for a preference.",
+ _ =>
+ $"Ingestion failed fast: embedding generation failed for '{input.SourceKey}'."
+ };
+
+ private sealed record LearnedEmbeddingInput(
+ MemoryItemKind Kind,
+ string SourceKey,
+ string Text,
+ Entity? Entity = null,
+ ExtractedFact? Fact = null,
+ ExtractedPreference? Preference = null);
+}
diff --git a/src/AgentMemory.Core/Extraction/PersistenceStage.cs b/src/AgentMemory.Core/Extraction/PersistenceStage.cs
index 548381ed..b085e800 100644
--- a/src/AgentMemory.Core/Extraction/PersistenceStage.cs
+++ b/src/AgentMemory.Core/Extraction/PersistenceStage.cs
@@ -13,7 +13,7 @@ namespace AgentMemory.Core.Extraction;
/// Embeds and persists the resolved items from .
/// Responsibility: generate embeddings, upsert to repositories, wire EXTRACTED_FROM provenance.
///
-internal sealed class PersistenceStage : IPersistenceStage
+internal sealed partial class PersistenceStage : IPersistenceStage
{
private readonly IEmbeddingOrchestrator _embeddingOrchestrator;
private readonly IEntityRepository _entityRepository;
@@ -23,6 +23,7 @@ internal sealed class PersistenceStage : IPersistenceStage
private readonly IClock _clock;
private readonly IIdGenerator _idGenerator;
private readonly ExtractionOptions _options;
+ private readonly IMemoryPersistenceTransaction _persistenceTransaction;
private readonly ILogger _logger;
public PersistenceStage(
@@ -34,6 +35,7 @@ public PersistenceStage(
IClock clock,
IIdGenerator idGenerator,
ILogger logger,
+ IMemoryPersistenceTransaction persistenceTransaction,
IOptions? extractionOptions = null)
{
_embeddingOrchestrator = embeddingOrchestrator;
@@ -44,6 +46,7 @@ public PersistenceStage(
_clock = clock;
_idGenerator = idGenerator;
_logger = logger;
+ _persistenceTransaction = persistenceTransaction ?? throw new ArgumentNullException(nameof(persistenceTransaction));
_options = extractionOptions?.Value ?? new ExtractionOptions();
}
@@ -53,11 +56,9 @@ public async Task PersistAsync(
MemoryTrustLevel trustLevel = MemoryTrustLevel.Untrusted,
CancellationToken cancellationToken = default)
{
- // Spans the whole persistence stage. The four per-kind blocks below are sequential loops that
- // embed and upsert one item at a time, so the stage's cost grows with how much the turn produced
- // -- the candidate counts are tagged here so that growth is attributable without needing four
- // more spans. (Per-kind TIMING would mean restructuring those loops; the counts plus the stage
- // total answer "is persistence expensive, and because of how many of what" already.)
+ // External embedding work is deliberately completed before the storage transaction opens.
+ // Holding a database transaction while waiting on a model/provider would amplify contention
+ // and make provider latency part of the database failure surface.
using var activity = AgentMemoryDiagnostics.Source.StartActivity("memory.persist.total");
if (activity is not null)
{
@@ -67,70 +68,126 @@ public async Task PersistAsync(
activity.SetTag("memory.persist.relationships", extraction.FilteredRelationships.Count);
}
+ var prepared = await PrepareEmbeddingsAsync(extraction, cancellationToken).ConfigureAwait(false);
+ if (_options.FailureMode == IngestionFailureMode.FailFast)
+ {
+ try
+ {
+ return await _persistenceTransaction.ExecuteAsync(
+ ct => PersistPreparedAsync(extraction, ownerId, trustLevel, prepared, ct),
+ cancellationToken).ConfigureAwait(false);
+ }
+ catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
+ {
+ throw;
+ }
+ catch (MemoryIngestionException)
+ {
+ throw;
+ }
+ catch (Exception ex)
+ {
+ // Transaction-entry, commit, and rollback-confirmation failures occur outside the
+ // per-item catch blocks below. Preserve the documented fail-fast boundary while
+ // retaining the provider/transaction failure as the inner cause. Outcomes created
+ // inside the rolled-back transaction are deliberately excluded as non-durable.
+ var completedOutcomes = extraction.Outcomes.Concat(prepared.Outcomes).ToList();
+ throw new MemoryIngestionException(
+ "Atomic memory persistence failed.", completedOutcomes, ex);
+ }
+ }
+
+ if (!_options.UseCoalescedPersistenceTransactions ||
+ !_persistenceTransaction.SupportsAtomicRollback)
+ {
+ return await PersistPreparedAsync(
+ extraction, ownerId, trustLevel, prepared, cancellationToken).ConfigureAwait(false);
+ }
+
+ try
+ {
+ return await _persistenceTransaction.ExecuteAsync(
+ async ct =>
+ {
+ var result = await PersistPreparedAsync(
+ extraction, ownerId, trustLevel, prepared, ct).ConfigureAwait(false);
+ if (result.Outcomes.Any(outcome => outcome.Status == IngestionItemStatus.Failed))
+ throw new ReplayBestEffortPersistenceException();
+ return result;
+ },
+ cancellationToken).ConfigureAwait(false);
+ }
+ catch (ReplayBestEffortPersistenceException)
+ {
+ // ExecuteAsync may surface this marker only after its provider rollback completed. Reuse
+ // the already prepared embeddings and replay through today's item-isolated best-effort path.
+ return await PersistPreparedAsync(
+ extraction, ownerId, trustLevel, prepared, cancellationToken).ConfigureAwait(false);
+ }
+ }
+
+ private sealed class ReplayBestEffortPersistenceException : Exception;
+
+ private async Task PersistPreparedAsync(
+ ExtractionStageResult extraction,
+ string? ownerId,
+ MemoryTrustLevel trustLevel,
+ PreparedEmbeddings prepared,
+ CancellationToken cancellationToken)
+ {
var sourceMessageIds = extraction.SourceMessageIds;
var failFast = _options.FailureMode == IngestionFailureMode.FailFast;
var outcomes = new List(extraction.Outcomes);
+ outcomes.AddRange(prepared.Outcomes);
// 1. Embed + upsert entities; build a name→persisted Entity map for relationship resolution.
var persistedEntityMap = new Dictionary(StringComparer.OrdinalIgnoreCase);
- foreach (var (name, entity) in extraction.ResolvedEntityMap)
+ var entityInputs = prepared.Entities.Select(pair =>
+ {
+ var effectiveTrustLevel = MaxTrustLevel(pair.Value.Metadata.GetTrustLevel(), trustLevel);
+ return (Name: pair.Key, Item: pair.Value with
+ {
+ OwnerId = ownerId,
+ Metadata = pair.Value.Metadata.WithTrustLevel(effectiveTrustLevel)
+ });
+ }).ToList();
+
+ async Task RecordPersistedEntityAsync(string name, Entity persisted)
{
- // Trust is monotonic, never silently downgraded: when entity resolution (auto-merge/SAME_AS)
- // resolves this mention onto an EXISTING, previously-persisted entity, `entity` already carries
- // that entity's own prior Metadata/trust level. An unrelated later mention at a lower trust
- // level (e.g. an ordinary chat turn) must not erase a deliberately-elevated trust stamp (e.g.
- // from a curated ApplicationTrusted import) -- take whichever of the two is higher.
- var effectiveTrustLevel = MaxTrustLevel(entity.Metadata.GetTrustLevel(), trustLevel);
- var entityToSave = entity with { OwnerId = ownerId, Metadata = entity.Metadata.WithTrustLevel(effectiveTrustLevel) };
+ persistedEntityMap[name] = persisted;
+ RecordSuccess(outcomes, MemoryItemKind.Entity, name, persisted.EntityId);
- if (entityToSave.Embedding is null)
+ foreach (var msgId in ExplicitProvenanceMessageIds(_entityRepository, sourceMessageIds))
{
try
{
- var embedding = await _embeddingOrchestrator.EmbedEntityAsync(
- entityToSave.Name, cancellationToken).ConfigureAwait(false);
- entityToSave = entityToSave with { Embedding = embedding };
+ await _entityRepository.CreateExtractedFromRelationshipAsync(
+ persisted.EntityId, msgId, cancellationToken: cancellationToken).ConfigureAwait(false);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { throw; }
catch (Exception ex)
{
- _logger.LogError(ex, "Error generating embedding for entity '{Name}'.", name);
- RecordFailureAndMaybeThrow(outcomes, failFast, MemoryItemKind.Entity, IngestionStage.Embedding,
- MemoryErrorCodes.EmbeddingGenerationFailed, name, null, ex,
- $"Ingestion failed fast: embedding generation failed for entity '{name}'.");
- continue; // no embedding — nothing to persist for this entity
+ _logger.LogWarning(ex,
+ "Failed to create EXTRACTED_FROM for entity '{Id}' → message '{MsgId}'.",
+ persisted.EntityId, msgId);
+ RecordFailureAndMaybeThrow(outcomes, failFast, MemoryItemKind.Entity, IngestionStage.Provenance,
+ MemoryErrorCodes.ProvenancePersistenceFailed, name, persisted.EntityId, ex,
+ $"Ingestion failed fast: provenance failed for entity '{name}'.");
}
}
+ _logger.LogDebug("Persisted entity '{Name}' (id={Id}).", persisted.Name, persisted.EntityId);
+ }
+
+ async Task PersistEntityIndividuallyAsync(string name, Entity item)
+ {
try
{
- entityToSave = await _entityRepository.UpsertAsync(entityToSave, cancellationToken).ConfigureAwait(false);
- persistedEntityMap[name] = entityToSave;
- RecordSuccess(outcomes, MemoryItemKind.Entity, name, entityToSave.EntityId);
-
- foreach (var msgId in sourceMessageIds)
- {
- try
- {
- await _entityRepository.CreateExtractedFromRelationshipAsync(
- entityToSave.EntityId, msgId, cancellationToken: cancellationToken).ConfigureAwait(false);
- }
- catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { throw; }
- catch (Exception ex)
- {
- _logger.LogWarning(ex,
- "Failed to create EXTRACTED_FROM for entity '{Id}' → message '{MsgId}'.",
- entityToSave.EntityId, msgId);
- RecordFailureAndMaybeThrow(outcomes, failFast, MemoryItemKind.Entity, IngestionStage.Provenance,
- MemoryErrorCodes.ProvenancePersistenceFailed, name, entityToSave.EntityId, ex,
- $"Ingestion failed fast: provenance failed for entity '{name}'.");
- }
- }
-
- _logger.LogDebug("Persisted entity '{Name}' (id={Id}).", entityToSave.Name, entityToSave.EntityId);
+ var persisted = await _entityRepository.UpsertAsync(item, cancellationToken).ConfigureAwait(false);
+ await RecordPersistedEntityAsync(name, persisted).ConfigureAwait(false);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { throw; }
- catch (MemoryIngestionException) { throw; } // already recorded + wrapped above — propagate as-is
+ catch (MemoryIngestionException) { throw; }
catch (Exception ex)
{
_logger.LogError(ex, "Error persisting entity '{Name}'.", name);
@@ -140,79 +197,61 @@ await _entityRepository.CreateExtractedFromRelationshipAsync(
}
}
- // 2. Embed + upsert facts.
- var persistedFactCount = 0;
- foreach (var extracted in extraction.FilteredFacts)
+ Dictionary? batchedEntitiesById = null;
+ var fusedEntityRepository = _options.UseCoalescedPersistenceTransactions
+ ? _entityRepository as IFusedBatchMemoryRepository : null;
+ var batchEntityRepository = _entityRepository as IBatchMemoryRepository;
+ var canBatchEntities = _options.EnableBatchMemoryUpserts && !failFast &&
+ entityInputs.Count > 0 &&
+ entityInputs.Select(input => input.Item.EntityId).Distinct(StringComparer.Ordinal).Count() == entityInputs.Count &&
+ (fusedEntityRepository is not null || (entityInputs.Count > 1 && batchEntityRepository is not null));
+ if (canBatchEntities)
{
- // factSourceKey (outcome/log identification) and the embedding below are both computed from the
- // freshly-extracted casing, even though the fact ultimately persisted may use an existing
- // record's casing instead when the #92 Phase 5 pre-fetch finds a case-insensitive match (see
- // below) -- a disclosed, cosmetic-only inconsistency (found in a post-Phase-5 holistic audit):
- // an outcome/log entry for a casing-only re-extraction won't textually match what was persisted,
- // and the surviving node's Embedding and Subject/Predicate/Object can reflect two different
- // casings of the same triple. Embeddings are semantically robust to case, so this hasn't been
- // observed to affect retrieval quality; not fixed here to keep this phase's blast radius narrow.
- var factSourceKey = $"{extracted.Subject} {extracted.Predicate} {extracted.Object}";
-
- float[] factEmbedding;
try
{
- factEmbedding = await _embeddingOrchestrator.EmbedFactAsync(
- extracted.Subject, extracted.Predicate, extracted.Object, cancellationToken).ConfigureAwait(false);
+ var items = entityInputs.Select(input => input.Item).ToList();
+ var persisted = fusedEntityRepository is not null
+ ? await fusedEntityRepository.UpsertFusedBatchAsync(items, cancellationToken)
+ .ConfigureAwait(false)
+ : await batchEntityRepository!.UpsertBatchAsync(items, cancellationToken)
+ .ConfigureAwait(false);
+ batchedEntitiesById = persisted.ToDictionary(entity => entity.EntityId, StringComparer.Ordinal);
+ if (entityInputs.Any(input => !batchedEntitiesById.ContainsKey(input.Item.EntityId)))
+ throw new InvalidOperationException("The entity batch result omitted one or more input identifiers.");
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { throw; }
catch (Exception ex)
{
- _logger.LogError(ex, "Error generating embedding for fact '{Key}'.", factSourceKey);
- RecordFailureAndMaybeThrow(outcomes, failFast, MemoryItemKind.Fact, IngestionStage.Embedding,
- MemoryErrorCodes.EmbeddingGenerationFailed, factSourceKey, null, ex,
- $"Ingestion failed fast: embedding generation failed for fact '{factSourceKey}'.");
- continue;
+ _logger.LogWarning(ex,
+ "Atomic entity batch failed; replaying {Count} entities through the item path.",
+ entityInputs.Count);
+ batchedEntitiesById = null;
}
+ }
+
+ if (batchedEntitiesById is not null)
+ {
+ foreach (var input in entityInputs)
+ await RecordPersistedEntityAsync(input.Name, batchedEntitiesById[input.Item.EntityId]).ConfigureAwait(false);
+ }
+ else
+ {
+ foreach (var input in entityInputs)
+ await PersistEntityIndividuallyAsync(input.Name, input.Item).ConfigureAwait(false);
+ }
+ // 2. Embed + upsert facts.
+ var persistedFactCount = 0;
+ async Task<(Fact Item, string SourceKey)?> PrepareFactAsync(PreparedFact preparedFact)
+ {
+ var extracted = preparedFact.Item;
+ var factSourceKey = $"{extracted.Subject} {extracted.Predicate} {extracted.Object}";
try
{
- // Trust is monotonic for owner-scoped facts too (#92 Phase 5), mirroring entities (Phase 3):
- // the repository's Upsert MERGEs on the exact {subject,predicate,object,owner} triple and its
- // Cypher ON MATCH unconditionally overwrites metadata, so re-extracting the identical triple
- // at a lower trust level (e.g. an ordinary chat turn re-stating a fact originally imported at
- // ApplicationTrusted) would otherwise silently erase the earlier elevation. Unlike entities,
- // facts have no upstream resolution step that hands PersistenceStage the prior record for
- // free, so this pre-fetch is the "one extra round-trip" the Phase 3 doc flagged as needed.
- // A lookup failure falls through to the same catch below as an ordinary persistence failure.
- //
- // Disclosed, unaddressed limitation (found in a post-Phase-5 holistic audit): this pre-fetch
- // and the Upsert below are two separate, non-atomic Neo4j round-trips, not one atomic
- // read-modify-write. Two concurrent extractions racing on the identical triple (e.g. a
- // curated ApplicationTrusted import racing an ordinary chat-turn extraction) could each read
- // the same stale prior state and independently compute their own "effective" trust, so
- // whichever Upsert commits last wins outright rather than the two being reconciled -- a
- // narrow, real gap in "never decreases" under genuine concurrency on the same triple.
- // Matches this codebase's existing precedent of disclosing rather than solving multi-step,
- // non-atomic writes (see the threat model's TT-12, record+provenance-edge non-atomicity).
- //
- // Only performed when ownerId is set (string.IsNullOrEmpty, matching how the rest of this
- // codebase treats an empty owner id the same as a null one -- e.g. DefaultMemoryIsolationPolicy):
- // FindByTripleAsync's MemoryScope? parameter follows the read/recall convention where an
- // unscoped lookup (null, or a scope with no OwnerId) means "search across every owner" -- the
- // opposite of what a null ownerId means on the WRITE side (the shared/global bucket). Passing
- // an owner-less scope here would risk adopting another owner's trust level into a shared fact
- // -- a cross-tenant leak. Unlike FindDuplicateAsync (whose raw ownerId parameter is documented
- // as "null -> shared bucket only"), there is no existing repository primitive for a safe
- // shared-bucket-only lookup, so shared/global facts don't get this protection yet -- a
- // disclosed, narrower-than-ideal limitation for this phase.
- //
- // includeShared: false -- deliberately excludes shared/global facts from the pre-fetch even
- // though MemoryScope.For defaults to including them. The default is right for READS (surface
- // everything the caller may see), but wrong here: with no ORDER BY, a shared fact and this
- // owner's own fact could both match the same triple, and picking up the shared one would
- // graft an unrelated record's ENTIRE metadata (not just its trust level) onto this owner's
- // fact -- conflating two conceptually distinct records that merely share text.
- //
- // FindByTripleAsync matches case-insensitively but Upsert's MERGE key is an exact-string
- // match -- if a match is found, this fact is built from the EXISTING record's Subject/
- // Predicate/Object (not the freshly-extracted casing) so the subsequent Upsert's MERGE
- // still targets the SAME node instead of creating a same-triple, different-casing duplicate.
+ // Trust is monotonic for owner-scoped facts. The pre-fetch deliberately excludes shared
+ // facts and carries an existing triple's casing forward so an exact MERGE cannot create a
+ // casing-only duplicate. Rank 20 will make this read-modify-write atomic; feat-04 leaves
+ // that owner boundary and trust behavior unchanged.
Fact? existingFact = string.IsNullOrEmpty(ownerId)
? null
: await _factRepository.FindByTripleAsync(
@@ -225,7 +264,7 @@ await _entityRepository.CreateExtractedFromRelationshipAsync(
? MemoryTrustMetadataExtensions.CreateWithTrustLevel(effectiveFactTrustLevel)
: existingFact.Metadata.WithTrustLevel(effectiveFactTrustLevel);
- var fact = new Fact
+ return (new Fact
{
FactId = _idGenerator.GenerateId(),
Subject = existingFact?.Subject ?? extracted.Subject,
@@ -234,129 +273,254 @@ await _entityRepository.CreateExtractedFromRelationshipAsync(
Confidence = extracted.Confidence,
ValidFrom = extracted.ValidFrom,
ValidUntil = extracted.ValidUntil,
- Embedding = factEmbedding,
+ Embedding = preparedFact.Embedding,
OwnerId = ownerId,
SourceMessageIds = sourceMessageIds,
CreatedAtUtc = _clock.UtcNow,
Metadata = factMetadata
- };
-
- // Facts MERGE on the natural {subject,predicate,object,owner_key} triple, and ON MATCH
- // deliberately never rewrites the surviving node's id (Neo4jFactRepository's own contract) --
- // so on a re-extraction hit, fact.FactId (the freshly-generated guid above) is orphaned and
- // was never actually persisted. Reassign from the repository's return value, mirroring the
- // entity block above, so RecordSuccess and the EXTRACTED_FROM loop below use the real,
- // surviving node's id.
- fact = await _factRepository.UpsertAsync(fact, cancellationToken).ConfigureAwait(false);
- RecordSuccess(outcomes, MemoryItemKind.Fact, factSourceKey, fact.FactId);
-
- foreach (var msgId in sourceMessageIds)
- {
- try
- {
- await _factRepository.CreateExtractedFromRelationshipAsync(
- fact.FactId, msgId, cancellationToken).ConfigureAwait(false);
- }
- catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { throw; }
- catch (Exception ex)
- {
- _logger.LogWarning(ex,
- "Failed to create EXTRACTED_FROM for fact '{Id}' → message '{MsgId}'.",
- fact.FactId, msgId);
- RecordFailureAndMaybeThrow(outcomes, failFast, MemoryItemKind.Fact, IngestionStage.Provenance,
- MemoryErrorCodes.ProvenancePersistenceFailed, factSourceKey, fact.FactId, ex,
- $"Ingestion failed fast: provenance failed for fact '{factSourceKey}'.");
- }
- }
-
- persistedFactCount++;
- _logger.LogDebug("Persisted fact '{S} {P} {O}'.", fact.Subject, fact.Predicate, fact.Object);
+ }, factSourceKey);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { throw; }
- catch (MemoryIngestionException) { throw; }
catch (Exception ex)
{
- _logger.LogError(ex, "Error persisting fact '{Key}'.", factSourceKey);
+ _logger.LogError(ex, "Error preparing fact '{Key}' for persistence.", factSourceKey);
RecordFailureAndMaybeThrow(outcomes, failFast, MemoryItemKind.Fact, IngestionStage.Persistence,
MemoryErrorCodes.FactPersistenceFailed, factSourceKey, null, ex,
$"Ingestion failed fast: persistence failed for fact '{factSourceKey}'.");
+ return null;
}
}
- // 3. Embed + upsert preferences.
- var persistedPrefCount = 0;
- foreach (var extracted in extraction.FilteredPreferences)
+ async Task RecordPersistedFactAsync(string sourceKey, Fact persisted)
+ {
+ // Fact upsert MERGEs on the natural triple and may return an older stable id. Always use
+ // the repository result for outcomes and provenance rather than the fresh caller id.
+ RecordSuccess(outcomes, MemoryItemKind.Fact, sourceKey, persisted.FactId);
+
+ foreach (var msgId in ExplicitProvenanceMessageIds(_factRepository, sourceMessageIds))
+ {
+ try
+ {
+ await _factRepository.CreateExtractedFromRelationshipAsync(
+ persisted.FactId, msgId, cancellationToken).ConfigureAwait(false);
+ }
+ catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { throw; }
+ catch (Exception ex)
+ {
+ _logger.LogWarning(ex,
+ "Failed to create EXTRACTED_FROM for fact '{Id}' → message '{MsgId}'.",
+ persisted.FactId, msgId);
+ RecordFailureAndMaybeThrow(outcomes, failFast, MemoryItemKind.Fact, IngestionStage.Provenance,
+ MemoryErrorCodes.ProvenancePersistenceFailed, sourceKey, persisted.FactId, ex,
+ $"Ingestion failed fast: provenance failed for fact '{sourceKey}'.");
+ }
+ }
+
+ persistedFactCount++;
+ _logger.LogDebug("Persisted fact '{S} {P} {O}'.",
+ persisted.Subject, persisted.Predicate, persisted.Object);
+ }
+
+ async Task PersistFactIndividuallyAsync(Fact item, string sourceKey)
{
- float[] prefEmbedding;
try
{
- prefEmbedding = await _embeddingOrchestrator.EmbedPreferenceAsync(
- extracted.PreferenceText, cancellationToken).ConfigureAwait(false);
+ var persisted = await _factRepository.UpsertAsync(item, cancellationToken).ConfigureAwait(false);
+ await RecordPersistedFactAsync(sourceKey, persisted).ConfigureAwait(false);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { throw; }
+ catch (MemoryIngestionException) { throw; }
catch (Exception ex)
{
- _logger.LogError(ex, "Error generating embedding for preference '{Text}'.", extracted.PreferenceText);
- RecordFailureAndMaybeThrow(outcomes, failFast, MemoryItemKind.Preference, IngestionStage.Embedding,
- MemoryErrorCodes.EmbeddingGenerationFailed, extracted.PreferenceText, null, ex,
- "Ingestion failed fast: embedding generation failed for a preference.");
- continue;
+ _logger.LogError(ex, "Error persisting fact '{Key}'.", sourceKey);
+ RecordFailureAndMaybeThrow(outcomes, failFast, MemoryItemKind.Fact, IngestionStage.Persistence,
+ MemoryErrorCodes.FactPersistenceFailed, sourceKey, null, ex,
+ $"Ingestion failed fast: persistence failed for fact '{sourceKey}'.");
}
+ }
- try
+ static (string Subject, string Predicate, string Object, string? OwnerId) FactKey(Fact fact) =>
+ (fact.Subject, fact.Predicate, fact.Object, fact.OwnerId);
+
+ var distinctExtractedTriples = extraction.FilteredFacts
+ .Select(fact => (fact.Subject, fact.Predicate, fact.Object))
+ .Distinct(FactTripleComparer.OrdinalIgnoreCase)
+ .Count() == extraction.FilteredFacts.Count;
+ var fusedFactRepository = _options.UseCoalescedPersistenceTransactions
+ ? _factRepository as IFusedBatchMemoryRepository : null;
+ var batchFactRepository = _factRepository as IBatchMemoryRepository;
+ var canAttemptFactBatch = _options.EnableBatchMemoryUpserts && !failFast && distinctExtractedTriples &&
+ (fusedFactRepository is not null ||
+ (prepared.Facts.Count > 1 && batchFactRepository is not null));
+
+ if (canAttemptFactBatch)
+ {
+ var factInputs = new List<(Fact Item, string SourceKey)>(prepared.Facts.Count);
+ foreach (var preparedFact in prepared.Facts)
+ {
+ if (await PrepareFactAsync(preparedFact).ConfigureAwait(false) is { } input)
+ factInputs.Add(input);
+ }
+
+ Dictionary<(string Subject, string Predicate, string Object, string? OwnerId), Fact>? batchedFactsByKey = null;
+ if (factInputs.Count > 0 &&
+ factInputs.Select(input => FactKey(input.Item)).Distinct().Count() == factInputs.Count)
{
- var preference = new Preference
+ try
{
- PreferenceId = _idGenerator.GenerateId(),
- Category = extracted.Category,
- PreferenceText = extracted.PreferenceText,
- Context = extracted.Context,
- Confidence = extracted.Confidence,
- Embedding = prefEmbedding,
- OwnerId = ownerId,
- SourceMessageIds = sourceMessageIds,
- CreatedAtUtc = _clock.UtcNow,
- Metadata = MemoryTrustMetadataExtensions.CreateWithTrustLevel(trustLevel)
- };
+ var items = factInputs.Select(input => input.Item).ToList();
+ var persisted = fusedFactRepository is not null
+ ? await fusedFactRepository.UpsertFusedBatchAsync(items, cancellationToken).ConfigureAwait(false)
+ : await batchFactRepository!.UpsertBatchAsync(items, cancellationToken).ConfigureAwait(false);
+ batchedFactsByKey = persisted.ToDictionary(FactKey);
+ if (factInputs.Any(input => !batchedFactsByKey.ContainsKey(FactKey(input.Item))))
+ throw new InvalidOperationException("The fact batch result omitted one or more input triples.");
+ }
+ catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { throw; }
+ catch (Exception ex)
+ {
+ _logger.LogWarning(ex,
+ "Atomic fact batch failed; replaying {Count} facts through the item path.",
+ factInputs.Count);
+ batchedFactsByKey = null;
+ }
+ }
- await _preferenceRepository.UpsertAsync(preference, cancellationToken).ConfigureAwait(false);
- RecordSuccess(outcomes, MemoryItemKind.Preference, extracted.PreferenceText, preference.PreferenceId);
+ if (batchedFactsByKey is not null)
+ {
+ foreach (var input in factInputs)
+ await RecordPersistedFactAsync(
+ input.SourceKey, batchedFactsByKey[FactKey(input.Item)]).ConfigureAwait(false);
+ }
+ else
+ {
+ foreach (var input in factInputs)
+ await PersistFactIndividuallyAsync(input.Item, input.SourceKey).ConfigureAwait(false);
+ }
+ }
+ else
+ {
+ // Preserve the exact original read→write order for non-capable repositories, disabled/fail-fast
+ // mode, and duplicate triples. The ordering is observable because the next fact's trust/casing
+ // pre-fetch may intentionally see the fact just written by the previous item.
+ foreach (var preparedFact in prepared.Facts)
+ {
+ if (await PrepareFactAsync(preparedFact).ConfigureAwait(false) is { } input)
+ await PersistFactIndividuallyAsync(input.Item, input.SourceKey).ConfigureAwait(false);
+ }
+ }
+ // 3. Embed + upsert preferences.
+ var preferenceInputs = prepared.Preferences.Select(preparedPreference =>
+ {
+ var extracted = preparedPreference.Item;
+ return (Item: new Preference
+ {
+ PreferenceId = _idGenerator.GenerateId(),
+ Category = extracted.Category,
+ PreferenceText = extracted.PreferenceText,
+ Context = extracted.Context,
+ Confidence = extracted.Confidence,
+ Embedding = preparedPreference.Embedding,
+ OwnerId = ownerId,
+ SourceMessageIds = sourceMessageIds,
+ CreatedAtUtc = _clock.UtcNow,
+ Metadata = MemoryTrustMetadataExtensions.CreateWithTrustLevel(trustLevel)
+ }, SourceKey: extracted.PreferenceText);
+ }).ToList();
- foreach (var msgId in sourceMessageIds)
+ var persistedPrefCount = 0;
+
+ async Task RecordPersistedPreferenceAsync(string sourceKey, Preference persisted)
+ {
+ RecordSuccess(outcomes, MemoryItemKind.Preference, sourceKey, persisted.PreferenceId);
+
+ foreach (var msgId in ExplicitProvenanceMessageIds(_preferenceRepository, sourceMessageIds))
+ {
+ try
{
- try
- {
- await _preferenceRepository.CreateExtractedFromRelationshipAsync(
- preference.PreferenceId, msgId, cancellationToken).ConfigureAwait(false);
- }
- catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { throw; }
- catch (Exception ex)
- {
- _logger.LogWarning(ex,
- "Failed to create EXTRACTED_FROM for preference '{Id}' → message '{MsgId}'.",
- preference.PreferenceId, msgId);
- RecordFailureAndMaybeThrow(outcomes, failFast, MemoryItemKind.Preference, IngestionStage.Provenance,
- MemoryErrorCodes.ProvenancePersistenceFailed, extracted.PreferenceText, preference.PreferenceId, ex,
- "Ingestion failed fast: provenance failed for a preference.");
- }
+ await _preferenceRepository.CreateExtractedFromRelationshipAsync(
+ persisted.PreferenceId, msgId, cancellationToken).ConfigureAwait(false);
}
+ catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { throw; }
+ catch (Exception ex)
+ {
+ _logger.LogWarning(ex,
+ "Failed to create EXTRACTED_FROM for preference '{Id}' → message '{MsgId}'.",
+ persisted.PreferenceId, msgId);
+ RecordFailureAndMaybeThrow(outcomes, failFast, MemoryItemKind.Preference, IngestionStage.Provenance,
+ MemoryErrorCodes.ProvenancePersistenceFailed, sourceKey, persisted.PreferenceId, ex,
+ "Ingestion failed fast: provenance failed for a preference.");
+ }
+ }
+
+ persistedPrefCount++;
+ _logger.LogDebug("Persisted preference in category '{Category}'.", persisted.Category);
+ }
- persistedPrefCount++;
- _logger.LogDebug("Persisted preference in category '{Category}'.", preference.Category);
+ async Task PersistPreferenceIndividuallyAsync(Preference item, string sourceKey)
+ {
+ try
+ {
+ var persisted = await _preferenceRepository.UpsertAsync(item, cancellationToken).ConfigureAwait(false);
+ await RecordPersistedPreferenceAsync(sourceKey, persisted).ConfigureAwait(false);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { throw; }
catch (MemoryIngestionException) { throw; }
catch (Exception ex)
{
- _logger.LogError(ex, "Error persisting preference '{Text}'.", extracted.PreferenceText);
+ _logger.LogError(ex, "Error persisting preference '{Text}'.", sourceKey);
RecordFailureAndMaybeThrow(outcomes, failFast, MemoryItemKind.Preference, IngestionStage.Persistence,
- MemoryErrorCodes.PreferencePersistenceFailed, extracted.PreferenceText, null, ex,
+ MemoryErrorCodes.PreferencePersistenceFailed, sourceKey, null, ex,
"Ingestion failed fast: persistence failed for a preference.");
}
}
+ Dictionary? batchedPreferencesById = null;
+ var fusedPreferenceRepository = _options.UseCoalescedPersistenceTransactions
+ ? _preferenceRepository as IFusedBatchMemoryRepository : null;
+ var batchPreferenceRepository = _preferenceRepository as IBatchMemoryRepository;
+ var canBatchPreferences = _options.EnableBatchMemoryUpserts && !failFast &&
+ preferenceInputs.Count > 0 &&
+ preferenceInputs.Select(input => input.Item.PreferenceId).Distinct(StringComparer.Ordinal).Count() == preferenceInputs.Count &&
+ (fusedPreferenceRepository is not null ||
+ (preferenceInputs.Count > 1 && batchPreferenceRepository is not null));
+ if (canBatchPreferences)
+ {
+ try
+ {
+ var items = preferenceInputs.Select(input => input.Item).ToList();
+ var persisted = fusedPreferenceRepository is not null
+ ? await fusedPreferenceRepository.UpsertFusedBatchAsync(items, cancellationToken).ConfigureAwait(false)
+ : await batchPreferenceRepository!.UpsertBatchAsync(items, cancellationToken).ConfigureAwait(false);
+ batchedPreferencesById = persisted.ToDictionary(
+ preference => preference.PreferenceId, StringComparer.Ordinal);
+ if (preferenceInputs.Any(input => !batchedPreferencesById.ContainsKey(input.Item.PreferenceId)))
+ throw new InvalidOperationException("The preference batch result omitted one or more input identifiers.");
+ }
+ catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { throw; }
+ catch (Exception ex)
+ {
+ _logger.LogWarning(ex,
+ "Atomic preference batch failed; replaying {Count} preferences through the item path.",
+ preferenceInputs.Count);
+ batchedPreferencesById = null;
+ }
+ }
+
+ if (batchedPreferencesById is not null)
+ {
+ foreach (var input in preferenceInputs)
+ await RecordPersistedPreferenceAsync(
+ input.SourceKey, batchedPreferencesById[input.Item.PreferenceId]).ConfigureAwait(false);
+ }
+ else
+ {
+ foreach (var input in preferenceInputs)
+ await PersistPreferenceIndividuallyAsync(input.Item, input.SourceKey).ConfigureAwait(false);
+ }
// 4. Persist relationships — resolve entity IDs from the upserted entity map.
- var persistedRelCount = 0;
+ var relationshipInputs = new List<(Relationship Item, string SourceKey)>(
+ extraction.FilteredRelationships.Count);
foreach (var extracted in extraction.FilteredRelationships)
{
var relSourceKey = $"{extracted.SourceEntity}-{extracted.RelationshipType}->{extracted.TargetEntity}";
@@ -395,43 +559,86 @@ await _preferenceRepository.CreateExtractedFromRelationshipAsync(
continue;
}
- try
+ relationshipInputs.Add((new Relationship
{
- var relationship = new Relationship
- {
- RelationshipId = _idGenerator.GenerateId(),
- SourceEntityId = sourceEntity.EntityId,
- TargetEntityId = targetEntity.EntityId,
- RelationshipType = extracted.RelationshipType,
- Description = extracted.Description,
- Confidence = extracted.Confidence,
- Attributes = extracted.Attributes,
- OwnerId = ownerId,
- SourceMessageIds = sourceMessageIds,
- CreatedAtUtc = _clock.UtcNow
- };
+ RelationshipId = _idGenerator.GenerateId(),
+ SourceEntityId = sourceEntity.EntityId,
+ TargetEntityId = targetEntity.EntityId,
+ RelationshipType = extracted.RelationshipType,
+ Description = extracted.Description,
+ Confidence = extracted.Confidence,
+ Attributes = extracted.Attributes,
+ OwnerId = ownerId,
+ SourceMessageIds = sourceMessageIds,
+ CreatedAtUtc = _clock.UtcNow
+ }, relSourceKey));
+ }
- await _relationshipRepository.UpsertAsync(relationship, cancellationToken).ConfigureAwait(false);
- persistedRelCount++;
- RecordSuccess(outcomes, MemoryItemKind.Relationship, relSourceKey, relationship.RelationshipId);
+ var persistedRelCount = 0;
+
+ void RecordPersistedRelationship(string sourceKey, Relationship persisted)
+ {
+ persistedRelCount++;
+ RecordSuccess(outcomes, MemoryItemKind.Relationship, sourceKey, persisted.RelationshipId);
+ _logger.LogDebug("Persisted relationship '{SourceKey}'.", sourceKey);
+ }
- _logger.LogDebug(
- "Persisted relationship '{Src}-{Type}->{Tgt}'.",
- extracted.SourceEntity, extracted.RelationshipType, extracted.TargetEntity);
+ async Task PersistRelationshipIndividuallyAsync(Relationship item, string sourceKey)
+ {
+ try
+ {
+ var persisted = await _relationshipRepository.UpsertAsync(item, cancellationToken).ConfigureAwait(false);
+ RecordPersistedRelationship(sourceKey, persisted);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { throw; }
- catch (MemoryIngestionException) { throw; } // consistent with the other item kinds (#101 review)
+ catch (MemoryIngestionException) { throw; }
catch (Exception ex)
{
- _logger.LogError(ex,
- "Error persisting relationship '{Src}->{Tgt}'.",
- extracted.SourceEntity, extracted.TargetEntity);
+ _logger.LogError(ex, "Error persisting relationship '{SourceKey}'.", sourceKey);
RecordFailureAndMaybeThrow(outcomes, failFast, MemoryItemKind.Relationship, IngestionStage.Persistence,
- MemoryErrorCodes.RelationshipPersistenceFailed, relSourceKey, null, ex,
- $"Ingestion failed fast: persistence failed for relationship '{relSourceKey}'.");
+ MemoryErrorCodes.RelationshipPersistenceFailed, sourceKey, null, ex,
+ $"Ingestion failed fast: persistence failed for relationship '{sourceKey}'.");
+ }
+ }
+
+ Dictionary? batchedRelationshipsById = null;
+ var canBatchRelationships = _options.EnableBatchMemoryUpserts && !failFast &&
+ relationshipInputs.Count > 1 &&
+ relationshipInputs.Select(input => input.Item.RelationshipId).Distinct(StringComparer.Ordinal).Count() == relationshipInputs.Count &&
+ _relationshipRepository is IBatchMemoryRepository;
+ if (canBatchRelationships)
+ {
+ try
+ {
+ var persisted = await ((IBatchMemoryRepository)_relationshipRepository)
+ .UpsertBatchAsync(relationshipInputs.Select(input => input.Item).ToList(), cancellationToken)
+ .ConfigureAwait(false);
+ batchedRelationshipsById = persisted.ToDictionary(
+ relationship => relationship.RelationshipId, StringComparer.Ordinal);
+ if (relationshipInputs.Any(input => !batchedRelationshipsById.ContainsKey(input.Item.RelationshipId)))
+ throw new InvalidOperationException("The relationship batch result omitted one or more input identifiers.");
+ }
+ catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { throw; }
+ catch (Exception ex)
+ {
+ _logger.LogWarning(ex,
+ "Atomic relationship batch failed; replaying {Count} relationships through the item path.",
+ relationshipInputs.Count);
+ batchedRelationshipsById = null;
}
}
+ if (batchedRelationshipsById is not null)
+ {
+ foreach (var input in relationshipInputs)
+ RecordPersistedRelationship(
+ input.SourceKey, batchedRelationshipsById[input.Item.RelationshipId]);
+ }
+ else
+ {
+ foreach (var input in relationshipInputs)
+ await PersistRelationshipIndividuallyAsync(input.Item, input.SourceKey).ConfigureAwait(false);
+ }
return new PersistenceResult
{
EntityCount = persistedEntityMap.Count,
@@ -442,12 +649,117 @@ await _preferenceRepository.CreateExtractedFromRelationshipAsync(
};
}
+ private async Task PrepareEmbeddingsIndividuallyAsync(
+ ExtractionStageResult extraction,
+ CancellationToken cancellationToken)
+ {
+ var failFast = _options.FailureMode == IngestionFailureMode.FailFast;
+ var outcomes = new List();
+ var entities = new Dictionary(StringComparer.OrdinalIgnoreCase);
+ var facts = new List(extraction.FilteredFacts.Count);
+ var preferences = new List(extraction.FilteredPreferences.Count);
+
+ foreach (var (name, entity) in extraction.ResolvedEntityMap)
+ {
+ if (entity.Embedding is not null)
+ {
+ entities[name] = entity;
+ continue;
+ }
+
+ try
+ {
+ var embedding = await _embeddingOrchestrator.EmbedEntityAsync(
+ entity.Name, cancellationToken).ConfigureAwait(false);
+ entities[name] = entity with { Embedding = embedding };
+ }
+ catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { throw; }
+ catch (Exception ex)
+ {
+ _logger.LogError(ex, "Error generating embedding for entity '{Name}'.", name);
+ RecordFailureAndMaybeThrow(outcomes, failFast, MemoryItemKind.Entity, IngestionStage.Embedding,
+ MemoryErrorCodes.EmbeddingGenerationFailed, name, null, ex,
+ $"Ingestion failed fast: embedding generation failed for entity '{name}'.");
+ }
+ }
+
+ foreach (var extracted in extraction.FilteredFacts)
+ {
+ var sourceKey = $"{extracted.Subject} {extracted.Predicate} {extracted.Object}";
+ try
+ {
+ var embedding = await _embeddingOrchestrator.EmbedFactAsync(
+ extracted.Subject, extracted.Predicate, extracted.Object, cancellationToken).ConfigureAwait(false);
+ facts.Add(new PreparedFact(extracted, embedding));
+ }
+ catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { throw; }
+ catch (Exception ex)
+ {
+ _logger.LogError(ex, "Error generating embedding for fact '{Key}'.", sourceKey);
+ RecordFailureAndMaybeThrow(outcomes, failFast, MemoryItemKind.Fact, IngestionStage.Embedding,
+ MemoryErrorCodes.EmbeddingGenerationFailed, sourceKey, null, ex,
+ $"Ingestion failed fast: embedding generation failed for fact '{sourceKey}'.");
+ }
+ }
+
+ foreach (var extracted in extraction.FilteredPreferences)
+ {
+ try
+ {
+ var embedding = await _embeddingOrchestrator.EmbedPreferenceAsync(
+ extracted.PreferenceText, cancellationToken).ConfigureAwait(false);
+ preferences.Add(new PreparedPreference(extracted, embedding));
+ }
+ catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { throw; }
+ catch (Exception ex)
+ {
+ _logger.LogError(ex, "Error generating embedding for preference '{Text}'.", extracted.PreferenceText);
+ RecordFailureAndMaybeThrow(outcomes, failFast, MemoryItemKind.Preference, IngestionStage.Embedding,
+ MemoryErrorCodes.EmbeddingGenerationFailed, extracted.PreferenceText, null, ex,
+ "Ingestion failed fast: embedding generation failed for a preference.");
+ }
+ }
+
+ return new PreparedEmbeddings(entities, facts, preferences, outcomes);
+ }
+
+ private sealed class FactTripleComparer : IEqualityComparer<(string Subject, string Predicate, string Object)>
+ {
+ public static FactTripleComparer OrdinalIgnoreCase { get; } = new();
+
+ public bool Equals(
+ (string Subject, string Predicate, string Object) left,
+ (string Subject, string Predicate, string Object) right) =>
+ StringComparer.OrdinalIgnoreCase.Equals(left.Subject, right.Subject) &&
+ StringComparer.OrdinalIgnoreCase.Equals(left.Predicate, right.Predicate) &&
+ StringComparer.OrdinalIgnoreCase.Equals(left.Object, right.Object);
+
+ public int GetHashCode((string Subject, string Predicate, string Object) value) =>
+ HashCode.Combine(
+ StringComparer.OrdinalIgnoreCase.GetHashCode(value.Subject),
+ StringComparer.OrdinalIgnoreCase.GetHashCode(value.Predicate),
+ StringComparer.OrdinalIgnoreCase.GetHashCode(value.Object));
+ }
+ private sealed record PreparedEmbeddings(
+ IReadOnlyDictionary Entities,
+ IReadOnlyList Facts,
+ IReadOnlyList Preferences,
+ IReadOnlyList Outcomes);
+
+ private sealed record PreparedFact(ExtractedFact Item, float[] Embedding);
+
+ private sealed record PreparedPreference(ExtractedPreference Item, float[] Embedding);
+
///
/// Trust is monotonic (#92 Phase 3): re-touching an already-persisted entity must never silently lower
/// its trust level below whatever it already had.
///
private static MemoryTrustLevel MaxTrustLevel(MemoryTrustLevel a, MemoryTrustLevel b) => a > b ? a : b;
+ private static IEnumerable ExplicitProvenanceMessageIds(
+ object repository, IReadOnlyList sourceMessageIds) =>
+ repository is IUpsertPersistsProvenance ? Array.Empty() : sourceMessageIds;
+
/// Appends a outcome (#101).
private static void RecordSuccess(
List outcomes, MemoryItemKind kind, string? sourceKey, string? persistedId) =>
diff --git a/src/AgentMemory.Core/Memory/MemoryPredicateSeedVocabulary.cs b/src/AgentMemory.Core/Memory/MemoryPredicateSeedVocabulary.cs
new file mode 100644
index 00000000..9abc27f3
--- /dev/null
+++ b/src/AgentMemory.Core/Memory/MemoryPredicateSeedVocabulary.cs
@@ -0,0 +1,66 @@
+namespace AgentMemory.Core.Memory;
+
+///
+/// The starting set of relation names offered to extraction.
+///
+///
+///
+/// Curated once rather than mined per run, for two reasons. A vocabulary that accumulated during
+/// a run would make each call's prompt depend on which concurrent extraction finished first, so the
+/// same input could produce different prompts — the precise property that made an earlier
+/// Structured score sequence unattributable. And a reviewed list can be checked by a human for the
+/// one mistake that matters here: silently omitting one side of an opposing pair.
+///
+///
+/// Drawn from relations actually observed in extracted graphs. Deliberately small: it is injected
+/// into every extraction prompt, and a list approaching the 421-predicates-per-700-facts figure that
+/// motivated it would consume the budget it exists to improve.
+///
+///
+/// Opposing relations are both present by design. bought/sold and
+/// likes/dislikes are one embedding threshold apart and mean opposite things; offering
+/// only one would invite the extractor to collapse them and invert facts.
+///
+///
+public static class MemoryPredicateSeedVocabulary
+{
+ ///
+ /// Derived from the single relation table, never authored separately.
+ ///
+ ///
+ /// This list was previously maintained by hand alongside the query lexicon, and the two drifted:
+ /// **13 relations became resolvable at query time that the extractor was never offered**, so the
+ /// graph could not contain them however well retrieval worked. `assembled` was one of them, which
+ /// is why assembly was filed under `completed` and the furniture question could not be answered
+ /// from the graph. One relation known to two layers must have one definition.
+ ///
+ /// Only the canonical keys cross over. Surface forms stay read-side: they never enter an
+ /// extraction prompt, where they would cost tokens on every call and invite the extractor to
+ /// choose inconsistently between buy, buys and purchased - the opposite of
+ /// the consolidation this vocabulary exists to produce.
+ ///
+ ///
+ private static readonly string[] Seed =
+ [.. MemoryRelationSeedTable.Table.Keys
+ .Where(relation => !MemoryRelationSeedTable.RetiredRelations.Contains(relation))
+ .OrderBy(relation => relation, StringComparer.Ordinal)];
+
+ ///
+ /// Content hash of this vocabulary, for recording which table produced a given extracted graph.
+ ///
+ ///
+ /// This list is injected into every extraction prompt, so changing it changes what is stored. Two
+ /// graphs built under different vocabularies are not comparable, and without this the artifact
+ /// would not say which one produced it.
+ ///
+ public static string Fingerprint { get; } = MemoryVocabularyFingerprint.Of(Seed);
+
+ /// A vocabulary pre-populated with the curated seed relations.
+ public static MemoryPredicateVocabulary Create()
+ {
+ var vocabulary = new MemoryPredicateVocabulary();
+ foreach (var predicate in Seed)
+ vocabulary.Admit(predicate);
+ return vocabulary;
+ }
+}
diff --git a/src/AgentMemory.Core/Memory/MemoryPredicateVocabulary.cs b/src/AgentMemory.Core/Memory/MemoryPredicateVocabulary.cs
new file mode 100644
index 00000000..b55acd6e
--- /dev/null
+++ b/src/AgentMemory.Core/Memory/MemoryPredicateVocabulary.cs
@@ -0,0 +1,81 @@
+using System.Collections.Concurrent;
+
+namespace AgentMemory.Core.Memory;
+
+///
+/// The set of relation names extraction has already established, so it reuses them instead of
+/// inventing a new phrasing per sentence.
+///
+///
+///
+/// Why this exists. collapses spelling
+/// (were_born_in → were born in). It cannot collapse phrasing. Measured on a
+/// live extracted graph, one real-world event — a birth — arrived as was born,
+/// was born in, were born in, had and welcomed, so retrieving any single
+/// relation gathered at most three of five and counting questions were unanswerable. That graph held
+/// 700 facts under 421 distinct predicates: a vocabulary almost as large as the data, which is what
+/// happens when nothing tells the extractor which relations already exist.
+///
+///
+/// Deterministic only. Admission matches on the canonical form and nothing else. No embedding
+/// or similarity folding: bought/sold and likes/dislikes sit one
+/// threshold apart and mean opposite things, and merging them would silently invert stored facts in
+/// a way ordinary tests would not catch. Narrowing the vocabulary is the extractor's job, guided by
+/// what it is shown; it is never this type's job to guess.
+///
+///
+/// First spelling wins, so an established relation never drifts between runs and queries can
+/// rely on its name. Growth is capped because the vocabulary is injected into the extraction prompt,
+/// and an uncapped one would trend toward one predicate per fact and consume the budget it exists to
+/// improve. A predicate beyond the cap is still returned and usable — it is simply not established.
+///
+///
+public sealed class MemoryPredicateVocabulary
+{
+ /// Keyed on the canonical form; the value is the established surface spelling.
+ private readonly ConcurrentDictionary _established;
+ private readonly int _maximumSize;
+
+ ///
+ /// Cap on established relations. The vocabulary is injected into the extraction prompt, so an
+ /// uncapped one would trend toward one predicate per fact and consume the budget it improves.
+ ///
+ public MemoryPredicateVocabulary(int maximumSize = 256)
+ {
+ ArgumentOutOfRangeException.ThrowIfNegativeOrZero(maximumSize);
+ _maximumSize = maximumSize;
+ _established = new ConcurrentDictionary(StringComparer.Ordinal);
+ }
+
+ /// Number of established relations.
+ public int Count => _established.Count;
+
+ ///
+ /// Returns the established spelling for , establishing it if the
+ /// relation is new and there is room.
+ ///
+ ///
+ /// The established predicate when one exists or is created; otherwise
+ /// unchanged, so a full vocabulary degrades to today's behaviour rather than losing a fact.
+ ///
+ public string Admit(string? predicate)
+ {
+ var canonical = MemoryTripleCanonicalizer.Canonical(predicate);
+ if (canonical.Length == 0)
+ return string.Empty;
+
+ if (_established.TryGetValue(canonical, out var established))
+ return established;
+
+ // Racing writers may briefly exceed the cap; the bound is a budget guard, not an invariant,
+ // and rejecting a predicate is never worth a lock on the extraction hot path.
+ if (_established.Count >= _maximumSize)
+ return predicate!;
+
+ return _established.GetOrAdd(canonical, predicate!.Trim());
+ }
+
+ /// The established relations, ordered so an injected prompt is reproducible.
+ public IReadOnlyList Snapshot() =>
+ _established.Values.OrderBy(value => value, StringComparer.Ordinal).ToArray();
+}
diff --git a/src/AgentMemory.Core/Memory/MemoryRelationLexicon.cs b/src/AgentMemory.Core/Memory/MemoryRelationLexicon.cs
new file mode 100644
index 00000000..7cae2069
--- /dev/null
+++ b/src/AgentMemory.Core/Memory/MemoryRelationLexicon.cs
@@ -0,0 +1,308 @@
+using System.Collections.Frozen;
+
+namespace AgentMemory.Core.Memory;
+
+///
+/// Resolves the verbs a question uses onto the canonical predicates a graph stores.
+///
+///
+///
+/// J2.1. Predicate expansion makes one relation complete, but it can only expand predicates
+/// that similarity already surfaced in the top-K. A question naming several relations
+/// ("did I buy, assemble, sell, or fix") therefore reaches only whichever of them retrieval happened
+/// to nominate. This supplies the relations from the question instead.
+///
+///
+/// Read-side only, and that asymmetry is the safety argument. Clustering predicates over stored
+/// facts was rejected because merging bought onto sold corrupts meaning irreversibly. A
+/// wrong entry here costs precision on one query and can never alter a stored fact. Fuzzy is
+/// unacceptable at write time and tolerable at read time.
+///
+///
+/// The authored direction is canonical → surface forms; the lookup index is derived, so
+/// the two can never disagree. Irregular forms are listed explicitly rather than inferred, and
+/// suffix stripping is only a fallback for forms the table does not list.
+///
+///
+internal sealed class MemoryRelationLexicon
+{
+ /// Longest multi-word surface form, so the harvester knows its window.
+ private const int MaximumPhraseWords = 4;
+
+ ///
+ /// Words that mark a sentence as being about the owner of the memory rather than an instruction.
+ ///
+ ///
+ /// Deliberately narrow. Second person is excluded because "can you tell me…" is the commonest
+ /// assistant-request opener there is, and admitting it would reopen the exact hole this closes.
+ ///
+ private static readonly FrozenSet FirstPersonMarkers =
+ new[] { "i", "me", "my", "mine", "myself", "we", "us", "our", "ours" }
+ .ToFrozenSet(StringComparer.Ordinal);
+
+ ///
+ /// Words that mark a sentence as a question rather than an instruction.
+ ///
+ ///
+ /// Needed because a memory question need not mention its owner: "How many babies were born to
+ /// friends and family" is about the user's history and contains no first person at all. It is
+ /// also the one question predicate expansion is measured to flip, so a gate that dropped it would
+ /// have traded away the only proven win in this track.
+ ///
+ private static readonly FrozenSet Interrogatives =
+ new[]
+ {
+ "what", "when", "where", "who", "whom", "whose", "which", "how", "why",
+ "did", "do", "does", "was", "were", "is", "are", "has", "have", "had", "can", "could"
+ }.ToFrozenSet(StringComparer.Ordinal);
+
+ private readonly FrozenDictionary _surfaceToCanonical;
+ private readonly FrozenDictionary _canonicalToStoredForms;
+ private readonly FrozenSet _canonical;
+ private readonly FrozenSet _queryStopForms;
+
+ private MemoryRelationLexicon(
+ FrozenDictionary surfaceToCanonical,
+ FrozenDictionary canonicalToStoredForms,
+ FrozenSet canonical,
+ FrozenSet queryStopForms,
+ IReadOnlyList ambiguousSurfaceForms)
+ {
+ _queryStopForms = queryStopForms;
+ _surfaceToCanonical = surfaceToCanonical;
+ _canonicalToStoredForms = canonicalToStoredForms;
+ _canonical = canonical;
+ AmbiguousSurfaceForms = ambiguousSurfaceForms;
+ }
+
+ internal static MemoryRelationLexicon Default { get; } = Build(MemoryRelationSeedTable.Table);
+
+ ///
+ /// Surface forms that were claimed by more than one canonical relation and therefore dropped.
+ ///
+ ///
+ /// Exposed rather than silently discarded: a duplicate in a hand-authored table is an authoring
+ /// mistake, and a test asserts this is empty for the shipped table. Dropping keeps the runtime
+ /// safe; exposing keeps the mistake visible.
+ ///
+ internal IReadOnlyList AmbiguousSurfaceForms { get; }
+
+ internal IReadOnlyCollection CanonicalRelations => _canonical;
+
+ /// Resolves one surface form, or null when the table does not know it.
+ ///
+ /// A null is load-bearing: it is what lets the caller fall back to today's top-K-derived
+ /// predicates, which is what makes this incapable of being worse than current behaviour.
+ ///
+ internal string? Resolve(string? surfaceForm)
+ {
+ var normalized = MemoryTripleCanonicalizer.Canonical(surfaceForm);
+ if (normalized.Length == 0 || _queryStopForms.Contains(normalized))
+ return null;
+ if (_surfaceToCanonical.TryGetValue(normalized, out var canonical))
+ return canonical;
+
+ // Fallback only. Listed irregulars have already matched above.
+ var stemmed = Stem(normalized);
+ if (stemmed is null)
+ return null;
+
+ // The stem is re-checked against the stop list, not just the raw form. Without this a
+ // suppressed bare verb is handed straight back through its own inflections - "works" and
+ // "working" both stem to a suppressed "work" - which silently defeats every suppression
+ // decision in the vocabulary rather than only the one being read.
+ if (_queryStopForms.Contains(stemmed))
+ return null;
+
+ return _surfaceToCanonical.TryGetValue(stemmed, out var stemMatch) ? stemMatch : null;
+ }
+
+
+ ///
+ /// Whether a stored predicate key is a form the vocabulary knows.
+ ///
+ ///
+ /// Deliberately not . That method answers a query-side question and rejects
+ /// stop forms on purpose, so a question mentioning "is" does not expand into the whole graph.
+ /// Those same forms are perfectly legitimate stored predicates — the measured graph holds
+ /// has with 1,583 facts and is with 1,118 — and scoring storage coverage with the
+ /// query-side method reports them as unknown vocabulary when they are nothing of the kind.
+ ///
+ /// Written for J1.5 gate 1 after exactly that mistake made the gate read 81.5% when the genuine
+ /// gap was far smaller. The two questions are different and need different methods; the fix is
+ /// not to make the stop forms resolvable, which would defeat the suppression they exist for.
+ ///
+ ///
+ internal bool IsKnownStoredForm(string? predicateKey)
+ {
+ var normalized = MemoryTripleCanonicalizer.Canonical(predicateKey);
+ if (normalized.Length == 0)
+ return false;
+ if (_surfaceToCanonical.ContainsKey(normalized) || _canonical.Contains(normalized))
+ return true;
+
+ var stemmed = Stem(normalized);
+ return stemmed is not null &&
+ (_surfaceToCanonical.ContainsKey(stemmed) || _canonical.Contains(stemmed));
+ }
+
+ ///
+ /// Every form of a relation that could appear as a stored predicate_key, including itself.
+ ///
+ ///
+ /// The write-side canonicalizer folds case and separators but deliberately never morphology, so
+ /// one relation is stored under several keys: the measured graph holds planned with 839
+ /// facts and plans with 14 as separate keys. Expanding on the canonical name alone would
+ /// silently miss the smaller bucket - precisely the completeness failure expansion exists to
+ /// prevent. An unknown relation yields just itself, so callers need no special case.
+ ///
+ internal IReadOnlyList StoredFormsOf(string? relation)
+ {
+ var canonical = MemoryTripleCanonicalizer.Canonical(relation);
+ if (canonical.Length == 0)
+ return [];
+ return _canonicalToStoredForms.TryGetValue(canonical, out var forms)
+ ? forms
+ : [canonical];
+ }
+
+ ///
+ /// Harvests every relation a question names, distinct and in order of first appearance.
+ ///
+ internal IReadOnlyList ResolveQuestion(string? question)
+ {
+ if (string.IsNullOrWhiteSpace(question))
+ return [];
+
+ // Tokenized on non-letters rather than through the predicate canonicalizer, which folds
+ // separators but leaves sentence punctuation intact: "buy," would then never match "buy".
+ var words = new string(question
+ .Select(character => char.IsLetter(character) ? char.ToLowerInvariant(character) : ' ')
+ .ToArray())
+ .Split(' ', StringSplitOptions.RemoveEmptyEntries);
+ if (words.Length == 0)
+ return [];
+
+ // Reject the imperative-instruction shape. Two independent reviews measured the same bound:
+ // per-form suppression cannot reach it, because `create`, `build`, `save`, `read` and `start`
+ // are all legitimate relations that cannot be deleted, yet "Create a summary" is not a
+ // question about anyone's past. It is a property of the sentence, not of any single verb,
+ // which is why no amount of per-form editing reached it.
+ //
+ // The discriminator is mood, NOT first person. A first-person test was written first and
+ // measured against the benchmark before being trusted: it would have blocked "How many babies
+ // were born to friends and family members" - a question with no first-person marker at all,
+ // and the single question that predicate expansion is proven to flip. An imperative is a bare
+ // verb with no subject, so the test is that the sentence OPENS on a relation verb while
+ // carrying neither an interrogative nor a first-person marker.
+ if (Resolve(words[0]) is not null &&
+ !words.Any(FirstPersonMarkers.Contains) &&
+ !words.Any(Interrogatives.Contains))
+ {
+ return [];
+ }
+
+ var resolved = new List();
+ var seen = new HashSet(StringComparer.Ordinal);
+
+ for (var index = 0; index < words.Length; index++)
+ {
+ // Longest phrase first: "is interested in" must win over "is", or a multi-word relation
+ // can never be reached.
+ for (var length = Math.Min(MaximumPhraseWords, words.Length - index); length >= 1; length--)
+ {
+ var phrase = string.Join(' ', words, index, length);
+ if (Resolve(phrase) is not { } canonical)
+ continue;
+ if (seen.Add(canonical))
+ resolved.Add(canonical);
+ index += length - 1;
+ break;
+ }
+ }
+
+ return resolved;
+ }
+
+ private static MemoryRelationLexicon Build(
+ IReadOnlyDictionary table)
+ {
+ var surfaceToCanonical = new Dictionary(StringComparer.Ordinal);
+ var ambiguous = new SortedSet(StringComparer.Ordinal);
+
+ foreach (var (canonicalRaw, surfaceForms) in table)
+ {
+ var canonical = MemoryTripleCanonicalizer.Canonical(canonicalRaw);
+ // A relation always resolves to itself, so the table never has to repeat its own name.
+ foreach (var surface in surfaceForms.Append(canonical))
+ {
+ var key = MemoryTripleCanonicalizer.Canonical(surface);
+ if (key.Length == 0)
+ continue;
+ if (surfaceToCanonical.TryGetValue(key, out var existing) &&
+ !string.Equals(existing, canonical, StringComparison.Ordinal))
+ {
+ ambiguous.Add(key);
+ continue;
+ }
+
+ surfaceToCanonical[key] = canonical;
+ }
+ }
+
+ // A form claimed by two relations is removed outright rather than awarded to whichever was
+ // authored first, which would make the result depend on table order.
+ foreach (var key in ambiguous)
+ surfaceToCanonical.Remove(key);
+
+ // The inverse index is DERIVED from the surviving forward map, never authored separately, so
+ // the two directions cannot disagree and a dropped ambiguous form stays dropped in both.
+ var canonicalToStoredForms = surfaceToCanonical
+ .GroupBy(pair => pair.Value, StringComparer.Ordinal)
+ .ToFrozenDictionary(
+ group => group.Key,
+ group => group.Select(pair => pair.Key)
+ .OrderBy(form => form, StringComparer.Ordinal)
+ .ToArray(),
+ StringComparer.Ordinal);
+
+ return new MemoryRelationLexicon(
+ surfaceToCanonical.ToFrozenDictionary(StringComparer.Ordinal),
+ canonicalToStoredForms,
+ table.Keys.Select(MemoryTripleCanonicalizer.Canonical)
+ .ToFrozenSet(StringComparer.Ordinal),
+ MemoryRelationSeedTable.QueryStopForms
+ .Select(MemoryTripleCanonicalizer.Canonical)
+ .ToFrozenSet(StringComparer.Ordinal),
+ [.. ambiguous]);
+ }
+
+ ///
+ /// Light, deterministic suffix stripping for forms the table does not list.
+ ///
+ ///
+ /// The length guards are not cosmetic. is is the single most common predicate in the
+ /// measured graph - 1,213 facts, 26% of all of them - and naive -s stripping would reduce
+ /// it to i and lose a quarter of the graph.
+ ///
+ private static string? Stem(string value)
+ {
+ if (value.Length <= 4)
+ return null;
+
+ if (value.EndsWith("ing", StringComparison.Ordinal) && value.Length > 6)
+ return value[..^3];
+ if (value.EndsWith("ed", StringComparison.Ordinal) && value.Length > 5)
+ return value[..^2];
+ if (value.EndsWith("es", StringComparison.Ordinal) && value.Length > 5)
+ return value[..^2];
+ if (value.EndsWith('s') &&
+ !value.EndsWith("ss", StringComparison.Ordinal) &&
+ value.Length > 4)
+ {
+ return value[..^1];
+ }
+
+ return null;
+ }
+}
diff --git a/src/AgentMemory.Core/Memory/MemoryRelationSeedTable.cs b/src/AgentMemory.Core/Memory/MemoryRelationSeedTable.cs
new file mode 100644
index 00000000..a5f2204f
--- /dev/null
+++ b/src/AgentMemory.Core/Memory/MemoryRelationSeedTable.cs
@@ -0,0 +1,86 @@
+namespace AgentMemory.Core.Memory;
+
+///
+/// The reviewed canonical → surface forms table behind .
+///
+///
+///
+/// Authored in one direction only. The lookup index is derived at load, so the extraction vocabulary
+/// and the query lexicon cannot drift apart.
+///
+///
+/// Canonical names are written in stored predicate_key form - lowercase, separators folded to
+/// single spaces - because resolution that produced keys the graph cannot match would be worthless.
+///
+///
+/// Grounded in measurement, not intuition. The canonical set covers the predicates actually
+/// observed in an extracted graph (4,659 facts over 10 owners), including the frequent forms the
+/// extractor invented outside the offered vocabulary - wants, is interested in,
+/// uses, asked about, requested, considered. Inflectional variants are the
+/// bulk of that tail (plans beside planned, was beside is), and they are
+/// resolved here rather than by enlarging the write-side vocabulary.
+///
+///
+/// Deliberately excluded: genuinely ambiguous forms. got could be bought or
+/// received; a form claimed by two relations is dropped at load and reported, so an authoring
+/// mistake is visible rather than silently resolved in table order.
+///
+///
+/// The data itself lives in relation-vocabulary.json, embedded in this assembly. See the
+/// README beside it for where each relation came from, under which licence, and what filtering was
+/// applied — provenance ships with the artifact because it draws on schema.org and Wikidata.
+///
+///
+internal static class MemoryRelationSeedTable
+{
+ ///
+ /// Relations still resolvable at query time but no longer offered to extraction.
+ ///
+ ///
+ /// Retiring a relation must stop new writes without making facts already stored under it
+ /// unreachable - a graph does not rewrite itself when the vocabulary changes.
+ ///
+ /// Two mechanisms, and knowing which is in use matters. Demotion removes the key and
+ /// keeps its name as a surface form of the relation it merged into, so the old facts are reached
+ /// through the survivor; this is what finished and welcomed did. Flagging
+ /// keeps the key but excludes it from the extraction vocabulary, which is what the filter over
+ /// this set does. Demotion is preferred where a survivor exists, because it leaves one name for
+ /// one meaning; flagging is for a relation being withdrawn with nothing to merge into.
+ ///
+ ///
+ internal static IReadOnlySet RetiredRelations { get; } =
+ RelationVocabularyDocument.Load().Retired.ToHashSet(StringComparer.Ordinal);
+
+ ///
+ /// Content hash of the query lexicon, recorded in run reports.
+ ///
+ ///
+ /// Surface forms never enter a prompt, but they change what a question resolves to and therefore
+ /// what is retrieved, so a run measured under a different table is not comparable to one measured
+ /// under this one.
+ ///
+ internal static string Fingerprint => MemoryVocabularyFingerprint.OfTable(Table);
+
+ ///
+ /// The reviewed relation table, loaded from the embedded JSON artifact.
+ ///
+ ///
+ /// Authored as JSON rather than C# so it is diffable in review, can be regenerated by the unifier,
+ /// and can carry the per-relation source and licence provenance that a C# array cannot express -
+ /// this vocabulary draws on schema.org and Wikidata and ships inside a package.
+ ///
+ internal static IReadOnlyDictionary Table { get; } =
+ RelationVocabularyDocument.Load().Canonical.ToDictionary(
+ entry => entry.Key,
+ entry => entry.Value.SurfaceForms.Concat(entry.Value.StoredOnly)
+ .Distinct(StringComparer.Ordinal).ToArray(),
+ StringComparer.Ordinal);
+
+ ///
+ /// Forms that expansion may fetch but a question must never resolve to.
+ ///
+ internal static IReadOnlySet QueryStopForms { get; } =
+ RelationVocabularyDocument.Load().Canonical
+ .SelectMany(entry => entry.Value.StoredOnly)
+ .ToHashSet(StringComparer.Ordinal);
+}
diff --git a/src/AgentMemory.Core/Memory/MemoryTripleCanonicalizer.cs b/src/AgentMemory.Core/Memory/MemoryTripleCanonicalizer.cs
new file mode 100644
index 00000000..765b726e
--- /dev/null
+++ b/src/AgentMemory.Core/Memory/MemoryTripleCanonicalizer.cs
@@ -0,0 +1,111 @@
+using System.Text;
+
+namespace AgentMemory.Core.Memory;
+
+///
+/// Canonical forms for the fact triple's subject, predicate and object.
+///
+///
+///
+/// Facts already deduplicate at write time — the repository MERGEs on
+/// {subject, predicate, object, owner_key} — but that key uses the raw strings, so trivial
+/// surface differences defeat it. Measured on a real extracted graph:
+/// "Ava and Lily"/were_born_in/"April" and "Ava and Lily"/were born in/"April" became
+/// two nodes, as did "User"/is planning and "user"/is planning. That graph held 575
+/// facts across 407 distinct predicates — 1.41 facts per predicate — which also makes any
+/// query keyed on a relation impossible.
+///
+///
+/// Canonicalization is therefore not a new deduplication mechanism; it repairs the one that exists.
+///
+///
+/// Deterministic only, by design. No embedding or fuzzy similarity is used to fold one
+/// predicate into another. Relations such as bought and sold are semantically adjacent
+/// and opposite; merging them would silently corrupt meaning in a way ordinary tests would not
+/// catch. Only differences that cannot change meaning are collapsed: surrounding whitespace, letter
+/// case, and word separators.
+///
+///
+/// Compute once, in C#, at write time. Never recompute this in Cypher: .NET's
+/// and Cypher's toLower() disagree on U+0130 (Turkish
+/// dotted capital I), so a value canonicalized on one side and matched on the other would produce two
+/// keys for one relation — reintroducing the exact fragmentation this removes.
+///
+///
+public static class MemoryTripleCanonicalizer
+{
+ ///
+ /// Returns the canonical form used for identity: trimmed, lower-cased invariantly, with word
+ /// separators unified and runs of whitespace collapsed.
+ ///
+ ///
+ /// The original text is always retained separately; this value is for matching, never display.
+ ///
+ ///
+ /// Canonical form for a value — a subject or object. Trims, lower-cases invariantly and
+ /// collapses whitespace, but never rewrites punctuation.
+ ///
+ ///
+ /// Separator folding is correct for predicates, where was_born and was born are one
+ /// relation under two naming conventions. It is corrupting for values: -5 and
+ /// 5 would fold into one fact, silently merging a quantity with its negation. Values carry
+ /// meaning in their punctuation; identifiers do not.
+ ///
+ public static string CanonicalValue(string? value)
+ {
+ if (string.IsNullOrWhiteSpace(value))
+ return string.Empty;
+
+ var builder = new StringBuilder(value.Length);
+ var pendingSpace = false;
+ foreach (var character in value.ToLowerInvariant())
+ {
+ if (char.IsWhiteSpace(character))
+ {
+ pendingSpace = builder.Length > 0;
+ continue;
+ }
+
+ if (pendingSpace)
+ {
+ builder.Append(' ');
+ pendingSpace = false;
+ }
+
+ builder.Append(character);
+ }
+
+ return builder.ToString();
+ }
+
+ /// Canonical form for a predicate, folding word separators.
+ public static string Canonical(string? value)
+ {
+ if (string.IsNullOrWhiteSpace(value))
+ return string.Empty;
+
+ var lowered = value.ToLowerInvariant();
+ var builder = new StringBuilder(lowered.Length);
+ var pendingSeparator = false;
+ foreach (var character in lowered)
+ {
+ // '_' and '-' are word separators in extracted predicates ("was_born" / "was born"),
+ // never meaningful punctuation, so they normalize to the same break as whitespace.
+ if (char.IsWhiteSpace(character) || character is '_' or '-')
+ {
+ pendingSeparator = builder.Length > 0;
+ continue;
+ }
+
+ if (pendingSeparator)
+ {
+ builder.Append(' ');
+ pendingSeparator = false;
+ }
+
+ builder.Append(character);
+ }
+
+ return builder.ToString();
+ }
+}
diff --git a/src/AgentMemory.Core/Memory/MemoryVocabularyFingerprint.cs b/src/AgentMemory.Core/Memory/MemoryVocabularyFingerprint.cs
new file mode 100644
index 00000000..a16933ae
--- /dev/null
+++ b/src/AgentMemory.Core/Memory/MemoryVocabularyFingerprint.cs
@@ -0,0 +1,66 @@
+using System.Security.Cryptography;
+using System.Text;
+
+namespace AgentMemory.Core.Memory;
+
+///
+/// A stable content hash of a relation vocabulary or lexicon.
+///
+///
+///
+/// The extraction vocabulary decides what is stored and the query lexicon decides what is
+/// retrieved, so two runs made under different tables are not comparable. Recording the hash is
+/// what lets a report say which table produced a given graph, and it is the same class of defect as
+/// retrieval flags that were absent from a run fingerprint: a setting that changes the outcome but
+/// leaves no trace in the artifact.
+///
+///
+/// Order-independent by construction, because a vocabulary is a set: reordering entries is an
+/// authoring change with no meaning, and if it altered the hash then every cosmetic edit would
+/// invalidate comparisons for nothing.
+///
+///
+internal static class MemoryVocabularyFingerprint
+{
+ /// Hashes a flat set of relation names.
+ internal static string Of(IEnumerable entries)
+ {
+ ArgumentNullException.ThrowIfNull(entries);
+ var normalized = entries
+ .Select(MemoryTripleCanonicalizer.Canonical)
+ .Where(entry => entry.Length > 0)
+ .Distinct(StringComparer.Ordinal)
+ .OrderBy(entry => entry, StringComparer.Ordinal);
+ return Hash(string.Join('\n', normalized));
+ }
+
+ /// Hashes a canonical-to-surface-forms table.
+ ///
+ /// Each relation is hashed together with its own forms, so moving a surface form from one relation
+ /// to another changes the hash even though the entry count is unchanged. A hash over counts, or
+ /// over the two sides separately, would miss exactly that.
+ ///
+ internal static string OfTable(IReadOnlyDictionary table)
+ {
+ ArgumentNullException.ThrowIfNull(table);
+ var lines = table
+ .Select(entry =>
+ {
+ var canonical = MemoryTripleCanonicalizer.Canonical(entry.Key);
+ var forms = (entry.Value ?? [])
+ .Select(MemoryTripleCanonicalizer.Canonical)
+ .Where(form => form.Length > 0)
+ .Distinct(StringComparer.Ordinal)
+ .OrderBy(form => form, StringComparer.Ordinal);
+ return $"{canonical}>{string.Join(',', forms)}";
+ })
+ .OrderBy(line => line, StringComparer.Ordinal);
+ return Hash(string.Join('\n', lines));
+ }
+
+ // ToHexString().ToLowerInvariant() rather than ToHexStringLower(): Core multi-targets down to
+ // net8.0, where the lowercase overload does not exist.
+ private static string Hash(string content) =>
+ Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(content)))
+ .ToLowerInvariant();
+}
diff --git a/src/AgentMemory.Core/Memory/README.md b/src/AgentMemory.Core/Memory/README.md
new file mode 100644
index 00000000..98f3e28b
--- /dev/null
+++ b/src/AgentMemory.Core/Memory/README.md
@@ -0,0 +1,138 @@
+# Relation vocabulary
+
+`relation-vocabulary.json` is the single reviewed source for **which relations this library knows**.
+It is embedded into `AgentMemory.Core` and parsed once at first use.
+
+## Why it exists
+
+Extraction previously invented a relation name per phrasing. One measured graph held **700 facts under
+421 distinct predicates**, with a single birth arriving as `was born`, `was born in`, `were born in`,
+`had` and `welcomed`. Offering a controlled vocabulary at extraction time normalises this at the point
+of writing, which is the only place it can be done safely — merging predicates *after* the fact would
+eventually merge `bought` onto `sold` and invert the meaning.
+
+## The one-table rule
+
+```
+canonical relation ──► surface forms
+ │ │
+ │ └── read side: the query lexicon, derived at load
+ └── write side: the extraction vocabulary, injected into the extraction prompt
+```
+
+**Keys are the extraction vocabulary. The inverse index is the query lexicon, and it is derived, never
+authored.** Both sides come from this one file, and tests enforce that they agree in both directions.
+
+This rule exists because it was broken. The two sides were briefly maintained as separate lists and
+drifted: **13 relations became resolvable at query time that the extractor was never offered**, so the
+graph could not contain them however well retrieval worked. `assembled` was one of them — which is why
+assembly was filed under `completed`, and why a benchmark question about furniture bought, assembled,
+sold or fixed could not be answered from the graph at all.
+
+**Only keys cross over.** Surface forms stay read-side: they never enter an extraction prompt, where
+they would cost tokens on every call and invite the extractor to choose inconsistently between `buy`,
+`buys` and `purchased` — the opposite of the consolidation the vocabulary exists to produce.
+
+## Where the data comes from
+
+| source | licence | what it contributed | fetched |
+|---|---|---|---|
+| [schema.org Action hierarchy](https://schema.org/Action) | CC BY-SA 3.0 | canonical keys for the **event** family — 34 relations, incl. the whole trade/transfer group | 2026-08-08 |
+| [Wikidata](https://query.wikidata.org/) property aliases (`skos:altLabel`, SPARQL) | CC0 | surface forms for the **state** family — 6 relations, 141 alias rows over 10 targeted properties | 2026-08-08 |
+| [PARAREL](https://github.com/yanaiela/pararel) paraphrase patterns | MIT | verb phrases per relation — **4 relations**; best-shaped source, its patterns *are* verb phrases (`is originally from` → `was born`, `passed away in` → `died`) | 2026-08-08 |
+| [Rel2Text](https://github.com/kasnerz/rel2text) crowd verbalisations | Apache-2.0 | delexicalised phrases, `state==ok` rows only — **7 relations** | 2026-08-08 |
+| [FewRel `pid2name.json`](https://github.com/thunlp/FewRel) | MIT | **surveyed, near-zero yield** — see below | 2026-08-08 |
+| hand-authored | — | **60 relations** — the majority, and the opposite of what the plan assumed. No surveyed source covers domestic life | — |
+
+### Why the two families are seeded differently
+
+**schema.org Actions model events; a memory graph stores events *and* states.** Measured against the
+top-50 predicates of a real graph, schema.org covers **38.0% by relation and 40.7% by fact mass**, and
+**44.7% of fact mass has no Action mapping at all**. The unmapped set is not a tail of oddities — it is
+the state/identity backbone: `is`, `is a`, `owns`, `has`, `works at`, `knows`, `belongs to`. The single
+most common predicate, `is`, is **26% of every fact in the graph** and schema.org has no Action for it,
+because it is the copula.
+
+So the state family is seeded from schema.org and Wikidata **properties** instead, which is where
+relations of that shape actually live: `P108 employer`, `P551 residence`, `P1830 owner of`, `P26 spouse`.
+
+### Why FewRel contributed almost nothing
+
+Its 744 Wikidata properties were matched against our relations and produced **28 apparent hits of which
+all but one are spurious substring collisions** — `has pet`, `has melody`, `has grammatical mood`,
+`has superpartner`, `has anatomical branch`, and `studied by` matching `died`. The one genuine match is
+`P2283 uses`.
+
+This is the same finding as the wider dataset survey, now verified directly: every relation-extraction
+corpus reviewed (TACRED, DocRED, REDFM, FewRel, T-REx, NYT10, Google-RE, Wiki-NRE) is **encyclopedic or
+newswire**. Their inventories describe grammar, physics, anatomy and geography — not what a person
+bought, planned or fixed. **The hand-authored delta is therefore the largest component, which is the
+opposite of what was originally assumed.**
+
+### Sources deliberately excluded
+
+| source | reason |
+|---|---|
+| Wiki-NRE, Google-RE, NYT10 | no licence stated anywhere |
+| REBEL | README self-contradicts on non-commercial use |
+| TACRED | paid ($25 for non-members) |
+
+## What was done to the data
+
+1. **schema.org** — Action hierarchy fetched in full (~110 descendant types) and mapped by hand to our
+ relations. Each mapping is recorded per relation in `sources` as `schema.org:`.
+2. **Wikidata** — English `skos:altLabel` aliases fetched by SPARQL for ten targeted properties.
+ Filtered to **verb-like forms only**: at most three words, letters and spaces only, and not starting
+ with `of`. A question contains "worked at"; it does not contain "alma mater" or "of employer".
+3. **Provenance is earned, not asserted.** A `wikidata:` source is recorded only when that property
+ actually contributed a form not already present. An early build claimed Wikidata provenance while
+ containing none of its data, because a file-path error was silently swallowed; the generator now
+ fails closed instead.
+4. **Determinism** — keys and forms are sorted; regenerating from unchanged inputs is byte-identical.
+ A vocabulary that varied per run would reintroduce the non-determinism it exists to remove.
+
+## Invariants, enforced by tests
+
+These fail CI rather than throwing inside a consumer's process on first use:
+
+- every relation declares at least one source, and a `family` of `event` or `state`
+- canonical keys are already in stored `predicate_key` form, so resolution produces keys the graph can match
+- no surface form is claimed by two relations; ambiguity is dropped, never guessed
+- no surface form collides with a *different* relation's canonical key
+- opposing pairs are both present — `bought`/`sold`, `likes`/`dislikes`, `borrowed`/`lent`, `gave`/`received`
+- everything offered to extraction resolves at query time, and everything resolvable is offered, unless
+ explicitly listed in `retired`
+
+## `retired`
+
+Currently **`finished`, `welcomed`**. A graph does not rewrite itself when a vocabulary
+changes, so removing a key must stop new writes without making facts already stored under it
+unreachable.
+
+Two mechanisms. **Demotion** removes the key and keeps its name as a surface form of the relation it
+merged into, so old facts are reached through the survivor — this is what `finished` (into `completed`)
+and `welcomed` (into `was born`) did. **Flagging** keeps the key and excludes it from the extraction
+vocabulary. Demotion is preferred where a survivor exists, because it leaves one name for one meaning.
+
+## `storedOnly`
+
+32 forms across the table. These are fetched by expansion but never trigger retrieval from a
+question. Two groups: the copulas (`is`, `was`, `had`, …), which appear in nearly every question and
+would expand `is` — 26% of the measured graph — on all of them; and bare verbs that double as
+assistant boilerplate (`plan`, `give`, `tell`, `know`, `need`, `want`, `find`, `work`, `order`, `own`,
+`used`, `change`, `go to`). Each is re-admitted through a question-anchored phrase such as
+`do i work` / `did i work`, so recall survives while the boilerplate stays silent. Bare `plan` was the
+worst case: `planned` is the largest measured bucket at 839 facts, so *"which phone plan am I on?"*
+could displace most of a retrieval budget on a question that had nothing to do with plans.
+
+## Known limitations
+
+- **Mined aliases are noisy.** Wikidata contributed nouns as well as verbs — `works at` acquired
+ `location` and `organisation`, `belongs to` acquired `club`. A question rarely contains these, and a
+ wrong surface form is not harmless: resolution expands a **whole relation** into a fixed retrieval
+ budget, so one bad alias can displace correct items. These are under review.
+- **Size.** 101 relations against a ~400 reviewability ceiling, with 619 surface forms. The ceiling applies to *keys*, which cost
+ prompt tokens on every extraction call; surface forms are read-side and far cheaper.
+- **Changing this file changes what gets extracted**, and only takes effect on a fresh build of the
+ memory graph. Its content hash is recorded in evaluation reports so two graphs built under different
+ vocabularies are never compared as though equivalent.
diff --git a/src/AgentMemory.Core/Memory/RelationVocabularyDocument.cs b/src/AgentMemory.Core/Memory/RelationVocabularyDocument.cs
new file mode 100644
index 00000000..334660e1
--- /dev/null
+++ b/src/AgentMemory.Core/Memory/RelationVocabularyDocument.cs
@@ -0,0 +1,97 @@
+using System.Reflection;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+namespace AgentMemory.Core.Memory;
+
+/// One canonical relation, its provenance, and the forms a question may use for it.
+internal sealed class RelationVocabularyEntry
+{
+ /// event or state.
+ ///
+ /// The split is not decorative. schema.org models actions and has no vocabulary for the
+ /// state/identity relations that carry 44.7% of this corpus's fact mass, so the two families are
+ /// seeded from different sources and reviewed against different expectations.
+ ///
+ [JsonPropertyName("family")]
+ public string Family { get; init; } = "event";
+
+ /// Where the relation came from, e.g. schema.org:BuyAction, wikidata:P108.
+ /// Licence provenance ships with the package; it is part of the artifact, not a footnote.
+ [JsonPropertyName("sources")]
+ public IReadOnlyList Sources { get; init; } = [];
+
+ [JsonPropertyName("surfaceForms")]
+ public IReadOnlyList SurfaceForms { get; init; } = [];
+
+ ///
+ /// Forms that are expansion targets but must never trigger retrieval from a question.
+ ///
+ ///
+ /// Surface forms carry two jobs at once: they decide what a question retrieves, and they are the
+ /// stored predicate keys expansion fetches. The copulas need the second and are actively harmful
+ /// in the first — was, had and have appear in almost every question a person
+ /// asks, and each one would expand is, which is 26% of the measured graph, exhausting the
+ /// shared budget before any correct relation is reached. Deleting them is not an option either,
+ /// because facts really are stored under them.
+ ///
+ [JsonPropertyName("storedOnly")]
+ public IReadOnlyList StoredOnly { get; init; } = [];
+}
+
+internal sealed class RelationVocabularyDocument
+{
+ private static readonly Lazy Cached = new(Parse);
+
+ [JsonPropertyName("schemaVersion")]
+ public int SchemaVersion { get; init; }
+
+ [JsonPropertyName("retired")]
+ public IReadOnlyList Retired { get; init; } = [];
+
+ ///
+ /// Relations the extractor may write but a question must never expand wholesale, each with its
+ /// reason.
+ ///
+ ///
+ /// They remain reachable by top-K similarity; what they are exempt from is expansion. The
+ /// copulas would otherwise flood a fixed budget with almost no meaning per fact — the J1.6 build
+ /// measured `has` absorbing 518 facts. Declared explicitly rather than left implicit in
+ /// storedOnly, so the asymmetry is a recorded decision instead of an accident.
+ ///
+ [JsonPropertyName("expansionExempt")]
+ public IReadOnlyDictionary ExpansionExempt { get; init; } =
+ new Dictionary(StringComparer.Ordinal);
+
+ [JsonPropertyName("canonical")]
+ public IReadOnlyDictionary Canonical { get; init; } =
+ new Dictionary(StringComparer.Ordinal);
+
+ /// The reviewed vocabulary, parsed once.
+ ///
+ /// Embedded rather than read from disk: this ships as a NuGet package, so a file dependency would
+ /// be a deployment hazard and would add a startup I/O failure mode to a library. Parsed once into
+ /// immutable structures, after which lookups are O(1) forever.
+ ///
+ internal static RelationVocabularyDocument Load() => Cached.Value;
+
+ private static RelationVocabularyDocument Parse()
+ {
+ const string ResourceName = "AgentMemory.Core.Memory.relation-vocabulary.json";
+ var assembly = typeof(RelationVocabularyDocument).Assembly;
+ using var stream = assembly.GetManifestResourceStream(ResourceName)
+ ?? throw new InvalidOperationException(
+ $"The relation vocabulary resource '{ResourceName}' is missing from {assembly.GetName().Name}. " +
+ "It must be embedded, not shipped alongside.");
+
+ var document = JsonSerializer.Deserialize(
+ stream,
+ new JsonSerializerOptions { PropertyNameCaseInsensitive = true })
+ ?? throw new InvalidOperationException("The relation vocabulary resource is empty.");
+
+ if (document.Canonical.Count == 0)
+ throw new InvalidOperationException("The relation vocabulary declares no relations.");
+
+ return document;
+ }
+}
diff --git a/src/AgentMemory.Core/Memory/relation-vocabulary.json b/src/AgentMemory.Core/Memory/relation-vocabulary.json
new file mode 100644
index 00000000..62adb79c
--- /dev/null
+++ b/src/AgentMemory.Core/Memory/relation-vocabulary.json
@@ -0,0 +1,1660 @@
+{
+ "schemaVersion": 1,
+ "about": "Canonical relation -> surface forms. Keys are the extraction vocabulary; the query lexicon is the inverse index, derived at load and never authored, so the two cannot drift apart. Only keys reach the extraction prompt; surface forms are read-side only.",
+ "sourceLicences": {
+ "schema.org": "CC BY-SA 3.0 - Action hierarchy, seeds the event family",
+ "wikidata": "CC0 - property aliases via SPARQL skos:altLabel, seeds the state family",
+ "fewrel": "MIT - pid2name.json surveyed; 744 relations, one genuine domain match (P2283 uses)",
+ "hand-authored": "domestic-life relations no surveyed source provides (assembled, fixed)"
+ },
+ "excludedSources": {
+ "wiki-nre": "no licence",
+ "google-re": "no licence",
+ "nyt10": "no licence",
+ "rebel": "README self-contradicts on non-commercial",
+ "tacred": "paid"
+ },
+ "retired": [
+ "finished"
+ ],
+ "canonical": {
+ "adopted": {
+ "family": "event",
+ "sources": [
+ "hand-authored"
+ ],
+ "surfaceForms": [
+ "adopt",
+ "adopting",
+ "adopts"
+ ]
+ },
+ "allergic to": {
+ "family": "state",
+ "sources": [
+ "hand-authored"
+ ],
+ "surfaceForms": [
+ "allergy to",
+ "is allergic to"
+ ]
+ },
+ "applied for": {
+ "family": "event",
+ "sources": [
+ "hand-authored"
+ ],
+ "surfaceForms": [
+ "applied",
+ "applied to",
+ "applies for",
+ "apply",
+ "apply for",
+ "apply to",
+ "applying for"
+ ]
+ },
+ "arrived": {
+ "family": "event",
+ "sources": [
+ "schema.org:ArriveAction"
+ ],
+ "surfaceForms": [
+ "arrive",
+ "arrives",
+ "arriving",
+ "delivered",
+ "deliveries",
+ "delivery"
+ ]
+ },
+ "asked about": {
+ "family": "event",
+ "sources": [
+ "schema.org:AskAction"
+ ],
+ "surfaceForms": [
+ "ask",
+ "ask about",
+ "asked",
+ "asking",
+ "asking about",
+ "asks",
+ "asks about"
+ ]
+ },
+ "assembled": {
+ "family": "event",
+ "sources": [
+ "hand-authored"
+ ],
+ "surfaceForms": [
+ "assemble",
+ "assembles",
+ "assembling",
+ "put together",
+ "puts together",
+ "putting together"
+ ]
+ },
+ "ate": {
+ "family": "event",
+ "sources": [
+ "hand-authored"
+ ],
+ "surfaceForms": [
+ "eat",
+ "eaten",
+ "eating",
+ "eats"
+ ]
+ },
+ "attended": {
+ "family": "event",
+ "sources": [
+ "wikidata:P1344",
+ "rel2text"
+ ],
+ "surfaceForms": [
+ "attend",
+ "attending",
+ "attends",
+ "compete in",
+ "competed in",
+ "competes in",
+ "competing in",
+ "did i take part",
+ "do i take part",
+ "i take part",
+ "is a student at",
+ "participate",
+ "participate in",
+ "participated",
+ "participated in",
+ "participated in the",
+ "participates",
+ "participates in",
+ "participating",
+ "participating in",
+ "present at",
+ "take part",
+ "take part in",
+ "takes part",
+ "taking part",
+ "taking part in",
+ "took part",
+ "took part in",
+ "took part in the",
+ "was educated at"
+ ],
+ "storedOnly": [
+ "played in the"
+ ]
+ },
+ "avoids": {
+ "family": "state",
+ "sources": [
+ "schema.org:IgnoreAction"
+ ],
+ "surfaceForms": [
+ "avoid",
+ "avoided",
+ "avoiding"
+ ]
+ },
+ "believes": {
+ "family": "state",
+ "sources": [
+ "hand-authored"
+ ],
+ "surfaceForms": [
+ "believe",
+ "believed",
+ "believing"
+ ]
+ },
+ "belongs to": {
+ "family": "state",
+ "sources": [
+ "wikidata:P463",
+ "pararel",
+ "rel2text"
+ ],
+ "surfaceForms": [
+ "belong to",
+ "belonged to",
+ "is a member of",
+ "is a member of the",
+ "is affiliated with",
+ "is member of",
+ "join",
+ "joined",
+ "joining",
+ "joins",
+ "member of",
+ "played for the"
+ ]
+ },
+ "booked": {
+ "family": "event",
+ "sources": [
+ "hand-authored"
+ ],
+ "surfaceForms": [
+ "book a",
+ "book an",
+ "book me",
+ "book the",
+ "book us",
+ "booking",
+ "bookings",
+ "reservation",
+ "reservations",
+ "reserve",
+ "reserved",
+ "reserves",
+ "reserving"
+ ]
+ },
+ "borrowed": {
+ "family": "event",
+ "sources": [
+ "schema.org:BorrowAction"
+ ],
+ "surfaceForms": [
+ "borrow",
+ "borrowing",
+ "borrows"
+ ]
+ },
+ "bought": {
+ "family": "event",
+ "sources": [
+ "schema.org:BuyAction"
+ ],
+ "surfaceForms": [
+ "buy",
+ "buying",
+ "buys",
+ "did i order",
+ "do i order",
+ "ordered",
+ "ordering",
+ "orders",
+ "purchase",
+ "purchased",
+ "purchases",
+ "purchasing"
+ ],
+ "storedOnly": [
+ "order"
+ ]
+ },
+ "broke": {
+ "family": "event",
+ "sources": [
+ "hand-authored"
+ ],
+ "surfaceForms": [
+ "break",
+ "breaking",
+ "breaks",
+ "broke down",
+ "broken"
+ ]
+ },
+ "called": {
+ "family": "event",
+ "sources": [
+ "hand-authored"
+ ],
+ "surfaceForms": [
+ "call",
+ "calling",
+ "calls",
+ "phoned",
+ "rang"
+ ]
+ },
+ "cancelled": {
+ "family": "event",
+ "sources": [
+ "schema.org:CancelAction"
+ ],
+ "surfaceForms": [
+ "cancel",
+ "canceled",
+ "canceling",
+ "cancelling",
+ "cancels"
+ ]
+ },
+ "changed to": {
+ "family": "event",
+ "sources": [
+ "schema.org:ReplaceAction"
+ ],
+ "surfaceForms": [
+ "change to",
+ "changed",
+ "changes to",
+ "changing to",
+ "did i change"
+ ],
+ "storedOnly": [
+ "change"
+ ]
+ },
+ "cleaned": {
+ "family": "event",
+ "sources": [
+ "hand-authored"
+ ],
+ "surfaceForms": [
+ "clean"
+ ]
+ },
+ "commutes": {
+ "family": "event",
+ "sources": [
+ "hand-authored"
+ ],
+ "surfaceForms": [
+ "commute",
+ "commuted",
+ "commuting"
+ ]
+ },
+ "completed": {
+ "family": "event",
+ "sources": [
+ "schema.org:AchieveAction"
+ ],
+ "surfaceForms": [
+ "complete",
+ "completes",
+ "completing",
+ "finish",
+ "finished",
+ "finishes",
+ "finishing"
+ ]
+ },
+ "considered": {
+ "family": "event",
+ "sources": [
+ "schema.org:AssessAction"
+ ],
+ "surfaceForms": [
+ "consider",
+ "considering",
+ "considers",
+ "is considering"
+ ]
+ },
+ "cooked": {
+ "family": "event",
+ "sources": [
+ "hand-authored"
+ ],
+ "surfaceForms": [
+ "bake",
+ "baked",
+ "bakes",
+ "baking",
+ "cook",
+ "cooking",
+ "cooks"
+ ]
+ },
+ "costs": {
+ "family": "state",
+ "sources": [
+ "hand-authored"
+ ],
+ "surfaceForms": [
+ "cost",
+ "cost me",
+ "priced at"
+ ]
+ },
+ "created": {
+ "family": "event",
+ "sources": [
+ "schema.org:CreateAction"
+ ],
+ "surfaceForms": [
+ "build",
+ "builds",
+ "built",
+ "create",
+ "creates",
+ "creating"
+ ]
+ },
+ "decided": {
+ "family": "event",
+ "sources": [
+ "hand-authored"
+ ],
+ "surfaceForms": [
+ "choose",
+ "chooses",
+ "choosing",
+ "chose",
+ "decide",
+ "decides",
+ "deciding"
+ ]
+ },
+ "decreased to": {
+ "family": "event",
+ "sources": [
+ "hand-authored"
+ ],
+ "surfaceForms": [
+ "decrease",
+ "decrease to",
+ "decreased",
+ "decreases",
+ "decreases to",
+ "decreasing",
+ "drop to",
+ "dropped to",
+ "drops to",
+ "fell to",
+ "go down",
+ "goes down",
+ "going down",
+ "gone down",
+ "went down"
+ ]
+ },
+ "diagnosed with": {
+ "family": "state",
+ "sources": [
+ "hand-authored"
+ ],
+ "surfaceForms": [
+ "diagnose with",
+ "diagnosed",
+ "was diagnosed with"
+ ]
+ },
+ "died": {
+ "family": "event",
+ "sources": [
+ "pararel"
+ ],
+ "surfaceForms": [
+ "die",
+ "died at",
+ "died in",
+ "dies",
+ "dying",
+ "lost their life at",
+ "pass away",
+ "passed away",
+ "passed away at",
+ "passed away in",
+ "passes away",
+ "passing away",
+ "succumbed at"
+ ],
+ "storedOnly": [
+ "expired at"
+ ]
+ },
+ "dislikes": {
+ "family": "state",
+ "sources": [
+ "schema.org:DislikeAction"
+ ],
+ "surfaceForms": [
+ "dislike",
+ "disliked",
+ "disliking",
+ "hate",
+ "hated",
+ "hates",
+ "hating"
+ ]
+ },
+ "divorced": {
+ "family": "event",
+ "sources": [
+ "hand-authored"
+ ],
+ "surfaceForms": [
+ "divorce",
+ "divorces",
+ "divorcing"
+ ]
+ },
+ "drank": {
+ "family": "event",
+ "sources": [
+ "hand-authored"
+ ],
+ "surfaceForms": [
+ "drink",
+ "drunk"
+ ]
+ },
+ "earned": {
+ "family": "event",
+ "sources": [
+ "hand-authored"
+ ],
+ "surfaceForms": [
+ "earn",
+ "income",
+ "salary"
+ ]
+ },
+ "exercised": {
+ "family": "event",
+ "sources": [
+ "hand-authored"
+ ],
+ "surfaceForms": [
+ "did i work out",
+ "do i work out",
+ "exercise",
+ "exercises",
+ "exercising",
+ "work out",
+ "worked out",
+ "working out",
+ "works out"
+ ]
+ },
+ "expires": {
+ "family": "state",
+ "sources": [
+ "hand-authored"
+ ],
+ "surfaceForms": [
+ "expiration",
+ "expire",
+ "expired",
+ "expiring",
+ "expiry"
+ ]
+ },
+ "feels": {
+ "family": "state",
+ "sources": [
+ "hand-authored"
+ ],
+ "surfaceForms": [
+ "feel",
+ "feeling",
+ "felt"
+ ]
+ },
+ "finds": {
+ "family": "event",
+ "sources": [
+ "schema.org:FindAction"
+ ],
+ "surfaceForms": [
+ "did i find",
+ "finding",
+ "found"
+ ],
+ "storedOnly": [
+ "find"
+ ]
+ },
+ "fixed": {
+ "family": "event",
+ "sources": [
+ "hand-authored"
+ ],
+ "surfaceForms": [
+ "fix",
+ "fixes",
+ "fixing",
+ "mend",
+ "mended",
+ "mending",
+ "mends",
+ "repair",
+ "repaired",
+ "repairing",
+ "repairs"
+ ]
+ },
+ "gave": {
+ "family": "event",
+ "sources": [
+ "schema.org:GiveAction"
+ ],
+ "surfaceForms": [
+ "did i give",
+ "donate",
+ "donated",
+ "donates",
+ "donating",
+ "gifted",
+ "gifting"
+ ],
+ "storedOnly": [
+ "give",
+ "gives",
+ "giving"
+ ]
+ },
+ "has": {
+ "family": "state",
+ "sources": [
+ "hand-authored"
+ ],
+ "surfaceForms": [],
+ "storedOnly": [
+ "had",
+ "has",
+ "have",
+ "having"
+ ]
+ },
+ "heard": {
+ "family": "event",
+ "sources": [
+ "hand-authored"
+ ],
+ "surfaceForms": [
+ "hear",
+ "hearing",
+ "hears"
+ ]
+ },
+ "helped": {
+ "family": "event",
+ "sources": [
+ "hand-authored"
+ ],
+ "surfaceForms": [
+ "help",
+ "helping",
+ "helps",
+ "assist",
+ "assisted",
+ "assisting",
+ "assists"
+ ]
+ },
+ "hired": {
+ "family": "event",
+ "sources": [
+ "hand-authored"
+ ],
+ "surfaceForms": [
+ "hire",
+ "hiring"
+ ]
+ },
+ "increased to": {
+ "family": "event",
+ "sources": [
+ "hand-authored"
+ ],
+ "surfaceForms": [
+ "go up",
+ "goes up",
+ "going up",
+ "gone up",
+ "increase",
+ "increase to",
+ "increased",
+ "increases",
+ "increases to",
+ "increasing",
+ "rose to",
+ "went up"
+ ]
+ },
+ "injured": {
+ "family": "event",
+ "sources": [
+ "hand-authored"
+ ],
+ "surfaceForms": [
+ "hurt",
+ "hurts",
+ "injure",
+ "injures",
+ "injuring",
+ "injury",
+ "sprained"
+ ]
+ },
+ "installed": {
+ "family": "event",
+ "sources": [
+ "hand-authored"
+ ],
+ "surfaceForms": [
+ "install",
+ "installing",
+ "installs"
+ ]
+ },
+ "invested in": {
+ "family": "event",
+ "sources": [
+ "hand-authored"
+ ],
+ "surfaceForms": [
+ "invest",
+ "invest in",
+ "invested",
+ "investing",
+ "investing in",
+ "invests",
+ "invests in"
+ ]
+ },
+ "is": {
+ "family": "state",
+ "sources": [
+ "hand-authored"
+ ],
+ "surfaceForms": [],
+ "storedOnly": [
+ "am",
+ "are",
+ "be",
+ "been",
+ "is",
+ "was",
+ "were"
+ ]
+ },
+ "is a": {
+ "family": "state",
+ "sources": [
+ "hand-authored"
+ ],
+ "surfaceForms": [],
+ "storedOnly": [
+ "is a",
+ "is an",
+ "was a",
+ "was an"
+ ]
+ },
+ "is interested in": {
+ "family": "state",
+ "sources": [
+ "hand-authored"
+ ],
+ "surfaceForms": [
+ "interested",
+ "interested in",
+ "is interested"
+ ]
+ },
+ "knows": {
+ "family": "state",
+ "sources": [
+ "hand-authored"
+ ],
+ "surfaceForms": [
+ "did i know",
+ "do i know",
+ "knew",
+ "knowing"
+ ],
+ "storedOnly": [
+ "know"
+ ]
+ },
+ "learned": {
+ "family": "event",
+ "sources": [
+ "hand-authored"
+ ],
+ "surfaceForms": [
+ "learn",
+ "learning",
+ "learns",
+ "learnt",
+ "studied",
+ "studies",
+ "study",
+ "studying"
+ ]
+ },
+ "led": {
+ "family": "event",
+ "sources": [
+ "hand-authored"
+ ],
+ "surfaceForms": [
+ "lead",
+ "leading",
+ "leads",
+ "managed",
+ "manages",
+ "managing"
+ ]
+ },
+ "lent": {
+ "family": "event",
+ "sources": [
+ "schema.org:LendAction"
+ ],
+ "surfaceForms": [
+ "lend",
+ "lending",
+ "lends",
+ "loaned"
+ ]
+ },
+ "likes": {
+ "family": "state",
+ "sources": [
+ "schema.org:LikeAction"
+ ],
+ "surfaceForms": [
+ "did i like",
+ "do i like",
+ "enjoy",
+ "enjoyed",
+ "enjoys",
+ "liked",
+ "liking",
+ "love",
+ "loved",
+ "loves"
+ ],
+ "storedOnly": [
+ "like"
+ ]
+ },
+ "listened to": {
+ "family": "event",
+ "sources": [
+ "hand-authored"
+ ],
+ "surfaceForms": [
+ "listen",
+ "listen to",
+ "listening to",
+ "listens to"
+ ]
+ },
+ "lives in": {
+ "family": "state",
+ "sources": [
+ "wikidata:P551",
+ "rel2text"
+ ],
+ "surfaceForms": [
+ "has resided in",
+ "live",
+ "live in",
+ "lived in",
+ "lives at",
+ "living in",
+ "resided at",
+ "resided in",
+ "resident in",
+ "resident of",
+ "resides in",
+ "residing in",
+ "stays in"
+ ]
+ },
+ "lost": {
+ "family": "event",
+ "sources": [
+ "hand-authored"
+ ],
+ "surfaceForms": [
+ "lose",
+ "loses",
+ "losing",
+ "misplaced"
+ ]
+ },
+ "married": {
+ "family": "event",
+ "sources": [
+ "schema.org:MarryAction",
+ "wikidata:P26",
+ "rel2text"
+ ],
+ "surfaceForms": [
+ "is married to",
+ "is the spouse of",
+ "marital partner",
+ "marriage partner",
+ "married partner",
+ "married to",
+ "marries",
+ "marry",
+ "marrying",
+ "spouse",
+ "spouses",
+ "wedded",
+ "wedded to"
+ ]
+ },
+ "met": {
+ "family": "event",
+ "sources": [
+ "schema.org:MeetAction"
+ ],
+ "surfaceForms": [
+ "meet",
+ "meeting",
+ "meets"
+ ]
+ },
+ "moved to": {
+ "family": "event",
+ "sources": [
+ "schema.org:MoveAction"
+ ],
+ "surfaceForms": [
+ "move",
+ "move to",
+ "moved",
+ "moves to",
+ "moving to",
+ "relocate to",
+ "relocated to",
+ "relocates to",
+ "relocating to"
+ ]
+ },
+ "noticed": {
+ "family": "event",
+ "sources": [
+ "hand-authored"
+ ],
+ "surfaceForms": [
+ "notice",
+ "noticing",
+ "notices"
+ ]
+ },
+ "owes": {
+ "family": "state",
+ "sources": [
+ "hand-authored"
+ ],
+ "surfaceForms": [
+ "owe",
+ "owed",
+ "owing"
+ ]
+ },
+ "owns": {
+ "family": "state",
+ "sources": [
+ "wikidata:P1830",
+ "rel2text"
+ ],
+ "surfaceForms": [
+ "did i own",
+ "do i own",
+ "is owned by",
+ "is the owner of",
+ "owned",
+ "owning",
+ "owns property",
+ "owns the",
+ "possess",
+ "possesses"
+ ],
+ "storedOnly": [
+ "own"
+ ]
+ },
+ "paid": {
+ "family": "event",
+ "sources": [
+ "hand-authored"
+ ],
+ "surfaceForms": [
+ "paid for",
+ "pay",
+ "pay for",
+ "paying",
+ "paying for",
+ "pays",
+ "spend",
+ "spending",
+ "spends",
+ "spent"
+ ]
+ },
+ "planned": {
+ "family": "event",
+ "sources": [
+ "schema.org:PlanAction"
+ ],
+ "surfaceForms": [
+ "did i plan",
+ "do i plan",
+ "plan to",
+ "planning",
+ "plans"
+ ],
+ "storedOnly": [
+ "plan"
+ ]
+ },
+ "played": {
+ "family": "event",
+ "sources": [
+ "hand-authored"
+ ],
+ "surfaceForms": [
+ "play",
+ "playing",
+ "plays"
+ ]
+ },
+ "prefers": {
+ "family": "state",
+ "sources": [
+ "schema.org:ChooseAction"
+ ],
+ "surfaceForms": [
+ "prefer",
+ "preferred",
+ "preferring"
+ ]
+ },
+ "prescribed": {
+ "family": "event",
+ "sources": [
+ "hand-authored"
+ ],
+ "surfaceForms": [
+ "prescribe",
+ "prescribes",
+ "prescribing"
+ ]
+ },
+ "promised": {
+ "family": "event",
+ "sources": [
+ "hand-authored"
+ ],
+ "surfaceForms": [
+ "promise",
+ "promises"
+ ]
+ },
+ "promoted": {
+ "family": "event",
+ "sources": [
+ "hand-authored"
+ ],
+ "surfaceForms": [
+ "promote",
+ "promotes",
+ "promoting",
+ "promotion"
+ ]
+ },
+ "provided": {
+ "family": "event",
+ "sources": [
+ "hand-authored"
+ ],
+ "surfaceForms": [
+ "provide",
+ "providing",
+ "provides",
+ "supplied",
+ "supplies",
+ "supplying"
+ ]
+ },
+ "rated": {
+ "family": "event",
+ "sources": [
+ "schema.org:ReviewAction"
+ ],
+ "surfaceForms": [
+ "did i rate",
+ "do i rate",
+ "rate it",
+ "rate my",
+ "rate that",
+ "rate the",
+ "rate them",
+ "rate this",
+ "rated it",
+ "rating"
+ ]
+ },
+ "read": {
+ "family": "event",
+ "sources": [
+ "hand-authored"
+ ],
+ "surfaceForms": [
+ "reading",
+ "reads"
+ ]
+ },
+ "received": {
+ "family": "event",
+ "sources": [
+ "schema.org:ReceiveAction"
+ ],
+ "surfaceForms": [
+ "receive",
+ "receives",
+ "receiving"
+ ]
+ },
+ "recommends": {
+ "family": "event",
+ "sources": [
+ "schema.org:EndorseAction"
+ ],
+ "surfaceForms": [
+ "recommend",
+ "recommended",
+ "recommending"
+ ]
+ },
+ "related to": {
+ "family": "state",
+ "sources": [
+ "hand-authored"
+ ],
+ "surfaceForms": [
+ "relates to"
+ ],
+ "storedOnly": [
+ "related",
+ "related to"
+ ]
+ },
+ "renewed": {
+ "family": "event",
+ "sources": [
+ "hand-authored"
+ ],
+ "surfaceForms": [
+ "renew"
+ ]
+ },
+ "rented": {
+ "family": "event",
+ "sources": [
+ "schema.org:RentAction"
+ ],
+ "surfaceForms": [
+ "lease",
+ "leased",
+ "leases",
+ "rent",
+ "renting",
+ "rents"
+ ]
+ },
+ "rented out": {
+ "family": "event",
+ "sources": [
+ "hand-authored"
+ ],
+ "surfaceForms": [
+ "leased out",
+ "let out",
+ "rent out",
+ "renting out",
+ "rents out"
+ ]
+ },
+ "requested": {
+ "family": "event",
+ "sources": [
+ "schema.org:AskAction"
+ ],
+ "surfaceForms": [
+ "ask for",
+ "asked for",
+ "asking for",
+ "asks for",
+ "request",
+ "requesting",
+ "requests"
+ ]
+ },
+ "requires": {
+ "family": "state",
+ "sources": [
+ "hand-authored"
+ ],
+ "surfaceForms": [
+ "did i need",
+ "do i need",
+ "needed",
+ "needs",
+ "require",
+ "required",
+ "requiring"
+ ],
+ "storedOnly": [
+ "need"
+ ]
+ },
+ "returned": {
+ "family": "event",
+ "sources": [
+ "schema.org:ReturnAction"
+ ],
+ "surfaceForms": [
+ "return",
+ "returning",
+ "returns"
+ ]
+ },
+ "returned from": {
+ "family": "event",
+ "sources": [
+ "hand-authored"
+ ],
+ "surfaceForms": [
+ "came back from",
+ "come back from",
+ "get back from",
+ "gets back from",
+ "got back from",
+ "return from",
+ "returning from",
+ "returns from"
+ ]
+ },
+ "saved": {
+ "family": "event",
+ "sources": [
+ "hand-authored"
+ ],
+ "surfaceForms": [
+ "save",
+ "saves",
+ "saving",
+ "savings"
+ ]
+ },
+ "scheduled": {
+ "family": "event",
+ "sources": [
+ "schema.org:ScheduleAction"
+ ],
+ "surfaceForms": [
+ "appointment",
+ "appointments",
+ "schedule",
+ "schedules",
+ "scheduling"
+ ]
+ },
+ "sent": {
+ "family": "event",
+ "sources": [
+ "hand-authored"
+ ],
+ "surfaceForms": [
+ "did i text",
+ "email",
+ "emailed",
+ "emailing",
+ "emails",
+ "messaged",
+ "send",
+ "sending",
+ "sends",
+ "texted",
+ "texting",
+ "texts"
+ ],
+ "storedOnly": [
+ "text"
+ ]
+ },
+ "signed": {
+ "family": "event",
+ "sources": [
+ "hand-authored"
+ ],
+ "surfaceForms": [
+ "sign",
+ "sign up for",
+ "signed up for"
+ ]
+ },
+ "slept": {
+ "family": "event",
+ "sources": [
+ "hand-authored"
+ ],
+ "surfaceForms": [
+ "sleep"
+ ]
+ },
+ "sold": {
+ "family": "event",
+ "sources": [
+ "schema.org:SellAction"
+ ],
+ "surfaceForms": [
+ "sell",
+ "selling",
+ "sells"
+ ]
+ },
+ "started": {
+ "family": "event",
+ "sources": [
+ "schema.org:ActivateAction"
+ ],
+ "surfaceForms": [
+ "began",
+ "begin",
+ "begins",
+ "begun",
+ "start",
+ "starting",
+ "starts"
+ ]
+ },
+ "stayed at": {
+ "family": "event",
+ "sources": [
+ "hand-authored"
+ ],
+ "surfaceForms": [
+ "stay",
+ "stay at",
+ "stay in",
+ "stayed",
+ "stayed in",
+ "staying",
+ "staying at",
+ "stays at"
+ ]
+ },
+ "stopped": {
+ "family": "event",
+ "sources": [
+ "hand-authored"
+ ],
+ "surfaceForms": [
+ "gave up",
+ "give up",
+ "gives up",
+ "giving up",
+ "quit",
+ "quits",
+ "quitting",
+ "resign",
+ "stop",
+ "stopping",
+ "stops"
+ ]
+ },
+ "subscribed to": {
+ "family": "state",
+ "sources": [
+ "hand-authored"
+ ],
+ "surfaceForms": [
+ "subscribe to",
+ "subscribes to",
+ "subscribing to",
+ "subscription",
+ "subscriptions"
+ ]
+ },
+ "threw away": {
+ "family": "event",
+ "sources": [
+ "hand-authored"
+ ],
+ "surfaceForms": [
+ "get rid of",
+ "got rid of",
+ "threw out",
+ "throw away",
+ "throw out",
+ "throwing away",
+ "throwing out",
+ "thrown away",
+ "thrown out"
+ ]
+ },
+ "told": {
+ "family": "event",
+ "sources": [
+ "hand-authored"
+ ],
+ "surfaceForms": [
+ "did i tell",
+ "said",
+ "saying",
+ "says",
+ "tell her",
+ "tell him",
+ "telling",
+ "tells"
+ ],
+ "storedOnly": [
+ "tell"
+ ]
+ },
+ "took": {
+ "family": "event",
+ "sources": [
+ "hand-authored"
+ ],
+ "surfaceForms": [
+ "did i take",
+ "do i take",
+ "i take",
+ "takes",
+ "taking"
+ ]
+ },
+ "travelled to": {
+ "family": "event",
+ "sources": [
+ "schema.org:TravelAction"
+ ],
+ "surfaceForms": [
+ "been to",
+ "did i go to",
+ "do i go to",
+ "drive to",
+ "drives to",
+ "driving to",
+ "drove to",
+ "flew",
+ "flew to",
+ "flies",
+ "flies to",
+ "fly",
+ "fly to",
+ "flying",
+ "flying to",
+ "gone to",
+ "travel",
+ "travel to",
+ "traveled to",
+ "traveling to",
+ "travelled",
+ "travelling",
+ "travelling to",
+ "travels to",
+ "went to"
+ ],
+ "storedOnly": [
+ "go to"
+ ]
+ },
+ "tried": {
+ "family": "event",
+ "sources": [
+ "hand-authored"
+ ],
+ "surfaceForms": [
+ "sampled",
+ "tasted",
+ "tries",
+ "try",
+ "trying"
+ ]
+ },
+ "updated to": {
+ "family": "event",
+ "sources": [
+ "schema.org:UpdateAction"
+ ],
+ "surfaceForms": [
+ "update",
+ "update to",
+ "updated",
+ "updates",
+ "updates to",
+ "updating"
+ ]
+ },
+ "uses": {
+ "family": "state",
+ "sources": [
+ "schema.org:UseAction"
+ ],
+ "surfaceForms": [
+ "use",
+ "using"
+ ],
+ "storedOnly": [
+ "used"
+ ]
+ },
+ "visited": {
+ "family": "event",
+ "sources": [
+ "hand-authored"
+ ],
+ "surfaceForms": [
+ "visit",
+ "visiting",
+ "visits"
+ ]
+ },
+ "wakes": {
+ "family": "event",
+ "sources": [
+ "hand-authored"
+ ],
+ "surfaceForms": [
+ "wake",
+ "wake up",
+ "wakes up",
+ "waking",
+ "woke"
+ ]
+ },
+ "wants": {
+ "family": "state",
+ "sources": [
+ "schema.org:WantAction"
+ ],
+ "surfaceForms": [
+ "did i want",
+ "do i want",
+ "wanted",
+ "wanting",
+ "wish",
+ "wishes"
+ ],
+ "storedOnly": [
+ "want"
+ ]
+ },
+ "was born": {
+ "family": "event",
+ "sources": [
+ "pararel",
+ "rel2text"
+ ],
+ "surfaceForms": [
+ "born",
+ "born in",
+ "is native to",
+ "is originally from",
+ "originated from",
+ "originates from",
+ "was born in",
+ "was native to",
+ "was originally from",
+ "were born",
+ "were born in"
+ ]
+ },
+ "watched": {
+ "family": "event",
+ "sources": [
+ "schema.org:WatchAction"
+ ],
+ "surfaceForms": [
+ "watch",
+ "watches",
+ "watching"
+ ]
+ },
+ "weighs": {
+ "family": "state",
+ "sources": [
+ "hand-authored"
+ ],
+ "surfaceForms": [
+ "weigh",
+ "weight"
+ ]
+ },
+ "welcomed": {
+ "family": "event",
+ "sources": [
+ "hand-authored"
+ ],
+ "surfaceForms": [
+ "welcome",
+ "welcomes",
+ "welcoming"
+ ]
+ },
+ "works at": {
+ "family": "state",
+ "sources": [
+ "wikidata:P108",
+ "wikidata:P937",
+ "pararel",
+ "rel2text"
+ ],
+ "surfaceForms": [
+ "did i work",
+ "do i work",
+ "employed at",
+ "employed by",
+ "found employment in",
+ "is an employee of",
+ "is employed by",
+ "is employed by the",
+ "location of work",
+ "took up work in",
+ "used to work in",
+ "was employed in",
+ "who is employed by",
+ "who works for",
+ "work at",
+ "worked",
+ "worked at",
+ "worked for",
+ "worked from",
+ "worked in",
+ "working at",
+ "working for",
+ "working from",
+ "workplace",
+ "works from",
+ "works in"
+ ],
+ "storedOnly": [
+ "work",
+ "working",
+ "works for"
+ ]
+ },
+ "works on": {
+ "family": "state",
+ "sources": [
+ "hand-authored"
+ ],
+ "surfaceForms": [
+ "work on",
+ "worked on",
+ "working on",
+ "works on"
+ ]
+ }
+ },
+ "expansionExempt": {
+ "is": "copula; 340 facts. Expanding it wholesale floods the budget and carries almost no meaning.",
+ "is a": "copula variant; same reason as `is`.",
+ "has": "generic possession; absorbed 518 facts in the J1.6 build. Reachable by similarity, never expanded."
+ }
+}
diff --git a/src/AgentMemory.Core/Resolution/CompositeEntityResolver.Batch.cs b/src/AgentMemory.Core/Resolution/CompositeEntityResolver.Batch.cs
new file mode 100644
index 00000000..fdce224a
--- /dev/null
+++ b/src/AgentMemory.Core/Resolution/CompositeEntityResolver.Batch.cs
@@ -0,0 +1,174 @@
+using AgentMemory.Abstractions.Domain;
+using AgentMemory.Abstractions.Options;
+
+namespace AgentMemory.Core.Resolution;
+
+internal sealed partial class CompositeEntityResolver
+{
+ private readonly AsyncLocal _candidateBatch = new();
+
+ public IDisposable BeginBatch()
+ {
+ if (!_options.UseBatchEntityResolutionSnapshots)
+ return NoopBatchLease.Instance;
+ if (_candidateBatch.Value is not null)
+ throw new InvalidOperationException("An entity-resolution batch is already active in this async flow.");
+
+ var state = new CandidateBatchState();
+ _candidateBatch.Value = state;
+ return new CandidateBatchLease(this, state);
+ }
+
+ public async Task PrepareCandidatesAsync(
+ IReadOnlyCollection entityTypes,
+ MemoryScope? scope = null,
+ CancellationToken cancellationToken = default)
+ {
+ ArgumentNullException.ThrowIfNull(entityTypes);
+ var state = _candidateBatch.Value;
+ if (state is null || entityTypes.Count == 0)
+ return;
+
+ var loads = entityTypes
+ .Where(type => !string.IsNullOrWhiteSpace(type))
+ .Distinct(StringComparer.OrdinalIgnoreCase)
+ .Select(type => state.GetOrAddAsync(
+ CandidateBatchKey.Create(type, scope),
+ () => LoadCandidatesAsync(type, scope, cancellationToken)))
+ .ToArray();
+ await Task.WhenAll(loads).ConfigureAwait(false);
+ }
+
+ public void InvalidateBatch() => _candidateBatch.Value?.Invalidate();
+
+ private async Task ResolveAndRememberAsync(
+ ExtractedEntity extractedEntity,
+ IReadOnlyList sourceMessageIds,
+ MemoryScope? scope,
+ bool persistResolution,
+ CancellationToken cancellationToken)
+ {
+ var entity = await ResolveEntityCoreAsync(
+ extractedEntity,
+ sourceMessageIds,
+ scope,
+ persistResolution,
+ cancellationToken).ConfigureAwait(false);
+ _candidateBatch.Value?.Remember(
+ CandidateBatchKey.Create(extractedEntity.Type, scope), entity);
+ return entity;
+ }
+
+ private async Task> GetBatchCandidatesAsync(
+ string type,
+ MemoryScope? scope,
+ CancellationToken cancellationToken)
+ {
+ var state = _candidateBatch.Value;
+ if (state is null)
+ return await LoadCandidatesAsync(type, scope, cancellationToken).ConfigureAwait(false);
+
+ return await state.GetOrAddAsync(
+ CandidateBatchKey.Create(type, scope),
+ () => LoadCandidatesAsync(type, scope, cancellationToken)).ConfigureAwait(false);
+ }
+
+ private async Task> LoadCandidatesAsync(
+ string type,
+ MemoryScope? scope,
+ CancellationToken cancellationToken)
+ {
+ // Candidate reads stay owner-scoped. Type-strict=false retains the historical best-effort
+ // GetByType behavior because the repository has no unfiltered GetAll contract.
+ return await _entityRepository.GetByTypeAsync(type, scope, cancellationToken).ConfigureAwait(false);
+ }
+
+ private void EndBatch(CandidateBatchState state)
+ {
+ if (!ReferenceEquals(_candidateBatch.Value, state))
+ return;
+ state.Dispose();
+ _candidateBatch.Value = null;
+ }
+
+ private readonly record struct CandidateBatchKey(
+ string Type,
+ string? OwnerId,
+ bool IncludeShared)
+ {
+ public static CandidateBatchKey Create(string type, MemoryScope? scope) =>
+ new(type.ToUpperInvariant(), scope?.OwnerId, scope?.IncludeShared ?? true);
+ }
+
+ private sealed class CandidateBatchState : IDisposable
+ {
+ private readonly object _gate = new();
+ private readonly Dictionary>> _snapshots = [];
+ private bool _disposed;
+
+ public Task> GetOrAddAsync(
+ CandidateBatchKey key,
+ Func>> loader)
+ {
+ lock (_gate)
+ {
+ ObjectDisposedException.ThrowIf(_disposed, this);
+ if (_snapshots.TryGetValue(key, out var existing))
+ return existing;
+ var created = LoadAsync(loader);
+ _snapshots.Add(key, created);
+ return created;
+ }
+ }
+
+ public void Remember(CandidateBatchKey key, Entity entity)
+ {
+ lock (_gate)
+ {
+ if (_disposed || !_snapshots.TryGetValue(key, out var snapshot) ||
+ !snapshot.IsCompletedSuccessfully)
+ return;
+
+ var candidates = snapshot.Result;
+ var index = candidates.FindIndex(candidate => candidate.EntityId == entity.EntityId);
+ if (index >= 0)
+ candidates[index] = entity;
+ else
+ candidates.Add(entity);
+ }
+ }
+
+ public void Invalidate()
+ {
+ lock (_gate)
+ _snapshots.Clear();
+ }
+
+ public void Dispose()
+ {
+ lock (_gate)
+ {
+ _disposed = true;
+ _snapshots.Clear();
+ }
+ }
+
+ private static async Task> LoadAsync(Func>> loader) =>
+ (await loader().ConfigureAwait(false)).ToList();
+ }
+
+ private sealed class CandidateBatchLease(
+ CompositeEntityResolver owner,
+ CandidateBatchState state) : IDisposable
+ {
+ private CompositeEntityResolver? _owner = owner;
+
+ public void Dispose() => Interlocked.Exchange(ref _owner, null)?.EndBatch(state);
+ }
+
+ private sealed class NoopBatchLease : IDisposable
+ {
+ public static NoopBatchLease Instance { get; } = new();
+ public void Dispose() { }
+ }
+}
diff --git a/src/AgentMemory.Core/Resolution/CompositeEntityResolver.cs b/src/AgentMemory.Core/Resolution/CompositeEntityResolver.cs
index 44fcf3e3..a15c8b9f 100644
--- a/src/AgentMemory.Core/Resolution/CompositeEntityResolver.cs
+++ b/src/AgentMemory.Core/Resolution/CompositeEntityResolver.cs
@@ -21,7 +21,7 @@ namespace AgentMemory.Core.Resolution;
/// entity — this is intentional "shared knowledge grows collaboratively" behavior, not a cross-owner
/// leak (a future opt-in option could make shared knowledge read-only per owner if a deployment needs it).
///
-internal sealed class CompositeEntityResolver : IEntityResolver
+internal sealed partial class CompositeEntityResolver : IEntityResolver, IExtractionEntityResolver
{
private readonly IEntityRepository _entityRepository;
private readonly IEmbeddingOrchestrator _embeddingOrchestrator;
@@ -49,12 +49,33 @@ public CompositeEntityResolver(
_logger = logger;
}
- ///
- public async Task ResolveEntityAsync(
+ public Task ResolveEntityAsync(
ExtractedEntity extractedEntity,
IReadOnlyList sourceMessageIds,
MemoryScope? scope = null,
- CancellationToken cancellationToken = default)
+ CancellationToken cancellationToken = default) =>
+ ResolveAndRememberAsync(
+ extractedEntity, sourceMessageIds, scope, persistResolution: true, cancellationToken);
+
+ public Task ResolveForPersistenceAsync(
+ ExtractedEntity extractedEntity,
+ IReadOnlyList sourceMessageIds,
+ MemoryScope? scope = null,
+ CancellationToken cancellationToken = default) =>
+ ResolveAndRememberAsync(
+ extractedEntity, sourceMessageIds, scope, persistResolution: false, cancellationToken);
+
+ ///
+ /// Resolves an entity either for a direct caller (preserving the historical persist-on-resolve
+ /// behavior) or for fail-fast ExtractionStage, which must remain side-effect free until
+ /// PersistenceStage opens the logical transaction.
+ ///
+ private async Task ResolveEntityCoreAsync(
+ ExtractedEntity extractedEntity,
+ IReadOnlyList sourceMessageIds,
+ MemoryScope? scope,
+ bool persistResolution,
+ CancellationToken cancellationToken)
{
var candidates = await GetCandidatesAsync(extractedEntity.Type, scope, cancellationToken)
.ConfigureAwait(false);
@@ -77,7 +98,7 @@ public async Task ResolveEntityAsync(
}
if (resolutionResult is null)
- return await CreateNewEntityAsync(extractedEntity, sourceMessageIds, scope, cancellationToken)
+ return await CreateNewEntityAsync(extractedEntity, sourceMessageIds, scope, persistResolution, cancellationToken)
.ConfigureAwait(false);
var matched = resolutionResult.ResolvedEntity;
@@ -117,8 +138,9 @@ public async Task ResolveEntityAsync(
mergedEntity = mergedEntity with { Embedding = freshEmbedding };
}
- return await _entityRepository.UpsertAsync(mergedEntity, cancellationToken)
- .ConfigureAwait(false);
+ return persistResolution
+ ? await _entityRepository.UpsertAsync(mergedEntity, cancellationToken).ConfigureAwait(false)
+ : mergedEntity;
}
// >= SameAsThreshold and < AutoMergeThreshold: flag for SAME_AS — caller handles relationship
@@ -136,7 +158,7 @@ public async Task ResolveEntityAsync(
"No match above SameAs threshold for '{Name}' — creating new entity.",
extractedEntity.Name);
- return await CreateNewEntityAsync(extractedEntity, sourceMessageIds, scope, cancellationToken)
+ return await CreateNewEntityAsync(extractedEntity, sourceMessageIds, scope, persistResolution, cancellationToken)
.ConfigureAwait(false);
}
@@ -164,22 +186,11 @@ public async Task> FindPotentialDuplicatesAsync(
return results;
}
- private async Task> GetCandidatesAsync(
+ private Task> GetCandidatesAsync(
string type,
MemoryScope? scope,
- CancellationToken cancellationToken)
- {
- // The candidate set MUST be owner-scoped (R1): without it, an incoming entity could match and
- // auto-merge onto another owner's private entity (a cross-owner write-path leak). A null scope
- // (single-tenant / no owner context) preserves the legacy unscoped behavior.
- if (_options.EntityResolution.TypeStrictFiltering)
- return await _entityRepository.GetByTypeAsync(type, scope, cancellationToken).ConfigureAwait(false);
-
- // Without type filtering, SearchByVectorAsync is impractical here without an embedding;
- // GetByTypeAsync with empty type returns all in many impls, so we fall back gracefully.
- // For a complete impl, a GetAllAsync method would be ideal — use GetByTypeAsync("") as best effort.
- return await _entityRepository.GetByTypeAsync(type, scope, cancellationToken).ConfigureAwait(false);
- }
+ CancellationToken cancellationToken) =>
+ GetBatchCandidatesAsync(type, scope, cancellationToken);
private IReadOnlyList BuildMatchers()
{
@@ -202,6 +213,7 @@ private async Task CreateNewEntityAsync(
ExtractedEntity extracted,
IReadOnlyList sourceMessageIds,
MemoryScope? scope,
+ bool persistResolution,
CancellationToken cancellationToken)
{
var entity = new Entity
@@ -223,6 +235,8 @@ private async Task CreateNewEntityAsync(
CreatedAtUtc = _clock.UtcNow
};
- return await _entityRepository.UpsertAsync(entity, cancellationToken).ConfigureAwait(false);
+ return persistResolution
+ ? await _entityRepository.UpsertAsync(entity, cancellationToken).ConfigureAwait(false)
+ : entity;
}
}
diff --git a/src/AgentMemory.Core/Resolution/IExtractionEntityResolver.cs b/src/AgentMemory.Core/Resolution/IExtractionEntityResolver.cs
new file mode 100644
index 00000000..32004dea
--- /dev/null
+++ b/src/AgentMemory.Core/Resolution/IExtractionEntityResolver.cs
@@ -0,0 +1,22 @@
+using AgentMemory.Abstractions.Domain;
+using AgentMemory.Abstractions.Options;
+
+namespace AgentMemory.Core.Resolution;
+
+internal interface IExtractionEntityResolver
+{
+ IDisposable BeginBatch();
+
+ Task PrepareCandidatesAsync(
+ IReadOnlyCollection entityTypes,
+ MemoryScope? scope = null,
+ CancellationToken cancellationToken = default);
+
+ void InvalidateBatch();
+
+ Task ResolveForPersistenceAsync(
+ ExtractedEntity extractedEntity,
+ IReadOnlyList sourceMessageIds,
+ MemoryScope? scope = null,
+ CancellationToken cancellationToken = default);
+}
diff --git a/src/AgentMemory.Core/ServiceCollectionExtensions.cs b/src/AgentMemory.Core/ServiceCollectionExtensions.cs
index 5a13c05b..62f51cbe 100644
--- a/src/AgentMemory.Core/ServiceCollectionExtensions.cs
+++ b/src/AgentMemory.Core/ServiceCollectionExtensions.cs
@@ -17,6 +17,71 @@ namespace AgentMemory.Core;
///
public static class ServiceCollectionExtensions
{
+ ///
+ /// Registers all Core memory services from a fully-constructed .
+ ///
+ ///
+ /// The Action<MemoryOptions> overload cannot configure anything.
+ /// is a record whose properties are all init-only, so a
+ /// configure lambda cannot assign them — options.EnableGraphRag = true is a compile error,
+ /// and the options = options with { ... } form that does compile rebinds the parameter
+ /// local and is discarded the moment the lambda returns. Code written that way builds, runs, and
+ /// silently keeps every default.
+ ///
+ /// This overload takes the instance directly, which is how every working call site in this
+ /// repository already builds options, and applies the same validators the other overload
+ /// registers — a supplied instance is checked, not trusted.
+ ///
+ ///
+ /// Added rather than changing the properties to set: that would alter the setter signature
+ /// of a public type on a SemVer-locked surface and break binary compatibility for anyone already
+ /// compiled against it. This is purely additive.
+ ///
+ ///
+ /// Limitation, stated rather than left to be discovered. This replaces the registration of
+ /// only. A host resolving IOptionsMonitor<MemoryOptions>
+ /// or IOptionsSnapshot<MemoryOptions> would still go through the options factory and
+ /// receive defaults. Nothing in this product resolves either — every consumer takes
+ /// IOptions<MemoryOptions> — but a host that does would see two different values for
+ /// the same options type, which is worth knowing before it happens rather than after.
+ ///
+ ///
+ public static IServiceCollection AddAgentMemoryCore(
+ this IServiceCollection services,
+ MemoryOptions options)
+ {
+ ArgumentNullException.ThrowIfNull(services);
+ ArgumentNullException.ThrowIfNull(options);
+
+ // Register everything the lambda overload does, including the validator chain, then take over
+ // the resolution of IOptions. An exact closed-generic registration wins over
+ // the open IOptions<> one, so the supplied instance is what every consumer resolves.
+ services.AddAgentMemoryCore(_ => { });
+ services.AddSingleton>(serviceProvider =>
+ {
+ // The validators are the whole reason this is a factory rather than Options.Create: an
+ // instance that skipped them would fail closed nowhere and misconfigure silently at the
+ // first affected call, which is the exact failure the validator chain exists to prevent.
+ var failures = serviceProvider
+ .GetServices>()
+ .Select(validator => validator.Validate(Microsoft.Extensions.Options.Options.DefaultName, options))
+ .Where(result => result.Failed)
+ // Failures is only guaranteed non-null once Failed is true, which the Where above
+ // establishes but the compiler cannot see.
+ .SelectMany(result => result.Failures ?? [])
+ .ToArray();
+ if (failures.Length > 0)
+ {
+ throw new OptionsValidationException(
+ Microsoft.Extensions.Options.Options.DefaultName, typeof(MemoryOptions), failures);
+ }
+
+ return Microsoft.Extensions.Options.Options.Create(options);
+ });
+
+ return services;
+ }
+
///
/// Registers all Core memory services.
/// Adapters (repositories, IEmbeddingGenerator, etc.) must be registered separately.
@@ -168,6 +233,7 @@ public static IServiceCollection AddAgentMemoryCore(
// Extraction pipeline stages.
// IExtractionStage receives IEnumerable — all registered extractor implementations.
+ services.TryAddSingleton();
services.TryAddScoped();
services.TryAddScoped();
@@ -185,7 +251,8 @@ public static IServiceCollection AddAgentMemoryCore(
sp.GetRequiredService(),
sp.GetRequiredService>(),
sp.GetRequiredService(),
- sp.GetService>()));
+ sp.GetService>(),
+ sp.GetServices()));
// Embedding orchestrator — centralizes embedding generation logic.
services.TryAddScoped();
diff --git a/src/AgentMemory.Core/Services/EmbeddingOrchestrator.cs b/src/AgentMemory.Core/Services/EmbeddingOrchestrator.cs
index 001732c3..1c19c61a 100644
--- a/src/AgentMemory.Core/Services/EmbeddingOrchestrator.cs
+++ b/src/AgentMemory.Core/Services/EmbeddingOrchestrator.cs
@@ -36,7 +36,7 @@ public async Task EmbedAsync(string text, CancellationToken cancellatio
try
{
- var result = await _generator.GenerateAsync([text], cancellationToken: cancellationToken).ConfigureAwait(false);
+ var result = await _generator.GenerateAsync([CapInputLength(text)], cancellationToken: cancellationToken).ConfigureAwait(false);
return result[0].Vector.ToArray();
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
@@ -70,7 +70,7 @@ public async Task> EmbedBatchAsync(IReadOnlyList
if (!string.IsNullOrWhiteSpace(texts[i]))
{
nonBlankIndices.Add(i);
- nonBlankTexts.Add(texts[i]);
+ nonBlankTexts.Add(CapInputLength(texts[i]));
}
}
@@ -97,10 +97,46 @@ public async Task> EmbedBatchAsync(IReadOnlyList
}
catch (Exception ex)
{
- _logger.LogWarning(ex, "Batch embedding generation failed for {Count} texts; returning empty vectors.", nonBlankTexts.Count);
- // results already initialized to empty vectors — positional alignment preserved.
+ // One unusable input fails the whole provider call, so leaving every slot empty lets a
+ // single bad message silently blank an entire batch — which is exactly how one
+ // 41,855-character turn made all 596 messages of a LongMemEval question unsearchable.
+ // Retry item by item so the blast radius is the offending input, not its neighbours.
+ _logger.LogWarning(
+ ex,
+ "Batch embedding generation failed for {Count} texts; retrying individually so one bad input cannot blank the rest.",
+ nonBlankTexts.Count);
+
+ for (int j = 0; j < nonBlankIndices.Count; j++)
+ {
+ results[nonBlankIndices[j]] = await EmbedAsync(nonBlankTexts[j], cancellationToken)
+ .ConfigureAwait(false);
+ }
}
return results;
}
+
+ ///
+ /// Caps embedding input length. Embedding models reject inputs beyond a fixed token budget
+ /// (ada-002 allows 8,191), and the provider rejects the whole request when a single
+ /// input exceeds it. Stored content is never truncated — only the text handed to the embedder —
+ /// so an unusually long message stays complete and remains searchable.
+ ///
+ ///
+ /// Deliberately conservative and character-based: an exact tokenizer is not available at this
+ /// abstraction boundary, and ~4 characters per token is a safe upper bound for the models in use.
+ ///
+ private const int MaxEmbeddingInputCharacters = 32_000;
+
+ private string CapInputLength(string text)
+ {
+ if (text.Length <= MaxEmbeddingInputCharacters)
+ return text;
+
+ _logger.LogWarning(
+ "Embedding input of {Length} characters exceeds the {Cap}-character cap; embedding its leading section. Stored content is unchanged.",
+ text.Length,
+ MaxEmbeddingInputCharacters);
+ return text[..MaxEmbeddingInputCharacters];
+ }
}
diff --git a/src/AgentMemory.Core/Services/LongTermMemoryService.cs b/src/AgentMemory.Core/Services/LongTermMemoryService.cs
index 463aa47a..d5bda3b9 100644
--- a/src/AgentMemory.Core/Services/LongTermMemoryService.cs
+++ b/src/AgentMemory.Core/Services/LongTermMemoryService.cs
@@ -1,6 +1,7 @@
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using AgentMemory.Abstractions.Domain;
+using AgentMemory.Core.Memory;
using AgentMemory.Abstractions.Options;
using AgentMemory.Abstractions.Repositories;
using AgentMemory.Abstractions.Services;
@@ -12,6 +13,10 @@ namespace AgentMemory.Core.Services;
///
internal sealed class LongTermMemoryService : ILongTermMemoryService
{
+ private const int FactDedupLockStripeCount = 256;
+ private static readonly SemaphoreSlim[] FactDedupLockStripes =
+ Enumerable.Range(0, FactDedupLockStripeCount).Select(_ => new SemaphoreSlim(1, 1)).ToArray();
+
private readonly IEntityRepository _entityRepo;
private readonly IFactRepository _factRepo;
private readonly IPreferenceRepository _prefRepo;
@@ -232,7 +237,17 @@ public async Task AddFactAsync(
// (zero-dimension) vector, which would otherwise be handed to db.index.vector.queryNodes and throw a
// dimension mismatch — aborting the whole add. An empty embedding has no semantic signal, so skip
// dedup and fall through to a plain create (the node persists with a NULL, re-queueable embedding).
- if (_options.DeduplicateOnCreate && embedding is { Length: > 0 })
+ var toSave = embedding is null ? fact : fact with { Embedding = embedding };
+ if (!_options.DeduplicateOnCreate || embedding is not { Length: > 0 })
+ return await _factRepo.UpsertAsync(toSave, cancellationToken).ConfigureAwait(false);
+
+ // Find + reinforce/create is otherwise a TOCTOU race across concurrent request scopes. A bounded
+ // process-wide stripe set serializes the same owner + case-insensitive subject/predicate key without
+ // retaining one lock per memory forever. This deliberately provides in-process session correctness;
+ // it does not claim distributed coordination between separate application instances.
+ var dedupLock = FactDedupLock(toSave);
+ await dedupLock.WaitAsync(cancellationToken).ConfigureAwait(false);
+ try
{
var dup = await _factRepo.FindDuplicateAsync(
fact.Subject, fact.Predicate, embedding, fact.OwnerId,
@@ -251,10 +266,22 @@ public async Task AddFactAsync(
// create the new node instead of failing the add.
_logger.LogDebug("Dedup target fact {Id} vanished before reinforce; creating new node.", dup.FactId);
}
+
+ return await _factRepo.UpsertAsync(toSave, cancellationToken).ConfigureAwait(false);
}
+ finally
+ {
+ dedupLock.Release();
+ }
+ }
- var toSave = embedding is null ? fact : fact with { Embedding = embedding };
- return await _factRepo.UpsertAsync(toSave, cancellationToken).ConfigureAwait(false);
+ private static SemaphoreSlim FactDedupLock(Fact fact)
+ {
+ var hash = new HashCode();
+ hash.Add(fact.OwnerId, StringComparer.Ordinal);
+ hash.Add(fact.Subject, StringComparer.OrdinalIgnoreCase);
+ hash.Add(fact.Predicate, StringComparer.OrdinalIgnoreCase);
+ return FactDedupLockStripes[(uint)hash.ToHashCode() % FactDedupLockStripeCount];
}
/// Reinforced confidence on a dedup hit: max(existing, incoming) + configured bump, capped at 1.0.
@@ -292,15 +319,91 @@ public Task> GetFactsBySubjectAsync(
}
///
- public async Task> SearchFactsAsync(
+ public Task> SearchFactsAsync(
float[] queryEmbedding,
int limit = 10,
double minScore = 0.0,
MemoryScope? scope = null,
- CancellationToken cancellationToken = default)
+ CancellationToken cancellationToken = default) =>
+ SearchFactsAsync(
+ queryEmbedding, limit, minScore, scope, false, 0, cancellationToken);
+
+ ///
+ /// Fact recall with optional canonical-predicate expansion (G5 "hard" tier).
+ ///
+ ///
+ /// A separate overload rather than optional parameters on the interface method: adding optional
+ /// parameters to a published interface breaks every implementor, and the interface is locked
+ /// under SemVer.
+ ///
+ public Task> SearchFactsAsync(
+ float[] queryEmbedding,
+ int limit,
+ double minScore,
+ MemoryScope? scope,
+ bool expandByPredicate,
+ int expansionLimit,
+ CancellationToken cancellationToken) =>
+ SearchFactsAsync(
+ queryEmbedding, limit, minScore, scope, expandByPredicate, expansionLimit,
+ Array.Empty(), cancellationToken);
+
+ ///
+ public async Task> SearchFactsAsync(
+ float[] queryEmbedding,
+ int limit,
+ double minScore,
+ MemoryScope? scope,
+ bool expandByPredicate,
+ int expansionLimit,
+ IReadOnlyList questionRelations,
+ CancellationToken cancellationToken)
{
- var scored = await _factRepo.SearchByVectorAsync(queryEmbedding, limit, minScore, Resolve(scope, nameof(SearchFactsAsync)), cancellationToken).ConfigureAwait(false);
- return scored.Select(r => r.Fact).ToList();
+ ArgumentNullException.ThrowIfNull(questionRelations);
+ var resolved = Resolve(scope, nameof(SearchFactsAsync));
+ var scored = await _factRepo.SearchByVectorAsync(queryEmbedding, limit, minScore, resolved, cancellationToken).ConfigureAwait(false);
+ var top = scored.Select(r => r.Fact).ToList();
+ // A question that names its relations outright does not need the top-K to nominate them, so an
+ // empty top-K is only a dead end when there is nothing else to expand on.
+ if (!expandByPredicate || (top.Count == 0 && questionRelations.Count == 0))
+ return top;
+
+ // G5 "hard" tier. Similarity decides *which* relation matters; this returns that relation
+ // whole. Top-K is a relevance cutoff and carries no completeness guarantee, so a question
+ // like "how many babies were born" is unanswerable from it - miss one of five and the count
+ // is four. Expansion is additive: the similarity-ranked facts stay, in order, at the front.
+ var predicates = top
+ .Select(fact => MemoryTripleCanonicalizer.Canonical(fact.Predicate))
+ // J2.2. Relations the question named, each widened to every form it could be stored under:
+ // the write-side canonicalizer never folds morphology, so one relation lives under several
+ // keys and expanding only the canonical name would miss the smaller buckets.
+ .Concat(questionRelations.SelectMany(
+ relation => MemoryRelationLexicon.Default.StoredFormsOf(relation)))
+ .Where(predicate => predicate.Length > 0)
+ .Distinct(StringComparer.Ordinal)
+ .ToArray();
+ var expanded = await _factRepo.SearchByCanonicalPredicatesAsync(
+ predicates, expansionLimit, resolved, cancellationToken).ConfigureAwait(false);
+
+ var seen = top.Select(fact => fact.FactId).ToHashSet(StringComparer.Ordinal);
+ foreach (var fact in expanded)
+ {
+ if (!seen.Add(fact.FactId))
+ continue;
+
+ // Marked because expansion returns a relation across the whole owner, so a fact may
+ // legitimately carry provenance outside the current query's window. A consumer
+ // resolving provenance must be able to tell that apart from a source that genuinely
+ // cannot be resolved, which is corruption — marking the former keeps the latter
+ // detectable rather than silencing both.
+ var metadata = new Dictionary(fact.Metadata, StringComparer.Ordinal)
+ {
+ [Fact.RetrievalSourceMetadataKey] = Fact.RetrievalSourcePredicateExpansion
+ };
+ top.Add(fact with { Metadata = metadata });
+ }
+
+ return top;
}
///
diff --git a/src/AgentMemory.Core/Services/MemoryContextAssembler.cs b/src/AgentMemory.Core/Services/MemoryContextAssembler.cs
index 5efc2fb1..4c1bcd12 100644
--- a/src/AgentMemory.Core/Services/MemoryContextAssembler.cs
+++ b/src/AgentMemory.Core/Services/MemoryContextAssembler.cs
@@ -5,6 +5,7 @@
using AgentMemory.Abstractions.Exceptions;
using AgentMemory.Abstractions.Options;
using AgentMemory.Abstractions.Services;
+using AgentMemory.Core.Memory;
using AgentMemory.Core.Services.Budgeting;
namespace AgentMemory.Core.Services;
@@ -156,11 +157,23 @@ public async Task AssembleContextAsync(
IReadOnlyList recentMessages = Array.Empty();
IReadOnlyList relevantMessages = Array.Empty();
+ IReadOnlyList<(Message Message, double Score)> relevantMessageScores =
+ Array.Empty<(Message, double)>();
IReadOnlyList entities = Array.Empty();
IReadOnlyList preferences = Array.Empty();
IReadOnlyList facts = Array.Empty();
IReadOnlyList traces = Array.Empty();
+ // J2.2 resolution, computed once at method scope so it can be both used by the fact search
+ // and reported on the context. Which relations a question resolved to is the difference
+ // between "expansion had nothing to expand" and "expansion ran and did not help", and those
+ // need opposite responses. It was computed inline and discarded, so no report could tell
+ // them apart.
+ var resolvedQueryRelations = recallOpts.ExpandFactsByPredicate &&
+ recallOpts.ResolveQueryRelations
+ ? MemoryRelationLexicon.Default.ResolveQuestion(request.Query)
+ : Array.Empty();
+
if (includeMemory)
{
// Generate embedding if not provided (only needed for memory-layer semantic search).
@@ -193,8 +206,14 @@ public async Task AssembleContextAsync(
var relevantTask = hasEmbedding && recallOpts.MaxRelevantMessages > 0
? TimedAsync("memory.recall.messages",
- () => _shortTerm.SearchMessagesAsync(request.SessionId, queryEmbedding, recallOpts.MaxRelevantMessages, minScore, cancellationToken))
- : Empty();
+ () => SearchRelevantMessagesAsync(
+ request.SessionId,
+ queryEmbedding,
+ recallOpts.MaxRelevantMessages,
+ minScore,
+ recallOpts.IncludeDiagnostics,
+ cancellationToken))
+ : Task.FromResult(RelevantMessageSearchResult.Empty);
// D3 — apply the per-request query intent (latest/analog) as an ambient ranking override for
// the long-term vector searches below. The long-term repositories read it synchronously while
@@ -215,12 +234,38 @@ public async Task AssembleContextAsync(
var factsTask = hasEmbedding && recallOpts.MaxFacts > 0
? TimedAsync("memory.recall.facts",
- () => _longTerm.SearchFactsAsync(queryEmbedding, recallOpts.MaxFacts, minScore, scope, cancellationToken))
+ // Only the expansion path takes the wider overload. Off by default, the call is
+ // byte-for-byte the original, so no existing behaviour or contract shifts.
+ // Hoisted out of the call so the decision is observable. Which relations a
+ // question resolved to is the difference between "expansion had nothing to
+ // expand" and "expansion ran and did not help", and the two need opposite
+ // responses. It was computed inline and discarded, so no report could tell them
+ // apart - a multi-session question failing because its verb is absent from the
+ // table looked identical to one failing for any other reason.
+ () => recallOpts.ExpandFactsByPredicate
+ ? _longTerm.SearchFactsAsync(
+ queryEmbedding, recallOpts.MaxFacts, minScore, scope,
+ true, recallOpts.MaxExpandedFacts,
+ // J2.2. Empty unless explicitly enabled, and an unrecognised verb resolves
+ // to nothing, so both the option-off and the no-match paths reproduce the
+ // previous call exactly.
+ resolvedQueryRelations,
+ cancellationToken)
+ : _longTerm.SearchFactsAsync(
+ queryEmbedding, recallOpts.MaxFacts, minScore, scope, cancellationToken))
: Empty();
var tracesTask = hasEmbedding && recallOpts.MaxTraces > 0
? TimedAsync("memory.recall.traces",
- () => _reasoning.SearchSimilarTracesAsync(queryEmbedding, null, recallOpts.MaxTraces, minScore, scope, cancellationToken))
+ () => _reasoning.SearchSimilarTracesAsync(
+ queryEmbedding,
+ // K5: was a hardcoded null, so the outcome filter the repository and its
+ // Cypher already supported could never be reached from automatic recall.
+ // A recalled trace is shown to the reader as precedent with nothing marking
+ // it as a failure, so imitating reasoning that did not work is worse than
+ // recalling nothing. Default stays null - today's behaviour - until measured.
+ recallOpts.SuccessfulTracesOnly,
+ recallOpts.MaxTraces, minScore, scope, cancellationToken))
: Empty();
if (overrideRanking) _rankingContext!.Current = null;
@@ -230,7 +275,9 @@ await Task.WhenAll(
preferencesTask, factsTask, tracesTask).ConfigureAwait(false);
recentMessages = await recentTask.ConfigureAwait(false);
- relevantMessages = await relevantTask.ConfigureAwait(false);
+ var relevantResult = await relevantTask.ConfigureAwait(false);
+ relevantMessages = relevantResult.Messages;
+ relevantMessageScores = relevantResult.ScoredMessages;
entities = await entitiesTask.ConfigureAwait(false);
preferences = await preferencesTask.ConfigureAwait(false);
facts = await factsTask.ConfigureAwait(false);
@@ -241,11 +288,18 @@ await Task.WhenAll(
await graphRagTask.ConfigureAwait(false);
string? graphRagContext = null;
+ // K4: the items are retained, not only their joined text. They already carry SourceNodeIds,
+ // Score and Metadata, and discarding them here is what left GraphRAG unattributable, and so
+ // unmeasurable, in every quality run to date. The joined string is byte-identical to before.
+ IReadOnlyList graphRagItems = Array.Empty();
if (graphRagTask != null)
{
var graphRagResult = await graphRagTask.ConfigureAwait(false);
if (graphRagResult?.Items is { Count: > 0 } items)
+ {
graphRagContext = string.Join("\n\n", items.Select(i => i.Text));
+ graphRagItems = items;
+ }
}
// Apply context budget if configured
@@ -266,17 +320,27 @@ await Task.WhenAll(
+ ContextBudgetEstimator.EstimateChars(traces)
+ (graphRagContext?.Length ?? 0);
+ var rankedRelevantItems = recallOpts.IncludeDiagnostics
+ ? BuildRankedItems(relevantMessages, relevantMessageScores)
+ : Array.Empty();
+
var context = new MemoryContext
{
SessionId = request.SessionId,
AssembledAtUtc = _clock.UtcNow,
RecentMessages = new MemoryContextSection { Items = recentMessages },
- RelevantMessages = new MemoryContextSection { Items = relevantMessages },
+ RelevantMessages = new MemoryContextSection
+ {
+ Items = relevantMessages,
+ RankedItems = rankedRelevantItems
+ },
RelevantEntities = new MemoryContextSection { Items = entities },
RelevantPreferences = new MemoryContextSection { Items = preferences },
RelevantFacts = new MemoryContextSection { Items = facts },
SimilarTraces = new MemoryContextSection { Items = traces },
GraphRagContext = graphRagContext,
+ GraphRagItems = graphRagItems,
+ ResolvedQueryRelations = resolvedQueryRelations,
BlendMode = blendMode,
Truncated = truncated
};
@@ -463,6 +527,69 @@ private static async Task TimedAsync(string spanName, Func> factor
}
}
+ private async Task SearchRelevantMessagesAsync(
+ string sessionId,
+ float[] queryEmbedding,
+ int limit,
+ double minScore,
+ bool includeDiagnostics,
+ CancellationToken cancellationToken)
+ {
+ if (includeDiagnostics && _shortTerm is IScoredMessageSearch scoredSearch)
+ {
+ var scoredMessages = await scoredSearch.SearchMessagesWithScoresAsync(
+ sessionId, queryEmbedding, limit, minScore, cancellationToken).ConfigureAwait(false);
+ return new RelevantMessageSearchResult(
+ scoredMessages.Select(result => result.Message).ToArray(),
+ scoredMessages);
+ }
+
+ var messages = await _shortTerm.SearchMessagesAsync(
+ sessionId, queryEmbedding, limit, minScore, cancellationToken).ConfigureAwait(false);
+ return new RelevantMessageSearchResult(messages, Array.Empty<(Message, double)>());
+ }
+
+ internal static IReadOnlyList BuildRankedItems(
+ IReadOnlyList contextMessages,
+ IReadOnlyList<(Message Message, double Score)> retrievedMessages)
+ {
+ if (contextMessages.Count == 0 || retrievedMessages.Count == 0)
+ return Array.Empty();
+
+ var retrievedById = retrievedMessages
+ .Select((result, index) => new
+ {
+ result.Message.MessageId,
+ result.Score,
+ RetrievalRank = index + 1
+ })
+ .ToDictionary(result => result.MessageId, StringComparer.Ordinal);
+
+ var ranked = new List(contextMessages.Count);
+ for (var index = 0; index < contextMessages.Count; index++)
+ {
+ var message = contextMessages[index];
+ if (!retrievedById.TryGetValue(message.MessageId, out var retrieved))
+ continue;
+ ranked.Add(new MemoryContextRankedItem(
+ message.MessageId,
+ retrieved.Score,
+ retrieved.RetrievalRank,
+ ContextRank: index + 1));
+ }
+
+ return ranked.AsReadOnly();
+ }
+
+ private sealed record RelevantMessageSearchResult(
+ IReadOnlyList Messages,
+ IReadOnlyList<(Message Message, double Score)> ScoredMessages)
+ {
+ public static RelevantMessageSearchResult Empty { get; } = new(
+ Array.Empty(),
+ Array.Empty<(Message, double)>());
+ }
+
private sealed record AssembledSections(
IReadOnlyList Recent,
IReadOnlyList Relevant,
diff --git a/src/AgentMemory.Core/Services/MemoryDecayService.cs b/src/AgentMemory.Core/Services/MemoryDecayService.cs
index 2c3081a7..3e7ca06d 100644
--- a/src/AgentMemory.Core/Services/MemoryDecayService.cs
+++ b/src/AgentMemory.Core/Services/MemoryDecayService.cs
@@ -92,8 +92,16 @@ internal double ComputeScore(
double daysSinceAccess = Math.Max(0, (now - reference).TotalDays);
double lambda = Math.Log(2) / _options.DecayHalfLifeDays;
- return confidence * Math.Exp(-lambda * daysSinceAccess)
- + _options.AccessBoostFactor * accessCount;
+ // BUG-R7. The access term was linear, unbounded, and — unlike confidence — never decayed, so
+ // access_count alone decided retention: 10,000 accesses scored 2,000 against a [0,1] cosine
+ // blend, and a single recall reached 0.2, permanently above MinRetentionScore (0.1) however
+ // stale the memory became. Damp it logarithmically, cap its contribution, and decay it on the
+ // same curve as confidence, so frequent access slows forgetting instead of preventing it.
+ double boost = Math.Min(
+ _options.AccessBoostFactor * Math.Log(1 + Math.Max(0, accessCount)),
+ _options.MaxAccessBoost);
+
+ return (confidence + boost) * Math.Exp(-lambda * daysSinceAccess);
}
///
diff --git a/src/AgentMemory.Core/Services/MemoryExtractionPipeline.Batch.cs b/src/AgentMemory.Core/Services/MemoryExtractionPipeline.Batch.cs
new file mode 100644
index 00000000..ea7c5635
--- /dev/null
+++ b/src/AgentMemory.Core/Services/MemoryExtractionPipeline.Batch.cs
@@ -0,0 +1,109 @@
+using System.Diagnostics;
+using AgentMemory.Abstractions.Domain;
+
+namespace AgentMemory.Core.Services;
+
+internal sealed partial class MemoryExtractionPipeline
+{
+ public async Task> ExtractBatchAsync(
+ IReadOnlyList requests,
+ int maxSessionsPerBatch,
+ int maxInputTokens,
+ CancellationToken cancellationToken = default)
+ {
+ ArgumentNullException.ThrowIfNull(requests);
+ ArgumentOutOfRangeException.ThrowIfNegativeOrZero(maxSessionsPerBatch);
+ ArgumentOutOfRangeException.ThrowIfNegativeOrZero(maxInputTokens);
+ if (requests.Count == 0)
+ return [];
+
+ var ordered = requests
+ .Select((request, index) => new
+ {
+ Request = request,
+ Index = index,
+ Timestamp = request.Messages
+ .Select(message => message.TimestampUtc)
+ .DefaultIfEmpty(DateTimeOffset.MinValue)
+ .Min(),
+ })
+ .OrderBy(item => item.Timestamp)
+ .ThenBy(item => item.Index)
+ .Select(item => item.Request)
+ .ToArray();
+
+ var batchExtractor = _multiSessionExtractors.FirstOrDefault(extractor => extractor.IsEnabled);
+ if (batchExtractor is null)
+ {
+ var fallback = new List(ordered.Length);
+ foreach (var request in ordered)
+ fallback.Add(await ExtractAsync(request, cancellationToken).ConfigureAwait(false));
+ return fallback;
+ }
+
+ var extractedBySession = await batchExtractor.ExtractAsync(
+ ordered,
+ maxSessionsPerBatch,
+ maxInputTokens,
+ cancellationToken).ConfigureAwait(false);
+
+ var persisted = new List(ordered.Length);
+ using var resolutionBatch = _extractionStage.BeginResolutionBatch();
+ foreach (var request in ordered)
+ {
+ if (!extractedBySession.TryGetValue(request.SessionId, out var extracted))
+ throw new InvalidOperationException(
+ $"Validated batch output is missing source session '{request.SessionId}'.");
+
+ var sw = Stopwatch.StartNew();
+ var scope = _isolationPolicy.ResolveReadScope(
+ explicitScope: null,
+ request.UserId,
+ nameof(ExtractBatchAsync),
+ MemoryOperationAccess.Tenant);
+ var staged = await _extractionStage.ProcessUnifiedAsync(
+ request.Messages,
+ extracted,
+ request.TypesToExtract,
+ scope,
+ cancellationToken).ConfigureAwait(false);
+
+ var ownerId = _isolationPolicy.ResolveWriteOwner(
+ request.UserId,
+ nameof(ExtractBatchAsync),
+ MemoryOperationAccess.Tenant);
+ var trustLevel = request.TrustLevel ?? _options.DefaultTrustLevel;
+ var result = await _persistenceStage.PersistAsync(
+ staged,
+ ownerId,
+ trustLevel,
+ cancellationToken).ConfigureAwait(false);
+ sw.Stop();
+ if (result.Outcomes.Any(outcome => outcome.Status == IngestionItemStatus.Failed))
+ _extractionStage.InvalidateResolutionBatch();
+
+
+ persisted.Add(new ExtractionResult
+ {
+ Entities = staged.RawEntities,
+ Facts = staged.RawFacts,
+ Preferences = staged.RawPreferences,
+ Relationships = staged.RawRelationships,
+ SourceMessageIds = staged.SourceMessageIds,
+ Status = ComputeStatus(result.Outcomes),
+ Outcomes = result.Outcomes,
+ Metadata = new Dictionary
+ {
+ ["sessionId"] = request.SessionId,
+ ["postBatchProcessingTimeMs"] = sw.ElapsedMilliseconds,
+ ["entityCount"] = result.EntityCount,
+ ["factCount"] = result.FactCount,
+ ["preferenceCount"] = result.PreferenceCount,
+ ["relationshipCount"] = result.RelationshipCount,
+ },
+ });
+ }
+
+ return persisted;
+ }
+}
diff --git a/src/AgentMemory.Core/Services/MemoryExtractionPipeline.cs b/src/AgentMemory.Core/Services/MemoryExtractionPipeline.cs
index 42c09666..6fab1abe 100644
--- a/src/AgentMemory.Core/Services/MemoryExtractionPipeline.cs
+++ b/src/AgentMemory.Core/Services/MemoryExtractionPipeline.cs
@@ -13,13 +13,14 @@ namespace AgentMemory.Core.Services;
/// merge, filter, validate, resolve) followed by (embed, upsert,
/// wire provenance). Implements the public interface.
///
-internal sealed class MemoryExtractionPipeline : IMemoryExtractionPipeline
+internal sealed partial class MemoryExtractionPipeline : IMemoryExtractionPipeline
{
private readonly IExtractionStage _extractionStage;
private readonly IPersistenceStage _persistenceStage;
private readonly ILogger _logger;
private readonly IMemoryIsolationPolicy _isolationPolicy;
private readonly ExtractionOptions _options;
+ private readonly IReadOnlyList _multiSessionExtractors;
// Internal ctor: the stage interfaces are internal to Core, so this type is activated by an
// explicit factory in AddAgentMemoryCore (the default DI activator only selects public ctors).
@@ -28,13 +29,16 @@ internal MemoryExtractionPipeline(
IPersistenceStage persistenceStage,
ILogger logger,
IMemoryIsolationPolicy isolationPolicy,
- IOptions? extractionOptions = null)
+ IOptions? extractionOptions = null,
+ IEnumerable? multiSessionExtractors = null)
{
_extractionStage = extractionStage;
_persistenceStage = persistenceStage;
_logger = logger;
_isolationPolicy = isolationPolicy;
_options = extractionOptions?.Value ?? new ExtractionOptions();
+ _multiSessionExtractors = (multiSessionExtractors ?? [])
+ .ToList().AsReadOnly();
}
///
diff --git a/src/AgentMemory.Core/Services/PassThroughMemoryPersistenceTransaction.cs b/src/AgentMemory.Core/Services/PassThroughMemoryPersistenceTransaction.cs
new file mode 100644
index 00000000..61b010aa
--- /dev/null
+++ b/src/AgentMemory.Core/Services/PassThroughMemoryPersistenceTransaction.cs
@@ -0,0 +1,17 @@
+using AgentMemory.Core.Extraction;
+
+namespace AgentMemory.Core.Services;
+
+/// Portable fallback for stores that do not expose a transaction coordinator.
+internal sealed class PassThroughMemoryPersistenceTransaction : IMemoryPersistenceTransaction
+{
+ public bool SupportsAtomicRollback => false;
+
+ public Task ExecuteAsync(
+ Func> work,
+ CancellationToken cancellationToken = default)
+ {
+ cancellationToken.ThrowIfCancellationRequested();
+ return work(cancellationToken);
+ }
+}
diff --git a/src/AgentMemory.Core/Services/ShortTermMemoryService.cs b/src/AgentMemory.Core/Services/ShortTermMemoryService.cs
index 60b644e7..d89e0546 100644
--- a/src/AgentMemory.Core/Services/ShortTermMemoryService.cs
+++ b/src/AgentMemory.Core/Services/ShortTermMemoryService.cs
@@ -10,7 +10,7 @@ namespace AgentMemory.Core.Services;
///
/// Service for short-term (conversational) memory operations.
///
-internal sealed class ShortTermMemoryService : IShortTermMemoryService
+internal sealed class ShortTermMemoryService : IShortTermMemoryService, IScoredMessageSearch
{
private readonly IConversationRepository _conversationRepo;
private readonly IMessageRepository _messageRepo;
@@ -90,21 +90,90 @@ public async Task> AddMessagesAsync(
CancellationToken cancellationToken = default)
{
var messageList = messages.ToList();
- var results = new List(messageList.Count);
+ if (!_options.GenerateEmbeddings)
+ {
+ _logger.LogDebug("Batch adding {Count} messages", messageList.Count);
+ return await _messageRepo.AddBatchAsync(messageList, cancellationToken).ConfigureAwait(false);
+ }
- foreach (var message in messageList)
+ if (!_options.UseBatchEmbeddingRequests)
{
- var finalMessage = message;
- if (_options.GenerateEmbeddings && message.Embedding is null)
+ var legacyResults = new List(messageList.Count);
+ foreach (var message in messageList)
{
- var embedding = await _embeddingOrchestrator.EmbedMessageAsync(message.Content, cancellationToken).ConfigureAwait(false);
- finalMessage = message with { Embedding = embedding };
+ var finalMessage = message;
+ if (message.Embedding is null)
+ {
+ var embedding = await _embeddingOrchestrator
+ .EmbedMessageAsync(message.Content, cancellationToken)
+ .ConfigureAwait(false);
+ finalMessage = message with { Embedding = embedding };
+ }
+ legacyResults.Add(finalMessage);
+ }
+
+ _logger.LogDebug("Batch adding {Count} messages", legacyResults.Count);
+ return await _messageRepo.AddBatchAsync(legacyResults, cancellationToken).ConfigureAwait(false);
+ }
+
+ var missingIndices = Enumerable.Range(0, messageList.Count)
+ .Where(index => messageList[index].Embedding is null)
+ .ToArray();
+ if (missingIndices.Length > 0)
+ {
+ var texts = missingIndices.Select(index => messageList[index].Content).ToArray();
+ var embeddings = await _embeddingOrchestrator
+ .EmbedBatchAsync(texts, cancellationToken)
+ .ConfigureAwait(false);
+ if (embeddings.Count != missingIndices.Length)
+ {
+ throw new InvalidOperationException(
+ $"Batch embedding returned {embeddings.Count} vectors for " +
+ $"{missingIndices.Length} messages; positional alignment cannot be guaranteed.");
+ }
+
+ for (var index = 0; index < missingIndices.Length; index++)
+ {
+ var messageIndex = missingIndices[index];
+ var vector = embeddings[index];
+
+ // BUG-M1. The count check above cannot see an empty vector, and an empty list is not
+ // null — so such a message passes recall's `embedding IS NOT NULL` filter, then
+ // cosine yields null and the score comparison drops it. The message would be stored
+ // and permanently unretrievable with no error raised. Replay that slot alone, and if
+ // it still comes back unusable, fail the write: a message is the only copy of itself,
+ // so losing the write is recoverable and storing an unsearchable one is not.
+ // Blank content has no embedding by definition — the orchestrator returns an empty
+ // vector for it deliberately. That is a property of the input, not a provider
+ // failure, so it is stored as-is and simply never matches a semantic search.
+ var contentIsBlank = string.IsNullOrWhiteSpace(messageList[messageIndex].Content);
+ if (!contentIsBlank && (vector is null || vector.Length == 0))
+ {
+ _logger.LogWarning(
+ "Batch embedding returned an unusable vector for message {MessageId}; replaying it individually.",
+ messageList[messageIndex].MessageId);
+ vector = await _embeddingOrchestrator
+ .EmbedMessageAsync(messageList[messageIndex].Content, cancellationToken)
+ .ConfigureAwait(false);
+
+ if (vector is null || vector.Length == 0)
+ {
+ throw new InvalidOperationException(
+ $"Embedding for message {messageList[messageIndex].MessageId} was empty after an " +
+ "individual replay, although its content is not blank; refusing to persist a " +
+ "message that could never be retrieved.");
+ }
+ }
+
+ messageList[messageIndex] = messageList[messageIndex] with
+ {
+ Embedding = vector,
+ };
}
- results.Add(finalMessage);
}
- _logger.LogDebug("Batch adding {Count} messages", results.Count);
- return await _messageRepo.AddBatchAsync(results, cancellationToken).ConfigureAwait(false);
+ _logger.LogDebug("Batch adding {Count} messages", messageList.Count);
+ return await _messageRepo.AddBatchAsync(messageList, cancellationToken).ConfigureAwait(false);
}
///
@@ -147,11 +216,25 @@ public async Task> SearchMessagesAsync(
double minScore = 0.0,
CancellationToken cancellationToken = default)
{
- var scored = await _messageRepo.SearchByVectorAsync(
- queryEmbedding, sessionId, limit, minScore, null, cancellationToken).ConfigureAwait(false);
- return scored.Select(r => r.Message).ToList();
+ var scored = await SearchMessagesWithScoresAsync(
+ sessionId, queryEmbedding, limit, minScore, cancellationToken).ConfigureAwait(false);
+ return scored.Select(result => result.Message).ToList();
}
+ ///
+ /// Returns the repository's existing ranked message results without a second query. This internal
+ /// contract is used only when a recall explicitly requests diagnostics; the public short-term service
+ /// remains source-compatible for custom implementations.
+ ///
+ public Task> SearchMessagesWithScoresAsync(
+ string? sessionId,
+ float[] queryEmbedding,
+ int limit,
+ double minScore,
+ CancellationToken cancellationToken) =>
+ _messageRepo.SearchByVectorAsync(
+ queryEmbedding, sessionId, limit, minScore, null, cancellationToken);
+
///
public async Task ClearSessionAsync(
string sessionId,
@@ -177,3 +260,17 @@ public async Task> GetRecentMessagesAsOfAsync(
return await _messageRepo.GetRecentBySessionAsOfAsync(sessionId, asOf, cappedLimit, cancellationToken).ConfigureAwait(false);
}
}
+
+///
+/// Internal scored-search capability implemented by the built-in short-term memory service. Keeping this
+/// separate from avoids a breaking interface addition for providers.
+///
+internal interface IScoredMessageSearch
+{
+ Task> SearchMessagesWithScoresAsync(
+ string? sessionId,
+ float[] queryEmbedding,
+ int limit,
+ double minScore,
+ CancellationToken cancellationToken);
+}
diff --git a/src/AgentMemory.Extraction.Llm/AgentMemory.Extraction.Llm.csproj b/src/AgentMemory.Extraction.Llm/AgentMemory.Extraction.Llm.csproj
index c4e99457..c66f6d2f 100644
--- a/src/AgentMemory.Extraction.Llm/AgentMemory.Extraction.Llm.csproj
+++ b/src/AgentMemory.Extraction.Llm/AgentMemory.Extraction.Llm.csproj
@@ -17,6 +17,7 @@
+
diff --git a/src/AgentMemory.Extraction.Llm/Internal/LlmExtractionRunner.cs b/src/AgentMemory.Extraction.Llm/Internal/LlmExtractionRunner.cs
index 4d2091f2..18c93e23 100644
--- a/src/AgentMemory.Extraction.Llm/Internal/LlmExtractionRunner.cs
+++ b/src/AgentMemory.Extraction.Llm/Internal/LlmExtractionRunner.cs
@@ -38,14 +38,16 @@ internal async Task> RunAsync(
string userInstruction,
string conversationText,
Func> project,
- CancellationToken cancellationToken)
+ CancellationToken cancellationToken,
+ bool failOnParseExhaustion = false,
+ ChatResponseFormat? responseFormat = null)
{
var chatMessages = new List
{
new(ChatRole.System, systemPrompt),
new(ChatRole.User, $"{userInstruction}\n\n{conversationText}")
};
- var chatOptions = BuildChatOptions();
+ var chatOptions = BuildChatOptions(responseFormat);
int maxAttempts = _options.MaxRetries < 0 ? 1 : _options.MaxRetries + 1;
@@ -53,7 +55,8 @@ internal async Task> RunAsync(
{
cancellationToken.ThrowIfCancellationRequested();
- var response = await _chatClient.GetResponseAsync(chatMessages, chatOptions, cancellationToken)
+ var response = await GetResponseWithTransportRetryAsync(
+ chatMessages, chatOptions, cancellationToken)
.ConfigureAwait(false);
var raw = response.Text;
@@ -72,15 +75,116 @@ internal async Task> RunAsync(
"That response was not valid JSON. Reply with ONLY the JSON object — no markdown fences, no prose."));
}
}
+ if (failOnParseExhaustion)
+ throw new FormatException("LLM extraction exhausted its parse retries without valid JSON.");
+
return Array.Empty();
}
- private ChatOptions BuildChatOptions()
+
+ ///
+ /// Calls the provider, retrying transport failures with backoff.
+ ///
+ ///
+ /// Separate from the parse-retry loop above, and for a different failure. That loop re-prompts a
+ /// model that answered with unparseable JSON; this one re-sends an identical request that never
+ /// got an answer at all. Before this existed there was no transport retry anywhere in the
+ /// extraction path, and two 614-call preparations died mid-run on a single transient, at 37 and
+ /// 26 minutes each.
+ ///
+ /// A is deliberately not retried: it is caused by the request's own
+ /// shape, so re-sending it unchanged cannot help. That mirrors the batch splitter, which splits
+ /// on exactly that set and nothing else. Cancellation is never retried — retrying it would make
+ /// the preparation watchdog's timeout unenforceable.
+ ///
+ ///
+
+ ///
+ /// Whether a provider failure is worth re-sending an identical request for.
+ ///
+ ///
+ /// Retrying a permanent failure is not merely useless, it is expensive: an n=50 preparation spent
+ /// its 60-minute budget re-sending requests the provider had already rejected with
+ /// HTTP 400, and the watchdog fired with 7 failures and 544 of 614 calls done. A 400 says
+ /// the request is wrong — most often too large — and the same request will be just as wrong the
+ /// third time.
+ ///
+ /// Retryable: 408, 429, and 5xx, plus transport-level exceptions that never reached the service
+ /// and so carry no status. Everything else is permanent. An oversized request is separately
+ /// recoverable by splitting the batch, which is a different mechanism and the right one.
+ ///
+ ///
+ internal static bool IsTransient(Exception exception)
+ {
+ var status = TryGetStatus(exception);
+ if (status is null)
+ return true; // never reached the service: a connection reset, a DNS failure, a timeout
+ return status is 408 or 429 || status >= 500;
+ }
+
+ ///
+ /// The HTTP status behind a provider exception, or null when the call never got one.
+ ///
+ ///
+ /// Read reflectively rather than by referencing System.ClientModel: the status lives on
+ /// ClientResultException.Status for Azure/OpenAI clients and on
+ /// HttpRequestException.StatusCode for raw HTTP, and this library should not take a
+ /// package dependency to classify an error.
+ ///
+ internal static int? TryGetStatus(Exception exception)
+ {
+ if (exception is HttpRequestException { StatusCode: { } code })
+ return (int)code;
+
+ var property = exception.GetType().GetProperty("Status");
+ if (property?.GetValue(exception) is int status && status > 0)
+ return status;
+
+ return exception.InnerException is null ? null : TryGetStatus(exception.InnerException);
+ }
+
+ private async Task GetResponseWithTransportRetryAsync(
+ List chatMessages,
+ ChatOptions chatOptions,
+ CancellationToken cancellationToken)
+ {
+ int maxAttempts = _options.MaxRetries < 0 ? 1 : _options.MaxRetries + 1;
+ for (int attempt = 1; ; attempt++)
+ {
+ try
+ {
+ return await _chatClient
+ .GetResponseAsync(chatMessages, chatOptions, cancellationToken)
+ .ConfigureAwait(false);
+ }
+ catch (OperationCanceledException)
+ {
+ throw;
+ }
+ catch (FormatException)
+ {
+ throw;
+ }
+ catch (Exception exception) when (attempt < maxAttempts && IsTransient(exception))
+ {
+ _logger.LogWarning(
+ exception,
+ "LLM extraction transport failure (attempt {Attempt}/{MaxAttempts}); retrying.",
+ attempt, maxAttempts);
+ await Task.Delay(TimeSpan.FromMilliseconds(200 * attempt), cancellationToken)
+ .ConfigureAwait(false);
+ }
+ }
+ }
+
+ private ChatOptions BuildChatOptions(ChatResponseFormat? responseFormat)
{
var opts = new ChatOptions { Temperature = _options.Temperature };
if (!string.IsNullOrEmpty(_options.ModelId))
opts.ModelId = _options.ModelId;
+ if (_options.UseJsonResponseFormat)
+ opts.ResponseFormat = responseFormat ?? ChatResponseFormat.Json;
return opts;
}
diff --git a/src/AgentMemory.Extraction.Llm/Internal/LlmResponseModels.cs b/src/AgentMemory.Extraction.Llm/Internal/LlmResponseModels.cs
index b53095b2..642d7b03 100644
--- a/src/AgentMemory.Extraction.Llm/Internal/LlmResponseModels.cs
+++ b/src/AgentMemory.Extraction.Llm/Internal/LlmResponseModels.cs
@@ -4,6 +4,9 @@ namespace AgentMemory.Extraction.Llm.Internal;
internal sealed class LlmEntityDto
{
+ [JsonPropertyName("source_session")]
+ public string? SourceSession { get; set; }
+
[JsonPropertyName("name")]
public string Name { get; set; } = "";
@@ -25,6 +28,9 @@ internal sealed class LlmEntityDto
internal sealed class LlmFactDto
{
+ [JsonPropertyName("source_session")]
+ public string? SourceSession { get; set; }
+
[JsonPropertyName("subject")]
public string Subject { get; set; } = "";
@@ -40,6 +46,9 @@ internal sealed class LlmFactDto
internal sealed class LlmPreferenceDto
{
+ [JsonPropertyName("source_session")]
+ public string? SourceSession { get; set; }
+
[JsonPropertyName("category")]
public string Category { get; set; } = "";
@@ -55,6 +64,9 @@ internal sealed class LlmPreferenceDto
internal sealed class LlmRelationshipDto
{
+ [JsonPropertyName("source_session")]
+ public string? SourceSession { get; set; }
+
[JsonPropertyName("source")]
public string Source { get; set; } = "";
@@ -73,6 +85,9 @@ internal sealed class LlmRelationshipDto
internal sealed class LlmExtractionResponse
{
+ [JsonPropertyName("processed_source_sessions")]
+ public List? ProcessedSourceSessions { get; set; }
+
[JsonPropertyName("entities")]
public List Entities { get; set; } = new();
diff --git a/src/AgentMemory.Extraction.Llm/LlmExtractionBatchConcurrencyLimiter.cs b/src/AgentMemory.Extraction.Llm/LlmExtractionBatchConcurrencyLimiter.cs
new file mode 100644
index 00000000..b1e6699a
--- /dev/null
+++ b/src/AgentMemory.Extraction.Llm/LlmExtractionBatchConcurrencyLimiter.cs
@@ -0,0 +1,42 @@
+using Microsoft.Extensions.Options;
+
+namespace AgentMemory.Extraction.Llm;
+
+///
+/// Process-local provider-batch limiter shared by all scoped multi-session extractors in one
+/// service provider. A zero configured limit keeps the historical uncapped cross-scope behavior.
+///
+internal sealed class LlmExtractionBatchConcurrencyLimiter : IDisposable
+{
+ private readonly SemaphoreSlim? _gate;
+
+ public LlmExtractionBatchConcurrencyLimiter(
+ IOptions options)
+ {
+ ArgumentNullException.ThrowIfNull(options);
+ var maximum = options.Value.MaxConcurrentExtractionBatches;
+ if (maximum > 0)
+ _gate = new SemaphoreSlim(maximum, maximum);
+ }
+
+ internal async Task RunAsync(
+ Func> operation,
+ CancellationToken cancellationToken)
+ {
+ ArgumentNullException.ThrowIfNull(operation);
+ if (_gate is null)
+ return await operation().ConfigureAwait(false);
+
+ await _gate.WaitAsync(cancellationToken).ConfigureAwait(false);
+ try
+ {
+ return await operation().ConfigureAwait(false);
+ }
+ finally
+ {
+ _gate.Release();
+ }
+ }
+
+ public void Dispose() => _gate?.Dispose();
+}
diff --git a/src/AgentMemory.Extraction.Llm/LlmExtractionBatchDiagnostics.cs b/src/AgentMemory.Extraction.Llm/LlmExtractionBatchDiagnostics.cs
new file mode 100644
index 00000000..5ce9235e
--- /dev/null
+++ b/src/AgentMemory.Extraction.Llm/LlmExtractionBatchDiagnostics.cs
@@ -0,0 +1,62 @@
+using System.Collections.Concurrent;
+
+namespace AgentMemory.Extraction.Llm;
+
+internal sealed class LlmExtractionBatchDiagnostics
+{
+ private const int MaximumDetails = 32;
+ private readonly ConcurrentQueue _details = new();
+ private long _splits;
+ private long _droppedDetails;
+
+ internal void RecordSplit(Exception exception, int sourceSessions)
+ {
+ ArgumentNullException.ThrowIfNull(exception);
+ ArgumentOutOfRangeException.ThrowIfNegativeOrZero(sourceSessions);
+ Interlocked.Increment(ref _splits);
+ _details.Enqueue(new LlmExtractionBatchSplitDetail(
+ Classify(exception),
+ sourceSessions,
+ exception.GetType().FullName ?? exception.GetType().Name));
+ while (_details.Count > MaximumDetails && _details.TryDequeue(out _))
+ Interlocked.Increment(ref _droppedDetails);
+ }
+
+ internal LlmExtractionBatchDiagnosticsSnapshot Snapshot() =>
+ new(
+ Interlocked.Read(ref _splits),
+ _details.ToArray(),
+ Interlocked.Read(ref _droppedDetails));
+
+ private static string Classify(Exception exception) =>
+ exception.Message switch
+ {
+ "Processed-session acknowledgement is incomplete or invalid." =>
+ "acknowledgement",
+ "A learned item has a missing or unknown source-session key." =>
+ "source-session-key",
+ "Batch exceeds the configured input-token budget." =>
+ "token-budget",
+ _ when exception is FormatException => "parse-or-format",
+ _ when exception is OperationCanceledException => "cancellation",
+ _ => "other"
+ };
+}
+
+internal sealed record LlmExtractionBatchDiagnosticsSnapshot(
+ long Splits,
+ IReadOnlyList Details,
+ long DroppedDetails)
+{
+ internal LlmExtractionBatchDiagnosticsSnapshot Delta(
+ LlmExtractionBatchDiagnosticsSnapshot baseline) =>
+ new(
+ Splits - baseline.Splits,
+ Details.Skip(Math.Min(Details.Count, baseline.Details.Count)).ToArray(),
+ DroppedDetails - baseline.DroppedDetails);
+}
+
+internal sealed record LlmExtractionBatchSplitDetail(
+ string Reason,
+ int SourceSessions,
+ string ExceptionType);
diff --git a/src/AgentMemory.Extraction.Llm/LlmExtractionOptions.cs b/src/AgentMemory.Extraction.Llm/LlmExtractionOptions.cs
index b01b2aea..e2107a89 100644
--- a/src/AgentMemory.Extraction.Llm/LlmExtractionOptions.cs
+++ b/src/AgentMemory.Extraction.Llm/LlmExtractionOptions.cs
@@ -15,6 +15,38 @@ public sealed class LlmExtractionOptions
///
public int MaxRetries { get; set; } = 2;
+ ///
+ /// Whether extraction requests should ask the chat provider for a JSON response.
+ /// Disable only for providers that do not support the portable response-format hint.
+ ///
+ public bool UseJsonResponseFormat { get; set; } = true;
+
+ ///
+ /// Uses one typed model response for entities, facts, preferences, and relationships.
+ /// Disabled by default until the unified path passes live extraction-quality acceptance;
+ /// the existing four-category extraction path remains the compatibility control.
+ ///
+ public bool UseUnifiedExtraction { get; set; }
+
+ ///
+ /// Enables token-bounded multi-session unified extraction through
+ /// IMemoryExtractionPipeline.ExtractBatchAsync. Disabled by default; single-session
+ /// extraction behavior is unchanged.
+ ///
+ public bool UseMultiSessionBatchExtraction { get; set; }
+
+ ///
+ /// Maximum number of planned multi-session batches that one extraction operation may send
+ /// concurrently. The default of one preserves the historical sequential provider-call order.
+ ///
+ public int MaxConcurrentBatchesPerExtraction { get; set; } = 1;
+
+ ///
+ /// Optional process-local cap shared by all multi-session extraction operations registered in
+ /// the same service provider. Zero disables the shared cap.
+ ///
+ public int MaxConcurrentExtractionBatches { get; set; }
+
///
/// Model identifier to use. null (the default) means use the IChatClient default.
///
@@ -49,4 +81,15 @@ public sealed class LlmExtractionOptions
/// When null the extractor's built-in default prompt is used.
///
public string? PreferenceExtractionPrompt { get; set; }
+
+ ///
+ /// Offers the established relation vocabulary to the extractor so it reuses relation names
+ /// instead of inventing a phrasing per sentence. Default off.
+ ///
+ ///
+ /// QUALITY-RISK: it changes what the model emits, and it lengthens the prompt, which moves the
+ /// frozen batch plan's estimated input totals. Opt-in so the effect can be measured against an
+ /// unchanged control before it becomes the default.
+ ///
+ public bool UsePredicateVocabulary { get; set; }
}
diff --git a/src/AgentMemory.Extraction.Llm/LlmMultiSessionExtractionResponseContract.cs b/src/AgentMemory.Extraction.Llm/LlmMultiSessionExtractionResponseContract.cs
new file mode 100644
index 00000000..cfd7d58d
--- /dev/null
+++ b/src/AgentMemory.Extraction.Llm/LlmMultiSessionExtractionResponseContract.cs
@@ -0,0 +1,112 @@
+using System.Text.Json;
+using Microsoft.Extensions.AI;
+
+namespace AgentMemory.Extraction.Llm;
+
+internal static class LlmMultiSessionExtractionResponseContract
+{
+ internal const string Version = "batch-source-alias-schema-v1";
+
+ internal static string Alias(int zeroBasedIndex) => $"s{zeroBasedIndex + 1}";
+
+ internal static ChatResponseFormat CreateResponseFormat(int sourceSessions)
+ {
+ ArgumentOutOfRangeException.ThrowIfNegativeOrZero(sourceSessions);
+ return ChatResponseFormat.ForJsonSchema(
+ CreateSchema(sourceSessions),
+ "agent_memory_multi_session_v1",
+ "Structured memory extracted independently for each source-session alias.");
+ }
+
+ internal static JsonElement CreateSchema(int sourceSessions)
+ {
+ ArgumentOutOfRangeException.ThrowIfNegativeOrZero(sourceSessions);
+ var aliases = Enumerable.Range(0, sourceSessions).Select(Alias).ToArray();
+
+ Dictionary StringSchema() => new() { ["type"] = "string" };
+ Dictionary NullableStringSchema() =>
+ new() { ["type"] = new[] { "string", "null" } };
+ Dictionary NumberSchema() => new() { ["type"] = "number" };
+ Dictionary AliasSchema() => new()
+ {
+ ["type"] = "string",
+ ["enum"] = aliases
+ };
+ Dictionary ArraySchema(object items) => new()
+ {
+ ["type"] = "array",
+ ["items"] = items
+ };
+ Dictionary ObjectSchema(
+ Dictionary properties,
+ params string[] required) => new()
+ {
+ ["type"] = "object",
+ ["properties"] = properties,
+ ["required"] = required,
+ ["additionalProperties"] = false
+ };
+
+ var entity = ObjectSchema(
+ new Dictionary
+ {
+ ["source_session"] = AliasSchema(),
+ ["name"] = StringSchema(),
+ ["type"] = StringSchema(),
+ ["subtype"] = NullableStringSchema(),
+ ["description"] = NullableStringSchema(),
+ ["confidence"] = NumberSchema(),
+ ["aliases"] = ArraySchema(StringSchema())
+ },
+ "source_session", "name", "type", "subtype", "description", "confidence", "aliases");
+ var fact = ObjectSchema(
+ new Dictionary
+ {
+ ["source_session"] = AliasSchema(),
+ ["subject"] = StringSchema(),
+ ["predicate"] = StringSchema(),
+ ["object"] = StringSchema(),
+ ["confidence"] = NumberSchema()
+ },
+ "source_session", "subject", "predicate", "object", "confidence");
+ var preference = ObjectSchema(
+ new Dictionary
+ {
+ ["source_session"] = AliasSchema(),
+ ["category"] = StringSchema(),
+ ["preference"] = StringSchema(),
+ ["context"] = NullableStringSchema(),
+ ["confidence"] = NumberSchema()
+ },
+ "source_session", "category", "preference", "context", "confidence");
+ var relationship = ObjectSchema(
+ new Dictionary
+ {
+ ["source_session"] = AliasSchema(),
+ ["source"] = StringSchema(),
+ ["target"] = StringSchema(),
+ ["relation_type"] = StringSchema(),
+ ["description"] = NullableStringSchema(),
+ ["confidence"] = NumberSchema()
+ },
+ "source_session", "source", "target", "relation_type", "description", "confidence");
+
+ return JsonSerializer.SerializeToElement(
+ ObjectSchema(
+ new Dictionary
+ {
+ ["processed_source_sessions"] = new Dictionary
+ {
+ ["type"] = "array",
+ ["items"] = AliasSchema(),
+ ["minItems"] = sourceSessions,
+ ["maxItems"] = sourceSessions
+ },
+ ["entities"] = ArraySchema(entity),
+ ["facts"] = ArraySchema(fact),
+ ["preferences"] = ArraySchema(preference),
+ ["relations"] = ArraySchema(relationship)
+ },
+ "processed_source_sessions", "entities", "facts", "preferences", "relations"));
+ }
+}
diff --git a/src/AgentMemory.Extraction.Llm/LlmMultiSessionUnifiedMemoryExtractor.cs b/src/AgentMemory.Extraction.Llm/LlmMultiSessionUnifiedMemoryExtractor.cs
new file mode 100644
index 00000000..9d5a7e90
--- /dev/null
+++ b/src/AgentMemory.Extraction.Llm/LlmMultiSessionUnifiedMemoryExtractor.cs
@@ -0,0 +1,475 @@
+using System.Text;
+using AgentMemory.Abstractions.Diagnostics;
+using AgentMemory.Abstractions.Domain;
+using AgentMemory.Abstractions.Services;
+using AgentMemory.Extraction.Llm.Internal;
+using AgentMemory.Core.Memory;
+using Microsoft.Extensions.AI;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+
+namespace AgentMemory.Extraction.Llm;
+
+///
+/// Token-bounded multi-session unified extraction. Invalid or partial batch responses are never
+/// accepted: a multi-session batch is split recursively, while an invalid single-session response
+/// fails the operation.
+///
+internal sealed class LlmMultiSessionUnifiedMemoryExtractor : IMultiSessionUnifiedMemoryExtractor
+{
+ private const string SystemPrompt =
+ """
+ You extract structured long-term memory from multiple independent source sessions.
+ Return JSON only. Include processed_source_sessions containing every supplied source_session.
+ Every entity, fact, preference, and relation must include its source_session.
+ Use exactly this shape:
+ {"processed_source_sessions":["..."],"entities":[{"source_session":"...","name":"...","type":"PERSON|ORGANIZATION|LOCATION|EVENT|OBJECT","confidence":0.9,"aliases":[]}],"facts":[{"source_session":"...","subject":"...","predicate":"...","object":"...","confidence":0.9}],"preferences":[{"source_session":"...","category":"...","preference":"...","confidence":0.85}],"relations":[{"source_session":"...","source":"...","target":"...","relation_type":"...","confidence":0.8}]}
+ Sessions are independent. Never combine facts or entities across source_session values.
+ Use empty arrays when a category has no supported memory. Do not emit prose or markdown.
+ """;
+
+ ///
+ /// The system prompt, with the established relation vocabulary offered when one is supplied.
+ ///
+ ///
+ /// Extraction invents a predicate per sentence when nothing tells it which relations exist —
+ /// measured at 700 facts under 421 distinct predicates, with a single birth expressed as
+ /// "was born", "was born in", "were born in", "had" and "welcomed", which left counting
+ /// questions unanswerable even once a relation could be retrieved whole. Reconciling phrasings
+ /// afterwards cannot be done safely, since "bought" and "sold" are one similarity threshold
+ /// apart, so the vocabulary is applied at generation instead.
+ ///
+ /// The extractor is told to prefer these relations, never to be limited to them: a model
+ /// restricted to a fixed list would drop facts that genuinely need a new relation. An empty
+ /// vocabulary yields the original prompt byte-for-byte, so callers that do not use this are
+ /// unaffected — including the frozen batch plan, whose estimated input totals depend on prompt
+ /// size.
+ ///
+ ///
+ internal static string BuildSystemPrompt(MemoryPredicateVocabulary? vocabulary)
+ {
+ var established = vocabulary?.Snapshot() ?? [];
+ if (established.Count == 0)
+ return SystemPrompt;
+
+ return SystemPrompt +
+ "\nEstablished relation predicates, in order of preference: " +
+ string.Join(", ", established) +
+ ".\nReuse an established predicate whenever it fits; introduce a new one only when none does.";
+ }
+
+ private const string UserInstruction =
+ "Extract every source session independently and acknowledge all processed source sessions:";
+
+ /// The vocabulary offered to the model, or null when the option is off.
+ ///
+ /// Built from the curated seed on each use rather than cached, so the size the plan estimates
+ /// and the string actually sent can never diverge — an estimate taken from a different prompt
+ /// than the request would corrupt the frozen plan's token accounting silently.
+ ///
+ private MemoryPredicateVocabulary? ActiveVocabulary =>
+ _options.UsePredicateVocabulary ? MemoryPredicateSeedVocabulary.Create() : null;
+
+ private readonly IChatClient _chatClient;
+ private readonly LlmExtractionOptions _options;
+ private readonly ILogger _logger;
+ private readonly LlmExtractionBatchConcurrencyLimiter? _concurrencyLimiter;
+ private readonly LlmExtractionBatchDiagnostics? _batchDiagnostics;
+
+ public LlmMultiSessionUnifiedMemoryExtractor(
+ IChatClient chatClient,
+ IOptions options,
+ ILogger logger,
+ LlmExtractionBatchConcurrencyLimiter? concurrencyLimiter = null,
+ LlmExtractionBatchDiagnostics? batchDiagnostics = null)
+ {
+ _chatClient = chatClient;
+ _options = options.Value;
+ _logger = logger;
+ _concurrencyLimiter = concurrencyLimiter;
+ _batchDiagnostics = batchDiagnostics;
+ ArgumentOutOfRangeException.ThrowIfNegativeOrZero(
+ _options.MaxConcurrentBatchesPerExtraction);
+ }
+
+ public bool IsEnabled =>
+ _options.UseUnifiedExtraction && _options.UseMultiSessionBatchExtraction;
+
+ public MultiSessionExtractionPlan Plan(
+ IReadOnlyList requests,
+ int maxSessionsPerBatch,
+ int maxInputTokens)
+ {
+ ArgumentNullException.ThrowIfNull(requests);
+ ArgumentOutOfRangeException.ThrowIfNegativeOrZero(maxSessionsPerBatch);
+ ArgumentOutOfRangeException.ThrowIfNegativeOrZero(maxInputTokens);
+
+ var duplicate = requests.GroupBy(request => request.SessionId, StringComparer.Ordinal)
+ .FirstOrDefault(group => group.Count() != 1);
+ if (duplicate is not null)
+ throw new ArgumentException($"Source session key '{duplicate.Key}' is not unique.", nameof(requests));
+
+ var batches = PlanBatches(requests, maxSessionsPerBatch, maxInputTokens)
+ .Select(batch => new MultiSessionExtractionBatchPlan(
+ batch.Select(request => request.SessionId).ToArray(),
+ EstimateInputTokens(batch)))
+ .ToArray();
+ return new MultiSessionExtractionPlan(batches);
+ }
+
+ public async Task> ExtractAsync(
+ IReadOnlyList requests,
+ int maxSessionsPerBatch,
+ int maxInputTokens,
+ CancellationToken cancellationToken = default)
+ {
+ var plan = Plan(requests, maxSessionsPerBatch, maxInputTokens);
+ var requestsBySession = requests.ToDictionary(
+ request => request.SessionId,
+ StringComparer.Ordinal);
+ var extractedByBatch =
+ new IReadOnlyDictionary?[plan.BatchCount];
+ var concurrency = Math.Min(
+ _options.MaxConcurrentBatchesPerExtraction,
+ plan.BatchCount);
+
+ if (concurrency <= 1)
+ {
+ for (var index = 0; index < plan.BatchCount; index++)
+ {
+ extractedByBatch[index] = await ExtractPlannedBatchAsync(
+ plan.Batches[index],
+ requestsBySession,
+ maxInputTokens,
+ cancellationToken)
+ .ConfigureAwait(false);
+ }
+ }
+ else
+ {
+ await Parallel.ForEachAsync(
+ Enumerable.Range(0, plan.BatchCount),
+ new ParallelOptions
+ {
+ MaxDegreeOfParallelism = concurrency,
+ CancellationToken = cancellationToken
+ },
+ async (index, itemCancellationToken) =>
+ {
+ extractedByBatch[index] = await ExtractPlannedBatchAsync(
+ plan.Batches[index],
+ requestsBySession,
+ maxInputTokens,
+ itemCancellationToken)
+ .ConfigureAwait(false);
+ }).ConfigureAwait(false);
+ }
+
+ var unordered = new Dictionary(StringComparer.Ordinal);
+ foreach (var extracted in extractedByBatch)
+ {
+ if (extracted is null)
+ throw new InvalidOperationException(
+ "Multi-session extraction did not complete every planned batch.");
+ foreach (var pair in extracted)
+ unordered.Add(pair.Key, pair.Value);
+ }
+
+ if (unordered.Count != requests.Count)
+ throw new InvalidOperationException(
+ $"Multi-session extraction returned {unordered.Count} sessions for {requests.Count} inputs.");
+
+ var results = new Dictionary(StringComparer.Ordinal);
+ foreach (var request in requests)
+ results.Add(request.SessionId, unordered[request.SessionId]);
+ return results;
+ }
+
+ private async Task>
+ ExtractPlannedBatchAsync(
+ MultiSessionExtractionBatchPlan plannedBatch,
+ IReadOnlyDictionary requestsBySession,
+ int maxInputTokens,
+ CancellationToken cancellationToken)
+ {
+ var batch = plannedBatch.SourceSessionIds
+ .Select(sessionId => requestsBySession[sessionId])
+ .ToArray();
+ return await ExtractOrSplitAsync(
+ batch,
+ maxInputTokens,
+ cancellationToken)
+ .ConfigureAwait(false);
+ }
+
+ private async Task> ExtractOrSplitAsync(
+ IReadOnlyList batch,
+ int maxInputTokens,
+ CancellationToken cancellationToken)
+ {
+ try
+ {
+ if (EstimateInputTokens(batch) > maxInputTokens)
+ throw new BatchValidationException("Batch exceeds the configured input-token budget.");
+ return await ExtractBatchAsync(batch, cancellationToken).ConfigureAwait(false);
+ }
+ catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
+ {
+ throw;
+ }
+ // FormatException only, deliberately. Every failure the batch's own shape causes arrives as
+ // one - BatchValidationException derives from it, covering the token budget, the
+ // acknowledgement check and the source-session-key check, as does an unparseable response -
+ // and halving the batch is a real remedy for each. A provider transport failure is not that:
+ // re-sending each half puts the same request shape at the same endpoint that just failed, so
+ // the split neither diagnoses nor fixes it, and it doubles the call count. That broke a
+ // 37-minute n=50 preparation at question 20 ("observed 14 calls ... expected exactly 12")
+ // over one ClientResultException.
+ //
+ // NOTE, verified rather than assumed: there is currently NO transport retry on this path.
+ // LlmExtractionRunner honours MaxRetries, but only by re-prompting on a parse failure - its
+ // GetResponseAsync call sits outside any catch, so a transport exception propagates
+ // immediately. Splitting was therefore the only thing resembling a retry for transports, and
+ // it was a bad one: it re-sent to the endpoint that had just failed and doubled the call
+ // count. Removing it does not remove a working recovery; it removes a misleading one. A real
+ // transport retry is tracked separately, because it must be reconciled with the harness's
+ // exact-call-count invariant rather than quietly breaking it.
+ catch (Exception ex) when (batch.Count > 1 && IsBatchShapeFailure(ex))
+ {
+ _batchDiagnostics?.RecordSplit(ex, batch.Count);
+ _logger.LogWarning(
+ ex,
+ "Multi-session extraction batch of {Count} did not pass validation; splitting.",
+ batch.Count);
+ var midpoint = batch.Count / 2;
+ var left = await ExtractOrSplitAsync(batch.Take(midpoint).ToArray(), maxInputTokens, cancellationToken)
+ .ConfigureAwait(false);
+ var right = await ExtractOrSplitAsync(batch.Skip(midpoint).ToArray(), maxInputTokens, cancellationToken)
+ .ConfigureAwait(false);
+ return left.Concat(right).ToDictionary(pair => pair.Key, pair => pair.Value, StringComparer.Ordinal);
+ }
+ }
+
+ private async Task> ExtractBatchAsync(
+ IReadOnlyList batch,
+ CancellationToken cancellationToken)
+ {
+ var estimatedInputTokens = EstimateInputTokens(batch);
+ using var activity = AgentMemoryDiagnostics.Source.StartActivity("memory.extract.unified_batch");
+ activity?.SetTag("memory.extract.source_sessions", batch.Count);
+ activity?.SetTag("memory.extract.estimated_input_tokens", estimatedInputTokens);
+ var runner = new LlmExtractionRunner(_chatClient, _options, _logger);
+
+ Task>> RunProviderAsync() =>
+ runner.RunAsync(
+ BuildSystemPrompt(ActiveVocabulary),
+ UserInstruction,
+ BuildBatchText(batch),
+ response => new[] { ProjectAndValidate(response, batch) },
+ cancellationToken,
+ failOnParseExhaustion: true,
+ responseFormat: LlmMultiSessionExtractionResponseContract.CreateResponseFormat(batch.Count));
+
+ var projected = _concurrencyLimiter is null
+ ? await RunProviderAsync().ConfigureAwait(false)
+ : await _concurrencyLimiter.RunAsync(RunProviderAsync, cancellationToken)
+ .ConfigureAwait(false);
+ return projected.Single();
+ }
+
+ private static IReadOnlyDictionary ProjectAndValidate(
+ LlmExtractionResponse response,
+ IReadOnlyList batch)
+ {
+ var sourceSessions = batch.Select((request, index) =>
+ new BatchSourceSession(
+ LlmMultiSessionExtractionResponseContract.Alias(index), request)).ToArray();
+ var expected = sourceSessions.Select(item => item.Alias).ToHashSet(StringComparer.Ordinal);
+ var acknowledged = (response.ProcessedSourceSessions ?? [])
+ .ToHashSet(StringComparer.Ordinal);
+ if (!acknowledged.SetEquals(expected) || response.ProcessedSourceSessions!.Count != expected.Count)
+ throw new BatchValidationException("Processed-session acknowledgement is incomplete or invalid.");
+
+ var results = expected.ToDictionary(
+ key => key,
+ _ => new Accumulator(),
+ StringComparer.Ordinal);
+
+ foreach (var item in response.Entities ?? [])
+ {
+ var target = GetAccumulator(results, item.SourceSession);
+ if (!string.IsNullOrWhiteSpace(item.Name) && !string.IsNullOrWhiteSpace(item.Type))
+ target.Entities.Add(new ExtractedEntity
+ {
+ Name = item.Name,
+ Type = NormalizeType(item.Type),
+ Subtype = item.Subtype,
+ Description = item.Description,
+ Confidence = item.Confidence,
+ Aliases = item.Aliases,
+ });
+ }
+ foreach (var item in response.Facts ?? [])
+ {
+ var target = GetAccumulator(results, item.SourceSession);
+ if (!string.IsNullOrWhiteSpace(item.Subject) &&
+ !string.IsNullOrWhiteSpace(item.Predicate) &&
+ !string.IsNullOrWhiteSpace(item.Object))
+ target.Facts.Add(new ExtractedFact
+ {
+ Subject = item.Subject,
+ Predicate = item.Predicate,
+ Object = item.Object,
+ Confidence = item.Confidence,
+ });
+ }
+ foreach (var item in response.Preferences ?? [])
+ {
+ var target = GetAccumulator(results, item.SourceSession);
+ if (!string.IsNullOrWhiteSpace(item.Preference))
+ target.Preferences.Add(new ExtractedPreference
+ {
+ Category = item.Category,
+ PreferenceText = item.Preference,
+ Context = item.Context,
+ Confidence = item.Confidence,
+ });
+ }
+ foreach (var item in response.Relations ?? [])
+ {
+ var target = GetAccumulator(results, item.SourceSession);
+ if (!string.IsNullOrWhiteSpace(item.Source) &&
+ !string.IsNullOrWhiteSpace(item.Target) &&
+ !string.IsNullOrWhiteSpace(item.RelationType))
+ target.Relationships.Add(new ExtractedRelationship
+ {
+ SourceEntity = item.Source,
+ TargetEntity = item.Target,
+ RelationshipType = item.RelationType,
+ Description = item.Description,
+ Confidence = item.Confidence,
+ });
+ }
+
+ return sourceSessions.ToDictionary(
+ item => item.Request.SessionId,
+ item => results[item.Alias].ToResult(),
+ StringComparer.Ordinal);
+ }
+
+ private static Accumulator GetAccumulator(
+ IReadOnlyDictionary results,
+ string? sourceSession)
+ {
+ if (string.IsNullOrWhiteSpace(sourceSession) || !results.TryGetValue(sourceSession, out var target))
+ throw new BatchValidationException("A learned item has a missing or unknown source-session key.");
+ return target;
+ }
+
+ private IReadOnlyList> PlanBatches(
+ IReadOnlyList requests,
+ int maxSessionsPerBatch,
+ int maxInputTokens)
+ {
+ var batches = new List>();
+ var current = new List();
+ foreach (var request in requests)
+ {
+ if (EstimateInputTokens([request]) > maxInputTokens)
+ throw new InvalidOperationException(
+ $"Source session '{request.SessionId}' exceeds the configured input-token budget.");
+
+ var candidate = current.Append(request).ToArray();
+ if (current.Count > 0 &&
+ (candidate.Length > maxSessionsPerBatch || EstimateInputTokens(candidate) > maxInputTokens))
+ {
+ batches.Add(current.ToArray());
+ current.Clear();
+ }
+ current.Add(request);
+ }
+ if (current.Count > 0)
+ batches.Add(current.ToArray());
+ return batches;
+ }
+
+ private int EstimateInputTokens(IReadOnlyList batch) =>
+ checked(
+ Encoding.UTF8.GetByteCount(BuildSystemPrompt(ActiveVocabulary)) +
+ Encoding.UTF8.GetByteCount(UserInstruction) +
+ Encoding.UTF8.GetByteCount(BuildBatchText(batch)) +
+ 35);
+
+ private static string BuildBatchText(IReadOnlyList batch)
+ {
+ var builder = new StringBuilder();
+ for (var index = 0; index < batch.Count; index++)
+ {
+ var request = batch[index];
+ builder.Append("");
+ foreach (var message in request.Messages)
+ {
+ builder.Append('[').Append(message.TimestampUtc.ToString("O")).Append("] ")
+ .Append(message.Role).Append(": ").AppendLine(message.Content);
+ }
+ builder.AppendLine("");
+ }
+ return builder.ToString();
+ }
+
+ private static string NormalizeType(string type) => type.ToUpperInvariant() switch
+ {
+ "CONCEPT" => "OBJECT",
+ "PLACE" => "LOCATION",
+ "COMPANY" => "ORGANIZATION",
+ "INDIVIDUAL" => "PERSON",
+ var value => value,
+ };
+ private sealed record BatchSourceSession(
+ string Alias,
+ ExtractionRequest Request);
+
+
+ private sealed class Accumulator
+ {
+ public List Entities { get; } = [];
+ public List Facts { get; } = [];
+ public List Preferences { get; } = [];
+ public List Relationships { get; } = [];
+
+ public UnifiedExtractionResult ToResult() => new()
+ {
+ Entities = Entities,
+ Facts = Facts,
+ Preferences = Preferences,
+ Relationships = Relationships,
+ };
+ }
+
+
+ ///
+ /// Whether a failure is caused by the batch's own shape, and so is worth splitting for.
+ ///
+ ///
+ /// Two families qualify. covers the validation and parse failures
+ /// this class raises itself. A permanent 4xx qualifies too, and missing it cost a full
+ /// 60-minute preparation: the provider rejected oversized batches with HTTP 400, the splitter
+ /// had been narrowed to FormatException only so it declined to help, and the transport retry
+ /// re-sent each rejected request until the watchdog fired.
+ ///
+ /// 408 and 429 are excluded deliberately — they are transient and belong to the retry policy, and
+ /// splitting on a rate limit would answer congestion by sending more requests.
+ ///
+ ///
+ internal static bool IsBatchShapeFailure(Exception exception)
+ {
+ if (exception is FormatException)
+ return true;
+
+ var status = Internal.LlmExtractionRunner.TryGetStatus(exception);
+ return status is >= 400 and < 500 and not 408 and not 429;
+ }
+
+ private sealed class BatchValidationException(string message) : FormatException(message);
+}
diff --git a/src/AgentMemory.Extraction.Llm/LlmUnifiedMemoryExtractor.cs b/src/AgentMemory.Extraction.Llm/LlmUnifiedMemoryExtractor.cs
new file mode 100644
index 00000000..3744b94b
--- /dev/null
+++ b/src/AgentMemory.Extraction.Llm/LlmUnifiedMemoryExtractor.cs
@@ -0,0 +1,111 @@
+using AgentMemory.Abstractions.Diagnostics;
+using AgentMemory.Abstractions.Domain;
+using AgentMemory.Abstractions.Services;
+using AgentMemory.Core.Extraction;
+using AgentMemory.Extraction.Llm.Internal;
+using Microsoft.Extensions.AI;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+
+namespace AgentMemory.Extraction.Llm;
+
+internal sealed class LlmUnifiedMemoryExtractor : IUnifiedMemoryExtractor
+{
+ private const string SystemPrompt =
+ """
+ You extract structured long-term memory from a conversation.
+ Return JSON only with all four arrays: entities, facts, preferences, relations.
+ Use exactly this shape:
+ {"entities":[{"name":"...","type":"PERSON|ORGANIZATION|LOCATION|EVENT|OBJECT","confidence":0.9,"aliases":[]}],"facts":[{"subject":"...","predicate":"...","object":"...","confidence":0.9}],"preferences":[{"category":"...","preference":"...","confidence":0.85}],"relations":[{"source":"...","target":"...","relation_type":"...","confidence":0.8}]}
+ Use empty arrays when a category has no supported memory. Do not emit prose or markdown.
+ """;
+
+ private readonly LlmExtractionOptions _options;
+ private readonly LlmExtractionRunner _runner;
+
+ public LlmUnifiedMemoryExtractor(
+ IChatClient chatClient,
+ IOptions options,
+ ILogger logger)
+ {
+ _options = options.Value;
+ _runner = new LlmExtractionRunner(chatClient, _options, logger);
+ }
+
+ public bool IsEnabled => _options.UseUnifiedExtraction;
+
+ public async Task ExtractAsync(
+ IReadOnlyList messages,
+ CancellationToken cancellationToken = default)
+ {
+ if (messages.Count == 0)
+ return new UnifiedExtractionResult();
+
+ using var activity = AgentMemoryDiagnostics.Source.StartActivity("memory.extract.unified");
+ var results = await _runner.RunAsync(
+ SystemPrompt,
+ "Extract all supported memory from this conversation:",
+ ConversationTextBuilder.Build(messages),
+ response => new[] { Project(response) },
+ cancellationToken,
+ failOnParseExhaustion: true).ConfigureAwait(false);
+ return results.Count == 1 ? results[0] : new UnifiedExtractionResult();
+ }
+
+ private static UnifiedExtractionResult Project(LlmExtractionResponse response) =>
+ new()
+ {
+ Entities = (response.Entities ?? [])
+ .Where(item => !string.IsNullOrWhiteSpace(item.Name) && !string.IsNullOrWhiteSpace(item.Type))
+ .Select(item => new ExtractedEntity
+ {
+ Name = item.Name,
+ Type = NormalizeType(item.Type),
+ Subtype = item.Subtype,
+ Description = item.Description,
+ Confidence = item.Confidence,
+ Aliases = item.Aliases,
+ }).ToArray(),
+ Facts = (response.Facts ?? [])
+ .Where(item => !string.IsNullOrWhiteSpace(item.Subject) &&
+ !string.IsNullOrWhiteSpace(item.Predicate) &&
+ !string.IsNullOrWhiteSpace(item.Object))
+ .Select(item => new ExtractedFact
+ {
+ Subject = item.Subject,
+ Predicate = item.Predicate,
+ Object = item.Object,
+ Confidence = item.Confidence,
+ }).ToArray(),
+ Preferences = (response.Preferences ?? [])
+ .Where(item => !string.IsNullOrWhiteSpace(item.Preference))
+ .Select(item => new ExtractedPreference
+ {
+ Category = item.Category,
+ PreferenceText = item.Preference,
+ Context = item.Context,
+ Confidence = item.Confidence,
+ }).ToArray(),
+ Relationships = (response.Relations ?? [])
+ .Where(item => !string.IsNullOrWhiteSpace(item.Source) &&
+ !string.IsNullOrWhiteSpace(item.Target) &&
+ !string.IsNullOrWhiteSpace(item.RelationType))
+ .Select(item => new ExtractedRelationship
+ {
+ SourceEntity = item.Source,
+ TargetEntity = item.Target,
+ RelationshipType = item.RelationType,
+ Description = item.Description,
+ Confidence = item.Confidence,
+ }).ToArray(),
+ };
+
+ private static string NormalizeType(string type) => type.ToUpperInvariant() switch
+ {
+ "CONCEPT" => "OBJECT",
+ "PLACE" => "LOCATION",
+ "COMPANY" => "ORGANIZATION",
+ "INDIVIDUAL" => "PERSON",
+ var value => value,
+ };
+}
diff --git a/src/AgentMemory.Extraction.Llm/ServiceCollectionExtensions.cs b/src/AgentMemory.Extraction.Llm/ServiceCollectionExtensions.cs
index 03963277..550298c5 100644
--- a/src/AgentMemory.Extraction.Llm/ServiceCollectionExtensions.cs
+++ b/src/AgentMemory.Extraction.Llm/ServiceCollectionExtensions.cs
@@ -22,6 +22,10 @@ public static IServiceCollection AddLlmExtraction(
llmOptions
.Validate(o => o.Temperature >= 0.0f, "LlmExtraction Temperature must be non-negative.")
.Validate(o => o.MaxRetries >= 0, "LlmExtraction MaxRetries must be non-negative.")
+ .Validate(o => o.MaxConcurrentBatchesPerExtraction > 0,
+ "LlmExtraction MaxConcurrentBatchesPerExtraction must be positive.")
+ .Validate(o => o.MaxConcurrentExtractionBatches >= 0,
+ "LlmExtraction MaxConcurrentExtractionBatches must be non-negative.")
.ValidateOnStart();
// Replace (not TryAdd) so the real extractors authoritatively override the Core no-op stub
@@ -33,6 +37,10 @@ public static IServiceCollection AddLlmExtraction(
services.Replace(ServiceDescriptor.Scoped());
services.Replace(ServiceDescriptor.Scoped());
services.Replace(ServiceDescriptor.Scoped());
+ services.TryAddSingleton();
+ services.TryAddScoped();
+ services.TryAddSingleton();
+ services.TryAddScoped();
return services;
}
diff --git a/src/AgentMemory.Neo4j/Infrastructure/CanonicalKeyBackfillRow.cs b/src/AgentMemory.Neo4j/Infrastructure/CanonicalKeyBackfillRow.cs
new file mode 100644
index 00000000..b63159d0
--- /dev/null
+++ b/src/AgentMemory.Neo4j/Infrastructure/CanonicalKeyBackfillRow.cs
@@ -0,0 +1,11 @@
+namespace AgentMemory.Neo4j.Infrastructure;
+
+///
+/// One pre-canonical fact awaiting key backfill. Named rather than anonymous so the migration's
+/// read shape is part of the type system and can be stubbed in tests.
+///
+internal sealed record CanonicalKeyBackfillRow(
+ string Id,
+ string Subject,
+ string Predicate,
+ string Object);
diff --git a/src/AgentMemory.Neo4j/Infrastructure/INeo4jAtomicTransactionRunner.cs b/src/AgentMemory.Neo4j/Infrastructure/INeo4jAtomicTransactionRunner.cs
new file mode 100644
index 00000000..f7026f8d
--- /dev/null
+++ b/src/AgentMemory.Neo4j/Infrastructure/INeo4jAtomicTransactionRunner.cs
@@ -0,0 +1,13 @@
+namespace AgentMemory.Neo4j.Infrastructure;
+
+/// Capability interface for joining repository calls into one logical write transaction.
+public interface INeo4jAtomicTransactionRunner
+{
+ ///
+ /// Executes a logical persistence unit in one write transaction. Repository calls made from
+ /// through the paired join it.
+ ///
+ Task ExecuteAtomicWriteAsync(
+ Func> work,
+ CancellationToken cancellationToken = default);
+}
diff --git a/src/AgentMemory.Neo4j/Infrastructure/Neo4jMemoryPersistenceTransaction.cs b/src/AgentMemory.Neo4j/Infrastructure/Neo4jMemoryPersistenceTransaction.cs
new file mode 100644
index 00000000..bdc92c07
--- /dev/null
+++ b/src/AgentMemory.Neo4j/Infrastructure/Neo4jMemoryPersistenceTransaction.cs
@@ -0,0 +1,53 @@
+using AgentMemory.Core.Extraction;
+
+namespace AgentMemory.Neo4j.Infrastructure;
+
+///
+/// Runs memory persistence inside one Neo4j write transaction when the configured runner can
+/// provide one, and passes work straight through when it cannot.
+///
+///
+/// is a public extension point, registered with
+/// TryAddSingleton precisely so a host can substitute its own implementation. This type is
+/// then registered unconditionally with Replace, so it receives whatever the host supplied.
+///
+/// It previously hard-cast that runner to — an interface
+/// this library added later — and threw when the cast failed. That turned a documented, deliberately
+/// overridable seam into a startup crash for any host that had already exercised it: the substitution
+/// was legal when they wrote it and became fatal on upgrade, with no compile-time signal.
+///
+///
+/// Atomicity is optional by design, which is why carries
+/// at all and why PersistenceStage already branches on
+/// it. So a runner without atomic support degrades to pass-through and reports that honestly, rather
+/// than claiming a rollback guarantee it cannot keep or refusing to start.
+///
+///
+internal sealed class Neo4jMemoryPersistenceTransaction : IMemoryPersistenceTransaction
+{
+ private readonly INeo4jAtomicTransactionRunner? _atomicRunner;
+
+ public Neo4jMemoryPersistenceTransaction(INeo4jTransactionRunner transactionRunner)
+ {
+ ArgumentNullException.ThrowIfNull(transactionRunner);
+ _atomicRunner = transactionRunner as INeo4jAtomicTransactionRunner;
+ }
+
+ /// True only when the configured runner can actually roll back.
+ public bool SupportsAtomicRollback => _atomicRunner is not null;
+
+ public Task ExecuteAsync(
+ Func> work,
+ CancellationToken cancellationToken = default)
+ {
+ ArgumentNullException.ThrowIfNull(work);
+
+ if (_atomicRunner is not null)
+ return _atomicRunner.ExecuteAtomicWriteAsync(work, cancellationToken);
+
+ // No coordinator: run the work as-is. Callers that need all-or-nothing check
+ // SupportsAtomicRollback first, so this cannot silently downgrade a guarantee.
+ cancellationToken.ThrowIfCancellationRequested();
+ return work(cancellationToken);
+ }
+}
diff --git a/src/AgentMemory.Neo4j/Infrastructure/Neo4jOptions.cs b/src/AgentMemory.Neo4j/Infrastructure/Neo4jOptions.cs
index 9400e5fb..647b64f3 100644
--- a/src/AgentMemory.Neo4j/Infrastructure/Neo4jOptions.cs
+++ b/src/AgentMemory.Neo4j/Infrastructure/Neo4jOptions.cs
@@ -16,6 +16,13 @@ public class Neo4jOptions
///
public int EmbeddingDimensions { get; set; } = 1536;
+ ///
+ /// Persists a message batch, its embeddings, ordering links, and read-back in one Cypher query.
+ /// Enabled by default; disable only for compatibility diagnosis or controlled A/B measurement.
+ /// The legacy path remains available and preserves its original multi-query behavior.
+ ///
+ public bool UseOptimizedMessageBatchWrites { get; set; } = true;
+
///
/// When (the default), schema bootstrap verifies that every existing vector
/// index was created with and throws
diff --git a/src/AgentMemory.Neo4j/Infrastructure/Neo4jTransactionRunner.cs b/src/AgentMemory.Neo4j/Infrastructure/Neo4jTransactionRunner.cs
index 9d35bc25..6d2b3eb8 100644
--- a/src/AgentMemory.Neo4j/Infrastructure/Neo4jTransactionRunner.cs
+++ b/src/AgentMemory.Neo4j/Infrastructure/Neo4jTransactionRunner.cs
@@ -27,11 +27,12 @@ namespace AgentMemory.Neo4j.Infrastructure;
/// the cost is one check per transaction.
///
///
-internal sealed class Neo4jTransactionRunner : INeo4jTransactionRunner
+internal sealed class Neo4jTransactionRunner : INeo4jTransactionRunner, INeo4jAtomicTransactionRunner
{
private readonly INeo4jSessionFactory _sessionFactory;
private readonly ILogger _logger;
+ private readonly AsyncLocal _ambientWriteTransaction = new();
public Neo4jTransactionRunner(INeo4jSessionFactory sessionFactory, ILogger logger)
{
_sessionFactory = sessionFactory;
@@ -41,15 +42,20 @@ public Neo4jTransactionRunner(INeo4jSessionFactory sessionFactory, ILogger ReadAsync(Func> work, CancellationToken cancellationToken = default)
{
cancellationToken.ThrowIfCancellationRequested();
+ if (_ambientWriteTransaction.Value is { } ambient)
+ return await work(ambient).ConfigureAwait(false);
+
using var activity = AgentMemoryDiagnostics.Source.StartActivity("memory.db.tx", ActivityKind.Client);
activity?.SetTag("db.mode", "read");
var payload = activity is null ? null : new PayloadAccumulator();
+ var transactionEntryStartedAt = activity is null ? 0 : Stopwatch.GetTimestamp();
var session = _sessionFactory.OpenSession(AccessMode.Read);
await using var _ = session.ConfigureAwait(false); // ConfigureAwait the disposal without rebinding session's type
try
{
- return await session.ExecuteReadAsync(Instrument(work, "read", activity, payload)).ConfigureAwait(false);
+ return await session.ExecuteReadAsync(
+ Instrument(work, "read", activity, payload, transactionEntryStartedAt)).ConfigureAwait(false);
}
catch (Exception ex)
{
@@ -75,15 +81,20 @@ await ReadAsync(async tx =>
public async Task WriteAsync(Func> work, CancellationToken cancellationToken = default)
{
cancellationToken.ThrowIfCancellationRequested();
+ if (_ambientWriteTransaction.Value is { } ambient)
+ return await work(ambient).ConfigureAwait(false);
+
using var activity = AgentMemoryDiagnostics.Source.StartActivity("memory.db.tx", ActivityKind.Client);
activity?.SetTag("db.mode", "write");
var payload = activity is null ? null : new PayloadAccumulator();
+ var transactionEntryStartedAt = activity is null ? 0 : Stopwatch.GetTimestamp();
var session = _sessionFactory.OpenSession(AccessMode.Write);
await using var _ = session.ConfigureAwait(false); // ConfigureAwait the disposal without rebinding session's type
try
{
- return await session.ExecuteWriteAsync(Instrument(work, "write", activity, payload)).ConfigureAwait(false);
+ return await session.ExecuteWriteAsync(
+ Instrument(work, "write", activity, payload, transactionEntryStartedAt)).ConfigureAwait(false);
}
catch (Exception ex)
{
@@ -97,6 +108,77 @@ public async Task WriteAsync(Func> work, Cancel
}
}
+ public async Task ExecuteAtomicWriteAsync(
+ Func> work,
+ CancellationToken cancellationToken = default)
+ {
+ cancellationToken.ThrowIfCancellationRequested();
+
+ // Nested logical units join the outer unit. This also keeps the transaction boundary
+ // well-defined when a higher-level persistence workflow composes another one.
+ if (_ambientWriteTransaction.Value is not null)
+ return await work(cancellationToken).ConfigureAwait(false);
+
+ using var activity = AgentMemoryDiagnostics.Source.StartActivity("memory.db.tx", ActivityKind.Client);
+ activity?.SetTag("db.mode", "write");
+ activity?.SetTag("db.transaction.logical_unit", true);
+ var payload = activity is null ? null : new PayloadAccumulator();
+ var transactionEntryStartedAt = activity is null ? 0 : Stopwatch.GetTimestamp();
+
+ var session = _sessionFactory.OpenSession(AccessMode.Write);
+ await using var _ = session.ConfigureAwait(false);
+ IAsyncTransaction? transaction = null;
+ try
+ {
+ // Explicit rather than managed transaction: the callback mutates in-memory outcome state
+ // and must execute exactly once. The caller owns any whole-operation retry after rollback.
+ transaction = await session.BeginTransactionAsync().ConfigureAwait(false);
+ activity?.SetTag(
+ "db.transaction_entry_ms_est",
+ Stopwatch.GetElapsedTime(transactionEntryStartedAt).TotalMilliseconds);
+
+ IAsyncQueryRunner ambientRunner = activity is null
+ ? transaction
+ : new CountingQueryRunner(transaction, "write", activity, payload!);
+ _ambientWriteTransaction.Value = ambientRunner;
+
+ var result = await work(cancellationToken).ConfigureAwait(false);
+ await transaction.CommitAsync().ConfigureAwait(false);
+ return result;
+ }
+ catch (Exception ex)
+ {
+ activity?.SetStatus(ActivityStatusCode.Error);
+ Exception? rollbackFailure = null;
+ if (transaction is not null)
+ {
+ try
+ {
+ await transaction.RollbackAsync().ConfigureAwait(false);
+ }
+ catch (Exception rollbackException)
+ {
+ rollbackFailure = rollbackException;
+ _logger.LogWarning(rollbackException, "Failed to roll back atomic memory transaction.");
+ }
+ }
+
+ if (rollbackFailure is not null)
+ throw new AggregateException(
+ "Atomic memory transaction failed and rollback could not be confirmed.", ex, rollbackFailure);
+
+ _logger.LogError(ex, "Error executing atomic memory transaction.");
+ throw;
+ }
+ finally
+ {
+ _ambientWriteTransaction.Value = null;
+ if (transaction is not null)
+ await transaction.DisposeAsync().ConfigureAwait(false);
+ TagPayload(activity, payload);
+ }
+ }
+
public async Task WriteAsync(Func work, CancellationToken cancellationToken = default)
{
await WriteAsync(async tx =>
@@ -116,13 +198,25 @@ private static Func> Instrument(
Func> work,
string mode,
Activity? transaction,
- PayloadAccumulator? payload) =>
+ PayloadAccumulator? payload,
+ long transactionEntryStartedAt) =>
transaction is null
? work
- : runner => work(new CountingQueryRunner(runner, mode, transaction, payload!));
+ : runner =>
+ {
+ // The driver's public API exposes acquisition counts and a timeout, but not wait duration.
+ // This upper-bound estimate starts immediately before ExecuteRead/WriteAsync and stops when
+ // its transaction callback begins. It therefore includes connection acquisition, routing,
+ // and transaction begin; the `_est` suffix is permanent and prevents a pure-pool-wait claim.
+ transaction.SetTag(
+ "db.transaction_entry_ms_est",
+ Stopwatch.GetElapsedTime(transactionEntryStartedAt).TotalMilliseconds);
+ return work(new CountingQueryRunner(runner, mode, transaction, payload!));
+ };
private static void TagPayload(Activity? activity, PayloadAccumulator? payload)
{
+
if (activity is null || payload is null) return;
activity.SetTag("db.records", payload.RecordCount);
activity.SetTag("db.bytes_est", payload.BytesEstimate);
diff --git a/src/AgentMemory.Neo4j/Infrastructure/RerankParameters.cs b/src/AgentMemory.Neo4j/Infrastructure/RerankParameters.cs
index e74ecec0..82de8f8b 100644
--- a/src/AgentMemory.Neo4j/Infrastructure/RerankParameters.cs
+++ b/src/AgentMemory.Neo4j/Infrastructure/RerankParameters.cs
@@ -25,6 +25,7 @@ public static void Add(
parameters["now"] = DateTimeOffset.UtcNow.ToString("O");
parameters["lambda"] = Math.Log(2) / halfLife;
parameters["boostFactor"] = decay.AccessBoostFactor;
+ parameters["maxBoost"] = decay.MaxAccessBoost;
parameters["tmpWeight"] = ranking.EffectiveRecencyWeight;
}
}
diff --git a/src/AgentMemory.Neo4j/Infrastructure/SchemaBootstrapper.cs b/src/AgentMemory.Neo4j/Infrastructure/SchemaBootstrapper.cs
index 7de276c2..9fba026c 100644
--- a/src/AgentMemory.Neo4j/Infrastructure/SchemaBootstrapper.cs
+++ b/src/AgentMemory.Neo4j/Infrastructure/SchemaBootstrapper.cs
@@ -1,6 +1,7 @@
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using AgentMemory.Abstractions.Exceptions;
+using AgentMemory.Core.Memory;
using AgentMemory.Neo4j.Queries;
using Neo4j.Driver;
@@ -14,6 +15,9 @@ internal sealed class SchemaBootstrapper : ISchemaBootstrapper
private readonly int _embeddingDimensions;
private readonly bool _validateVectorIndexDimensions;
+ /// Bounded so a large store migrates in pages rather than one transaction.
+ internal const int CanonicalKeyBackfillBatchSize = 500;
+
public SchemaBootstrapper(
INeo4jTransactionRunner txRunner,
IOptions options,
@@ -27,6 +31,76 @@ public SchemaBootstrapper(
_vectorIndexes = SchemaQueries.BuildVectorIndexes(_embeddingDimensions);
}
+ ///
+ /// Facts written before canonical identity carry no *_key properties, so a re-extracted
+ /// triple never MERGEs onto them and predicate expansion cannot see them. Backfills the keys
+ /// during bootstrap, before any write can occur on an upgraded store.
+ ///
+ ///
+ /// Computed in C# and never in Cypher: toLower() and
+ /// disagree on U+0130, so a Cypher backfill would write keys the write path never matches.
+ /// Idempotent by construction — it selects on predicate_key IS NULL, so a re-run over a
+ /// migrated store does nothing.
+ ///
+ internal async Task BackfillCanonicalFactKeysAsync(
+ int batchSize,
+ CancellationToken cancellationToken = default)
+ {
+ ArgumentOutOfRangeException.ThrowIfNegativeOrZero(batchSize);
+ var migrated = 0;
+
+ while (true)
+ {
+ cancellationToken.ThrowIfCancellationRequested();
+ var pending = await _txRunner.ReadAsync(async runner =>
+ {
+ var cursor = await runner.RunAsync(
+ FactQueries.SelectFactsMissingCanonicalKeys,
+ new { limit = batchSize }).ConfigureAwait(false);
+ var records = await cursor.ToListAsync().ConfigureAwait(false);
+ return records.Select(record => new CanonicalKeyBackfillRow(
+ record["id"].As(),
+ record["subject"].As(),
+ record["predicate"].As(),
+ record["object"].As())).ToList();
+ }, cancellationToken).ConfigureAwait(false);
+
+ if (pending.Count == 0)
+ break;
+
+ var items = pending.Select(fact => new Dictionary
+ {
+ ["id"] = fact.Id,
+ ["subject_key"] = MemoryTripleCanonicalizer.CanonicalValue(fact.Subject),
+ ["predicate_key"] = MemoryTripleCanonicalizer.Canonical(fact.Predicate),
+ ["object_key"] = MemoryTripleCanonicalizer.CanonicalValue(fact.Object)
+ }).ToList();
+
+ await _txRunner.WriteAsync(async runner =>
+ {
+ var cursor = await runner.RunAsync(
+ FactQueries.ApplyCanonicalKeys, new { items }).ConfigureAwait(false);
+ await cursor.ConsumeAsync().ConfigureAwait(false);
+ return true;
+ }, cancellationToken).ConfigureAwait(false);
+
+ migrated += pending.Count;
+
+ // A short final batch means the last page was reached; anything else would re-query for
+ // a page that cannot exist.
+ if (pending.Count < batchSize)
+ break;
+ }
+
+ if (migrated > 0)
+ {
+ _logger.LogInformation(
+ "Backfilled canonical identity keys onto {Count} pre-existing facts.", migrated);
+ }
+
+ return migrated;
+ }
+
public async Task BootstrapAsync(CancellationToken cancellationToken = default)
{
_logger.LogInformation(
@@ -63,10 +137,52 @@ public async Task BootstrapAsync(CancellationToken cancellationToken = default)
// already exists, so an embedder/dimension change leaves stale indexes that would only fail at
// query time. Verify dimensions now and surface an actionable error listing every mismatch.
await ValidateVectorIndexDimensionsAsync(cancellationToken).ConfigureAwait(false);
+ await ValidateNoFailedIndexesAsync(cancellationToken).ConfigureAwait(false);
+
+ // Ordering matters and is the whole point: a fact written between an upgrade and the
+ // backfill would MERGE onto a fresh node and duplicate anyway. Bootstrap runs before any
+ // repository write, so this is the only place it is guaranteed to precede them.
+ await BackfillCanonicalFactKeysAsync(CanonicalKeyBackfillBatchSize, cancellationToken)
+ .ConfigureAwait(false);
_logger.LogInformation("Schema bootstrap complete.");
}
+ ///
+ /// Surfaces indexes that reached the terminal FAILED state. Only vector dimensions were checked
+ /// before, so a range index that could not populate — Neo4j caps index keys at roughly 8 KB, and
+ /// nothing bounds fact property length — degraded invisibly: queries still succeeded through full
+ /// scans, so the symptom was gradual slowness rather than an error.
+ ///
+ private async Task ValidateNoFailedIndexesAsync(CancellationToken cancellationToken)
+ {
+ cancellationToken.ThrowIfCancellationRequested();
+
+ var failed = await _txRunner.ReadAsync(
+ async runner =>
+ {
+ var cursor = await runner.RunAsync(SchemaQueries.ShowIndexStates).ConfigureAwait(false);
+ var records = await cursor.ToListAsync().ConfigureAwait(false);
+ // Filtered in memory rather than in Cypher: a literal in a WHERE clause trips the
+ // repository's parameterization guard, and POPULATING is a normal transient state.
+ return records
+ .Where(record => string.Equals(
+ record["state"].As(), "FAILED", StringComparison.OrdinalIgnoreCase))
+ .Select(record => $"{record["name"].As()} ({record["type"].As()})")
+ .ToArray();
+ },
+ cancellationToken).ConfigureAwait(false) ?? [];
+
+ if (failed.Length > 0)
+ {
+ throw new InvalidOperationException(
+ $"Neo4j reports {failed.Length} index(es) in the FAILED state: {string.Join(", ", failed)}. " +
+ "A failed index does not stop queries — they fall back to full scans — so this would " +
+ "otherwise surface only as unexplained slowness. Drop and recreate the index; if it " +
+ "covers long text properties, note that Neo4j limits index keys to roughly 8 KB.");
+ }
+ }
+
private async Task ValidateVectorIndexDimensionsAsync(CancellationToken cancellationToken)
{
if (!_validateVectorIndexDimensions)
diff --git a/src/AgentMemory.Neo4j/Infrastructure/ServiceCollectionExtensions.cs b/src/AgentMemory.Neo4j/Infrastructure/ServiceCollectionExtensions.cs
index 05a54846..7f24815f 100644
--- a/src/AgentMemory.Neo4j/Infrastructure/ServiceCollectionExtensions.cs
+++ b/src/AgentMemory.Neo4j/Infrastructure/ServiceCollectionExtensions.cs
@@ -2,6 +2,7 @@
using Microsoft.Extensions.DependencyInjection.Extensions;
using Neo4j.Driver;
using AgentMemory.Abstractions.Repositories;
+using AgentMemory.Core.Extraction;
using AgentMemory.Abstractions.Services;
using AgentMemory.Neo4j.Repositories;
using AgentMemory.Neo4j.Services;
@@ -36,7 +37,10 @@ public static IServiceCollection AddNeo4jAgentMemory(
// INeo4jDriverFactory still owns creation and disposal.
services.TryAddSingleton(sp => sp.GetRequiredService().GetDriver());
services.TryAddSingleton();
- services.TryAddTransient();
+ // Singleton so repository instances in the same async flow can join the transaction opened
+ // for one logical extraction-persistence operation.
+ services.TryAddSingleton();
+ services.Replace(ServiceDescriptor.Singleton());
services.TryAddTransient();
services.TryAddTransient();
diff --git a/src/AgentMemory.Neo4j/Queries/CypherQueryRegistry.cs b/src/AgentMemory.Neo4j/Queries/CypherQueryRegistry.cs
index 8771dfb8..7d5355d2 100644
--- a/src/AgentMemory.Neo4j/Queries/CypherQueryRegistry.cs
+++ b/src/AgentMemory.Neo4j/Queries/CypherQueryRegistry.cs
@@ -39,7 +39,18 @@ internal static string FingerprintFor(string? cypher)
: "DecayQueries.UpdateAccessTimestamp";
}
- if (Has("CALL db.index.vector.queryNodes('message_embedding_idx'") &&
+ if (Has("WITH $messages AS messages") &&
+ Has("[msg IN $messages | msg.id] AS batchIds") &&
+ Has("WITH DISTINCT msg.id AS id"))
+ {
+ return "MessageQueries.AddBatchOptimized";
+ }
+
+ var isMessageVectorSearch =
+ Has("CALL db.index.vector.queryNodes('message_embedding_idx'") ||
+ (Has("MATCH (:Conversation {session_id: $sessionId})-[:HAS_MESSAGE]->(node:Message)") &&
+ Has("vector.similarity.cosine(node.embedding, $embedding)"));
+ if (isMessageVectorSearch &&
Has("RETURN node, score"))
{
return "MessageQueries.SearchByVector";
@@ -55,6 +66,12 @@ internal static string FingerprintFor(string? cypher)
return "EntityQueries.SearchByVector";
}
+ if (Has("MATCH (node:Fact)") &&
+ Has("vector.similarity.cosine(node.embedding, $embedding)") &&
+ Has("toLower(node.subject) = toLower($subject)") &&
+ Has("toLower(node.predicate) = toLower($predicate)"))
+ return "FactQueries.FindDuplicate";
+
if (Has("CALL db.index.vector.queryNodes('fact_embedding_idx'"))
{
if (Has("node.created_at <= datetime($systemAsOf)"))
diff --git a/src/AgentMemory.Neo4j/Queries/DecayQueries.cs b/src/AgentMemory.Neo4j/Queries/DecayQueries.cs
index 3c35d2a4..c5044c77 100644
--- a/src/AgentMemory.Neo4j/Queries/DecayQueries.cs
+++ b/src/AgentMemory.Neo4j/Queries/DecayQueries.cs
@@ -98,7 +98,12 @@ private static string BuildPrune(string a, string label, bool hasOwnerFilter, bo
// Clamp daysSince to >= 0 so the prune score matches the C# read-path score exactly for nodes
// with a future last_accessed_at (a negative exponent would otherwise inflate the score).
" WITH " + a + ", conf, ac, CASE WHEN rawDays < 0 THEN 0.0 ELSE rawDays END AS daysSince\n" +
- " WHERE (COALESCE(conf, 0.5) * exp(-$lambda * daysSince) + $boostFactor * ac) < $minScore\n" +
+ // BUG-R7: the access term is damped (log), capped ($maxBoost), and decayed on the same
+ // curve as confidence, so a single recall can no longer hold a stale node above $minScore
+ // forever. Must stay identical to MemoryDecayService.ComputeScore and VectorRerank.
+ " WHERE ((COALESCE(conf, 0.5) + CASE WHEN $boostFactor * log(1 + ac) > $maxBoost" +
+ " THEN $maxBoost ELSE $boostFactor * log(1 + ac) END)" +
+ " * exp(-$lambda * daysSince)) < $minScore\n" +
" " + action + "\n" +
" RETURN count(*) AS pruned";
}
diff --git a/src/AgentMemory.Neo4j/Queries/FactQueries.cs b/src/AgentMemory.Neo4j/Queries/FactQueries.cs
index 0282649e..894d37e9 100644
--- a/src/AgentMemory.Neo4j/Queries/FactQueries.cs
+++ b/src/AgentMemory.Neo4j/Queries/FactQueries.cs
@@ -9,12 +9,83 @@ namespace AgentMemory.Neo4j.Queries;
///
internal static class FactQueries
{
+ // ── Canonical-key backfill (Phase 1.1) ─────────────────────────────
+
+ /// Facts written before canonical identity, in bounded batches.
+ ///
+ /// Selecting on predicate_key IS NULL makes the backfill idempotent by construction: once
+ /// every fact is keyed, a re-run selects nothing. Bounded so a large store migrates in batches
+ /// rather than one transaction.
+ ///
+ public const string SelectFactsMissingCanonicalKeys = @"
+ MATCH (f:Fact)
+ WHERE f.predicate_key IS NULL
+ RETURN f.id AS id, f.subject AS subject, f.predicate AS predicate, f.object AS object
+ LIMIT $limit";
+
+ ///
+ /// Writes canonical keys computed in C# onto facts identified by id.
+ ///
+ ///
+ /// Deliberately contains no toLower() or string rewriting: Cypher's toLower() and
+ /// .NET's ToLowerInvariant() disagree on U+0130, so a key computed here would not match
+ /// the one the write path produces, silently reintroducing the duplication canonical identity
+ /// exists to remove. That is also why this is not a .cypher migration file.
+ ///
+ public const string ApplyCanonicalKeys = @"
+ UNWIND $items AS item
+ MATCH (f:Fact {id: item.id})
+ SET f.subject_key = item.subject_key,
+ f.predicate_key = item.predicate_key,
+ f.object_key = item.object_key
+ RETURN count(f) AS updated";
+
+ // ── Predicate expansion (G3B.13) ───────────────────────────────────
+
+ ///
+ /// Every fact an owner holds under the given canonical predicates, bounded.
+ ///
+ ///
+ /// Top-K vector search answers "what is most relevant"; it cannot answer "how many", because a
+ /// relevance cutoff gives no completeness guarantee — miss one of five births and the count is
+ /// four. This retrieves a relation whole so aggregation questions become answerable, and
+ /// composes with top-K rather than replacing it: similarity finds which predicate matters, this
+ /// makes that predicate complete.
+ ///
+ /// Matches predicate_key, never the raw predicate: raw text would reinstate the exact
+ /// fragmentation canonical identity removed, where "were_born_in" and "were born in" fail to
+ /// find each other. Owner-scoped and explicitly limited — unbounded completeness over a graph of
+ /// ~1,000 facts would simply exhaust the answer budget.
+ ///
+ ///
+ public static string SearchByCanonicalPredicates(bool hasOwnerFilter, bool includeShared)
+ {
+ // Mirrors GetBySubject's owner-conditional shape rather than inventing its own. The first
+ // version hard-coded `f.owner_key = $ownerKey` with `scope.OwnerId ?? OwnerKeyShared`, which
+ // (a) never matched shared facts even when IncludeShared was set, silently breaking the
+ // "relation whole" guarantee, and (b) coerced a null-owner scope to the shared bucket, so it
+ // returned nothing exactly where top-K returned everything.
+ var owner = !hasOwnerFilter ? string.Empty
+ : includeShared ? " AND (f.owner_id = $ownerId OR f.owner_id IS NULL)"
+ : " AND f.owner_id = $ownerId";
+ return $@"
+ MATCH (f:Fact)
+ WHERE f.predicate_key IN $predicateKeys
+ AND f.invalidated_at IS NULL{owner}
+ RETURN f
+ ORDER BY f.confidence DESC, f.id ASC
+ LIMIT $limit";
+ }
+
// ── UpsertAsync ────────────────────────────────────────────────────
/// Merge a fact by subject/predicate/object triple, setting all properties.
public const string Upsert = @"
- MERGE (f:Fact {subject: $subject, predicate: $predicate, object: $object, owner_key: $ownerKey})
+ MERGE (f:Fact {subject_key: $subjectKey, predicate_key: $predicateKey, object_key: $objectKey, owner_key: $ownerKey})
ON CREATE SET
+ f.subject = $subject,
+ f.predicate = $predicate,
+ f.object = $object,
f.id = $id,
f.owner_id = $ownerId,
f.category = $category,
@@ -49,8 +120,11 @@ ON MATCH SET
///
public const string UpsertBatch = @"
UNWIND $items AS item
- MERGE (f:Fact {subject: item.subject, predicate: item.predicate, object: item.object, owner_key: item.owner_key})
+ MERGE (f:Fact {subject_key: item.subject_key, predicate_key: item.predicate_key, object_key: item.object_key, owner_key: item.owner_key})
ON CREATE SET
+ f.subject = item.subject,
+ f.predicate = item.predicate,
+ f.object = item.object,
f.id = item.id,
f.owner_id = item.owner_id,
f.category = item.category,
@@ -90,18 +164,19 @@ public static string GetBySubject(bool hasOwnerFilter, bool includeShared)
// ── Dedup-on-create ────────────────────────────────────────────────
///
- /// Finds the most-similar existing fact with the same subject+predicate within the same owner
- /// (matched by owner_key) whose cosine score ≥ $threshold — used to reinforce instead
- /// of creating a near-duplicate node. Over-fetches candidates, returns top 1.
+ /// Scopes live candidates to the same owner + case-insensitive subject/predicate before exact cosine
+ /// scoring, then returns the best match above $threshold. Scoped exact scoring gives a caller
+ /// holding the process-local dedup lock read-after-commit behavior without vector-index refresh lag.
///
- public static string FindDuplicate(int topK) => $@"
- CALL db.index.vector.queryNodes('fact_embedding_idx', {topK}, $embedding)
- YIELD node, score
- WHERE score >= $threshold
- AND node.invalidated_at IS NULL
+ public static string FindDuplicate() => @"
+ MATCH (node:Fact)
+ WHERE node.invalidated_at IS NULL
+ AND node.owner_key = $ownerKey
AND toLower(node.subject) = toLower($subject)
AND toLower(node.predicate) = toLower($predicate)
- AND node.owner_key = $ownerKey
+ AND node.embedding IS NOT NULL
+ WITH node, vector.similarity.cosine(node.embedding, $embedding) AS score
+ WHERE score >= $threshold
RETURN node, score
ORDER BY score DESC
LIMIT 1";
diff --git a/src/AgentMemory.Neo4j/Queries/FusedPersistenceQueries.cs b/src/AgentMemory.Neo4j/Queries/FusedPersistenceQueries.cs
new file mode 100644
index 00000000..fa9622ca
--- /dev/null
+++ b/src/AgentMemory.Neo4j/Queries/FusedPersistenceQueries.cs
@@ -0,0 +1,122 @@
+namespace AgentMemory.Neo4j.Queries;
+
+///
+/// Bounded memory-kind writes used by the coalesced extraction path. Each query folds node mutation,
+/// embedding and message provenance into one round trip so an outer transaction does not retain locks
+/// across per-item follow-up queries.
+///
+internal static class FusedPersistenceQueries
+{
+ public const string EntityUpsertBatch = @"
+ UNWIND $items AS item
+ MERGE (e:Entity {id: item.id})
+ ON CREATE SET
+ e.owner_id = item.owner_id,
+ e.name = item.name,
+ e.canonical_name = item.canonical_name,
+ e.type = item.type,
+ e.subtype = item.subtype,
+ e.description = item.description,
+ e.confidence = item.confidence,
+ e.aliases = item.aliases,
+ e.attributes = item.attributes,
+ e.source_message_ids = item.source_message_ids,
+ e.created_at = datetime(item.created_at),
+ e.metadata = item.metadata
+ ON MATCH SET
+ e.name = item.name,
+ e.canonical_name = item.canonical_name,
+ e.type = item.type,
+ e.subtype = item.subtype,
+ e.description = item.description,
+ e.confidence = item.confidence,
+ e.aliases = item.aliases,
+ e.attributes = item.attributes,
+ e.source_message_ids = item.source_message_ids,
+ e.metadata = item.metadata,
+ e.updated_at = datetime()
+ SET e.embedding = CASE
+ WHEN item.embedding IS NOT NULL AND size(item.embedding) > 0 THEN item.embedding
+ ELSE e.embedding END
+ FOREACH (_ IN CASE
+ WHEN item.latitude IS NOT NULL AND item.longitude IS NOT NULL THEN [1]
+ ELSE [] END |
+ SET e.location = point({latitude: item.latitude, longitude: item.longitude}))
+ SET e:$(item.labels)
+ WITH e, item
+ CALL (e, item) {
+ UNWIND item.source_message_ids AS msgId
+ MATCH (m:Message {id: msgId})
+ MERGE (e)-[:EXTRACTED_FROM]->(m)
+ RETURN count(*) AS linked
+ }
+ RETURN e";
+
+ public const string FactUpsertBatch = @"
+ UNWIND $items AS item
+ MERGE (f:Fact {subject_key: item.subject_key, predicate_key: item.predicate_key, object_key: item.object_key, owner_key: item.owner_key})
+ ON CREATE SET
+ f.subject = item.subject,
+ f.predicate = item.predicate,
+ f.object = item.object,
+ f.id = item.id,
+ f.owner_id = item.owner_id,
+ f.category = item.category,
+ f.confidence = item.confidence,
+ f.valid_from = CASE WHEN item.valid_from IS NOT NULL THEN datetime(item.valid_from) ELSE null END,
+ f.valid_until = CASE WHEN item.valid_until IS NOT NULL THEN datetime(item.valid_until) ELSE null END,
+ f.source_message_ids = item.source_message_ids,
+ f.created_at = datetime(item.created_at),
+ f.metadata = item.metadata
+ ON MATCH SET
+ f.category = item.category,
+ f.confidence = item.confidence,
+ f.valid_from = CASE WHEN item.valid_from IS NOT NULL THEN datetime(item.valid_from) ELSE f.valid_from END,
+ f.valid_until = CASE WHEN item.valid_until IS NOT NULL THEN datetime(item.valid_until) ELSE f.valid_until END,
+ f.source_message_ids = item.source_message_ids,
+ f.updated_at = datetime(item.updated_at),
+ f.metadata = item.metadata,
+ f.invalidated_at = null
+ SET f.embedding = CASE
+ WHEN item.embedding IS NOT NULL AND size(item.embedding) > 0 THEN item.embedding
+ ELSE f.embedding END
+ WITH f, item
+ CALL (f, item) {
+ UNWIND item.source_message_ids AS msgId
+ MATCH (m:Message {id: msgId})
+ MERGE (f)-[:EXTRACTED_FROM]->(m)
+ RETURN count(*) AS linked
+ }
+ RETURN f";
+
+ public const string PreferenceUpsertBatch = @"
+ UNWIND $items AS item
+ MERGE (p:Preference {id: item.id})
+ ON CREATE SET
+ p.owner_id = item.owner_id,
+ p.category = item.category,
+ p.preference = item.preference,
+ p.context = item.context,
+ p.confidence = item.confidence,
+ p.source_message_ids = item.source_message_ids,
+ p.created_at = datetime(item.created_at),
+ p.metadata = item.metadata
+ ON MATCH SET
+ p.category = item.category,
+ p.preference = item.preference,
+ p.context = item.context,
+ p.confidence = item.confidence,
+ p.source_message_ids = item.source_message_ids,
+ p.metadata = item.metadata
+ SET p.embedding = CASE
+ WHEN item.embedding IS NOT NULL AND size(item.embedding) > 0 THEN item.embedding
+ ELSE p.embedding END
+ WITH p, item
+ CALL (p, item) {
+ UNWIND item.source_message_ids AS msgId
+ MATCH (m:Message {id: msgId})
+ MERGE (p)-[:EXTRACTED_FROM]->(m)
+ RETURN count(*) AS linked
+ }
+ RETURN p";
+}
diff --git a/src/AgentMemory.Neo4j/Queries/MessageQueries.cs b/src/AgentMemory.Neo4j/Queries/MessageQueries.cs
index fef1f6c8..f3417bb8 100644
--- a/src/AgentMemory.Neo4j/Queries/MessageQueries.cs
+++ b/src/AgentMemory.Neo4j/Queries/MessageQueries.cs
@@ -11,7 +11,8 @@ internal static class MessageQueries
{
// ── AddAsync ───────────────────────────────────────────────────────
- /// Create a message and link it to its conversation via HAS_MESSAGE. The conversation
+ /// Create a message, persist its optional embedding, and maintain its conversation/order
+ /// links in one query. The conversation
/// is MERGE-d so persisting a message never silently no-ops when the conversation was not
/// explicitly created first (e.g. from the MAF context/history providers); a thin conversation
/// is created and later enriched by ConversationQueries.Upsert.
@@ -36,8 +37,26 @@ ON CREATE SET
m.timestamp = datetime($timestamp),
m.tool_call_ids = $toolCallIds,
m.metadata = $metadata
+ WITH conv, m, m { .* } AS persisted
+ SET m.embedding = CASE
+ WHEN $embedding IS NOT NULL THEN $embedding
+ ELSE m.embedding
+ END
MERGE (conv)-[:HAS_MESSAGE]->(m)
- RETURN m";
+ WITH conv, m, persisted
+ OPTIONAL MATCH (conv)-[:FIRST_MESSAGE]->(first:Message)
+ FOREACH (_ IN CASE WHEN first IS NULL THEN [1] ELSE [] END |
+ MERGE (conv)-[:FIRST_MESSAGE]->(m)
+ )
+ WITH conv, m, persisted
+ OPTIONAL MATCH (conv)-[:HAS_MESSAGE]->(prev:Message)
+ WHERE prev.id <> $id
+ WITH m, persisted, prev ORDER BY prev.timestamp DESC
+ WITH m, persisted, head(collect(prev)) AS prev
+ FOREACH (_ IN CASE WHEN prev IS NULL THEN [] ELSE [1] END |
+ MERGE (prev)-[:NEXT_MESSAGE]->(m)
+ )
+ RETURN persisted AS m";
/// Link the first message in a conversation via FIRST_MESSAGE.
public const string CreateFirstMessageLink = @"
@@ -88,6 +107,62 @@ ON CREATE SET
MERGE (conv)-[:HAS_MESSAGE]->(m)
RETURN m";
+ ///
+ /// One-query batch write preserving 's behavior: first-write-wins message
+ /// properties, unconditional overwrite for supplied embeddings, intra-batch ordering, connection to
+ /// the prior conversation tail, and ordered read-back. The input must already be timestamp ordered.
+ ///
+ public static string AddBatchOptimized { get; } = @"
+ WITH $messages AS messages,
+ [msg IN $messages | msg.id] AS batchIds
+ UNWIND messages AS msg
+ MERGE (conv:Conversation {id: msg.conversation_id})
+ ON CREATE SET conv.session_id = msg.session_id,
+ conv.created_at = datetime(msg.timestamp),
+ conv.updated_at = datetime(msg.timestamp)
+ MERGE (m:Message {id: msg.id})
+ ON CREATE SET
+ m.conversation_id = msg.conversation_id,
+ m.session_id = msg.session_id,
+ m.role = msg.role,
+ m.content = msg.content,
+ m.timestamp = datetime(msg.timestamp),
+ m.tool_call_ids = msg.tool_call_ids,
+ m.metadata = msg.metadata
+ FOREACH (_ IN CASE WHEN msg.embedding IS NULL THEN [] ELSE [1] END |
+ SET m.embedding = msg.embedding
+ )
+ MERGE (conv)-[:HAS_MESSAGE]->(m)
+ WITH messages, batchIds
+ CALL {
+ WITH messages
+ UNWIND CASE
+ WHEN size(messages) > 1 THEN range(1, size(messages) - 1)
+ ELSE []
+ END AS i
+ MATCH (prev:Message {id: messages[i - 1].id})
+ MATCH (next:Message {id: messages[i].id})
+ MERGE (prev)-[:NEXT_MESSAGE]->(next)
+ RETURN count(*) AS linked
+ }
+ WITH messages, batchIds
+ MATCH (conv:Conversation {id: messages[0].conversation_id})
+ MATCH (first:Message {id: messages[0].id})
+ OPTIONAL MATCH (conv)-[:HAS_MESSAGE]->(prev:Message)
+ WHERE NOT prev.id IN batchIds
+ WITH messages, first, prev
+ ORDER BY prev.timestamp DESC
+ WITH messages, first, head(collect(prev)) AS prev
+ FOREACH (_ IN CASE WHEN prev IS NULL THEN [] ELSE [1] END |
+ MERGE (prev)-[:NEXT_MESSAGE]->(first)
+ )
+ WITH messages
+ UNWIND messages AS msg
+ WITH DISTINCT msg.id AS id
+ MATCH (m:Message {id: id})
+ RETURN m
+ ORDER BY m.timestamp";
+
/// Create NEXT_MESSAGE link between two specific messages. MERGE (not CREATE) for the same
/// idempotency guarantee as .
public const string CreateNextMessageLink =
@@ -142,22 +217,39 @@ RETURN m
///
/// Builds a vector similarity search query for messages with optional session and metadata filters.
- /// The value is embedded in the CALL as a literal integer.
+ /// Session-scoped search uses the indexed Conversation session id, traverses HAS_MESSAGE, and
+ /// calculates exact cosine inside that session; unscoped search uses the global vector index.
///
- /// When true, adds an AND clause for node.session_id = $sessionId.
+ /// When true, scopes traversal through Conversation.session_id.
///
/// Optional pre-formatted AND condition lines from .
///
- /// Number of candidates to retrieve from the vector index.
- public static string SearchByVector(bool hasSessionFilter, string? metadataFilterFragment = null, int topK = 10) =>
- new CypherBuilder()
+ /// Number of candidates to retrieve from the unscoped vector index.
+ public static string SearchByVector(bool hasSessionFilter, string? metadataFilterFragment = null, int topK = 10)
+ {
+ if (hasSessionFilter)
+ {
+ return $$"""
+ MATCH (:Conversation {session_id: $sessionId})-[:HAS_MESSAGE]->(node:Message)
+ WHERE node.embedding IS NOT NULL AND size(node.embedding) = size($embedding)
+ {{metadataFilterFragment}}
+ WITH node, vector.similarity.cosine(node.embedding, $embedding) AS score
+ WHERE score >= $minScore
+ RETURN node, score
+ ORDER BY score DESC
+ LIMIT $limit
+ """;
+ }
+
+ return new CypherBuilder()
.WithVectorSearch("message_embedding_idx", "$embedding", "node", topK)
.Where("score >= $minScore")
- .And("node.session_id = $sessionId", when: hasSessionFilter)
.AndRawFragment(metadataFilterFragment)
.Return("node, score")
.OrderBy("score DESC")
+ .Limit("$limit", when: !string.IsNullOrWhiteSpace(metadataFilterFragment))
.Build();
+ }
// ── DeleteBySessionAsync ───────────────────────────────────────────
diff --git a/src/AgentMemory.Neo4j/Queries/PreferenceQueries.cs b/src/AgentMemory.Neo4j/Queries/PreferenceQueries.cs
index 474a13a9..b98c1d4e 100644
--- a/src/AgentMemory.Neo4j/Queries/PreferenceQueries.cs
+++ b/src/AgentMemory.Neo4j/Queries/PreferenceQueries.cs
@@ -28,6 +28,27 @@ ON MATCH SET
p.metadata = $metadata
RETURN p";
+ /// Batch upsert preferences by id via UNWIND.
+ public const string UpsertBatch = @"
+ UNWIND $items AS item
+ MERGE (p:Preference {id: item.id})
+ ON CREATE SET
+ p.owner_id = item.owner_id,
+ p.category = item.category,
+ p.preference = item.preference,
+ p.context = item.context,
+ p.confidence = item.confidence,
+ p.source_message_ids = item.source_message_ids,
+ p.created_at = datetime(item.created_at),
+ p.metadata = item.metadata
+ ON MATCH SET
+ p.category = item.category,
+ p.preference = item.preference,
+ p.context = item.context,
+ p.confidence = item.confidence,
+ p.source_message_ids = item.source_message_ids,
+ p.metadata = item.metadata
+ RETURN p";
/// Set the embedding vector on a Preference node.
public const string SetEmbedding = "MATCH (p:Preference {id: $id}) SET p.embedding = $embedding";
diff --git a/src/AgentMemory.Neo4j/Queries/RelationshipQueries.cs b/src/AgentMemory.Neo4j/Queries/RelationshipQueries.cs
index 78e9fd25..c050bf4c 100644
--- a/src/AgentMemory.Neo4j/Queries/RelationshipQueries.cs
+++ b/src/AgentMemory.Neo4j/Queries/RelationshipQueries.cs
@@ -40,6 +40,37 @@ ON MATCH SET
r.metadata = $metadata
RETURN r";
+ /// Batch merge RELATED_TO relationships by id via UNWIND.
+ public const string UpsertBatch = @"
+ UNWIND $items AS item
+ MERGE (s:Entity {id: item.source_entity_id})
+ MERGE (t:Entity {id: item.target_entity_id})
+ MERGE (s)-[r:RELATED_TO {id: item.id}]->(t)
+ ON CREATE SET
+ r.relation_type = item.relation_type,
+ r.owner_id = item.owner_id,
+ r.source_entity_id = item.source_entity_id,
+ r.target_entity_id = item.target_entity_id,
+ r.confidence = item.confidence,
+ r.description = item.description,
+ r.valid_from = CASE WHEN item.valid_from IS NOT NULL THEN datetime(item.valid_from) ELSE null END,
+ r.valid_until = CASE WHEN item.valid_until IS NOT NULL THEN datetime(item.valid_until) ELSE null END,
+ r.attributes = item.attributes,
+ r.source_message_ids = item.source_message_ids,
+ r.created_at = datetime(item.created_at),
+ r.updated_at = datetime(item.updated_at),
+ r.metadata = item.metadata
+ ON MATCH SET
+ r.relation_type = item.relation_type,
+ r.confidence = item.confidence,
+ r.description = item.description,
+ r.valid_from = CASE WHEN item.valid_from IS NOT NULL THEN datetime(item.valid_from) ELSE null END,
+ r.valid_until = CASE WHEN item.valid_until IS NOT NULL THEN datetime(item.valid_until) ELSE null END,
+ r.attributes = item.attributes,
+ r.source_message_ids = item.source_message_ids,
+ r.updated_at = datetime(item.updated_at),
+ r.metadata = item.metadata
+ RETURN r";
// ── GetByIdAsync ───────────────────────────────────────────────────
/// Get a single RELATED_TO relationship by id.
diff --git a/src/AgentMemory.Neo4j/Queries/SchemaQueries.cs b/src/AgentMemory.Neo4j/Queries/SchemaQueries.cs
index d795acc1..d1a68c83 100644
--- a/src/AgentMemory.Neo4j/Queries/SchemaQueries.cs
+++ b/src/AgentMemory.Neo4j/Queries/SchemaQueries.cs
@@ -224,6 +224,17 @@ public static IReadOnlyList BootstrapStatements(int dimensions)
"SHOW VECTOR INDEXES YIELD name, options " +
"RETURN name AS name, options['indexConfig']['vector.dimensions'] AS dimensions";
+ ///
+ /// Lists indexes in the terminal FAILED state. Bootstrap previously validated only vector-index
+ /// dimensions, so a range index that failed to populate — for example when a composite key
+ /// exceeds Neo4j's ~8 KB key-size limit — degraded silently: queries kept working through full
+ /// scans and nothing ever reported the index missing. POPULATING is deliberately not treated as
+ /// a failure; it is the normal asynchronous build state.
+ ///
+ public const string ShowIndexStates =
+ "SHOW INDEXES YIELD name, state, type " +
+ "RETURN name AS name, state AS state, type AS type";
+
// ── Schema-conformance introspection (CLI `schema-check`) ────
/// Lists the names of all constraints in the current database.
diff --git a/src/AgentMemory.Neo4j/Queries/VectorRerank.cs b/src/AgentMemory.Neo4j/Queries/VectorRerank.cs
index abeab423..316a1a95 100644
--- a/src/AgentMemory.Neo4j/Queries/VectorRerank.cs
+++ b/src/AgentMemory.Neo4j/Queries/VectorRerank.cs
@@ -19,9 +19,16 @@ namespace AgentMemory.Neo4j.Queries;
///
internal static class VectorRerank
{
- // confidence·e^(−λ·daysSince) + boost·accessCount, with COALESCE fallbacks matching the prune.
+ // (confidence + damped, capped accessBoost)·e^(−λ·daysSince), with COALESCE fallbacks matching the
+ // prune. BUG-R7: the boost was linear, uncapped, and outside the decay, so at the shipped 0.2
+ // factor five accesses drove it to 1.0 — where the clamp below pinned retention at its maximum
+ // permanently and every frequently-recalled item tied at the ceiling, destroying the confidence
+ // and recency signal this blend exists to carry.
private const string RetentionExpr =
- "COALESCE(node.confidence, 0.5) * exp(-$lambda * daysSince) + $boostFactor * COALESCE(node.access_count, 0)";
+ "(COALESCE(node.confidence, 0.5) + " +
+ "CASE WHEN $boostFactor * log(1 + COALESCE(node.access_count, 0)) > $maxBoost " +
+ "THEN $maxBoost ELSE $boostFactor * log(1 + COALESCE(node.access_count, 0)) END) " +
+ "* exp(-$lambda * daysSince)";
///
/// Appends the (optional) recency-rerank blend, then RETURN / ORDER BY / LIMIT, and builds the query.
diff --git a/src/AgentMemory.Neo4j/Repositories/Neo4jEntityRepository.Fused.cs b/src/AgentMemory.Neo4j/Repositories/Neo4jEntityRepository.Fused.cs
new file mode 100644
index 00000000..ce2b0787
--- /dev/null
+++ b/src/AgentMemory.Neo4j/Repositories/Neo4jEntityRepository.Fused.cs
@@ -0,0 +1,59 @@
+using AgentMemory.Abstractions.Domain;
+using AgentMemory.Neo4j.Queries;
+using Microsoft.Extensions.Logging;
+using Neo4j.Driver;
+using static AgentMemory.Neo4j.Repositories.Neo4jRecordMapper;
+
+namespace AgentMemory.Neo4j.Repositories;
+
+internal sealed partial class Neo4jEntityRepository
+{
+ public async Task> UpsertFusedBatchAsync(
+ IReadOnlyList entities,
+ CancellationToken cancellationToken = default)
+ {
+ if (entities.Count == 0) return Array.Empty();
+
+ _logger.LogDebug("Fused batch upserting {Count} entities", entities.Count);
+ var items = entities.Select(entity => new Dictionary
+ {
+ ["id"] = entity.EntityId,
+ ["owner_id"] = entity.OwnerId,
+ ["name"] = entity.Name,
+ ["canonical_name"] = entity.CanonicalName,
+ ["type"] = entity.Type,
+ ["subtype"] = entity.Subtype,
+ ["description"] = entity.Description,
+ ["confidence"] = entity.Confidence,
+ ["aliases"] = entity.Aliases.ToList(),
+ ["attributes"] = SerializeMetadata(entity.Attributes),
+ ["source_message_ids"] = entity.SourceMessageIds.ToList(),
+ ["created_at"] = entity.CreatedAtUtc.ToString("O"),
+ ["metadata"] = SerializeMetadata(entity.Metadata),
+ ["embedding"] = entity.Embedding is { Length: > 0 } ? entity.Embedding.ToList() : null,
+ ["latitude"] = entity.Latitude,
+ ["longitude"] = entity.Longitude,
+ ["labels"] = BuildDynamicLabels(entity.Type, entity.Subtype),
+ }).ToList();
+
+ return await _tx.WriteAsync(async runner =>
+ {
+ var cursor = await runner.RunAsync(FusedPersistenceQueries.EntityUpsertBatch, new { items })
+ .ConfigureAwait(false);
+ var records = await cursor.ToListAsync().ConfigureAwait(false);
+ var byId = entities.ToDictionary(entity => entity.EntityId, StringComparer.Ordinal);
+ return records.Select(record =>
+ {
+ var node = record["e"].As();
+ var id = node["id"].As();
+ if (!byId.TryGetValue(id, out var source))
+ return MapToEntity(node, ReadEmbedding(node));
+ return MapToEntity(node, source.Embedding) with
+ {
+ Latitude = source.Latitude,
+ Longitude = source.Longitude,
+ };
+ }).ToList();
+ }, cancellationToken).ConfigureAwait(false);
+ }
+}
diff --git a/src/AgentMemory.Neo4j/Repositories/Neo4jEntityRepository.cs b/src/AgentMemory.Neo4j/Repositories/Neo4jEntityRepository.cs
index 978999d2..95520890 100644
--- a/src/AgentMemory.Neo4j/Repositories/Neo4jEntityRepository.cs
+++ b/src/AgentMemory.Neo4j/Repositories/Neo4jEntityRepository.cs
@@ -4,6 +4,7 @@
using AgentMemory.Abstractions.Options;
using AgentMemory.Abstractions.Repositories;
using AgentMemory.Abstractions.Services;
+using AgentMemory.Core.Extraction;
using AgentMemory.Neo4j.Infrastructure;
using AgentMemory.Neo4j.Queries;
using Neo4j.Driver;
@@ -11,7 +12,8 @@
namespace AgentMemory.Neo4j.Repositories;
-internal sealed class Neo4jEntityRepository : IEntityRepository
+internal sealed partial class Neo4jEntityRepository : IEntityRepository, IUpsertPersistsProvenance,
+ IBatchMemoryRepository, IFusedBatchMemoryRepository
{
private const int OwnerOverFetchFactor = Neo4jFactRepository.OwnerOverFetchFactor;
private const int OwnerOverFetchFloor = Neo4jFactRepository.OwnerOverFetchFloor;
@@ -44,19 +46,19 @@ public async Task UpsertAsync(Entity entity, CancellationToken cancellat
{
var parameters = new Dictionary
{
- ["id"] = entity.EntityId,
- ["ownerId"] = entity.OwnerId,
- ["name"] = entity.Name,
- ["canonicalName"] = (object?)entity.CanonicalName,
- ["type"] = entity.Type,
- ["subtype"] = (object?)entity.Subtype,
- ["description"] = (object?)entity.Description,
- ["confidence"] = entity.Confidence,
- ["aliases"] = entity.Aliases.ToList(),
- ["attributes"] = SerializeMetadata(entity.Attributes),
+ ["id"] = entity.EntityId,
+ ["ownerId"] = entity.OwnerId,
+ ["name"] = entity.Name,
+ ["canonicalName"] = (object?)entity.CanonicalName,
+ ["type"] = entity.Type,
+ ["subtype"] = (object?)entity.Subtype,
+ ["description"] = (object?)entity.Description,
+ ["confidence"] = entity.Confidence,
+ ["aliases"] = entity.Aliases.ToList(),
+ ["attributes"] = SerializeMetadata(entity.Attributes),
["sourceMessageIds"] = entity.SourceMessageIds.ToList(),
- ["createdAtUtc"] = entity.CreatedAtUtc.ToString("O"),
- ["metadata"] = SerializeMetadata(entity.Metadata)
+ ["createdAtUtc"] = entity.CreatedAtUtc.ToString("O"),
+ ["metadata"] = SerializeMetadata(entity.Metadata)
};
var cursor = await runner.RunAsync(EntityQueries.Upsert, parameters).ConfigureAwait(false);
@@ -178,7 +180,7 @@ public async Task> GetByNameAsync(
var records = await cursor.ToListAsync().ConfigureAwait(false);
return records.Select(r =>
{
- var node = r["node"].As();
+ var node = r["node"].As();
var score = r["score"].As();
return (MapToEntity(node, ReadEmbedding(node)), score);
}).ToList();
@@ -270,9 +272,9 @@ await _tx.WriteAsync(async runner =>
var records = await cursor.ToListAsync().ConfigureAwait(false);
return records.Select(r =>
{
- var node = r["other"].As();
+ var node = r["other"].As();
var confidence = r["confidence"].As();
- var matchType = r["matchType"].As();
+ var matchType = r["matchType"].As();
return (MapToEntity(node, ReadEmbedding(node)), confidence, matchType);
}).ToList();
}, cancellationToken).ConfigureAwait(false);
@@ -286,19 +288,19 @@ public async Task> UpsertBatchAsync(IReadOnlyList
var items = entities.Select(e => new Dictionary
{
- ["id"] = e.EntityId,
- ["owner_id"] = e.OwnerId,
- ["name"] = e.Name,
- ["canonical_name"] = (object?)e.CanonicalName,
- ["type"] = e.Type,
- ["subtype"] = (object?)e.Subtype,
- ["description"] = (object?)e.Description,
- ["confidence"] = e.Confidence,
- ["aliases"] = e.Aliases.ToList(),
- ["attributes"] = SerializeMetadata(e.Attributes),
+ ["id"] = e.EntityId,
+ ["owner_id"] = e.OwnerId,
+ ["name"] = e.Name,
+ ["canonical_name"] = (object?)e.CanonicalName,
+ ["type"] = e.Type,
+ ["subtype"] = (object?)e.Subtype,
+ ["description"] = (object?)e.Description,
+ ["confidence"] = e.Confidence,
+ ["aliases"] = e.Aliases.ToList(),
+ ["attributes"] = SerializeMetadata(e.Attributes),
["source_message_ids"] = e.SourceMessageIds.ToList(),
- ["created_at"] = e.CreatedAtUtc.ToString("O"),
- ["metadata"] = SerializeMetadata(e.Metadata)
+ ["created_at"] = e.CreatedAtUtc.ToString("O"),
+ ["metadata"] = SerializeMetadata(e.Metadata)
}).ToList();
return await _tx.WriteAsync(async runner =>
@@ -349,7 +351,7 @@ await runner.RunAsync(
return records.Select(r =>
{
var node = r["e"].As();
- var id = node["id"].As();
+ var id = node["id"].As();
if (!byId.TryGetValue(id, out var src))
return MapToEntity(node, null);
return MapToEntity(node, src.Embedding) with { Latitude = src.Latitude, Longitude = src.Longitude };
@@ -424,35 +426,35 @@ private static Entity MapToEntity(INode node, float[]? embedding)
if (node.Properties.TryGetValue("location", out var locValue) && locValue is Point pt)
{
// WGS-84: X = longitude, Y = latitude
- latitude = pt.Y;
+ latitude = pt.Y;
longitude = pt.X;
}
return new Entity
{
- EntityId = node["id"].As(),
- OwnerId = node.Properties.TryGetValue("owner_id", out var oid) ? oid.As() : null,
- Name = node["name"].As(),
- CanonicalName = node.Properties.TryGetValue("canonical_name", out var cn) ? cn.As() : null,
- Type = node["type"].As(),
- Subtype = node.Properties.TryGetValue("subtype", out var st) ? st.As() : null,
- Description = node.Properties.TryGetValue("description", out var desc) ? desc.As() : null,
- Confidence = node["confidence"].As(),
- Embedding = embedding,
- Latitude = latitude,
- Longitude = longitude,
- Aliases = node.Properties.TryGetValue("aliases", out var al)
+ EntityId = node["id"].As(),
+ OwnerId = node.Properties.TryGetValue("owner_id", out var oid) ? oid.As() : null,
+ Name = node["name"].As(),
+ CanonicalName = node.Properties.TryGetValue("canonical_name", out var cn) ? cn.As() : null,
+ Type = node["type"].As(),
+ Subtype = node.Properties.TryGetValue("subtype", out var st) ? st.As() : null,
+ Description = node.Properties.TryGetValue("description", out var desc) ? desc.As() : null,
+ Confidence = node["confidence"].As(),
+ Embedding = embedding,
+ Latitude = latitude,
+ Longitude = longitude,
+ Aliases = node.Properties.TryGetValue("aliases", out var al)
? al.As>().Select(a => a.ToString()!).ToList()
: Array.Empty(),
- Attributes = DeserializeMetadata(node.Properties.TryGetValue("attributes", out var attr) ? attr.As() : null),
+ Attributes = DeserializeMetadata(node.Properties.TryGetValue("attributes", out var attr) ? attr.As() : null),
SourceMessageIds = node.Properties.TryGetValue("source_message_ids", out var sm)
? sm.As>().Select(v => v.ToString()!).ToList()
: Array.Empty(),
- CreatedAtUtc = Neo4jDateTimeHelper.ReadDateTimeOffset(node["created_at"]),
- UpdatedAtUtc = node.Properties.TryGetValue("updated_at", out var ua) && ua is not null
+ CreatedAtUtc = Neo4jDateTimeHelper.ReadDateTimeOffset(node["created_at"]),
+ UpdatedAtUtc = node.Properties.TryGetValue("updated_at", out var ua) && ua is not null
? Neo4jDateTimeHelper.ReadNullableDateTimeOffset(ua)
: null,
- Metadata = DeserializeMetadata(node.Properties.TryGetValue("metadata", out var md) ? md.As() : null)
+ Metadata = DeserializeMetadata(node.Properties.TryGetValue("metadata", out var md) ? md.As() : null)
};
}
@@ -497,7 +499,11 @@ public async Task> SearchByLocationAsync(
var cursor = hasOwner
? await runner.RunAsync(cypher, new Dictionary
{
- ["lat"] = latitude, ["lon"] = longitude, ["radiusMeters"] = radiusKm * 1000.0, ["limit"] = limit, ["ownerId"] = scope!.OwnerId!,
+ ["lat"] = latitude,
+ ["lon"] = longitude,
+ ["radiusMeters"] = radiusKm * 1000.0,
+ ["limit"] = limit,
+ ["ownerId"] = scope!.OwnerId!,
}).ConfigureAwait(false)
: await runner.RunAsync(cypher, new { lat = latitude, lon = longitude, radiusMeters = radiusKm * 1000.0, limit }).ConfigureAwait(false);
var records = await cursor.ToListAsync().ConfigureAwait(false);
@@ -530,7 +536,12 @@ public async Task> SearchInBoundingBoxAsync(
var cursor = hasOwner
? await runner.RunAsync(cypher, new Dictionary
{
- ["minLat"] = minLat, ["minLon"] = minLon, ["maxLat"] = maxLat, ["maxLon"] = maxLon, ["limit"] = limit, ["ownerId"] = scope!.OwnerId!,
+ ["minLat"] = minLat,
+ ["minLon"] = minLon,
+ ["maxLat"] = maxLat,
+ ["maxLon"] = maxLon,
+ ["limit"] = limit,
+ ["ownerId"] = scope!.OwnerId!,
}).ConfigureAwait(false)
: await runner.RunAsync(cypher, new { minLat, minLon, maxLat, maxLon, limit }).ConfigureAwait(false);
var records = await cursor.ToListAsync().ConfigureAwait(false);
@@ -722,9 +733,9 @@ public async Task> GetEntitiesFromMessageAsync(
var cypher = TemporalQueries.SearchEntitiesAsOf(hasOwner, includeShared, topK);
var parameters = new Dictionary
{
- ["embedding"] = queryEmbedding.ToList(),
- ["limit"] = limit,
- ["minScore"] = minScore,
+ ["embedding"] = queryEmbedding.ToList(),
+ ["limit"] = limit,
+ ["minScore"] = minScore,
// D6: entities have only the transaction clock, so the AsOf timestamp binds $systemAsOf.
["systemAsOf"] = asOf.UtcDateTime.ToString("O")
};
@@ -736,7 +747,7 @@ public async Task> GetEntitiesFromMessageAsync(
var records = await cursor.ToListAsync().ConfigureAwait(false);
return records.Select(r =>
{
- var node = r["node"].As();
+ var node = r["node"].As();
var score = r["score"].As();
return (MapToEntity(node, ReadEmbedding(node)), score);
}).ToList();
diff --git a/src/AgentMemory.Neo4j/Repositories/Neo4jFactRepository.Fused.cs b/src/AgentMemory.Neo4j/Repositories/Neo4jFactRepository.Fused.cs
new file mode 100644
index 00000000..76d0dcaf
--- /dev/null
+++ b/src/AgentMemory.Neo4j/Repositories/Neo4jFactRepository.Fused.cs
@@ -0,0 +1,70 @@
+using AgentMemory.Abstractions.Domain;
+using AgentMemory.Core.Memory;
+using AgentMemory.Neo4j.Queries;
+using Microsoft.Extensions.Logging;
+using Neo4j.Driver;
+using static AgentMemory.Neo4j.Repositories.Neo4jRecordMapper;
+
+namespace AgentMemory.Neo4j.Repositories;
+
+internal sealed partial class Neo4jFactRepository
+{
+ public async Task> UpsertFusedBatchAsync(
+ IReadOnlyList facts,
+ CancellationToken cancellationToken = default)
+ {
+ if (facts.Count == 0) return Array.Empty();
+
+ _logger.LogDebug("Fused batch upserting {Count} facts", facts.Count);
+ var deduped = facts
+ .GroupBy(fact => TripleKey(
+ fact.Subject, fact.Predicate, fact.Object, fact.OwnerId ?? OwnerKeyShared))
+ .Select(group => group.Last())
+ .ToList();
+ var updatedAt = DateTimeOffset.UtcNow.ToString("O");
+ var items = deduped.Select(fact => new Dictionary
+ {
+ ["id"] = fact.FactId,
+ ["subject"] = fact.Subject,
+ ["predicate"] = fact.Predicate,
+ // The fused batch writer is the path extraction actually uses; the non-fused Upsert
+ // carried these keys while this one did not, so canonical identity never reached a real
+ // cold build.
+ ["subject_key"] = MemoryTripleCanonicalizer.CanonicalValue(fact.Subject),
+ ["predicate_key"] = MemoryTripleCanonicalizer.Canonical(fact.Predicate),
+ ["object_key"] = MemoryTripleCanonicalizer.CanonicalValue(fact.Object),
+ ["object"] = fact.Object,
+ ["owner_id"] = fact.OwnerId,
+ ["owner_key"] = fact.OwnerId ?? OwnerKeyShared,
+ ["category"] = fact.Category,
+ ["confidence"] = fact.Confidence,
+ ["valid_from"] = fact.ValidFrom?.ToString("O"),
+ ["valid_until"] = fact.ValidUntil?.ToString("O"),
+ ["source_message_ids"] = fact.SourceMessageIds.ToList(),
+ ["created_at"] = fact.CreatedAtUtc.ToString("O"),
+ ["updated_at"] = updatedAt,
+ ["metadata"] = SerializeMetadata(fact.Metadata),
+ ["embedding"] = fact.Embedding is { Length: > 0 } ? fact.Embedding.ToList() : null,
+ }).ToList();
+
+ return await _tx.WriteAsync(async runner =>
+ {
+ var cursor = await runner.RunAsync(FusedPersistenceQueries.FactUpsertBatch, new { items })
+ .ConfigureAwait(false);
+ var records = await cursor.ToListAsync().ConfigureAwait(false);
+ var embeddingByTriple = deduped.ToDictionary(
+ fact => TripleKey(fact.Subject, fact.Predicate, fact.Object, fact.OwnerId ?? OwnerKeyShared),
+ fact => fact.Embedding);
+ return records.Select(record =>
+ {
+ var node = record["f"].As();
+ var key = TripleKey(
+ node["subject"].As(),
+ node["predicate"].As(),
+ node["object"].As(),
+ node["owner_key"].As());
+ return MapToFact(node, embeddingByTriple.TryGetValue(key, out var embedding) ? embedding : null);
+ }).ToList();
+ }, cancellationToken).ConfigureAwait(false);
+ }
+}
diff --git a/src/AgentMemory.Neo4j/Repositories/Neo4jFactRepository.cs b/src/AgentMemory.Neo4j/Repositories/Neo4jFactRepository.cs
index 1866c1cd..e30330a9 100644
--- a/src/AgentMemory.Neo4j/Repositories/Neo4jFactRepository.cs
+++ b/src/AgentMemory.Neo4j/Repositories/Neo4jFactRepository.cs
@@ -1,9 +1,11 @@
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using AgentMemory.Abstractions.Domain;
+using AgentMemory.Core.Memory;
using AgentMemory.Abstractions.Options;
using AgentMemory.Abstractions.Repositories;
using AgentMemory.Abstractions.Services;
+using AgentMemory.Core.Extraction;
using AgentMemory.Neo4j.Infrastructure;
using AgentMemory.Neo4j.Queries;
using Neo4j.Driver;
@@ -11,7 +13,8 @@
namespace AgentMemory.Neo4j.Repositories;
-internal sealed class Neo4jFactRepository : IFactRepository
+internal sealed partial class Neo4jFactRepository : IFactRepository, IUpsertPersistsProvenance,
+ IBatchMemoryRepository, IFusedBatchMemoryRepository
{
// Owner-scoped vector search over-fetches candidates (topK > limit) so an owner filter is not
// starved by higher-scoring foreign rows; the post-WHERE then LIMITs to the requested count (R1).
@@ -50,20 +53,24 @@ public async Task UpsertAsync(Fact fact, CancellationToken cancellationTok
{
var parameters = new Dictionary
{
- ["id"] = fact.FactId,
- ["subject"] = fact.Subject,
- ["predicate"] = fact.Predicate,
- ["object"] = fact.Object,
- ["ownerId"] = fact.OwnerId,
- ["ownerKey"] = fact.OwnerId ?? OwnerKeyShared,
- ["category"] = fact.Category,
- ["confidence"] = fact.Confidence,
- ["validFrom"] = (object?)(fact.ValidFrom?.ToString("O")),
- ["validUntil"] = (object?)(fact.ValidUntil?.ToString("O")),
+ ["id"] = fact.FactId,
+ ["subject"] = fact.Subject,
+ ["predicate"] = fact.Predicate,
+ // Identity is the canonical trio; the raw strings above stay for display and audit.
+ ["subjectKey"] = MemoryTripleCanonicalizer.CanonicalValue(fact.Subject),
+ ["predicateKey"] = MemoryTripleCanonicalizer.Canonical(fact.Predicate),
+ ["objectKey"] = MemoryTripleCanonicalizer.CanonicalValue(fact.Object),
+ ["object"] = fact.Object,
+ ["ownerId"] = fact.OwnerId,
+ ["ownerKey"] = fact.OwnerId ?? OwnerKeyShared,
+ ["category"] = fact.Category,
+ ["confidence"] = fact.Confidence,
+ ["validFrom"] = (object?)(fact.ValidFrom?.ToString("O")),
+ ["validUntil"] = (object?)(fact.ValidUntil?.ToString("O")),
["sourceMessageIds"] = fact.SourceMessageIds.ToList(),
- ["createdAtUtc"] = fact.CreatedAtUtc.ToString("O"),
- ["updatedAtUtc"] = DateTimeOffset.UtcNow.ToString("O"),
- ["metadata"] = SerializeMetadata(fact.Metadata)
+ ["createdAtUtc"] = fact.CreatedAtUtc.ToString("O"),
+ ["updatedAtUtc"] = DateTimeOffset.UtcNow.ToString("O"),
+ ["metadata"] = SerializeMetadata(fact.Metadata)
};
var cursor = await runner.RunAsync(FactQueries.Upsert, parameters).ConfigureAwait(false);
@@ -116,20 +123,23 @@ public async Task> UpsertBatchAsync(IReadOnlyList fact
var updatedAt = DateTimeOffset.UtcNow.ToString("O");
var items = deduped.Select(f => new Dictionary
{
- ["id"] = f.FactId,
- ["subject"] = f.Subject,
- ["predicate"] = f.Predicate,
- ["object"] = f.Object,
- ["owner_id"] = f.OwnerId,
- ["owner_key"] = f.OwnerId ?? OwnerKeyShared,
- ["category"] = f.Category,
- ["confidence"] = f.Confidence,
- ["valid_from"] = (object?)(f.ValidFrom?.ToString("O")),
- ["valid_until"] = (object?)(f.ValidUntil?.ToString("O")),
+ ["id"] = f.FactId,
+ ["subject"] = f.Subject,
+ ["predicate"] = f.Predicate,
+ ["subject_key"] = MemoryTripleCanonicalizer.CanonicalValue(f.Subject),
+ ["predicate_key"] = MemoryTripleCanonicalizer.Canonical(f.Predicate),
+ ["object_key"] = MemoryTripleCanonicalizer.CanonicalValue(f.Object),
+ ["object"] = f.Object,
+ ["owner_id"] = f.OwnerId,
+ ["owner_key"] = f.OwnerId ?? OwnerKeyShared,
+ ["category"] = f.Category,
+ ["confidence"] = f.Confidence,
+ ["valid_from"] = (object?)(f.ValidFrom?.ToString("O")),
+ ["valid_until"] = (object?)(f.ValidUntil?.ToString("O")),
["source_message_ids"] = f.SourceMessageIds.ToList(),
- ["created_at"] = f.CreatedAtUtc.ToString("O"),
- ["updated_at"] = updatedAt,
- ["metadata"] = SerializeMetadata(f.Metadata)
+ ["created_at"] = f.CreatedAtUtc.ToString("O"),
+ ["updated_at"] = updatedAt,
+ ["metadata"] = SerializeMetadata(f.Metadata)
}).ToList();
return await _tx.WriteAsync(async runner =>
@@ -178,7 +188,7 @@ await runner.RunAsync(
return records.Select(r =>
{
var node = r["f"].As();
- var key = TripleKey(node["subject"].As(), node["predicate"].As(),
+ var key = TripleKey(node["subject"].As(), node["predicate"].As(),
node["object"].As(), node["owner_key"].As());
return MapToFact(node, embeddingByTriple.TryGetValue(key, out var emb) ? emb : null);
}).ToList();
@@ -255,17 +265,13 @@ public async Task> GetBySubjectAsync(
var records = await cursor.ToListAsync().ConfigureAwait(false);
return records.Select(r =>
{
- var node = r["node"].As();
+ var node = r["node"].As();
var score = r["score"].As();
return (MapToFact(node, ReadEmbedding(node)), score);
}).ToList();
}, cancellationToken).ConfigureAwait(false);
}
- // Small candidate set for dedup lookups: a near-duplicate by subject+predicate is rare, so a
- // modest over-fetch is enough to find the best match above the (high) similarity threshold.
- private const int DedupOverFetch = 10;
-
public async Task FindDuplicateAsync(
string subject, string predicate, float[] embedding, string? ownerId, double threshold,
CancellationToken cancellationToken = default)
@@ -273,14 +279,14 @@ public async Task> GetBySubjectAsync(
// Boundary invariant: a zero-dimension (empty/degraded) embedding can't address the vector index;
// there is no duplicate to find, so short-circuit (caller then creates a new node).
if (embedding is not { Length: > 0 }) return null;
- var cypher = FactQueries.FindDuplicate(DedupOverFetch);
+ var cypher = FactQueries.FindDuplicate();
var parameters = new Dictionary
{
- ["embedding"] = embedding.ToList(),
- ["threshold"] = threshold,
- ["subject"] = subject,
- ["predicate"] = predicate,
- ["ownerKey"] = ownerId ?? OwnerKeyShared,
+ ["embedding"] = embedding.ToList(),
+ ["threshold"] = threshold,
+ ["subject"] = subject,
+ ["predicate"] = predicate,
+ ["ownerKey"] = ownerId ?? OwnerKeyShared,
};
return await _tx.ReadAsync(async runner =>
@@ -354,25 +360,25 @@ private static (string, string, string, string) TripleKey(
private static Fact MapToFact(INode node, float[]? embedding) =>
new()
{
- FactId = node["id"].As(),
- Subject = node["subject"].As(),
- Predicate = node["predicate"].As(),
- Object = node["object"].As(),
- OwnerId = node.Properties.TryGetValue("owner_id", out var oid) ? oid.As() : null,
- Category = node.Properties.TryGetValue("category", out var cat) ? cat.As() : null,
- Confidence = node["confidence"].As(),
- ValidFrom = node.Properties.TryGetValue("valid_from", out var vf)
+ FactId = node["id"].As(),
+ Subject = node["subject"].As(),
+ Predicate = node["predicate"].As(),
+ Object = node["object"].As(),
+ OwnerId = node.Properties.TryGetValue("owner_id", out var oid) ? oid.As() : null,
+ Category = node.Properties.TryGetValue("category", out var cat) ? cat.As() : null,
+ Confidence = node["confidence"].As(),
+ ValidFrom = node.Properties.TryGetValue("valid_from", out var vf)
? Neo4jDateTimeHelper.ReadNullableDateTimeOffset(vf)
: null,
- ValidUntil = node.Properties.TryGetValue("valid_until", out var vu)
+ ValidUntil = node.Properties.TryGetValue("valid_until", out var vu)
? Neo4jDateTimeHelper.ReadNullableDateTimeOffset(vu)
: null,
- Embedding = embedding,
+ Embedding = embedding,
SourceMessageIds = node.Properties.TryGetValue("source_message_ids", out var sm)
? sm.As>().Select(v => v.ToString()!).ToList()
: Array.Empty(),
- CreatedAtUtc = Neo4jDateTimeHelper.ReadDateTimeOffset(node["created_at"]),
- Metadata = DeserializeMetadata(node.Properties.TryGetValue("metadata", out var md) ? md.As() : null)
+ CreatedAtUtc = Neo4jDateTimeHelper.ReadDateTimeOffset(node["created_at"]),
+ Metadata = DeserializeMetadata(node.Properties.TryGetValue("metadata", out var md) ? md.As() : null)
};
private static float[]? ReadEmbedding(INode node)
@@ -518,10 +524,10 @@ public async Task SupersedeAsync(string loserFactId, string winnerFactId,
var cypher = TemporalQueries.SearchFactsAsOf(hasOwner, includeShared, topK);
var parameters = new Dictionary
{
- ["embedding"] = queryEmbedding.ToList(),
- ["limit"] = limit,
- ["minScore"] = minScore,
- ["validAsOf"] = asOf.UtcDateTime.ToString("O"),
+ ["embedding"] = queryEmbedding.ToList(),
+ ["limit"] = limit,
+ ["minScore"] = minScore,
+ ["validAsOf"] = asOf.UtcDateTime.ToString("O"),
["systemAsOf"] = (systemAsOf ?? asOf).UtcDateTime.ToString("O")
};
if (hasOwner) parameters["ownerId"] = scope!.OwnerId;
@@ -532,10 +538,49 @@ public async Task SupersedeAsync(string loserFactId, string winnerFactId,
var records = await cursor.ToListAsync().ConfigureAwait(false);
return records.Select(r =>
{
- var node = r["node"].As();
+ var node = r["node"].As();
var score = r["score"].As();
return (MapToFact(node, ReadEmbedding(node)), score);
}).ToList();
}, cancellationToken).ConfigureAwait(false);
}
-}
\ No newline at end of file
+
+ ///
+ public async Task> SearchByCanonicalPredicatesAsync(
+ IReadOnlyList canonicalPredicates,
+ int limit,
+ MemoryScope scope,
+ CancellationToken cancellationToken = default)
+ {
+ ArgumentNullException.ThrowIfNull(canonicalPredicates);
+ ArgumentOutOfRangeException.ThrowIfNegativeOrZero(limit);
+ if (canonicalPredicates.Count == 0)
+ return Array.Empty();
+
+ // Same scope semantics as every other fact read: an owner filter only when one was asked
+ // for, and shared facts included unless explicitly excluded.
+ var hasOwner = scope?.HasOwnerFilter == true;
+ var includeShared = scope?.IncludeShared ?? true;
+ var parameters = new Dictionary
+ {
+ ["predicateKeys"] = canonicalPredicates.ToArray(),
+ ["limit"] = limit
+ };
+ if (hasOwner) parameters["ownerId"] = scope!.OwnerId;
+
+ return await _tx.ReadAsync(async runner =>
+ {
+ var cursor = await runner.RunAsync(
+ FactQueries.SearchByCanonicalPredicates(hasOwner, includeShared),
+ parameters).ConfigureAwait(false);
+ var records = await cursor.ToListAsync().ConfigureAwait(false);
+ return (IReadOnlyList)records
+ .Select(record =>
+ {
+ var node = record["f"].As();
+ return MapToFact(node, ReadEmbedding(node));
+ })
+ .ToList();
+ }, cancellationToken).ConfigureAwait(false);
+ }
+}
diff --git a/src/AgentMemory.Neo4j/Repositories/Neo4jMessageRepository.cs b/src/AgentMemory.Neo4j/Repositories/Neo4jMessageRepository.cs
index 6213dca1..29b6bcf2 100644
--- a/src/AgentMemory.Neo4j/Repositories/Neo4jMessageRepository.cs
+++ b/src/AgentMemory.Neo4j/Repositories/Neo4jMessageRepository.cs
@@ -1,4 +1,5 @@
using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
using AgentMemory.Abstractions.Domain;
using AgentMemory.Abstractions.Repositories;
using AgentMemory.Neo4j.Infrastructure;
@@ -10,13 +11,22 @@ namespace AgentMemory.Neo4j.Repositories;
internal sealed class Neo4jMessageRepository : IMessageRepository
{
+ // Metadata-only filters run after the global vector candidate pool, so those searches must
+ // over-fetch before filtering. Session-scoped searches use an exact in-session query instead.
+ private const int ScopedOverFetchFactor = 5;
+ private const int ScopedOverFetchFloor = 50;
private readonly INeo4jTransactionRunner _tx;
private readonly ILogger _logger;
+ private readonly bool _useOptimizedMessageBatchWrites;
- public Neo4jMessageRepository(INeo4jTransactionRunner tx, ILogger logger)
+ public Neo4jMessageRepository(
+ INeo4jTransactionRunner tx,
+ ILogger logger,
+ IOptions? options = null)
{
_tx = tx;
_logger = logger;
+ _useOptimizedMessageBatchWrites = options?.Value.UseOptimizedMessageBatchWrites ?? true;
}
public async Task AddAsync(Message message, CancellationToken cancellationToken = default)
@@ -35,30 +45,20 @@ public async Task AddAsync(Message message, CancellationToken cancellat
["content"] = message.Content,
["timestamp"] = message.TimestampUtc.ToString("O"),
["toolCallIds"] = message.ToolCallIds?.ToList() ?? new List(),
- ["metadata"] = SerializeMetadata(message.Metadata)
+ ["metadata"] = SerializeMetadata(message.Metadata),
+ ["embedding"] = message.Embedding is { Length: > 0 }
+ ? message.Embedding.ToList()
+ : null
};
var cursor = await runner.RunAsync(MessageQueries.Add, createParams).ConfigureAwait(false);
var record = await cursor.SingleAsync().ConfigureAwait(false);
- var node = record["m"].As();
+ var returned = record["m"];
+ var properties = returned is INode node
+ ? node.Properties
+ : returned.As>();
- // Only persist a real (non-empty) vector; a degraded empty embedding leaves `embedding` NULL.
- if (message.Embedding is { Length: > 0 })
- {
- await runner.RunAsync(
- SharedFragments.SetMessageEmbedding,
- new { id = message.MessageId, embedding = message.Embedding.ToList() }).ConfigureAwait(false);
- }
-
- // Create FIRST_MESSAGE if this is the first message in the conversation
- await runner.RunAsync(
- MessageQueries.CreateFirstMessageLink,
- new { conversationId = message.ConversationId, id = message.MessageId }).ConfigureAwait(false);
-
- // Establish NEXT_MESSAGE link from the previous last message
- await runner.RunAsync(MessageQueries.LinkNextMessage, new { conversationId = message.ConversationId, id = message.MessageId }).ConfigureAwait(false);
-
- return MapToMessage(node, message.Embedding);
+ return MapToMessage(properties, message.Embedding);
}, cancellationToken).ConfigureAwait(false);
}
@@ -78,9 +78,32 @@ public async Task> AddBatchAsync(IEnumerable mes
["content"] = m.Content,
["timestamp"] = m.TimestampUtc.ToString("O"),
["tool_call_ids"] = m.ToolCallIds?.ToList() ?? new List(),
- ["metadata"] = SerializeMetadata(m.Metadata)
+ ["metadata"] = SerializeMetadata(m.Metadata),
+ ["embedding"] = m.Embedding is { Length: > 0 }
+ ? m.Embedding.ToList()
+ : null
}).ToList();
+ var embeddingMap = ordered.ToDictionary(m => m.MessageId, m => m.Embedding);
+ if (_useOptimizedMessageBatchWrites)
+ {
+ return await _tx.WriteAsync(async runner =>
+ {
+ var cursor = await runner.RunAsync(
+ MessageQueries.AddBatchOptimized,
+ new { messages = msgParams }).ConfigureAwait(false);
+ var records = await cursor.ToListAsync().ConfigureAwait(false);
+ return records.Select(record =>
+ {
+ var node = record["m"].As();
+ var id = node["id"].As();
+ return MapToMessage(
+ node,
+ embeddingMap.TryGetValue(id, out var embedding) ? embedding : null);
+ }).ToList();
+ }, cancellationToken).ConfigureAwait(false);
+ }
+
return await _tx.WriteAsync(async runner =>
{
var cursor = await runner.RunAsync(MessageQueries.AddBatch, new { messages = msgParams }).ConfigureAwait(false);
@@ -122,7 +145,6 @@ await runner.RunAsync(
new { ids = ordered.Select(m => m.MessageId).ToList() }).ConfigureAwait(false);
var records = await readCursor.ToListAsync().ConfigureAwait(false);
- var embeddingMap = ordered.ToDictionary(m => m.MessageId, m => m.Embedding);
return records.Select(r =>
{
var node = r["m"].As();
@@ -208,13 +230,17 @@ public async Task> GetAllBySessionAsync(string sessionId,
_logger.LogDebug("Vector search messages, sessionId={SessionId}, limit={Limit}", sessionId, limit);
var (filterClause, filterParams) = MetadataFilterBuilder.Build(metadataFilters, nodeAlias: "node");
-
- var cypher = MessageQueries.SearchByVector(sessionId is not null, filterClause, limit);
+ var hasMetadataFilter = !string.IsNullOrWhiteSpace(filterClause);
+ var topK = sessionId is null && hasMetadataFilter
+ ? Math.Max(limit * ScopedOverFetchFactor, limit + ScopedOverFetchFloor)
+ : limit;
+ var cypher = MessageQueries.SearchByVector(sessionId is not null, filterClause, topK);
var parameters = new Dictionary
{
["embedding"] = queryEmbedding.ToList(),
- ["minScore"] = minScore
+ ["minScore"] = minScore,
+ ["limit"] = limit
};
if (sessionId is not null) parameters["sessionId"] = sessionId;
foreach (var (k, v) in filterParams) parameters[k] = v;
@@ -281,19 +307,22 @@ public async Task> GetRecentBySessionAsOfAsync(
}
private static Message MapToMessage(INode node, float[]? embedding) =>
+ MapToMessage(node.Properties, embedding);
+
+ private static Message MapToMessage(IReadOnlyDictionary properties, float[]? embedding) =>
new()
{
- MessageId = node["id"].As(),
- ConversationId = node["conversation_id"].As(),
- SessionId = node["session_id"].As(),
- Role = node["role"].As(),
- Content = node["content"].As(),
- TimestampUtc = Neo4jDateTimeHelper.ReadDateTimeOffset(node["timestamp"]),
+ MessageId = properties["id"].As(),
+ ConversationId = properties["conversation_id"].As(),
+ SessionId = properties["session_id"].As(),
+ Role = properties["role"].As(),
+ Content = properties["content"].As(),
+ TimestampUtc = Neo4jDateTimeHelper.ReadDateTimeOffset(properties["timestamp"]),
Embedding = embedding,
- ToolCallIds = node.Properties.TryGetValue("tool_call_ids", out var tc)
+ ToolCallIds = properties.TryGetValue("tool_call_ids", out var tc)
? tc.As>().Select(v => v.ToString()!).ToList()
: [],
- Metadata = DeserializeMetadata(node.Properties.TryGetValue("metadata", out var md) ? md.As() : null)
+ Metadata = DeserializeMetadata(properties.TryGetValue("metadata", out var md) ? md.As() : null)
};
private static float[]? ReadEmbedding(INode node)
diff --git a/src/AgentMemory.Neo4j/Repositories/Neo4jPreferenceRepository.Fused.cs b/src/AgentMemory.Neo4j/Repositories/Neo4jPreferenceRepository.Fused.cs
new file mode 100644
index 00000000..657f0133
--- /dev/null
+++ b/src/AgentMemory.Neo4j/Repositories/Neo4jPreferenceRepository.Fused.cs
@@ -0,0 +1,48 @@
+using AgentMemory.Abstractions.Domain;
+using AgentMemory.Neo4j.Queries;
+using Microsoft.Extensions.Logging;
+using Neo4j.Driver;
+using static AgentMemory.Neo4j.Repositories.Neo4jRecordMapper;
+
+namespace AgentMemory.Neo4j.Repositories;
+
+internal sealed partial class Neo4jPreferenceRepository
+{
+ public async Task> UpsertFusedBatchAsync(
+ IReadOnlyList preferences,
+ CancellationToken cancellationToken = default)
+ {
+ if (preferences.Count == 0) return Array.Empty();
+
+ _logger.LogDebug("Fused batch upserting {Count} preferences", preferences.Count);
+ var items = preferences.Select(preference => new Dictionary
+ {
+ ["id"] = preference.PreferenceId,
+ ["owner_id"] = preference.OwnerId,
+ ["category"] = preference.Category,
+ ["preference"] = preference.PreferenceText,
+ ["context"] = preference.Context,
+ ["confidence"] = preference.Confidence,
+ ["source_message_ids"] = preference.SourceMessageIds.ToList(),
+ ["created_at"] = preference.CreatedAtUtc.ToString("O"),
+ ["metadata"] = SerializeMetadata(preference.Metadata),
+ ["embedding"] = preference.Embedding is { Length: > 0 }
+ ? preference.Embedding.ToList() : null,
+ }).ToList();
+
+ return await _tx.WriteAsync(async runner =>
+ {
+ var cursor = await runner.RunAsync(FusedPersistenceQueries.PreferenceUpsertBatch, new { items })
+ .ConfigureAwait(false);
+ var records = await cursor.ToListAsync().ConfigureAwait(false);
+ var byId = preferences.ToDictionary(preference => preference.PreferenceId, StringComparer.Ordinal);
+ return records.Select(record =>
+ {
+ var node = record["p"].As();
+ var id = node["id"].As();
+ return MapToPreference(node, byId.TryGetValue(id, out var source)
+ ? source.Embedding : ReadEmbedding(node));
+ }).ToList();
+ }, cancellationToken).ConfigureAwait(false);
+ }
+}
diff --git a/src/AgentMemory.Neo4j/Repositories/Neo4jPreferenceRepository.cs b/src/AgentMemory.Neo4j/Repositories/Neo4jPreferenceRepository.cs
index a8dd03f0..525c2c3b 100644
--- a/src/AgentMemory.Neo4j/Repositories/Neo4jPreferenceRepository.cs
+++ b/src/AgentMemory.Neo4j/Repositories/Neo4jPreferenceRepository.cs
@@ -4,6 +4,7 @@
using AgentMemory.Abstractions.Options;
using AgentMemory.Abstractions.Repositories;
using AgentMemory.Abstractions.Services;
+using AgentMemory.Core.Extraction;
using AgentMemory.Neo4j.Infrastructure;
using AgentMemory.Neo4j.Queries;
using Neo4j.Driver;
@@ -11,7 +12,8 @@
namespace AgentMemory.Neo4j.Repositories;
-internal sealed class Neo4jPreferenceRepository : IPreferenceRepository
+internal sealed partial class Neo4jPreferenceRepository : IPreferenceRepository, IUpsertPersistsProvenance,
+ IBatchMemoryRepository, IFusedBatchMemoryRepository
{
private const int OwnerOverFetchFactor = Neo4jFactRepository.OwnerOverFetchFactor;
private const int OwnerOverFetchFloor = Neo4jFactRepository.OwnerOverFetchFloor;
@@ -44,15 +46,15 @@ public async Task UpsertAsync(Preference preference, CancellationTok
{
var parameters = new Dictionary
{
- ["id"] = preference.PreferenceId,
- ["ownerId"] = preference.OwnerId,
- ["category"] = preference.Category,
- ["preferenceText"] = preference.PreferenceText,
- ["context"] = (object?)preference.Context,
- ["confidence"] = preference.Confidence,
+ ["id"] = preference.PreferenceId,
+ ["ownerId"] = preference.OwnerId,
+ ["category"] = preference.Category,
+ ["preferenceText"] = preference.PreferenceText,
+ ["context"] = (object?)preference.Context,
+ ["confidence"] = preference.Confidence,
["sourceMessageIds"] = preference.SourceMessageIds.ToList(),
- ["createdAtUtc"] = preference.CreatedAtUtc.ToString("O"),
- ["metadata"] = SerializeMetadata(preference.Metadata)
+ ["createdAtUtc"] = preference.CreatedAtUtc.ToString("O"),
+ ["metadata"] = SerializeMetadata(preference.Metadata)
};
var cursor = await runner.RunAsync(PreferenceQueries.Upsert, parameters).ConfigureAwait(false);
@@ -80,6 +82,55 @@ await runner.RunAsync(
}, cancellationToken).ConfigureAwait(false);
}
+ public async Task> UpsertBatchAsync(
+ IReadOnlyList preferences,
+ CancellationToken cancellationToken = default)
+ {
+ if (preferences.Count == 0) return Array.Empty();
+
+ _logger.LogDebug("Batch upserting {Count} preferences", preferences.Count);
+ var items = preferences.Select(preference => new Dictionary
+ {
+ ["id"] = preference.PreferenceId,
+ ["owner_id"] = preference.OwnerId,
+ ["category"] = preference.Category,
+ ["preference"] = preference.PreferenceText,
+ ["context"] = preference.Context,
+ ["confidence"] = preference.Confidence,
+ ["source_message_ids"] = preference.SourceMessageIds.ToList(),
+ ["created_at"] = preference.CreatedAtUtc.ToString("O"),
+ ["metadata"] = SerializeMetadata(preference.Metadata)
+ }).ToList();
+
+ return await _tx.WriteAsync(async runner =>
+ {
+ var cursor = await runner.RunAsync(PreferenceQueries.UpsertBatch, new { items }).ConfigureAwait(false);
+ var records = await cursor.ToListAsync().ConfigureAwait(false);
+
+ foreach (var preference in preferences.Where(item => item.Embedding is { Length: > 0 }))
+ {
+ await runner.RunAsync(
+ PreferenceQueries.SetEmbedding,
+ new { id = preference.PreferenceId, embedding = preference.Embedding!.ToList() }).ConfigureAwait(false);
+ }
+
+ foreach (var preference in preferences.Where(item => item.SourceMessageIds.Count > 0))
+ {
+ await runner.RunAsync(
+ PreferenceQueries.CreateExtractedFromMessages,
+ new { id = preference.PreferenceId, sourceMessageIds = preference.SourceMessageIds.ToList() })
+ .ConfigureAwait(false);
+ }
+
+ var byId = preferences.ToDictionary(item => item.PreferenceId, StringComparer.Ordinal);
+ return records.Select(record =>
+ {
+ var node = record["p"].As();
+ var id = node["id"].As();
+ return MapToPreference(node, byId.TryGetValue(id, out var source) ? source.Embedding : null);
+ }).ToList();
+ }, cancellationToken).ConfigureAwait(false);
+ }
public async Task GetByIdAsync(string preferenceId, CancellationToken cancellationToken = default)
{
_logger.LogDebug("Getting preference {Id}", preferenceId);
@@ -150,7 +201,7 @@ public async Task> GetByCategoryAsync(
var records = await cursor.ToListAsync().ConfigureAwait(false);
return records.Select(r =>
{
- var node = r["node"].As();
+ var node = r["node"].As();
var score = r["score"].As