diff --git a/tests/AgentMemory.Tests.Unit.LongMemEval/TimestampedHistoryInjectionTests.cs b/tests/AgentMemory.Tests.Unit.LongMemEval/TimestampedHistoryInjectionTests.cs new file mode 100644 index 00000000..97c30c2a --- /dev/null +++ b/tests/AgentMemory.Tests.Unit.LongMemEval/TimestampedHistoryInjectionTests.cs @@ -0,0 +1,262 @@ +using AgentEval.Core; +using AgentMemory.Abstractions.Domain; +using AgentMemory.Abstractions.Services; +using AgentMemory.LongMemEval; +using FluentAssertions; +using Microsoft.Extensions.AI; +using NSubstitute; +using Xunit; + +namespace AgentMemory.Tests.Unit.LongMemEval; + +/// +/// 30.9c prereq A: the adapter accepts TypedMemEval's timestamped channel +/// () and honours its two clocks. +/// +/// +/// +/// The free-checks pass found this interface implemented nowhere: the tg work delivered +/// dates as text, and AgentEval's Prospective vertical refuses to run — before its first provider +/// call, deliberately — against an agent without the typed channel. These tests pin the semantics +/// the channel promises: a turn's instant becomes the stored message's own valid time, no date +/// text is ever added to message content (TimestampsOnly grounding exists precisely to remove the +/// in-text crutch), and the history's QueryTime anchors the recall and the answer call. +/// +/// +public sealed class TimestampedHistoryInjectionTests +{ + private static readonly DateTimeOffset FirstTurnTime = + new(2023, 5, 1, 10, 0, 0, TimeSpan.Zero); + + private static readonly DateTimeOffset SecondTurnTime = + new(2023, 6, 11, 9, 30, 0, TimeSpan.Zero); + + private static readonly DateTimeOffset QueryTime = + new(2023, 7, 1, 12, 0, 0, TimeSpan.Zero); + + private static TimestampedConversationHistory TwoTurnHistory() => new() + { + Turns = + [ + new TimestampedConversationTurn( + "Alice moved to Zurich in March.", "Noted.", FirstTurnTime, 0), + new TimestampedConversationTurn( + "Remind me to renew the allotment lease.", "Will do.", SecondTurnTime, 1) + ], + QueryTime = QueryTime + }; + + private static IChatClient AnsweringChat(Action>? capture = null) + { + var chat = Substitute.For(); + chat.GetResponseAsync( + Arg.Any>(), + Arg.Any(), + Arg.Any()) + .Returns(call => + { + capture?.Invoke(call.Arg>().ToArray()); + return new ChatResponse(new ChatMessage(ChatRole.Assistant, "answer")); + }); + return chat; + } + + private static RecallResult RecallOf(string sessionId, params Message[] items) => new() + { + Context = new MemoryContext + { + SessionId = sessionId, + AssembledAtUtc = DateTimeOffset.UnixEpoch, + RelevantMessages = new MemoryContextSection { Items = items } + }, + TotalItemsRetrieved = items.Length + }; + + private static Message Message(string sessionId, string role, string content) => new() + { + MessageId = Guid.NewGuid().ToString("N"), + SessionId = sessionId, + ConversationId = sessionId, + Role = role, + Content = content, + TimestampUtc = DateTimeOffset.UnixEpoch + }; + + [Fact] + public async Task StoresTurnInstantsAsMessageValidTime_AndAppendsNoDateTextToContent() + { + var memory = Substitute.For(); + IReadOnlyList? stored = null; + memory.AddMessagesAsync(Arg.Any>(), Arg.Any()) + .Returns(call => + { + stored = call.Arg>().ToArray(); + return stored; + }); + memory.RecallAsOfAsync( + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any()) + .Returns(call => RecallOf( + call.Arg().SessionId, + Message(call.Arg().SessionId, "user", "Alice moved to Zurich in March."))); + + var adapter = new AgentMemoryLongMemEvalAdapter(memory, AnsweringChat(), "typed-run"); + await adapter.ResetSessionAsync(); + adapter.InjectTimestampedConversationHistory(TwoTurnHistory()); + + await adapter.InvokeAsync("Where does Alice live?"); + + stored.Should().NotBeNull().And.HaveCount(4); + // (a) The turn's instant is the stored message's own clock — the product's valid time — + // for BOTH halves of the pair, replacing the epoch + ordinal counter. + stored!.Select(message => message.TimestampUtc).Should().Equal( + FirstTurnTime, FirstTurnTime, SecondTurnTime, SecondTurnTime); + // (b) The content is byte-identical to the injected turns: no session-date header, no + // "Current Date:" line, no timestamp rendered into the text. Dates are structural here. + stored.Select(message => message.Content).Should().Equal( + "Alice moved to Zurich in March.", + "Noted.", + "Remind me to renew the allotment lease.", + "Will do."); + } + + [Fact] + public async Task AnchorsRecallAndAnswerAtQueryTime() + { + var testStartUtc = DateTimeOffset.UtcNow; + var memory = Substitute.For(); + memory.AddMessagesAsync(Arg.Any>(), Arg.Any()) + .Returns(call => call.Arg>().ToArray()); + DateTimeOffset? recordedAsOf = null; + DateTimeOffset? recordedSystemAsOf = null; + RecallRequest? recordedRequest = null; + memory.RecallAsOfAsync( + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any()) + .Returns(call => + { + recordedRequest = call.Arg(); + recordedAsOf = call.ArgAt(1); + recordedSystemAsOf = call.ArgAt(2); + return RecallOf( + recordedRequest.SessionId, + Message(recordedRequest.SessionId, "user", "Remind me to renew the allotment lease.")); + }); + IReadOnlyList? answerPrompt = null; + var adapter = new AgentMemoryLongMemEvalAdapter( + memory, AnsweringChat(prompt => answerPrompt = prompt), "typed-run"); + await adapter.ResetSessionAsync(); + adapter.InjectTimestampedConversationHistory(TwoTurnHistory()); + + await adapter.InvokeAsync("Has the lease renewal come due yet?"); + + // (c) QueryTime reaches the recall path as the VALID-time clock: "what was true at the + // question's now". The ordinary recall path must not run at all for a timestamped question. + recordedAsOf.Should().Be(QueryTime); + recordedRequest.Should().NotBeNull(); + await memory.DidNotReceive().RecallAsync( + Arg.Any(), Arg.Any()); + // The TRANSACTION clock stays at the machine's now: the corpus was ingested moments ago, so + // a transaction clock bound to the 2023 QueryTime would erase everything just stored. + recordedSystemAsOf.Should().NotBeNull(); + recordedSystemAsOf!.Value.Should().BeOnOrAfter(testStartUtc); + // The answer prompt's "now" is rendered from the typed channel's QueryTime — data the + // system under test legitimately holds — not from evaluator-side knowledge. + answerPrompt.Should().NotBeNull(); + answerPrompt!.Select(message => message.Text).Should().Contain(text => + text!.Contains( + $"Current date: {AgentMemoryLongMemEvalAdapter.FormatQueryTime(QueryTime)}", + StringComparison.Ordinal)); + } + + [Fact] + public async Task ResetClearsTheTimestampedState_SoThePlainChannelKeepsItsOwnClocks() + { + var memory = Substitute.For(); + IReadOnlyList? stored = null; + memory.AddMessagesAsync(Arg.Any>(), Arg.Any()) + .Returns(call => + { + stored = call.Arg>().ToArray(); + return stored; + }); + memory.RecallAsOfAsync( + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any()) + .Returns(call => RecallOf( + call.Arg().SessionId, + Message(call.Arg().SessionId, "user", "one"))); + memory.RecallAsync(Arg.Any(), Arg.Any()) + .Returns(call => RecallOf( + call.Arg().SessionId, + Message(call.Arg().SessionId, "user", "two"))); + var adapter = new AgentMemoryLongMemEvalAdapter(memory, AnsweringChat(), "typed-run"); + + // Injected but never invoked — the runner's shape when a question dies between injection + // and the agent call. Only ResetSessionAsync stands between this question's clocks and the + // next one's: InvokeAsync consumes the pending state itself, so a sequence that invokes + // before resetting cannot observe whether reset clears anything. + await adapter.ResetSessionAsync(); + adapter.InjectTimestampedConversationHistory(TwoTurnHistory()); + await adapter.ResetSessionAsync(); + adapter.InjectConversationHistory([("two", "second")]); + await adapter.InvokeAsync("second question"); + + // A leaked QueryTime would send the plain question down RecallAsOfAsync; a leaked turn + // clock would stamp the plain question's messages with the previous corpus's dates. + await memory.Received(1).RecallAsync(Arg.Any(), Arg.Any()); + await memory.DidNotReceive().RecallAsOfAsync( + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any()); + stored.Should().NotBeNull(); + stored!.Select(message => message.TimestampUtc).Should().Equal( + DateTimeOffset.UnixEpoch, DateTimeOffset.UnixEpoch.AddSeconds(1)); + } + + [Fact] + public void DoubleInjectionIsRefused_AcrossChannels() + { + var adapter = new AgentMemoryLongMemEvalAdapter( + Substitute.For(), Substitute.For(), "typed-run"); + adapter.InjectConversationHistory([("one", "first")]); + + var act = () => adapter.InjectTimestampedConversationHistory(TwoTurnHistory()); + + act.Should().Throw().WithMessage("*more than once*"); + } + + [Theory] + [InlineData(true, false, 0)] + [InlineData(false, true, 0)] + [InlineData(false, false, 3)] + public void RefusesOptionsThePointInTimeRecallPathWouldSilentlyIgnore( + bool expandFactsByPredicate, bool resolveQueryRelations, int graphRagItems) + { + // The dead-option shape: RecallAsOfAsync implements neither predicate expansion nor + // query-relation resolution nor GraphRAG, so a timestamped run configured with them would + // report a measurement of options that never executed. ResolveQueryRelations rides with + // expansion enabled so the combination is constructible at all. + var adapter = new AgentMemoryLongMemEvalAdapter( + Substitute.For(), + Substitute.For(), + "typed-run", + new LongMemEvalAdapterOptions + { + ExpandFactsByPredicate = expandFactsByPredicate || resolveQueryRelations, + ResolveQueryRelations = resolveQueryRelations, + GraphRagItems = graphRagItems + }); + + var act = () => adapter.InjectTimestampedConversationHistory(TwoTurnHistory()); + + act.Should().Throw().WithMessage("*silently ignored*"); + } +} diff --git a/tests/AgentMemory.Tests.Unit.LongMemEval/TypedMemEvalCommandLineTests.cs b/tests/AgentMemory.Tests.Unit.LongMemEval/TypedMemEvalCommandLineTests.cs new file mode 100644 index 00000000..b0bd3fc5 --- /dev/null +++ b/tests/AgentMemory.Tests.Unit.LongMemEval/TypedMemEvalCommandLineTests.cs @@ -0,0 +1,184 @@ +using System.Reflection; +using AgentEval.Memory.External.TypedMemEval; +using AgentMemory.LongMemEval; +using FluentAssertions; +using Xunit; + +namespace AgentMemory.Tests.Unit.LongMemEval; + +/// +/// 30.9c prereq C: every option the --typedmemeval verb advertises survives its parser and +/// lands on its options record. +/// +/// +/// The discipline is ExtractionSeedCommandLineTests', for the reason that file documents: an +/// option in KnownOptions is a promise that the verb honours it — the argument validator's +/// whole purpose is refusing options that would be ignored, so a name in that list and absent from +/// the options record is the validator lying. +/// +public sealed class TypedMemEvalCommandLineTests +{ + private static TypedMemEvalProgram.TypedMemEvalRunOptions Parse(params string[] args) + => TypedMemEvalProgram.Parse(args); + + [Theory] + [InlineData("prospective", TypedMemEvalVertical.Prospective)] + [InlineData("episodic", TypedMemEvalVertical.Episodic)] + [InlineData("arithmetic", TypedMemEvalVertical.Arithmetic)] + [InlineData("workingmemory", TypedMemEvalVertical.WorkingMemory)] + [InlineData("forgetting", TypedMemEvalVertical.Forgetting)] + public void EachVerticalSlugParsesToItsVertical(string slug, TypedMemEvalVertical expected) + { + Parse("--typedmemeval", slug).Verticals.Should().Equal(expected); + } + + [Fact] + public void AllRunsEveryVerticalInDeclarationOrder() + { + // Read from the descriptor table, never a literal list: when 0.23 (or any later revision) + // changes the family, this test must keep describing the shipped set. + Parse("--typedmemeval", "all").Verticals.Should().Equal( + TypedMemEvalVerticals.All.Select(descriptor => descriptor.Vertical)); + } + + [Fact] + public void AnUnknownVerticalIsRefusedWithTheKnownSlugs() + { + var act = () => Parse("--typedmemeval", "prospektive"); + + act.Should().Throw() + .WithMessage("*prospektive*").And.Message.Should().Contain("prospective"); + } + + [Fact] + public void AMissingVerticalValueIsRefused() + { + var act = () => Parse("--typedmemeval"); + + act.Should().Throw().WithMessage("*--typedmemeval requires a value*"); + } + + [Fact] + public void DefaultsAreWholeCorpusUnseededSingleRunAgentArm() + { + var options = Parse("--typedmemeval", "forgetting"); + + options.MaxQuestions.Should().BeNull("null runs the whole corpus, whatever size it ships at"); + options.RandomSeed.Should().BeNull(); + options.AnswerSeed.Should().BeNull(); + options.Runs.Should().Be(1); + options.Oracle.Should().BeFalse(); + options.Control.Should().BeFalse(); + } + + [Fact] + public void MaxQuestionsIsCarried() + { + Parse("--typedmemeval", "forgetting", "--max-questions", "4") + .MaxQuestions.Should().Be(4); + } + + [Theory] + [InlineData("0")] + [InlineData("-3")] + [InlineData("four")] + public void MaxQuestionsRefusesNonPositiveOrNonNumericValues(string value) + { + var act = () => Parse("--typedmemeval", "forgetting", "--max-questions", value); + + act.Should().Throw().WithMessage("*--max-questions*positive*"); + } + + [Theory] + [InlineData("--random-seed")] + [InlineData("--answer-seed")] + public void ASeedMayBeZeroOrNegative_AndNonNumericIsRefusedNotDropped(string option) + { + // A sampling seed is not a count; and a value the parser cannot read must stop the run + // rather than fall back to "unseeded" while the operator believes otherwise. + int? Of(TypedMemEvalProgram.TypedMemEvalRunOptions options) => + option == "--random-seed" ? options.RandomSeed : options.AnswerSeed; + + Of(Parse("--typedmemeval", "forgetting", option, "0")).Should().Be(0); + Of(Parse("--typedmemeval", "forgetting", option, "-7")).Should().Be(-7); + var act = () => Parse("--typedmemeval", "forgetting", option, "not-a-number"); + act.Should().Throw().WithMessage($"*{option}*integer*"); + } + + [Fact] + public void RunsIsCarried_AndBandingWithoutASeedIsRefusedBeforeAnySpend() + { + Parse("--typedmemeval", "forgetting", "--runs", "3", "--random-seed", "42") + .Runs.Should().Be(3); + + // TypedMemEvalRunSet.Summarize refuses to band runs that drew different questions; an + // unseeded multi-run would discover that only AFTER paying for every run. + var act = () => Parse("--typedmemeval", "forgetting", "--runs", "3"); + act.Should().Throw().WithMessage("*--runs*--random-seed*"); + } + + [Fact] + public void OracleAndControlFlagsAreCarried() + { + Parse("--typedmemeval", "forgetting", "--oracle").Oracle.Should().BeTrue(); + Parse("--typedmemeval", "prospective", "--control").Control.Should().BeTrue(); + } + + [Theory] + [InlineData("episodic")] + [InlineData("all")] + public void ControlIsTheProspectivePairsArmOnly(string vertical) + { + var act = () => Parse("--typedmemeval", vertical, "--control"); + + act.Should().Throw().WithMessage("*--control*prospective*"); + } + + [Fact] + public void AnUnknownOptionIsRefusedWithASuggestion() + { + // The same failure mode the shared validator exists for: a typo must stop the run, not be + // silently ignored while the report claims a measurement nobody configured. + var act = () => Parse("--typedmemeval", "forgetting", "--max-question", "4"); + + act.Should().Throw().WithMessage("*Did you mean --max-questions?*"); + } + + [Fact] + public void TheMainVerbAdvertisesTheTypedMemEvalSwitch() + { + // Dispatch happens before the main parser runs, so without this listing a typo'd verb + // switch would fall through to the default verb and silently measure something else. + var known = (string[])typeof(LongMemEvalProgram) + .GetField("KnownOptions", BindingFlags.NonPublic | BindingFlags.Static)! + .GetValue(null)!; + + known.Should().Contain("--typedmemeval"); + } + + [Fact] + public void EveryAdvertisedOptionIsCarriedOnTheOptionsRecord() + { + // THE drift guard, in the ExtractionSeedCommandLineTests shape: an option listed in + // KnownOptions and absent from the options record is the validator lying. The map below + // must name every known option, so adding an option without carrying it fails here. + var carriedBy = new Dictionary(StringComparer.Ordinal) + { + ["--typedmemeval"] = "Verticals", + ["--max-questions"] = "MaxQuestions", + ["--random-seed"] = "RandomSeed", + ["--answer-seed"] = "AnswerSeed", + ["--runs"] = "Runs", + ["--oracle"] = "Oracle", + ["--control"] = "Control", + }; + + TypedMemEvalProgram.KnownOptions.Should().BeEquivalentTo( + carriedBy.Keys, + "every advertised option must be mapped to the record property that honours it"); + var properties = typeof(TypedMemEvalProgram.TypedMemEvalRunOptions) + .GetProperties(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic) + .Select(property => property.Name); + properties.Should().Contain(carriedBy.Values); + } +} diff --git a/tests/AgentMemory.Tests.Unit.LongMemEval/TypedMemEvalEvidenceWiringTests.cs b/tests/AgentMemory.Tests.Unit.LongMemEval/TypedMemEvalEvidenceWiringTests.cs new file mode 100644 index 00000000..1a572770 --- /dev/null +++ b/tests/AgentMemory.Tests.Unit.LongMemEval/TypedMemEvalEvidenceWiringTests.cs @@ -0,0 +1,240 @@ +using System.Reflection; +using System.Text.Json; +using AgentEval.Memory.External.LongMemEval; +using AgentEval.Memory.External.Models; +using AgentEval.Memory.External.TypedMemEval; +using AgentMemory.Abstractions.Domain; +using AgentMemory.Abstractions.Services; +using AgentMemory.LongMemEval; +using FluentAssertions; +using Microsoft.Extensions.AI; +using NSubstitute; +using Xunit; + +namespace AgentMemory.Tests.Unit.LongMemEval; + +/// +/// 30.9c prereq B: the evidence envelope works for TypedMemEval's embedded corpora. +/// +/// +/// +/// The envelope path previously required an evidence index built from a dataset file +/// (LongMemEvalEvidenceIndex.Load(datasetPath, …)). TypedMemEval corpora are embedded +/// resources, so every typed run necessarily ran without an index, attached no envelope, and +/// reported attribution Unobserved — the free-checks pass named this exact gap. These tests +/// pin the new construction path end to end: corpus-sourced entries, the replicated option +/// mapping, the QueryTime-salted fingerprint the Prospective pairs require, and a full offline +/// stub-agent + stub-judge run whose attribution is observed. +/// +/// +/// Version-agnostic by design: no corpus SHA-256 or question-count literal appears here — counts, +/// ids and hashes are read from at run time, so the 0.23 corpus +/// revision (new ids, new hashes) must not touch this file. +/// +/// +public sealed class TypedMemEvalEvidenceWiringTests +{ + [Fact] + public void OptionMappingMatchesAgentEvalsInternalMapper_ForEveryVerticalAndArm() + { + // The runner maps its facade through an INTERNAL method this harness cannot call, so the + // harness replicates the mapping — and a replica without a drift guard is a lie waiting for + // the next AgentEval release. This invokes the real internal mapper by reflection and holds + // the replica to property-for-property equality across every vertical and both arms. + var toExternal = typeof(TypedMemEvalOptions).GetMethod( + "ToExternalOptions", BindingFlags.Instance | BindingFlags.NonPublic); + toExternal.Should().NotBeNull( + "the pinned AgentEval package is expected to map its facade through " + + "TypedMemEvalOptions.ToExternalOptions; if this fails after a version bump, re-verify " + + "the mapping and update TypedMemEvalOptionMapping to match"); + + TypedMemEvalOptions[] facades = + [ + new(), + new() + { + MaxQuestions = 7, + RandomSeed = 3, + AnswerSeed = -2, + AnswerTemperature = 0.5, + JudgeTemperature = 0.1, + RetainRawJudgeResponse = true, + EvidenceTopK = 25 + }, + new() { TemporalGrounding = TemporalGroundingMode.TimestampsOnly }, + new() { TemporalGrounding = TemporalGroundingMode.TimestampsAndText, ControlArm = true }, + new() { IncludeTimestamps = false }, + ]; + + foreach (var descriptor in TypedMemEvalVerticals.All) + foreach (var facade in facades) + { + var expected = (ExternalBenchmarkOptions)toExternal!.Invoke(facade, [descriptor])!; + var actual = TypedMemEvalOptionMapping.ToExternalOptions(facade, descriptor); + actual.Should().BeEquivalentTo( + expected, + because: $"the replica must match AgentEval's own mapping for {descriptor.Slug}"); + } + } + + [Fact] + public async Task OfflineTypedRunReportsObservedAttribution_OnceTheEmbeddedIndexIsWired() + { + // The free-checks probe proved this exact offline path runs (Forgetting, MaxQuestions=4, + // stub judge answering {"outcome":"abstained"}) — and that without an index its attribution + // is Unobserved on every question. This is the same run with the index wired. + var facade = new TypedMemEvalOptions { MaxQuestions = 4, RandomSeed = 20260815 }; + var memory = Substitute.For(); + IReadOnlyList lastStored = []; + memory.AddMessagesAsync(Arg.Any>(), Arg.Any()) + .Returns(call => + { + lastStored = call.Arg>().ToArray(); + return lastStored; + }); + memory.RecallAsync(Arg.Any(), Arg.Any()) + .Returns(call => + { + var request = call.Arg(); + var items = lastStored.Take(5).ToArray(); + return new RecallResult + { + Context = new MemoryContext + { + SessionId = request.SessionId, + AssembledAtUtc = DateTimeOffset.UnixEpoch, + RelevantMessages = new MemoryContextSection + { + Items = items, + RankedItems = items + .Select((message, index) => new MemoryContextRankedItem( + message.MessageId, 0.9 - index * 0.01, index + 1, index + 1)) + .ToArray() + } + }, + TotalItemsRetrieved = items.Length + }; + }); + var answerChat = Substitute.For(); + answerChat.GetResponseAsync( + Arg.Any>(), + Arg.Any(), + Arg.Any()) + .Returns(new ChatResponse( + new ChatMessage(ChatRole.Assistant, "I don't have that information."))); + var judgeChat = Substitute.For(); + judgeChat.GetResponseAsync( + Arg.Any>(), + Arg.Any(), + Arg.Any()) + .Returns(new ChatResponse( + new ChatMessage(ChatRole.Assistant, """{"outcome":"abstained"}"""))); + + var adapter = new AgentMemoryLongMemEvalAdapter( + memory, + answerChat, + "typed-offline", + new LongMemEvalAdapterOptions + { + // THE wiring under test: an index built from the embedded corpus, not from a file. + EvidenceIndex = LongMemEvalEvidenceIndex.CreateTypedMemEval( + TypedMemEvalVertical.Forgetting, facade) + }); + var result = await new TypedMemEvalRunner(judgeChat) + .RunAsync(adapter, TypedMemEvalVertical.Forgetting, facade); + + result.QuestionResults.Should().NotBeEmpty(); + result.QuestionResults.Should().OnlyContain(question => + question.ExecutionStatus == QuestionExecutionStatus.Completed); + // The point of the prerequisite: with the embedded-corpus index wired, the adapter attaches + // the envelope and NOT ONE question reports the attribution channel as unobserved. + result.QuestionResults.Should().OnlyContain(question => + question.TypedOutcome != null && + question.TypedOutcome!.Attribution != TypedMemEvalEvidenceAttribution.Unobserved); + result.TypedOutcomes.Should().NotBeNull(); + result.TypedOutcomes!.Attribution.Unobserved.Should().Be(0); + result.TypedOutcomes.Attribution.ObservedShare.Should().Be(1.0); + } + + [Fact] + public void ProspectivePairArms_ResolveDistinctly_BecauseTheFingerprintCarriesQueryTime() + { + // Two Prospective pair arms are one haystack and one question text asked at two instants. + // A fingerprint over the turns alone makes them a single entry with two candidate + // questions and an identical prompt — unresolvable. The query instant is identity. + var options = TypedMemEvalOptionMapping.ToExternalOptions( + new TypedMemEvalOptions { TemporalGrounding = TemporalGroundingMode.TimestampsOnly }, + TypedMemEvalVerticals.For(TypedMemEvalVertical.Prospective)); + var arms = new[] + { + PairArm("tme-test-a", "2026/06/04 (Thu) 21:30"), + PairArm("tme-test-b", "2026/06/16 (Tue) 21:30"), + }; + var index = LongMemEvalEvidenceIndex.CreateTimestamped(arms, options); + + foreach (var arm in arms) + { + var history = LongMemEvalHistoryFormatter.FormatTimestamped(arm, options); + var pairs = history.Turns + .Select(turn => (turn.UserMessage, turn.AssistantResponse)) + .ToArray(); + + var resolved = index.Resolve(pairs, history.QueryTime, arm.Question); + + resolved.QuestionId.Should().Be(arm.QuestionId); + resolved.QuestionDate.Should().Be(arm.QuestionDate); + } + } + + [Fact] + public void CreateTypedMemEval_AlignsWithTheRunnersTimestampedInjection_ForProspective() + { + // Selection, formatting, and prompt construction must all reproduce what the runner will + // inject, or Resolve throws mid-run. This drives the real embedded Prospective corpus + // through the same seeded selection and resolves every drawn question. + var facade = new TypedMemEvalOptions { MaxQuestions = 3, RandomSeed = 7 }; + var descriptor = TypedMemEvalVerticals.For(TypedMemEvalVertical.Prospective); + var options = TypedMemEvalOptionMapping.ToExternalOptions(facade, descriptor); + var entries = TypedMemEvalCorpus.Load(TypedMemEvalVertical.Prospective, options); + var index = LongMemEvalEvidenceIndex.CreateTypedMemEval( + TypedMemEvalVertical.Prospective, facade); + + entries.Should().NotBeEmpty(); + foreach (var entry in entries) + { + var history = LongMemEvalHistoryFormatter.FormatTimestamped(entry, options); + var pairs = history.Turns + .Select(turn => (turn.UserMessage, turn.AssistantResponse)) + .ToArray(); + + // Under TimestampsOnly the runner sends the bare question — no "Current Date:" prefix. + var resolved = index.Resolve(pairs, history.QueryTime, entry.Question); + + resolved.QuestionId.Should().Be(entry.QuestionId); + } + } + + private static LongMemEvalEntry PairArm(string id, string questionDate) => new() + { + QuestionId = id, + QuestionType = "prospective", + Question = "Has the reminder about the lease renewal come due yet?", + AnswerRaw = JsonSerializer.SerializeToElement("yes"), + QuestionDate = questionDate, + HaystackDates = ["2026/05/01 (Fri) 09:00"], + HaystackSessionIds = ["pair-session-1"], + HaystackSessions = + [ + [ + new LongMemEvalTurn + { + Role = "user", + Content = "Remind me to renew the lease on June 10.", + HasAnswer = true + }, + new LongMemEvalTurn { Role = "assistant", Content = "I will remind you." } + ] + ], + AnswerSessionIds = ["pair-session-1"] + }; +} diff --git a/tests/AgentMemory.Tests.Unit.LongMemEval/TypedMemEvalSeedOverlapGuardTests.cs b/tests/AgentMemory.Tests.Unit.LongMemEval/TypedMemEvalSeedOverlapGuardTests.cs new file mode 100644 index 00000000..2c9e4067 --- /dev/null +++ b/tests/AgentMemory.Tests.Unit.LongMemEval/TypedMemEvalSeedOverlapGuardTests.cs @@ -0,0 +1,111 @@ +using AgentEval.Memory.External.Models; +using AgentEval.Memory.External.TypedMemEval; +using AgentMemory.LongMemEval; +using FluentAssertions; +using Xunit; + +namespace AgentMemory.Tests.Unit.LongMemEval; + +/// +/// The double-count guard: the twelve time-grounded probe questions were carried into +/// TypedMemEval-Prospective, so a result set spanning both corpora double-counts them in any +/// total. AgentEval ships the detector (); +/// these tests hold the promise that OUR report assembly actually consults it — the +/// ship-but-unreachable shape is the one this repo keeps finding, so the wire itself is guarded. +/// +public sealed class TypedMemEvalSeedOverlapGuardTests +{ + private static ExternalBenchmarkResult ResultFor(string datasetIdentifier) => new() + { + BenchmarkId = "seed-overlap-guard-fixture", + BenchmarkName = "seed overlap guard fixture", + OverallAccuracy = null, + TaskAveragedAccuracy = null, + PerTypeResults = [], + QuestionResults = [], + Duration = TimeSpan.Zero, + Options = new ExternalBenchmarkOptions(), + Provenance = new BenchmarkRunProvenance + { + Mode = RunProvenanceMode.Full, + DatasetIdentifier = datasetIdentifier + } + }; + + [Fact] + public void WarnsWithAgentEvalsOwnText_WhenBothOverlappingCorporaAppear() + { + var results = new[] + { + ResultFor(TypedMemEvalRunSet.TimeGroundedCorpusId), + ResultFor(TypedMemEvalVerticals.For(TypedMemEvalVertical.Prospective).CorpusId), + }; + using var output = new StringWriter(); + + TypedMemEvalProgram.WarnOnSeedOverlap(results, output).Should().BeTrue(); + + // The message is upstream's, not ours: assert the load-bearing phrase, not the wording. + output.ToString().Should().Contain("typedmemeval: WARNING") + .And.Contain("double-count"); + } + + [Fact] + public void StaysSilent_ForTypedMemEvalOnlyResultSets() + { + var results = TypedMemEvalVerticals.All + .Select(descriptor => ResultFor(descriptor.CorpusId)) + .ToArray(); + using var output = new StringWriter(); + + TypedMemEvalProgram.WarnOnSeedOverlap(results, output).Should().BeFalse(); + + output.ToString().Should().BeEmpty( + "a guard that warns on every run trains its reader to ignore it"); + } + + [Fact] + public void StaysSilent_ForTheTimeGroundedCorpusAlone() + { + using var output = new StringWriter(); + + TypedMemEvalProgram + .WarnOnSeedOverlap([ResultFor(TypedMemEvalRunSet.TimeGroundedCorpusId)], output) + .Should().BeFalse(); + + output.ToString().Should().BeEmpty(); + } + + /// + /// The wire, not the method: RunAsync must call the guard where results are assembled. + /// Raw-source assertion with comment lines stripped — a substring guard that a comment can + /// satisfy is the likeliest way this wire actually gets cut (the Wave C–E review found exactly + /// that failure on the voting guard). + /// + [Fact] + public void TheVerbsResultAssemblyConsultsTheGuard() + { + var source = File.ReadAllLines(TypedMemEvalProgramSourcePath()) + .Where(line => !line.TrimStart().StartsWith("//", StringComparison.Ordinal)) + .ToArray(); + + source.Count(line => line.Contains("WarnOnSeedOverlap(", StringComparison.Ordinal)) + .Should().BeGreaterThan( + 1, + "the definition alone is ship-but-unreachable; RunAsync must call it where " + + "results are assembled"); + } + + private static string TypedMemEvalProgramSourcePath() + { + var directory = new DirectoryInfo(AppContext.BaseDirectory); + while (directory is not null && + !File.Exists(Path.Combine(directory.FullName, "AgentMemory.slnx"))) + { + directory = directory.Parent; + } + + directory.Should().NotBeNull("the repo root (AgentMemory.slnx) must be findable from the test bin"); + return Path.Combine( + directory!.FullName, "tools", "AgentMemory.LongMemEval", "TypedMemEvalProgram.cs"); + } +} diff --git a/tools/AgentMemory.LongMemEval/AgentMemory.LongMemEval.csproj b/tools/AgentMemory.LongMemEval/AgentMemory.LongMemEval.csproj index 0c8f1448..9367bc0b 100644 --- a/tools/AgentMemory.LongMemEval/AgentMemory.LongMemEval.csproj +++ b/tools/AgentMemory.LongMemEval/AgentMemory.LongMemEval.csproj @@ -30,7 +30,13 @@ - + + diff --git a/tools/AgentMemory.LongMemEval/AgentMemoryLongMemEvalAdapter.cs b/tools/AgentMemory.LongMemEval/AgentMemoryLongMemEvalAdapter.cs index 934bced5..21327949 100644 --- a/tools/AgentMemory.LongMemEval/AgentMemoryLongMemEvalAdapter.cs +++ b/tools/AgentMemory.LongMemEval/AgentMemoryLongMemEvalAdapter.cs @@ -18,6 +18,7 @@ namespace AgentMemory.LongMemEval; public sealed partial class AgentMemoryLongMemEvalAdapter : IEvaluableAgent, IHistoryInjectableAgent, + ITimestampedHistoryInjectableAgent, ISessionResettableAgent { internal const string SystemPrompt = @@ -57,6 +58,18 @@ private string AnswerTextOf(ChatResponse response) private readonly object _stateLock = new(); private readonly List _telemetry = []; private IReadOnlyList<(string UserMessage, string AssistantResponse)>? _pendingHistory; + + /// + /// One instant per pending turn pair, aligned with . Null when the + /// history arrived through the untimestamped channel. + /// + private IReadOnlyList? _pendingTurnTimestamps; + + /// + /// The instant the pending question is asked, from the typed channel. Null when the history + /// arrived untimestamped. + /// + private DateTimeOffset? _pendingQueryTime; private int _questionNumber; private string _sessionId; private string _ownerId; @@ -200,6 +213,68 @@ public void InjectConversationHistory( } } + /// + /// Accepts history whose turns carry real instants (TypedMemEval's TimestampsOnly channel). + /// + /// + /// + /// Each turn's becomes the stored messages' + /// own clock (Message.TimestampUtc — the product's valid time), replacing the epoch + + /// injection-ordinal clock the untimestamped channel uses. No date text is added to any + /// message content: the whole point of the TimestampsOnly grounding mode is that dates are + /// structural, and a harness that re-printed them into the text would hand back the crutch the + /// mode exists to remove. + /// + /// + /// is carried to the question call as the + /// answer-time anchor: recall runs through RecallAsOfAsync with QueryTime as the + /// valid-time clock (so a fact's validity window is judged against the question's "now", not the + /// machine's), and the answer prompt's "Current date" line is rendered from it rather than from + /// evaluator-side knowledge. + /// + /// + /// Refused up front when the adapter options include surfaces the point-in-time recall path does + /// not implement (predicate expansion, query-relation resolution, GraphRAG): accepting them + /// would run a question whose configuration silently did nothing — the dead-option shape this + /// codebase exists to refuse. + /// + /// + public void InjectTimestampedConversationHistory(TimestampedConversationHistory history) + { + ArgumentNullException.ThrowIfNull(history); + ArgumentNullException.ThrowIfNull(history.Turns); + if (_options.ExpandFactsByPredicate || _options.ResolveQueryRelations || + _options.GraphRagItems > 0) + { + throw new InvalidOperationException( + "Timestamped LongMemEval history anchors recall at RecallAsOfAsync, which does not " + + "implement predicate expansion, query-relation resolution, or GraphRAG; refusing to " + + "run a question whose options would be silently ignored."); + } + + var pairs = new (string UserMessage, string AssistantResponse)[history.Turns.Count]; + var timestamps = new DateTimeOffset[history.Turns.Count]; + for (var index = 0; index < history.Turns.Count; index++) + { + var turn = history.Turns[index]; + pairs[index] = (turn.UserMessage, turn.AssistantResponse); + timestamps[index] = turn.Timestamp; + } + + lock (_stateLock) + { + if (_pendingHistory is not null) + { + throw new InvalidOperationException( + "LongMemEval history was injected more than once for the same question."); + } + + _pendingHistory = pairs; + _pendingTurnTimestamps = timestamps; + _pendingQueryTime = history.QueryTime; + } + } + public Task ResetSessionAsync(CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); @@ -209,6 +284,8 @@ public Task ResetSessionAsync(CancellationToken cancellationToken = default) _sessionId = ScopeId("session", _questionNumber); _ownerId = ScopeId("owner", _questionNumber); _pendingHistory = null; + _pendingTurnTimestamps = null; + _pendingQueryTime = null; } return Task.CompletedTask; @@ -222,6 +299,8 @@ public async Task InvokeAsync( var timings = new LongMemEvalStageTimingCollector(); IReadOnlyList<(string UserMessage, string AssistantResponse)> history; + IReadOnlyList? turnTimestamps; + DateTimeOffset? queryTime; string sessionId; string ownerId; int questionNumber; @@ -236,7 +315,11 @@ public async Task InvokeAsync( "LongMemEval question cannot run with empty conversation history."); } + turnTimestamps = _pendingTurnTimestamps; + queryTime = _pendingQueryTime; _pendingHistory = null; + _pendingTurnTimestamps = null; + _pendingQueryTime = null; sessionId = _sessionId; ownerId = _ownerId; questionNumber = _questionNumber; @@ -245,8 +328,13 @@ public async Task InvokeAsync( LongMemEvalEvidenceQuestion? evidenceQuestion = null; try { + // A timestamped question must resolve through the QueryTime-aware fingerprint: + // Prospective pair arms share one haystack and differ only in the query instant, so a + // fingerprint over the turns alone would make the two arms indistinguishable. if (_options.EvidenceIndex is not null) - evidenceQuestion = _options.EvidenceIndex.Resolve(history, prompt); + evidenceQuestion = queryTime is { } evidenceQueryTime + ? _options.EvidenceIndex.Resolve(history, evidenceQueryTime, prompt) + : _options.EvidenceIndex.Resolve(history, prompt); } catch (Exception) when (!cancellationToken.IsCancellationRequested) { @@ -256,7 +344,8 @@ public async Task InvokeAsync( var originsByMessageId = new Dictionary(StringComparer.Ordinal); var messages = BuildMessages( - _runId, history, sessionId, ownerId, questionNumber, evidenceQuestion, originsByMessageId); + _runId, history, sessionId, ownerId, questionNumber, evidenceQuestion, originsByMessageId, + turnTimestamps); LongMemEvalPreparedQuestion? preparedQuestion = null; if (_options.PreparedMemory) @@ -602,6 +691,33 @@ _chatClient is LongMemEvalChatCallMeter callMeter ? prompt : await _options.QueryFormulator.DeriveAsync(prompt, cancellationToken).ConfigureAwait(false); + var recallRequest = new RecallRequest + { + SessionId = sessionId, + UserId = ownerId, + Query = retrievalQuery, + Options = new RecallOptions + { + MaxRecentMessages = 0, + MaxRelevantMessages = requestedMessages, + MaxEntities = budget.Entities, + MaxPreferences = budget.Preferences, + MaxFacts = budget.Facts, + MaxTraces = 0, + // G5 "hard" tier: a relation returned whole, for the aggregation + // questions top-K structurally cannot answer. + ExpandFactsByPredicate = _options.ExpandFactsByPredicate, + // J2.2: also expand on the relations the question itself names, for + // the multi-relation case top-K structurally cannot nominate. + ResolveQueryRelations = _options.ResolveQueryRelations, + MaxExpandedFacts = _options.MaxExpandedFacts, + MaxGraphRagItems = budget.GraphRag, + MinSimilarityScore = _options.MinSimilarityScore, + BlendMode = BlendModeFor(budget.GraphRag), + IncludeDiagnostics = evidenceQuestion is not null + } + }; + RecallResult recall; try { @@ -609,34 +725,16 @@ _chatClient is LongMemEvalChatCallMeter callMeter LongMemEvalStage.Retrieval, () => LongMemEvalRuntime.ExecuteStageAsync( "retrieval", - () => _memory.RecallAsync( - new RecallRequest - { - SessionId = sessionId, - UserId = ownerId, - Query = retrievalQuery, - Options = new RecallOptions - { - MaxRecentMessages = 0, - MaxRelevantMessages = requestedMessages, - MaxEntities = budget.Entities, - MaxPreferences = budget.Preferences, - MaxFacts = budget.Facts, - MaxTraces = 0, - // G5 "hard" tier: a relation returned whole, for the aggregation - // questions top-K structurally cannot answer. - ExpandFactsByPredicate = _options.ExpandFactsByPredicate, - // J2.2: also expand on the relations the question itself names, for - // the multi-relation case top-K structurally cannot nominate. - ResolveQueryRelations = _options.ResolveQueryRelations, - MaxExpandedFacts = _options.MaxExpandedFacts, - MaxGraphRagItems = budget.GraphRag, - MinSimilarityScore = _options.MinSimilarityScore, - BlendMode = BlendModeFor(budget.GraphRag), - IncludeDiagnostics = evidenceQuestion is not null - } - }, - cancellationToken))).ConfigureAwait(false); + // A timestamped question anchors recall at its own "now". QueryTime is the + // VALID-time clock — it decides which facts' validity windows contain the + // question's instant, which is precisely the prospective-memory semantics the + // typed channel exists to measure. The TRANSACTION clock must stay at the + // machine's now: the corpus was ingested moments ago, so bounding created_at at + // a 2023 QueryTime would erase everything just stored and score an empty memory. + () => queryTime is { } asOf + ? _memory.RecallAsOfAsync( + recallRequest, asOf, DateTimeOffset.UtcNow, cancellationToken) + : _memory.RecallAsync(recallRequest, cancellationToken))).ConfigureAwait(false); } catch (Exception) when (!cancellationToken.IsCancellationRequested) { @@ -767,8 +865,15 @@ _chatClient is LongMemEvalChatCallMeter callMeter }; } + // The answer-time anchor. A timestamped question renders "now" from the QueryTime the typed + // channel delivered — data the system under test legitimately holds — rather than from the + // evaluator-side index, which the untimestamped path (whose only source is the evidence + // index) still uses. var answerPrompt = BuildAnswerPrompt( - recall.Context, prompt, evidenceQuestion?.QuestionDate, originsByMessageId); + recall.Context, + prompt, + queryTime is { } answerNow ? FormatQueryTime(answerNow) : evidenceQuestion?.QuestionDate, + originsByMessageId); LongMemEvalRetrievalEvidence? retrievalEvidence = null; AgentEval.Memory.External.Models.QuestionEvidenceEnvelope? normalizedEvidence = null; if (evidenceQuestion is not null) @@ -1333,7 +1438,8 @@ internal static List BuildMessages( string ownerId, int questionNumber, LongMemEvalEvidenceQuestion? evidenceQuestion, - IDictionary originsByMessageId) + IDictionary originsByMessageId, + IReadOnlyList? turnTimestamps = null) { var expectedCount = history.Count * 2; if (evidenceQuestion is not null && evidenceQuestion.Messages.Count != expectedCount) @@ -1341,6 +1447,11 @@ internal static List BuildMessages( throw new InvalidOperationException( $"LongMemEval evidence contained {evidenceQuestion.Messages.Count} origins for {expectedCount} injected messages."); } + if (turnTimestamps is not null && turnTimestamps.Count != history.Count) + { + throw new InvalidOperationException( + $"LongMemEval timestamped history carried {turnTimestamps.Count} instants for {history.Count} turns."); + } var result = new List(expectedCount); var ordinal = 0; @@ -1392,12 +1503,26 @@ Message Message(string role, string content) ConversationId = sessionId, Role = role, Content = content, - TimestampUtc = DateTimeOffset.UnixEpoch.AddSeconds(current), + // The typed channel's whole claim: the turn's own instant becomes the message's + // valid time. Both halves of a pair share the turn's instant, and the untimestamped + // channel keeps its epoch + injection-ordinal clock so every sealed measurement + // stays byte-comparable with itself. + TimestampUtc = turnTimestamps is null + ? DateTimeOffset.UnixEpoch.AddSeconds(current) + : turnTimestamps[current / 2], Metadata = metadata }; } } + /// + /// Renders a QueryTime for the answer prompt's "Current date" line, in the corpus's own date + /// style so the anchor and the per-message timestamps read as one convention. + /// + internal static string FormatQueryTime(DateTimeOffset queryTime) => + queryTime.UtcDateTime.ToString( + "yyyy/MM/dd (ddd) HH:mm", System.Globalization.CultureInfo.InvariantCulture); + /// /// G3B.2. The source timestamp AgentMemory persisted with the message and returns through recall. /// diff --git a/tools/AgentMemory.LongMemEval/LongMemEvalEvidenceIndex.cs b/tools/AgentMemory.LongMemEval/LongMemEvalEvidenceIndex.cs index 5f100162..70c837b1 100644 --- a/tools/AgentMemory.LongMemEval/LongMemEvalEvidenceIndex.cs +++ b/tools/AgentMemory.LongMemEval/LongMemEvalEvidenceIndex.cs @@ -1,8 +1,10 @@ +using System.Globalization; using System.Security.Cryptography; using System.Text; using System.Text.Json.Serialization; using AgentEval.Memory.External.LongMemEval; using AgentEval.Memory.External.Models; +using AgentEval.Memory.External.TypedMemEval; using AgentMemory.Abstractions.Domain; namespace AgentMemory.LongMemEval; @@ -45,6 +47,75 @@ public static LongMemEvalEvidenceIndex Load( internal static LongMemEvalEvidenceIndex Create( IReadOnlyList entries, ExternalBenchmarkOptions options) + => CreateCore(entries, options, entry => + { + var formatted = LongMemEvalHistoryFormatter.Format(entry, options); + return (formatted, Fingerprint(formatted), BuildInvocationPrompt(entry)); + }); + + /// + /// Builds the index for entries a runner will inject through the timestamped channel + /// (ITimestampedHistoryInjectableAgent), i.e. any run whose temporal grounding is not + /// . + /// + /// + /// + /// Three things differ from , each forced by what the runner actually sends. + /// The formatted content comes from FormatTimestamped — under TimestampsOnly the session + /// boundary marker drops its date, so an index built from the dated Format output would + /// never align. The invocation prompt is grounding-aware — under TimestampsOnly the runner sends + /// the bare question, because printing "Current Date:" would hand back the crutch the mode + /// removes. And the fingerprint is salted with the history's QueryTime — Prospective pair arms + /// share one haystack and one question text and differ only in the query instant, so a + /// fingerprint over the turns alone cannot tell the arms apart. + /// + /// + internal static LongMemEvalEvidenceIndex CreateTimestamped( + IReadOnlyList entries, + ExternalBenchmarkOptions options) + => CreateCore(entries, options, entry => + { + var history = LongMemEvalHistoryFormatter.FormatTimestamped(entry, options); + var formatted = history.Turns + .Select(turn => (turn.UserMessage, turn.AssistantResponse)) + .ToArray(); + return (formatted, + Fingerprint(formatted, history.QueryTime), + BuildTimestampedInvocationPrompt(entry, options)); + }); + + /// + /// Builds the evidence index for one TypedMemEval vertical from its embedded corpus, selecting + /// and formatting entries exactly as TypedMemEvalRunner will for the same facade options. + /// + /// + /// This is the construction path the envelope pipeline was missing: requires + /// a dataset file, TypedMemEval corpora are embedded resources, and without an index the + /// adapter attaches no evidence envelope — every typed run reported attribution + /// Unobserved. Question counts, ids and hashes are all read from the corpus at run time, + /// never hard-coded, so the 0.23 corpus revision changes nothing here. + /// + internal static LongMemEvalEvidenceIndex CreateTypedMemEval( + TypedMemEvalVertical vertical, + TypedMemEvalOptions? facade = null) + { + facade ??= new TypedMemEvalOptions(); + facade.Validate(); + var descriptor = TypedMemEvalVerticals.For(vertical); + var options = TypedMemEvalOptionMapping.ToExternalOptions(facade, descriptor); + var entries = TypedMemEvalCorpus.Load(vertical, options); + return options.TemporalGrounding == TemporalGroundingMode.None + ? Create(entries, options) + : CreateTimestamped(entries, options); + } + + private static LongMemEvalEvidenceIndex CreateCore( + IReadOnlyList entries, + ExternalBenchmarkOptions options, + Func Formatted, + string Fingerprint, + string InvocationPrompt)> format) { ArgumentNullException.ThrowIfNull(entries); ArgumentNullException.ThrowIfNull(options); @@ -54,9 +125,8 @@ internal static LongMemEvalEvidenceIndex Create( var questions = new List(entries.Count); foreach (var entry in entries) { - var formatted = LongMemEvalHistoryFormatter.Format(entry, options); - var question = BuildQuestion(entry, formatted, options); - var fingerprint = Fingerprint(formatted); + var (formatted, fingerprint, invocationPrompt) = format(entry); + var question = BuildQuestion(entry, formatted, options, invocationPrompt); if (!byHistory.TryGetValue(fingerprint, out var matching)) { matching = []; @@ -81,8 +151,26 @@ public LongMemEvalEvidenceQuestion Resolve( { ArgumentNullException.ThrowIfNull(history); ArgumentException.ThrowIfNullOrWhiteSpace(prompt); + return ResolveCore(Fingerprint(history), prompt); + } + + /// + /// Resolves a question injected through the timestamped channel. The query instant is part of + /// the identity: Prospective pair arms share one haystack and one question text, so without it + /// the two arms are a single fingerprint and neither can be attributed. + /// + public LongMemEvalEvidenceQuestion Resolve( + IReadOnlyList<(string UserMessage, string AssistantResponse)> history, + DateTimeOffset queryTime, + string prompt) + { + ArgumentNullException.ThrowIfNull(history); + ArgumentException.ThrowIfNullOrWhiteSpace(prompt); + return ResolveCore(Fingerprint(history, queryTime), prompt); + } - var fingerprint = Fingerprint(history); + private LongMemEvalEvidenceQuestion ResolveCore(string fingerprint, string prompt) + { lock (_gate) { if (!_questionsByHistory.TryGetValue(fingerprint, out var candidates)) @@ -122,7 +210,8 @@ public LongMemEvalEvidenceQuestion GetByQuestionId(string questionId) private static LongMemEvalEvidenceQuestion BuildQuestion( LongMemEvalEntry entry, IReadOnlyList<(string UserMessage, string AssistantResponse)> formatted, - ExternalBenchmarkOptions options) + ExternalBenchmarkOptions options, + string invocationPrompt) { var sessions = entry.HaystackSessions ?? throw new InvalidOperationException($"LongMemEval question {entry.QuestionId} has no sessions."); @@ -248,7 +337,7 @@ LongMemEvalMessageOrigin Origin( entry.QuestionId, entry.QuestionType, entry.Question, - BuildInvocationPrompt(entry), + invocationPrompt, entry.Answer, entry.QuestionDate ?? string.Empty, entry.IsAbstention, @@ -262,6 +351,17 @@ private static string BuildInvocationPrompt(LongMemEvalEntry entry) => ? entry.Question : $"Current Date: {entry.QuestionDate}\n\n{entry.Question}"; + /// + /// The prompt TypedMemEvalRunner.BuildPrompt sends for a structurally-injected question: + /// under TimestampsOnly the query time arrives through the typed channel and the prompt is the + /// bare question; under TimestampsAndText the "Current Date:" prefix survives. + /// + private static string BuildTimestampedInvocationPrompt( + LongMemEvalEntry entry, ExternalBenchmarkOptions options) => + options.TemporalGrounding == TemporalGroundingMode.TimestampsOnly + ? entry.Question + : BuildInvocationPrompt(entry); + private static bool IsSessionBoundary((string UserMessage, string AssistantResponse) turn) => turn.UserMessage.StartsWith("--- Session ", StringComparison.Ordinal) && turn.UserMessage.EndsWith(" ---", StringComparison.Ordinal) && @@ -275,6 +375,23 @@ private static bool IsSessionBoundary((string UserMessage, string AssistantRespo internal static string Fingerprint( IReadOnlyList<(string UserMessage, string AssistantResponse)> history) + => Fingerprint(history, salt: null); + + /// + /// Fingerprint for a timestamped history: the turns plus the query instant. Two + /// Prospective pair arms are the same turns asked at two different instants, so the instant is + /// part of the question's identity, not metadata about it. + /// + internal static string Fingerprint( + IReadOnlyList<(string UserMessage, string AssistantResponse)> history, + DateTimeOffset queryTime) + => Fingerprint( + history, + $"query-time:{queryTime.UtcTicks.ToString(CultureInfo.InvariantCulture)}"); + + private static string Fingerprint( + IReadOnlyList<(string UserMessage, string AssistantResponse)> history, + string? salt) { var builder = new StringBuilder(); foreach (var (user, assistant) in history) @@ -283,6 +400,9 @@ internal static string Fingerprint( Append(assistant); } + if (salt is not null) + Append(salt); + return Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(builder.ToString()))); void Append(string value) => builder.Append(value.Length).Append(':').Append(value).Append('|'); diff --git a/tools/AgentMemory.LongMemEval/Program.cs b/tools/AgentMemory.LongMemEval/Program.cs index 4f46c8ee..efdb3123 100644 --- a/tools/AgentMemory.LongMemEval/Program.cs +++ b/tools/AgentMemory.LongMemEval/Program.cs @@ -145,6 +145,14 @@ public static async Task RunAsync(string[] args) .ConfigureAwait(false); } + if (args.Contains("--typedmemeval", StringComparer.Ordinal)) + { + // 30.9c. TypedMemEval verticals (embedded AgentEval corpora): typed-outcome runs, + // oracle ceilings, the Prospective control arm, and N-run bands. Cited as + // TypedMemEval- (AgentEval), never as LongMemEval. + return await TypedMemEvalProgram.RunAsync(args).ConfigureAwait(false); + } + try { @@ -464,7 +472,7 @@ await File.WriteAllTextAsync( private static readonly string[] KnownOptions = [ "--reference-arm", "--surface-probe", "--predicate-distribution", "--prepared-pair", - "--procedural-benefit", "--attempts", + "--procedural-benefit", "--typedmemeval", "--attempts", "--oracle-decomposition", "--max-sub-questions", "--question-ids", "--no-content", "--oracle-precision", "--distractor-sessions", "--gold-fraction", "--oracle-representation", "--capture-headroom", "--artifacts", @@ -685,6 +693,18 @@ formatter boilerplate (session boundaries and padding), keeps retrieval order, a scored as wrong. Cannot be combined with --memory-mode, --prepared-pair, --exclude-synthetic-messages, or a non-none --oracle. + --typedmemeval runs a + TypedMemEval vertical from AgentEval's embedded corpora against the structured memory stack: + [--max-questions N] [--random-seed N] [--answer-seed N] [--runs N] [--oracle] [--control] + --oracle runs the perfect-retrieval ceiling (LongMemEvalOracleOptions.GoldOnly; no memory + store, no container). --control runs the Prospective pair's control arm (dates re-added to + the text) and is valid only with the prospective vertical. --runs N repeats the identical + configuration and, past one run, prints the TypedMemEvalRunSet band with QuestionsWithFlips; + it requires --random-seed so every run draws the same questions. Each run's native + ExternalBenchmarkResult JSON lands under artifacts/evaluation/, named by vertical, arm, + seed, run index, and UTC stamp. TypedMemEval results are never LongMemEval results: do not + sum or average across the two families. + --prepared-pair prepares structured memory once, freezes it, clones it, and evaluates isolated Structured and Hybrid arms. Supplying both diagnostic selectors with --prepared-pair runs exactly one extraction unit and can never emit a report or execute recall/judging. --preflight-only freezes the exact prepared-pair batch plan, proves zero provider calls/writes, diff --git a/tools/AgentMemory.LongMemEval/TypedMemEvalOptionMapping.cs b/tools/AgentMemory.LongMemEval/TypedMemEvalOptionMapping.cs new file mode 100644 index 00000000..439854ff --- /dev/null +++ b/tools/AgentMemory.LongMemEval/TypedMemEvalOptionMapping.cs @@ -0,0 +1,70 @@ +using AgentEval.Memory.External.Models; +using AgentEval.Memory.External.TypedMemEval; + +namespace AgentMemory.LongMemEval; + +/// +/// Replicates AgentEval's internal TypedMemEvalOptions.ToExternalOptions mapping, so the +/// evidence index selects and formats exactly the entries the runner will inject. +/// +/// +/// +/// The runner maps its facade onto through an +/// internal method, so this harness cannot call it — but the evidence index must be built +/// from the same selection (MaxQuestions, RandomSeed, stratification) and the same formatting +/// (session boundaries, timestamps, grounding), or fingerprints will never align and every typed +/// run degrades to attribution Unobserved. The invariants replicated here are the family's +/// own, stated in the facade's documentation: stratified sampling on, session boundaries +/// preserved, the corpus id as the dataset mode (with a -control suffix for the control +/// arm), the vertical's required grounding unless overridden, and TextBlob injection forced up to +/// structured whenever grounding is active. +/// +/// +/// Drift is guarded, not hoped away: a test invokes the real internal mapper by reflection and +/// asserts property-for-property equality with this replica, so an AgentEval release that changes +/// the mapping fails a named test instead of silently mis-aligning the index. +/// +/// +internal static class TypedMemEvalOptionMapping +{ + internal static ExternalBenchmarkOptions ToExternalOptions( + TypedMemEvalOptions facade, + TypedMemEvalVerticalDescriptor descriptor) + { + ArgumentNullException.ThrowIfNull(facade); + ArgumentNullException.ThrowIfNull(descriptor); + + var grounding = facade.TemporalGrounding ?? descriptor.RequiredGrounding; + var injection = grounding != TemporalGroundingMode.None && + facade.HistoryInjectionMode == HistoryInjectionMode.TextBlob + ? HistoryInjectionMode.StructuredChatHistory + : facade.HistoryInjectionMode; + + return new ExternalBenchmarkOptions + { + MaxQuestions = facade.MaxQuestions, + RandomSeed = facade.RandomSeed, + StratifiedSampling = true, + PreserveSessionBoundaries = true, + IncludeTimestamps = facade.IncludeTimestamps, + AnswerTemperature = facade.AnswerTemperature, + AnswerSeed = facade.AnswerSeed, + TemporalGrounding = grounding, + HistoryInjectionMode = injection, + DatasetMode = facade.ControlArm + ? $"{descriptor.CorpusId}-control" + : descriptor.CorpusId, + DatasetPath = null, + RunProvenanceMode = RunProvenanceMode.Full, + JudgeVerdictProtocol = JudgeVerdictProtocol.StructuredJson, + JudgeTemperature = facade.JudgeTemperature, + JudgeMaxOutputTokens = facade.JudgeMaxOutputTokens, + MaxJudgeRetries = facade.MaxJudgeRetries, + JudgeFailurePolicy = facade.JudgeFailurePolicy, + JudgeEvidenceMode = facade.JudgeEvidenceMode, + RetainRawJudgeResponse = facade.RetainRawJudgeResponse, + EvidenceCaptureMode = facade.EvidenceCaptureMode, + EvidenceTopK = facade.EvidenceTopK + }; + } +} diff --git a/tools/AgentMemory.LongMemEval/TypedMemEvalProgram.cs b/tools/AgentMemory.LongMemEval/TypedMemEvalProgram.cs new file mode 100644 index 00000000..de6d210d --- /dev/null +++ b/tools/AgentMemory.LongMemEval/TypedMemEvalProgram.cs @@ -0,0 +1,416 @@ +using System.Globalization; +using System.Text.Json; +using AgentEval.Memory.External.Models; +using AgentEval.Memory.External.TypedMemEval; +using AgentMemory.Abstractions.Services; +using Azure; +using Azure.AI.OpenAI; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.DependencyInjection; + +namespace AgentMemory.LongMemEval; + +/// +/// The --typedmemeval verb: runs one TypedMemEval vertical (or all five) against the +/// structured AgentMemory stack, the oracle ceiling, or the Prospective control arm. +/// +/// +/// +/// Citation rule (ADR-026): results are cited as "TypedMemEval-<Vertical> (AgentEval)" +/// and are never summed or averaged with LongMemEval numbers. Nothing here hard-codes a corpus +/// hash or question count — identity comes from the result's own provenance, so the 0.23 corpus +/// revision changes file names and hashes without changing this code. +/// +/// +/// --runs N repeats the identical configuration sequentially and, past one run, prints the +/// band — including QuestionsWithFlips, the +/// number that says whether a narrow band is evidence or coincidence. Banding requires a +/// --random-seed, because unseeded runs draw different questions and +/// rightly refuses to band different experiments. +/// +/// +/// The memory arm always runs the structured mode: it is the stack under measurement, and +/// the timestamped channel's point-in-time recall performs no semantic message search, so a raw or +/// hybrid arm under it could never fill a message budget honestly. +/// +/// +internal static class TypedMemEvalProgram +{ + /// Answer-context item budget, matching the main verb's default. + private const int DefaultMaxRelevant = 30; + + /// + /// Every option this verb accepts. An option listed here is a promise that the verb honours + /// it — the guard tests hold each name against a property on the options record. + /// + internal static readonly string[] KnownOptions = + [ + "--typedmemeval", "--max-questions", "--random-seed", "--answer-seed", + "--runs", "--oracle", "--control", + ]; + + public static async Task RunAsync(string[] args) + { + try + { + var options = Parse(args); + + var endpoint = RequiredEnvironment("AZURE_OPENAI_ENDPOINT"); + var apiKey = RequiredEnvironment("AZURE_OPENAI_API_KEY"); + var deployment = RequiredEnvironment("AZURE_OPENAI_DEPLOYMENT"); + var azureClient = new AzureOpenAIClient(new Uri(endpoint), new AzureKeyCredential(apiKey)); + using var answerChatClient = new LongMemEvalChatCallMeter( + azureClient.GetChatClient(deployment).AsIChatClient()); + using var judgeChatClient = new LongMemEvalChatCallMeter( + azureClient.GetChatClient(deployment).AsIChatClient()); + + // The oracle arm reads projected gold directly: no memory store, no embeddings, no + // container. Everything below the profile exists only for the memory arm. + LongMemEvalMemoryProfile? profile = null; + LongMemEvalChatCallMeter? extractionChatClient = null; + string? extractionDeployment = null; + try + { + if (!options.Oracle) + { + var embeddingDeployment = RequiredEnvironment("AZURE_OPENAI_EMBEDDING_DEPLOYMENT"); + extractionDeployment = + Environment.GetEnvironmentVariable("AZURE_OPENAI_EXTRACTION_DEPLOYMENT") + ?? deployment; + extractionChatClient = new LongMemEvalChatCallMeter( + new ProviderCompatibleExtractionChatClient( + azureClient.GetChatClient(extractionDeployment).AsIChatClient())); + var embeddingGenerator = azureClient + .GetEmbeddingClient(embeddingDeployment) + .AsIEmbeddingGenerator(); + var embeddingDimensions = await LongMemEvalRuntime + .ProbeEmbeddingDimensionsAsync(embeddingGenerator) + .ConfigureAwait(false); + profile = await LongMemEvalMemoryProfile + .StartAsync( + embeddingGenerator, + extractionChatClient, + LongMemEvalMemoryMode.Structured, + extractionDeployment, + embeddingDimensions, + Console.Out, + CancellationToken.None) + .ConfigureAwait(false); + } + + var assembledResults = new List(); + foreach (var vertical in options.Verticals) + { + assembledResults.AddRange(await RunVerticalAsync( + vertical, options, answerChatClient, judgeChatClient, deployment, profile) + .ConfigureAwait(false)); + } + + // The double-count guard runs over everything this invocation assembled: the + // twelve tg probe questions carried into Prospective make any cross-corpus total + // a double count, and the detector is upstream's, not a local id comparison. + WarnOnSeedOverlap(assembledResults, Console.Out); + } + finally + { + if (profile is not null) + await profile.DisposeAsync().ConfigureAwait(false); + extractionChatClient?.Dispose(); + } + + return 0; + } + catch (Exception exception) + { + Console.Error.WriteLine($"typedmemeval: {exception.Message}"); + return 1; + } + } + + private static async Task> RunVerticalAsync( + TypedMemEvalVertical vertical, + TypedMemEvalRunOptions options, + IChatClient answerChatClient, + IChatClient judgeChatClient, + string deployment, + LongMemEvalMemoryProfile? profile) + { + var descriptor = TypedMemEvalVerticals.For(vertical); + var results = new List(options.Runs); + for (var runIndex = 1; runIndex <= options.Runs; runIndex++) + { + var facade = BuildFacade(options); + var runner = new TypedMemEvalRunner(judgeChatClient); + var startedUtc = DateTimeOffset.UtcNow; + Console.WriteLine( + $"typedmemeval: {descriptor.DisplayName} run {runIndex}/{options.Runs} — " + + $"{(options.Oracle ? "oracle (GoldOnly)" : "structured memory arm")}" + + $"{(options.Control ? ", control arm (TimestampsAndText)" : string.Empty)}, " + + $"seed {options.RandomSeed?.ToString(CultureInfo.InvariantCulture) ?? "unseeded"}, " + + $"max questions {options.MaxQuestions?.ToString(CultureInfo.InvariantCulture) ?? "all"}."); + + ExternalBenchmarkResult result; + if (options.Oracle) + { + result = await runner + .RunOracleAsync( + answerChatClient, vertical, facade, LongMemEvalOracleOptions.GoldOnly) + .ConfigureAwait(false); + } + else + { + // Fresh adapter and fresh evidence index per run: Resolve consumes its entries, and + // a distinct runId keeps every run's owners and sessions isolated in the shared + // store. The container itself is reused — isolation is by scope, not by teardown. + var runId = + $"typedmemeval-{descriptor.Slug}{(options.Control ? "-control" : string.Empty)}" + + $"-run{runIndex}-{startedUtc:yyyyMMddTHHmmssZ}"; + var adapter = new AgentMemoryLongMemEvalAdapter( + profile!.Services.GetRequiredService(), + answerChatClient, + runId, + new LongMemEvalAdapterOptions + { + MaxRelevantMessages = DefaultMaxRelevant, + MemoryMode = LongMemEvalMemoryMode.Structured, + MinSimilarityScore = 0, + ModelId = deployment, + EvidenceIndex = LongMemEvalEvidenceIndex.CreateTypedMemEval(vertical, facade), + EvidenceDetail = LongMemEvalEvidenceDetail.Identifiers, + RequireGraphReadBack = true, + GraphProbe = new Neo4jLongMemEvalGraphProbe( + profile.Services.GetRequiredService()), + ExtractionProgress = (completed, total) => Console.WriteLine( + $"typedmemeval: extraction units {completed}/{total}.") + }); + result = await runner.RunAsync(adapter, vertical, facade).ConfigureAwait(false); + } + + var destination = Persist(result, descriptor, options, runIndex, startedUtc); + PrintRun(result, destination); + results.Add(result); + } + + if (results.Count > 1) + PrintBand(TypedMemEvalRunSet.Summarize(results)); + + return results; + } + + private static TypedMemEvalOptions BuildFacade(TypedMemEvalRunOptions options) => new() + { + MaxQuestions = options.MaxQuestions, + RandomSeed = options.RandomSeed, + AnswerSeed = options.AnswerSeed, + // Null takes the vertical's own required grounding — the probe arm. The control arm is the + // family's published pairing: same corpus, same hash, dates re-added to the text, and the + // run labelled so it can never be banded with the probe by accident. + TemporalGrounding = options.Control ? TemporalGroundingMode.TimestampsAndText : null, + ControlArm = options.Control + }; + + /// + /// Writes the runner's own result JSON under artifacts/evaluation/, named by vertical, + /// arm, seed, run index, and UTC stamp — everything needed to pair files into a band later. + /// + private static string Persist( + ExternalBenchmarkResult result, + TypedMemEvalVerticalDescriptor descriptor, + TypedMemEvalRunOptions options, + int runIndex, + DateTimeOffset startedUtc) + { + var name = + $"typedmemeval-{descriptor.Slug}" + + (options.Control ? "-control" : string.Empty) + + (options.Oracle ? "-oracle" : string.Empty) + + $"-seed{options.RandomSeed?.ToString(CultureInfo.InvariantCulture) ?? "unseeded"}" + + $"-run{runIndex}" + + $"-{startedUtc:yyyyMMddTHHmmssZ}.json"; + var destination = Path.GetFullPath(Path.Combine("artifacts", "evaluation", name)); + Directory.CreateDirectory(Path.GetDirectoryName(destination)!); + File.WriteAllText( + destination, + JsonSerializer.Serialize(result, new JsonSerializerOptions { WriteIndented = true }) + + Environment.NewLine); + return destination; + } + + private static void PrintRun(ExternalBenchmarkResult result, string destination) + { + if (result.TypedOutcomes is { } typed) + { + var counts = typed.Outcomes; + Console.WriteLine( + $"typedmemeval: N={counts.N} correct={counts.Correct} wrong={counts.Wrong} " + + $"abstained={counts.Abstained} missed={counts.Missed} premature={counts.Premature} " + + $"inconclusive={counts.Inconclusive} unrun={counts.Unrun}"); + Console.WriteLine( + $"typedmemeval: attribution observed share {typed.Attribution.ObservedShare:P0} " + + $"(present {typed.Attribution.EvidencePresent}, absent {typed.Attribution.EvidenceAbsent}, " + + $"unobserved {typed.Attribution.Unobserved})"); + if (typed.Coverage.MeanOverGoldBearing is { } realised) + { + var floor = typed.Coverage.CalibratedFloorMean; + Console.WriteLine( + $"typedmemeval: realised coverage (gold-bearing mean) {realised:F3}" + + (floor is { } f + ? $" vs calibrated BM25 floor {f:F3}" + : string.Empty)); + } + } + + Console.WriteLine($"typedmemeval: report {destination}"); + } + + private static void PrintBand(TypedMemEvalRunSetSummary band) + { + Console.WriteLine( + $"typedmemeval: band over {band.Runs} runs of {band.Vertical} " + + $"(corpus {band.CorpusSha256})"); + foreach (var pair in band.Outcomes.OrderBy(pair => pair.Key)) + { + Console.WriteLine( + $"typedmemeval: {pair.Key,-12} {pair.Value.Minimum}..{pair.Value.Maximum} " + + $"(mean {pair.Value.Mean.ToString("F1", CultureInfo.InvariantCulture)}, " + + $"width {pair.Value.Width})"); + } + + Console.WriteLine( + $"typedmemeval: questions compared {band.QuestionsCompared}; " + + $"questions with flips {band.QuestionsWithFlips}" + + (band.QuestionsWithFlips > 0 + ? $" [{string.Join(", ", band.FlippedQuestionIds)}]" + : string.Empty)); + if (band.AtMinimumRunCount) + { + Console.WriteLine( + "typedmemeval: two runs is the banding floor (three recommended); a zero-width " + + "band here can be coincidence, so read it as weak evidence."); + } + } + + internal static TypedMemEvalRunOptions Parse(string[] args) + { + LongMemEvalArgumentValidator.Validate(args, KnownOptions); + + string? Value(string name) + { + var index = Array.IndexOf(args, name); + if (index < 0) return null; + if (index + 1 >= args.Length) + throw new ArgumentException($"{name} requires a value."); + return args[index + 1]; + } + + var verticals = ParseVerticals( + Value("--typedmemeval") + ?? throw new ArgumentException( + "--typedmemeval requires a vertical " + + $"({VerticalSlugs()}) or 'all'.")); + var options = new TypedMemEvalRunOptions( + verticals, + ParsePositive(Value("--max-questions"), "--max-questions"), + ParseAnyInteger(Value("--random-seed"), "--random-seed"), + ParseAnyInteger(Value("--answer-seed"), "--answer-seed"), + ParsePositive(Value("--runs"), "--runs") ?? 1, + Array.IndexOf(args, "--oracle") >= 0, + Array.IndexOf(args, "--control") >= 0); + + // Validated at parse time, before any container, client, or provider call exists: a run + // set that cannot be banded, or a control arm with no pair to control, must stop here. + if (options.Runs > 1 && options.RandomSeed is null) + { + throw new ArgumentException( + "--runs above 1 requires --random-seed: unseeded runs draw different questions, and " + + "TypedMemEvalRunSet.Summarize refuses to band different samples."); + } + if (options.Control && + (options.Verticals.Count != 1 || + options.Verticals[0] != TypedMemEvalVertical.Prospective)) + { + throw new ArgumentException( + "--control is the Prospective pair's control arm (dates re-added to the text); " + + "combine it with --typedmemeval prospective."); + } + + return options; + } + + private static IReadOnlyList ParseVerticals(string value) + { + if (string.Equals(value, "all", StringComparison.OrdinalIgnoreCase)) + return TypedMemEvalVerticals.All.Select(descriptor => descriptor.Vertical).ToArray(); + + var descriptor = TypedMemEvalVerticals.All.FirstOrDefault(candidate => + string.Equals(candidate.Slug, value.Trim(), StringComparison.OrdinalIgnoreCase)); + return descriptor is null + ? throw new ArgumentException( + $"Unknown TypedMemEval vertical '{value}'. " + + $"--typedmemeval must name a vertical ({VerticalSlugs()}) or 'all'.") + : [descriptor.Vertical]; + } + + private static string VerticalSlugs() => + string.Join("|", TypedMemEvalVerticals.All.Select(descriptor => descriptor.Slug)); + + private static int? ParsePositive(string? value, string option) + { + if (value is null) return null; + if (!int.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed) || + parsed <= 0) + { + throw new ArgumentException($"{option} must be a positive integer."); + } + + return parsed; + } + + /// + /// A sampling seed is not a count: zero and negative values are legal, and a value the parser + /// cannot read must stop the run rather than fall back to "unseeded" while the operator + /// believes otherwise. + /// + private static int? ParseAnyInteger(string? value, string option) + { + if (value is null) return null; + return int.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed) + ? parsed + : throw new ArgumentException($"{option} must be an integer."); + } + + private static string RequiredEnvironment(string name) => + Environment.GetEnvironmentVariable(name) is { Length: > 0 } value + ? value + : throw new InvalidOperationException( + $"{name} is required; refusing to create a synthetic TypedMemEval score."); + + /// + /// AgentEval's double-count guard, applied at the one place this verb assembles results across + /// verticals: the twelve time-grounded probe questions were carried into + /// TypedMemEval-Prospective, so a result set containing both corpora double-counts them in any + /// total. is the upstream authority on the + /// corpus identifiers; this method only decides where its warning surfaces. + /// + /// True when the warning was printed — the tests' observable. + internal static bool WarnOnSeedOverlap( + IEnumerable results, TextWriter output) + { + if (TypedMemEvalRunSet.DetectSeedOverlap(results) is { } warning) + { + output.WriteLine($"typedmemeval: WARNING — {warning}"); + return true; + } + + return false; + } + + internal sealed record TypedMemEvalRunOptions( + IReadOnlyList Verticals, + int? MaxQuestions, + int? RandomSeed, + int? AnswerSeed, + int Runs, + bool Oracle, + bool Control); +}