Skip to content
Merged
Show file tree
Hide file tree
Changes from 15 commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
e18d11f
Add expanded, un-marked corpus generator + open-extraction reader mode
hardcoreerik Jul 3, 2026
db5c3bc
Generate the 85 host-templated questions for the expanded corpus
hardcoreerik Jul 3, 2026
dcffd05
External authorship for the 65 remaining question categories, plus a …
hardcoreerik Jul 3, 2026
0673278
Split the verified 150-question suite into development/held-out sets
hardcoreerik Jul 3, 2026
01f3fd0
CF benchmark remediation: JSON recovery, baseline completion, admissi…
hardcoreerik Jul 3, 2026
4497278
Add CF-7 gate re-run script and update bench help text
hardcoreerik Jul 3, 2026
8590775
Clear two CS8601 nullable warnings and ignore publish-* variants
hardcoreerik Jul 3, 2026
ab7f364
docs: add TheOrc Warpath gamification whitepaper
hardcoreerik Jul 3, 2026
9f1d830
Foundry F-1: frozen toolcaller-v0 tool inventory, capture schema, val…
hardcoreerik Jul 3, 2026
18f45ea
Merge origin/master (Foundry F-0 docs, PR #33) into cf-benchmark-reme…
hardcoreerik Jul 3, 2026
4e9c2ac
Link F-1 docs and CF-7 re-run script from their doc indexes
hardcoreerik Jul 3, 2026
6dd6f48
Capture real toolcaller-v0 data from live swarm tool-call decisions
hardcoreerik Jul 3, 2026
c55e505
Rewrite B2 top-k RAG baseline: IDF-weighted scoring, budget-fill inst…
hardcoreerik Jul 4, 2026
c68e01c
Fix Context Fabric evidence-pack selection: IDF-weighted scoring, no …
hardcoreerik Jul 4, 2026
3ef5fb0
Fix Exhaustive-answer over-inclusion: entity-scoped vs category-wide …
hardcoreerik Jul 4, 2026
40d79e1
Address Grok adversarial review of the Exhaustive-answer fix
hardcoreerik Jul 4, 2026
2e18805
Make Foundry F-1 dataset capture opt-in with a status bar indicator
hardcoreerik Jul 4, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -52,8 +52,9 @@ OrchestratorIDE.SwarmBenchmarks/
*.docx
*.doc

# Publish output (large binary artifacts — run dotnet publish locally)
publish/
# Publish output (large binary artifacts — run dotnet publish locally).
# Matches publish/ and per-machine variants like publish-4b/, publish-hardcorepc/.
publish*/

# Secrets / local config
appsettings.local.json
Expand Down
1 change: 1 addition & 0 deletions OrchestratorIDE.Avalonia/OrchestratorIDE.Avalonia.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -269,6 +269,7 @@

<!-- Services / Swarm -->
<Compile Include="..\OrchestratorIDE\Services\Swarm\DatasetCapture.cs" />
<Compile Include="..\OrchestratorIDE\Services\Swarm\ToolcallerDatasetCapture.cs" />
<Compile Include="..\OrchestratorIDE\Services\Swarm\EvalRubric.cs" />
<Compile Include="..\OrchestratorIDE\Services\Swarm\FileOwnershipLedger.cs" />
<Compile Include="..\OrchestratorIDE\Services\Swarm\OllamaReviewService.cs" />
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,11 @@
<Compile Include="..\OrchestratorIDE\Core\Runtime\HeadlessAgentLoop.cs" Link="Runtime\HeadlessAgentLoop.cs" />
<Compile Include="..\OrchestratorIDE\Services\ContextFabric\ContextFabricContracts.cs" Link="ContextFabric\ContextFabricContracts.cs" />
<Compile Include="..\OrchestratorIDE\Services\ContextFabric\DeterministicFabricCorpus.cs" Link="ContextFabric\DeterministicFabricCorpus.cs" />
<Compile Include="..\OrchestratorIDE\Services\ContextFabric\DeterministicExpandedFabricCorpus.cs" Link="ContextFabric\DeterministicExpandedFabricCorpus.cs" />
<Compile Include="..\OrchestratorIDE\Services\ContextFabric\ExpandedFabricQuestionGenerator.cs" Link="ContextFabric\ExpandedFabricQuestionGenerator.cs" />
<Compile Include="..\OrchestratorIDE\Services\ContextFabric\ExpandedFabricLedgerExport.cs" Link="ContextFabric\ExpandedFabricLedgerExport.cs" />
<Compile Include="..\OrchestratorIDE\Services\ContextFabric\ExpandedFabricAuthoredQuestionMerger.cs" Link="ContextFabric\ExpandedFabricAuthoredQuestionMerger.cs" />
<Compile Include="..\OrchestratorIDE\Services\ContextFabric\ExpandedFabricQuestionSplitter.cs" Link="ContextFabric\ExpandedFabricQuestionSplitter.cs" />
<Compile Include="..\OrchestratorIDE\Services\ContextFabric\ContextFabricValidation.cs" Link="ContextFabric\ContextFabricValidation.cs" />
<Compile Include="..\OrchestratorIDE\Services\ContextFabric\ContextFabricBenchmarkExpansionRunner.cs" Link="ContextFabric\ContextFabricBenchmarkExpansionRunner.cs" />
<Compile Include="..\OrchestratorIDE\Services\ContextFabric\ContextFabricBenchmarkExpansionWriter.cs" Link="ContextFabric\ContextFabricBenchmarkExpansionWriter.cs" />
Expand Down
134 changes: 134 additions & 0 deletions OrchestratorIDE.UnitTests/ContextFabricB2TopKRagTests.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,134 @@
// Copyright (C) 2025-present hardcoreerik / TheOrc contributors
// SPDX-License-Identifier: AGPL-3.0-or-later
using NUnit.Framework;
using OrchestratorIDE.Services.ContextFabric;

namespace OrchestratorIDE.UnitTests;

/// <summary>
/// Covers the B2 "conventional top-k RAG" baseline rewrite: IDF-weighted, stopword-aware segment
/// scoring, and greedy budget-filling selection instead of a fixed segment count.
/// </summary>
[TestFixture]
public sealed class ContextFabricB2TopKRagTests
{
// FabricContextBudget enforces ContextLimit >= 2048 and EvidenceLimit (ContextLimit minus its
// own 1536/512 reserves) > 0, so tests must keep ContextLimit comfortably above 2048 and
// instead tune AnswerMaxTokens (a separate FabricRunOptions field the runner's own budget
// math actually subtracts) to get a tight or exhausted effective budget.
private static FabricRunOptions Options(int answerMaxTokens) =>
new(new FabricContextBudget(ContextLimit: 2200), AnswerMaxTokens: answerMaxTokens);

[Test]
public void BuildTopKText_PrefersSegmentWithRareDistinctiveTerm_OverCommonWordOverlap()
{
// "checksum" and "CK-991" are rare/distinctive; "the", "was", "and" appear everywhere and
// must not drive the ranking.
var target = new FabricSegment("seg-target", 1, "Target",
"The archive recorded the checksum CK-991 for this shipment.",
FabricHashing.Sha256("target"), 20);
var distractor = new FabricSegment("seg-distractor", 2, "Distractor",
"The archive and the depot were and the records were the same as the other archive.",
FabricHashing.Sha256("distractor"), 20);
var corpus = new FabricCorpus("corpus-1", "doc-1", "gen-1", "digest-1", "1.0",
[target, distractor], 40);
var question = new FabricBenchmarkQuestion(
"q-1", FabricQuestionKind.LocalFact, "What checksum was recorded for the shipment?",
["CK-991"], ["seg-target"]);
var fixture = new FabricBenchmarkFixture(corpus, [question]);

// AnswerMaxTokens tuned so the effective budget (~25 tokens) fits one 20-token segment
// but not both (40 total) -- forces the ranking to actually decide, rather than both
// segments trivially fitting.
var runner = new ContextFabricBaselineRunner(new ScriptedFabricRuntime(), Options(answerMaxTokens: 1652));
var text = runner.BuildTopKText(fixture, question);

Assert.That(text, Does.Contain("CK-991"));
Assert.That(text, Does.Not.Contain(distractor.Text));
}

[Test]
public void BuildTopKText_SelectsMoreThanFourSegments_WhenBudgetAllowsAndAllAreRelevant()
{
// Six short segments, each sharing a distinctive term with the question. The old
// implementation hard-capped at 4 regardless of budget; this must not.
var segments = Enumerable.Range(1, 6)
.Select(i => new FabricSegment($"seg-{i}", i, $"Section {i}",
$"Station Bravo logged a distinct reading labeled MARK-{i:D3} during this cycle.",
FabricHashing.Sha256($"seg-{i}"), 20))
.ToArray();
var corpus = new FabricCorpus("corpus-2", "doc-2", "gen-2", "digest-2", "1.0", segments, 120);
var question = new FabricBenchmarkQuestion(
"q-2", FabricQuestionKind.Exhaustive,
"What readings did Station Bravo log across the cycle?",
["MARK-001"], ["seg-1"]);
var fixture = new FabricBenchmarkFixture(corpus, [question]);

// Generous budget: 8192 limit easily fits all six ~20-token segments plus overhead.
var runner = new ContextFabricBaselineRunner(new ScriptedFabricRuntime(), FabricRunOptions.Default);
var text = runner.BuildTopKText(fixture, question);

var includedCount = segments.Count(s => text.Contains(s.Text, StringComparison.Ordinal));
Assert.That(includedCount, Is.GreaterThan(4));
}

[Test]
public void BuildTopKText_ReturnsEmpty_WhenQuestionHasNoNonStopwordTerms()
{
var segment = new FabricSegment("seg-a", 1, "A", "Some genuinely distinctive content ABC-123.",
FabricHashing.Sha256("a"), 10);
var corpus = new FabricCorpus("corpus-3", "doc-3", "gen-3", "digest-3", "1.0", [segment], 10);
// Every word here is a stopword per the runner's list.
var question = new FabricBenchmarkQuestion(
"q-3", FabricQuestionKind.LocalFact, "What was this and that with the same?",
["ABC-123"], ["seg-a"]);
var fixture = new FabricBenchmarkFixture(corpus, [question]);

var runner = new ContextFabricBaselineRunner(new ScriptedFabricRuntime(), Options(answerMaxTokens: 1500));
var text = runner.BuildTopKText(fixture, question);

Assert.That(text, Is.Empty);
}

[Test]
public void BuildTopKText_ReturnsEmpty_WhenBudgetIsExhausted()
{
var segment = new FabricSegment("seg-a", 1, "A", "A checksum CK-500 was recorded here.",
FabricHashing.Sha256("a"), 20);
var corpus = new FabricCorpus("corpus-4", "doc-4", "gen-4", "digest-4", "1.0", [segment], 20);
var question = new FabricBenchmarkQuestion(
"q-4", FabricQuestionKind.LocalFact, "What checksum was recorded?",
["CK-500"], ["seg-a"]);
var fixture = new FabricBenchmarkFixture(corpus, [question]);

// AnswerMaxTokens tuned so ContextLimit(2200) - AnswerMaxTokens - question - 512 is negative.
var runner = new ContextFabricBaselineRunner(new ScriptedFabricRuntime(), Options(answerMaxTokens: 1700));
var text = runner.BuildTopKText(fixture, question);

Assert.That(text, Is.Empty);
}

[Test]
public void BuildTopKText_SkipsOverBudgetSegment_ButStillFitsShorterLowerRankedOne()
{
// The highest-scoring segment is too large to fit; a shorter, lower-scoring but still
// relevant segment should still be included rather than leaving the budget unused.
var big = new FabricSegment("seg-big", 1, "Big",
"checksum CK-777 checksum CK-777 checksum CK-777 padding padding padding padding padding padding padding",
FabricHashing.Sha256("big"), 500);
var small = new FabricSegment("seg-small", 2, "Small", "checksum CK-777 noted briefly.",
FabricHashing.Sha256("small"), 15);
var corpus = new FabricCorpus("corpus-5", "doc-5", "gen-5", "digest-5", "1.0", [big, small], 515);
var question = new FabricBenchmarkQuestion(
"q-5", FabricQuestionKind.LocalFact, "What checksum was noted?", ["CK-777"], ["seg-small"]);
var fixture = new FabricBenchmarkFixture(corpus, [question]);

// AnswerMaxTokens tuned so the effective budget (~50 tokens) fits "small" (15 tokens) but
// not "big" (500 tokens), even though "big" scores higher (three checksum mentions).
var runner = new ContextFabricBaselineRunner(new ScriptedFabricRuntime(), Options(answerMaxTokens: 1632));
var text = runner.BuildTopKText(fixture, question);

Assert.That(text, Does.Contain("noted briefly"));
Assert.That(text, Does.Not.Contain("padding"));
}
Comment on lines +111 to +133

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🎯 Functional Correctness | 🟠 Major | ⚡ Quick win

Test doesn't actually exercise its stated scenario — Tokenize dedupes per segment, so "big" doesn't outscore "small".

Tokenize returns a HashSet<string> per segment (ContextFabricBaselineRunner.cs line 478-483), so repeated occurrences of "checksum" in big don't increase its score — only presence matters. Computing IDF scores here: checksum has document-frequency 2 (df across both segments), noted has df 1 (only in small). That gives small a higher score (0.5 + 1.0 = 1.5) than big (0.5), not lower as the in-line comment claims ("even though 'big' scores higher"). The test still passes because the budget happens to only fit small regardless of ranking, but it never actually exercises the intended case — a higher-ranked-but-oversized segment being skipped in favor of a genuinely lower-ranked-but-smaller one that still fits.

🧪 Suggested fix: make "big" genuinely outrank "small"
-        var big = new FabricSegment("seg-big", 1, "Big",
-            "checksum CK-777 checksum CK-777 checksum CK-777 padding padding padding padding padding padding padding",
-            FabricHashing.Sha256("big"), 500);
-        var small = new FabricSegment("seg-small", 2, "Small", "checksum CK-777 noted briefly.",
-            FabricHashing.Sha256("small"), 15);
+        // "big" must match strictly more distinct question terms than "small" to actually outrank
+        // it under presence/IDF scoring (Tokenize dedupes per segment, so repeated mentions of the
+        // same term do not increase score).
+        var big = new FabricSegment("seg-big", 1, "Big",
+            "checksum CK-777 was noted and verified with padding padding padding padding padding padding padding",
+            FabricHashing.Sha256("big"), 500);
+        var small = new FabricSegment("seg-small", 2, "Small", "checksum CK-777 noted briefly.",
+            FabricHashing.Sha256("small"), 15);
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
[Test]
public void BuildTopKText_SkipsOverBudgetSegment_ButStillFitsShorterLowerRankedOne()
{
// The highest-scoring segment is too large to fit; a shorter, lower-scoring but still
// relevant segment should still be included rather than leaving the budget unused.
var big = new FabricSegment("seg-big", 1, "Big",
"checksum CK-777 checksum CK-777 checksum CK-777 padding padding padding padding padding padding padding",
FabricHashing.Sha256("big"), 500);
var small = new FabricSegment("seg-small", 2, "Small", "checksum CK-777 noted briefly.",
FabricHashing.Sha256("small"), 15);
var corpus = new FabricCorpus("corpus-5", "doc-5", "gen-5", "digest-5", "1.0", [big, small], 515);
var question = new FabricBenchmarkQuestion(
"q-5", FabricQuestionKind.LocalFact, "What checksum was noted?", ["CK-777"], ["seg-small"]);
var fixture = new FabricBenchmarkFixture(corpus, [question]);
// AnswerMaxTokens tuned so the effective budget (~50 tokens) fits "small" (15 tokens) but
// not "big" (500 tokens), even though "big" scores higher (three checksum mentions).
var runner = new ContextFabricBaselineRunner(new ScriptedFabricRuntime(), Options(answerMaxTokens: 1632));
var text = runner.BuildTopKText(fixture, question);
Assert.That(text, Does.Contain("noted briefly"));
Assert.That(text, Does.Not.Contain("padding"));
}
[Test]
public void BuildTopKText_SkipsOverBudgetSegment_ButStillFitsShorterLowerRankedOne()
{
// The highest-scoring segment is too large to fit; a shorter, lower-scoring but still
// relevant segment should still be included rather than leaving the budget unused.
// "big" must match strictly more distinct question terms than "small" to actually outrank
// it under presence/IDF scoring (Tokenize dedupes per segment, so repeated mentions of the
// same term do not increase score).
var big = new FabricSegment("seg-big", 1, "Big",
"checksum CK-777 was noted and verified with padding padding padding padding padding padding padding",
FabricHashing.Sha256("big"), 500);
var small = new FabricSegment("seg-small", 2, "Small", "checksum CK-777 noted briefly.",
FabricHashing.Sha256("small"), 15);
var corpus = new FabricCorpus("corpus-5", "doc-5", "gen-5", "digest-5", "1.0", [big, small], 515);
var question = new FabricBenchmarkQuestion(
"q-5", FabricQuestionKind.LocalFact, "What checksum was noted?", ["CK-777"], ["seg-small"]);
var fixture = new FabricBenchmarkFixture(corpus, [question]);
// AnswerMaxTokens tuned so the effective budget (~50 tokens) fits "small" (15 tokens) but
// not "big" (500 tokens), even though "big" scores higher (three checksum mentions).
var runner = new ContextFabricBaselineRunner(new ScriptedFabricRuntime(), Options(answerMaxTokens: 1632));
var text = runner.BuildTopKText(fixture, question);
Assert.That(text, Does.Contain("noted briefly"));
Assert.That(text, Does.Not.Contain("padding"));
}
🤖 Prompt for AI Agents
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In `@OrchestratorIDE.UnitTests/ContextFabricB2TopKRagTests.cs` around lines 111 -
133, The test in
BuildTopKText_SkipsOverBudgetSegment_ButStillFitsShorterLowerRankedOne does not
match the intended ranking because Tokenize dedupes terms per segment, so
repeated “checksum” in the big segment does not raise its score. Update the test
data so the segment referenced by big genuinely outranks small under the
BuildTopKText scoring logic in ContextFabricBaselineRunner, while still being
too large for the token budget; keep small as the shorter fallback that fits and
is selected when big is skipped.

}
85 changes: 83 additions & 2 deletions OrchestratorIDE.UnitTests/ContextFabricCf0Tests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,18 @@ public void DeterministicCorpus_RebuildsIdentically_AndExceeds8KContext()
Assert.That(first.Corpus.GenerationId, Is.EqualTo(second.Corpus.GenerationId));
Assert.That(first.Corpus.Segments.Select(segment => segment.SegmentId),
Is.EqualTo(second.Corpus.Segments.Select(segment => segment.SegmentId)));
Assert.That(first.Questions.Select(question => question.Kind),
Is.EquivalentTo(Enum.GetValues<FabricQuestionKind>()));
// The frozen 5-question fixture predates the Paraphrased/GlobalSynthesis kinds added
// for the expanded corpus's question suite (docs' 7-category, 150-question spec) --
// it only ever needs to cover its own 5 hardcoded kinds, not every enum value.
Assert.That(first.Questions.Select(question => question.Kind), Is.EquivalentTo(
new[]
{
FabricQuestionKind.LocalFact,
FabricQuestionKind.MultiHop,
FabricQuestionKind.Contradiction,
FabricQuestionKind.Exhaustive,
FabricQuestionKind.Unanswerable,
}));
});
}

Expand Down Expand Up @@ -161,6 +171,77 @@ public void JsonParser_RepairsUnterminatedObject_WhenThePayloadIsOtherwiseComple
Assert.That(parsed.Summary, Is.EqualTo("ok"));
}

[Test]
public void JsonParser_SanitizesKeywordSuffixArtifact_FalseC()
{
// "falseC" is the classic token-boundary artifact: the literal token "false" immediately
// followed by the first character of the next word token (e.g. "Charles"). The sanitizer
// must strip the suffix without touching "false" inside string values.
var parsed = FabricJson.ParseModelObject<FabricAnswerDraft>(
"{\"schemaVersion\":\"cf0-answer-1.0\",\"answer\":\"ok\",\"abstained\":falseC,\"claims\":[]}");

Assert.That(parsed.Answer, Is.EqualTo("ok"));
Assert.That(parsed.Abstained, Is.False);
}

[Test]
public void JsonParser_SanitizesKeywordSuffixArtifact_TrueX()
{
// "trueX" outside a string should collapse to the keyword "true".
var parsed = FabricJson.ParseModelObject<FabricAnswerDraft>(
"{\"schemaVersion\":\"cf0-answer-1.0\",\"answer\":\"ok\",\"abstained\":trueX,\"claims\":[]}");
Assert.That(parsed.Abstained, Is.True);
}

[Test]
public void JsonParser_SanitizesKeywordSuffixArtifact_NullValue()
{
// "nullValue" outside a string should collapse to "null". Test via a nullable string field
// because null → non-nullable bool is a deserializer type error, not a JSON syntax error.
var parsed = FabricJson.ParseModelObject<FabricAnswerDraft>(
"{\"schemaVersion\":\"cf0-answer-1.0\",\"answer\":nullValue,\"abstained\":false,\"claims\":[]}");
Assert.That(parsed.Answer, Is.Null);
}

[Test]
public void JsonParser_SanitizesKeywordSuffix_DoesNotCorruptStringContents()
{
// "falsehood" and "trueColor" inside JSON string values must be preserved verbatim —
// the sanitizer is only allowed to modify tokens that appear outside string boundaries.
var parsed = FabricJson.ParseModelObject<FabricAnswerDraft>(
"{\"schemaVersion\":\"cf0-answer-1.0\",\"answer\":\"falsehood is trueColor nullValue\",\"abstained\":false,\"claims\":[]}");

Assert.That(parsed.Answer, Is.EqualTo("falsehood is trueColor nullValue"));
Assert.That(parsed.Abstained, Is.False);
}

[Test]
public void JsonParser_HandlesTrailingCommaInObject()
{
// Models sometimes emit a trailing comma after the last key-value pair.
var parsed = FabricJson.ParseModelObject<FabricReductionDraft>(
"{\"schemaVersion\":\"cf0-reduction-1.0\",\"summary\":\"ok\",\"claimIds\":[],\"conflicts\":[],}");

Assert.That(parsed.Summary, Is.EqualTo("ok"));
}

[Test]
public void JsonParser_HandlesTrailingCommaInArray()
{
var parsed = FabricJson.ParseModelObject<FabricReductionDraft>(
"{\"schemaVersion\":\"cf0-reduction-1.0\",\"summary\":\"ok\",\"claimIds\":[\"a\",\"b\",],\"conflicts\":[]}");

Assert.That(parsed.ClaimIds, Is.EqualTo(new[] { "a", "b" }));
}

[Test]
public void JsonParser_TrySanitizeLiteralSuffixes_ReturnsNullWhenNothingChanged()
{
// If the input JSON is already clean, the sanitizer must return null (no copy allocated).
var clean = "{\"schemaVersion\":\"cf0-reduction-1.0\",\"summary\":\"ok\",\"claimIds\":[],\"conflicts\":[]}";
Assert.That(FabricJson.TrySanitizeLiteralSuffixes(clean), Is.Null);
}

[Test]
public void ContextBudget_RejectsImpossibleConfiguration()
{
Expand Down
Loading
Loading