CF benchmark remediation: honest evidence-selection fix + Foundry F-1 start

.gitignore

@@ -52,8 +52,9 @@ OrchestratorIDE.SwarmBenchmarks/
 *.docx
 *.doc
 
-# Publish output (large binary artifacts — run dotnet publish locally)
-publish/
+# Publish output (large binary artifacts — run dotnet publish locally).
+# Matches publish/ and per-machine variants like publish-4b/, publish-hardcorepc/.
+publish*/
 
 # Secrets / local config
 appsettings.local.json

OrchestratorIDE.Avalonia/MainWindow.axaml

@@ -586,6 +586,24 @@
                     </StackPanel>
                 </Border>
 
+                <!-- Dataset gathering indicator — visible only while Foundry F-1 capture is on -->
+                <Border x:Name="BdrDatasetCapture"
+                        DockPanel.Dock="Right"
+                        Background="#2A1E00"
+                        CornerRadius="3" Padding="6,0" Margin="4,2,4,2"
+                        Cursor="Hand" IsVisible="False"
+                        ToolTip.Tip="Foundry F-1 dataset capture is on — real tool-call decisions are being staged locally. Click to open Settings."
+                        PointerPressed="BdrDatasetCapture_Click">
+                    <StackPanel Orientation="Horizontal">
+                        <Ellipse Width="7" Height="7" Fill="#E0A030"
+                                 VerticalAlignment="Center" Margin="0,0,5,0"/>
+                        <TextBlock Text="Dataset Gathering Active"
+                                   Foreground="#E0C080"
+                                   FontSize="13" FontWeight="SemiBold"
+                                   VerticalAlignment="Center"/>
+                    </StackPanel>
+                </Border>
+
                 <!-- Update badge — collapsed until newer release detected -->
                 <Border x:Name="BdrUpdateBadge"
                         DockPanel.Dock="Right"

OrchestratorIDE.Avalonia/MainWindow.axaml.cs

@@ -122,6 +122,9 @@ public MainWindow()
             SetStatus("Recording saved — F12 to record again");
         });
 
+        Services.Swarm.ToolcallerDatasetCapture.IsEnabled = _settings.ToolcallerDatasetCaptureEnabled;
+        BdrDatasetCapture.IsVisible = _settings.ToolcallerDatasetCaptureEnabled;
+
         _approvals = new ApprovalQueue();
         _registry  = new ToolRegistry(_approvals);
         _context   = new ContextManager(32_768);
@@ -1993,6 +1996,11 @@ private void OnSettingsSaved(AppSettings newSettings)
         if (_hiveWorkerAgent is not null)
             _hiveWorkerAgent.AutoResyncEnabled = newSettings.HiveDevAutoResyncEnabled;
 
+        // Dataset capture is opt-in and must never be silent -- flip the static flag and the
+        // status bar pill together, immediately, whenever the setting changes.
+        Services.Swarm.ToolcallerDatasetCapture.IsEnabled = newSettings.ToolcallerDatasetCaptureEnabled;
+        BdrDatasetCapture.IsVisible = newSettings.ToolcallerDatasetCaptureEnabled;
+
         if (newSettings.Backend != oldBackend ||
             (newSettings.Backend == InferenceBackend.LlamaCpp &&
              _llamaServer != null &&
@@ -2348,6 +2356,9 @@ private void ShowUpdateBadge(UpdateChecker.UpdateResult result)
     private void BdrUpdateBadge_Click(object? sender, PointerPressedEventArgs e)
         => OpenSelfUpdateDialog(_settings.LastKnownLatestVersion ?? "");
 
+    private void BdrDatasetCapture_Click(object? sender, PointerPressedEventArgs e)
+        => BtnSettings_Click(this, new RoutedEventArgs());
+
     private void OpenSelfUpdateDialog(string latestVersion)
     {
         SelfUpdateWindow.ShowWindow(this, _settings);

OrchestratorIDE.Avalonia/OrchestratorIDE.Avalonia.csproj

@@ -269,6 +269,7 @@
 
     <!-- Services / Swarm -->
     <Compile Include="..\OrchestratorIDE\Services\Swarm\DatasetCapture.cs" />
+    <Compile Include="..\OrchestratorIDE\Services\Swarm\ToolcallerDatasetCapture.cs" />
     <Compile Include="..\OrchestratorIDE\Services\Swarm\EvalRubric.cs" />
     <Compile Include="..\OrchestratorIDE\Services\Swarm\FileOwnershipLedger.cs" />
     <Compile Include="..\OrchestratorIDE\Services\Swarm\OllamaReviewService.cs" />

OrchestratorIDE.Avalonia/UI/Panels/SettingsPanel.axaml

@@ -548,6 +548,20 @@
                     </DockPanel>
                 </Border>
 
+                <Border Classes="SettingRowBorder">
+                    <DockPanel LastChildFill="True">
+                        <ToggleButton x:Name="TglToolcallerDatasetCapture"
+                                      DockPanel.Dock="Right"
+                                      Theme="{StaticResource ToggleStyle}"
+                                      IsChecked="False"/>
+                        <StackPanel Margin="0,0,8,0">
+                            <TextBlock Text="Foundry F-1 dataset capture" Classes="SettingLabel"/>
+                            <TextBlock Text="Opt-in, off by default. When on, stages real swarm tool-call decisions (tool name, arguments, policy outcome) as theorc-toolcaller-v0 training examples in .orc/swarm/dataset-staging/toolcaller/. This is local-only and gitignored, but captures are NOT sanitized -- they can contain real file paths, shell commands, and file contents. Never share raw captures; review before using them to train anything. Shown as &quot;Dataset Gathering Active&quot; in the status bar while on."
+                                       Classes="SettingHint" TextWrapping="Wrap"/>
+                        </StackPanel>
+                    </DockPanel>
+                </Border>
+
                 <StackPanel Classes="SettingRow">
                     <TextBlock Text="Native Model Root" Classes="SettingLabel"/>
                     <TextBlock Text="Folder scanned for GGUF/LoRA role bindings. Empty uses the normal model storage folder."

OrchestratorIDE.Avalonia/UI/Panels/SettingsPanel.axaml.cs

@@ -133,6 +133,7 @@ public void LoadSettings(AppSettings s)
         TglNativeHiveWorker.IsChecked = s.ExperimentalNativeHiveWorkerEnabled;
         TglAutoResync.IsChecked       = s.HiveDevAutoResyncEnabled;
         TglNativeMainChat.IsChecked   = s.ExperimentalNativeMainChatEnabled;
+        TglToolcallerDatasetCapture.IsChecked = s.ToolcallerDatasetCaptureEnabled;
         TbNativeRuntimeModelRoot.Text = s.NativeRuntimeModelRoot;
         TbNativeRuntimeContextSize.Text = s.NativeRuntimeContextSize.ToString();
         TbNativeRuntimeGpuLayers.Text = s.NativeRuntimeGpuLayers.ToString();
@@ -310,6 +311,7 @@ private AppSettings ReadSettings()
         s.ExperimentalNativeHiveWorkerEnabled = TglNativeHiveWorker.IsChecked == true;
         s.HiveDevAutoResyncEnabled            = TglAutoResync.IsChecked == true;
         s.ExperimentalNativeMainChatEnabled   = TglNativeMainChat.IsChecked == true;
+        s.ToolcallerDatasetCaptureEnabled     = TglToolcallerDatasetCapture.IsChecked == true;
         s.NativeRuntimeModelRoot = TbNativeRuntimeModelRoot.Text?.Trim() ?? "";
         s.NativeRuntimeContextSize = int.TryParse(TbNativeRuntimeContextSize.Text, out var nativeCtx)
             ? Math.Max(512, nativeCtx)

OrchestratorIDE.NativeRuntime/OrchestratorIDE.NativeRuntime.csproj

@@ -35,6 +35,11 @@
     <Compile Include="..\OrchestratorIDE\Core\Runtime\HeadlessAgentLoop.cs" Link="Runtime\HeadlessAgentLoop.cs" />
     <Compile Include="..\OrchestratorIDE\Services\ContextFabric\ContextFabricContracts.cs" Link="ContextFabric\ContextFabricContracts.cs" />
     <Compile Include="..\OrchestratorIDE\Services\ContextFabric\DeterministicFabricCorpus.cs" Link="ContextFabric\DeterministicFabricCorpus.cs" />
+    <Compile Include="..\OrchestratorIDE\Services\ContextFabric\DeterministicExpandedFabricCorpus.cs" Link="ContextFabric\DeterministicExpandedFabricCorpus.cs" />
+    <Compile Include="..\OrchestratorIDE\Services\ContextFabric\ExpandedFabricQuestionGenerator.cs" Link="ContextFabric\ExpandedFabricQuestionGenerator.cs" />
+    <Compile Include="..\OrchestratorIDE\Services\ContextFabric\ExpandedFabricLedgerExport.cs" Link="ContextFabric\ExpandedFabricLedgerExport.cs" />
+    <Compile Include="..\OrchestratorIDE\Services\ContextFabric\ExpandedFabricAuthoredQuestionMerger.cs" Link="ContextFabric\ExpandedFabricAuthoredQuestionMerger.cs" />
+    <Compile Include="..\OrchestratorIDE\Services\ContextFabric\ExpandedFabricQuestionSplitter.cs" Link="ContextFabric\ExpandedFabricQuestionSplitter.cs" />
     <Compile Include="..\OrchestratorIDE\Services\ContextFabric\ContextFabricValidation.cs" Link="ContextFabric\ContextFabricValidation.cs" />
     <Compile Include="..\OrchestratorIDE\Services\ContextFabric\ContextFabricBenchmarkExpansionRunner.cs" Link="ContextFabric\ContextFabricBenchmarkExpansionRunner.cs" />
     <Compile Include="..\OrchestratorIDE\Services\ContextFabric\ContextFabricBenchmarkExpansionWriter.cs" Link="ContextFabric\ContextFabricBenchmarkExpansionWriter.cs" />

OrchestratorIDE.UnitTests/ContextFabricB2TopKRagTests.cs

@@ -0,0 +1,134 @@
+// Copyright (C) 2025-present hardcoreerik / TheOrc contributors
+// SPDX-License-Identifier: AGPL-3.0-or-later
+using NUnit.Framework;
+using OrchestratorIDE.Services.ContextFabric;
+
+namespace OrchestratorIDE.UnitTests;
+
+/// <summary>
+/// Covers the B2 "conventional top-k RAG" baseline rewrite: IDF-weighted, stopword-aware segment
+/// scoring, and greedy budget-filling selection instead of a fixed segment count.
+/// </summary>
+[TestFixture]
+public sealed class ContextFabricB2TopKRagTests
+{
+    // FabricContextBudget enforces ContextLimit >= 2048 and EvidenceLimit (ContextLimit minus its
+    // own 1536/512 reserves) > 0, so tests must keep ContextLimit comfortably above 2048 and
+    // instead tune AnswerMaxTokens (a separate FabricRunOptions field the runner's own budget
+    // math actually subtracts) to get a tight or exhausted effective budget.
+    private static FabricRunOptions Options(int answerMaxTokens) =>
+        new(new FabricContextBudget(ContextLimit: 2200), AnswerMaxTokens: answerMaxTokens);
+
+    [Test]
+    public void BuildTopKText_PrefersSegmentWithRareDistinctiveTerm_OverCommonWordOverlap()
+    {
+        // "checksum" and "CK-991" are rare/distinctive; "the", "was", "and" appear everywhere and
+        // must not drive the ranking.
+        var target = new FabricSegment("seg-target", 1, "Target",
+            "The archive recorded the checksum CK-991 for this shipment.",
+            FabricHashing.Sha256("target"), 20);
+        var distractor = new FabricSegment("seg-distractor", 2, "Distractor",
+            "The archive and the depot were and the records were the same as the other archive.",
+            FabricHashing.Sha256("distractor"), 20);
+        var corpus = new FabricCorpus("corpus-1", "doc-1", "gen-1", "digest-1", "1.0",
+            [target, distractor], 40);
+        var question = new FabricBenchmarkQuestion(
+            "q-1", FabricQuestionKind.LocalFact, "What checksum was recorded for the shipment?",
+            ["CK-991"], ["seg-target"]);
+        var fixture = new FabricBenchmarkFixture(corpus, [question]);
+
+        // AnswerMaxTokens tuned so the effective budget (~25 tokens) fits one 20-token segment
+        // but not both (40 total) -- forces the ranking to actually decide, rather than both
+        // segments trivially fitting.
+        var runner = new ContextFabricBaselineRunner(new ScriptedFabricRuntime(), Options(answerMaxTokens: 1652));
+        var text = runner.BuildTopKText(fixture, question);
+
+        Assert.That(text, Does.Contain("CK-991"));
+        Assert.That(text, Does.Not.Contain(distractor.Text));
+    }
+
+    [Test]
+    public void BuildTopKText_SelectsMoreThanFourSegments_WhenBudgetAllowsAndAllAreRelevant()
+    {
+        // Six short segments, each sharing a distinctive term with the question. The old
+        // implementation hard-capped at 4 regardless of budget; this must not.
+        var segments = Enumerable.Range(1, 6)
+            .Select(i => new FabricSegment($"seg-{i}", i, $"Section {i}",
+                $"Station Bravo logged a distinct reading labeled MARK-{i:D3} during this cycle.",
+                FabricHashing.Sha256($"seg-{i}"), 20))
+            .ToArray();
+        var corpus = new FabricCorpus("corpus-2", "doc-2", "gen-2", "digest-2", "1.0", segments, 120);
+        var question = new FabricBenchmarkQuestion(
+            "q-2", FabricQuestionKind.Exhaustive,
+            "What readings did Station Bravo log across the cycle?",
+            ["MARK-001"], ["seg-1"]);
+        var fixture = new FabricBenchmarkFixture(corpus, [question]);
+
+        // Generous budget: 8192 limit easily fits all six ~20-token segments plus overhead.
+        var runner = new ContextFabricBaselineRunner(new ScriptedFabricRuntime(), FabricRunOptions.Default);
+        var text = runner.BuildTopKText(fixture, question);
+
+        var includedCount = segments.Count(s => text.Contains(s.Text, StringComparison.Ordinal));
+        Assert.That(includedCount, Is.GreaterThan(4));
+    }
+
+    [Test]
+    public void BuildTopKText_ReturnsEmpty_WhenQuestionHasNoNonStopwordTerms()
+    {
+        var segment = new FabricSegment("seg-a", 1, "A", "Some genuinely distinctive content ABC-123.",
+            FabricHashing.Sha256("a"), 10);
+        var corpus = new FabricCorpus("corpus-3", "doc-3", "gen-3", "digest-3", "1.0", [segment], 10);
+        // Every word here is a stopword per the runner's list.
+        var question = new FabricBenchmarkQuestion(
+            "q-3", FabricQuestionKind.LocalFact, "What was this and that with the same?",
+            ["ABC-123"], ["seg-a"]);
+        var fixture = new FabricBenchmarkFixture(corpus, [question]);
+
+        var runner = new ContextFabricBaselineRunner(new ScriptedFabricRuntime(), Options(answerMaxTokens: 1500));
+        var text = runner.BuildTopKText(fixture, question);
+
+        Assert.That(text, Is.Empty);
+    }
+
+    [Test]
+    public void BuildTopKText_ReturnsEmpty_WhenBudgetIsExhausted()
+    {
+        var segment = new FabricSegment("seg-a", 1, "A", "A checksum CK-500 was recorded here.",
+            FabricHashing.Sha256("a"), 20);
+        var corpus = new FabricCorpus("corpus-4", "doc-4", "gen-4", "digest-4", "1.0", [segment], 20);
+        var question = new FabricBenchmarkQuestion(
+            "q-4", FabricQuestionKind.LocalFact, "What checksum was recorded?",
+            ["CK-500"], ["seg-a"]);
+        var fixture = new FabricBenchmarkFixture(corpus, [question]);
+
+        // AnswerMaxTokens tuned so ContextLimit(2200) - AnswerMaxTokens - question - 512 is negative.
+        var runner = new ContextFabricBaselineRunner(new ScriptedFabricRuntime(), Options(answerMaxTokens: 1700));
+        var text = runner.BuildTopKText(fixture, question);
+
+        Assert.That(text, Is.Empty);
+    }
+
+    [Test]
+    public void BuildTopKText_SkipsOverBudgetSegment_ButStillFitsShorterLowerRankedOne()
+    {
+        // The highest-scoring segment is too large to fit; a shorter, lower-scoring but still
+        // relevant segment should still be included rather than leaving the budget unused.
+        var big = new FabricSegment("seg-big", 1, "Big",
+            "checksum CK-777 checksum CK-777 checksum CK-777 padding padding padding padding padding padding padding",
+            FabricHashing.Sha256("big"), 500);
+        var small = new FabricSegment("seg-small", 2, "Small", "checksum CK-777 noted briefly.",
+            FabricHashing.Sha256("small"), 15);
+        var corpus = new FabricCorpus("corpus-5", "doc-5", "gen-5", "digest-5", "1.0", [big, small], 515);
+        var question = new FabricBenchmarkQuestion(
+            "q-5", FabricQuestionKind.LocalFact, "What checksum was noted?", ["CK-777"], ["seg-small"]);
+        var fixture = new FabricBenchmarkFixture(corpus, [question]);
+
+        // AnswerMaxTokens tuned so the effective budget (~50 tokens) fits "small" (15 tokens) but
+        // not "big" (500 tokens), even though "big" scores higher (three checksum mentions).
+        var runner = new ContextFabricBaselineRunner(new ScriptedFabricRuntime(), Options(answerMaxTokens: 1632));
+        var text = runner.BuildTopKText(fixture, question);
+
+        Assert.That(text, Does.Contain("noted briefly"));
+        Assert.That(text, Does.Not.Contain("padding"));
+    }
+}