feat(ab-testing): improve performance and info

src/Features/PerformanceOverlay.cpp

@@ -1195,6 +1195,10 @@ void PerformanceOverlay::DrawABTestSection(const std::vector<DrawCallRow>& allRo
 			this->settingsDiff.clear();
 			this->settingsDiffLoaded = false;
 			showSettingsDiff = false;
+			// Also clear cached snapshots so diff does not linger after user opts to clear
+			if (abTestingManager) {
+				abTestingManager->ClearCachedSnapshots();
+			}
 			ImGui::EndGroup();
 			ImGui::Separator();
 			return;
@@ -1203,9 +1207,15 @@ void PerformanceOverlay::DrawABTestSection(const std::vector<DrawCallRow>& allRo
 		// --- Settings diff section (inline, toggled) ---
 		if (showSettingsDiff) {
 			if (!this->settingsDiffLoaded) {
-				std::filesystem::path userPath = Util::PathHelpers::GetDataPath() / "SKSE/Plugins/CommunityShaders/SettingsUser.json";
-				std::filesystem::path testPath = Util::PathHelpers::GetDataPath() / "SKSE/Plugins/CommunityShaders/SettingsTest.json";
-				this->settingsDiff = Util::FileSystem::LoadJsonDiff(userPath, testPath);
+				// Use cached memory data from ABTestingManager instead of loading from disk
+				if (abTestingManager && abTestingManager->HasCachedSnapshots()) {
+					// Pull structured diff entries directly from ABTestingManager (in-memory snapshots)
+					this->settingsDiff = abTestingManager->GetConfigDiffEntries();
+				} else {
+					std::filesystem::path userPath = Util::PathHelpers::GetDataPath() / "SKSE/Plugins/CommunityShaders/SettingsUser.json";
+					std::filesystem::path testPath = Util::PathHelpers::GetDataPath() / "SKSE/Plugins/CommunityShaders/SettingsTest.json";
+					this->settingsDiff = Util::FileSystem::LoadJsonDiff(userPath, testPath);
+				}
 				this->settingsDiffLoaded = true;
 			}
 			ImGui::TextUnformatted("Differences between USER (A) and TEST (B) configs:");

src/Features/PerformanceOverlay/ABTesting/ABTesting.cpp

@@ -2,8 +2,11 @@
 #include "Features/PerformanceOverlay.h"
 #include "Menu.h"
 #include "State.h"
+#include "Utils/FileSystem.h"
 #include "Utils/UI.h"
+#include <cmath>
 #include <fmt/format.h>
+#include <fstream>
 #include <imgui.h>
 
 ABTestingManager* ABTestingManager::GetSingleton()
@@ -23,8 +26,26 @@ void ABTestingManager::Enable()
 		auto* state = globals::state;
 		auto& performanceOverlay = globals::features::performanceOverlay;
 
-		logger::info("Saving current settings for Variant B (TEST) and starting test with interval {}.", testInterval);
-		state->Save(State::ConfigMode::TEST);
+		logger::info("Starting A/B testing with current settings as Variant B (TEST), interval {} seconds.", testInterval);
+
+		// Preserve overlay enabled state before config operations
+		bool overlayWasEnabled = performanceOverlay.settings.ShowInOverlay;
+
+		// Save current settings as TEST variant (Variant B) in memory
+		testConfigSnapshot = nlohmann::json::object();
+		state->SaveToJson(testConfigSnapshot);
+		hasTestSnapshot = true;
+
+		// Load and cache USER settings in memory (to avoid disk I/O during swaps)
+		userConfigSnapshot = nlohmann::json::object();
+		state->Load(State::ConfigMode::USER);
+		state->SaveToJson(userConfigSnapshot);
+		hasUserSnapshot = true;
+
+		// Load TEST variant to start with (user's configured test settings)
+		state->LoadFromJson(testConfigSnapshot);
+		usingTestConfig = true;
+
 		abTestingEnabled = true;
 
 		// Initialize QueryPerformanceCounter timing
@@ -33,9 +54,10 @@ void ABTestingManager::Enable()
 		}
 		QueryPerformanceCounter(&lastTestSwitch);
 
-		// Preserve overlay enabled state
-		bool overlayWasEnabled = performanceOverlay.settings.ShowInOverlay;
+		// Restore overlay enabled state after config operations
 		performanceOverlay.settings.ShowInOverlay = overlayWasEnabled;
+
+		logger::info("A/B Testing enabled - starting with Variant B (TEST). Both variants cached in memory for unbiased swapping.");
 	}
 }
 
@@ -45,12 +67,20 @@ void ABTestingManager::Disable()
 		auto* state = globals::state;
 		auto& performanceOverlay = globals::features::performanceOverlay;
 
-		logger::info("Disabling A/B testing. Will restore to Variant B (TEST) config.");
-		state->Load(State::ConfigMode::TEST);  // restore last settings before entering test mode
-		abTestingEnabled = false;
-
 		// Preserve overlay enabled state
 		bool overlayWasEnabled = performanceOverlay.settings.ShowInOverlay;
+
+		logger::info("Disabling A/B testing. Restoring to Variant B (TEST) config from memory.");
+
+		// Restore TEST config from memory snapshot (no disk read)
+		if (hasTestSnapshot) {
+			state->LoadFromJson(testConfigSnapshot);
+		} else {
+			logger::warn("No TEST snapshot available, staying with current config.");
+		}
+
+		abTestingEnabled = false;
+
 		performanceOverlay.settings.ShowInOverlay = overlayWasEnabled;
 	}
 }
@@ -63,20 +93,39 @@ void ABTestingManager::Update()
 	auto* state = globals::state;
 	auto& performanceOverlay = globals::features::performanceOverlay;
 
-	// Preserve overlay enabled state when switching configs
+	// Check if it's time to swap
 	LARGE_INTEGER currentTime;
 	QueryPerformanceCounter(&currentTime);
 	float seconds = (currentTime.QuadPart - lastTestSwitch.QuadPart) / static_cast<float>(timingFrequency.QuadPart);
 	auto remaining = static_cast<float>(testInterval) - seconds;
 
 	if (remaining < 0.0f) {
 		bool overlayWasEnabled = performanceOverlay.settings.ShowInOverlay;
+
+		// Swap between variants
 		usingTestConfig = !usingTestConfig;
-		logger::info("Swapping to {} (A/B Test): {}",
-			usingTestConfig ? "Variant B (TEST)" : "Variant A (USER)",
-			usingTestConfig ? "TEST config" : "USER config");
-		state->Load(usingTestConfig ? State::ConfigMode::TEST : State::ConfigMode::USER);
-		performanceOverlay.settings.ShowInOverlay = overlayWasEnabled;  // Restore overlay state
+		logger::info("A/B Test swap to {} (from memory snapshot)",
+			usingTestConfig ? "Variant B (TEST)" : "Variant A (USER)");
+
+		if (usingTestConfig) {
+			// Swap to TEST - load from memory snapshot (no disk I/O)
+			if (hasTestSnapshot) {
+				state->LoadFromJson(testConfigSnapshot);
+			} else {
+				logger::error("TEST snapshot missing! Cannot swap to Variant B.");
+				usingTestConfig = false;  // Stay on USER
+			}
+		} else {
+			// Swap to USER - load from memory snapshot (no disk I/O)
+			if (hasUserSnapshot) {
+				state->LoadFromJson(userConfigSnapshot);
+			} else {
+				logger::error("USER snapshot missing! Cannot swap to Variant A.");
+				usingTestConfig = true;  // Stay on TEST
+			}
+		}
+
+		performanceOverlay.settings.ShowInOverlay = overlayWasEnabled;
 		QueryPerformanceCounter(&lastTestSwitch);
 
 		// Notify the A/B test aggregator of the variant switch
@@ -102,14 +151,80 @@ void ABTestingManager::DrawSettingsUI()
 
 	if (auto _tt = Util::HoverTooltipWrapper()) {
 		ImGui::Text(
-			"Sets number of seconds before toggling between Variant A (USER) and Variant B (TEST) config for A/B testing. "
-			"0 disables. Non-zero will enable A/B testing mode. "
-			"Enabling will save current settings as TEST config (Variant B). "
-			"This has no impact if no settings are changed. "
-			"Variant A = USER config, Variant B = TEST config.");
+			"A/B Testing compares two configurations by automatically swapping between them.\n"
+			"Workflow: Configure your test settings, then enable A/B testing.\n"
+			"- Variant B (TEST) = Your current settings when you enable testing\n"
+			"- Variant A (USER) = Your previously saved user configuration\n"
+			"Testing starts with Variant B, then swaps every N seconds.\n"
+			"Set to 0 to disable and restore TEST settings.");
 	}
 }
 
+std::vector<std::string> ABTestingManager::GetConfigDifferencesForDisplay() const
+{
+	std::vector<std::string> differences;
+
+	if (!hasTestSnapshot || !hasUserSnapshot)
+		return differences;
+
+	auto diffEntries = Util::FileSystem::DiffJson(userConfigSnapshot, testConfigSnapshot, 0.0001f);
+
+	// Format diff entries for display
+	for (const auto& entry : diffEntries) {
+		std::string path = entry.path;
+		std::string aVal = entry.aValue;
+		std::string bVal = entry.bValue;
+
+		// Clean up JSON path (remove leading slash and simplify)
+		if (!path.empty() && path[0] == '/') {
+			path = path.substr(1);
+			// Replace slashes with dots for readability
+			std::replace(path.begin(), path.end(), '/', '.');
+		}
+
+		// Skip version changes (not relevant to user)
+		if (path == "Version")
+			continue;
+
+		// Truncate long values
+		if (aVal.length() > 30)
+			aVal = aVal.substr(0, 27) + "...";
+		if (bVal.length() > 30)
+			bVal = bVal.substr(0, 27) + "...";
+
+		// Format: "path: oldValue -> newValue"
+		differences.push_back(fmt::format("{}: {} -> {}", path, aVal, bVal));
+	}
+
+	return differences;
+}
+
+std::vector<SettingsDiffEntry> ABTestingManager::GetConfigDiffEntries(float epsilon) const
+{
+	if (!hasTestSnapshot || !hasUserSnapshot)
+		return {};
+
+	return Util::FileSystem::DiffJson(userConfigSnapshot, testConfigSnapshot, epsilon);
+}
+
+void ABTestingManager::ClearCachedSnapshots()
+{
+	try {
+		testConfigSnapshot.clear();
+		userConfigSnapshot.clear();
+		hasTestSnapshot = false;
+		hasUserSnapshot = false;
+	} catch (...) {
+		// No-op if clear fails
+	}
+}
+
+std::vector<std::string> ABTestingManager::GetConfigDifferences() const
+{
+	// Keep this for DrawOverlayUI to avoid circular dependencies
+	return GetConfigDifferencesForDisplay();
+}
+
 void ABTestingManager::DrawOverlayUI()
 {
 	if (!abTestingEnabled)
@@ -120,16 +235,40 @@ void ABTestingManager::DrawOverlayUI()
 	float seconds = (currentTime.QuadPart - lastTestSwitch.QuadPart) / static_cast<float>(timingFrequency.QuadPart);
 	auto remaining = static_cast<float>(testInterval) - seconds;
 
-	ImGui::SetNextWindowBgAlpha(1.0f);
+	// Match CS menu background alpha (0.85f from FullPalette[ImGuiCol_ChildBg])
+	ImGui::SetNextWindowBgAlpha(0.85f);
 	ImGui::SetNextWindowPos(ImVec2(10, 10));
-	if (!ImGui::Begin("Testing", nullptr, ImGuiWindowFlags_NoTitleBar | ImGuiWindowFlags_AlwaysAutoResize | ImGuiWindowFlags_NoResize | ImGuiWindowFlags_NoMove | ImGuiWindowFlags_NoSavedSettings)) {
+	if (!ImGui::Begin("A/B Testing", nullptr, ImGuiWindowFlags_NoTitleBar | ImGuiWindowFlags_AlwaysAutoResize | ImGuiWindowFlags_NoResize | ImGuiWindowFlags_NoMove | ImGuiWindowFlags_NoSavedSettings)) {
 		ImGui::End();
 		return;
 	}
 
 	remaining = std::max(0.0f, remaining);
-	ImGui::Text(fmt::format("{} : {:.1f} seconds left",
+
+	// Show current variant and time
+	ImGui::Text(fmt::format("{} : {:.1f}s left",
 		usingTestConfig ? "Variant B (TEST)" : "Variant A (USER)", remaining)
 			.c_str());
+
+	// Show what changed (for both variants)
+	if (hasTestSnapshot) {
+		auto differences = GetConfigDifferences();
+
+		if (!differences.empty()) {
+			ImGui::Separator();
+
+			constexpr size_t MAX_CHANGES_DISPLAYED = 10;  // Show max 10 individual changes, otherwise show count
+			if (differences.size() <= MAX_CHANGES_DISPLAYED) {
+				ImGui::TextColored(ImVec4(0.7f, 0.9f, 1.0f, 1.0f), "Changes from USER:");
+				for (const auto& diff : differences) {
+					ImGui::BulletText("%s", diff.c_str());
+				}
+			} else {
+				ImGui::TextColored(ImVec4(0.7f, 0.9f, 1.0f, 1.0f),
+					"%zu settings changed", differences.size());
+			}
+		}
+	}
+
 	ImGui::End();
 }

src/Features/PerformanceOverlay/ABTesting/ABTesting.h

@@ -1,5 +1,6 @@
 #pragma once
 #include "ABTestAggregator.h"
+#include "Utils/FileSystem.h"
 #include <nlohmann/json.hpp>
 #include <vector>
 
@@ -27,11 +28,30 @@ class ABTestingManager
 	// Access to aggregator
 	ABTestAggregator& GetAggregator() { return aggregator; }
 
+	// Access to cached settings for performance overlay (in-memory, no disk I/O)
+	bool HasCachedSnapshots() const { return hasTestSnapshot && hasUserSnapshot; }
+	const nlohmann::json& GetUserSnapshot() const { return userConfigSnapshot; }
+	const nlohmann::json& GetTestSnapshot() const { return testConfigSnapshot; }
+	std::vector<std::string> GetConfigDifferencesForDisplay() const;
+	std::vector<SettingsDiffEntry> GetConfigDiffEntries(float epsilon = 0.0001f) const;
+
+	// Clear cached snapshots explicitly (used when overlay results are cleared)
+	void ClearCachedSnapshots();
+
 private:
 	uint32_t testInterval = 0;
 	bool abTestingEnabled = false;
 	bool usingTestConfig = false;
 	LARGE_INTEGER timingFrequency = { 0 };
 	LARGE_INTEGER lastTestSwitch = { 0 };
 	ABTestAggregator aggregator;
+
+	// In-memory storage for both variants to avoid disk I/O during swapping
+	nlohmann::json testConfigSnapshot;
+	nlohmann::json userConfigSnapshot;
+	bool hasTestSnapshot = false;
+	bool hasUserSnapshot = false;
+
+	// Track what changed between USER and TEST configs
+	std::vector<std::string> GetConfigDifferences() const;
 };

src/Menu/AdvancedSettingsRenderer.cpp

@@ -59,7 +59,6 @@ void AdvancedSettingsRenderer::RenderAdvancedSettings(
 			ImGui::EndChild();
 			ImGui::EndTabItem();
 		}
-
 		// Shader Debug Tab
 		if (MenuFonts::BeginTabItemWithFont("Shader Debug", Menu::FontRole::Subheading)) {
 			if (ImGui::BeginChild("##ShaderDebugContent", ImVec2(0, 0), false)) {
@@ -69,6 +68,15 @@ void AdvancedSettingsRenderer::RenderAdvancedSettings(
 			ImGui::EndTabItem();
 		}
 
+		// Testing Tab (for A/B Testing and related settings)
+		if (MenuFonts::BeginTabItemWithFont("Testing", Menu::FontRole::Subheading)) {
+			if (ImGui::BeginChild("##Testing", ImVec2(0, 0), false)) {
+				RenderTestingSection();
+			}
+			ImGui::EndChild();
+			ImGui::EndTabItem();
+		}
+
 		ImGui::EndTabBar();
 	}
 }
@@ -129,10 +137,6 @@ void AdvancedSettingsRenderer::RenderLoggingSection()
 			"The more threads the faster compilation will finish but may make the system unresponsive. ");
 	}
 
-	// A/B Testing settings
-	auto* abTestingManager = ABTestingManager::GetSingleton();
-	abTestingManager->DrawSettingsUI();
-
 	// Dump Ini Settings button
 	if (ImGui::Button("Dump Ini Settings", { -1, 0 })) {
 		Util::DumpSettingsOptions();
@@ -565,3 +569,10 @@ void AdvancedSettingsRenderer::RenderDeveloperSection()
 		}
 	}
 }
+
+void AdvancedSettingsRenderer::RenderTestingSection()
+{
+	// A/B Testing settings
+	auto* abTestingManager = ABTestingManager::GetSingleton();
+	abTestingManager->DrawSettingsUI();
+}

src/Menu/AdvancedSettingsRenderer.h

@@ -19,4 +19,5 @@ class AdvancedSettingsRenderer
 	static void RenderPBRSection(const std::function<void()>& drawTruePBRSettings);
 	static void RenderDisableAtBootSection(const std::function<void()>& drawDisableAtBootSettings);
 	static void RenderDeveloperSection();
+	static void RenderTestingSection();
 };