feat: add renderdoc capture

package/SKSE/Plugins/Renderdoc/LICENSE.md

@@ -0,0 +1,25 @@
+# The MIT License (MIT)
+
+Copyright (c) 2015-2025 Baldur Karlsson
+
+Copyright (c) 2014 Crytek
+
+Copyright (c) 1998-2018 [Third party code and tools](docs/credits_acknowledgements.rst)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

src/FrameAnnotations.cpp

@@ -11,7 +11,7 @@ namespace FrameAnnotations
 	{
 		static void thunk(RE::BSShader* shader, RE::BSRenderPass* pass, uint32_t renderFlags)
 		{
-			if (globals::state->frameAnnotations) {
+			if (globals::state->frameDebugging) {
 				const std::string passName = std::format("[{}:{:X}] <{}> {}", magic_enum::enum_name(ShaderType), pass->passEnum,
 					pass->accumulationHint, pass->geometry->name.c_str());
 				globals::state->BeginPerfEvent(passName);
@@ -30,7 +30,7 @@ namespace FrameAnnotations
 		{
 			func(shader, pass, renderFlags);
 
-			if (globals::state->frameAnnotations) {
+			if (globals::state->frameDebugging) {
 				globals::state->EndPerfEvent();
 			}
 		}
@@ -72,7 +72,7 @@ namespace FrameAnnotations
 	{
 		static void thunk(RE::BSGraphics::BSShaderAccumulator* shaderAccumulator, uint32_t renderFlags)
 		{
-			const bool frameAnnotations = globals::state->frameAnnotations;
+			const bool frameAnnotations = globals::state->frameDebugging;
 			if (frameAnnotations) {
 				globals::state->BeginPerfEvent(std::format("BSShaderAccumulator::FinishAccumulatingDispatch [{}] <{}>",
 					static_cast<uint32_t>(shaderAccumulator->GetRuntimeData().renderMode), renderFlags));
@@ -150,7 +150,7 @@ namespace FrameAnnotations
 			void* passIndexList,
 			uint32_t renderFlags)
 		{
-			const bool frameAnnotations = globals::state->frameAnnotations;
+			const bool frameAnnotations = globals::state->frameDebugging;
 			if (frameAnnotations) {
 				globals::state->BeginPerfEvent(std::format("BSBatchRenderer::RenderBatches ({:X})[{}] <{}>", *currentPass, *bucketIndex,
 					renderFlags));
@@ -262,7 +262,7 @@ namespace FrameAnnotations
 	{
 		static void thunk(void* shaderAccumulator, uint32_t firstPass, uint32_t lastPass, uint32_t renderFlags, int groupIndex)
 		{
-			const bool frameAnnotations = globals::state->frameAnnotations;
+			const bool frameAnnotations = globals::state->frameDebugging;
 			if (frameAnnotations) {
 				globals::state->BeginPerfEvent(std::format("BSShaderAccumulator::RenderBatches ({:X}:{:X})[{}] <{}>", firstPass, lastPass, groupIndex,
 					renderFlags));
@@ -281,7 +281,7 @@ namespace FrameAnnotations
 	{
 		static void thunk(void* passList, uint32_t renderFlags)
 		{
-			const bool frameAnnotations = globals::state->frameAnnotations;
+			const bool frameAnnotations = globals::state->frameDebugging;
 			if (frameAnnotations) {
 				globals::state->BeginPerfEvent(std::format("BSShaderAccumulator::RenderPersistentPassList <{}>", renderFlags));
 			}
@@ -310,7 +310,7 @@ namespace FrameAnnotations
 
 	void OnPostPostLoad()
 	{
-		if (!globals::state->frameAnnotations)
+		if (!globals::state->frameDebugging)
 			return;
 
 		stl::write_vfunc<0x6, BSShader_SetupGeometry<RE::BSShader::Type::Lighting>>(
@@ -924,7 +924,7 @@ namespace FrameAnnotations
 
 	void OnDataLoaded()
 	{
-		if (!globals::state->frameAnnotations)
+		if (!globals::state->frameDebugging)
 			return;
 
 		auto renderer = globals::game::renderer;

src/Menu.cpp

@@ -14,6 +14,7 @@
 #include "Feature.h"
 #include "FeatureIssues.h"
 #include "FeatureVersions.h"
+#include "RenderDoc.h"
 #include "ShaderCache.h"
 #include "State.h"
 #include "Streamline.h"
@@ -1479,6 +1480,7 @@ void Menu::DrawAdvancedSettings()
 		if (ImGui::Button("Dump Ini Settings", { -1, 0 })) {
 			Util::DumpSettingsOptions();
 		}
+
 		if (!shaderCache->blockedKey.empty()) {
 			auto blockingButtonString = std::format("Stop Blocking {} Shaders", shaderCache->blockedIDs.size());
 			if (ImGui::Button(blockingButtonString.c_str(), { -1, 0 })) {
@@ -1505,7 +1507,54 @@ void Menu::DrawAdvancedSettings()
 			ImGui::Text(std::format("Shader Compiler : {}", shaderCache->GetShaderStatsString()).c_str());
 			ImGui::TreePop();
 		}
-		ImGui::Checkbox("Frame Annotations", &globals::state->frameAnnotations);
+		ImGui::Checkbox("Frame Debugging", &globals::state->frameDebugging);
+		if (auto _tt = Util::HoverTooltipWrapper()) {
+			ImGui::Text("Enables frame debugging for providing reports to the Community Shaders team.");
+		}
+
+		auto renderDoc = RenderDoc::GetSingleton();
+
+		bool frameDebuggingActive = globals::state->frameDebugging;
+		bool renderDocActive = renderDoc->IsAvailable();
+
+		static uint32_t clearedCaptures = 0;
+
+		if (frameDebuggingActive && !renderDocActive) {
+			ImGui::TextColored(settings.Theme.StatusPalette.RestartNeeded, "Requires restart to enable frame debugging.");
+		} else if (!frameDebuggingActive && renderDocActive) {
+			ImGui::TextColored(settings.Theme.StatusPalette.Warning, "Requires restart to disable frame debugging, performance will be severely impacted.");
+		} else if (frameDebuggingActive && renderDocActive) {
+			ImGui::TextColored(settings.Theme.StatusPalette.InfoColor, "Frame debugging is active.");
+
+			ImGui::SameLine();
+
+			if (ImGui::Button("Create Capture")) {
+				renderDoc->TriggerCapture();
+			}
+			if (auto _tt = Util::HoverTooltipWrapper()) {
+				ImGui::Text("Start a RenderDoc frame capture for debugging. This will capture the next frame and save it to a file next to SkyrimSE.exe.");
+			}
+
+			auto numCaptures = renderDoc->GetNumCaptures();
+			if (numCaptures > clearedCaptures) {
+				auto capturePath = renderDoc->GetCapturePath(numCaptures - 1);
+				ImGui::TextColored(settings.Theme.StatusPalette.SuccessColor, std::format("Saved to {}", capturePath).c_str());
+			}
+		}
+
+		auto captureDiskStorage = renderDoc->CalculateCapturesDiskUsage();
+
+		if (captureDiskStorage > 0) {
+			ImGui::TextColored(settings.Theme.StatusPalette.Warning, std::format("Frame captures disk usage: {} MB", captureDiskStorage).c_str());
+		} else {
+			ImGui::TextColored(settings.Theme.StatusPalette.InfoColor, std::format("Frame captures disk usage: {} MB", captureDiskStorage).c_str());
+		}
+
+		ImGui::SameLine();
+		if (ImGui::Button("Clear Captures")) {
+			renderDoc->ClearFrameCaptures();
+			clearedCaptures = renderDoc->GetNumCaptures();
+		}
 	}
 
 	if (ImGui::CollapsingHeader("Replace Original Shaders", ImGuiTreeNodeFlags_DefaultOpen | ImGuiTreeNodeFlags_OpenOnArrow | ImGuiTreeNodeFlags_OpenOnDoubleClick)) {
@@ -1671,7 +1720,11 @@ void Menu::DrawOverlay()
 	auto failed = shaderCache->GetFailedTasks();
 	auto hide = shaderCache->IsHideErrors();
 	auto* abTestingManager = ABTestingManager::GetSingleton();
-	if (!(shaderCache->IsCompiling() || IsEnabled || abTestingManager->IsEnabled() || (failed && !hide) || PerformanceOverlay::GetSingleton()->settings.ShowInOverlay)) {
+	auto* renderDoc = RenderDoc::GetSingleton();
+	bool renderDocAvailable = renderDoc->IsAvailable();
+	const auto renderDocInformation = "WARNING: Frame debugging is active, performance will be severely impacted.\nPress F12, Print Screen or press the Capture button in the Advanced menu.\nDisable frame debugging in the Advanced menu.";
+
+	if (!(shaderCache->IsCompiling() || IsEnabled || abTestingManager->IsEnabled() || (failed && !hide) || PerformanceOverlay::GetSingleton()->settings.ShowInOverlay || renderDocAvailable)) {
 		auto& io = ImGui::GetIO();
 		io.ClearInputKeys();
 		io.ClearEventsQueue();
@@ -1721,6 +1774,9 @@ void Menu::DrawOverlay()
 			ImGui::TextUnformatted("WARNING: Uncompiled shaders will have visual errors or cause stuttering when loading.");
 		}
 
+		if (renderDocAvailable)
+			ImGui::TextColored(settings.Theme.StatusPalette.Warning, renderDocInformation);
+
 		ImGui::End();
 	} else if (failed) {
 		if (!hide) {
@@ -1737,8 +1793,19 @@ void Menu::DrawOverlay()
 				ImGui::TextColored(themeSettings.StatusPalette.Error, "Features that may have modified shaders detected. Check Feature Issues in the Menu.");
 			}
 
+			if (renderDocAvailable)
+				ImGui::TextColored(settings.Theme.StatusPalette.Warning, renderDocInformation);
+
 			ImGui::End();
 		}
+	} else if (renderDocAvailable) {
+		ImGui::SetNextWindowPos(ImVec2(10, 10));
+		if (!ImGui::Begin("ShaderCompilationInfo", nullptr, ImGuiWindowFlags_NoTitleBar | ImGuiWindowFlags_AlwaysAutoResize | ImGuiWindowFlags_NoResize | ImGuiWindowFlags_NoMove | ImGuiWindowFlags_NoSavedSettings)) {
+			ImGui::End();
+			return;
+		}
+		ImGui::TextColored(settings.Theme.StatusPalette.Warning, renderDocInformation);
+		ImGui::End();
 	}
 
 	if (IsEnabled) {

src/RenderDoc.cpp

@@ -0,0 +1,139 @@
+#include "RenderDoc.h"
+
+constexpr const wchar_t* RENDERDOC_DLL_PATH = L"Data\\SKSE\\Plugins\\Renderdoc\\renderdoc.dll";
+constexpr const char* CAPTURE_PATH_TEMPLATE = ".\\Data\\SKSE\\Plugins\\CommunityShaders\\Captures\\";
+
+bool RenderDoc::Initialize()
+{
+	// Try to load the RenderDoc DLL
+	renderDocModule = LoadLibraryW(RENDERDOC_DLL_PATH);
+	if (!renderDocModule) {
+		logger::debug("[RenderDoc] renderdoc.dll not found");
+		return false;
+	}
+
+	// Get the API function pointer
+	auto RENDERDOC_GetAPI = (pRENDERDOC_GetAPI)GetProcAddress(renderDocModule, "RENDERDOC_GetAPI");
+	if (!RENDERDOC_GetAPI) {
+		logger::warn("[RenderDoc] Failed to get RENDERDOC_GetAPI function");
+		FreeLibrary(renderDocModule);
+		renderDocModule = nullptr;
+		return false;
+	}
+
+	// Get the API interface
+	int ret = RENDERDOC_GetAPI(eRENDERDOC_API_Version_1_6_0, (void**)&renderDocApi);
+	if (ret != 1 || !renderDocApi) {
+		logger::warn("[RenderDoc] Failed to get API interface");
+		FreeLibrary(renderDocModule);
+		renderDocModule = nullptr;
+		renderDocApi = nullptr;
+		return false;
+	}
+
+	renderDocApi->SetCaptureFilePathTemplate(CAPTURE_PATH_TEMPLATE);
+
+	try {
+		std::filesystem::create_directories(CAPTURE_PATH_TEMPLATE);
+	} catch (const std::exception& e) {
+		logger::warn("[RenderDoc] Failed to create capture directory: {}", e.what());
+	}
+
+	renderDocApi->MaskOverlayBits(eRENDERDOC_Overlay_None, eRENDERDOC_Overlay_None);
+
+	logger::info("[RenderDoc] Successfully initialized");
+	return true;
+}
+
+void RenderDoc::TriggerCapture()
+{
+	if (!IsAvailable()) {
+		logger::warn("[RenderDoc] Cannot trigger capture - RenderDoc not available");
+		return;
+	}
+
+	if (IsCapturing()) {
+		logger::warn("[RenderDoc] Capture already in progress");
+		return;
+	}
+
+	renderDocApi->TriggerCapture();
+	logger::info("[RenderDoc] Frame capture triggered");
+}
+
+bool RenderDoc::IsCapturing() const
+{
+	if (!IsAvailable()) {
+		return false;
+	}
+
+	return renderDocApi->IsFrameCapturing() != 0;
+}
+
+std::string RenderDoc::GetCapturePath(uint32_t a_index)
+{
+	if (!IsAvailable()) {
+		return "";
+	}
+
+	char path[1024] = {};
+	uint32_t pathLength = sizeof(path);
+	uint64_t timestamp;
+
+	renderDocApi->GetCapture(a_index, path, &pathLength, &timestamp);
+
+	if (pathLength == 0 || pathLength >= sizeof(path)) {
+		return "invalid path";
+	}
+
+	return std::string{ path };
+}
+
+uint32_t RenderDoc::GetNumCaptures() const
+{
+	if (!IsAvailable()) {
+		return 0;
+	}
+
+	return renderDocApi->GetNumCaptures();
+}
+
+uint32_t RenderDoc::CalculateCapturesDiskUsage()
+{
+	try {
+		auto path = std::filesystem::path{ ".\\Data\\SKSE\\Plugins\\CommunityShaders\\Captures\\" };
+
+		uintmax_t totalBytes = 0;
+
+		if (std::filesystem::exists(path)) {
+			for (const auto& entry : std::filesystem::directory_iterator(path)) {
+				if (entry.is_regular_file()) {
+					totalBytes += entry.file_size();
+				}
+			}
+		}
+
+		return static_cast<uint32_t>(totalBytes / (1024 * 1024));  // Convert to MB
+	} catch (const std::exception& e) {
+		logger::error("[RenderDoc] Failed to calculate disk usage: {}", e.what());
+		return 0;
+	}
+}
+
+void RenderDoc::ClearFrameCaptures()
+{
+	try {
+		auto path = std::filesystem::path{ ".\\Data\\SKSE\\Plugins\\CommunityShaders\\Captures\\" };
+
+		if (std::filesystem::exists(path)) {
+			for (const auto& entry : std::filesystem::directory_iterator(path)) {
+				if (entry.is_regular_file()) {
+					std::filesystem::remove(entry.path());
+					logger::info("[RenderDoc] Deleted: {}", entry.path().string());
+				}
+			}
+		}
+	} catch (const std::exception& e) {
+		logger::error("[RenderDoc] Failed to clear frame captures: {}", e.what());
+	}
+}