Skip to content

fix(VR): apply per eye upscaling #1819

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
alandtse merged 2 commits into from
Feb 6, 2026
Merged

fix(VR): apply per eye upscaling #1819

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5365d46
fix(VR): apply per eye upscaling
alandtse Feb 5, 2026
4f463ac
chore: address ai comments
alandtse Feb 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions features/Upscaling/Shaders/Upscaling/ClearHMDMaskCS.hlsl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
// Zeros color in the HMD hidden area per eye.
// Prevents DLSS/FSR from temporally accumulating the engine's sky/ambient clear color
// into visible pixels during head movement ("light blue border" ghosting).
// depth == 0.0 is the unrendered/hidden area value (Skyrim reversed-Z: far plane = 0).
// DepthIn is the combined stereo depth buffer; DepthOffsetX selects the eye's half.
// ColorInOut is the isolated per-eye buffer; ColorOffsetX is always 0.

cbuffer ClearHMDMaskCB : register(b0)
{
uint DepthOffsetX; // X offset into combined stereo depth (0 = left, eyeWidth = right)
uint ColorOffsetX; // X offset into color target (always 0 for per-eye buffers)
uint pad0;
uint pad1;
};

Texture2D<float> DepthIn : register(t0);
RWTexture2D<float4> ColorInOut : register(u0);

[numthreads(8, 8, 1)] void main(uint3 dispatchID : SV_DispatchThreadID)
{
// Read from stereo depth, write to potentially stereo color
if (DepthIn[dispatchID.xy + uint2(DepthOffsetX, 0)] == 0.0)
ColorInOut[dispatchID.xy + uint2(ColorOffsetX, 0)] = float4(0.0, 0.0, 0.0, 0.0);
}
306 changes: 302 additions & 4 deletions src/Features/Upscaling.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
#include "Upscaling/DX12SwapChain.h"
#include "Upscaling/FidelityFX.h"
#include "Upscaling/Streamline.h"
#include "Utils/UI.h"
#include "VR.h"
#include <Windows.h>
#include <algorithm>
Expand Down Expand Up @@ -182,12 +183,13 @@ void Upscaling::DrawSettings()
// Check the current upscale method
auto upscaleMethod = GetUpscaleMethod();

// Display warning for DLSS resolution limits
if (upscaleMethod == UpscaleMethod::kDLSS) {
// Display warning for DLSS resolution limits (non-VR only; VR handles this automatically)
if (!globals::game::isVR && upscaleMethod == UpscaleMethod::kDLSS) {
auto screenSize = globals::state->screenSize;
if (screenSize.x > streamline.MAX_RESOLUTION || screenSize.y > streamline.MAX_RESOLUTION) {
ImGui::PushStyleColor(ImGuiCol_Text, Util::Colors::GetWarning());
ImGui::Text("Warning: Requested resolution %.0f x %.0f exceeds maximum supported resolution %d x %d for DLSS.", screenSize.x, screenSize.y, streamline.MAX_RESOLUTION, streamline.MAX_RESOLUTION);
ImGui::Text("Warning: Requested resolution %.0f x %.0f exceeds maximum supported resolution %d x %d for DLSS.",
screenSize.x, screenSize.y, streamline.MAX_RESOLUTION, streamline.MAX_RESOLUTION);
ImGui::Text("DLSS will not function. Lower your resolution or select a different upscaling method.");
ImGui::PopStyleColor();
}
Expand Down Expand Up @@ -312,6 +314,61 @@ void Upscaling::DrawSettings()
if (auto _tt = Util::HoverTooltipWrapper()) {
ImGui::Text("Streamline logging controls the verbosity of NVIDIA Streamline backend logs. Useful for debugging issues with DLSS/DLSS-G.");
}

// VR Debug visualization -- per-eye buffers and native inputs
if (globals::game::isVR) {
ImGui::Separator();
static float debugRescale = 0.15f;
ImGui::SliderFloat("View Resize", &debugRescale, 0.05f, 1.f);

if (ImGui::TreeNode("Upscaling Intermediates")) {
if (vrIntermediateColorIn[0] && vrIntermediateColorOut[0]) {
BUFFER_VIEWER_NODE_TITLE(vrIntermediateColorIn[0], "Left Eye In", debugRescale)
BUFFER_VIEWER_NODE_TITLE(vrIntermediateColorIn[1], "Right Eye In", debugRescale)
BUFFER_VIEWER_NODE_TITLE(vrIntermediateColorOut[0], "Left Eye Out", debugRescale)
BUFFER_VIEWER_NODE_TITLE(vrIntermediateColorOut[1], "Right Eye Out", debugRescale)
BUFFER_VIEWER_NODE_TITLE(vrIntermediateMotionVectors[0], "Left Eye MVec", debugRescale)
BUFFER_VIEWER_NODE_TITLE(vrIntermediateMotionVectors[1], "Right Eye MVec", debugRescale)
BUFFER_VIEWER_NODE_TITLE(vrIntermediateReactiveMask[0], "Left Eye Reactive", debugRescale)
BUFFER_VIEWER_NODE_TITLE(vrIntermediateReactiveMask[1], "Right Eye Reactive", debugRescale)
} else {
ImGui::TextDisabled("VR intermediates not yet created (enter game world)");
}
ImGui::TreePop();
}

if (ImGui::TreeNode("Native Inputs")) {
auto renderer = globals::game::renderer;
auto& main = renderer->GetRuntimeData().renderTargets[RE::RENDER_TARGETS::kMAIN];
auto& mvec = renderer->GetRuntimeData().renderTargets[RE::RENDER_TARGETS::kMOTION_VECTOR];
auto& depth = renderer->GetDepthStencilData().depthStencils[RE::RENDER_TARGETS_DEPTHSTENCIL::kMAIN];

auto DisplayRT = [&](const char* label, ID3D11Texture2D* tex, ID3D11ShaderResourceView* srv) {
if (srv && tex) {
D3D11_TEXTURE2D_DESC desc;
tex->GetDesc(&desc);
char buf[128];
snprintf(buf, sizeof(buf), "%s (%ux%u)", label, desc.Width, desc.Height);
if (ImGui::TreeNode(buf)) {
ImGui::Image(srv, { desc.Width * debugRescale, desc.Height * debugRescale });
ImGui::TreePop();
}
}
};

DisplayRT("kMAIN (Color Input)", (ID3D11Texture2D*)main.texture, (ID3D11ShaderResourceView*)main.SRV);
DisplayRT("Motion Vectors", (ID3D11Texture2D*)mvec.texture, (ID3D11ShaderResourceView*)mvec.SRV);
DisplayRT("Depth", depth.texture, depth.depthSRV);

if (reactiveMaskTexture)
BUFFER_VIEWER_NODE_TITLE(reactiveMaskTexture, "Reactive Mask", debugRescale)
if (transparencyCompositionMaskTexture)
BUFFER_VIEWER_NODE_TITLE(transparencyCompositionMaskTexture, "Transparency Mask", debugRescale)

ImGui::TreePop();
}
}

ImGui::Separator();
// FidelityFX section
if (ImGui::Selectable("AMD FidelityFX DLLs (click to open folder)")) {
Expand Down Expand Up @@ -598,6 +655,18 @@ void Upscaling::CheckResources(UpscaleMethod a_upscalemethod)
streamline.DestroyDLSSResources();
else if (previousUpscaleMode == UpscaleMethod::kFSR)
fidelityFX.DestroyFSRResources();

if (globals::game::isVR) {
for (int i = 0; i < 2; i++) {
vrIntermediateColorIn[i].reset();
vrIntermediateColorOut[i].reset();
vrIntermediateDepth[i].reset();
vrIntermediateMotionVectors[i].reset();
vrIntermediateReactiveMask[i].reset();
vrIntermediateTransparencyMask[i].reset();
}
vrResourcesAllocated[0] = vrResourcesAllocated[1] = false;
}
}
if (a_upscalemethod == UpscaleMethod::kFSR)
fidelityFX.CreateFSRResources();
Expand Down Expand Up @@ -675,6 +744,231 @@ ID3D11VertexShader* Upscaling::GetUpscaleVS()
return upscaleVS.get();
}

eastl::unique_ptr<Texture2D> Upscaling::CreateTextureFromSource(ID3D11Resource* src, uint32_t width, uint32_t height,
bool copyBindFlags, bool createSRV, bool createUAV, const char* name)
{
D3D11_TEXTURE2D_DESC srcDesc;
static_cast<ID3D11Texture2D*>(src)->GetDesc(&srcDesc);

D3D11_TEXTURE2D_DESC desc = {};
desc.Width = width;
desc.Height = height;
desc.MipLevels = 1;
desc.ArraySize = 1;
desc.Format = srcDesc.Format;
desc.SampleDesc.Count = 1;
desc.Usage = D3D11_USAGE_DEFAULT;
desc.BindFlags = copyBindFlags ? srcDesc.BindFlags : (D3D11_BIND_SHADER_RESOURCE | D3D11_BIND_UNORDERED_ACCESS);

auto tex = eastl::make_unique<Texture2D>(desc);

if (name) {
Util::SetResourceName(tex->resource.get(), name);
}

if (createSRV) {
D3D11_SHADER_RESOURCE_VIEW_DESC srvDesc = {};
srvDesc.Format = srcDesc.Format;
srvDesc.ViewDimension = D3D11_SRV_DIMENSION_TEXTURE2D;
srvDesc.Texture2D.MostDetailedMip = 0;
srvDesc.Texture2D.MipLevels = 1;
tex->CreateSRV(srvDesc);
}
if (createUAV) {
D3D11_UNORDERED_ACCESS_VIEW_DESC uavDesc = {};
uavDesc.Format = srcDesc.Format;
uavDesc.ViewDimension = D3D11_UAV_DIMENSION_TEXTURE2D;
uavDesc.Texture2D.MipSlice = 0;
tex->CreateUAV(uavDesc);
}
return tex;
}

void Upscaling::CreateVRIntermediateTextures(uint32_t inWidth, uint32_t inHeight, uint32_t outWidth, uint32_t outHeight,
ID3D11Resource* colorSrc, ID3D11Resource* mvecSrc, ID3D11Resource* reactiveSrc, ID3D11Resource* transparencySrc)
{
// All buffers are per-eye: Streamline validates all extents against the input color texture
// dimensions, so every tagged resource must be isolated per-eye at {0,0}.
for (int i = 0; i < 2; i++) {
std::string suffix = (i == 0) ? "Left" : "Right";

vrIntermediateColorIn[i] = CreateTextureFromSource(colorSrc, inWidth, inHeight, false, true, true, ("Upscale_ColorIn_" + suffix).c_str());
vrIntermediateColorOut[i] = CreateTextureFromSource(colorSrc, outWidth, outHeight, false, true, false, ("Upscale_ColorOut_" + suffix).c_str());

// Depth: R32_TYPELESS base (matches kMAIN), with R32_FLOAT SRV for ClearHMDMaskCS.
// CopySubresourceRegion requires matching typeless formats; SRV reinterprets as R32_FLOAT.
{
D3D11_TEXTURE2D_DESC depthDesc = {};
depthDesc.Width = inWidth;
depthDesc.Height = inHeight;
depthDesc.MipLevels = 1;
depthDesc.ArraySize = 1;
depthDesc.Format = DXGI_FORMAT_R32_TYPELESS;
depthDesc.SampleDesc.Count = 1;
depthDesc.Usage = D3D11_USAGE_DEFAULT;
depthDesc.BindFlags = D3D11_BIND_SHADER_RESOURCE;
vrIntermediateDepth[i] = eastl::make_unique<Texture2D>(depthDesc);

Util::SetResourceName(vrIntermediateDepth[i]->resource.get(), ("Upscale_Depth_" + suffix).c_str());

D3D11_SHADER_RESOURCE_VIEW_DESC srvDesc = {};
srvDesc.Format = DXGI_FORMAT_R32_FLOAT;
srvDesc.ViewDimension = D3D11_SRV_DIMENSION_TEXTURE2D;
srvDesc.Texture2D.MipLevels = 1;
vrIntermediateDepth[i]->CreateSRV(srvDesc);
}

vrIntermediateMotionVectors[i] = CreateTextureFromSource(mvecSrc, inWidth, inHeight, false, true, false, ("Upscale_MVec_" + suffix).c_str());
vrIntermediateReactiveMask[i] = CreateTextureFromSource(reactiveSrc, inWidth, inHeight, false, true, false, ("Upscale_Reactive_" + suffix).c_str());
vrIntermediateTransparencyMask[i] = CreateTextureFromSource(transparencySrc, inWidth, inHeight, false, true, false, ("Upscale_Transparency_" + suffix).c_str());
}

logger::info("[Upscaling] Created VR intermediate textures: per-eye in {}x{}, out {}x{}",
inWidth, inHeight, outWidth, outHeight);
}

void Upscaling::PreparePerEyeInputs(ID3D11Resource* colorSrc, ID3D11Resource* depthSrc, ID3D11Resource* mvecSrc,
ID3D11Resource* reactiveSrc, ID3D11Resource* transparencySrc)
{
if (!globals::game::isVR)
return;

auto state = globals::state;
if (state->frameAnnotations)
state->BeginPerfEvent("VR Upscaling Prepare");

auto context = globals::d3d::context;
auto screenSize = state->screenSize;
auto renderSize = Util::ConvertToDynamic(screenSize);

uint32_t eyeWidthOut = (uint32_t)(screenSize.x / 2);
uint32_t eyeHeightOut = (uint32_t)screenSize.y;
uint32_t eyeWidthIn = (uint32_t)(renderSize.x / 2);
uint32_t eyeHeightIn = (uint32_t)renderSize.y;

bool needsRecreate = !vrIntermediateColorIn[0] || !vrIntermediateColorOut[0];
if (!needsRecreate) {
needsRecreate = (vrIntermediateColorIn[0]->desc.Width != eyeWidthIn ||
vrIntermediateColorIn[0]->desc.Height != eyeHeightIn ||
vrIntermediateColorOut[0]->desc.Width != eyeWidthOut ||
vrIntermediateColorOut[0]->desc.Height != eyeHeightOut);
}
if (needsRecreate) {
logger::info("[Upscaling] (Re)creating VR intermediates: per-eye in {}x{}, out {}x{}",
eyeWidthIn, eyeHeightIn, eyeWidthOut, eyeHeightOut);
CreateVRIntermediateTextures(eyeWidthIn, eyeHeightIn, eyeWidthOut, eyeHeightOut,
colorSrc, mvecSrc, reactiveSrc, transparencySrc);
vrResourcesAllocated[0] = vrResourcesAllocated[1] = false;
}

// Extract both eyes' inputs from combined stereo buffers
for (uint32_t i = 0; i < 2; ++i) {
uint32_t offsetXIn = (i == 1) ? eyeWidthIn : 0;
D3D11_BOX srcBox = { offsetXIn, 0, 0, offsetXIn + eyeWidthIn, eyeHeightIn, 1 };

context->CopySubresourceRegion(vrIntermediateColorIn[i]->resource.get(), 0, 0, 0, 0, colorSrc, 0, &srcBox);
context->CopySubresourceRegion(vrIntermediateDepth[i]->resource.get(), 0, 0, 0, 0, depthSrc, 0, &srcBox);
context->CopySubresourceRegion(vrIntermediateMotionVectors[i]->resource.get(), 0, 0, 0, 0, mvecSrc, 0, &srcBox);
context->CopySubresourceRegion(vrIntermediateTransparencyMask[i]->resource.get(), 0, 0, 0, 0, transparencySrc, 0, &srcBox);
context->CopySubresourceRegion(vrIntermediateReactiveMask[i]->resource.get(), 0, 0, 0, 0, reactiveSrc, 0, &srcBox);
}

// Zero color where depth == 0 (HMD hidden area) in each per-eye buffer.
// Depth is read from the combined stereo SRV at the per-eye offset; color is written
// to the isolated per-eye UAV (ColorOffsetX = 0).
auto& depthTexture = globals::game::renderer->GetDepthStencilData().depthStencils[RE::RENDER_TARGETS_DEPTHSTENCIL::kMAIN];

for (uint32_t i = 0; i < 2; ++i) {
uint32_t depthOffset = (i == 1) ? eyeWidthIn : 0;
ClearHMDMask(vrIntermediateColorIn[i]->uav.get(), depthTexture.depthSRV,
eyeWidthIn, eyeHeightIn, depthOffset, 0);
}

if (state->frameAnnotations)
state->EndPerfEvent();
}

void Upscaling::FinalizePerEyeOutputs(ID3D11Resource* colorDst)
{
if (!globals::game::isVR)
return;

auto state = globals::state;
if (state->frameAnnotations)
state->BeginPerfEvent("VR Upscaling Finalize");

auto context = globals::d3d::context;
auto screenSize = state->screenSize;

uint32_t eyeWidthOut = (uint32_t)(screenSize.x / 2);
uint32_t eyeHeightOut = (uint32_t)screenSize.y;

// Write upscaled outputs back
for (uint32_t i = 0; i < 2; ++i) {
uint32_t offsetXOut = (i == 1) ? eyeWidthOut : 0;
D3D11_BOX outBox = { 0, 0, 0, eyeWidthOut, eyeHeightOut, 1 };
context->CopySubresourceRegion(colorDst, 0, offsetXOut, 0, 0, vrIntermediateColorOut[i]->resource.get(), 0, &outBox);
}

if (state->frameAnnotations)
state->EndPerfEvent();
}

void Upscaling::ClearHMDMask(ID3D11UnorderedAccessView* colorUAV, ID3D11ShaderResourceView* depthSRV,
uint32_t eyeWidth, uint32_t eyeHeight, uint32_t depthOffsetX, uint32_t colorOffsetX)
{
if (!globals::game::isVR)
return;

auto context = globals::d3d::context;

if (!vrClearHMDMaskCS) {
vrClearHMDMaskCS.attach((ID3D11ComputeShader*)Util::CompileShader(L"Data/Shaders/Upscaling/ClearHMDMaskCS.hlsl", {}, "cs_5_0"));

D3D11_BUFFER_DESC cbDesc = {};
cbDesc.ByteWidth = 16; // 4 uints
cbDesc.Usage = D3D11_USAGE_DYNAMIC;
cbDesc.BindFlags = D3D11_BIND_CONSTANT_BUFFER;
cbDesc.CPUAccessFlags = D3D11_CPU_ACCESS_WRITE;
DX::ThrowIfFailed(globals::d3d::device->CreateBuffer(&cbDesc, nullptr, vrClearHMDMaskCB.put()));
Comment thread
alandtse marked this conversation as resolved.
}

if (vrClearHMDMaskCS) {
auto dispatchX = (eyeWidth + 7) / 8;
auto dispatchY = (eyeHeight + 7) / 8;

context->CSSetShader(vrClearHMDMaskCS.get(), nullptr, 0);

ID3D11ShaderResourceView* srvs[1] = { depthSRV };
context->CSSetShaderResources(0, 1, srvs);

ID3D11UnorderedAccessView* uavs[1] = { colorUAV };
context->CSSetUnorderedAccessViews(0, 1, uavs, nullptr);

D3D11_MAPPED_SUBRESOURCE mapped{};
context->Map(vrClearHMDMaskCB.get(), 0, D3D11_MAP_WRITE_DISCARD, 0, &mapped);

uint32_t offsets[4] = { depthOffsetX, colorOffsetX, 0, 0 };

memcpy(mapped.pData, offsets, sizeof(offsets));
context->Unmap(vrClearHMDMaskCB.get(), 0);

ID3D11Buffer* cbs[1] = { vrClearHMDMaskCB.get() };
context->CSSetConstantBuffers(0, 1, cbs);

context->Dispatch(dispatchX, dispatchY, 1);

// Unbind
ID3D11ShaderResourceView* nullSRV[1] = { nullptr };
ID3D11UnorderedAccessView* nullUAV[1] = { nullptr };
ID3D11Buffer* nullCB[1] = { nullptr };
context->CSSetShaderResources(0, 1, nullSRV);
context->CSSetUnorderedAccessViews(0, 1, nullUAV, nullptr);
context->CSSetConstantBuffers(0, 1, nullCB);
context->CSSetShader(nullptr, nullptr, 0);
}
}

int32_t GetJitterPhaseCount(int32_t renderWidth, int32_t displayWidth)
{
const float basePhaseCount = 8.0f;
Expand Down Expand Up @@ -1139,7 +1433,11 @@ void Upscaling::LoadUpscalingSDKs()

void Upscaling::CheckFrameConstants()
{
streamline.CheckFrameConstants();
// In VR, constants are set per-eye in the Upscale() loop
// Skip the early call from DeferredPasses to avoid issues
if (globals::game::isVR)
return;
streamline.CheckFrameConstants(streamline.viewport, 0);
}

void Upscaling::SetUIBuffer()
Expand Down
Loading