Skip to content

fix(lighting): create Masks2 RT and divide SSGI AO by vertexAO #2411

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
SkrubbySkrubInAShrub merged 5 commits into from
Jun 7, 2026
Merged

fix(lighting): create Masks2 RT and divide SSGI AO by vertexAO #2411

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
f2d0a52
fix(lighting): create Masks2 RT and divide SSGI AO by vertexAO
claude May 24, 2026
88b8872
Merge branch 'dev' into claude/lighting-vertexao-linearise
doodlum Jun 1, 2026
b212d7f
Merge branch 'dev' into claude/lighting-vertexao-linearise
doodlum Jun 5, 2026
5b4fb0d
Merge branch 'dev' into claude/lighting-vertexao-linearise
doodlum Jun 6, 2026
457f0c6
fix(deferred): use R16_UNORM for MASKS2 render target
doodlum Jun 6, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions package/Shaders/DeferredCompositeCS.hlsl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ Texture2D<float3> SpecularTexture : register(t0);
Texture2D<unorm float3> AlbedoTexture : register(t1);
Texture2D<unorm float3> NormalRoughnessTexture : register(t2);
Texture2D<float3> MasksTexture : register(t3);
Texture2D<unorm float> Masks2Texture : register(t9);

RWTexture2D<float4> MainRW : register(u0);
RWTexture2D<float4> NormalTAAMaskSpecularMaskRW : register(u1);
Expand Down Expand Up @@ -139,6 +140,11 @@ void SampleSSGISpecular(uint2 pixCoord, sh2 lobe, inout float ao, out float3 il,
float3 ssgiIl;
SampleSSGI(dispatchID.xy, normalWS, ssgiAo, ssgiIl);

// Masks2.x stores 1 - vertexAO (Lighting.hlsl only); cleared to 0 for
// pixels with no vertex AO contribution, so vertexAO defaults to 1.
float vertexAO = 1.0 - Masks2Texture[dispatchID.xy].x;
ssgiAo = saturate(ssgiAo / max(vertexAO, EPSILON_DIVISION));

float3 linAlbedo = Color::IrradianceToLinear(albedo / Color::PBRLightingScale);
float3 multiBounceSSGIAo = MultiBounceAO(linAlbedo, ssgiAo);

Expand Down
37 changes: 8 additions & 29 deletions package/Shaders/Lighting.hlsl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -344,9 +344,7 @@ struct PS_OUTPUT
float4 Specular: SV_Target4;
float4 Reflectance: SV_Target5;
float4 Masks: SV_Target6;
# if defined(SNOW)
float4 Parameters: SV_Target7;
# endif
float4 Masks2: SV_Target7;
};
#else
struct PS_OUTPUT
Expand Down Expand Up @@ -2184,9 +2182,6 @@ PS_OUTPUT main(PS_INPUT input, bool frontFace : SV_IsFrontFace)
rawBaseColor = Triplanar::SampleStochasticBias(TexColorSampler, SampColorSampler, projWorldPos, triWeights, ProjectedUVParams2.y, SharedData::MipBias, screenNoise);
baseColor = float4(Color::Diffuse(rawBaseColor.rgb), rawBaseColor.a);
worldNormal.xyz = projectedNormal;
# if defined(SNOW)
psout.Parameters.y = 1;
# endif // SNOW
# elif !defined(FACEGEN) && !defined(MULTI_LAYER_PARALLAX) && !defined(PARALLAX) && !defined(SPARKLE)
if (ProjectedUVParams3.w > 0.5) {
float diffuseNormalScale = ProjectedUVParams3.x * ProjectedUVParams.z;
Expand Down Expand Up @@ -2215,18 +2210,12 @@ PS_OUTPUT main(PS_INPUT input, bool frontFace : SV_IsFrontFace)

# if defined(SNOW)
useSnowDecalSpecular = true;
psout.Parameters.y = GetSnowParameterY(projectedMaterialWeight, baseColor.w);
# endif // SNOW
} else {
if (projWeight > 0) {
baseColor.xyz = Color::Diffuse(ProjectedUVParams2.xyz);
# if defined(SNOW)
useSnowDecalSpecular = true;
psout.Parameters.y = GetSnowParameterY(projWeight, baseColor.w);
# endif // SNOW
} else {
# if defined(SNOW)
psout.Parameters.y = 0;
# endif // SNOW
}
}
Expand All @@ -2236,12 +2225,6 @@ PS_OUTPUT main(PS_INPUT input, bool frontFace : SV_IsFrontFace)
# endif // SPECULAR
# endif // SPARKLE

# elif defined(SNOW)
# if defined(LANDSCAPE)
psout.Parameters.y = landSnowMask;
# else
psout.Parameters.y = baseColor.w;
# endif // LANDSCAPE
# endif // SNOW

# if defined(WORLD_MAP)
Expand Down Expand Up @@ -3077,13 +3060,14 @@ PS_OUTPUT main(PS_INPUT input, bool frontFace : SV_IsFrontFace)

# if defined(HAIR)
float3 vertexColor = lerp(1, Color::ColorToLinear(TintColor.xyz), Color::ColorToLinear(input.Color.y));
float vertexAO = 1;
# if defined(CS_HAIR)
if (SharedData::hairSpecularSettings.Enabled)
vertexColor = 1;
# endif
# elif defined(SKYLIGHTING)
float3 vertexColor = input.Color.xyz;
float vertexAO = max(max(vertexColor.r, vertexColor.g), vertexColor.b);
float vertexAO = Color::ColorToLinear(max(max(vertexColor.r, vertexColor.g), vertexColor.b).xxx).x;
# if defined(TRUE_PBR)
vertexAO = lerp(1, vertexAO, SharedData::truePBRSettings.VertexAOStrength);
vertexColor = 1;
Expand All @@ -3096,6 +3080,7 @@ PS_OUTPUT main(PS_INPUT input, bool frontFace : SV_IsFrontFace)
# else
float3 vertexColor = input.Color.xyz;
# endif
float vertexAO = Color::ColorToLinear(max(max(vertexColor.r, vertexColor.g), vertexColor.b).xxx).x;
# endif // defined (HAIR)

# if defined(IBL)
Expand Down Expand Up @@ -3493,16 +3478,6 @@ PS_OUTPUT main(PS_INPUT input, bool frontFace : SV_IsFrontFace)
PomOffsetTex[uint2(input.Position.xy)] = hasPOM ? pixelOffset : Stereo::POM_NO_DATA;
# endif

# if defined(SNOW)
# if defined(TRUE_PBR)
psout.Parameters.x = Color::RGBToLuminanceAlternative(specularColor);
psout.Parameters.y = 0;
# else
psout.Parameters.x = Color::RGBToLuminanceAlternative(lightsSpecularColor);
# endif
psout.Parameters.w = psout.Diffuse.w;
# endif

float masksZ = Color::RGBToYCoCg(directionalAmbientColor).x;

# if defined(SSS) && defined(SKIN)
Expand All @@ -3511,6 +3486,10 @@ PS_OUTPUT main(PS_INPUT input, bool frontFace : SV_IsFrontFace)
psout.Masks = float4(0, 0, masksZ, psout.Diffuse.w);
# endif

// Stored as 1 - vertexAO so the cleared default (0) means no occlusion
// for pixels that do not write to this RT (sky, water, grass, effects).
psout.Masks2 = float4(1.0 - vertexAO, 0, 0, 0);

float stochasticBlend = (screenNoise * screenNoise) < psout.Diffuse.w ? 1.0 : 0.0;
psout.NormalGlossiness.w = stochasticBlend;
# endif
Expand Down
16 changes: 8 additions & 8 deletions package/Shaders/RunGrass.hlsl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -358,9 +358,7 @@ struct PS_OUTPUT
float4 Reflectance: SV_Target5;
# endif // TRUE_PBR
float4 Masks: SV_Target6;
# if defined(TRUE_PBR)
float4 Parameters: SV_Target7;
# endif // TRUE_PBR
float4 Masks2: SV_Target7;
# endif // RENDER_DEPTH
};
#else
Expand All @@ -374,6 +372,7 @@ struct PS_OUTPUT
float4 Normal: SV_Target2;
float4 Albedo: SV_Target3;
float4 Masks: SV_Target6;
float4 Masks2: SV_Target7;
# endif
};
#endif
Expand Down Expand Up @@ -633,15 +632,15 @@ PS_OUTPUT main(PS_INPUT input, bool frontFace : SV_IsFrontFace)
lightsDiffuseColor += dirLightColor * dirDetailedShadow * saturate(dirLightAngle) * Color::VanillaNormalization();

float3 vertexColor = Color::ColorToLinear(input.Color.xyz);
vertexColor /= max(max(max(vertexColor.r, vertexColor.g), vertexColor.b), EPSILON_DIVISION);
float vertexAO = max(max(vertexColor.r, vertexColor.g), vertexColor.b);
vertexColor /= max(vertexAO, EPSILON_DIVISION);

# if defined(SKYLIGHTING)
# if defined(VR)
float3 positionMSSkylight = input.WorldPosition.xyz + FrameBuffer::CameraPosAdjust[eyeIndex].xyz - FrameBuffer::CameraPosAdjust[0].xyz;
# else
float3 positionMSSkylight = input.WorldPosition.xyz;
# endif
float vertexAO = max(max(vertexColor.r, vertexColor.g), vertexColor.b);
float skylightingDiffuse = Skylighting::GetVertexSkylightingDiffuse(positionMSSkylight, normal, vertexAO);
# endif // SKYLIGHTING

Expand Down Expand Up @@ -787,14 +786,14 @@ PS_OUTPUT main(PS_INPUT input, bool frontFace : SV_IsFrontFace)
psout.Albedo = float4(Color::IrradianceToGamma(indirectDiffuseLobeWeight), 1);
psout.NormalGlossiness = float4(GBuffer::EncodeNormal(normalVS), 1 - pbrSurfaceProperties.Roughness, 1);
psout.Reflectance = float4(indirectSpecularLobeWeight, 1);
psout.Parameters = float4(0, 0, 1, 1);
# else
psout.Albedo = float4(albedo, 1);
psout.NormalGlossiness = float4(GBuffer::EncodeNormal(normalVS), specColor.w, 1);
# endif

psout.Specular = float4(specularColor, 1);
psout.Masks = float4(0, 0, Color::RGBToYCoCg(directionalAmbientColor).x, 0);
psout.Masks2 = float4(1.0 - vertexAO, 0, 0, 0);
# endif
return psout;
}
Expand Down Expand Up @@ -900,15 +899,15 @@ PS_OUTPUT main(PS_INPUT input)
float3 normal = -normalize(cross(ddx, ddy));

float3 vertexColor = Color::ColorToLinear(input.Color.xyz);
vertexColor /= max(max(max(vertexColor.r, vertexColor.g), vertexColor.b), EPSILON_DIVISION);
float vertexAO = max(max(vertexColor.r, vertexColor.g), vertexColor.b);
vertexColor /= max(vertexAO, EPSILON_DIVISION);

# if defined(SKYLIGHTING)
# if defined(VR)
float3 positionMSSkylight = input.WorldPosition.xyz + FrameBuffer::CameraPosAdjust[eyeIndex].xyz - FrameBuffer::CameraPosAdjust[0].xyz;
# else
float3 positionMSSkylight = input.WorldPosition.xyz;
# endif
float vertexAO = max(max(vertexColor.r, vertexColor.g), vertexColor.b);
float skylightingDiffuse = Skylighting::GetVertexSkylightingDiffuse(positionMSSkylight, normal, vertexAO);
# endif // SKYLIGHTING

Expand Down Expand Up @@ -950,6 +949,7 @@ PS_OUTPUT main(PS_INPUT input)

psout.Albedo = float4(albedo, 1);
psout.Masks = float4(0, 0, Color::RGBToYCoCg(directionalAmbientColor).x, 0);
psout.Masks2 = float4(1.0 - vertexAO, 0, 0, 0);
# endif

return psout;
Expand Down
7 changes: 5 additions & 2 deletions src/Deferred.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,8 @@ void Deferred::SetupResources()
SetupRenderTarget(NORMALROUGHNESS, texDesc, srvDesc, rtvDesc, uavDesc, DXGI_FORMAT_R10G10B10A2_UNORM, D3D11_BIND_RENDER_TARGET | D3D11_BIND_SHADER_RESOURCE);
// Masks
SetupRenderTarget(MASKS, texDesc, srvDesc, rtvDesc, uavDesc, DXGI_FORMAT_R11G11B10_FLOAT, D3D11_BIND_RENDER_TARGET | D3D11_BIND_SHADER_RESOURCE);
// Masks2 (vertexAO; fp16 to allow blending)
SetupRenderTarget(MASKS2, texDesc, srvDesc, rtvDesc, uavDesc, DXGI_FORMAT_R16_UNORM, D3D11_BIND_RENDER_TARGET | D3D11_BIND_SHADER_RESOURCE);

// TAA Water Buffers
SetupRenderTarget(RE::RENDER_TARGETS::kWATER_1, texDesc, srvDesc, rtvDesc, uavDesc, DXGI_FORMAT_R16G16B16A16_FLOAT, D3D11_BIND_RENDER_TARGET | D3D11_BIND_SHADER_RESOURCE);
Expand Down Expand Up @@ -243,7 +245,7 @@ void Deferred::StartDeferred()
SPECULAR,
REFLECTANCE,
MASKS,
RE::RENDER_TARGET::kNONE
MASKS2
};

for (uint i = 2; i < 8; i++) {
Expand Down Expand Up @@ -317,6 +319,7 @@ void Deferred::DeferredPasses()
auto albedo = renderer->GetRuntimeData().renderTargets[ALBEDO];
auto normalRoughness = renderer->GetRuntimeData().renderTargets[NORMALROUGHNESS];
auto masks = renderer->GetRuntimeData().renderTargets[MASKS];
auto masks2 = renderer->GetRuntimeData().renderTargets[MASKS2];

auto main = renderer->GetRuntimeData().renderTargets[forwardRenderTargets[0]];
auto normals = renderer->GetRuntimeData().renderTargets[forwardRenderTargets[2]];
Expand Down Expand Up @@ -361,7 +364,7 @@ void Deferred::DeferredPasses()
dynamicCubemaps.loaded ? dynamicCubemaps.envTexture->srv.get() : nullptr, // t6 EnvTexture
dynamicCubemaps.loaded ? dynamicCubemaps.envReflectionsTexture->srv.get() : nullptr, // t7 EnvReflectionsTexture
dynamicCubemaps.loaded && skylighting.loaded ? skylighting.texProbeArray->srv.get() : nullptr, // t8 SkylightingProbeArray
nullptr, // t9 unused
masks2.SRV, // t9 Masks2Texture (vertexAO in .x)
ssgi_ao, // t10 SsgiAoTexture
ssgi_hq_spec ? nullptr : ssgi_y, // t11 SsgiYTexture
ssgi_hq_spec ? nullptr : ssgi_cocg, // t12 SsgiCoCgTexture
Expand Down
Loading