fix: fixes for flickering lighting

features/Light Limit Fix/Shaders/LightLimitFix/LightLimitFix.hlsli

@@ -8,6 +8,8 @@ namespace LightLimitFix
 	{
 		uint NumStrictLights;
 		int RoomIndex;
+		uint ShadowBitMask;
+		uint pad0;
 		Light StrictLights[15];
 	};
 
@@ -113,6 +115,11 @@ namespace LightLimitFix
 
 	bool IsLightIgnored(Light light)
 	{
+		if (light.lightFlags & LightLimitFix::LightFlags::Shadow)
+		{
+			return !(ShadowBitMask & (1 << light.shadowLightIndex));
+		}
+
 		bool lightIgnored = false;
 		if ((light.lightFlags & LightFlags::PortalStrict) && RoomIndex >= 0) {
 			lightIgnored = true;

package/Shaders/Lighting.hlsl

@@ -64,10 +64,11 @@ struct VS_OUTPUT
 	float2
 #endif  // (defined (PROJECTED_UV) && !defined(SKINNED)) || defined(LANDSCAPE)
 		TexCoord0 : TEXCOORD0;
-#if defined(ENVMAP)
-	precise
-#endif  // ENVMAP
+
+#if defined(WORLD_MAP)
 		float3 InputPosition : TEXCOORD4;
+#endif
+
 #if defined(SKINNED) || !defined(MODELSPACENORMALS)
 	float3 TBN0 : TEXCOORD1;
 	float3 TBN1 : TEXCOORD2;
@@ -81,21 +82,12 @@ struct VS_OUTPUT
 #elif defined(PROJECTED_UV) && !defined(SKINNED)
 	float3 TexProj : TEXCOORD7;
 #endif  // EYE
-	float3 ScreenNormalTransform0 : TEXCOORD8;
-	float3 ScreenNormalTransform1 : TEXCOORD9;
-	float3 ScreenNormalTransform2 : TEXCOORD10;
-	// #if !defined(VR)  // Position is normally not in VR, but perhaps we can use
-	// it.
+
 	float4 WorldPosition : POSITION1;
 	float4 PreviousWorldPosition : POSITION2;
-	// #endif // !VR
 	float4 Color : COLOR0;
 	float4 FogParam : COLOR1;
-#if !defined(VR)
-	row_major float3x4 World[1] : POSITION3;
-#else
-	row_major float3x4 World[2] : POSITION3;
-#endif  // VR
+
 #if defined(VR)
 	float ClipDistance : SV_ClipDistance0;  // o11
 	float CullDistance : SV_CullDistance0;  // p11
@@ -242,12 +234,8 @@ VS_OUTPUT main(VS_INPUT input)
 #	endif
 	vsout.TexCoord0.xy = uv;
 
-#	if defined(ENVMAP) || defined(MULTI_LAYER_PARALLAX) || defined(SKINNED)
-	vsout.InputPosition.xyz = worldPosition.xyz;
-#	elif defined(WORLD_MAP)
+#	if defined(WORLD_MAP)
 	vsout.InputPosition.xyz = WorldMapOverlayParameters.xyz + worldPosition.xyz;
-#	else
-	vsout.InputPosition.xyz = inputPosition.xyz;
 #	endif
 
 #	if defined(SKINNED)
@@ -309,26 +297,6 @@ VS_OUTPUT main(VS_INPUT input)
 	vsout.EyeNormal.xyz = normalize(worldPosition.xyz - mul(modelEyeCenter, transpose(worldMatrix)));
 #	endif  // EYE
 
-#	if defined(SKINNED)
-	float3x3 ScreenNormalTransform = mul(ScreenProj[eyeIndex], worldTbnTr);
-
-	vsout.ScreenNormalTransform0.xyz = ScreenNormalTransform[0];
-	vsout.ScreenNormalTransform1.xyz = ScreenNormalTransform[1];
-	vsout.ScreenNormalTransform2.xyz = ScreenNormalTransform[2];
-#	else
-	float3x4 transMat = mul(ScreenProj[eyeIndex], World[eyeIndex]);
-
-#		if defined(MODELSPACENORMALS)
-	vsout.ScreenNormalTransform0.xyz = transMat[0].xyz;
-	vsout.ScreenNormalTransform1.xyz = transMat[1].xyz;
-	vsout.ScreenNormalTransform2.xyz = transMat[2].xyz;
-#		else
-	vsout.ScreenNormalTransform0.xyz = mul(transMat[0].xyz, transpose(tbn));
-	vsout.ScreenNormalTransform1.xyz = mul(transMat[1].xyz, transpose(tbn));
-	vsout.ScreenNormalTransform2.xyz = mul(transMat[2].xyz, transpose(tbn));
-#		endif  // MODELSPACENORMALS
-#	endif      // SKINNED
-
 	vsout.WorldPosition = worldPosition;
 	vsout.PreviousWorldPosition = previousWorldPosition;
 
@@ -344,11 +312,6 @@ VS_OUTPUT main(VS_INPUT input)
 	vsout.FogParam.xyz = lerp(FogNearColor.xyz, FogFarColor.xyz, fogColorParam);
 	vsout.FogParam.w = fogColorParam;
 
-	vsout.World[0] = World[0];
-#	ifdef VR
-	vsout.World[1] = World[1];
-#	endif  // VR
-
 #	if defined(VR)
 	Stereo::VR_OUTPUT VRout = Stereo::GetVRVSOutput(vsout.Position, eyeIndex);
 	vsout.Position = VRout.VRPosition;
@@ -2505,7 +2468,7 @@ PS_OUTPUT main(PS_INPUT input, bool frontFace : SV_IsFrontFace)
 #		if !defined(LIGHT_LIMIT_FIX)
 	[loop] for (uint lightIndex = 0; lightIndex < numLights; lightIndex++)
 	{
-		float3 lightDirection = PointLightPosition[eyeIndex * numLights + lightIndex].xyz - input.InputPosition.xyz;
+		float3 lightDirection = PointLightPosition[eyeIndex * numLights + lightIndex].xyz - input.WorldPosition.xyz;
 		float lightDist = length(lightDirection);
 		float intensityFactor = saturate(lightDist / PointLightPosition[lightIndex].w);
 		if (intensityFactor == 1)
@@ -2623,9 +2586,11 @@ PS_OUTPUT main(PS_INPUT input, bool frontFace : SV_IsFrontFace)
 		float lightShadow = 1.0;
 
 		float shadowComponent = 1.0;
-		if (Permutation::PixelShaderDescriptor & Permutation::LightingFlags::DefShadow && light.lightFlags & LightLimitFix::LightFlags::Shadow) {
-			shadowComponent = shadowColor[light.shadowLightIndex];
-			lightShadow *= shadowComponent;
+		if (Permutation::PixelShaderDescriptor & Permutation::LightingFlags::DefShadow) {
+			if (light.lightFlags & LightLimitFix::LightFlags::Shadow) {
+				shadowComponent = shadowColor[light.shadowLightIndex];
+				lightShadow *= shadowComponent;
+			}
 		}
 
 		float3 normalizedLightDirection = normalize(lightDirection);

src/Features/LightLimitFix.cpp

@@ -289,6 +289,7 @@ void LightLimitFix::BSLightingShader_SetupGeometry_Before(RE::BSRenderPass* a_pa
 		return;
 
 	strictLightDataTemp.NumStrictLights = 0;
+	strictLightDataTemp.ShadowBitMask = 0;
 
 	strictLightDataTemp.RoomIndex = -1;
 	if (!roomNodes.empty()) {
@@ -343,6 +344,13 @@ void LightLimitFix::BSLightingShader_SetupGeometry_GeometrySetupConstantPointLig
 
 		strictLightDataTemp.StrictLights[i] = light;
 	}
+
+	for (uint32_t i = 0; i < a_pass->numShadowLights; i++) {
+		auto bsLight = a_pass->sceneLights[i + 1];
+		auto* shadowLight = static_cast<RE::BSShadowLight*>(bsLight);
+		GET_INSTANCE_MEMBER(shadowLightIndex, shadowLight);
+		strictLightDataTemp.ShadowBitMask |= (1 << shadowLightIndex);
+	}
 }
 
 void LightLimitFix::BSLightingShader_SetupGeometry_After(RE::BSRenderPass*)
@@ -358,15 +366,17 @@ void LightLimitFix::BSLightingShader_SetupGeometry_After(RE::BSRenderPass*)
 
 	auto shadowSceneNode = smState->shadowSceneNode[0];
 
-	const bool isEmpty = strictLightDataTemp.NumStrictLights == 0;
+	const auto isEmpty = strictLightDataTemp.NumStrictLights == 0;
 	const bool isWorld = accumulator->GetRuntimeData().activeShadowSceneNode == shadowSceneNode;
-	const int roomIndex = strictLightDataTemp.RoomIndex;
+	const auto roomIndex = strictLightDataTemp.RoomIndex;
+	const auto shadowBitMask = strictLightDataTemp.ShadowBitMask;
 
-	if (!isEmpty || (isEmpty && !wasEmpty) || isWorld != wasWorld || previousRoomIndex != roomIndex) {
+	if (!isEmpty || (isEmpty && !wasEmpty) || isWorld != wasWorld || previousRoomIndex != roomIndex || shadowBitMask != previousShadowBitMask) {
 		strictLightDataCB->Update(strictLightDataTemp);
 		wasEmpty = isEmpty;
 		wasWorld = isWorld;
 		previousRoomIndex = roomIndex;
+		previousShadowBitMask = shadowBitMask;
 	}
 
 	if (frameChecker.IsNewFrame()) {

src/Features/LightLimitFix.h

@@ -107,7 +107,8 @@ struct LightLimitFix : Feature
 	{
 		uint NumStrictLights;
 		int RoomIndex;
-		uint pad0[2];
+		uint ShadowBitMask;
+		uint pad0;
 		LightData StrictLights[15];
 	};
 
@@ -171,6 +172,8 @@ struct LightLimitFix : Feature
 	bool wasEmpty = false;
 	bool wasWorld = false;
 	int previousRoomIndex = -1;
+	uint previousShadowBitMask = 0;
+
 	Util::FrameChecker frameChecker;
 
 	virtual void SetupResources() override;

src/Hooks.cpp

@@ -1116,16 +1116,23 @@ namespace Hooks
 		// We offset from the base address of the containing function to the start of the patch
 		{
 			logger::info("Patching BSLightingShader::SetupGeometry::updateEyePosition");
-			uintptr_t setupGeometryUpdateRenderSpace = REL::RelocationID(100565, 107300).address() + REL::Relocate(0x76, 0x71, 0x65);
+			auto setupGeometryUpdateRenderSpace = REL::RelocationID(100565, 107300).address();
+
 			if (REL::Module::IsAE()) {
 				std::uint8_t patch[] = { 0x41, 0x83, 0xE7, 0x00 };  // and r15d, 0
-				REL::safe_write(setupGeometryUpdateRenderSpace, patch, sizeof(patch));
+				REL::safe_write(setupGeometryUpdateRenderSpace + 0x71, patch, sizeof(patch));
 			} else if (REL::Module::IsVR()) {
 				std::uint8_t patch[] = { 0x41, 0x83, 0xE4, 0x00 };  // and r12d, 0
-				REL::safe_write(setupGeometryUpdateRenderSpace, patch, sizeof(patch));
+				REL::safe_write(setupGeometryUpdateRenderSpace + 0x65, patch, sizeof(patch));
 			} else {
-				std::uint8_t patch[] = { 0x0F, 0x1F, 0x40, 0x00 };  // 4-byte NOP
-				REL::safe_write(setupGeometryUpdateRenderSpace, patch, sizeof(patch));
+				std::uint8_t patch1[] = { 0xB8, 0x00, 0x00 };  // mov eax, 0
+				REL::safe_write(setupGeometryUpdateRenderSpace + 0x73, patch1, sizeof(patch1));
+
+				std::uint8_t patch2[] = { 0x45, 0x31, 0xC9 };  // xor r9d, r9d (zeros r9d)
+				REL::safe_write(setupGeometryUpdateRenderSpace + 0x36D, patch2, sizeof(patch2));
+
+				std::uint8_t patch3[] = { 0x45, 0x31, 0xC0 };  // xor r8d, r8d (zeros r8d)
+				REL::safe_write(setupGeometryUpdateRenderSpace + 0x378, patch3, sizeof(patch3));
 			}
 		}
 	}