Fix blur background halo effect (#415)

Author: royshil

data/effects/kawase_blur.effect

@@ -71,6 +71,10 @@ float4 PSKawaseFocalBlur(VertDataOut v_in) : TARGET
 	return sum;
 }
 
+/**
+ * Standard Kawase blur
+ * While it's not being used, we keep it here for reference.
+ */
 float4 PSKawaseBlur(VertDataOut v_in) : TARGET
 {
 	// Calculate the blur value from neighboring pixels
@@ -83,6 +87,41 @@ float4 PSKawaseBlur(VertDataOut v_in) : TARGET
 	return sum;
 }
 
+/**
+ * Mask aware Kawase blur
+ * Only uses pixels which are in the masked area for blur. This prevents the "Halo Effect" on
+ * the border pixels of the mask.
+ */
+float4 PSKawaseBlurMaskAware(VertDataOut v_in) : TARGET
+{
+	if (focalmask.Sample(textureSampler, v_in.uv).r == 0) {
+		// No mask - return the original image value without any blur
+		return image.Sample(textureSampler, v_in.uv);
+	}
+
+	// Calculate the blur value from neighboring pixels
+
+	float alphaValue1 = focalmask.Sample(textureSampler, v_in.uv + float2( xOffset,  yOffset)).r;
+	float4 sum = image.Sample(textureSampler, v_in.uv + float2( xOffset,  yOffset)) * alphaValue1;
+	float pixelCounter = alphaValue1;
+
+	float alphaValue2 = focalmask.Sample(textureSampler, v_in.uv + float2(-xOffset,  yOffset)).r;
+	sum += image.Sample(textureSampler, v_in.uv + float2(-xOffset,  yOffset)) * alphaValue2;
+	pixelCounter += alphaValue2;
+
+	float alphaValue3 = focalmask.Sample(textureSampler, v_in.uv + float2( xOffset, -yOffset)).r;
+	sum += image.Sample(textureSampler, v_in.uv + float2( xOffset, -yOffset)) * alphaValue3;
+	pixelCounter += alphaValue3;
+
+	float alphaValue4 = focalmask.Sample(textureSampler, v_in.uv + float2(-xOffset, -yOffset)).r;
+	sum += image.Sample(textureSampler, v_in.uv + float2(-xOffset, -yOffset)) * alphaValue4;
+	pixelCounter += alphaValue4;
+
+	// Complement the blur pixels with a relative fraction of the center pixel
+	return (sum + image.Sample(textureSampler, v_in.uv) * (4.0 - pixelCounter)) * 0.25;
+}
+
+
 technique DrawFocalBlur
 {
 	pass
@@ -97,6 +136,6 @@ technique Draw
 	pass
 	{
 		vertex_shader = VSDefault(v_in);
-		pixel_shader  = PSKawaseBlur(v_in);
+		pixel_shader  = PSKawaseBlurMaskAware(v_in);
 	}
 }

data/effects/mask_alpha_filter.effect

@@ -40,9 +40,9 @@ float4 PSAlphaMaskRGBAWithBlur(VertDataOut v_in) : TARGET
 	return outputRGBA;
 }
 
-float4 PSAlphaMaskRGBAWithFocalBlur(VertDataOut v_in) : TARGET
+float4 PSTakeBlur(VertDataOut v_in) : TARGET
 {
-	// Return the blurred image, the focal mask is already applied to the blurred image
+	// Return the blurred image, assume any masking is already applied to the blurred image
 	return float4(blurredBackground.Sample(textureSampler, v_in.uv).rgb, 1.0);
 }
 
@@ -63,7 +63,7 @@ technique DrawWithBlur
 	pass
 	{
 		vertex_shader = VSDefault(v_in);
-		pixel_shader  = PSAlphaMaskRGBAWithBlur(v_in);
+		pixel_shader  = PSTakeBlur(v_in);
 	}
 }
 
@@ -72,7 +72,7 @@ technique DrawWithFocalBlur
 	pass
 	{
 		vertex_shader = VSDefault(v_in);
-		pixel_shader  = PSAlphaMaskRGBAWithFocalBlur(v_in);
+		pixel_shader  = PSTakeBlur(v_in);
 	}
 }