[Backport 8.x.x] Fixed decal duplication when editing a prefab in context (#1107)

JulienIgnace-Unity · sebastienlagarde · web-flow · commit da56fbd05c3c · 2020-07-07T20:01:42.000+02:00
* Implemented proper culling for decals when used in prefabs and edited in context.

* Compilation fix

* Update changelog

Co-authored-by: sebastienlagarde &lt;sebastien@unity3d.com&gt;
diff --git a/com.unity.render-pipelines.high-definition/CHANGELOG.md b/com.unity.render-pipelines.high-definition/CHANGELOG.md
@@ -73,6 +73,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - Fixed issue with depth pyramid generation and dynamic resolution.
 - Fixed a serialization issue, preventing quality level parameters to undo/redo and update scene view on change.
 - Fixed an issue where opening the look dev window with the light theme would make the window blink and eventually crash unity.
+- Fixed culling for decals when used in prefabs and edited in context.
 
 ### Changed
 - Shadowmask and realtime reflection probe property are hide in Quality settings
diff --git a/com.unity.render-pipelines.high-definition/Editor/Material/Decal/DecalProjectorEditor.cs b/com.unity.render-pipelines.high-definition/Editor/Material/Decal/DecalProjectorEditor.cs
@@ -259,7 +259,7 @@ void DrawHandles()
                     {
                         // Smoothly update the decal image projected
                         Matrix4x4 sizeOffset = Matrix4x4.Translate(decalProjector.decalOffset) * Matrix4x4.Scale(decalProjector.decalSize);
-                        DecalSystem.instance.UpdateCachedData(decalProjector.position, decalProjector.rotation, sizeOffset, decalProjector.drawDistance, decalProjector.fadeScale, decalProjector.uvScaleBias, decalProjector.affectsTransparency, decalProjector.Handle, decalProjector.gameObject.layer, HDUtils.GetSceneCullingMaskFromGameObject(decalProjector.gameObject), decalProjector.fadeFactor);
+                        DecalSystem.instance.UpdateCachedData(decalProjector.position, decalProjector.rotation, sizeOffset, decalProjector.drawDistance, decalProjector.fadeScale, decalProjector.uvScaleBias, decalProjector.affectsTransparency, decalProjector.Handle, decalProjector.gameObject.layer, decalProjector.gameObject.sceneCullingMask, decalProjector.fadeFactor);
                     }
                 }
             }
diff --git a/com.unity.render-pipelines.high-definition/Runtime/Material/Decal/DecalProjector.Migration.cs b/com.unity.render-pipelines.high-definition/Runtime/Material/Decal/DecalProjector.Migration.cs
@@ -37,7 +37,7 @@ enum Version
                 if (decal.m_Handle != null)
                     DecalSystem.instance.RemoveDecal(decal.m_Handle);
                 Matrix4x4 sizeOffset = Matrix4x4.Translate(decal.decalOffset) * Matrix4x4.Scale(decal.decalSize);
-                decal.m_Handle = DecalSystem.instance.AddDecal(decal.position, decal.rotation, Vector3.one, sizeOffset, decal.m_DrawDistance, decal.m_FadeScale, decal.uvScaleBias, decal.m_AffectsTransparency, decal.m_Material, decal.gameObject.layer, HDUtils.GetSceneCullingMaskFromGameObject(decal.gameObject), decal.m_FadeFactor);
+                decal.m_Handle = DecalSystem.instance.AddDecal(decal.position, decal.rotation, Vector3.one, sizeOffset, decal.m_DrawDistance, decal.m_FadeScale, decal.uvScaleBias, decal.m_AffectsTransparency, decal.m_Material, decal.gameObject.layer, decal.gameObject.sceneCullingMask, decal.m_FadeFactor);
             }),
             MigrationStep.New(Version.FixPivotPosition, (DecalProjector decal) =>
             {
@@ -69,7 +69,7 @@ enum Version
                 if (decal.m_Handle != null)
                     DecalSystem.instance.RemoveDecal(decal.m_Handle);
                 Matrix4x4 sizeOffset = Matrix4x4.Translate(decal.decalOffset) * Matrix4x4.Scale(decal.decalSize);
-                decal.m_Handle = DecalSystem.instance.AddDecal(decal.position, decal.rotation, Vector3.one, sizeOffset, decal.m_DrawDistance, decal.m_FadeScale, decal.uvScaleBias, decal.m_AffectsTransparency, decal.m_Material, decal.gameObject.layer, HDUtils.GetSceneCullingMaskFromGameObject(decal.gameObject), decal.m_FadeFactor);
+                decal.m_Handle = DecalSystem.instance.AddDecal(decal.position, decal.rotation, Vector3.one, sizeOffset, decal.m_DrawDistance, decal.m_FadeScale, decal.uvScaleBias, decal.m_AffectsTransparency, decal.m_Material, decal.gameObject.layer, decal.gameObject.sceneCullingMask, decal.m_FadeFactor);
             })
         );
 
diff --git a/com.unity.render-pipelines.high-definition/Runtime/Material/Decal/DecalProjector.cs b/com.unity.render-pipelines.high-definition/Runtime/Material/Decal/DecalProjector.cs
@@ -238,7 +238,7 @@ void OnEnable()
             }
 
             Matrix4x4 sizeOffset = Matrix4x4.Translate(decalOffset) * Matrix4x4.Scale(decalSize);
-            m_Handle = DecalSystem.instance.AddDecal(position, rotation, Vector3.one, sizeOffset, m_DrawDistance, m_FadeScale, uvScaleBias, m_AffectsTransparency, m_Material, gameObject.layer, HDUtils.GetSceneCullingMaskFromGameObject(gameObject), m_FadeFactor);
+            m_Handle = DecalSystem.instance.AddDecal(position, rotation, Vector3.one, sizeOffset, m_DrawDistance, m_FadeScale, uvScaleBias, m_AffectsTransparency, m_Material, gameObject.layer, gameObject.sceneCullingMask, m_FadeFactor);
             m_OldMaterial = m_Material;
 
 #if UNITY_EDITOR
@@ -260,7 +260,13 @@ void UpdateDecalVisibility()
             else if (m_Handle == null)
             {
                 Matrix4x4 sizeOffset = Matrix4x4.Translate(decalOffset) * Matrix4x4.Scale(decalSize);
-                m_Handle = DecalSystem.instance.AddDecal(position, rotation, Vector3.one, sizeOffset, m_DrawDistance, m_FadeScale, uvScaleBias, m_AffectsTransparency, m_Material, gameObject.layer, HDUtils.GetSceneCullingMaskFromGameObject(gameObject), m_FadeFactor);
+                m_Handle = DecalSystem.instance.AddDecal(position, rotation, Vector3.one, sizeOffset, m_DrawDistance, m_FadeScale, uvScaleBias, m_AffectsTransparency, m_Material, gameObject.layer, gameObject.sceneCullingMask, m_FadeFactor);
+            }
+            else
+            {
+                // Scene culling mask may have changed.
+                Matrix4x4 sizeOffset = Matrix4x4.Translate(decalOffset) * Matrix4x4.Scale(decalSize);
+                DecalSystem.instance.UpdateCachedData(position, rotation, sizeOffset, m_DrawDistance, m_FadeScale, uvScaleBias, m_AffectsTransparency, m_Handle, gameObject.layer, gameObject.sceneCullingMask, m_FadeFactor);
             }
         }
 #endif
@@ -299,7 +305,7 @@ internal void OnValidate()
 
                     if (m_Material != null)
                     {
-                        m_Handle = DecalSystem.instance.AddDecal(position, rotation, Vector3.one, sizeOffset, m_DrawDistance, m_FadeScale, uvScaleBias, m_AffectsTransparency, m_Material, gameObject.layer, HDUtils.GetSceneCullingMaskFromGameObject(gameObject), m_FadeFactor);
+                        m_Handle = DecalSystem.instance.AddDecal(position, rotation, Vector3.one, sizeOffset, m_DrawDistance, m_FadeScale, uvScaleBias, m_AffectsTransparency, m_Material, gameObject.layer, gameObject.sceneCullingMask, m_FadeFactor);
 
                         if (!DecalSystem.IsHDRenderPipelineDecal(m_Material.shader)) // non HDRP/decal shaders such as shader graph decal do not affect transparency
                         {
@@ -317,7 +323,7 @@ internal void OnValidate()
                 }
                 else // no material change, just update whatever else changed
                 {
-                    DecalSystem.instance.UpdateCachedData(position, rotation, sizeOffset, m_DrawDistance, m_FadeScale, uvScaleBias, m_AffectsTransparency, m_Handle, gameObject.layer, HDUtils.GetSceneCullingMaskFromGameObject(gameObject), m_FadeFactor);
+                    DecalSystem.instance.UpdateCachedData(position, rotation, sizeOffset, m_DrawDistance, m_FadeScale, uvScaleBias, m_AffectsTransparency, m_Handle, gameObject.layer, gameObject.sceneCullingMask, m_FadeFactor);
                 }
             }
         }
@@ -329,7 +335,7 @@ void Update() // only run in editor
             {
                 Matrix4x4 sizeOffset = Matrix4x4.Translate(decalOffset) * Matrix4x4.Scale(decalSize);
                 m_Layer = gameObject.layer;
-                DecalSystem.instance.UpdateCachedData(position, rotation, sizeOffset, m_DrawDistance, m_FadeScale, uvScaleBias, m_AffectsTransparency, m_Handle, gameObject.layer, HDUtils.GetSceneCullingMaskFromGameObject(gameObject), m_FadeFactor);
+                DecalSystem.instance.UpdateCachedData(position, rotation, sizeOffset, m_DrawDistance, m_FadeScale, uvScaleBias, m_AffectsTransparency, m_Handle, gameObject.layer, gameObject.sceneCullingMask, m_FadeFactor);
             }
         }
 #endif
@@ -341,7 +347,7 @@ void LateUpdate()
                 if (transform.hasChanged == true)
                 {
                     Matrix4x4 sizeOffset = Matrix4x4.Translate(decalOffset) * Matrix4x4.Scale(decalSize);
-                    DecalSystem.instance.UpdateCachedData(position, rotation, sizeOffset, m_DrawDistance, m_FadeScale, uvScaleBias, m_AffectsTransparency, m_Handle, gameObject.layer, HDUtils.GetSceneCullingMaskFromGameObject(gameObject), m_FadeFactor);
+                    DecalSystem.instance.UpdateCachedData(position, rotation, sizeOffset, m_DrawDistance, m_FadeScale, uvScaleBias, m_AffectsTransparency, m_Handle, gameObject.layer, gameObject.sceneCullingMask, m_FadeFactor);
                     transform.hasChanged = false;
                 }
             }
diff --git a/com.unity.render-pipelines.high-definition/Runtime/Material/Decal/DecalSystem.cs b/com.unity.render-pipelines.high-definition/Runtime/Material/Decal/DecalSystem.cs
@@ -496,6 +496,7 @@ public DecalHandle AddDecal(Matrix4x4 localToWorld, Quaternion rotation, Matrix4
                     m_CachedDrawDistances = newCachedDrawDistances;
                     m_CachedUVScaleBias = newCachedUVScaleBias;
                     m_CachedAffectsTransparency = newCachedAffectsTransparency;
+                    m_CachedLayerMask = newCachedLayerMask;
                     m_CachedSceneLayerMask = newCachedSceneLayerMask;
                     m_CachedFadeFactor = newCachedFadeFactor;
                 }
diff --git a/com.unity.render-pipelines.high-definition/Runtime/RenderPipeline/Utility/HDUtils.cs b/com.unity.render-pipelines.high-definition/Runtime/RenderPipeline/Utility/HDUtils.cs
@@ -942,15 +942,6 @@ internal static UInt64 GetSceneCullingMaskFromCamera(Camera camera)
 
         }
 
-        internal static ulong GetSceneCullingMaskFromGameObject(GameObject go)
-        {
-#if UNITY_EDITOR
-            return (go.scene == null) ? ~(ulong)0 : EditorSceneManager.GetSceneCullingMask(go.scene);
-#else
-            return 0;
-#endif
-        }
-
         internal static HDAdditionalCameraData TryGetAdditionalCameraDataOrDefault(Camera camera)
         {
             if (camera == null || camera.Equals(null))

Original file line number	Diff line number	Diff line change
`@@ -259,7 +259,7 @@ void DrawHandles()`
`259`	`259`	`{`
`260`	`260`	`// Smoothly update the decal image projected`
`261`	`261`	`Matrix4x4 sizeOffset = Matrix4x4.Translate(decalProjector.decalOffset) * Matrix4x4.Scale(decalProjector.decalSize);`
`262`		`- DecalSystem.instance.UpdateCachedData(decalProjector.position, decalProjector.rotation, sizeOffset, decalProjector.drawDistance, decalProjector.fadeScale, decalProjector.uvScaleBias, decalProjector.affectsTransparency, decalProjector.Handle, decalProjector.gameObject.layer, HDUtils.GetSceneCullingMaskFromGameObject(decalProjector.gameObject), decalProjector.fadeFactor);`
	`262`	`+ DecalSystem.instance.UpdateCachedData(decalProjector.position, decalProjector.rotation, sizeOffset, decalProjector.drawDistance, decalProjector.fadeScale, decalProjector.uvScaleBias, decalProjector.affectsTransparency, decalProjector.Handle, decalProjector.gameObject.layer, decalProjector.gameObject.sceneCullingMask, decalProjector.fadeFactor);`
`263`	`263`	`}`
`264`	`264`	`}`
`265`	`265`	`}`
Original file line number	Diff line number	Diff line change
`@@ -238,7 +238,7 @@ void OnEnable()`
`238`	`238`	`}`
`239`	`239`
`240`	`240`	`Matrix4x4 sizeOffset = Matrix4x4.Translate(decalOffset) * Matrix4x4.Scale(decalSize);`
`241`		`- m_Handle = DecalSystem.instance.AddDecal(position, rotation, Vector3.one, sizeOffset, m_DrawDistance, m_FadeScale, uvScaleBias, m_AffectsTransparency, m_Material, gameObject.layer, HDUtils.GetSceneCullingMaskFromGameObject(gameObject), m_FadeFactor);`
	`241`	`+ m_Handle = DecalSystem.instance.AddDecal(position, rotation, Vector3.one, sizeOffset, m_DrawDistance, m_FadeScale, uvScaleBias, m_AffectsTransparency, m_Material, gameObject.layer, gameObject.sceneCullingMask, m_FadeFactor);`
`242`	`242`	`m_OldMaterial = m_Material;`
`243`	`243`
`244`	`244`	`#if UNITY_EDITOR`
`@@ -260,7 +260,13 @@ void UpdateDecalVisibility()`
`260`	`260`	`else if (m_Handle == null)`
`261`	`261`	`{`
`262`	`262`	`Matrix4x4 sizeOffset = Matrix4x4.Translate(decalOffset) * Matrix4x4.Scale(decalSize);`
`263`		`- m_Handle = DecalSystem.instance.AddDecal(position, rotation, Vector3.one, sizeOffset, m_DrawDistance, m_FadeScale, uvScaleBias, m_AffectsTransparency, m_Material, gameObject.layer, HDUtils.GetSceneCullingMaskFromGameObject(gameObject), m_FadeFactor);`
	`263`	`+ m_Handle = DecalSystem.instance.AddDecal(position, rotation, Vector3.one, sizeOffset, m_DrawDistance, m_FadeScale, uvScaleBias, m_AffectsTransparency, m_Material, gameObject.layer, gameObject.sceneCullingMask, m_FadeFactor);`
	`264`	`+ }`
	`265`	`+ else`
	`266`	`+ {`
	`267`	`+ // Scene culling mask may have changed.`
	`268`	`+ Matrix4x4 sizeOffset = Matrix4x4.Translate(decalOffset) * Matrix4x4.Scale(decalSize);`
	`269`	`+ DecalSystem.instance.UpdateCachedData(position, rotation, sizeOffset, m_DrawDistance, m_FadeScale, uvScaleBias, m_AffectsTransparency, m_Handle, gameObject.layer, gameObject.sceneCullingMask, m_FadeFactor);`
`264`	`270`	`}`
`265`	`271`	`}`
`266`	`272`	`#endif`
`@@ -299,7 +305,7 @@ internal void OnValidate()`
`299`	`305`
`300`	`306`	`if (m_Material != null)`
`301`	`307`	`{`
`302`		`- m_Handle = DecalSystem.instance.AddDecal(position, rotation, Vector3.one, sizeOffset, m_DrawDistance, m_FadeScale, uvScaleBias, m_AffectsTransparency, m_Material, gameObject.layer, HDUtils.GetSceneCullingMaskFromGameObject(gameObject), m_FadeFactor);`
	`308`	`+ m_Handle = DecalSystem.instance.AddDecal(position, rotation, Vector3.one, sizeOffset, m_DrawDistance, m_FadeScale, uvScaleBias, m_AffectsTransparency, m_Material, gameObject.layer, gameObject.sceneCullingMask, m_FadeFactor);`
`303`	`309`
`304`	`310`	`if (!DecalSystem.IsHDRenderPipelineDecal(m_Material.shader)) // non HDRP/decal shaders such as shader graph decal do not affect transparency`
`305`	`311`	`{`
`@@ -317,7 +323,7 @@ internal void OnValidate()`
`317`	`323`	`}`
`318`	`324`	`else // no material change, just update whatever else changed`
`319`	`325`	`{`
`320`		`- DecalSystem.instance.UpdateCachedData(position, rotation, sizeOffset, m_DrawDistance, m_FadeScale, uvScaleBias, m_AffectsTransparency, m_Handle, gameObject.layer, HDUtils.GetSceneCullingMaskFromGameObject(gameObject), m_FadeFactor);`
	`326`	`+ DecalSystem.instance.UpdateCachedData(position, rotation, sizeOffset, m_DrawDistance, m_FadeScale, uvScaleBias, m_AffectsTransparency, m_Handle, gameObject.layer, gameObject.sceneCullingMask, m_FadeFactor);`
`321`	`327`	`}`
`322`	`328`	`}`
`323`	`329`	`}`
`@@ -329,7 +335,7 @@ void Update() // only run in editor`
`329`	`335`	`{`
`330`	`336`	`Matrix4x4 sizeOffset = Matrix4x4.Translate(decalOffset) * Matrix4x4.Scale(decalSize);`
`331`	`337`	`m_Layer = gameObject.layer;`
`332`		`- DecalSystem.instance.UpdateCachedData(position, rotation, sizeOffset, m_DrawDistance, m_FadeScale, uvScaleBias, m_AffectsTransparency, m_Handle, gameObject.layer, HDUtils.GetSceneCullingMaskFromGameObject(gameObject), m_FadeFactor);`
	`338`	`+ DecalSystem.instance.UpdateCachedData(position, rotation, sizeOffset, m_DrawDistance, m_FadeScale, uvScaleBias, m_AffectsTransparency, m_Handle, gameObject.layer, gameObject.sceneCullingMask, m_FadeFactor);`
`333`	`339`	`}`
`334`	`340`	`}`
`335`	`341`	`#endif`
`@@ -341,7 +347,7 @@ void LateUpdate()`
`341`	`347`	`if (transform.hasChanged == true)`
`342`	`348`	`{`
`343`	`349`	`Matrix4x4 sizeOffset = Matrix4x4.Translate(decalOffset) * Matrix4x4.Scale(decalSize);`
`344`		`- DecalSystem.instance.UpdateCachedData(position, rotation, sizeOffset, m_DrawDistance, m_FadeScale, uvScaleBias, m_AffectsTransparency, m_Handle, gameObject.layer, HDUtils.GetSceneCullingMaskFromGameObject(gameObject), m_FadeFactor);`
	`350`	`+ DecalSystem.instance.UpdateCachedData(position, rotation, sizeOffset, m_DrawDistance, m_FadeScale, uvScaleBias, m_AffectsTransparency, m_Handle, gameObject.layer, gameObject.sceneCullingMask, m_FadeFactor);`
`345`	`351`	`transform.hasChanged = false;`
`346`	`352`	`}`
`347`	`353`	`}`
Original file line number	Diff line number	Diff line change
`@@ -496,6 +496,7 @@ public DecalHandle AddDecal(Matrix4x4 localToWorld, Quaternion rotation, Matrix4`
`496`	`496`	`m_CachedDrawDistances = newCachedDrawDistances;`
`497`	`497`	`m_CachedUVScaleBias = newCachedUVScaleBias;`
`498`	`498`	`m_CachedAffectsTransparency = newCachedAffectsTransparency;`
	`499`	`+ m_CachedLayerMask = newCachedLayerMask;`
`499`	`500`	`m_CachedSceneLayerMask = newCachedSceneLayerMask;`
`500`	`501`	`m_CachedFadeFactor = newCachedFadeFactor;`
`501`	`502`	`}`
Original file line number	Diff line number	Diff line change
`@@ -942,15 +942,6 @@ internal static UInt64 GetSceneCullingMaskFromCamera(Camera camera)`
`942`	`942`
`943`	`943`	`}`
`944`	`944`
`945`		`- internal static ulong GetSceneCullingMaskFromGameObject(GameObject go)`
`946`		`- {`
`947`		`-#if UNITY_EDITOR`
`948`		`- return (go.scene == null) ? ~(ulong)0 : EditorSceneManager.GetSceneCullingMask(go.scene);`
`949`		`-#else`
`950`		`- return 0;`
`951`		`-#endif`
`952`		`- }`
`953`		`-`
`954`	`945`	`internal static HDAdditionalCameraData TryGetAdditionalCameraDataOrDefault(Camera camera)`
`955`	`946`	`{`
`956`	`947`	`if (camera == null \|\| camera.Equals(null))`