feat: (editor) use stencil outside screen in scene view for development

Assets/ProjectSettings/UISoftMask.asset

@@ -16,6 +16,6 @@ MonoBehaviour:
   m_StereoEnabled: 0
   m_FallbackBehavior: 0
   m_TransformSensitivity: 0
-  m_EnabledInEditMode: 1
+  m_UseStencilOutsideScreen: 1
   m_AutoIncludeShaders: 1
   m_StripShaderVariants: 1

Packages/src/Editor/MaskingShapeEditor.cs

@@ -11,23 +11,24 @@ public class MaskingShapeEditor : Editor
         private SerializedProperty _antiAliasingThreshold;
         private SerializedProperty _maskingMethod;
         private SerializedProperty _showMaskGraphic;
-        private SerializedProperty _softnessRange;
+        private SerializedProperty _softMaskingRange;
 
         protected void OnEnable()
         {
             _maskingMethod = serializedObject.FindProperty("m_MaskingMethod");
             _showMaskGraphic = serializedObject.FindProperty("m_ShowMaskGraphic");
             _alphaHitTest = serializedObject.FindProperty("m_AlphaHitTest");
             _antiAliasingThreshold = serializedObject.FindProperty("m_AntiAliasingThreshold");
-            _softnessRange = serializedObject.FindProperty("m_SoftnessRange");
+            _softMaskingRange = serializedObject.FindProperty("m_SoftMaskingRange");
         }
 
         public override void OnInspectorGUI()
         {
             var current = target as MaskingShape;
             if (!current) return;
 
-            Utils.GetStencilDepthAndMask(current.transform, false, out var mask);
+            var useStencil = UISoftMaskProjectSettings.useStencilOutsideScreen;
+            Utils.GetStencilBits(current.transform, false, useStencil, out var mask, out var _);
             var maskingMode = mask is SoftMask softMask ? softMask.GetActualMaskingMode() : SoftMask.MaskingMode.Normal;
             EditorGUI.BeginDisabledGroup(true);
             EditorGUILayout.EnumPopup(new GUIContent("Masking Mode"), maskingMode);
@@ -43,7 +44,7 @@ public override void OnInspectorGUI()
                 case SoftMask.MaskingMode.SoftMasking:
                     EditorGUILayout.PropertyField(_showMaskGraphic);
                     EditorGUILayout.PropertyField(_alphaHitTest);
-                    EditorGUILayout.PropertyField(_softnessRange);
+                    EditorGUILayout.PropertyField(_softMaskingRange);
                     break;
                 case SoftMask.MaskingMode.AntiAliasing:
                     EditorGUILayout.PropertyField(_alphaHitTest);

Packages/src/Runtime/Coffee.SoftMaskForUGUI.asmdef

@@ -1,13 +1,20 @@
 {
     "name": "Coffee.SoftMaskForUGUI",
-    "references": [],
+    "references": [
+        "Unity.TextMeshPro"
+    ],
     "includePlatforms": [],
     "excludePlatforms": [],
     "versionDefines": [
         {
             "name": "com.unity.modules.vr",
             "expression": "1.0.0",
             "define": "UNITY_MODULE_VR"
+        },
+        {
+            "name": "com.unity.textmeshpro",
+            "expression": "1.0.0",
+            "define": "TMP_ENABLE"
         }
     ]
 }

Packages/src/Runtime/Internal/Extensions/MeshExtensions.cs

@@ -0,0 +1,81 @@
+using UnityEngine;
+
+namespace Coffee.UISoftMaskInternal
+{
+    internal static class MeshExtensions
+    {
+        internal static readonly ObjectPool<Mesh> s_MeshPool = new ObjectPool<Mesh>(
+            () =>
+            {
+                var mesh = new Mesh
+                {
+                    hideFlags = HideFlags.DontSave | HideFlags.NotEditable
+                };
+                mesh.MarkDynamic();
+                return mesh;
+            },
+            mesh => mesh,
+            mesh =>
+            {
+                if (mesh)
+                {
+                    mesh.Clear();
+                }
+            });
+
+        public static Mesh Rent()
+        {
+            return s_MeshPool.Rent();
+        }
+
+        public static void Return(ref Mesh mesh)
+        {
+            s_MeshPool.Return(ref mesh);
+        }
+
+        public static void CopyTo(this Mesh self, Mesh dst)
+        {
+            if (!self || !dst) return;
+
+            var vector3List = ListPool<Vector3>.Rent();
+            var vector4List = ListPool<Vector4>.Rent();
+            var color32List = ListPool<Color32>.Rent();
+            var intList = ListPool<int>.Rent();
+
+            dst.Clear(false);
+
+            self.GetVertices(vector3List);
+            dst.SetVertices(vector3List);
+
+            self.GetTriangles(intList, 0);
+            dst.SetTriangles(intList, 0);
+
+            self.GetNormals(vector3List);
+            dst.SetNormals(vector3List);
+
+            self.GetTangents(vector4List);
+            dst.SetTangents(vector4List);
+
+            self.GetColors(color32List);
+            dst.SetColors(color32List);
+
+            self.GetUVs(0, vector4List);
+            dst.SetUVs(0, vector4List);
+
+            self.GetUVs(1, vector4List);
+            dst.SetUVs(1, vector4List);
+
+            self.GetUVs(2, vector4List);
+            dst.SetUVs(2, vector4List);
+
+            self.GetUVs(3, vector4List);
+            dst.SetUVs(3, vector4List);
+
+            dst.RecalculateBounds();
+            ListPool<Vector3>.Return(ref vector3List);
+            ListPool<Vector4>.Return(ref vector4List);
+            ListPool<Color32>.Return(ref color32List);
+            ListPool<int>.Return(ref intList);
+        }
+    }
+}

Packages/src/Runtime/Internal/Utilities/FrameCache.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using UnityEngine;
 
 namespace Coffee.UISoftMaskInternal
 {
@@ -13,6 +14,14 @@ static FrameCache()
             UIExtraCallbacks.onLateAfterCanvasRebuild += ClearAllCache;
         }
 
+#if UNITY_EDITOR
+        [RuntimeInitializeOnLoadMethod(RuntimeInitializeLoadType.SubsystemRegistration)]
+        private static void Clear()
+        {
+            s_Caches.Clear();
+        }
+#endif
+
         /// <summary>
         /// Tries to retrieve a value from the frame cache with a specified key.
         /// </summary>

Packages/src/Runtime/MaskingShape/MaskingShape.cs

@@ -27,17 +27,16 @@ public enum MaskingMethod
         private bool m_ShowMaskGraphic;
 
         [Tooltip("Enable alpha hit test.")] [SerializeField]
-        public bool m_AlphaHitTest;
+        private bool m_AlphaHitTest;
 
         [Tooltip("Enable anti-alias masking.")] [SerializeField] [Range(0f, 1f)]
-        public float m_AntiAliasingThreshold = 0.5f;
+        private float m_AntiAliasingThreshold;
 
         [Tooltip("The range for soft masking.")]
         [SerializeField]
-        public MinMax01 m_SoftnessRange = new MinMax01(0, 1f);
+        private MinMax01 m_SoftMaskingRange = new MinMax01(0, 1f);
 
         private bool _antiAliasingRegistered;
-
         private MaskingShapeContainer _container;
         private Graphic _graphic;
         private Mask _mask;
@@ -47,7 +46,7 @@ public enum MaskingMethod
         private Matrix4x4 _prevTransformMatrix;
         private UnityAction _setContainerDirty;
         private bool _shouldRecalculateStencil;
-        private int _stencilDepth;
+        private int _stencilBits;
         private Action _updateAntiAliasing;
         private UnityAction _updateContainer;
 
@@ -100,14 +99,14 @@ public float antiAliasingThreshold
         /// <summary>
         /// The range for soft masking.
         /// </summary>
-        public MinMax01 softnessRange
+        public MinMax01 softMaskingRange
         {
-            get => m_SoftnessRange;
+            get => m_SoftMaskingRange;
             set
             {
-                if (m_SoftnessRange.Approximately(value)) return;
+                if (m_SoftMaskingRange.Approximately(value)) return;
 
-                m_SoftnessRange = value;
+                m_SoftMaskingRange = value;
                 SetContainerDirty();
             }
         }
@@ -136,7 +135,7 @@ protected override void OnDisable()
             StencilMaterial.Remove(_maskMaterial);
             _maskMaterial = null;
 
-            SoftMaskUtils.meshPool.Return(ref _mesh);
+            MeshExtensions.Return(ref _mesh);
             SoftMaskUtils.materialPropertyBlockPool.Return(ref _mpb);
 
             SetContainerDirty();
@@ -220,26 +219,24 @@ Material IMaterialModifier.GetModifiedMaterial(Material baseMaterial)
 
             // Not in mask.
             RecalculateStencilIfNeeded();
-            if (_stencilDepth <= 0)
+            if (_stencilBits == 0 && !_mask)
             {
                 StencilMaterial.Remove(_maskMaterial);
                 _maskMaterial = null;
                 return null;
             }
 
-            var colorMask = m_ShowMaskGraphic ? ColorWriteMask.All : 0;
-            var stencilBit = 1 << (_stencilDepth - 1);
-
             // Mask material
             Material maskMat = null;
-            if (SoftMaskingEnabled())
+            var colorMask = m_ShowMaskGraphic ? ColorWriteMask.All : 0;
+            if (SoftMaskingEnabled() && !UISoftMaskProjectSettings.useStencilOutsideScreen)
             {
                 if (m_ShowMaskGraphic)
                 {
                     Profiler.BeginSample(
                         "(SM4UI)[MaskingShape)] GetModifiedMaterial > StencilMaterial.Add for SoftMask");
-                    maskMat = StencilMaterial.Add(baseMaterial, stencilBit, StencilOp.Keep, CompareFunction.Equal,
-                        colorMask, stencilBit, stencilBit);
+                    maskMat = StencilMaterial.Add(baseMaterial, _stencilBits, StencilOp.Keep, CompareFunction.Equal,
+                        colorMask, _stencilBits, _stencilBits);
                     Profiler.EndSample();
                 }
             }
@@ -249,12 +246,12 @@ Material IMaterialModifier.GetModifiedMaterial(Material baseMaterial)
                 switch (maskingMethod)
                 {
                     case MaskingMethod.Additive:
-                        maskMat = StencilMaterial.Add(baseMaterial, stencilBit, StencilOp.Replace,
-                            CompareFunction.NotEqual, colorMask, stencilBit, stencilBit);
+                        maskMat = StencilMaterial.Add(baseMaterial, _stencilBits, StencilOp.Replace,
+                            CompareFunction.NotEqual, colorMask, _stencilBits, _stencilBits);
                         break;
                     case MaskingMethod.Subtract:
-                        maskMat = StencilMaterial.Add(baseMaterial, stencilBit, StencilOp.Invert,
-                            CompareFunction.Equal, colorMask, stencilBit, stencilBit);
+                        maskMat = StencilMaterial.Add(baseMaterial, _stencilBits, StencilOp.Invert,
+                            CompareFunction.Equal, colorMask, _stencilBits, _stencilBits);
                         break;
                 }
 
@@ -278,7 +275,7 @@ void IMeshModifier.ModifyMesh(VertexHelper verts)
             Profiler.BeginSample("(SM4UI)[MaskingShape)] ModifyMesh");
             if (!_mesh)
             {
-                _mesh = SoftMaskUtils.meshPool.Rent();
+                _mesh = MeshExtensions.Rent();
             }
 
             _mesh.Clear(false);
@@ -306,13 +303,14 @@ private void RecalculateStencilIfNeeded()
             if (!isActiveAndEnabled)
             {
                 _mask = null;
-                _stencilDepth = -1;
+                _stencilBits = 0;
                 return;
             }
 
             if (!_shouldRecalculateStencil) return;
             _shouldRecalculateStencil = false;
-            _stencilDepth = Utils.GetStencilDepthAndMask(transform, true, out _mask);
+            var useStencil = UISoftMaskProjectSettings.useStencilOutsideScreen;
+            _stencilBits = Utils.GetStencilBits(transform, true, useStencil, out _mask, out var _);
         }
 
         private void SetContainerDirty()
@@ -328,7 +326,8 @@ private void UpdateContainer()
             Mask mask = null;
             if (isActiveAndEnabled)
             {
-                Utils.GetStencilDepthAndMask(transform, false, out mask);
+                var useStencil = UISoftMaskProjectSettings.useStencilOutsideScreen;
+                Utils.GetStencilBits(transform, false, useStencil, out mask, out var _);
             }
 
             var newContainer = mask.GetOrAddComponent<MaskingShapeContainer>();
@@ -373,7 +372,9 @@ internal bool IsInside(Vector2 sp, Camera eventCamera, float threshold = 0.01f)
 
         internal void DrawSoftMaskBuffer(CommandBuffer cb, int depth)
         {
-            if (!_mesh) return;
+            var texture = graphic.mainTexture;
+            var mesh = _mesh;
+            if (!mesh) return;
             if (!graphic.IsInScreen()) return;
 
             Profiler.BeginSample("(SM4UI)[MaskingShape)] DrawSoftMaskBuffer");
@@ -382,10 +383,10 @@ internal void DrawSoftMaskBuffer(CommandBuffer cb, int depth)
                 _mpb = SoftMaskUtils.materialPropertyBlockPool.Rent();
             }
 
-            SoftMaskUtils.ApplyMaterialPropertyBlock(_mpb, depth, graphic.mainTexture, softnessRange);
+            SoftMaskUtils.ApplyMaterialPropertyBlock(_mpb, depth, texture, softMaskingRange);
             var softMaterial = SoftMaskUtils.GetSoftMaskingMaterial(maskingMethod);
 
-            cb.DrawMesh(_mesh, transform.localToWorldMatrix, softMaterial, 0, 0, _mpb);
+            cb.DrawMesh(mesh, transform.localToWorldMatrix, softMaterial, 0, 0, _mpb);
             Profiler.EndSample();
         }