Update CharUnicodeInfo to Unicode 12.1; update StringInfo to UAX29

eng/Versions.props

@@ -61,6 +61,7 @@
     <SystemIOCompressionTestDataVersion>5.0.0-beta.19608.5</SystemIOCompressionTestDataVersion>
     <SystemIOPackagingTestDataVersion>5.0.0-beta.19608.5</SystemIOPackagingTestDataVersion>
     <SystemNetTestDataVersion>5.0.0-beta.19608.5</SystemNetTestDataVersion>
+    <SystemPrivateRuntimeUnicodeDataVersion>5.0.0-beta.19610.1</SystemPrivateRuntimeUnicodeDataVersion>
     <SystemSecurityCryptographyX509CertificatesTestDataVersion>5.0.0-beta.19608.5</SystemSecurityCryptographyX509CertificatesTestDataVersion>
     <SystemWindowsExtensionsTestDataVersion>5.0.0-beta.19608.5</SystemWindowsExtensionsTestDataVersion>
     <!-- Standard dependencies -->

src/coreclr/src/System.Private.CoreLib/Tools/GenUnicodeProp/CategoryCasingInfo.cs

@@ -0,0 +1,119 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Buffers.Binary;
+using System.Globalization;
+using System.Text;
+using System.Text.Unicode;
+
+namespace GenUnicodeProp
+{
+    /// <summary>
+    /// Contains information about a code point's Unicode category,
+    /// bidi class, and simple case mapping / folding.
+    /// </summary>
+    internal sealed class CategoryCasingInfo : IEquatable<CategoryCasingInfo>
+    {
+        private readonly (UnicodeCategory generalCategory,
+            StrongBidiCategory strongBidiCategory,
+            ushort offsetToSimpleUppercase,
+            ushort offsetToSimpleLowercase,
+            ushort offsetToSimpleTitlecase,
+            ushort offsetToSimpleCasefold,
+            bool isWhitespace) _data;
+
+        public CategoryCasingInfo(CodePoint codePoint)
+        {
+            _data.generalCategory = codePoint.GeneralCategory;
+
+            switch (codePoint.BidiClass)
+            {
+                case BidiClass.Left_To_Right:
+                    _data.strongBidiCategory = StrongBidiCategory.StrongLeftToRight;
+                    break;
+
+                case BidiClass.Right_To_Left:
+                case BidiClass.Arabic_Letter:
+                    _data.strongBidiCategory = StrongBidiCategory.StrongRightToLeft;
+                    break;
+
+                default:
+                    _data.strongBidiCategory = StrongBidiCategory.Other;
+                    break;
+            }
+
+            if (Program.IncludeCasingData)
+            {
+                _data.offsetToSimpleUppercase = (ushort)(codePoint.SimpleUppercaseMapping - codePoint.Value);
+                _data.offsetToSimpleLowercase = (ushort)(codePoint.SimpleLowercaseMapping - codePoint.Value);
+                _data.offsetToSimpleTitlecase = (ushort)(codePoint.SimpleTitlecaseMapping - codePoint.Value);
+                _data.offsetToSimpleCasefold = (ushort)(codePoint.SimpleCaseFoldMapping - codePoint.Value);
+            }
+            else
+            {
+                _data.offsetToSimpleUppercase = default;
+                _data.offsetToSimpleLowercase = default;
+                _data.offsetToSimpleTitlecase = default;
+                _data.offsetToSimpleCasefold = default;
+            }
+
+            _data.isWhitespace = codePoint.Flags.HasFlag(CodePointFlags.White_Space);
+        }
+
+        public override bool Equals(object obj) => Equals(obj as CategoryCasingInfo);
+
+        public bool Equals(CategoryCasingInfo other)
+        {
+            return !(other is null) && this._data.Equals(other._data);
+        }
+
+        public override int GetHashCode()
+        {
+            return _data.GetHashCode();
+        }
+
+        public static byte[] ToCategoryBytes(CategoryCasingInfo input)
+        {
+            // We're storing 3 pieces of information in 8 bits:
+            // bit 7 (high bit) = isWhitespace?
+            // bits 6..5 = restricted bidi class
+            // bits 4..0 = Unicode category
+
+            int combinedValue = Convert.ToInt32(input._data.isWhitespace) << 7;
+            combinedValue += (int)input._data.strongBidiCategory << 5;
+            combinedValue += (int)input._data.generalCategory;
+
+            return new byte[] { checked((byte)combinedValue) };
+        }
+
+        public static byte[] ToUpperBytes(CategoryCasingInfo input)
+        {
+            byte[] bytes = new byte[sizeof(ushort)];
+            BinaryPrimitives.WriteUInt16LittleEndian(bytes, input._data.offsetToSimpleUppercase);
+            return bytes;
+        }
+
+        public static byte[] ToLowerBytes(CategoryCasingInfo input)
+        {
+            byte[] bytes = new byte[sizeof(ushort)];
+            BinaryPrimitives.WriteUInt16LittleEndian(bytes, input._data.offsetToSimpleLowercase);
+            return bytes;
+        }
+
+        public static byte[] ToTitleBytes(CategoryCasingInfo input)
+        {
+            byte[] bytes = new byte[sizeof(ushort)];
+            BinaryPrimitives.WriteUInt16LittleEndian(bytes, input._data.offsetToSimpleTitlecase);
+            return bytes;
+        }
+
+        public static byte[] ToCaseFoldBytes(CategoryCasingInfo input)
+        {
+            byte[] bytes = new byte[sizeof(ushort)];
+            BinaryPrimitives.WriteUInt16LittleEndian(bytes, input._data.offsetToSimpleCasefold);
+            return bytes;
+        }
+    }
+}

src/coreclr/src/System.Private.CoreLib/Tools/GenUnicodeProp/DataTable.cs

@@ -170,30 +170,4 @@ public byte[][] GetBytes()
             return new[] { Level1Index.ToArray(), level2.ToArray(), Level3Data.ToArray() };
         }
     }
-
-    internal sealed class FlatDataTable
-    {
-        // If a codepoint does not have data, this specifies the default value.
-        private readonly string DefaultValue;
-        private readonly Func<string, byte[]> GetValueBytesCallback;
-
-        // This contains the data mapping between codepoints and values.
-        private readonly SortedDictionary<uint, string> RawData = new SortedDictionary<uint, string>();
-
-        public FlatDataTable(string defaultValue, Func<string, byte[]> getValueBytesCallback)
-        {
-            DefaultValue = defaultValue;
-            GetValueBytesCallback = getValueBytesCallback;
-        }
-
-        public void AddData(uint codepoint, string value) => RawData[codepoint] = value;
-
-        public byte[] GetBytesFlat()
-        {
-            var str = new List<byte>();
-            foreach (var v in RawData.Values)
-                str.AddRange(GetValueBytesCallback(v ?? DefaultValue));
-            return str.ToArray();
-        }
-    }
 }

src/coreclr/src/System.Private.CoreLib/Tools/GenUnicodeProp/GenUnicodeProp.csproj

@@ -2,7 +2,51 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>$(NetCoreAppCurrent)</TargetFramework>
+    <TargetFramework>netcoreapp3.0</TargetFramework>
+    <UnicodeUcdVersion>12.1</UnicodeUcdVersion>
   </PropertyGroup>
 
+  <ItemGroup>
+    <Compile Include="$(LibrariesProjectRoot)\Common\tests\CoreFx.Private.TestUtilities.Unicode\**\*.cs" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="System.Private.Runtime.UnicodeData" Version="$(SystemPrivateRuntimeUnicodeDataVersion)" ExcludeAssets="contentFiles" GeneratePathProperty="true" />
+    <EmbeddedResource Include="$(PkgSystem_Private_Runtime_UnicodeData)\contentFiles\any\any\$(UnicodeUcdVersion).0\ucd\CaseFolding.txt">
+      <Link>UnicodeData\CaseFolding.txt</Link>
+      <LogicalName>CaseFolding.txt</LogicalName>
+    </EmbeddedResource>
+    <EmbeddedResource Include="$(PkgSystem_Private_Runtime_UnicodeData)\contentFiles\any\any\$(UnicodeUcdVersion).0\ucd\PropList.txt">
+      <Link>UnicodeData\PropList.txt</Link>
+      <LogicalName>PropList.txt</LogicalName>
+    </EmbeddedResource>
+    <EmbeddedResource Include="$(PkgSystem_Private_Runtime_UnicodeData)\contentFiles\any\any\$(UnicodeUcdVersion).0\ucd\UnicodeData.txt">
+      <Link>UnicodeData\UnicodeData.txt</Link>
+      <LogicalName>UnicodeData.txt</LogicalName>
+    </EmbeddedResource>
+    <EmbeddedResource Include="$(PkgSystem_Private_Runtime_UnicodeData)\contentFiles\any\any\$(UnicodeUcdVersion).0\ucd\auxiliary\GraphemeBreakProperty.txt">
+      <Link>UnicodeData\GraphemeBreakProperty.txt</Link>
+      <LogicalName>GraphemeBreakProperty.txt</LogicalName>
+    </EmbeddedResource>
+    <EmbeddedResource Include="$(PkgSystem_Private_Runtime_UnicodeData)\contentFiles\any\any\$(UnicodeUcdVersion).0\ucd\extracted\DerivedBidiClass.txt">
+      <Link>UnicodeData\DerivedBidiClass.txt</Link>
+      <LogicalName>DerivedBidiClass.txt</LogicalName>
+    </EmbeddedResource>
+    <EmbeddedResource Include="$(PkgSystem_Private_Runtime_UnicodeData)\contentFiles\any\any\$(UnicodeUcdVersion).0\ucd\extracted\DerivedName.txt">
+      <Link>UnicodeData\DerivedName.txt</Link>
+      <LogicalName>DerivedName.txt</LogicalName>
+    </EmbeddedResource>
+    <EmbeddedResource Include="$(PkgSystem_Private_Runtime_UnicodeData)\contentFiles\any\any\emoji\$(UnicodeUcdVersion)\emoji-data.txt">
+      <Link>UnicodeData\emoji-data.txt</Link>
+      <LogicalName>emoji-data.txt</LogicalName>
+    </EmbeddedResource>
+  </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.DotNet.PlatformAbstractions" Version="$(MicrosoftDotNetPlatformAbstractionsVersion)" />
+    <PackageReference Include="Microsoft.DotNet.XUnitExtensions" Version="$(MicrosoftDotNetXUnitExtensionsVersion)" />
+    <PackageReference Include="xunit.core" Version="$(XUnitVersion)" ExcludeAssets="build" />
+    <PackageReference Include="xunit.assert" Version="$(XUnitVersion)" />
+  </ItemGroup>
+
 </Project>

src/coreclr/src/System.Private.CoreLib/Tools/GenUnicodeProp/NumericGraphemeInfo.cs

@@ -0,0 +1,69 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Buffers.Binary;
+using System.Runtime.CompilerServices;
+using System.Text.Unicode;
+
+namespace GenUnicodeProp
+{
+    /// <summary>
+    /// Contains information about a code point's numeric representation
+    /// and the manner in which it's treated for grapheme cluster segmentation
+    /// purposes.
+    /// </summary>
+    internal sealed class NumericGraphemeInfo : IEquatable<NumericGraphemeInfo>
+    {
+        public readonly (int decimalDigitValue,
+            int digitValue,
+            double numericValue,
+            GraphemeClusterBreakProperty graphemeClusterBreakProperty) _data;
+
+        public NumericGraphemeInfo(CodePoint codePoint)
+        {
+            _data.decimalDigitValue = codePoint.DecimalDigitValue;
+            _data.digitValue = codePoint.DigitValue;
+            _data.numericValue = codePoint.NumericValue;
+            _data.graphemeClusterBreakProperty = codePoint.GraphemeClusterBreakProperty;
+        }
+
+        public override bool Equals(object obj) => Equals(obj as NumericGraphemeInfo);
+
+        public bool Equals(NumericGraphemeInfo other)
+        {
+            return !(other is null) && this._data.Equals(other._data);
+        }
+
+        public override int GetHashCode()
+        {
+            return _data.GetHashCode();
+        }
+
+        public static byte[] ToDigitBytes(NumericGraphemeInfo input)
+        {
+            // Bits 4 .. 7 contain (decimalDigitValue + 1).
+            // Bits 0 .. 3 contain (digitValue + 1).
+            // This means that each nibble will have a value 0x0 .. 0xa, inclusive.
+
+            int adjustedDecimalDigitValue = input._data.decimalDigitValue + 1;
+            int adjustedDigitValue = input._data.digitValue + 1;
+
+            return new byte[] { (byte)((adjustedDecimalDigitValue << 4) | adjustedDigitValue) };
+        }
+
+        public static byte[] ToNumericBytes(NumericGraphemeInfo input)
+        {
+            byte[] bytes = new byte[sizeof(double)];
+            double value = input._data.numericValue;
+            BinaryPrimitives.WriteUInt64LittleEndian(bytes, Unsafe.As<double, ulong>(ref value));
+            return bytes;
+        }
+
+        public static byte[] ToGraphemeBytes(NumericGraphemeInfo input)
+        {
+            return new byte[] { checked((byte)input._data.graphemeClusterBreakProperty) };
+        }
+    }
+}