Merge pull request #978 from KrzysztofCwalina/EncodingCleanup

Encoding cleanup

src/System.Text.Formatting.Globalization/System/Text/Formatting/FormattingDataProvider.cs

@@ -45,7 +45,7 @@ private static EncodingData CreateEncoding(string localeId, Stream resourceStrea
 
             byte[] idBytes = new byte[maxIdLength];
             int idByteCount;
-            if (!localeId.TryFormat(new Span<byte>(idBytes), EncodingData.InvariantUtf8, out idByteCount))
+            if (!localeId.TryFormat(new Span<byte>(idBytes), EncodingData.TextEncoding.Utf8, out idByteCount))
             {
                 throw new Exception("bad locale id");
             }
@@ -92,7 +92,7 @@ private static EncodingData CreateEncoding(string localeId, Stream resourceStrea
                 Array.Copy(data, stringStart, utf16digitsAndSymbols[stringIndex], 0, stringLength);
             }
 
-            return new EncodingData(utf16digitsAndSymbols, EncodingData.Encoding.Utf16);
+            return new EncodingData(utf16digitsAndSymbols, EncodingData.TextEncoding.Utf16);
         }
 
         static ushort ReadUInt16At(byte[] data, int ushortIndex)

src/System.Text.Formatting/System/Text/Formatting/CompositeFormat.cs

@@ -141,7 +141,7 @@ static void Append<TFormatter>(this TFormatter formatter, string whole, int inde
             for (int i = 0; i < count; i++)
             {
                 int bytesWritten;
-                while (!whole[i+index].TryFormat(buffer, formatter.Encoding, out bytesWritten))
+                while (!whole[i+index].TryFormat(buffer, formatter.Encoding.Encoding, out bytesWritten))
                 {
                     Debug.Assert(false, "this should never happen"); // because I pre-resized the buffer to 4 bytes per char at the top of this method.
                 }

src/System.Text.Formatting/System/Text/Formatting/IOutputExtensions.cs

@@ -25,7 +25,7 @@ public static class IOutputExtensions
             return true;
         }
 
-        public static void Append<TFormatter>(this TFormatter formatter, string value, EncodingData.Encoding encoding) where TFormatter : IOutput
+        public static void Append<TFormatter>(this TFormatter formatter, string value, EncodingData.TextEncoding encoding) where TFormatter : IOutput
         {
             if (value.Length < 256) {
                 while (!formatter.TryAppend(value, encoding)) {
@@ -47,61 +47,61 @@ public static void Append<TFormatter>(this TFormatter formatter, string value, E
             }
         }
 
-        public static bool TryAppend<TFormatter>(this TFormatter formatter, string value, EncodingData.Encoding encoding) where TFormatter : IOutput
+        public static bool TryAppend<TFormatter>(this TFormatter formatter, string value, EncodingData.TextEncoding encoding) where TFormatter : IOutput
         {
             int bytesWritten;
-            if (!value.TryFormat(formatter.Buffer, encoding == EncodingData.Encoding.Utf8 ? EncodingData.InvariantUtf8 : EncodingData.InvariantUtf16, out bytesWritten)) {
+            if (!value.TryFormat(formatter.Buffer, encoding, out bytesWritten)) {
                 return false;
             }
             formatter.Advance(bytesWritten);
             return true;
         }
 
-        public static void Append<TFormatter>(this TFormatter formatter, ReadOnlySpan<char> value, EncodingData.Encoding encoding) where TFormatter : IOutput
+        public static void Append<TFormatter>(this TFormatter formatter, ReadOnlySpan<char> value, EncodingData.TextEncoding encoding) where TFormatter : IOutput
         {
             while (!formatter.TryAppend(value, encoding)) {
                 formatter.Enlarge();
             }
         }
 
-        public static bool TryAppend<TFormatter>(this TFormatter formatter, ReadOnlySpan<char> value, EncodingData.Encoding encoding) where TFormatter : IOutput
+        public static bool TryAppend<TFormatter>(this TFormatter formatter, ReadOnlySpan<char> value, EncodingData.TextEncoding encoding) where TFormatter : IOutput
         {
             int bytesWritten;
-            if (!value.TryFormat(formatter.Buffer, encoding==EncodingData.Encoding.Utf8?EncodingData.InvariantUtf8:EncodingData.InvariantUtf16, out bytesWritten)) {
+            if (!value.TryFormat(formatter.Buffer, encoding, out bytesWritten)) {
                 return false;
             }
             formatter.Advance(bytesWritten);
             return true;
         }
 
-        public static void Append<TFormatter>(this TFormatter formatter, char value, EncodingData.Encoding encoding) where TFormatter : IOutput
+        public static void Append<TFormatter>(this TFormatter formatter, char value, EncodingData.TextEncoding encoding) where TFormatter : IOutput
         {
             while (!formatter.TryAppend(value, encoding)) {
                 formatter.Enlarge();
             }
         }
 
-        public static bool TryAppend<TFormatter>(this TFormatter formatter, char value, EncodingData.Encoding encoding) where TFormatter : IOutput
+        public static bool TryAppend<TFormatter>(this TFormatter formatter, char value, EncodingData.TextEncoding encoding) where TFormatter : IOutput
         {
             int bytesWritten;
-            if (!value.TryFormat(formatter.Buffer, encoding == EncodingData.Encoding.Utf8 ? EncodingData.InvariantUtf8 : EncodingData.InvariantUtf16, out bytesWritten)) {
+            if (!value.TryFormat(formatter.Buffer, encoding, out bytesWritten)) {
                 return false;
             }
             formatter.Advance(bytesWritten);
             return true;
         }
 
-        public static void Append<TFormatter>(this TFormatter formatter, Utf8String value, EncodingData.Encoding encoding) where TFormatter : IOutput
+        public static void Append<TFormatter>(this TFormatter formatter, Utf8String value, EncodingData.TextEncoding encoding) where TFormatter : IOutput
         {
             while (!formatter.TryAppend(value, encoding)) {
                 formatter.Enlarge();
             }
         }
 
-        public static bool TryAppend<TFormatter>(this TFormatter formatter, Utf8String value, EncodingData.Encoding encoding) where TFormatter : IOutput
+        public static bool TryAppend<TFormatter>(this TFormatter formatter, Utf8String value, EncodingData.TextEncoding encoding) where TFormatter : IOutput
         {
             int bytesWritten;
-            if (!value.TryFormat(formatter.Buffer, encoding == EncodingData.Encoding.Utf8 ? EncodingData.InvariantUtf8 : EncodingData.InvariantUtf16, out bytesWritten)) {
+            if (!value.TryFormat(formatter.Buffer, encoding, out bytesWritten)) {
                 return false;
             }
             formatter.Advance(bytesWritten);

src/System.Text.Formatting/System/Text/Formatting/ITextOutputExtensions.cs

@@ -171,7 +171,7 @@ public static void Append<TFormatter>(this TFormatter formatter, char value) whe
         public static bool TryAppend<TFormatter>(this TFormatter formatter, char value) where TFormatter : ITextOutput
         {
             int bytesWritten;
-            if (!value.TryFormat(formatter.Buffer, formatter.Encoding, out bytesWritten)) {
+            if (!value.TryFormat(formatter.Buffer, formatter.Encoding.Encoding, out bytesWritten)) {
                 return false;
             }
             formatter.Advance(bytesWritten);
@@ -188,7 +188,7 @@ public static void Append<TFormatter>(this TFormatter formatter, ReadOnlySpan<ch
         public static bool TryAppend<TFormatter>(this TFormatter formatter, ReadOnlySpan<char> value) where TFormatter : ITextOutput
         {
             int bytesWritten;
-            if (!value.TryFormat(formatter.Buffer, formatter.Encoding, out bytesWritten)) {
+            if (!value.TryFormat(formatter.Buffer, formatter.Encoding.Encoding, out bytesWritten)) {
                 return false;
             }
             formatter.Advance(bytesWritten);
@@ -220,7 +220,7 @@ public static void Append<TFormatter>(this TFormatter formatter, string value) w
         public static bool TryAppend<TFormatter>(this TFormatter formatter, string value) where TFormatter : ITextOutput
         {
             int bytesWritten;
-            if (!value.TryFormat(formatter.Buffer, formatter.Encoding, out bytesWritten)) {
+            if (!value.TryFormat(formatter.Buffer, formatter.Encoding.Encoding, out bytesWritten)) {
                 return false;
             }
             formatter.Advance(bytesWritten);
@@ -237,7 +237,7 @@ public static void Append<TFormatter>(this TFormatter formatter, Utf8String valu
         public static bool TryAppend<TFormatter>(this TFormatter formatter, Utf8String value) where TFormatter : ITextOutput
         {
             int bytesWritten;
-            if (!value.TryFormat(formatter.Buffer, formatter.Encoding, out bytesWritten)) {
+            if (!value.TryFormat(formatter.Buffer, formatter.Encoding.Encoding, out bytesWritten)) {
                 return false;
             }
             formatter.Advance(bytesWritten);

src/System.Text.Http/System/Text/Http/HttpHeaderBuffer.cs

@@ -22,7 +22,7 @@ public void UpdateValue(string newValue)
             }
 
             int bytesWritten;
-            newValue.TryFormat(_bytes, _encoding, out bytesWritten);            
+            newValue.TryFormat(_bytes, _encoding.Encoding, out bytesWritten);            
 
             _bytes.SetFromRestOfSpanToEmpty(newValue.Length);
         }        

src/System.Text.Json.Dynamic/System/Text/Json/JsonDynamicObject.cs

@@ -169,7 +169,7 @@ public bool TryFormat(Span<byte> buffer, Format.Parsed format, EncodingData form
         {
             written = 0;
             int justWritten;
-            if(!'{'.TryFormat(buffer, formattingData, out justWritten)){
+            if(!'{'.TryFormat(buffer, formattingData.Encoding, out justWritten)){
                 return false;
             }
             written += justWritten;
@@ -182,7 +182,7 @@ public bool TryFormat(Span<byte> buffer, Format.Parsed format, EncodingData form
                 if(firstProperty) { firstProperty = false; }
                 else
                 {
-                    if (!','.TryFormat(buffer.Slice(written), formattingData, out justWritten))
+                    if (!','.TryFormat(buffer.Slice(written), formattingData.Encoding, out justWritten))
                     {
                         return false;
                     }
@@ -194,7 +194,7 @@ public bool TryFormat(Span<byte> buffer, Format.Parsed format, EncodingData form
                     written = 0; return false;
                 }
                 written += justWritten;
-                if (!':'.TryFormat(buffer.Slice(written), formattingData, out justWritten))
+                if (!':'.TryFormat(buffer.Slice(written), formattingData.Encoding, out justWritten))
                 {
                     return false;
                 }
@@ -206,7 +206,7 @@ public bool TryFormat(Span<byte> buffer, Format.Parsed format, EncodingData form
                 written += justWritten;
             }
 
-            if (!'}'.TryFormat(buffer.Slice(written), formattingData, out justWritten)){
+            if (!'}'.TryFormat(buffer.Slice(written), formattingData.Encoding, out justWritten)){
                 written = 0; return false;
             }
             written += justWritten;
@@ -268,11 +268,11 @@ public bool TryFormat(Span<byte> buffer, Format.Parsed format, EncodingData form
                     case JsonReader.JsonValueType.Object:
                         return _object.TryFormat(buffer, format, formattingData, out written);
                     case JsonReader.JsonValueType.Null:
-                        return "null".TryFormat(buffer, formattingData, out written);
+                        return "null".TryFormat(buffer, formattingData.Encoding, out written);
                     case JsonReader.JsonValueType.True:
-                        return "true".TryFormat(buffer, formattingData, out written);
+                        return "true".TryFormat(buffer, formattingData.Encoding, out written);
                     case JsonReader.JsonValueType.False:
-                        return "false".TryFormat(buffer, formattingData, out written);
+                        return "false".TryFormat(buffer, formattingData.Encoding, out written);
                     default:
                         throw new NotImplementedException();
                 }
@@ -348,7 +348,7 @@ public static bool TryFormatQuotedString(this Utf8String str, Span<byte> buffer,
             written = 0;
             int justWritten;
 
-            if (!'"'.TryFormat(buffer, formattingData, out justWritten))
+            if (!'"'.TryFormat(buffer, formattingData.Encoding, out justWritten))
             {
                 return false;
             }
@@ -360,7 +360,7 @@ public static bool TryFormatQuotedString(this Utf8String str, Span<byte> buffer,
             }
             written += justWritten;
 
-            if (!'"'.TryFormat(buffer.Slice(written), formattingData, out justWritten))
+            if (!'"'.TryFormat(buffer.Slice(written), formattingData.Encoding, out justWritten))
             {
                 return false;
             }

src/System.Text.Primitives/System/Text/EncodingData.cs

@@ -13,14 +13,16 @@ public struct EncodingData : IEquatable<EncodingData>
     {
         private static EncodingData s_invariantUtf16;
         private static EncodingData s_invariantUtf8;
+
         private byte[][] _digitsAndSymbols; // this could be flattened into a single array
-        private TrieNode[] _parsingTrie;
-        private Encoding _encoding;
+        private ParsingTrieNode[] _parsingTrie;
+        private TextEncoding _encoding;
 
-        public enum Encoding : byte
+        public enum TextEncoding : byte
         {
             Utf16 = 0,
             Utf8 = 1,
+            Ascii,
         }
 
         // The parsing trie is structured as an array, which means that there are two types of
@@ -40,25 +42,38 @@ public enum Encoding : byte
         // The index is formatted as such: 0xAABBCCDD, where AA = the min value,
         // BB = the index of the min value relative to the current node (1-indexed),
         // CC = the max value, and DD = the max value's index in the same coord-system as BB.
-        public struct TrieNode
+        internal struct ParsingTrieNode
         {
             public byte valueOrNumChildren;
             public int index;
         }
 
-        // TODO: make these private once bin file generator is used
-        public EncodingData(byte[][] digitsAndSymbols, TrieNode[] parsingTrie, Encoding encoding)
+        // this should be removed after CreateParsingTire is implemented
+        public EncodingData(byte[][] digitsAndSymbols, TextEncoding encoding, Tuple<byte, int>[] parsingTrie)
         {
             _digitsAndSymbols = digitsAndSymbols;
             _encoding = encoding;
-            _parsingTrie = parsingTrie;
+
+            var tire = new ParsingTrieNode[parsingTrie.Length];
+            for(int i=0; i<parsingTrie.Length; i++) {
+                tire[i] = new ParsingTrieNode() { valueOrNumChildren = parsingTrie[i].Item1, index = parsingTrie[i].Item2 };
+            }
+
+            _parsingTrie = tire;
         }
 
-        public EncodingData(byte[][] digitsAndSymbols, Encoding encoding)
+        public EncodingData(byte[][] digitsAndSymbols, TextEncoding encoding)
         {
             _digitsAndSymbols = digitsAndSymbols;
             _encoding = encoding;
             _parsingTrie = null;
+            _parsingTrie = CreateParsingTire(_digitsAndSymbols);
+        }
+
+        private ParsingTrieNode[] CreateParsingTire(byte[][] _digitsAndSymbols)
+        {
+            // TODO: this needs to be implemented;
+            return null;
         }
 
         // This binary search implementation returns an int representing either:
@@ -128,7 +143,7 @@ static EncodingData()
                 new byte[] { 101, 0, }, // e
             };
 
-            s_invariantUtf16 = new EncodingData(utf16digitsAndSymbols, Encoding.Utf16);
+            s_invariantUtf16 = new EncodingData(utf16digitsAndSymbols, TextEncoding.Utf16);
 
             var utf8digitsAndSymbols = new byte[][] {
                 new byte[] { 48, },
@@ -151,7 +166,7 @@ static EncodingData()
                 new byte[] { 101, }, // e
             };
 
-            s_invariantUtf8 = new EncodingData(utf8digitsAndSymbols, Encoding.Utf8);
+            s_invariantUtf8 = new EncodingData(utf8digitsAndSymbols, TextEncoding.Utf8);
         }
 
         public static EncodingData InvariantUtf16
@@ -348,14 +363,7 @@ public bool IsInvariantUtf8
             get { return _digitsAndSymbols == s_invariantUtf8._digitsAndSymbols; }
         }
 
-        public bool IsUtf16
-        {
-            get { return _encoding == Encoding.Utf16; }
-        }
-        public bool IsUtf8
-        {
-            get { return _encoding == Encoding.Utf8; }
-        }
+        public TextEncoding Encoding => _encoding;
 
         public static bool operator==(EncodingData left, EncodingData right)
         {

src/System.Text.Primitives/System/Text/Formatting/FloatFormatter.cs

@@ -44,7 +44,7 @@ public static bool TryFormatNumber(double value, bool isSingle, Span<byte> buffe
             // TODO: the lines below need to be replaced with properly implemented algorithm
             // the problem is the algorithm is complex, so I am commiting a stub for now
             var hack = value.ToString(format.Symbol.ToString());
-            return hack.TryFormat(buffer, formattingData, out bytesWritten);
+            return hack.TryFormat(buffer, formattingData.Encoding, out bytesWritten);
         }
     }
 }

src/System.Text.Primitives/System/Text/Formatting/IntegerFormatter.cs

@@ -64,11 +64,11 @@ internal static bool TryFormatUInt64(ulong value, byte numberOfBytes, Span<byte>
                 format.Symbol = 'G';
             }
 
-            if (format.IsHexadecimal && formattingData.IsUtf16) {
+            if (format.IsHexadecimal && formattingData.IsInvariantUtf16) {
                 return TryFormatHexadecimalInvariantCultureUtf16(value, buffer, format, out bytesWritten);
             }
 
-            if (format.IsHexadecimal && formattingData.IsUtf8) {
+            if (format.IsHexadecimal && formattingData.IsInvariantUtf8) {
                 return TryFormatHexadecimalInvariantCultureUtf8(value, buffer, format, out bytesWritten);
             }