Feature/Unicode escape support

src/SmartFormat.Tests/Core/LiteralTextTests.cs

@@ -33,8 +33,8 @@ public void FormatCharacterLiteralsAsString()
         [Test]
         public void AllSupportedCharacterLiteralsAsUnicode()
         {
-            const string formatWithFileBehavior = @"All supported literal characters: \' \"" \\ \a \b \f \n \r \t \v \0!";
-            const string formatWithCodeBehavior = "All supported literal characters: \' \" \\ \a \b \f \n \r \t \v \0!";
+            const string formatWithFileBehavior = @"All supported literal characters: \' \"" \\ \a \b \f \n \r \t \v \0 \u2022!";
+            const string formatWithCodeBehavior = "All supported literal characters: \' \" \\ \a \b \f \n \r \t \v \0 \u2022!";
 
             var formatter = Smart.CreateDefaultSmartFormat();
             formatter.Settings.Parser.ConvertCharacterStringLiterals = true;
@@ -43,15 +43,27 @@ public void AllSupportedCharacterLiteralsAsUnicode()
             Assert.AreEqual(formatWithCodeBehavior, result);
         }
 
-        [Test]
-        public void UnsupportedCharacterLiteralEscapeSequence()
+        [TestCase(@"Some text \u2022 with the value {0}", "Some text \u2022 with the value 123")]
+        [TestCase(@"\u2015 {0}", "\u2015 123")]
+        [TestCase(@"\u2010 {0} \u2015", "\u2010 123 \u2015")]
+        public void UnicodeEscapeSequenceIsParsed(string format, string expectedOutput)
+        {
+            var formatter = Smart.CreateDefaultSmartFormat();
+            formatter.Settings.Parser.ConvertCharacterStringLiterals = true;
+            Assert.AreEqual(expectedOutput, formatter.Format(format, 123));
+        }
+
+        [TestCase(@"Some text {0} \uABCP", @"\uABCP")]
+        [TestCase(@"Some text {0} \u-123", @"\u-123")]
+        [TestCase(@"\u", @"\u")]
+        [TestCase(@"\y", @"\y")]
+        [TestCase(@"Some text \uuuuu {0}", @"\uuuuu")]
+        public void UnsupportedCharacterLiteralEscapeSequence(string format, string exMessage)
         {
-            const string format = @"Not supported: \y - Supported: \a";
-
             var formatter = Smart.CreateDefaultSmartFormat();
             formatter.Settings.Parser.ConvertCharacterStringLiterals = true;
 
-            Assert.That( () => formatter.Format(format), Throws.ArgumentException.And.Message.Contains(@"\y"));
+            Assert.That( () => formatter.Format(format, 123), Throws.ArgumentException.And.Message.Contains(exMessage));
         }
 
         [Test]

src/SmartFormat.Tests/Core/ParserTests.cs

@@ -596,5 +596,26 @@ public void Parse_Options()
             Assert.That(placeholder.Format?.Items.Count, Is.EqualTo(3));
             Assert.That(string.Concat(placeholder.Format?.Items[0].ToString(), placeholder.Format?.Items[1].ToString(), placeholder.Format?.Items[2].ToString()), Is.EqualTo(literal.Replace("\\", "")));
         }
+
+        [TestCase(@"\u1234", @"\u1234", 0, true)]
+        [TestCase(@"\u1234abc", @"\u1234", 0, true)]
+        [TestCase(@"abc\u1234", @"\u1234", 1, true)]
+        [TestCase(@"abc\u1234def", @"\u1234", 1, true)]
+        [TestCase(@"\uwxyz", @"\uwxyz", 0, false)] // Parser accepts illegal sequence
+        [TestCase(@"\uw", @"\uw", 0, false)] // Parser accepts illegal sequence, even if shorter than 4 characters
+        public void Parse_Unicode(string formatString, string unicodeLiteral, int itemIndex, bool isLegal)
+        {
+            var parser = GetRegularParser();
+            var result = parser.ParseFormat(formatString, new[] {"d"});
+
+            var literal = result.Items[itemIndex];
+            Assert.That(literal, Is.TypeOf(typeof(LiteralText)));
+            Assert.That(literal.BaseString.Substring(literal.StartIndex, literal.Length), Is.EqualTo(unicodeLiteral));
+
+            if(isLegal) 
+                Assert.That(literal.ToString()[0], Is.EqualTo((char) int.Parse(unicodeLiteral.Substring(2), System.Globalization.NumberStyles.HexNumber)));
+            else
+                Assert.That(() => literal.ToString(), Throws.ArgumentException.And.Message.Contains(unicodeLiteral));
+        }
     }
 }

src/SmartFormat/Core/Parsing/EscapedLiteral.cs

@@ -5,6 +5,7 @@
 
 using System;
 using System.Collections.Generic;
+using System.Globalization;
 using System.Linq;
 
 namespace SmartFormat.Core.Parsing
@@ -53,6 +54,20 @@ public static bool TryGetChar(char input, out char result, bool includeFormatter
                 : GeneralLookupTable.TryGetValue(input, out result);
         }
 
+        private static char GetUnicode(string input, int startIndex)
+        {
+            var unicode = input.Length - startIndex >= 4
+                ? input.Substring(startIndex, 4)
+                : input.Substring(startIndex);
+
+            if (int.TryParse(unicode, NumberStyles.HexNumber, null, out var result))
+            {
+                return (char) result;
+            }
+
+            throw new ArgumentException($"Unrecognized escape sequence in literal: \"\\u{unicode}\"");
+        }
+
         /// <summary>
         /// Converts escaped characters in input with real characters, e.g. "\\" => "\", if '\' is the escape character.
         /// </summary>
@@ -84,6 +99,14 @@ public static ReadOnlySpan<char> UnEscapeCharLiterals(char escapingSequenceStart
 
                 if (input[inputIndex] == escapingSequenceStart)
                 {
+                    if (input[nextInputIndex] == 'u')
+                    {
+                        // GetUnicode will throw if code is illegal
+                        result[resultIndex++] = GetUnicode(input, nextInputIndex + 1);
+                        inputIndex += 5;  // move to last unicode character
+                        continue;
+                    }
+
                     if (TryGetChar(input[nextInputIndex], out var realChar, includeFormatterOptionChars))
                     {
                         result[resultIndex++] = realChar;

src/SmartFormat/Core/Parsing/Parser.cs

@@ -408,7 +408,7 @@ private void ParseAlternativeEscaping(string inputFormat, Format currentFormat,
             // index.NamedFormatterStart = PositionUndefined;
 
             // See what is the next character
-            var indexNextChar = index.Current + 1;
+            var indexNextChar = index.SafeAdd(index.Current, 1);
 
             // **** Alternative brace escaping with { or } following the escape character ****
             if (indexNextChar < inputFormat.Length && (inputFormat[indexNextChar] == _parserSettings.PlaceholderBeginChar || inputFormat[indexNextChar] == _parserSettings.PlaceholderEndChar))
@@ -425,10 +425,20 @@ private void ParseAlternativeEscaping(string inputFormat, Format currentFormat,
 
                 // Finish the last text item:
                 if (index.Current != index.LastEnd) currentFormat.Items.Add(new LiteralText(Settings, currentFormat, index.LastEnd, index.Current));
-                index.LastEnd = index.SafeAdd(index.Current, 2);
-
-                // Next add the character literal INCLUDING the escape character, which LiteralText will expect
-                currentFormat.Items.Add(new LiteralText(Settings, currentFormat, index.Current, index.LastEnd));
+
+                // Is this a unicode escape sequence?
+                if (inputFormat[indexNextChar] == 'u')
+                {
+                    // The next 4 characters must represent the unicode 
+                    index.LastEnd = index.SafeAdd(index.Current, 6);
+                    currentFormat.Items.Add(new LiteralText(Settings, currentFormat, index.Current, index.LastEnd));
+                }
+                else
+                {
+                    // Next add the character literal INCLUDING the escape character, which LiteralText will expect
+                    index.LastEnd = index.SafeAdd(index.Current, 2);
+                    currentFormat.Items.Add(new LiteralText(Settings, currentFormat, index.Current, index.LastEnd));
+                }
 
                 index.Current = index.SafeAdd(index.Current, 1);
             }