#13 - Detailed UTF-8 validation.

Tratcher · Tratcher · commit 98e9285fa8b2 · 2014-10-23T17:34:26.000-07:00
diff --git a/src/Microsoft.AspNet.WebSockets.Protocol/Utilities.cs b/src/Microsoft.AspNet.WebSockets.Protocol/Utilities.cs
@@ -70,49 +70,52 @@ public static WebSocketMessageType GetMessageType(int opCode)
             }
         }
 
-        // For now this is stateless and does not handle sequences spliced across messages.
-        // http://etutorials.org/Programming/secure+programming/Chapter+3.+Input+Validation/3.12+Detecting+Illegal+UTF-8+Characters/
+        // Performs a stateful validation of UTF-8 bytes.
+        // It checks for valid formatting, overlong encodings, surrogates, and value ranges.
         public static bool TryValidateUtf8(ArraySegment<byte> arraySegment, bool endOfMessage, Utf8MessageState state)
         {
             for (int i = arraySegment.Offset; i < arraySegment.Offset + arraySegment.Count; )
             {
+                // Have we started a character sequence yet?
                 if (!state.SequenceInProgress)
                 {
+                    // The first byte tells us how many bytes are in the sequence.
                     state.SequenceInProgress = true;
                     byte b = arraySegment.Array[i];
+                    i++;
                     if ((b & 0x80) == 0) // 0bbbbbbb, single byte
                     {
                         state.AdditionalBytesExpected = 0;
+                        state.CurrentDecodeBits = b & 0x7F;
+                        state.ExpectedValueMin = 0;
                     }
                     else if ((b & 0xC0) == 0x80)
                     {
-                        return false; // Misplaced 10bbbbbb byte. This cannot be the first byte.
+                        // Misplaced 10bbbbbb continuation byte. This cannot be the first byte.
+                        return false;
                     }
                     else if ((b & 0xE0) == 0xC0) // 110bbbbb 10bbbbbb
                     {
                         state.AdditionalBytesExpected = 1;
+                        state.CurrentDecodeBits = b & 0x1F;
+                        state.ExpectedValueMin = 0x80;
                     }
                     else if ((b & 0xF0) == 0xE0) // 1110bbbb 10bbbbbb 10bbbbbb
                     {
                         state.AdditionalBytesExpected = 2;
+                        state.CurrentDecodeBits = b & 0xF;
+                        state.ExpectedValueMin = 0x800;
                     }
                     else if ((b & 0xF8) == 0xF0) // 11110bbb 10bbbbbb 10bbbbbb 10bbbbbb
                     {
                         state.AdditionalBytesExpected = 3;
+                        state.CurrentDecodeBits = b & 0x7;
+                        state.ExpectedValueMin = 0x10000;
                     }
-                    else if ((b & 0xFC) == 0xF8) // 111110bb 10bbbbbb 10bbbbbb 10bbbbbb 10bbbbbb
-                    {
-                        state.AdditionalBytesExpected = 4;
-                    }
-                    else if ((b & 0xFE) == 0xFC) // 1111110b 10bbbbbb 10bbbbbb 10bbbbbb 10bbbbbb 10bbbbbb
-                    {
-                        state.AdditionalBytesExpected = 5;
-                    }
-                    else // 11111110 && 11111111 are not valid
+                    else // 111110bb & 1111110b & 11111110 && 11111111 are not valid
                     {
                         return false;
                     }
-                    i++;
                 }
                 while (state.AdditionalBytesExpected > 0 && i < arraySegment.Offset + arraySegment.Count)
                 {
@@ -121,12 +124,32 @@ public static bool TryValidateUtf8(ArraySegment<byte> arraySegment, bool endOfMe
                     {
                         return false;
                     }
-                    state.AdditionalBytesExpected--;
+
                     i++;
+                    state.AdditionalBytesExpected--;
+
+                    // Each continuation byte carries 6 bits of data 0x10bbbbbb.
+                    state.CurrentDecodeBits = (state.CurrentDecodeBits << 6) | b & 0x3F;
+
+                    if (state.AdditionalBytesExpected == 1 && state.CurrentDecodeBits >= 0x360 && state.CurrentDecodeBits <= 0x37F)
+                    {
+                        // This is going to end up in the range of 0xD800-0xDFFF UTF-16 surrogates that are not allowed in UTF-8;
+                        return false;
+                    }
+                    if (state.AdditionalBytesExpected == 2 && state.CurrentDecodeBits >= 0x110)
+                    {
+                        // This is going to be out of the upper Unicode bound 0x10FFFF.
+                        return false;
+                    }
                 }
                 if (state.AdditionalBytesExpected == 0)
                 {
                     state.SequenceInProgress = false;
+                    if (state.CurrentDecodeBits < state.ExpectedValueMin)
+                    {
+                        // Overlong encoding (e.g. using 2 bytes to encode something that only needed 1).
+                        return false;
+                    }
                 }
             }
             if (endOfMessage && state.SequenceInProgress)
@@ -140,6 +163,8 @@ public class Utf8MessageState
         {
             public bool SequenceInProgress { get; set; }
             public int AdditionalBytesExpected { get; set; }
+            public int ExpectedValueMin { get; set; }
+            public int CurrentDecodeBits { get; set; }
         }
     }
 }
diff --git a/test/Microsoft.AspNet.WebSockets.Protocol.Test/Utf8ValidationTests.cs b/test/Microsoft.AspNet.WebSockets.Protocol.Test/Utf8ValidationTests.cs
@@ -13,6 +13,8 @@ public class Utf8ValidationTests
         [InlineData(new byte[] { })]
         [InlineData(new byte[] { 0x48, 0x65, 0x6C, 0x6C, 0x6F, 0x20, 0x57, 0x6F, 0x72, 0x6C, 0x64 })] // Hello World
         [InlineData(new byte[] { 0x48, 0x65, 0x6C, 0x6C, 0x6F, 0x2D, 0xC2, 0xB5, 0x40, 0xC3, 0x9F, 0xC3, 0xB6, 0xC3, 0xA4, 0xC3, 0xBC, 0xC3, 0xA0, 0xC3, 0xA1 })] // "Hello-µ@ßöäüàá";
+        // [InlineData(new byte[] { 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0xf0, 0xa4, 0xad, 0xa2, 0x77, 0x6f, 0x72, 0x6c, 0x64 })] // "hello\U00024b62world"
+        [InlineData(new byte[] { 0xf0, 0xa4, 0xad, 0xa2 })] // "\U00024b62"
         public void ValidateSingleValidSegments_Valid(byte[] data)
         {
             var state = new Utilities.Utf8MessageState();
@@ -35,28 +37,39 @@ public void ValidateMultipleValidSegments_Valid(byte[] data1, byte[] data2, byte
         [InlineData(new byte[] { 0xfe })]
         [InlineData(new byte[] { 0xff })]
         [InlineData(new byte[] { 0xfe, 0xfe, 0xff, 0xff })]
-        // [InlineData(new byte[] { 0xc0, 0xaf })]
-        // [InlineData(new byte[] { 0xe0, 0x80, 0xaf })]
-        // [InlineData(new byte[] { 0xf4, 0x90, 0x80, 0x80 })]
-        // [InlineData(new byte[] { 0xf0, 0x80, 0x80, 0xaf })]
-        // [InlineData(new byte[] { 0xf8, 0x80, 0x80, 0x80, 0xaf })]
-        // [InlineData(new byte[] { 0xfc, 0x80, 0x80, 0x80, 0x80, 0xaf })]
-        // [InlineData(new byte[] { 0xc1, 0xbf })]
-        // [InlineData(new byte[] { 0xed, 0xa0, 0x80, 0x65, 0x64, 0x69, 0x74, 0x65, 0x64 })] // 0xEDA080 decodes to 0xD800, which is a reserved high surrogate character.
+        [InlineData(new byte[] { 0xc0, 0xb1 })] // Overlong Ascii
+        [InlineData(new byte[] { 0xc1, 0xb1 })] // Overlong Ascii
+        [InlineData(new byte[] { 0xe0, 0x80, 0xaf })] // Overlong
+        [InlineData(new byte[] { 0xf0, 0x80, 0x80, 0xaf })] // Overlong
+        [InlineData(new byte[] { 0xf8, 0x80, 0x80, 0x80, 0xaf })] // Overlong
+        [InlineData(new byte[] { 0xfc, 0x80, 0x80, 0x80, 0x80, 0xaf })] // Overlong
+        [InlineData(new byte[] { 0xed, 0xa0, 0x80, 0x65, 0x64, 0x69, 0x74, 0x65, 0x64 })] // 0xEDA080 decodes to 0xD800, which is a reserved high surrogate character.
         public void ValidateSingleInvalidSegment_Invalid(byte[] data)
         {
             var state = new Utilities.Utf8MessageState();
             Assert.False(Utilities.TryValidateUtf8(new ArraySegment<byte>(data), endOfMessage: true, state: state));
         }
-        /*
-        [Theory]
-        // [InlineData(true, new byte[] { 0xce, 0xba, 0xe1, 0xbd, 0xb9, 0xcf, 0x83, 0xce, 0xbc, 0xce, 0xb5, 0xf4 }, false, new byte[] { 0x90 }, true, new byte[] { })]
-        public void ValidateMultipleInvalidSegments_Invalid(bool valid1, byte[] data1, bool valid2, byte[] data2, bool valid3, byte[] data3)
+
+        [Fact]
+        public void ValidateIndividualInvalidSegments_Invalid()
         {
+            var data = new byte[] { 0xce, 0xba, 0xe1, 0xbd, 0xb9, 0xcf, 0x83, 0xce, 0xbc, 0xce, 0xb5, 0xed, 0xa0, 0x80, 0x65, 0x64, 0x69, 0x74, 0x65, 0x64 };
             var state = new Utilities.Utf8MessageState();
-            Assert.True(valid1 == Utilities.TryValidateUtf8(new ArraySegment<byte>(data1), endOfMessage: false, state: state), "1st");
-            Assert.True(valid2 == Utilities.TryValidateUtf8(new ArraySegment<byte>(data2), endOfMessage: false, state: state), "2nd");
-            Assert.True(valid3 == Utilities.TryValidateUtf8(new ArraySegment<byte>(data3), endOfMessage: true, state: state), "3rd");
-        }*/
+            for (int i = 0; i < 12; i++)
+            {
+                Assert.True(Utilities.TryValidateUtf8(new ArraySegment<byte>(data, i, 1), endOfMessage: false, state: state), i.ToString());
+            }
+            Assert.False(Utilities.TryValidateUtf8(new ArraySegment<byte>(data, 12, 1), endOfMessage: false, state: state), 12.ToString());
+        }
+
+        [Fact]
+        public void ValidateMultipleInvalidSegments_Invalid()
+        {
+            var data0 = new byte[] { 0xce, 0xba, 0xe1, 0xbd, 0xb9, 0xcf, 0x83, 0xce, 0xbc, 0xce, 0xb5, 0xf4 };
+            var data1 = new byte[] { 0x90 };
+            var state = new Utilities.Utf8MessageState();
+            Assert.True(Utilities.TryValidateUtf8(new ArraySegment<byte>(data0), endOfMessage: false, state: state));
+            Assert.False(Utilities.TryValidateUtf8(new ArraySegment<byte>(data1), endOfMessage: false, state: state));
+        }
     }
 }

Original file line number	Diff line number	Diff line change
`@@ -70,49 +70,52 @@ public static WebSocketMessageType GetMessageType(int opCode)`
`70`	`70`	`}`
`71`	`71`	`}`
`72`	`72`
`73`		`- // For now this is stateless and does not handle sequences spliced across messages.`
`74`		`- // http://etutorials.org/Programming/secure+programming/Chapter+3.+Input+Validation/3.12+Detecting+Illegal+UTF-8+Characters/`
	`73`	`+ // Performs a stateful validation of UTF-8 bytes.`
	`74`	`+ // It checks for valid formatting, overlong encodings, surrogates, and value ranges.`
`75`	`75`	`public static bool TryValidateUtf8(ArraySegment<byte> arraySegment, bool endOfMessage, Utf8MessageState state)`
`76`	`76`	`{`
`77`	`77`	`for (int i = arraySegment.Offset; i < arraySegment.Offset + arraySegment.Count; )`
`78`	`78`	`{`
	`79`	`+ // Have we started a character sequence yet?`
`79`	`80`	`if (!state.SequenceInProgress)`
`80`	`81`	`{`
	`82`	`+ // The first byte tells us how many bytes are in the sequence.`
`81`	`83`	`state.SequenceInProgress = true;`
`82`	`84`	`byte b = arraySegment.Array[i];`
	`85`	`+ i++;`
`83`	`86`	`if ((b & 0x80) == 0) // 0bbbbbbb, single byte`
`84`	`87`	`{`
`85`	`88`	`state.AdditionalBytesExpected = 0;`
	`89`	`+ state.CurrentDecodeBits = b & 0x7F;`
	`90`	`+ state.ExpectedValueMin = 0;`
`86`	`91`	`}`
`87`	`92`	`else if ((b & 0xC0) == 0x80)`
`88`	`93`	`{`
`89`		`- return false; // Misplaced 10bbbbbb byte. This cannot be the first byte.`
	`94`	`+ // Misplaced 10bbbbbb continuation byte. This cannot be the first byte.`
	`95`	`+ return false;`
`90`	`96`	`}`
`91`	`97`	`else if ((b & 0xE0) == 0xC0) // 110bbbbb 10bbbbbb`
`92`	`98`	`{`
`93`	`99`	`state.AdditionalBytesExpected = 1;`
	`100`	`+ state.CurrentDecodeBits = b & 0x1F;`
	`101`	`+ state.ExpectedValueMin = 0x80;`
`94`	`102`	`}`
`95`	`103`	`else if ((b & 0xF0) == 0xE0) // 1110bbbb 10bbbbbb 10bbbbbb`
`96`	`104`	`{`
`97`	`105`	`state.AdditionalBytesExpected = 2;`
	`106`	`+ state.CurrentDecodeBits = b & 0xF;`
	`107`	`+ state.ExpectedValueMin = 0x800;`
`98`	`108`	`}`
`99`	`109`	`else if ((b & 0xF8) == 0xF0) // 11110bbb 10bbbbbb 10bbbbbb 10bbbbbb`
`100`	`110`	`{`
`101`	`111`	`state.AdditionalBytesExpected = 3;`
	`112`	`+ state.CurrentDecodeBits = b & 0x7;`
	`113`	`+ state.ExpectedValueMin = 0x10000;`
`102`	`114`	`}`
`103`		`- else if ((b & 0xFC) == 0xF8) // 111110bb 10bbbbbb 10bbbbbb 10bbbbbb 10bbbbbb`
`104`		`- {`
`105`		`- state.AdditionalBytesExpected = 4;`
`106`		`- }`
`107`		`- else if ((b & 0xFE) == 0xFC) // 1111110b 10bbbbbb 10bbbbbb 10bbbbbb 10bbbbbb 10bbbbbb`
`108`		`- {`
`109`		`- state.AdditionalBytesExpected = 5;`
`110`		`- }`
`111`		`- else // 11111110 && 11111111 are not valid`
	`115`	`+ else // 111110bb & 1111110b & 11111110 && 11111111 are not valid`
`112`	`116`	`{`
`113`	`117`	`return false;`
`114`	`118`	`}`
`115`		`- i++;`
`116`	`119`	`}`
`117`	`120`	`while (state.AdditionalBytesExpected > 0 && i < arraySegment.Offset + arraySegment.Count)`
`118`	`121`	`{`
`@@ -121,12 +124,32 @@ public static bool TryValidateUtf8(ArraySegment<byte> arraySegment, bool endOfMe`
`121`	`124`	`{`
`122`	`125`	`return false;`
`123`	`126`	`}`
`124`		`- state.AdditionalBytesExpected--;`
	`127`	`+`
`125`	`128`	`i++;`
	`129`	`+ state.AdditionalBytesExpected--;`
	`130`	`+`
	`131`	`+ // Each continuation byte carries 6 bits of data 0x10bbbbbb.`
	`132`	`+ state.CurrentDecodeBits = (state.CurrentDecodeBits << 6) \| b & 0x3F;`
	`133`	`+`
	`134`	`+ if (state.AdditionalBytesExpected == 1 && state.CurrentDecodeBits >= 0x360 && state.CurrentDecodeBits <= 0x37F)`
	`135`	`+ {`
	`136`	`+ // This is going to end up in the range of 0xD800-0xDFFF UTF-16 surrogates that are not allowed in UTF-8;`
	`137`	`+ return false;`
	`138`	`+ }`
	`139`	`+ if (state.AdditionalBytesExpected == 2 && state.CurrentDecodeBits >= 0x110)`
	`140`	`+ {`
	`141`	`+ // This is going to be out of the upper Unicode bound 0x10FFFF.`
	`142`	`+ return false;`
	`143`	`+ }`
`126`	`144`	`}`
`127`	`145`	`if (state.AdditionalBytesExpected == 0)`
`128`	`146`	`{`
`129`	`147`	`state.SequenceInProgress = false;`
	`148`	`+ if (state.CurrentDecodeBits < state.ExpectedValueMin)`
	`149`	`+ {`
	`150`	`+ // Overlong encoding (e.g. using 2 bytes to encode something that only needed 1).`
	`151`	`+ return false;`
	`152`	`+ }`
`130`	`153`	`}`
`131`	`154`	`}`
`132`	`155`	`if (endOfMessage && state.SequenceInProgress)`
`@@ -140,6 +163,8 @@ public class Utf8MessageState`
`140`	`163`	`{`
`141`	`164`	`public bool SequenceInProgress { get; set; }`
`142`	`165`	`public int AdditionalBytesExpected { get; set; }`
	`166`	`+ public int ExpectedValueMin { get; set; }`
	`167`	`+ public int CurrentDecodeBits { get; set; }`
`143`	`168`	`}`
`144`	`169`	`}`
`145`	`170`	`}`