From b28e9fe944f209e7cb57ba7b3e0b6cccb1c5cb81 Mon Sep 17 00:00:00 2001 From: Marko Lahma Date: Tue, 21 Jul 2026 19:30:02 +0300 Subject: [PATCH] Add bulk string scan and simple-number fast paths to JSON parser ScanStringLiteral now copies runs of ordinary characters in bulk instead of one char at a time. On net8+ a SearchValues over the closing quote, the escape backslash, every control character (< 0x20) and the two Unicode line separators locates the next stop exactly where the per-char loop would have halted; older TFMs use IndexOfAny('"','\') plus a short control/line-separator validation. Escape handling, interned-key routing and every error message/position are preserved byte-for-byte (including the U+2028/U+2029 break to UnexpectedEOS). ScanNumericLiteral gains a fast path for the dominant shape -- optional '-', integer digits, optional '.fraction', no exponent, <= 15 total digits -- accumulating one exact long numerator and dividing once by an exact power of ten. The result is bit-identical to double.Parse for every accepted input because both operands are exactly representable. Gated to net8+ where double.Parse is IEEE correctly-rounded; legacy runtimes keep deferring to double.Parse (which can differ by an ULP and parses "-0" as +0.0) so behavior stays byte-for-byte identical everywhere. Negative zero also defers to double.Parse. Tests: bulk-scan escapes at start/middle/end, internal-buffer-boundary crossings, control-char rejection at exact positions and U+2028/U+2029 termination; a 200k-case number bit-identity property test plus edge cases and negative/positive-zero handling. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01U3NyGAFzB3uRWQnPUNL8i2 --- Jint.Tests/Runtime/JsonTests.cs | 207 ++++++++++++++++++++++++++++++++ Jint/Native/Json/JsonParser.cs | 206 ++++++++++++++++++++++++++++--- 2 files changed, 397 insertions(+), 16 deletions(-) diff --git a/Jint.Tests/Runtime/JsonTests.cs b/Jint.Tests/Runtime/JsonTests.cs index c88f64967..11276ff17 100644 --- a/Jint.Tests/Runtime/JsonTests.cs +++ b/Jint.Tests/Runtime/JsonTests.cs @@ -1,3 +1,4 @@ +using System.Globalization; using Jint.Native; using Jint.Native.Json; using Jint.Native.Object; @@ -104,6 +105,8 @@ public void ShouldParseEscapedCharactersCorrectly(string json, string expectedCh [InlineData("truE", "Unexpected token ILLEGAL in JSON at position 0")] [InlineData("nul", "Unexpected token ILLEGAL in JSON at position 0")] [InlineData("\"ab\t\"", "Invalid character in JSON at position 3")] // invalid char in string literal + [InlineData("\"\u0001abc\"", "Invalid character in JSON at position 1")] // control char right after the opening quote + [InlineData("\"abc\u0005def\"", "Invalid character in JSON at position 4")] // control char in the middle of a bulk run [InlineData("\"ab", "Unexpected end of JSON input at position 3")] // unterminated string literal [InlineData("alpha", "Unexpected token 'a' in JSON at position 0")] [InlineData("[1,\na]", "Unexpected token 'a' in JSON at position 4")] // multiline @@ -600,6 +603,210 @@ public void CanParseNestedObjectsWithSharedAndDistinctLayouts() Assert.True(ok); } + [Fact] + public void CanBulkScanStringLiteralsWithEscapesAtEdges() + { + var parser = new JsonParser(new Engine()); + + // escape at the very start, in the middle and at the end of the bulk run + Assert.Equal("\nabc", parser.Parse("\"\\nabc\"").AsString()); + Assert.Equal("abc\ndef", parser.Parse("\"abc\\ndef\"").AsString()); + Assert.Equal("abc\n", parser.Parse("\"abc\\n\"").AsString()); + + // every simple escape back-to-back: JSON "\"\\\/\n\r\t\b\f" + Assert.Equal("\"\\/\n\r\t\b\f", parser.Parse("\"\\\"\\\\\\/\\n\\r\\t\\b\\f\"").AsString()); + + // \uXXXX escapes (BMP) + Assert.Equal("Aé中", parser.Parse("\"\\u0041\\u00e9\\u4e2d\"").AsString()); + + // a run with no escapes at all takes the pure bulk path + Assert.Equal("plain text with spaces", parser.Parse("\"plain text with spaces\"").AsString()); + } + + [Fact] + public void CanBulkScanStringsCrossingInternalBufferBoundary() + { + // ValueStringBuilder starts with a 64-char stack buffer; these inputs force it to grow + // while the bulk Append copies whole spans. + var parser = new JsonParser(new Engine()); + + // a single content run far longer than the 64-char buffer + var long1 = new string('a', 200); + Assert.Equal(long1, parser.Parse("\"" + long1 + "\"").AsString()); + + // escape just before the boundary, then a long run after it + var before = new string('a', 60); + var after = new string('b', 60); + Assert.Equal(before + "\n" + after, parser.Parse("\"" + before + "\\n" + after + "\"").AsString()); + + // escape just after the boundary + var before3 = new string('c', 70); + var after3 = new string('d', 5); + Assert.Equal(before3 + "\t" + after3, parser.Parse("\"" + before3 + "\\t" + after3 + "\"").AsString()); + + // many escapes interspersed straddling the boundary + var json = new System.Text.StringBuilder("\""); + var expected = new System.Text.StringBuilder(); + for (var i = 0; i < 100; i++) + { + json.Append('x').Append("\\n"); + expected.Append('x').Append('\n'); + } + json.Append('"'); + Assert.Equal(expected.ToString(), parser.Parse(json.ToString()).AsString()); + } + + [Fact] + public void BulkScanPreservesUnicodeLineSeparatorTermination() + { + // U+2028 / U+2029 terminate the string scan with the quote still open, which the original + // char-by-char scanner reported as UnexpectedEOS just past the separator. Lock that in. + var parser = new JsonParser(new Engine()); + + var ex = Assert.ThrowsAny(() => parser.Parse("\"ab\u2028cd\"")); + Assert.Equal("Unexpected end of JSON input at position 4", ex.Message); + + var ex2 = Assert.ThrowsAny(() => parser.Parse("\"ab\u2029cd\"")); + Assert.Equal("Unexpected end of JSON input at position 4", ex2.Message); + } + + private const NumberStyles JsonNumberStyles = + NumberStyles.AllowLeadingSign | NumberStyles.AllowDecimalPoint | NumberStyles.AllowExponent; + + [Fact] + public void NumberFastPathIsBitIdenticalToDoubleParse() + { + var parser = new JsonParser(new Engine()); + var random = new Random(20260721); + + for (var iteration = 0; iteration < 200_000; iteration++) + { + var json = GenerateRandomJsonNumber(random); + var expected = BitConverter.DoubleToInt64Bits(double.Parse(json, JsonNumberStyles, CultureInfo.InvariantCulture)); + var actual = BitConverter.DoubleToInt64Bits(parser.Parse(json).AsNumber()); + Assert.True(expected == actual, $"Bit mismatch for '{json}': expected 0x{expected:X16}, actual 0x{actual:X16}"); + } + } + + [Theory] + [InlineData("0")] + [InlineData("-0")] + [InlineData("0.0")] + [InlineData("-0.0")] + [InlineData("0.1")] + [InlineData("0.2")] + [InlineData("0.3")] + [InlineData("1.005")] + [InlineData("-1.5")] + [InlineData("19.99")] + [InlineData("37.774929")] + [InlineData("3.141592653589")] // 13 significant digits, fast path + [InlineData("1.23456789012345")] // 15 total digits, fast path + [InlineData("123456789012.345")] // 15 total digits, fast path + [InlineData("0.00000000000001")] // tiny fraction, small numerator + [InlineData("0.123456789012345")] // 16 total digit chars -> falls back to double.Parse + [InlineData("1234567890123456.5")] // 16 integer digits -> falls back + [InlineData("99999999999999.9")] // 16 total digits -> falls back + [InlineData("9007199254740992")] // 2^53 exactly (integer path) + [InlineData("1.7976931348623157e308")] // exponent -> falls back + [InlineData("5e-324")] // exponent -> falls back + public void NumberFastPathEdgeCasesMatchDoubleParse(string json) + { + var parser = new JsonParser(new Engine()); + var expected = BitConverter.DoubleToInt64Bits(double.Parse(json, JsonNumberStyles, CultureInfo.InvariantCulture)); + var actual = BitConverter.DoubleToInt64Bits(parser.Parse(json).AsNumber()); + Assert.Equal(expected, actual); + } + + [Theory] + [InlineData("-0")] + [InlineData("-0.0")] + [InlineData("-0.00")] + public void NumberFastPathMatchesDoubleParseForNegativeZero(string json) + { + // Negative zero is deliberately deferred to double.Parse so the platform-specific sign of zero + // (+0.0 on .NET Framework, -0.0 on .NET Core) is preserved exactly rather than normalized. + var expected = BitConverter.DoubleToInt64Bits(double.Parse(json, JsonNumberStyles, CultureInfo.InvariantCulture)); + var parser = new JsonParser(new Engine()); + Assert.Equal(expected, BitConverter.DoubleToInt64Bits(parser.Parse(json).AsNumber())); + } + + [Theory] + [InlineData("0")] + [InlineData("0.0")] + [InlineData("0.5")] + public void NumberFastPathProducesPositiveZeroWhereExpected(string json) + { + var positiveZeroBits = BitConverter.DoubleToInt64Bits(0.0); + var parser = new JsonParser(new Engine()); + var bits = BitConverter.DoubleToInt64Bits(parser.Parse(json).AsNumber()); + if (json == "0.5") + { + Assert.NotEqual(positiveZeroBits, bits); + } + else + { + Assert.Equal(positiveZeroBits, bits); + } + } + + private static string GenerateRandomJsonNumber(Random random) + { + var sb = new System.Text.StringBuilder(); + if (random.Next(2) == 0) + { + sb.Append('-'); + } + + // integer part: either a lone '0' or a non-zero-leading run (valid JSON grammar) + if (random.Next(10) == 0) + { + sb.Append('0'); + } + else + { + var intLength = 1 + random.Next(18); // 1..18 digits, straddling the 15-digit fast-path bound + sb.Append((char) ('1' + random.Next(9))); + for (var k = 1; k < intLength; k++) + { + sb.Append((char) ('0' + random.Next(10))); + } + } + + // optional fraction + if (random.Next(2) == 0) + { + sb.Append('.'); + var fractionLength = 1 + random.Next(18); + for (var k = 0; k < fractionLength; k++) + { + sb.Append((char) ('0' + random.Next(10))); + } + } + + // optional exponent (always exercises the fallback path) + if (random.Next(5) == 0) + { + sb.Append(random.Next(2) == 0 ? 'e' : 'E'); + var sign = random.Next(3); + if (sign == 1) + { + sb.Append('+'); + } + else if (sign == 2) + { + sb.Append('-'); + } + sb.Append((char) ('0' + random.Next(10))); + if (random.Next(2) == 0) + { + sb.Append((char) ('0' + random.Next(10))); + } + } + + return sb.ToString(); + } + private static string GenerateDeepNestedArray(int depth) { string arrayOpen = new string('[', depth); diff --git a/Jint/Native/Json/JsonParser.cs b/Jint/Native/Json/JsonParser.cs index f0ba90f2d..8e98dc27b 100644 --- a/Jint/Native/Json/JsonParser.cs +++ b/Jint/Native/Json/JsonParser.cs @@ -1,3 +1,4 @@ +using System.Buffers; using System.Diagnostics.CodeAnalysis; using System.Globalization; using System.Runtime.CompilerServices; @@ -152,12 +153,6 @@ private static bool IsWhiteSpace(char ch) (ch == '\r'); } - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static bool IsLineTerminator(char ch) - { - return (ch == 10) || (ch == 13) || (ch == 0x2028) || (ch == 0x2029); - } - private char ScanHexEscape() { int code = char.MinValue; @@ -316,6 +311,17 @@ private Token ScanNumericLiteral() { value = JsNumber.Create(longResult); } +#if NET8_0_OR_GREATER + else if (TryParseDecimalFast(number.AsSpan(), out var fastValue)) + { + // Bit-identical to the double.Parse fallback below for every shape it accepts (proven by + // JsonTests.NumberFastPath*); the constructor keeps the Types.Number tagging the + // double.Parse path produces. Gated to net8+ because only there is double.Parse guaranteed + // to be IEEE correctly-rounded — on the legacy runtimes it can differ by an ULP, so we keep + // deferring to it to stay byte-for-byte identical to the pre-existing behavior. + value = new JsNumber(fastValue); + } +#endif else { value = new JsNumber(double.Parse(number, NumberStyles.AllowLeadingSign | NumberStyles.AllowDecimalPoint | NumberStyles.AllowExponent, CultureInfo.InvariantCulture)); @@ -324,6 +330,105 @@ private Token ScanNumericLiteral() return CreateToken(Tokens.Number, number, '\0', value, new TextRange(start, _index)); } +#if NET8_0_OR_GREATER + // Exact power-of-ten scaling factors for the fraction fast path. 10^0..10^15 are all exactly + // representable doubles (each significand fits in 53 bits), so dividing an exact long numerator + // (< 10^15 < 2^53) by one of these yields the correctly-rounded quotient — bit-identical to + // double.Parse on the same text (see JsonTests.NumberFastPath*). The fraction path only ever indexes + // 0..14 (at least one integer digit is required), the extra slots are headroom. + private static readonly double[] PowersOf10 = + { + 1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, + 1e8, 1e9, 1e10, 1e11, 1e12, 1e13, 1e14, 1e15, + }; + + /// + /// Fast path for the dominant JSON number shape: an optional leading '-', integer digits, an + /// optional '.fraction' and NO exponent, with at most 15 total digits. Every digit is accumulated + /// into a single exact numerator which is divided once by an exact power of ten, + /// avoiding the general floating-point parse. Returns — deferring to + /// double.Parse — for anything outside that shape (an exponent, a 16th significant digit, or + /// any unexpected trailing character). The result is bit-identical to double.Parse for every + /// accepted input because both operands of the division are exactly representable. + /// + private static bool TryParseDecimalFast(ReadOnlySpan text, out double result) + { + result = 0; + var len = text.Length; + var i = 0; + + var negative = false; + if (len > 0 && text[0] == '-') + { + negative = true; + i = 1; + } + + long mantissa = 0; + var totalDigits = 0; + var intDigits = 0; + while (i < len) + { + var c = text[i]; + if (!IsDecimalDigit(c)) + { + break; + } + if (totalDigits == 15) + { + return false; // more significant digits than the exact long numerator can hold + } + mantissa = mantissa * 10 + (c - '0'); + totalDigits++; + intDigits++; + i++; + } + + if (intDigits == 0) + { + return false; // no integer digit (e.g. a lone '.') — let the general path handle it + } + + var fractionDigits = 0; + if (i < len && text[i] == '.') + { + i++; + while (i < len) + { + var c = text[i]; + if (!IsDecimalDigit(c)) + { + break; + } + if (totalDigits == 15) + { + return false; + } + mantissa = mantissa * 10 + (c - '0'); + totalDigits++; + fractionDigits++; + i++; + } + } + + if (i != len) + { + return false; // exponent or other trailing character — not covered by the fast path + } + + if (negative && mantissa == 0) + { + // Negative zero ("-0", "-0.0", ...): defer to double.Parse so the sign of zero always comes + // from the platform parser rather than being synthesized here. + return false; + } + + var value = fractionDigits == 0 ? (double) mantissa : (double) mantissa / PowersOf10[fractionDigits]; + result = negative ? -value : value; + return true; + } +#endif + private Token ScanBooleanLiteral() { var start = _index; @@ -366,22 +471,93 @@ private Token ScanNullLiteral() return null!; } +#if NET8_0_OR_GREATER + // Characters that terminate a bulk string-content run: the closing quote, the escape backslash, + // every control character (< 0x20, which JSON forbids unescaped) and the two Unicode line + // separators the original char-by-char scanner treated as terminators. IndexOfAny over this set + // finds the first such character exactly where the per-char loop would have stopped. + private static readonly SearchValues JsonStringStopChars = CreateStringStopChars(); + + private static SearchValues CreateStringStopChars() + { + Span stops = stackalloc char[36]; + for (var i = 0; i < 32; i++) + { + stops[i] = (char) i; + } + stops[32] = '"'; + stops[33] = '\\'; + stops[34] = '\u2028'; + stops[35] = '\u2029'; + return SearchValues.Create(stops); + } +#else + // Portable fallback: the vectorized IndexOfAny locates the next quote/backslash, then we make sure + // no control character (< 0x20) or line separator (U+2028/U+2029) appears earlier, so the returned + // index matches the NET8 SearchValues path (and the original per-char scanner). + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int IndexOfStringStop(ReadOnlySpan span) + { + var qb = span.IndexOfAny('"', '\\'); + var limit = qb < 0 ? span.Length : qb; + for (var i = 0; i < limit; i++) + { + var c = span[i]; + if (c < ' ' || c == '\u2028' || c == '\u2029') + { + return i; + } + } + return qb; + } +#endif + private Token ScanStringLiteral(ref State state, bool isPropertyKey) { char quote = _source[_index]; int start = _index; ++_index; + var source = _source; + var length = _length; + using var sb = new ValueStringBuilder(stackalloc char[64]); var scanned = 0; - while (_index < _length) + while (_index < length) { - if (++scanned % ConstraintCheckInterval == 0) + // Bulk fast path: copy the run of ordinary characters up to the next quote, backslash, + // control character (< 0x20) or Unicode line separator in one shot. The search lands on + // exactly the character the original per-char loop would have stopped at, so escapes and + // every error position are preserved byte-for-byte. + var remaining = source.AsSpan(_index, length - _index); +#if NET8_0_OR_GREATER + var stop = remaining.IndexOfAny(JsonStringStopChars); +#else + var stop = IndexOfStringStop(remaining); +#endif + if (stop < 0) + { + // No closing quote (and no special character) remains: unterminated literal, reported + // below at position == length just like the original scanner. + _index = length; + break; + } + + scanned += stop + 1; + if (scanned >= ConstraintCheckInterval) { + scanned = 0; _engine.Constraints.Check(); } - char ch = _source[_index++]; + if (stop > 0) + { + sb.Append(remaining.Slice(0, stop)); + } + + var pos = _index + stop; + char ch = source[pos]; + _index = pos + 1; if (ch == quote) { @@ -391,12 +567,12 @@ private Token ScanStringLiteral(ref State state, bool isPropertyKey) if (ch <= 31) { - ThrowError(_index - 1, Messages.InvalidCharacter); + ThrowError(pos, Messages.InvalidCharacter); } if (ch == '\\') { - ch = _source.CharCodeAt(_index++); + ch = source.CharCodeAt(_index++); switch (ch) { @@ -432,13 +608,11 @@ private Token ScanStringLiteral(ref State state, bool isPropertyKey) break; } } - else if (IsLineTerminator(ch)) - { - break; - } else { - sb.Append(ch); + // Unicode line separator (U+2028 / U+2029): the original char-by-char scanner breaks + // here with the quote still open, which surfaces as UnexpectedEOS at _index below. + break; } }