diff --git a/src/libraries/Common/src/System/Text/Json/PooledByteBufferWriter.cs b/src/libraries/Common/src/System/Text/Json/PooledByteBufferWriter.cs index 865d13eabae76..75424aed0d3cc 100644 --- a/src/libraries/Common/src/System/Text/Json/PooledByteBufferWriter.cs +++ b/src/libraries/Common/src/System/Text/Json/PooledByteBufferWriter.cs @@ -22,11 +22,18 @@ internal sealed class PooledByteBufferWriter : IBufferWriter, IDisposable private const int MinimumBufferSize = 256; + // Value copied from Array.MaxLength in System.Private.CoreLib/src/libraries/System.Private.CoreLib/src/System/Array.cs. + public const int MaximumBufferSize = 0X7FFFFFC7; + private PooledByteBufferWriter() { +#if NETCOREAPP + // Ensure we are in sync with the Array.MaxLength implementation. + Debug.Assert(MaximumBufferSize == Array.MaxLength); +#endif } - public PooledByteBufferWriter(int initialCapacity) + public PooledByteBufferWriter(int initialCapacity) : this() { Debug.Assert(initialCapacity > 0); @@ -125,17 +132,16 @@ public void Advance(int count) Debug.Assert(_rentedBuffer != null); Debug.Assert(count >= 0); Debug.Assert(_index <= _rentedBuffer.Length - count); - _index += count; } - public Memory GetMemory(int sizeHint = 0) + public Memory GetMemory(int sizeHint = MinimumBufferSize) { CheckAndResizeBuffer(sizeHint); return _rentedBuffer.AsMemory(_index); } - public Span GetSpan(int sizeHint = 0) + public Span GetSpan(int sizeHint = MinimumBufferSize) { CheckAndResizeBuffer(sizeHint); return _rentedBuffer.AsSpan(_index); @@ -168,26 +174,28 @@ internal void WriteToStream(Stream destination) private void CheckAndResizeBuffer(int sizeHint) { Debug.Assert(_rentedBuffer != null); - Debug.Assert(sizeHint >= 0); + Debug.Assert(sizeHint > 0); + + int currentLength = _rentedBuffer.Length; + int availableSpace = currentLength - _index; - if (sizeHint == 0) + // If we've reached ~1GB written, grow to the maximum buffer + // length to avoid incessant minimal growths causing perf issues. + if (_index >= MaximumBufferSize / 2) { - sizeHint = MinimumBufferSize; + sizeHint = Math.Max(sizeHint, MaximumBufferSize - currentLength); } - int availableSpace = _rentedBuffer.Length - _index; - if (sizeHint > availableSpace) { - int currentLength = _rentedBuffer.Length; int growBy = Math.Max(sizeHint, currentLength); int newSize = currentLength + growBy; - if ((uint)newSize > int.MaxValue) + if ((uint)newSize > MaximumBufferSize) { newSize = currentLength + sizeHint; - if ((uint)newSize > int.MaxValue) + if ((uint)newSize > MaximumBufferSize) { ThrowHelper.ThrowOutOfMemoryException_BufferMaximumSizeExceeded((uint)newSize); } @@ -200,9 +208,9 @@ private void CheckAndResizeBuffer(int sizeHint) Debug.Assert(oldBuffer.Length >= _index); Debug.Assert(_rentedBuffer.Length >= _index); - Span previousBuffer = oldBuffer.AsSpan(0, _index); - previousBuffer.CopyTo(_rentedBuffer); - previousBuffer.Clear(); + Span oldBufferAsSpan = oldBuffer.AsSpan(0, _index); + oldBufferAsSpan.CopyTo(_rentedBuffer); + oldBufferAsSpan.Clear(); ArrayPool.Shared.Return(oldBuffer); } diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/Object.WriteTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/Object.WriteTests.cs index d2000d649acc0..77e4eb311a184 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/Object.WriteTests.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/Object.WriteTests.cs @@ -2,7 +2,9 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Collections.Generic; +using System.Linq; using System.Text.Encodings.Web; +using System.Text.Json.Tests; using Xunit; namespace System.Text.Json.Serialization.Tests @@ -146,5 +148,80 @@ public static void WriteObjectWithNumberHandling() var options = new JsonSerializerOptions { NumberHandling = JsonNumberHandling.AllowReadingFromString }; JsonSerializer.Serialize(new object(), options); } + + /// + /// This test is constrained to run on Windows and MacOSX because it causes + /// problems on Linux due to the way deferred memory allocation works. On Linux, the allocation can + /// succeed even if there is not enough memory but then the test may get killed by the OOM killer at the + /// time the memory is accessed which triggers the full memory allocation. + /// Also see + /// + [PlatformSpecific(TestPlatforms.Windows | TestPlatforms.OSX)] + [ConditionalFact(nameof(Utf8JsonWriterTests.IsX64))] + [OuterLoop] + public static void SerializeLargeListOfObjects() + { + Dto dto = new() + { + Prop1 = int.MaxValue, + Prop2 = int.MinValue, + Prop3 = "AC", + Prop4 = 500, + Prop5 = int.MaxValue / 2, + Prop6 = 250M, + Prop7 = 250M, + Prop8 = 250M, + Prop9 = 250M, + Prop10 = 250M, + Prop11 = 150M, + Prop12 = 150M, + Prop13 = DateTimeOffset.MaxValue, + Prop14 = DateTimeOffset.MaxValue, + Prop15 = DateTimeOffset.MaxValue, + Prop16 = DateTimeOffset.MaxValue, + Prop17 = 3, + Prop18 = DateTime.MaxValue, + Prop19 = DateTime.MaxValue, + Prop20 = 25000, + Prop21 = DateTime.MaxValue + }; + + // It takes a little over 4,338,000 items to reach a payload size above the Array.MaxLength value. + List items = Enumerable.Repeat(dto, 4_338_000).ToList(); + + try + { + JsonSerializer.SerializeToUtf8Bytes(items); + } + catch (OutOfMemoryException) { } + + items.AddRange(Enumerable.Repeat(dto, 1000).ToList()); + Assert.Throws(() => JsonSerializer.SerializeToUtf8Bytes(items)); + } + + class Dto + { + public int Prop1 { get; set; } + public int Prop2 { get; set; } + public string Prop3 { get; set; } + public int Prop4 { get; set; } + public long Prop5 { get; set; } + public decimal Prop6 { get; set; } + public decimal Prop7 { get; set; } + public decimal Prop8 { get; set; } + public decimal Prop9 { get; set; } + public decimal Prop10 { get; set; } + public decimal Prop11 { get; set; } + public decimal Prop12 { get; set; } + public DateTimeOffset Prop13 { get; set; } + public DateTimeOffset Prop14 { get; set; } + public DateTimeOffset Prop15 { get; set; } + public DateTimeOffset Prop16 { get; set; } + public int Prop17 { get; set; } + public DateTime Prop18 { get; set; } + public DateTime Prop19 { get; set; } + public int Prop20 { get; set; } + public DateTime Prop21 { get; set; } + } } }