Skip to content

Make stream buffer size configurable. #1286

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Jul 26, 2020
Merged

Make stream buffer size configurable. #1286

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
36b3fab
Make stream buffer size configurable. Fixes #1276
JimBobSquarePants Jul 24, 2020
8fe053f
Update ConfigurationTests.cs
JimBobSquarePants Jul 24, 2020
10b57fd
Throw, don't adjust.
JimBobSquarePants Jul 24, 2020
881b79b
Sneak in some docs fixes
JimBobSquarePants Jul 25, 2020
91a9783
Add sanitation and make tests parametric.
JimBobSquarePants Jul 26, 2020
6aa6472
Fix formatting.
JimBobSquarePants Jul 26, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 23 additions & 1 deletion src/ImageSharp/Configuration.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Net.Http;
using SixLabors.ImageSharp.Formats;
using SixLabors.ImageSharp.Formats.Bmp;
using SixLabors.ImageSharp.Formats.Gif;
Expand All @@ -27,6 +26,9 @@ public sealed class Configuration
/// </summary>
private static readonly Lazy<Configuration> Lazy = new Lazy<Configuration>(CreateDefaultInstance);

private const int MinStreamProcessingBufferSize = 128;
private const int DefaultStreamProcessingBufferSize = 8096;
private int streamProcessingBufferSize = DefaultStreamProcessingBufferSize;
private int maxDegreeOfParallelism = Environment.ProcessorCount;

/// <summary>
Expand Down Expand Up @@ -75,6 +77,25 @@ public int MaxDegreeOfParallelism
}
}

/// <summary>
/// Gets or sets the size of the buffer to use when working with streams.
/// Intitialized with <see cref="DefaultStreamProcessingBufferSize"/> by default
/// and can accept a minimum value of <see cref="MinStreamProcessingBufferSize"/>.
/// </summary>
public int StreamProcessingBufferSize
{
get => this.streamProcessingBufferSize;
set
{
if (value < MinStreamProcessingBufferSize)
{
value = MinStreamProcessingBufferSize;
}

this.streamProcessingBufferSize = value;
}
}

/// <summary>
/// Gets a set of properties for the Congiguration.
/// </summary>
Expand Down Expand Up @@ -145,6 +166,7 @@ public Configuration Clone()
return new Configuration
{
MaxDegreeOfParallelism = this.MaxDegreeOfParallelism,
StreamProcessingBufferSize = this.StreamProcessingBufferSize,
ImageFormatsManager = this.ImageFormatsManager,
MemoryAllocator = this.MemoryAllocator,
ImageOperationsProvider = this.ImageOperationsProvider,
Expand Down
8 changes: 4 additions & 4 deletions src/ImageSharp/Formats/Bmp/BmpDecoder.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ public Image<TPixel> Decode<TPixel>(Configuration configuration, Stream stream)

try
{
using var bufferedStream = new BufferedReadStream(stream);
using var bufferedStream = new BufferedReadStream(configuration, stream);
return decoder.Decode<TPixel>(bufferedStream);
}
catch (InvalidMemoryOperationException ex)
Expand All @@ -64,7 +64,7 @@ public async Task<Image<TPixel>> DecodeAsync<TPixel>(Configuration configuration

try
{
using var bufferedStream = new BufferedReadStream(stream);
using var bufferedStream = new BufferedReadStream(configuration, stream);
return await decoder.DecodeAsync<TPixel>(bufferedStream).ConfigureAwait(false);
}
catch (InvalidMemoryOperationException ex)
Expand All @@ -84,7 +84,7 @@ public IImageInfo Identify(Configuration configuration, Stream stream)
{
Guard.NotNull(stream, nameof(stream));

using var bufferedStream = new BufferedReadStream(stream);
using var bufferedStream = new BufferedReadStream(configuration, stream);
return new BmpDecoderCore(configuration, this).Identify(bufferedStream);
}

Expand All @@ -93,7 +93,7 @@ public Task<IImageInfo> IdentifyAsync(Configuration configuration, Stream stream
{
Guard.NotNull(stream, nameof(stream));

using var bufferedStream = new BufferedReadStream(stream);
using var bufferedStream = new BufferedReadStream(configuration, stream);
return new BmpDecoderCore(configuration, this).IdentifyAsync(bufferedStream);
}
}
Expand Down
8 changes: 4 additions & 4 deletions src/ImageSharp/Formats/Gif/GifDecoder.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ public Image<TPixel> Decode<TPixel>(Configuration configuration, Stream stream)

try
{
using var bufferedStream = new BufferedReadStream(stream);
using var bufferedStream = new BufferedReadStream(configuration, stream);
return decoder.Decode<TPixel>(bufferedStream);
}
catch (InvalidMemoryOperationException ex)
Expand All @@ -59,7 +59,7 @@ public async Task<Image<TPixel>> DecodeAsync<TPixel>(Configuration configuration

try
{
using var bufferedStream = new BufferedReadStream(stream);
using var bufferedStream = new BufferedReadStream(configuration, stream);
return await decoder.DecodeAsync<TPixel>(bufferedStream).ConfigureAwait(false);
}
catch (InvalidMemoryOperationException ex)
Expand All @@ -84,7 +84,7 @@ public IImageInfo Identify(Configuration configuration, Stream stream)

var decoder = new GifDecoderCore(configuration, this);

using var bufferedStream = new BufferedReadStream(stream);
using var bufferedStream = new BufferedReadStream(configuration, stream);
return decoder.Identify(bufferedStream);
}

Expand All @@ -95,7 +95,7 @@ public Task<IImageInfo> IdentifyAsync(Configuration configuration, Stream stream

var decoder = new GifDecoderCore(configuration, this);

using var bufferedStream = new BufferedReadStream(stream);
using var bufferedStream = new BufferedReadStream(configuration, stream);
return decoder.IdentifyAsync(bufferedStream);
}
}
Expand Down
8 changes: 4 additions & 4 deletions src/ImageSharp/Formats/Jpeg/JpegDecoder.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ public Image<TPixel> Decode<TPixel>(Configuration configuration, Stream stream)
using var decoder = new JpegDecoderCore(configuration, this);
try
{
using var bufferedStream = new BufferedReadStream(stream);
using var bufferedStream = new BufferedReadStream(configuration, stream);
return decoder.Decode<TPixel>(bufferedStream);
}
catch (InvalidMemoryOperationException ex)
Expand All @@ -53,7 +53,7 @@ public async Task<Image<TPixel>> DecodeAsync<TPixel>(Configuration configuration
using var decoder = new JpegDecoderCore(configuration, this);
try
{
using var bufferedStream = new BufferedReadStream(stream);
using var bufferedStream = new BufferedReadStream(configuration, stream);
return await decoder.DecodeAsync<TPixel>(bufferedStream).ConfigureAwait(false);
}
catch (InvalidMemoryOperationException ex)
Expand All @@ -77,7 +77,7 @@ public IImageInfo Identify(Configuration configuration, Stream stream)
Guard.NotNull(stream, nameof(stream));

using var decoder = new JpegDecoderCore(configuration, this);
using var bufferedStream = new BufferedReadStream(stream);
using var bufferedStream = new BufferedReadStream(configuration, stream);

return decoder.Identify(bufferedStream);
}
Expand All @@ -88,7 +88,7 @@ public Task<IImageInfo> IdentifyAsync(Configuration configuration, Stream stream
Guard.NotNull(stream, nameof(stream));

using var decoder = new JpegDecoderCore(configuration, this);
using var bufferedStream = new BufferedReadStream(stream);
using var bufferedStream = new BufferedReadStream(configuration, stream);

return decoder.IdentifyAsync(bufferedStream);
}
Expand Down
8 changes: 4 additions & 4 deletions src/ImageSharp/Formats/Png/PngDecoder.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ public Image<TPixel> Decode<TPixel>(Configuration configuration, Stream stream)

try
{
using var bufferedStream = new BufferedReadStream(stream);
using var bufferedStream = new BufferedReadStream(configuration, stream);
return decoder.Decode<TPixel>(bufferedStream);
}
catch (InvalidMemoryOperationException ex)
Expand All @@ -50,7 +50,7 @@ public async Task<Image<TPixel>> DecodeAsync<TPixel>(Configuration configuration

try
{
using var bufferedStream = new BufferedReadStream(stream);
using var bufferedStream = new BufferedReadStream(configuration, stream);
return await decoder.DecodeAsync<TPixel>(bufferedStream).ConfigureAwait(false);
}
catch (InvalidMemoryOperationException ex)
Expand All @@ -71,15 +71,15 @@ public async Task<Image<TPixel>> DecodeAsync<TPixel>(Configuration configuration
public IImageInfo Identify(Configuration configuration, Stream stream)
{
var decoder = new PngDecoderCore(configuration, this);
using var bufferedStream = new BufferedReadStream(stream);
using var bufferedStream = new BufferedReadStream(configuration, stream);
return decoder.Identify(bufferedStream);
}

/// <inheritdoc/>
public Task<IImageInfo> IdentifyAsync(Configuration configuration, Stream stream)
{
var decoder = new PngDecoderCore(configuration, this);
using var bufferedStream = new BufferedReadStream(stream);
using var bufferedStream = new BufferedReadStream(configuration, stream);
return decoder.IdentifyAsync(bufferedStream);
}
}
Expand Down
3 changes: 1 addition & 2 deletions src/ImageSharp/Formats/Png/PngDecoderCore.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Text;
using System.Threading.Tasks;
using SixLabors.ImageSharp.Formats.Png.Chunks;
using SixLabors.ImageSharp.Formats.Png.Filters;
using SixLabors.ImageSharp.Formats.Png.Zlib;
Expand Down Expand Up @@ -1027,7 +1026,7 @@ private void ReadInternationalTextChunk(PngMetadata metadata, ReadOnlySpan<byte>
private bool TryUncompressTextData(ReadOnlySpan<byte> compressedData, Encoding encoding, out string value)
{
using (var memoryStream = new MemoryStream(compressedData.ToArray()))
using (var bufferedStream = new BufferedReadStream(memoryStream))
using (var bufferedStream = new BufferedReadStream(this.Configuration, memoryStream))
using (var inflateStream = new ZlibInflateStream(bufferedStream))
{
if (!inflateStream.AllocateNewBytes(compressedData.Length, false))
Expand Down
8 changes: 4 additions & 4 deletions src/ImageSharp/Formats/Tga/TgaDecoder.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ public Image<TPixel> Decode<TPixel>(Configuration configuration, Stream stream)

try
{
using var bufferedStream = new BufferedReadStream(stream);
using var bufferedStream = new BufferedReadStream(configuration, stream);
return decoder.Decode<TPixel>(bufferedStream);
}
catch (InvalidMemoryOperationException ex)
Expand Down Expand Up @@ -52,7 +52,7 @@ public async Task<Image<TPixel>> DecodeAsync<TPixel>(Configuration configuration

try
{
using var bufferedStream = new BufferedReadStream(stream);
using var bufferedStream = new BufferedReadStream(configuration, stream);
return await decoder.DecodeAsync<TPixel>(bufferedStream).ConfigureAwait(false);
}
catch (InvalidMemoryOperationException ex)
Expand All @@ -75,7 +75,7 @@ public IImageInfo Identify(Configuration configuration, Stream stream)
{
Guard.NotNull(stream, nameof(stream));

using var bufferedStream = new BufferedReadStream(stream);
using var bufferedStream = new BufferedReadStream(configuration, stream);
return new TgaDecoderCore(configuration, this).Identify(bufferedStream);
}

Expand All @@ -84,7 +84,7 @@ public Task<IImageInfo> IdentifyAsync(Configuration configuration, Stream stream
{
Guard.NotNull(stream, nameof(stream));

using var bufferedStream = new BufferedReadStream(stream);
using var bufferedStream = new BufferedReadStream(configuration, stream);
return new TgaDecoderCore(configuration, this).IdentifyAsync(bufferedStream);
}
}
Expand Down
47 changes: 27 additions & 20 deletions src/ImageSharp/IO/BufferedReadStream.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,7 @@ namespace SixLabors.ImageSharp.IO
/// </summary>
internal sealed class BufferedReadStream : Stream
{
/// <summary>
/// The length, in bytes, of the underlying buffer.
/// </summary>
public const int BufferLength = 8192;

private const int MaxBufferIndex = BufferLength - 1;
private readonly int maxBufferIndex;

private readonly byte[] readBuffer;

Expand All @@ -38,9 +33,11 @@ internal sealed class BufferedReadStream : Stream
/// <summary>
/// Initializes a new instance of the <see cref="BufferedReadStream"/> class.
/// </summary>
/// <param name="configuration">The configuration which allows altering default behaviour or extending the library.</param>
/// <param name="stream">The input stream.</param>
public BufferedReadStream(Stream stream)
public BufferedReadStream(Configuration configuration, Stream stream)
{
Guard.NotNull(configuration, nameof(configuration));
Guard.IsTrue(stream.CanRead, nameof(stream), "Stream must be readable.");
Guard.IsTrue(stream.CanSeek, nameof(stream), "Stream must be seekable.");

Expand All @@ -55,16 +52,26 @@ public BufferedReadStream(Stream stream)
this.BaseStream = stream;
this.Position = (int)stream.Position;
this.Length = stream.Length;

this.readBuffer = ArrayPool<byte>.Shared.Rent(BufferLength);
this.BufferSize = configuration.StreamProcessingBufferSize;
this.maxBufferIndex = this.BufferSize - 1;
this.readBuffer = ArrayPool<byte>.Shared.Rent(this.BufferSize);
this.readBufferHandle = new Memory<byte>(this.readBuffer).Pin();
unsafe
{
this.pinnedReadBuffer = (byte*)this.readBufferHandle.Pointer;
}

// This triggers a full read on first attempt.
this.readBufferIndex = BufferLength;
this.readBufferIndex = this.BufferSize;
}

/// <summary>
/// Gets the size, in bytes, of the underlying buffer.
/// </summary>
public int BufferSize
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
get;
}

/// <inheritdoc/>
Expand All @@ -91,7 +98,7 @@ public override long Position
// Base stream seek will throw for us if invalid.
this.BaseStream.Seek(value, SeekOrigin.Begin);
this.readerPosition = value;
this.readBufferIndex = BufferLength;
this.readBufferIndex = this.BufferSize;
}
}
}
Expand Down Expand Up @@ -125,7 +132,7 @@ public override int ReadByte()

// Our buffer has been read.
// We need to refill and start again.
if (this.readBufferIndex > MaxBufferIndex)
if (this.readBufferIndex > this.maxBufferIndex)
{
this.FillReadBuffer();
}
Expand All @@ -142,14 +149,14 @@ public override int ReadByte()
public override int Read(byte[] buffer, int offset, int count)
{
// Too big for our buffer. Read directly from the stream.
if (count > BufferLength)
if (count > this.BufferSize)
{
return this.ReadToBufferDirectSlow(buffer, offset, count);
}

// Too big for remaining buffer but less than entire buffer length
// Copy to buffer then read from there.
if (count + this.readBufferIndex > BufferLength)
if (count + this.readBufferIndex > this.BufferSize)
{
return this.ReadToBufferViaCopySlow(buffer, offset, count);
}
Expand All @@ -164,14 +171,14 @@ public override int Read(Span<byte> buffer)
{
// Too big for our buffer. Read directly from the stream.
int count = buffer.Length;
if (count > BufferLength)
if (count > this.BufferSize)
{
return this.ReadToBufferDirectSlow(buffer);
}

// Too big for remaining buffer but less than entire buffer length
// Copy to buffer then read from there.
if (count + this.readBufferIndex > BufferLength)
if (count + this.readBufferIndex > this.BufferSize)
{
return this.ReadToBufferViaCopySlow(buffer);
}
Expand All @@ -192,7 +199,7 @@ public override void Flush()
}

// Reset to trigger full read on next attempt.
this.readBufferIndex = BufferLength;
this.readBufferIndex = this.BufferSize;
}

/// <inheritdoc/>
Expand Down Expand Up @@ -249,7 +256,7 @@ protected override void Dispose(bool disposing)
private bool IsInReadBuffer(long newPosition, out long index)
{
index = newPosition - this.readerPosition + this.readBufferIndex;
return index > -1 && index < BufferLength;
return index > -1 && index < this.BufferSize;
}

[MethodImpl(MethodImplOptions.NoInlining)]
Expand All @@ -267,10 +274,10 @@ private void FillReadBuffer()
int i;
do
{
i = baseStream.Read(this.readBuffer, n, BufferLength - n);
i = baseStream.Read(this.readBuffer, n, this.BufferSize - n);
n += i;
}
while (n < BufferLength && i > 0);
while (n < this.BufferSize && i > 0);

this.readBufferIndex = 0;
}
Expand Down
Loading