diff --git a/src/SharpCompress/Common/Rar/RarCryptoWrapper.cs b/src/SharpCompress/Common/Rar/RarCryptoWrapper.cs index 54ca27393..6e1ed222c 100644 --- a/src/SharpCompress/Common/Rar/RarCryptoWrapper.cs +++ b/src/SharpCompress/Common/Rar/RarCryptoWrapper.cs @@ -63,9 +63,9 @@ public override Task ReadAsync( int offset, int count, CancellationToken cancellationToken - ) => ReadAndDecryptAsync(buffer, offset, count, cancellationToken); + ) => ReadAndDecryptAsync(buffer, offset, count, cancellationToken).AsTask(); - private async Task ReadAndDecryptAsync( + private async ValueTask ReadAndDecryptAsync( byte[] buffer, int offset, int count, diff --git a/src/SharpCompress/Compressors/LZMA/LZ/LzBinTree.cs b/src/SharpCompress/Compressors/LZMA/LZ/LzBinTree.cs index bc0a8a7ae..85795e742 100644 --- a/src/SharpCompress/Compressors/LZMA/LZ/LzBinTree.cs +++ b/src/SharpCompress/Compressors/LZMA/LZ/LzBinTree.cs @@ -1,6 +1,7 @@ #nullable disable using System; +using System.Buffers; using System.IO; namespace SharpCompress.Compressors.LZMA.LZ; @@ -11,8 +12,8 @@ internal sealed class BinTree : InWindow private uint _cyclicBufferSize; private uint _matchMaxLen; - private uint[] _son; - private uint[] _hash; + private uint[] _son = []; + private uint[] _hash = []; private uint _cutValue = 0xFF; private uint _hashMask; @@ -108,7 +109,12 @@ uint keepAddBufferAfter var cyclicBufferSize = historySize + 1; if (_cyclicBufferSize != cyclicBufferSize) { - _son = new uint[(_cyclicBufferSize = cyclicBufferSize) * 2]; + if (_son.Length != 0) + { + ArrayPool.Shared.Return(_son); + } + _cyclicBufferSize = cyclicBufferSize; + _son = ArrayPool.Shared.Rent(checked((int)(_cyclicBufferSize * 2))); } var hs = K_BT2_HASH_SIZE; @@ -132,7 +138,27 @@ uint keepAddBufferAfter } if (hs != _hashSizeSum) { - _hash = new uint[_hashSizeSum = hs]; + if (_hash.Length != 0) + { + ArrayPool.Shared.Return(_hash); + } + _hashSizeSum = hs; + _hash = ArrayPool.Shared.Rent(checked((int)_hashSizeSum)); + } + } + + public override void Dispose() + { + base.Dispose(); + if (_son.Length != 0) + { + ArrayPool.Shared.Return(_son); + _son = []; + } + if (_hash.Length != 0) + { + ArrayPool.Shared.Return(_hash); + _hash = []; } } diff --git a/src/SharpCompress/Compressors/LZMA/LZ/LzInWindow.cs b/src/SharpCompress/Compressors/LZMA/LZ/LzInWindow.cs index 9df3c27d2..7ee8bc888 100644 --- a/src/SharpCompress/Compressors/LZMA/LZ/LzInWindow.cs +++ b/src/SharpCompress/Compressors/LZMA/LZ/LzInWindow.cs @@ -1,10 +1,12 @@ #nullable disable +using System; +using System.Buffers; using System.IO; namespace SharpCompress.Compressors.LZMA.LZ; -internal class InWindow +internal class InWindow : IDisposable { public byte[] _bufferBase; // pointer to buffer with data private Stream _stream; @@ -78,7 +80,22 @@ public virtual void ReadBlock() } } - private void Free() => _bufferBase = null; + private void Free() + { + if (_bufferBase is null) + { + return; + } + + ArrayPool.Shared.Return(_bufferBase); + _bufferBase = null; + } + + public virtual void Dispose() + { + ReleaseStream(); + Free(); + } public void Create(uint keepSizeBefore, uint keepSizeAfter, uint keepSizeReserv) { @@ -89,7 +106,7 @@ public void Create(uint keepSizeBefore, uint keepSizeAfter, uint keepSizeReserv) { Free(); _blockSize = blockSize; - _bufferBase = new byte[_blockSize]; + _bufferBase = ArrayPool.Shared.Rent(checked((int)_blockSize)); } _pointerToLastSafePosition = _blockSize - keepSizeAfter; _streamEndWasReached = false; diff --git a/src/SharpCompress/Compressors/LZMA/LZipStream.Async.cs b/src/SharpCompress/Compressors/LZMA/LZipStream.Async.cs index ae8e1074c..2cc7fb8d3 100644 --- a/src/SharpCompress/Compressors/LZMA/LZipStream.Async.cs +++ b/src/SharpCompress/Compressors/LZMA/LZipStream.Async.cs @@ -126,31 +126,40 @@ CancellationToken cancellationToken ) { // Read the header - byte[] header = new byte[6]; - var n = await stream - .ReadAsync(header, 0, header.Length, cancellationToken) - .ConfigureAwait(false); + var header = ArrayPool.Shared.Rent(6); + try + { + var n = await stream.ReadAsync(header, 0, 6, cancellationToken).ConfigureAwait(false); - // TODO: Handle reading only part of the header? + // TODO: Handle reading only part of the header? - if (n != 6) - { - return 0; - } + if (n != 6) + { + return 0; + } - if ( - header[0] != 'L' - || header[1] != 'Z' - || header[2] != 'I' - || header[3] != 'P' - || header[4] != 1 /* version 1 */ - ) + if ( + header[0] != 'L' + || header[1] != 'Z' + || header[2] != 'I' + || header[3] != 'P' + || header[4] != 1 /* version 1 */ + ) + { + return 0; + } + var basePower = header[5] & 0x1F; + var subtractionNumerator = (header[5] & 0xE0) >> 5; + if (basePower < 4 || basePower > 30) + { + return 0; + } + return (1 << basePower) - (subtractionNumerator * (1 << (basePower - 4))); + } + finally { - return 0; + ArrayPool.Shared.Return(header); } - var basePower = header[5] & 0x1F; - var subtractionNumerator = (header[5] & 0xE0) >> 5; - return (1 << basePower) - (subtractionNumerator * (1 << (basePower - 4))); } #if !LEGACY_DOTNET diff --git a/src/SharpCompress/Compressors/LZMA/Lzma2EncoderStream.cs b/src/SharpCompress/Compressors/LZMA/Lzma2EncoderStream.cs index f1ed808c6..51e1b7535 100644 --- a/src/SharpCompress/Compressors/LZMA/Lzma2EncoderStream.cs +++ b/src/SharpCompress/Compressors/LZMA/Lzma2EncoderStream.cs @@ -1,4 +1,5 @@ using System; +using System.Buffers; using System.IO; using System.Threading; using System.Threading.Tasks; @@ -26,11 +27,12 @@ internal sealed class Lzma2EncoderStream : Stream private readonly int _dictionarySize; private readonly int _numFastBytes; private readonly byte[] _buffer; + private readonly byte[] _properties; + private readonly Encoder _encoder; private int _bufferPosition; private bool _isFirstChunk = true; private bool _isDisposed; private byte _lzmaPropertiesByte; - private bool _lzmaPropertiesKnown; /// /// Creates a new LZMA2 encoder stream. @@ -43,14 +45,23 @@ public Lzma2EncoderStream(Stream output, int dictionarySize, int numFastBytes) _output = output; _dictionarySize = dictionarySize; _numFastBytes = numFastBytes; - _buffer = new byte[MAX_UNCOMPRESSED_CHUNK_SIZE]; + _buffer = ArrayPool.Shared.Rent(MAX_UNCOMPRESSED_CHUNK_SIZE); _bufferPosition = 0; + + var encoderProps = new LzmaEncoderProperties(eos: false, _dictionarySize, _numFastBytes); + _encoder = new Encoder(); + _encoder.SetCoderProperties(encoderProps.PropIDs, encoderProps.Properties); + + Span lzmaProperties = stackalloc byte[5]; + _encoder.WriteCoderProperties(lzmaProperties); + _lzmaPropertiesByte = lzmaProperties[0]; + _properties = [EncodeDictionarySize(_dictionarySize)]; } /// /// Gets the 1-byte LZMA2 properties (encoded dictionary size). /// - public byte[] Properties => [EncodeDictionarySize(_dictionarySize)]; + public byte[] Properties => _properties; public override bool CanRead => false; public override bool CanSeek => false; @@ -67,13 +78,13 @@ public override void Write(byte[] buffer, int offset, int count) { while (count > 0) { - var toCopy = Math.Min(count, _buffer.Length - _bufferPosition); + var toCopy = Math.Min(count, MAX_UNCOMPRESSED_CHUNK_SIZE - _bufferPosition); Buffer.BlockCopy(buffer, offset, _buffer, _bufferPosition, toCopy); _bufferPosition += toCopy; offset += toCopy; count -= toCopy; - if (_bufferPosition == _buffer.Length) + if (_bufferPosition == MAX_UNCOMPRESSED_CHUNK_SIZE) { FlushChunk(); } @@ -91,13 +102,13 @@ CancellationToken cancellationToken { cancellationToken.ThrowIfCancellationRequested(); - var toCopy = Math.Min(count, _buffer.Length - _bufferPosition); + var toCopy = Math.Min(count, MAX_UNCOMPRESSED_CHUNK_SIZE - _bufferPosition); Buffer.BlockCopy(buffer, offset, _buffer, _bufferPosition, toCopy); _bufferPosition += toCopy; offset += toCopy; count -= toCopy; - if (_bufferPosition == _buffer.Length) + if (_bufferPosition == MAX_UNCOMPRESSED_CHUNK_SIZE) { await FlushChunkAsync(cancellationToken).ConfigureAwait(false); } @@ -116,13 +127,13 @@ public override async ValueTask WriteAsync( { cancellationToken.ThrowIfCancellationRequested(); - var toCopy = Math.Min(count, _buffer.Length - _bufferPosition); + var toCopy = Math.Min(count, MAX_UNCOMPRESSED_CHUNK_SIZE - _bufferPosition); buffer.Slice(offset, toCopy).Span.CopyTo(_buffer.AsSpan(_bufferPosition, toCopy)); _bufferPosition += toCopy; offset += toCopy; count -= toCopy; - if (_bufferPosition == _buffer.Length) + if (_bufferPosition == MAX_UNCOMPRESSED_CHUNK_SIZE) { await FlushChunkAsync(cancellationToken).ConfigureAwait(false); } @@ -153,6 +164,8 @@ protected override void Dispose(bool disposing) // Write LZMA2 end marker _output.WriteByte(0x00); + _encoder.Dispose(); + ArrayPool.Shared.Return(_buffer); } base.Dispose(disposing); } @@ -179,6 +192,9 @@ public async ValueTask DisposeAsync() #else await _output.WriteAsync(_endMarker, 0, 1).ConfigureAwait(false); #endif + + _encoder.Dispose(); + ArrayPool.Shared.Return(_buffer); } #if !LEGACY_DOTNET || NETSTANDARD2_1 @@ -193,14 +209,15 @@ private void FlushChunk() return; } - var uncompressedData = _buffer.AsSpan(0, _bufferPosition); + var uncompressedSize = _bufferPosition; + var uncompressedData = _buffer.AsSpan(0, uncompressedSize); _bufferPosition = 0; // Try compressing the data - byte[] compressed; + (byte[] Buffer, int Length) compressed; try { - compressed = CompressBlock(uncompressedData); + compressed = CompressBlock(_buffer, uncompressedSize); } catch (Exception ex) when (ex is not OutOfMemoryException and not StackOverflowException) { @@ -215,7 +232,7 @@ private void FlushChunk() && compressed.Length < uncompressedData.Length ) { - WriteCompressedChunk(uncompressedData.Length, compressed); + WriteCompressedChunk(uncompressedData.Length, compressed.Buffer, compressed.Length); } else { @@ -230,13 +247,14 @@ private async ValueTask FlushChunkAsync(CancellationToken cancellationToken = de return; } - var uncompressedData = _buffer.AsMemory(0, _bufferPosition); + var uncompressedSize = _bufferPosition; + var uncompressedData = _buffer.AsMemory(0, uncompressedSize); _bufferPosition = 0; - byte[] compressed; + (byte[] Buffer, int Length) compressed; try { - compressed = CompressBlock(uncompressedData.Span); + compressed = CompressBlock(_buffer, uncompressedSize); } catch (Exception ex) when (ex is not OutOfMemoryException and not StackOverflowException) { @@ -250,7 +268,12 @@ await WriteUncompressedChunksAsync(uncompressedData, cancellationToken) && compressed.Length < uncompressedData.Length ) { - await WriteCompressedChunkAsync(uncompressedData.Length, compressed, cancellationToken) + await WriteCompressedChunkAsync( + uncompressedData.Length, + compressed.Buffer, + compressed.Length, + cancellationToken + ) .ConfigureAwait(false); } else @@ -260,45 +283,26 @@ await WriteUncompressedChunksAsync(uncompressedData, cancellationToken) } } - private byte[] CompressBlock(ReadOnlySpan data) + private (byte[] Buffer, int Length) CompressBlock(byte[] data, int length) { - var encoderProps = new LzmaEncoderProperties(eos: false, _dictionarySize, _numFastBytes); - - var encoder = new Encoder(); - encoder.SetCoderProperties(encoderProps.PropIDs, encoderProps.Properties); - - // Capture the LZMA properties byte (pb/lp/lc encoding) for the chunk header - if (!_lzmaPropertiesKnown) - { - var propBytes = new byte[5]; - encoder.WriteCoderProperties(propBytes); - _lzmaPropertiesByte = propBytes[0]; - _lzmaPropertiesKnown = true; - } - - using var inputMs = new MemoryStream(data.ToArray(), writable: false); + using var inputMs = new MemoryStream(data, 0, length, writable: false); using var outputMs = new PooledMemoryStream(); - encoder.Code(inputMs, outputMs, data.Length, -1, null); + _encoder.Code(inputMs, outputMs, length, -1, null); var fullCompressed = outputMs.ToArray(); // The LZMA range encoder flush writes trailing bytes the decoder doesn't consume. // Trial-decode to find the exact byte count the decoder needs, so the LZMA2 // chunk header reports a compressed size that matches what the decoder reads. - var consumed = FindConsumedBytes(fullCompressed, data.Length); - if (consumed < fullCompressed.Length) - { - return fullCompressed.AsSpan(0, consumed).ToArray(); - } - - return fullCompressed; + var consumed = FindConsumedBytes(fullCompressed, fullCompressed.Length, length); + return (fullCompressed, consumed); } - private int FindConsumedBytes(byte[] compressedData, int uncompressedSize) + private int FindConsumedBytes(byte[] compressedData, int compressedSize, int uncompressedSize) { // Build 5-byte LZMA property header: [pb/lp/lc byte] [dictSize as LE int32] - var props = new byte[5]; + Span props = stackalloc byte[5]; props[0] = _lzmaPropertiesByte; props[1] = (byte)_dictionarySize; props[2] = (byte)(_dictionarySize >> 8); @@ -308,9 +312,8 @@ private int FindConsumedBytes(byte[] compressedData, int uncompressedSize) var decoder = new Decoder(); decoder.SetDecoderProperties(props); - using var input = new MemoryStream(compressedData); - using var output = new PooledMemoryStream(); - decoder.Code(input, output, compressedData.Length, uncompressedSize, null); + using var input = new MemoryStream(compressedData, 0, compressedSize, writable: false); + decoder.Code(input, Stream.Null, compressedSize, uncompressedSize, null); return (int)input.Position; } @@ -319,10 +322,14 @@ private int FindConsumedBytes(byte[] compressedData, int uncompressedSize) /// Writes a compressed LZMA2 chunk. /// Header: [control] [uncompSize_hi] [uncompSize_lo] [compSize_hi] [compSize_lo] [props?] /// - private void WriteCompressedChunk(int uncompressedSize, byte[] compressedData) + private void WriteCompressedChunk( + int uncompressedSize, + byte[] compressedData, + int compressedSize + ) { var uncompSizeMinus1 = uncompressedSize - 1; - var compSizeMinus1 = compressedData.Length - 1; + var compSizeMinus1 = compressedSize - 1; // Each chunk is compressed independently with a fresh LZMA encoder, // so we must use 0xE0 (full reset: dictionary + state + properties) every time. @@ -341,32 +348,38 @@ private void WriteCompressedChunk(int uncompressedSize, byte[] compressedData) // 0xE0 (>= 0xC0) requires properties byte _output.WriteByte(_lzmaPropertiesByte); - _output.Write(compressedData, 0, compressedData.Length); + _output.Write(compressedData, 0, compressedSize); } private async ValueTask WriteCompressedChunkAsync( int uncompressedSize, byte[] compressedData, + int compressedSize, CancellationToken cancellationToken = default ) { var uncompSizeMinus1 = uncompressedSize - 1; - var compSizeMinus1 = compressedData.Length - 1; + var compSizeMinus1 = compressedSize - 1; var control = (byte)(0xE0 | ((uncompSizeMinus1 >> 16) & 0x1F)); _isFirstChunk = false; - var header = new[] + var header = ArrayPool.Shared.Rent(6); + try + { + header[0] = control; + header[1] = (byte)((uncompSizeMinus1 >> 8) & 0xFF); + header[2] = (byte)(uncompSizeMinus1 & 0xFF); + header[3] = (byte)((compSizeMinus1 >> 8) & 0xFF); + header[4] = (byte)(compSizeMinus1 & 0xFF); + header[5] = _lzmaPropertiesByte; + await _output.WriteAsync(header, 0, 6, cancellationToken).ConfigureAwait(false); + } + finally { - control, - (byte)((uncompSizeMinus1 >> 8) & 0xFF), - (byte)(uncompSizeMinus1 & 0xFF), - (byte)((compSizeMinus1 >> 8) & 0xFF), - (byte)(compSizeMinus1 & 0xFF), - _lzmaPropertiesByte, - }; - await _output.WriteAsync(header, 0, header.Length, cancellationToken).ConfigureAwait(false); + ArrayPool.Shared.Return(header); + } await _output - .WriteAsync(compressedData, 0, compressedData.Length, cancellationToken) + .WriteAsync(compressedData, 0, compressedSize, cancellationToken) .ConfigureAwait(false); } @@ -426,20 +439,37 @@ private async ValueTask WriteUncompressedChunksAsync( control = 0x02; } - var header = new[] + var header = ArrayPool.Shared.Rent(3); + try { - control, - (byte)((sizeMinus1 >> 8) & 0xFF), - (byte)(sizeMinus1 & 0xFF), - }; - await _output - .WriteAsync(header, 0, header.Length, cancellationToken) - .ConfigureAwait(false); + header[0] = control; + header[1] = (byte)((sizeMinus1 >> 8) & 0xFF); + header[2] = (byte)(sizeMinus1 & 0xFF); + await _output.WriteAsync(header, 0, 3, cancellationToken).ConfigureAwait(false); + } + finally + { + ArrayPool.Shared.Return(header); + } - var chunk = data.Slice(offset, chunkSize).ToArray(); +#if !LEGACY_DOTNET || NETSTANDARD2_1 await _output - .WriteAsync(chunk, 0, chunk.Length, cancellationToken) + .WriteAsync(data.Slice(offset, chunkSize), cancellationToken) .ConfigureAwait(false); +#else + var chunk = ArrayPool.Shared.Rent(chunkSize); + try + { + data.Slice(offset, chunkSize).CopyTo(chunk); + await _output + .WriteAsync(chunk, 0, chunkSize, cancellationToken) + .ConfigureAwait(false); + } + finally + { + ArrayPool.Shared.Return(chunk); + } +#endif offset += chunkSize; } } diff --git a/src/SharpCompress/Compressors/LZMA/LzmaDecoder.Async.cs b/src/SharpCompress/Compressors/LZMA/LzmaDecoder.Async.cs index 18e1475f2..7b81b8925 100644 --- a/src/SharpCompress/Compressors/LZMA/LzmaDecoder.Async.cs +++ b/src/SharpCompress/Compressors/LZMA/LzmaDecoder.Async.cs @@ -72,7 +72,7 @@ public async ValueTask DecodeNormalAsync( { symbol = (symbol << 1) - | await _decoders[symbol] + | await _decoders[_baseIndex + symbol] .DecodeAsync(rangeDecoder, cancellationToken) .ConfigureAwait(false); } while (symbol < 0x100); @@ -90,7 +90,7 @@ public async ValueTask DecodeWithMatchByteAsync( { var matchBit = (uint)(matchByte >> 7) & 1; matchByte <<= 1; - var bit = await _decoders[((1 + matchBit) << 8) + symbol] + var bit = await _decoders[_baseIndex + ((1 + matchBit) << 8) + symbol] .DecodeAsync(rangeDecoder, cancellationToken) .ConfigureAwait(false); symbol = (symbol << 1) | bit; @@ -100,7 +100,7 @@ public async ValueTask DecodeWithMatchByteAsync( { symbol = (symbol << 1) - | await _decoders[symbol] + | await _decoders[_baseIndex + symbol] .DecodeAsync(rangeDecoder, cancellationToken) .ConfigureAwait(false); } diff --git a/src/SharpCompress/Compressors/LZMA/LzmaDecoder.cs b/src/SharpCompress/Compressors/LZMA/LzmaDecoder.cs index 49a0e4c28..bdc189ff5 100644 --- a/src/SharpCompress/Compressors/LZMA/LzmaDecoder.cs +++ b/src/SharpCompress/Compressors/LZMA/LzmaDecoder.cs @@ -73,14 +73,19 @@ private partial class LiteralDecoder private partial struct Decoder2 { private BitDecoder[] _decoders; + private int _baseIndex; - public void Create() => _decoders = new BitDecoder[0x300]; + public void Create(BitDecoder[] decoders, int baseIndex) + { + _decoders = decoders; + _baseIndex = baseIndex; + } public void Init() { for (var i = 0; i < 0x300; i++) { - _decoders[i].Init(); + _decoders[_baseIndex + i].Init(); } } @@ -89,7 +94,7 @@ public byte DecodeNormal(RangeCoder.Decoder rangeDecoder) uint symbol = 1; do { - symbol = (symbol << 1) | _decoders[symbol].Decode(rangeDecoder); + symbol = (symbol << 1) | _decoders[_baseIndex + symbol].Decode(rangeDecoder); } while (symbol < 0x100); return (byte)symbol; } @@ -101,13 +106,15 @@ public byte DecodeWithMatchByte(RangeCoder.Decoder rangeDecoder, byte matchByte) { var matchBit = (uint)(matchByte >> 7) & 1; matchByte <<= 1; - var bit = _decoders[((1 + matchBit) << 8) + symbol].Decode(rangeDecoder); + var bit = _decoders[_baseIndex + ((1 + matchBit) << 8) + symbol] + .Decode(rangeDecoder); symbol = (symbol << 1) | bit; if (matchBit != bit) { while (symbol < 0x100) { - symbol = (symbol << 1) | _decoders[symbol].Decode(rangeDecoder); + symbol = + (symbol << 1) | _decoders[_baseIndex + symbol].Decode(rangeDecoder); } break; } @@ -117,6 +124,7 @@ public byte DecodeWithMatchByte(RangeCoder.Decoder rangeDecoder, byte matchByte) } private Decoder2[] _coders; + private BitDecoder[] _models; private int _numPrevBits; private int _numPosBits; private uint _posMask; @@ -131,10 +139,11 @@ public void Create(int numPosBits, int numPrevBits) _posMask = ((uint)1 << numPosBits) - 1; _numPrevBits = numPrevBits; var numStates = (uint)1 << (_numPrevBits + _numPosBits); + _models = new BitDecoder[checked((int)(numStates * 0x300))]; _coders = new Decoder2[numStates]; for (uint i = 0; i < numStates; i++) { - _coders[i].Create(); + _coders[i].Create(_models, checked((int)(i * 0x300))); } } @@ -446,7 +455,10 @@ internal bool Code(int dictionarySize, OutWindow outWindow, RangeCoder.Decoder r return false; } - public void SetDecoderProperties(byte[] properties) + public void SetDecoderProperties(byte[] properties) => + SetDecoderProperties(properties.AsSpan()); + + internal void SetDecoderProperties(ReadOnlySpan properties) { if (properties.Length < 1) { diff --git a/src/SharpCompress/Compressors/LZMA/LzmaEncoder.cs b/src/SharpCompress/Compressors/LZMA/LzmaEncoder.cs index 14d947079..c8d5279ba 100644 --- a/src/SharpCompress/Compressors/LZMA/LzmaEncoder.cs +++ b/src/SharpCompress/Compressors/LZMA/LzmaEncoder.cs @@ -1,6 +1,7 @@ #nullable disable using System; +using System.Buffers; using System.IO; using System.Threading; using System.Threading.Tasks; @@ -10,7 +11,7 @@ namespace SharpCompress.Compressors.LZMA; -internal partial class Encoder : ICoder, ISetCoderProperties, IWriteCoderProperties +internal partial class Encoder : ICoder, ISetCoderProperties, IWriteCoderProperties, IDisposable { private enum EMatchFinderType { @@ -86,14 +87,19 @@ private partial class LiteralEncoder public partial struct Encoder2 { private BitEncoder[] _encoders; + private int _baseIndex; - public void Create() => _encoders = new BitEncoder[0x300]; + public void Create(BitEncoder[] encoders, int baseIndex) + { + _encoders = encoders; + _baseIndex = baseIndex; + } public void Init() { for (var i = 0; i < 0x300; i++) { - _encoders[i].Init(); + _encoders[_baseIndex + i].Init(); } } @@ -103,7 +109,7 @@ public void Encode(RangeCoder.Encoder rangeEncoder, byte symbol) for (var i = 7; i >= 0; i--) { var bit = (uint)((symbol >> i) & 1); - _encoders[context].Encode(rangeEncoder, bit); + _encoders[_baseIndex + context].Encode(rangeEncoder, bit); context = (context << 1) | bit; } } @@ -118,7 +124,7 @@ public async ValueTask EncodeAsync( for (var i = 7; i >= 0; i--) { var bit = (uint)((symbol >> i) & 1); - await _encoders[context] + await _encoders[_baseIndex + context] .EncodeAsync(rangeEncoder, bit, cancellationToken) .ConfigureAwait(false); context = (context << 1) | bit; @@ -139,7 +145,7 @@ public void EncodeMatched(RangeCoder.Encoder rangeEncoder, byte matchByte, byte state += ((1 + matchBit) << 8); same = (matchBit == bit); } - _encoders[state].Encode(rangeEncoder, bit); + _encoders[_baseIndex + state].Encode(rangeEncoder, bit); context = (context << 1) | bit; } } @@ -163,7 +169,7 @@ public async ValueTask EncodeMatchedAsync( state += ((1 + matchBit) << 8); same = matchBit == bit; } - await _encoders[state] + await _encoders[_baseIndex + state] .EncodeAsync(rangeEncoder, bit, cancellationToken) .ConfigureAwait(false); context = (context << 1) | bit; @@ -181,7 +187,8 @@ public uint GetPrice(bool matchMode, byte matchByte, byte symbol) { var matchBit = (uint)(matchByte >> i) & 1; var bit = (uint)(symbol >> i) & 1; - price += _encoders[((1 + matchBit) << 8) + context].GetPrice(bit); + price += _encoders[_baseIndex + ((1 + matchBit) << 8) + context] + .GetPrice(bit); context = (context << 1) | bit; if (matchBit != bit) { @@ -193,7 +200,7 @@ public uint GetPrice(bool matchMode, byte matchByte, byte symbol) for (; i >= 0; i--) { var bit = (uint)(symbol >> i) & 1; - price += _encoders[context].GetPrice(bit); + price += _encoders[_baseIndex + context].GetPrice(bit); context = (context << 1) | bit; } return price; @@ -201,6 +208,7 @@ public uint GetPrice(bool matchMode, byte matchByte, byte symbol) } private Encoder2[] _coders; + private BitEncoder[] _models; private int _numPrevBits; private int _numPosBits; private uint _posMask; @@ -215,11 +223,35 @@ public void Create(int numPosBits, int numPrevBits) _posMask = ((uint)1 << numPosBits) - 1; _numPrevBits = numPrevBits; var numStates = (uint)1 << (_numPrevBits + _numPosBits); - _coders = new Encoder2[numStates]; + var requiredModelLength = checked((int)(numStates * 0x300)); + if (_models is null || _models.Length < requiredModelLength) + { + if (_models is not null) + { + ArrayPool.Shared.Return(_models); + } + _models = ArrayPool.Shared.Rent(requiredModelLength); + } + if (_coders is null || _coders.Length != numStates) + { + _coders = new Encoder2[numStates]; + } for (uint i = 0; i < numStates; i++) { - _coders[i].Create(); + _coders[i].Create(_models, checked((int)(i * 0x300))); + } + } + + public void Dispose() + { + if (_models is null) + { + return; } + + ArrayPool.Shared.Return(_models); + _models = null; + _coders = null; } public void Init() @@ -423,7 +455,7 @@ await base.EncodeAsync(rangeEncoder, symbol, posState, cancellationToken) private const uint K_NUM_OPTS = 1 << 12; - private class Optimal + private struct Optimal { public Base.State _state; @@ -564,16 +596,19 @@ private void Create() public Encoder() { - for (var i = 0; i < K_NUM_OPTS; i++) - { - _optimum[i] = new Optimal(); - } for (var i = 0; i < Base.K_NUM_LEN_TO_POS_STATES; i++) { _posSlotEncoder[i] = new BitTreeEncoder(Base.K_NUM_POS_SLOT_BITS); } } + public void Dispose() + { + _literalEncoder.Dispose(); + _matchFinder?.Dispose(); + _matchFinder = null; + } + private void SetWriteEndMarkerMode(bool writeEndMarker) => _writeEndMark = writeEndMarker; private void Init() @@ -859,7 +894,7 @@ private uint GetOptimum(uint position, out uint backRes) do { var curAndLenPrice = price + _repMatchLenEncoder.GetPrice(repLen - 2, posState); - var optimum = _optimum[repLen]; + ref var optimum = ref _optimum[repLen]; if (curAndLenPrice < optimum._price) { optimum._price = curAndLenPrice; @@ -884,7 +919,7 @@ private uint GetOptimum(uint position, out uint backRes) { var distance = _matchDistances[offs + 1]; var curAndLenPrice = normalMatchPrice + GetPosLenPrice(distance, len, posState); - var optimum = _optimum[len]; + ref var optimum = ref _optimum[len]; if (curAndLenPrice < optimum._price) { optimum._price = curAndLenPrice; @@ -1039,7 +1074,7 @@ private uint GetOptimum(uint position, out uint backRes) .GetSubCoder(position, _matchFinder.GetIndexByte(0 - 2)) .GetPrice(!state.IsCharState(), matchByte, currentByte); - var nextOptimum = _optimum[cur + 1]; + ref var nextOptimum = ref _optimum[cur + 1]; var nextIsChar = false; if (curAnd1Price < nextOptimum._price) @@ -1105,7 +1140,7 @@ private uint GetOptimum(uint position, out uint backRes) } var curAndLenPrice = nextRepMatchPrice + GetRepPrice(0, lenTest2, state2, posStateNext); - var optimum = _optimum[offset]; + ref var optimum = ref _optimum[offset]; if (curAndLenPrice < optimum._price) { optimum._price = curAndLenPrice; @@ -1136,7 +1171,7 @@ private uint GetOptimum(uint position, out uint backRes) } var curAndLenPrice = repMatchPrice + GetRepPrice(repIndex, lenTest, state, posState); - var optimum = _optimum[cur + lenTest]; + ref var optimum = ref _optimum[cur + lenTest]; if (curAndLenPrice < optimum._price) { optimum._price = curAndLenPrice; @@ -1200,7 +1235,7 @@ private uint GetOptimum(uint position, out uint backRes) } var curAndLenPrice = nextRepMatchPrice + GetRepPrice(0, lenTest2, state2, posStateNext); - var optimum = _optimum[cur + offset]; + ref var optimum = ref _optimum[cur + offset]; if (curAndLenPrice < optimum._price) { optimum._price = curAndLenPrice; @@ -1249,7 +1284,7 @@ private uint GetOptimum(uint position, out uint backRes) var curBack = _matchDistances[offs + 1]; var curAndLenPrice = normalMatchPrice + GetPosLenPrice(curBack, lenTest, posState); - var optimum = _optimum[cur + lenTest]; + ref var optimum = ref _optimum[cur + lenTest]; if (curAndLenPrice < optimum._price) { optimum._price = curAndLenPrice; @@ -1308,7 +1343,7 @@ private uint GetOptimum(uint position, out uint backRes) curAndLenPrice = nextRepMatchPrice + GetRepPrice(0, lenTest2, state2, posStateNext); - optimum = _optimum[cur + offset]; + optimum = ref _optimum[cur + offset]; if (curAndLenPrice < optimum._price) { optimum._price = curAndLenPrice; diff --git a/src/SharpCompress/Compressors/LZMA/LzmaStream.Async.cs b/src/SharpCompress/Compressors/LZMA/LzmaStream.Async.cs index 3adef73d7..4ad301270 100644 --- a/src/SharpCompress/Compressors/LZMA/LzmaStream.Async.cs +++ b/src/SharpCompress/Compressors/LZMA/LzmaStream.Async.cs @@ -83,11 +83,11 @@ Stream outputStream private async ValueTask DecodeChunkHeaderAsync(CancellationToken cancellationToken = default) { - var controlBuffer = new byte[1]; + var headerBuffer = GetAsyncHeaderBuffer(); await _inputStream! - .ReadExactAsync(controlBuffer, 0, 1, cancellationToken) + .ReadExactAsync(headerBuffer, 0, 1, cancellationToken) .ConfigureAwait(false); - var control = controlBuffer[0]; + var control = headerBuffer[0]; _inputPosition++; if (control == 0x00) @@ -112,26 +112,25 @@ await _inputStream! _uncompressedChunk = false; _availableBytes = (control & 0x1F) << 16; - var buffer = new byte[2]; await _inputStream! - .ReadExactAsync(buffer, 0, 2, cancellationToken) + .ReadExactAsync(headerBuffer, 0, 2, cancellationToken) .ConfigureAwait(false); - _availableBytes += (buffer[0] << 8) + buffer[1] + 1; + _availableBytes += (headerBuffer[0] << 8) + headerBuffer[1] + 1; _inputPosition += 2; await _inputStream! - .ReadExactAsync(buffer, 0, 2, cancellationToken) + .ReadExactAsync(headerBuffer, 0, 2, cancellationToken) .ConfigureAwait(false); - _rangeDecoderLimit = (buffer[0] << 8) + buffer[1] + 1; + _rangeDecoderLimit = (headerBuffer[0] << 8) + headerBuffer[1] + 1; _inputPosition += 2; if (control >= 0xC0) { _needProps = false; await _inputStream! - .ReadExactAsync(controlBuffer, 0, 1, cancellationToken) + .ReadExactAsync(headerBuffer, 0, 1, cancellationToken) .ConfigureAwait(false); - Properties[0] = controlBuffer[0]; + Properties[0] = headerBuffer[0]; _inputPosition++; _decoder = new Decoder(); @@ -156,11 +155,10 @@ await _inputStream! else { _uncompressedChunk = true; - var buffer = new byte[2]; await _inputStream! - .ReadExactAsync(buffer, 0, 2, cancellationToken) + .ReadExactAsync(headerBuffer, 0, 2, cancellationToken) .ConfigureAwait(false); - _availableBytes = (buffer[0] << 8) + buffer[1] + 1; + _availableBytes = (headerBuffer[0] << 8) + headerBuffer[1] + 1; _inputPosition += 2; } } @@ -436,6 +434,7 @@ public async ValueTask DisposeAsync() if (_encoder != null) { _position = await _encoder.CodeAsync(null, true).ConfigureAwait(false); + _encoder.Dispose(); } if (!_leaveOpen) @@ -450,6 +449,7 @@ public async ValueTask DisposeAsync() } } await _outWindow.DisposeAsync().ConfigureAwait(false); + ReturnAsyncHeaderBuffer(); #if !LEGACY_DOTNET || NETSTANDARD2_1 await base.DisposeAsync().ConfigureAwait(false); diff --git a/src/SharpCompress/Compressors/LZMA/LzmaStream.cs b/src/SharpCompress/Compressors/LZMA/LzmaStream.cs index 12f6884e0..38bb2eca3 100644 --- a/src/SharpCompress/Compressors/LZMA/LzmaStream.cs +++ b/src/SharpCompress/Compressors/LZMA/LzmaStream.cs @@ -1,4 +1,5 @@ using System; +using System.Buffers; using System.Buffers.Binary; using System.IO; using System.Threading; @@ -33,6 +34,7 @@ public partial class LzmaStream : Stream, IStreamStack, IAsyncDisposable private bool _needProps = true; private readonly Encoder? _encoder; + private byte[]? _asyncHeaderBuffer; private bool _isDisposed; private LzmaStream( @@ -205,16 +207,31 @@ protected override void Dispose(bool disposing) if (_encoder != null) { _position = _encoder.Code(null, true); + _encoder.Dispose(); } if (!_leaveOpen) { _inputStream?.Dispose(); } _outWindow.Dispose(); + ReturnAsyncHeaderBuffer(); } base.Dispose(disposing); } + private byte[] GetAsyncHeaderBuffer() => _asyncHeaderBuffer ??= ArrayPool.Shared.Rent(6); + + private void ReturnAsyncHeaderBuffer() + { + if (_asyncHeaderBuffer is null) + { + return; + } + + ArrayPool.Shared.Return(_asyncHeaderBuffer); + _asyncHeaderBuffer = null; + } + public override long Length => _position + _availableBytes; public override long Position diff --git a/src/SharpCompress/Compressors/LZMA/RangeCoder/RangeCoder.Async.cs b/src/SharpCompress/Compressors/LZMA/RangeCoder/RangeCoder.Async.cs index cd3fc6a50..61db6eaec 100644 --- a/src/SharpCompress/Compressors/LZMA/RangeCoder/RangeCoder.Async.cs +++ b/src/SharpCompress/Compressors/LZMA/RangeCoder/RangeCoder.Async.cs @@ -9,6 +9,8 @@ namespace SharpCompress.Compressors.LZMA.RangeCoder; internal partial class Encoder { + private byte[] SingleByteBuffer => _singleByteBuffer ??= new byte[1]; + public async ValueTask ShiftLowAsync(CancellationToken cancellationToken = default) { if ((uint)_low < 0xFF000000 || (uint)(_low >> 32) == 1) @@ -17,7 +19,8 @@ public async ValueTask ShiftLowAsync(CancellationToken cancellationToken = defau do { var b = (byte)(temp + (_low >> 32)); - var buffer = new[] { b }; + var buffer = SingleByteBuffer; + buffer[0] = b; await _stream.WriteAsync(buffer, 0, 1, cancellationToken).ConfigureAwait(false); temp = 0xFF; } while (--_cacheSize != 0); @@ -78,13 +81,15 @@ public async ValueTask FlushStreamAsync(CancellationToken cancellationToken = de internal partial class Decoder { + private byte[] SingleByteBuffer => _singleByteBuffer ??= new byte[1]; + public async ValueTask InitAsync(Stream stream, CancellationToken cancellationToken = default) { _stream = stream; _code = 0; _range = 0xFFFFFFFF; - var buffer = new byte[1]; + var buffer = SingleByteBuffer; for (var i = 0; i < 5; i++) { var read = await _stream @@ -103,7 +108,7 @@ public async ValueTask NormalizeAsync(CancellationToken cancellationToken = defa { while (_range < K_TOP_VALUE) { - var buffer = new byte[1]; + var buffer = SingleByteBuffer; var read = await _stream .ReadAsync(buffer, 0, 1, cancellationToken) .ConfigureAwait(false); @@ -121,7 +126,7 @@ public async ValueTask Normalize2Async(CancellationToken cancellationToken = def { if (_range < K_TOP_VALUE) { - var buffer = new byte[1]; + var buffer = SingleByteBuffer; var read = await _stream .ReadAsync(buffer, 0, 1, cancellationToken) .ConfigureAwait(false); @@ -143,7 +148,7 @@ public async ValueTask DecodeDirectBitsAsync( var range = _range; var code = _code; uint result = 0; - var buffer = new byte[1]; + var buffer = SingleByteBuffer; for (var i = numTotalBits; i > 0; i--) { range >>= 1; diff --git a/src/SharpCompress/Compressors/LZMA/RangeCoder/RangeCoder.cs b/src/SharpCompress/Compressors/LZMA/RangeCoder/RangeCoder.cs index e29a1c375..147146c44 100644 --- a/src/SharpCompress/Compressors/LZMA/RangeCoder/RangeCoder.cs +++ b/src/SharpCompress/Compressors/LZMA/RangeCoder/RangeCoder.cs @@ -15,8 +15,7 @@ internal partial class Encoder public uint _range; private uint _cacheSize; private byte _cache; - - //long StartPosition; + private byte[] _singleByteBuffer = new byte[1]; public void SetStream(Stream stream) => _stream = stream; @@ -88,6 +87,7 @@ internal partial class Decoder public Stream _stream; public long _total; + private byte[] _singleByteBuffer; public void Init(Stream stream) { @@ -180,6 +180,4 @@ public uint DecodeBit(uint size0, int numTotalBits) } public bool IsFinished => _code == 0; - - // ulong GetProcessedSize() {return Stream.GetProcessedSize(); } } diff --git a/src/SharpCompress/Polyfills/AsyncEnumerableExtensions.cs b/src/SharpCompress/Polyfills/AsyncEnumerableExtensions.cs index 51b6bfc89..df716ecd9 100644 --- a/src/SharpCompress/Polyfills/AsyncEnumerableExtensions.cs +++ b/src/SharpCompress/Polyfills/AsyncEnumerableExtensions.cs @@ -20,7 +20,6 @@ public static class EnumerableExtensions { public static async IAsyncEnumerable ToAsyncEnumerable(this IEnumerable source) { - await Task.Yield(); foreach (var item in source) { yield return item; diff --git a/tests/SharpCompress.Performance/Program.cs b/tests/SharpCompress.Performance/Program.cs index 35e257b8a..ab940ed66 100644 --- a/tests/SharpCompress.Performance/Program.cs +++ b/tests/SharpCompress.Performance/Program.cs @@ -1,4 +1,5 @@ using System; +using System.Threading.Tasks; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Running; @@ -8,12 +9,12 @@ namespace SharpCompress.Performance; public class Program { - public static void Main(string[] args) + public static async Task Main(string[] args) { // Check if profiling mode is requested if (args.Length > 0 && args[0].Equals("--profile", StringComparison.OrdinalIgnoreCase)) { - RunWithProfiler(args); + await RunWithProfiler(args); return; } @@ -29,7 +30,7 @@ public static void Main(string[] args) BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args, config); } - private static void RunWithProfiler(string[] args) + private static async Task RunWithProfiler(string[] args) { var profileType = "cpu"; // Default to CPU profiling var outputPath = "./profiler-snapshots"; @@ -59,16 +60,11 @@ private static void RunWithProfiler(string[] args) Console.WriteLine(); // Run a sample benchmark with profiling - RunSampleBenchmarkWithProfiler(profileType, outputPath); + await RunSampleBenchmarkWithProfiler(profileType, outputPath); } - private static void RunSampleBenchmarkWithProfiler(string profileType, string outputPath) + private static async Task RunSampleBenchmarkWithProfiler(string profileType, string outputPath) { - Console.WriteLine("Running sample benchmark with profiler..."); - Console.WriteLine("Note: JetBrains profiler requires the profiler tools to be installed."); - Console.WriteLine("Install from: https://www.jetbrains.com/profiler/"); - Console.WriteLine(); - try { IDisposable? profiler = null; @@ -85,13 +81,13 @@ private static void RunSampleBenchmarkWithProfiler(string profileType, string ou using (profiler) { // Run a simple benchmark iteration - var zipBenchmark = new Benchmarks.ZipBenchmarks(); + var zipBenchmark = new Benchmarks.SevenZipBenchmarks(); zipBenchmark.Setup(); Console.WriteLine("Running benchmark iterations..."); - for (int i = 0; i < 10; i++) + for (int i = 0; i < 100; i++) { - zipBenchmark.ZipExtractArchiveApi(); + await zipBenchmark.SevenZipLzma2ExtractAsync_Reader(); if (i % 3 == 0) { Console.Write(".");