From 1b2ba921bbd47f2ecd041b6a29159d83fbb6fb95 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 27 Oct 2025 08:48:01 +0000 Subject: [PATCH 01/12] Initial plan From 43aa2bad2252ac114cdb11136347fea9cd5caecf Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 27 Oct 2025 09:00:38 +0000 Subject: [PATCH 02/12] Integrate async/await support from PR #976 as baseline Co-authored-by: adamhathcock <527620+adamhathcock@users.noreply.github.com> --- AGENTS.md | 87 ++++++---- README.md | 78 +++++++++ USAGE.md | 143 +++++++++++++++++ .../Archives/GZip/GZipArchiveEntry.cs | 8 + src/SharpCompress/Archives/IArchiveEntry.cs | 8 + .../Archives/Rar/RarArchiveEntry.cs | 5 + .../Archives/SevenZip/SevenZipArchiveEntry.cs | 5 + .../Archives/Tar/TarArchiveEntry.cs | 5 + .../Archives/Zip/ZipArchiveEntry.cs | 5 + src/SharpCompress/Common/ExtractionMethods.cs | 113 +++++++++++++ src/SharpCompress/IO/SharpCompressStream.cs | 151 ++++++++++++++++-- src/SharpCompress/IO/SourceStream.cs | 102 ++++++++++++ src/SharpCompress/Readers/AbstractReader.cs | 38 +++++ src/SharpCompress/Readers/IReader.cs | 9 ++ .../Readers/IReaderExtensions.cs | 62 +++++++ src/SharpCompress/Utility.cs | 135 ++++++++++++++++ src/SharpCompress/Writers/AbstractWriter.cs | 15 ++ src/SharpCompress/Writers/IWriter.cs | 8 + .../Writers/IWriterExtensions.cs | 69 ++++++++ src/SharpCompress/packages.lock.json | 6 +- tests/SharpCompress.Test/AsyncTests.cs | 133 +++++++++++++++ 21 files changed, 1136 insertions(+), 49 deletions(-) create mode 100644 tests/SharpCompress.Test/AsyncTests.cs diff --git a/AGENTS.md b/AGENTS.md index d6d39153a..a47fa0297 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,45 +1,64 @@ -# Agent usage and commands +--- +description: 'Guidelines for building C# applications' +applyTo: '**/*.cs' +--- -This document explains how maintainers and contributors can instruct the GitHub Copilot coding agent in this repository. +# C# Development -Supported instruction channels -- PR front-matter (YAML at top of PR body) — preferred for reproducibility. -- PR comment using slash-style commands (e.g. `/copilot run apply-fixes`). -- Add a label that triggers a run (e.g. `copilot: run`). +## C# Instructions +- Always use the latest version C#, currently C# 13 features. +- Write clear and concise comments for each function. -Example PR front-matter (place at the top of the PR body): +## General Instructions +- Make only high confidence suggestions when reviewing code changes. +- Write code with good maintainability practices, including comments on why certain design decisions were made. +- Handle edge cases and write clear exception handling. +- For libraries or external dependencies, mention their usage and purpose in comments. -```yaml -copilot: - run: "apply-fixes" - target_branch: "master" - auto_merge: false - run_tests: true - required_approvals: 1 -``` +## Naming Conventions -Example slash command via PR comment: -- `/copilot run apply-fixes --target=master --run-tests` +- Follow PascalCase for component names, method names, and public members. +- Use camelCase for private fields and local variables. +- Prefix interface names with "I" (e.g., IUserService). -Recommended labels -- `copilot: run` -> instructs agent to run its default task on the PR -- `copilot: approve` -> if allowed by policy, agent may merge once checks pass +## Code Formatting -How to enable and grant permissions -1. Merge `.github/agents/copilot-agent.yml` into master. -2. As a repository administrator, install/authorize the GitHub Copilot coding agent app and grant it repository permissions that match the manifest (Contents: write, Pull requests: write, Checks: write, Actions: write/read, Issues: write). -3. Ensure Actions is enabled for the repository and branch protection rules are compatible with the manifest (or allow the agent to have the bypass when appropriate). +- Use CSharpier for all code formatting to ensure consistent style across the project. +- Install CSharpier globally: `dotnet tool install -g csharpier` +- Format files with: `dotnet csharpier format .` +- Configure your IDE to format on save using CSharpier. +- CSharpier configuration can be customized via `.csharpierrc` file in the project root. +- Trust CSharpier's opinionated formatting decisions to maintain consistency. -Safety & governance -- Keep allow paths narrow — only grant the agent write access where it needs it. -- Prefer `require_review_before_merge: true` during initial rollout. -- Use audit logs to review agent activity and require a human reviewer until you trust the automation. +## Project Setup and Structure -PR details -- Branch name: copilot-agent-config-and-docs -- Changes: add/modify .github/agents/copilot-agent.yml and add AGENTS.md at repo root -- This PR is intentionally limited to configuration and documentation; it does not add any workflows that push changes or perform merges. +- Guide users through creating a new .NET project with the appropriate templates. +- Explain the purpose of each generated file and folder to build understanding of the project structure. +- Demonstrate how to organize code using feature folders or domain-driven design principles. +- Show proper separation of concerns with models, services, and data access layers. +- Explain the Program.cs and configuration system in ASP.NET Core 9 including environment-specific settings. -If the repository settings or installed apps block the agent from running, include a clear note in the PR description describing actions an admin must take: enable Actions, install Copilot coding agent app, grant repo write permissions to agent, or run onboarding steps. +## Nullable Reference Types -Author: GitHub Copilot (@copilot) acting on behalf of adamhathcock. +- Declare variables non-nullable, and check for `null` at entry points. +- Always use `is null` or `is not null` instead of `== null` or `!= null`. +- Trust the C# null annotations and don't add null checks when the type system says a value cannot be null. + +## Testing + +- Always include test cases for critical paths of the application. +- Guide users through creating unit tests. +- Do not emit "Act", "Arrange" or "Assert" comments. +- Copy existing style in nearby files for test method names and capitalization. +- Explain integration testing approaches for API endpoints. +- Demonstrate how to mock dependencies for effective testing. +- Show how to test authentication and authorization logic. +- Explain test-driven development principles as applied to API development. + +## Performance Optimization + +- Guide users on implementing caching strategies (in-memory, distributed, response caching). +- Explain asynchronous programming patterns and why they matter for API performance. +- Demonstrate pagination, filtering, and sorting for large data sets. +- Show how to implement compression and other performance optimizations. +- Explain how to measure and benchmark API performance. diff --git a/README.md b/README.md index f27dbe609..188dbe8aa 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,8 @@ SharpCompress is a compression library in pure C# for .NET Framework 4.62, .NET The major feature is support for non-seekable streams so large files can be processed on the fly (i.e. download stream). +**NEW:** All I/O operations now support async/await for improved performance and scalability. See the [Async Usage](#async-usage) section below. + GitHub Actions Build - [![SharpCompress](https://github.com/adamhathcock/sharpcompress/actions/workflows/dotnetcore.yml/badge.svg)](https://github.com/adamhathcock/sharpcompress/actions/workflows/dotnetcore.yml) [![Static Badge](https://img.shields.io/badge/API%20Docs-DNDocs-190088?logo=readme&logoColor=white)](https://dndocs.com/d/sharpcompress/api/index.html) @@ -32,6 +34,82 @@ Hi everyone. I hope you're using SharpCompress and finding it useful. Please giv Please do not email me directly to ask for help. If you think there is a real issue, please report it here. +## Async Usage + +SharpCompress now provides full async/await support for all I/O operations, allowing for better performance and scalability in modern applications. + +### Async Reading Examples + +Extract entries asynchronously: +```csharp +using (Stream stream = File.OpenRead("archive.zip")) +using (var reader = ReaderFactory.Open(stream)) +{ + while (reader.MoveToNextEntry()) + { + if (!reader.Entry.IsDirectory) + { + // Async extraction + await reader.WriteEntryToDirectoryAsync( + @"C:\temp", + new ExtractionOptions() { ExtractFullPath = true, Overwrite = true }, + cancellationToken + ); + } + } +} +``` + +Extract all entries to directory asynchronously: +```csharp +using (Stream stream = File.OpenRead("archive.tar.gz")) +using (var reader = ReaderFactory.Open(stream)) +{ + await reader.WriteAllToDirectoryAsync( + @"C:\temp", + new ExtractionOptions() { ExtractFullPath = true, Overwrite = true }, + cancellationToken + ); +} +``` + +Open entry stream asynchronously: +```csharp +using (var archive = ZipArchive.Open("archive.zip")) +{ + foreach (var entry in archive.Entries.Where(e => !e.IsDirectory)) + { + using (var entryStream = await entry.OpenEntryStreamAsync(cancellationToken)) + { + // Process stream asynchronously + await entryStream.CopyToAsync(outputStream, cancellationToken); + } + } +} +``` + +### Async Writing Examples + +Write files asynchronously: +```csharp +using (Stream stream = File.OpenWrite("output.zip")) +using (var writer = WriterFactory.Open(stream, ArchiveType.Zip, CompressionType.Deflate)) +{ + await writer.WriteAsync("file1.txt", fileStream, DateTime.Now, cancellationToken); +} +``` + +Write all files from directory asynchronously: +```csharp +using (Stream stream = File.OpenWrite("output.tar.gz")) +using (var writer = WriterFactory.Open(stream, ArchiveType.Tar, new WriterOptions(CompressionType.GZip))) +{ + await writer.WriteAllAsync(@"D:\files", "*", SearchOption.AllDirectories, cancellationToken); +} +``` + +All async methods support `CancellationToken` for graceful cancellation of long-running operations. + ## Want to contribute? I'm always looking for help or ideas. Please submit code or email with ideas. Unfortunately, just letting me know you'd like to help is not enough because I really have no overall plan of what needs to be done. I'll definitely accept code submissions and add you as a member of the project! diff --git a/USAGE.md b/USAGE.md index cba49ed4b..a06d2e834 100644 --- a/USAGE.md +++ b/USAGE.md @@ -1,5 +1,18 @@ # SharpCompress Usage +## Async/Await Support + +SharpCompress now provides full async/await support for all I/O operations. All `Read`, `Write`, and extraction operations have async equivalents ending in `Async` that accept an optional `CancellationToken`. This enables better performance and scalability for I/O-bound operations. + +**Key Async Methods:** +- `reader.WriteEntryToAsync(stream, cancellationToken)` - Extract entry asynchronously +- `reader.WriteAllToDirectoryAsync(path, options, cancellationToken)` - Extract all asynchronously +- `writer.WriteAsync(filename, stream, modTime, cancellationToken)` - Write entry asynchronously +- `writer.WriteAllAsync(directory, pattern, searchOption, cancellationToken)` - Write directory asynchronously +- `entry.OpenEntryStreamAsync(cancellationToken)` - Open entry stream asynchronously + +See [Async Examples](#async-examples) section below for usage patterns. + ## Stream Rules (changed with 0.21) When dealing with Streams, the rule should be that you don't close a stream you didn't create. This, in effect, should mean you should always put a Stream in a using block to dispose it. @@ -172,3 +185,133 @@ foreach(var entry in tr.Entries) Console.WriteLine($"{entry.Key}"); } ``` + +## Async Examples + +### Async Reader Examples + +**Extract single entry asynchronously:** +```C# +using (Stream stream = File.OpenRead("archive.zip")) +using (var reader = ReaderFactory.Open(stream)) +{ + while (reader.MoveToNextEntry()) + { + if (!reader.Entry.IsDirectory) + { + using (var entryStream = reader.OpenEntryStream()) + { + using (var outputStream = File.Create("output.bin")) + { + await reader.WriteEntryToAsync(outputStream, cancellationToken); + } + } + } + } +} +``` + +**Extract all entries asynchronously:** +```C# +using (Stream stream = File.OpenRead("archive.tar.gz")) +using (var reader = ReaderFactory.Open(stream)) +{ + await reader.WriteAllToDirectoryAsync( + @"D:\temp", + new ExtractionOptions() + { + ExtractFullPath = true, + Overwrite = true + }, + cancellationToken + ); +} +``` + +**Open and process entry stream asynchronously:** +```C# +using (var archive = ZipArchive.Open("archive.zip")) +{ + foreach (var entry in archive.Entries.Where(e => !e.IsDirectory)) + { + using (var entryStream = await entry.OpenEntryStreamAsync(cancellationToken)) + { + // Process the decompressed stream asynchronously + await ProcessStreamAsync(entryStream, cancellationToken); + } + } +} +``` + +### Async Writer Examples + +**Write single file asynchronously:** +```C# +using (Stream archiveStream = File.OpenWrite("output.zip")) +using (var writer = WriterFactory.Open(archiveStream, ArchiveType.Zip, CompressionType.Deflate)) +{ + using (Stream fileStream = File.OpenRead("input.txt")) + { + await writer.WriteAsync("entry.txt", fileStream, DateTime.Now, cancellationToken); + } +} +``` + +**Write entire directory asynchronously:** +```C# +using (Stream stream = File.OpenWrite("backup.tar.gz")) +using (var writer = WriterFactory.Open(stream, ArchiveType.Tar, new WriterOptions(CompressionType.GZip))) +{ + await writer.WriteAllAsync( + @"D:\files", + "*", + SearchOption.AllDirectories, + cancellationToken + ); +} +``` + +**Write with progress tracking and cancellation:** +```C# +var cts = new CancellationTokenSource(); + +// Set timeout or cancel from UI +cts.CancelAfter(TimeSpan.FromMinutes(5)); + +using (Stream stream = File.OpenWrite("archive.zip")) +using (var writer = WriterFactory.Open(stream, ArchiveType.Zip, CompressionType.Deflate)) +{ + try + { + await writer.WriteAllAsync(@"D:\data", "*", SearchOption.AllDirectories, cts.Token); + } + catch (OperationCanceledException) + { + Console.WriteLine("Operation was cancelled"); + } +} +``` + +### Archive Async Examples + +**Extract from archive asynchronously:** +```C# +using (var archive = ZipArchive.Open("archive.zip")) +{ + using (var reader = archive.ExtractAllEntries()) + { + await reader.WriteAllToDirectoryAsync( + @"C:\output", + new ExtractionOptions() { ExtractFullPath = true, Overwrite = true }, + cancellationToken + ); + } +} +``` + +**Benefits of Async Operations:** +- Non-blocking I/O for better application responsiveness +- Improved scalability for server applications +- Support for cancellation via CancellationToken +- Better resource utilization in async/await contexts +- Compatible with modern .NET async patterns diff --git a/src/SharpCompress/Archives/GZip/GZipArchiveEntry.cs b/src/SharpCompress/Archives/GZip/GZipArchiveEntry.cs index f00e889cb..1cc115e4e 100644 --- a/src/SharpCompress/Archives/GZip/GZipArchiveEntry.cs +++ b/src/SharpCompress/Archives/GZip/GZipArchiveEntry.cs @@ -1,5 +1,7 @@ using System.IO; using System.Linq; +using System.Threading; +using System.Threading.Tasks; using SharpCompress.Common.GZip; namespace SharpCompress.Archives.GZip; @@ -20,6 +22,12 @@ public virtual Stream OpenEntryStream() return Parts.Single().GetCompressedStream().NotNull(); } + public virtual Task OpenEntryStreamAsync(CancellationToken cancellationToken = default) + { + // GZip synchronous implementation is fast enough, just wrap it + return Task.FromResult(OpenEntryStream()); + } + #region IArchiveEntry Members public IArchive Archive { get; } diff --git a/src/SharpCompress/Archives/IArchiveEntry.cs b/src/SharpCompress/Archives/IArchiveEntry.cs index 708753cbe..69b3a674e 100644 --- a/src/SharpCompress/Archives/IArchiveEntry.cs +++ b/src/SharpCompress/Archives/IArchiveEntry.cs @@ -1,4 +1,6 @@ using System.IO; +using System.Threading; +using System.Threading.Tasks; using SharpCompress.Common; namespace SharpCompress.Archives; @@ -11,6 +13,12 @@ public interface IArchiveEntry : IEntry /// Stream OpenEntryStream(); + /// + /// Opens the current entry as a stream that will decompress as it is read asynchronously. + /// Read the entire stream or use SkipEntry on EntryStream. + /// + Task OpenEntryStreamAsync(CancellationToken cancellationToken = default); + /// /// The archive can find all the parts of the archive needed to extract this entry. /// diff --git a/src/SharpCompress/Archives/Rar/RarArchiveEntry.cs b/src/SharpCompress/Archives/Rar/RarArchiveEntry.cs index 262d7cbe5..18a879695 100644 --- a/src/SharpCompress/Archives/Rar/RarArchiveEntry.cs +++ b/src/SharpCompress/Archives/Rar/RarArchiveEntry.cs @@ -2,6 +2,8 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Threading; +using System.Threading.Tasks; using SharpCompress.Common; using SharpCompress.Common.Rar; using SharpCompress.Common.Rar.Headers; @@ -84,6 +86,9 @@ public Stream OpenEntryStream() ); } + public Task OpenEntryStreamAsync(CancellationToken cancellationToken = default) => + Task.FromResult(OpenEntryStream()); + public bool IsComplete { get diff --git a/src/SharpCompress/Archives/SevenZip/SevenZipArchiveEntry.cs b/src/SharpCompress/Archives/SevenZip/SevenZipArchiveEntry.cs index 9824ca472..754c8c637 100644 --- a/src/SharpCompress/Archives/SevenZip/SevenZipArchiveEntry.cs +++ b/src/SharpCompress/Archives/SevenZip/SevenZipArchiveEntry.cs @@ -1,4 +1,6 @@ using System.IO; +using System.Threading; +using System.Threading.Tasks; using SharpCompress.Common.SevenZip; namespace SharpCompress.Archives.SevenZip; @@ -10,6 +12,9 @@ internal SevenZipArchiveEntry(SevenZipArchive archive, SevenZipFilePart part) public Stream OpenEntryStream() => FilePart.GetCompressedStream(); + public Task OpenEntryStreamAsync(CancellationToken cancellationToken = default) => + Task.FromResult(OpenEntryStream()); + public IArchive Archive { get; } public bool IsComplete => true; diff --git a/src/SharpCompress/Archives/Tar/TarArchiveEntry.cs b/src/SharpCompress/Archives/Tar/TarArchiveEntry.cs index 770a71095..e3d2219bb 100644 --- a/src/SharpCompress/Archives/Tar/TarArchiveEntry.cs +++ b/src/SharpCompress/Archives/Tar/TarArchiveEntry.cs @@ -1,5 +1,7 @@ using System.IO; using System.Linq; +using System.Threading; +using System.Threading.Tasks; using SharpCompress.Common; using SharpCompress.Common.Tar; @@ -12,6 +14,9 @@ internal TarArchiveEntry(TarArchive archive, TarFilePart? part, CompressionType public virtual Stream OpenEntryStream() => Parts.Single().GetCompressedStream().NotNull(); + public virtual Task OpenEntryStreamAsync(CancellationToken cancellationToken = default) => + Task.FromResult(OpenEntryStream()); + #region IArchiveEntry Members public IArchive Archive { get; } diff --git a/src/SharpCompress/Archives/Zip/ZipArchiveEntry.cs b/src/SharpCompress/Archives/Zip/ZipArchiveEntry.cs index 4c980f910..5ab93836a 100644 --- a/src/SharpCompress/Archives/Zip/ZipArchiveEntry.cs +++ b/src/SharpCompress/Archives/Zip/ZipArchiveEntry.cs @@ -1,5 +1,7 @@ using System.IO; using System.Linq; +using System.Threading; +using System.Threading.Tasks; using SharpCompress.Common.Zip; namespace SharpCompress.Archives.Zip; @@ -11,6 +13,9 @@ internal ZipArchiveEntry(ZipArchive archive, SeekableZipFilePart? part) public virtual Stream OpenEntryStream() => Parts.Single().GetCompressedStream().NotNull(); + public virtual Task OpenEntryStreamAsync(CancellationToken cancellationToken = default) => + Task.FromResult(OpenEntryStream()); + #region IArchiveEntry Members public IArchive Archive { get; } diff --git a/src/SharpCompress/Common/ExtractionMethods.cs b/src/SharpCompress/Common/ExtractionMethods.cs index 799d47b37..f81f82d4c 100644 --- a/src/SharpCompress/Common/ExtractionMethods.cs +++ b/src/SharpCompress/Common/ExtractionMethods.cs @@ -1,5 +1,7 @@ using System; using System.IO; +using System.Threading; +using System.Threading.Tasks; namespace SharpCompress.Common; @@ -116,4 +118,115 @@ Action openAndWrite entry.PreserveExtractionOptions(destinationFileName, options); } } + + public static async Task WriteEntryToDirectoryAsync( + IEntry entry, + string destinationDirectory, + ExtractionOptions? options, + Func writeAsync, + CancellationToken cancellationToken = default + ) + { + string destinationFileName; + var fullDestinationDirectoryPath = Path.GetFullPath(destinationDirectory); + + //check for trailing slash. + if ( + fullDestinationDirectoryPath[fullDestinationDirectoryPath.Length - 1] + != Path.DirectorySeparatorChar + ) + { + fullDestinationDirectoryPath += Path.DirectorySeparatorChar; + } + + if (!Directory.Exists(fullDestinationDirectoryPath)) + { + throw new ExtractionException( + $"Directory does not exist to extract to: {fullDestinationDirectoryPath}" + ); + } + + options ??= new ExtractionOptions() { Overwrite = true }; + + var file = Path.GetFileName(entry.Key.NotNull("Entry Key is null")).NotNull("File is null"); + file = Utility.ReplaceInvalidFileNameChars(file); + if (options.ExtractFullPath) + { + var folder = Path.GetDirectoryName(entry.Key.NotNull("Entry Key is null")) + .NotNull("Directory is null"); + var destdir = Path.GetFullPath(Path.Combine(fullDestinationDirectoryPath, folder)); + + if (!Directory.Exists(destdir)) + { + if (!destdir.StartsWith(fullDestinationDirectoryPath, StringComparison.Ordinal)) + { + throw new ExtractionException( + "Entry is trying to create a directory outside of the destination directory." + ); + } + + Directory.CreateDirectory(destdir); + } + destinationFileName = Path.Combine(destdir, file); + } + else + { + destinationFileName = Path.Combine(fullDestinationDirectoryPath, file); + } + + if (!entry.IsDirectory) + { + destinationFileName = Path.GetFullPath(destinationFileName); + + if ( + !destinationFileName.StartsWith( + fullDestinationDirectoryPath, + StringComparison.Ordinal + ) + ) + { + throw new ExtractionException( + "Entry is trying to write a file outside of the destination directory." + ); + } + await writeAsync(destinationFileName, options).ConfigureAwait(false); + } + else if (options.ExtractFullPath && !Directory.Exists(destinationFileName)) + { + Directory.CreateDirectory(destinationFileName); + } + } + + public static async Task WriteEntryToFileAsync( + IEntry entry, + string destinationFileName, + ExtractionOptions? options, + Func openAndWriteAsync, + CancellationToken cancellationToken = default + ) + { + if (entry.LinkTarget != null) + { + if (options?.WriteSymbolicLink is null) + { + throw new ExtractionException( + "Entry is a symbolic link but ExtractionOptions.WriteSymbolicLink delegate is null" + ); + } + options.WriteSymbolicLink(destinationFileName, entry.LinkTarget); + } + else + { + var fm = FileMode.Create; + options ??= new ExtractionOptions() { Overwrite = true }; + + if (!options.Overwrite) + { + fm = FileMode.CreateNew; + } + + await openAndWriteAsync(destinationFileName, fm).ConfigureAwait(false); + entry.PreserveExtractionOptions(destinationFileName, options); + } + } } diff --git a/src/SharpCompress/IO/SharpCompressStream.cs b/src/SharpCompress/IO/SharpCompressStream.cs index 5cd13d83b..238112654 100644 --- a/src/SharpCompress/IO/SharpCompressStream.cs +++ b/src/SharpCompress/IO/SharpCompressStream.cs @@ -4,6 +4,7 @@ using System.IO; using System.Text; using System.Threading; +using System.Threading.Tasks; namespace SharpCompress.IO; @@ -326,20 +327,146 @@ public override void Write(byte[] buffer, int offset, int count) _internalPosition += count; } + public override async Task ReadAsync( + byte[] buffer, + int offset, + int count, + CancellationToken cancellationToken + ) + { + if (count == 0) + return 0; + + if (_bufferingEnabled) + { + ValidateBufferState(); + + // Fill buffer if needed + if (_bufferedLength == 0) + { + _bufferedLength = await Stream + .ReadAsync(_buffer!, 0, _bufferSize, cancellationToken) + .ConfigureAwait(false); + _bufferPosition = 0; + } + int available = _bufferedLength - _bufferPosition; + int toRead = Math.Min(count, available); + if (toRead > 0) + { + Array.Copy(_buffer!, _bufferPosition, buffer, offset, toRead); + _bufferPosition += toRead; + _internalPosition += toRead; + return toRead; + } + // If buffer exhausted, refill + int r = await Stream + .ReadAsync(_buffer!, 0, _bufferSize, cancellationToken) + .ConfigureAwait(false); + if (r == 0) + return 0; + _bufferedLength = r; + _bufferPosition = 0; + if (_bufferedLength == 0) + { + return 0; + } + toRead = Math.Min(count, _bufferedLength); + Array.Copy(_buffer!, 0, buffer, offset, toRead); + _bufferPosition = toRead; + _internalPosition += toRead; + return toRead; + } + else + { + int read = await Stream + .ReadAsync(buffer, offset, count, cancellationToken) + .ConfigureAwait(false); + _internalPosition += read; + return read; + } + } + + public override async Task WriteAsync( + byte[] buffer, + int offset, + int count, + CancellationToken cancellationToken + ) + { + await Stream.WriteAsync(buffer, offset, count, cancellationToken).ConfigureAwait(false); + _internalPosition += count; + } + + public override async Task FlushAsync(CancellationToken cancellationToken) + { + await Stream.FlushAsync(cancellationToken).ConfigureAwait(false); + } + #if !NETFRAMEWORK && !NETSTANDARD2_0 - //public override int Read(Span buffer) - //{ - // int bytesRead = Stream.Read(buffer); - // _internalPosition += bytesRead; - // return bytesRead; - //} - - // public override void Write(ReadOnlySpan buffer) - // { - // Stream.Write(buffer); - // _internalPosition += buffer.Length; - // } + public override async ValueTask ReadAsync( + Memory buffer, + CancellationToken cancellationToken = default + ) + { + if (buffer.Length == 0) + return 0; + + if (_bufferingEnabled) + { + ValidateBufferState(); + + // Fill buffer if needed + if (_bufferedLength == 0) + { + _bufferedLength = await Stream + .ReadAsync(_buffer.AsMemory(0, _bufferSize), cancellationToken) + .ConfigureAwait(false); + _bufferPosition = 0; + } + int available = _bufferedLength - _bufferPosition; + int toRead = Math.Min(buffer.Length, available); + if (toRead > 0) + { + _buffer.AsSpan(_bufferPosition, toRead).CopyTo(buffer.Span); + _bufferPosition += toRead; + _internalPosition += toRead; + return toRead; + } + // If buffer exhausted, refill + int r = await Stream + .ReadAsync(_buffer.AsMemory(0, _bufferSize), cancellationToken) + .ConfigureAwait(false); + if (r == 0) + return 0; + _bufferedLength = r; + _bufferPosition = 0; + if (_bufferedLength == 0) + { + return 0; + } + toRead = Math.Min(buffer.Length, _bufferedLength); + _buffer.AsSpan(0, toRead).CopyTo(buffer.Span); + _bufferPosition = toRead; + _internalPosition += toRead; + return toRead; + } + else + { + int read = await Stream.ReadAsync(buffer, cancellationToken).ConfigureAwait(false); + _internalPosition += read; + return read; + } + } + + public override async ValueTask WriteAsync( + ReadOnlyMemory buffer, + CancellationToken cancellationToken = default + ) + { + await Stream.WriteAsync(buffer, cancellationToken).ConfigureAwait(false); + _internalPosition += buffer.Length; + } #endif } diff --git a/src/SharpCompress/IO/SourceStream.cs b/src/SharpCompress/IO/SourceStream.cs index 74eddec86..2c152917d 100644 --- a/src/SharpCompress/IO/SourceStream.cs +++ b/src/SharpCompress/IO/SourceStream.cs @@ -2,6 +2,8 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Threading; +using System.Threading.Tasks; using SharpCompress.Readers; namespace SharpCompress.IO; @@ -238,6 +240,106 @@ public override long Seek(long offset, SeekOrigin origin) public override void Write(byte[] buffer, int offset, int count) => throw new NotImplementedException(); + public override async Task ReadAsync( + byte[] buffer, + int offset, + int count, + CancellationToken cancellationToken + ) + { + if (count <= 0) + { + return 0; + } + + var total = count; + var r = -1; + + while (count != 0 && r != 0) + { + r = await Current + .ReadAsync( + buffer, + offset, + (int)Math.Min(count, Current.Length - Current.Position), + cancellationToken + ) + .ConfigureAwait(false); + count -= r; + offset += r; + + if (!IsVolumes && count != 0 && Current.Position == Current.Length) + { + var length = Current.Length; + + // Load next file if present + if (!SetStream(_stream + 1)) + { + break; + } + + // Current stream switched + // Add length of previous stream + _prevSize += length; + Current.Seek(0, SeekOrigin.Begin); + r = -1; //BugFix: reset to allow loop if count is still not 0 - was breaking split zipx (lzma xz etc) + } + } + + return total - count; + } + +#if !NETFRAMEWORK && !NETSTANDARD2_0 + + public override async ValueTask ReadAsync( + Memory buffer, + CancellationToken cancellationToken = default + ) + { + if (buffer.Length <= 0) + { + return 0; + } + + var total = buffer.Length; + var count = buffer.Length; + var offset = 0; + var r = -1; + + while (count != 0 && r != 0) + { + r = await Current + .ReadAsync( + buffer.Slice(offset, (int)Math.Min(count, Current.Length - Current.Position)), + cancellationToken + ) + .ConfigureAwait(false); + count -= r; + offset += r; + + if (!IsVolumes && count != 0 && Current.Position == Current.Length) + { + var length = Current.Length; + + // Load next file if present + if (!SetStream(_stream + 1)) + { + break; + } + + // Current stream switched + // Add length of previous stream + _prevSize += length; + Current.Seek(0, SeekOrigin.Begin); + r = -1; + } + } + + return total - count; + } + +#endif + public override void Close() { if (IsFileMode || !ReaderOptions.LeaveStreamOpen) //close if file mode or options specify it diff --git a/src/SharpCompress/Readers/AbstractReader.cs b/src/SharpCompress/Readers/AbstractReader.cs index fc6e3d1ca..7d3be5a4f 100644 --- a/src/SharpCompress/Readers/AbstractReader.cs +++ b/src/SharpCompress/Readers/AbstractReader.cs @@ -2,6 +2,8 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Threading; +using System.Threading.Tasks; using SharpCompress.Common; namespace SharpCompress.Readers; @@ -171,6 +173,33 @@ public void WriteEntryTo(Stream writableStream) _wroteCurrentEntry = true; } + public async Task WriteEntryToAsync( + Stream writableStream, + CancellationToken cancellationToken = default + ) + { + if (_wroteCurrentEntry) + { + throw new ArgumentException( + "WriteEntryToAsync or OpenEntryStream can only be called once." + ); + } + + if (writableStream is null) + { + throw new ArgumentNullException(nameof(writableStream)); + } + if (!writableStream.CanWrite) + { + throw new ArgumentException( + "A writable Stream was required. Use Cancel if that was intended." + ); + } + + await WriteAsync(writableStream, cancellationToken).ConfigureAwait(false); + _wroteCurrentEntry = true; + } + internal void Write(Stream writeStream) { var streamListener = this as IReaderExtractionListener; @@ -178,6 +207,15 @@ internal void Write(Stream writeStream) s.TransferTo(writeStream, Entry, streamListener); } + internal async Task WriteAsync(Stream writeStream, CancellationToken cancellationToken) + { + var streamListener = this as IReaderExtractionListener; + using Stream s = OpenEntryStream(); + await s + .TransferToAsync(writeStream, Entry, streamListener, cancellationToken) + .ConfigureAwait(false); + } + public EntryStream OpenEntryStream() { if (_wroteCurrentEntry) diff --git a/src/SharpCompress/Readers/IReader.cs b/src/SharpCompress/Readers/IReader.cs index 50fc7f4dd..b66636c46 100644 --- a/src/SharpCompress/Readers/IReader.cs +++ b/src/SharpCompress/Readers/IReader.cs @@ -1,5 +1,7 @@ using System; using System.IO; +using System.Threading; +using System.Threading.Tasks; using SharpCompress.Common; namespace SharpCompress.Readers; @@ -21,6 +23,13 @@ public interface IReader : IDisposable /// void WriteEntryTo(Stream writableStream); + /// + /// Decompresses the current entry to the stream asynchronously. This cannot be called twice for the current entry. + /// + /// + /// + Task WriteEntryToAsync(Stream writableStream, CancellationToken cancellationToken = default); + bool Cancelled { get; } void Cancel(); diff --git a/src/SharpCompress/Readers/IReaderExtensions.cs b/src/SharpCompress/Readers/IReaderExtensions.cs index 0e51d72af..6480df1d4 100644 --- a/src/SharpCompress/Readers/IReaderExtensions.cs +++ b/src/SharpCompress/Readers/IReaderExtensions.cs @@ -1,4 +1,6 @@ using System.IO; +using System.Threading; +using System.Threading.Tasks; using SharpCompress.Common; namespace SharpCompress.Readers; @@ -65,4 +67,64 @@ public static void WriteEntryToFile( reader.WriteEntryTo(fs); } ); + + /// + /// Extract to specific directory asynchronously, retaining filename + /// + public static async Task WriteEntryToDirectoryAsync( + this IReader reader, + string destinationDirectory, + ExtractionOptions? options = null, + CancellationToken cancellationToken = default + ) => + await ExtractionMethods + .WriteEntryToDirectoryAsync( + reader.Entry, + destinationDirectory, + options, + (fileName, opts) => reader.WriteEntryToFileAsync(fileName, opts, cancellationToken), + cancellationToken + ) + .ConfigureAwait(false); + + /// + /// Extract to specific file asynchronously + /// + public static async Task WriteEntryToFileAsync( + this IReader reader, + string destinationFileName, + ExtractionOptions? options = null, + CancellationToken cancellationToken = default + ) => + await ExtractionMethods + .WriteEntryToFileAsync( + reader.Entry, + destinationFileName, + options, + async (x, fm) => + { + using var fs = File.Open(destinationFileName, fm); + await reader.WriteEntryToAsync(fs, cancellationToken).ConfigureAwait(false); + }, + cancellationToken + ) + .ConfigureAwait(false); + + /// + /// Extract all remaining unread entries to specific directory asynchronously, retaining filename + /// + public static async Task WriteAllToDirectoryAsync( + this IReader reader, + string destinationDirectory, + ExtractionOptions? options = null, + CancellationToken cancellationToken = default + ) + { + while (reader.MoveToNextEntry()) + { + await reader + .WriteEntryToDirectoryAsync(destinationDirectory, options, cancellationToken) + .ConfigureAwait(false); + } + } } diff --git a/src/SharpCompress/Utility.cs b/src/SharpCompress/Utility.cs index 8f72cadce..c6cb3e954 100644 --- a/src/SharpCompress/Utility.cs +++ b/src/SharpCompress/Utility.cs @@ -4,6 +4,8 @@ using System.Collections.ObjectModel; using System.IO; using System.Text; +using System.Threading; +using System.Threading.Tasks; using SharpCompress.Readers; namespace SharpCompress; @@ -217,6 +219,89 @@ IReaderExtractionListener readerExtractionListener } } + public static async Task TransferToAsync( + this Stream source, + Stream destination, + long maxLength, + CancellationToken cancellationToken = default + ) + { + var array = ArrayPool.Shared.Rent(TEMP_BUFFER_SIZE); + try + { + var maxReadSize = array.Length; + long total = 0; + var remaining = maxLength; + if (remaining < maxReadSize) + { + maxReadSize = (int)remaining; + } + while ( + await ReadTransferBlockAsync(source, array, maxReadSize, cancellationToken) + .ConfigureAwait(false) + is var (success, count) + && success + ) + { + await destination + .WriteAsync(array, 0, count, cancellationToken) + .ConfigureAwait(false); + total += count; + if (remaining - count < 0) + { + break; + } + remaining -= count; + if (remaining < maxReadSize) + { + maxReadSize = (int)remaining; + } + } + return total; + } + finally + { + ArrayPool.Shared.Return(array); + } + } + + public static async Task TransferToAsync( + this Stream source, + Stream destination, + Common.Entry entry, + IReaderExtractionListener readerExtractionListener, + CancellationToken cancellationToken = default + ) + { + var array = ArrayPool.Shared.Rent(TEMP_BUFFER_SIZE); + try + { + var iterations = 0; + long total = 0; + int count; + while ( + ( + count = await source + .ReadAsync(array, 0, array.Length, cancellationToken) + .ConfigureAwait(false) + ) != 0 + ) + { + total += count; + await destination + .WriteAsync(array, 0, count, cancellationToken) + .ConfigureAwait(false); + iterations++; + readerExtractionListener.FireEntryExtractionProgress(entry, total, iterations); + } + return total; + } + finally + { + ArrayPool.Shared.Return(array); + } + } + private static bool ReadTransferBlock(Stream source, byte[] array, int maxSize, out int count) { var size = maxSize; @@ -228,6 +313,56 @@ private static bool ReadTransferBlock(Stream source, byte[] array, int maxSize, return count != 0; } + private static async Task<(bool success, int count)> ReadTransferBlockAsync( + Stream source, + byte[] array, + int maxSize, + CancellationToken cancellationToken + ) + { + var size = maxSize; + if (maxSize > array.Length) + { + size = array.Length; + } + var count = await source.ReadAsync(array, 0, size, cancellationToken).ConfigureAwait(false); + return (count != 0, count); + } + + public static async Task SkipAsync( + this Stream source, + long advanceAmount, + CancellationToken cancellationToken = default + ) + { + if (source.CanSeek) + { + source.Position += advanceAmount; + return; + } + + var array = ArrayPool.Shared.Rent(TEMP_BUFFER_SIZE); + try + { + while (advanceAmount > 0) + { + var toRead = (int)Math.Min(array.Length, advanceAmount); + var read = await source + .ReadAsync(array, 0, toRead, cancellationToken) + .ConfigureAwait(false); + if (read <= 0) + { + break; + } + advanceAmount -= read; + } + } + finally + { + ArrayPool.Shared.Return(array); + } + } + #if NET60_OR_GREATER public static bool ReadFully(this Stream stream, byte[] buffer) diff --git a/src/SharpCompress/Writers/AbstractWriter.cs b/src/SharpCompress/Writers/AbstractWriter.cs index 1820c9861..2f71c0cb5 100644 --- a/src/SharpCompress/Writers/AbstractWriter.cs +++ b/src/SharpCompress/Writers/AbstractWriter.cs @@ -1,5 +1,7 @@ using System; using System.IO; +using System.Threading; +using System.Threading.Tasks; using SharpCompress.Common; namespace SharpCompress.Writers; @@ -22,6 +24,19 @@ public abstract class AbstractWriter(ArchiveType type, WriterOptions writerOptio public abstract void Write(string filename, Stream source, DateTime? modificationTime); + public virtual async Task WriteAsync( + string filename, + Stream source, + DateTime? modificationTime, + CancellationToken cancellationToken = default + ) + { + // Default implementation calls synchronous version + // Derived classes should override for true async behavior + Write(filename, source, modificationTime); + await Task.CompletedTask.ConfigureAwait(false); + } + protected virtual void Dispose(bool isDisposing) { if (isDisposing) diff --git a/src/SharpCompress/Writers/IWriter.cs b/src/SharpCompress/Writers/IWriter.cs index bde2fcd98..3ca51bc3c 100644 --- a/src/SharpCompress/Writers/IWriter.cs +++ b/src/SharpCompress/Writers/IWriter.cs @@ -1,5 +1,7 @@ using System; using System.IO; +using System.Threading; +using System.Threading.Tasks; using SharpCompress.Common; namespace SharpCompress.Writers; @@ -8,4 +10,10 @@ public interface IWriter : IDisposable { ArchiveType WriterType { get; } void Write(string filename, Stream source, DateTime? modificationTime); + Task WriteAsync( + string filename, + Stream source, + DateTime? modificationTime, + CancellationToken cancellationToken = default + ); } diff --git a/src/SharpCompress/Writers/IWriterExtensions.cs b/src/SharpCompress/Writers/IWriterExtensions.cs index 56f3966a3..080312c77 100644 --- a/src/SharpCompress/Writers/IWriterExtensions.cs +++ b/src/SharpCompress/Writers/IWriterExtensions.cs @@ -1,6 +1,8 @@ using System; using System.IO; using System.Linq; +using System.Threading; +using System.Threading.Tasks; namespace SharpCompress.Writers; @@ -52,4 +54,71 @@ var file in Directory writer.Write(file.Substring(directory.Length), file); } } + + // Async extensions + public static Task WriteAsync( + this IWriter writer, + string entryPath, + Stream source, + CancellationToken cancellationToken = default + ) => writer.WriteAsync(entryPath, source, null, cancellationToken); + + public static async Task WriteAsync( + this IWriter writer, + string entryPath, + FileInfo source, + CancellationToken cancellationToken = default + ) + { + if (!source.Exists) + { + throw new ArgumentException("Source does not exist: " + source.FullName); + } + using var stream = source.OpenRead(); + await writer + .WriteAsync(entryPath, stream, source.LastWriteTime, cancellationToken) + .ConfigureAwait(false); + } + + public static Task WriteAsync( + this IWriter writer, + string entryPath, + string source, + CancellationToken cancellationToken = default + ) => writer.WriteAsync(entryPath, new FileInfo(source), cancellationToken); + + public static Task WriteAllAsync( + this IWriter writer, + string directory, + string searchPattern = "*", + SearchOption option = SearchOption.TopDirectoryOnly, + CancellationToken cancellationToken = default + ) => writer.WriteAllAsync(directory, searchPattern, null, option, cancellationToken); + + public static async Task WriteAllAsync( + this IWriter writer, + string directory, + string searchPattern = "*", + Func? fileSearchFunc = null, + SearchOption option = SearchOption.TopDirectoryOnly, + CancellationToken cancellationToken = default + ) + { + if (!Directory.Exists(directory)) + { + throw new ArgumentException("Directory does not exist: " + directory); + } + + fileSearchFunc ??= n => true; + foreach ( + var file in Directory + .EnumerateFiles(directory, searchPattern, option) + .Where(fileSearchFunc) + ) + { + await writer + .WriteAsync(file.Substring(directory.Length), file, cancellationToken) + .ConfigureAwait(false); + } + } } diff --git a/src/SharpCompress/packages.lock.json b/src/SharpCompress/packages.lock.json index 3d0d443d6..b85a38f73 100644 --- a/src/SharpCompress/packages.lock.json +++ b/src/SharpCompress/packages.lock.json @@ -335,9 +335,9 @@ "net8.0": { "Microsoft.NET.ILLink.Tasks": { "type": "Direct", - "requested": "[8.0.21, )", - "resolved": "8.0.21", - "contentHash": "s8H5PZQs50OcNkaB6Si54+v3GWM7vzs6vxFRMlD3aXsbM+aPCtod62gmK0BYWou9diGzmo56j8cIf/PziijDqQ==" + "requested": "[8.0.20, )", + "resolved": "8.0.20", + "contentHash": "Rhcto2AjGvTO62+/VTmBpumBOmqIGp7nYEbTbmEXkCq4yPGxV8whju3/HsIA/bKyo2+DggaYk5+/8sxb1AbPTw==" }, "Microsoft.SourceLink.GitHub": { "type": "Direct", diff --git a/tests/SharpCompress.Test/AsyncTests.cs b/tests/SharpCompress.Test/AsyncTests.cs new file mode 100644 index 000000000..a67f8f00d --- /dev/null +++ b/tests/SharpCompress.Test/AsyncTests.cs @@ -0,0 +1,133 @@ +using System; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using SharpCompress.Archives; +using SharpCompress.Archives.Zip; +using SharpCompress.Common; +using SharpCompress.IO; +using SharpCompress.Readers; +using SharpCompress.Writers; +using Xunit; + +namespace SharpCompress.Test; + +public class AsyncTests : TestBase +{ + [Fact] + public async Task Reader_Async_Extract_All() + { + var testArchive = Path.Combine(TEST_ARCHIVES_PATH, "Tar.tar.gz"); + using var stream = File.OpenRead(testArchive); + using var reader = ReaderFactory.Open(stream); + + await reader.WriteAllToDirectoryAsync( + SCRATCH_FILES_PATH, + new ExtractionOptions { ExtractFullPath = true, Overwrite = true } + ); + + // Just verify some files were extracted + var extractedFiles = Directory.GetFiles(SCRATCH_FILES_PATH, "*", SearchOption.AllDirectories); + Assert.True(extractedFiles.Length > 0, "No files were extracted"); + } + + [Fact] + public async Task Reader_Async_Extract_Single_Entry() + { + var testArchive = Path.Combine(TEST_ARCHIVES_PATH, "Tar.tar.gz"); + using var stream = File.OpenRead(testArchive); + using var reader = ReaderFactory.Open(stream); + + while (reader.MoveToNextEntry()) + { + if (!reader.Entry.IsDirectory) + { + var outputPath = Path.Combine(SCRATCH_FILES_PATH, reader.Entry.Key!); + Directory.CreateDirectory(Path.GetDirectoryName(outputPath)!); + + using var outputStream = File.Create(outputPath); + await reader.WriteEntryToAsync(outputStream); + break; // Just test one entry + } + } + } + + [Fact] + public async Task Archive_Entry_Async_Open_Stream() + { + var testArchive = Path.Combine(TEST_ARCHIVES_PATH, "Tar.tar.gz"); + using var archive = ArchiveFactory.Open(testArchive); + + foreach (var entry in archive.Entries.Where(e => !e.IsDirectory).Take(1)) + { + using var entryStream = await entry.OpenEntryStreamAsync(); + Assert.NotNull(entryStream); + Assert.True(entryStream.CanRead); + + // Read some data to verify it works + var buffer = new byte[1024]; + var read = await entryStream.ReadAsync(buffer, 0, buffer.Length); + Assert.True(read > 0); + } + } + + [Fact] + public async Task Writer_Async_Write_Single_File() + { + var outputPath = Path.Combine(SCRATCH_FILES_PATH, "async_test.zip"); + using (var stream = File.Create(outputPath)) + using (var writer = WriterFactory.Open(stream, ArchiveType.Zip, CompressionType.Deflate)) + { + var testFile = Path.Combine(TEST_ARCHIVES_PATH, "Tar.tar.gz"); + using var fileStream = File.OpenRead(testFile); + await writer.WriteAsync("test_entry.bin", fileStream, new DateTime(2023, 1, 1)); + } + + // Verify the archive was created and contains the entry + Assert.True(File.Exists(outputPath)); + using var archive = ZipArchive.Open(outputPath); + Assert.Single(archive.Entries.Where(e => !e.IsDirectory)); + } + + [Fact] + public async Task Async_With_Cancellation_Token() + { + using var cts = new CancellationTokenSource(); + cts.CancelAfter(10000); // 10 seconds should be plenty + + var testArchive = Path.Combine(TEST_ARCHIVES_PATH, "Tar.tar.gz"); + using var stream = File.OpenRead(testArchive); + using var reader = ReaderFactory.Open(stream); + + await reader.WriteAllToDirectoryAsync( + SCRATCH_FILES_PATH, + new ExtractionOptions { ExtractFullPath = true, Overwrite = true }, + cts.Token + ); + + // Just verify some files were extracted + var extractedFiles = Directory.GetFiles(SCRATCH_FILES_PATH, "*", SearchOption.AllDirectories); + Assert.True(extractedFiles.Length > 0, "No files were extracted"); + } + + [Fact] + public async Task Stream_Extensions_Async() + { + var testFile = Path.Combine(TEST_ARCHIVES_PATH, "Tar.tar.gz"); + using var inputStream = File.OpenRead(testFile); + var outputPath = Path.Combine(SCRATCH_FILES_PATH, "async_copy.bin"); + using var outputStream = File.Create(outputPath); + + // Test the async extension method + var buffer = new byte[8192]; + int bytesRead; + while ((bytesRead = await inputStream.ReadAsync(buffer, 0, buffer.Length)) > 0) + { + await outputStream.WriteAsync(buffer, 0, bytesRead); + } + + Assert.True(File.Exists(outputPath)); + Assert.True(new FileInfo(outputPath).Length > 0); + } +} From beeb37b4fd3e64eacb0ea2627e234ff4993adaa9 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 27 Oct 2025 09:11:29 +0000 Subject: [PATCH 03/12] Add async support to EntryStream, ZlibStream, and ZlibBaseStream Co-authored-by: adamhathcock <527620+adamhathcock@users.noreply.github.com> --- src/SharpCompress/Common/EntryStream.cs | 77 ++++++ .../Compressors/Deflate/ZlibBaseStream.cs | 227 ++++++++++++++++++ .../Compressors/Deflate/ZlibStream.cs | 86 +++++++ src/SharpCompress/Utility.cs | 22 ++ 4 files changed, 412 insertions(+) diff --git a/src/SharpCompress/Common/EntryStream.cs b/src/SharpCompress/Common/EntryStream.cs index a0fe736a4..cfaef0aa1 100644 --- a/src/SharpCompress/Common/EntryStream.cs +++ b/src/SharpCompress/Common/EntryStream.cs @@ -1,6 +1,8 @@ using System; using System.IO; using System.IO.Compression; +using System.Threading; +using System.Threading.Tasks; using SharpCompress.IO; using SharpCompress.Readers; @@ -51,6 +53,15 @@ public void SkipEntry() _completed = true; } + /// + /// Asynchronously skip the rest of the entry stream. + /// + public async Task SkipEntryAsync(CancellationToken cancellationToken = default) + { + await this.SkipAsync(cancellationToken).ConfigureAwait(false); + _completed = true; + } + protected override void Dispose(bool disposing) { if (!(_completed || _reader.Cancelled)) @@ -83,6 +94,40 @@ protected override void Dispose(bool disposing) _stream.Dispose(); } +#if !NETFRAMEWORK && !NETSTANDARD2_0 + public override async ValueTask DisposeAsync() + { + if (!(_completed || _reader.Cancelled)) + { + await SkipEntryAsync().ConfigureAwait(false); + } + + //Need a safe standard approach to this - it's okay for compression to overreads. Handling needs to be standardised + if (_stream is IStreamStack ss) + { + if (ss.BaseStream() is SharpCompress.Compressors.Deflate.DeflateStream deflateStream) + { + await deflateStream.FlushAsync().ConfigureAwait(false); + } + else if (ss.BaseStream() is SharpCompress.Compressors.LZMA.LzmaStream lzmaStream) + { + await lzmaStream.FlushAsync().ConfigureAwait(false); + } + } + + if (_isDisposed) + { + return; + } + _isDisposed = true; +#if DEBUG_STREAMS + this.DebugDispose(typeof(EntryStream)); +#endif + await base.DisposeAsync().ConfigureAwait(false); + await _stream.DisposeAsync().ConfigureAwait(false); + } +#endif + public override bool CanRead => true; public override bool CanSeek => false; @@ -91,6 +136,8 @@ protected override void Dispose(bool disposing) public override void Flush() { } + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + public override long Length => _stream.Length; public override long Position @@ -109,6 +156,36 @@ public override int Read(byte[] buffer, int offset, int count) return read; } + public override async Task ReadAsync( + byte[] buffer, + int offset, + int count, + CancellationToken cancellationToken + ) + { + var read = await _stream.ReadAsync(buffer, offset, count, cancellationToken).ConfigureAwait(false); + if (read <= 0) + { + _completed = true; + } + return read; + } + +#if !NETFRAMEWORK && !NETSTANDARD2_0 + public override async ValueTask ReadAsync( + Memory buffer, + CancellationToken cancellationToken = default + ) + { + var read = await _stream.ReadAsync(buffer, cancellationToken).ConfigureAwait(false); + if (read <= 0) + { + _completed = true; + } + return read; + } +#endif + public override int ReadByte() { var value = _stream.ReadByte(); diff --git a/src/SharpCompress/Compressors/Deflate/ZlibBaseStream.cs b/src/SharpCompress/Compressors/Deflate/ZlibBaseStream.cs index 155a3556a..52442f907 100644 --- a/src/SharpCompress/Compressors/Deflate/ZlibBaseStream.cs +++ b/src/SharpCompress/Compressors/Deflate/ZlibBaseStream.cs @@ -31,6 +31,8 @@ using System.Collections.Generic; using System.IO; using System.Text; +using System.Threading; +using System.Threading.Tasks; using SharpCompress.Common.Tar.Headers; using SharpCompress.IO; @@ -197,6 +199,57 @@ public override void Write(byte[] buffer, int offset, int count) } while (!done); } + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + // workitem 7159 + // calculate the CRC on the unccompressed data (before writing) + if (crc != null) + { + crc.SlurpBlock(buffer, offset, count); + } + + if (_streamMode == StreamMode.Undefined) + { + _streamMode = StreamMode.Writer; + } + else if (_streamMode != StreamMode.Writer) + { + throw new ZlibException("Cannot Write after Reading."); + } + + if (count == 0) + { + return; + } + + // first reference of z property will initialize the private var _z + z.InputBuffer = buffer; + _z.NextIn = offset; + _z.AvailableBytesIn = count; + var done = false; + do + { + _z.OutputBuffer = workingBuffer; + _z.NextOut = 0; + _z.AvailableBytesOut = _workingBuffer.Length; + var rc = (_wantCompress) ? _z.Deflate(_flushMode) : _z.Inflate(_flushMode); + if (rc != ZlibConstants.Z_OK && rc != ZlibConstants.Z_STREAM_END) + { + throw new ZlibException((_wantCompress ? "de" : "in") + "flating: " + _z.Message); + } + + await _stream.WriteAsync(_workingBuffer, 0, _workingBuffer.Length - _z.AvailableBytesOut, cancellationToken).ConfigureAwait(false); + + done = _z.AvailableBytesIn == 0 && _z.AvailableBytesOut != 0; + + // If GZIP and de-compress, we're done when 8 bytes remain. + if (_flavor == ZlibStreamFlavor.GZIP && !_wantCompress) + { + done = (_z.AvailableBytesIn == 8 && _z.AvailableBytesOut != 0); + } + } while (!done); + } + private void finish() { if (_z is null) @@ -335,6 +388,104 @@ private void finish() } } + private async Task finishAsync(CancellationToken cancellationToken = default) + { + if (_z is null) + { + return; + } + + if (_streamMode == StreamMode.Writer) + { + var done = false; + do + { + _z.OutputBuffer = workingBuffer; + _z.NextOut = 0; + _z.AvailableBytesOut = _workingBuffer.Length; + var rc = + (_wantCompress) ? _z.Deflate(FlushType.Finish) : _z.Inflate(FlushType.Finish); + + if (rc != ZlibConstants.Z_STREAM_END && rc != ZlibConstants.Z_OK) + { + var verb = (_wantCompress ? "de" : "in") + "flating"; + if (_z.Message is null) + { + throw new ZlibException(String.Format("{0}: (rc = {1})", verb, rc)); + } + throw new ZlibException(verb + ": " + _z.Message); + } + + if (_workingBuffer.Length - _z.AvailableBytesOut > 0) + { + await _stream.WriteAsync(_workingBuffer, 0, _workingBuffer.Length - _z.AvailableBytesOut, cancellationToken).ConfigureAwait(false); + } + + done = _z.AvailableBytesIn == 0 && _z.AvailableBytesOut != 0; + + // If GZIP and de-compress, we're done when 8 bytes remain. + if (_flavor == ZlibStreamFlavor.GZIP && !_wantCompress) + { + done = (_z.AvailableBytesIn == 8 && _z.AvailableBytesOut != 0); + } + } while (!done); + + await FlushAsync(cancellationToken).ConfigureAwait(false); + + // workitem 7159 + if (_flavor == ZlibStreamFlavor.GZIP) + { + if (_wantCompress) + { + // Emit the GZIP trailer: CRC32 and size mod 2^32 + byte[] intBuf = new byte[4]; + BinaryPrimitives.WriteInt32LittleEndian(intBuf, crc.Crc32Result); + await _stream.WriteAsync(intBuf, 0, 4, cancellationToken).ConfigureAwait(false); + var c2 = (int)(crc.TotalBytesRead & 0x00000000FFFFFFFF); + BinaryPrimitives.WriteInt32LittleEndian(intBuf, c2); + await _stream.WriteAsync(intBuf, 0, 4, cancellationToken).ConfigureAwait(false); + } + else + { + throw new ZlibException("Writing with decompression is not supported."); + } + } + } + // workitem 7159 + else if (_streamMode == StreamMode.Reader) + { + if (_flavor == ZlibStreamFlavor.GZIP) + { + if (!_wantCompress) + { + // workitem 8501: handle edge case (decompress empty stream) + if (_z.TotalBytesOut == 0L) + { + return; + } + + // Read and potentially verify the GZIP trailer: CRC32 and size mod 2^32 + byte[] trailer = new byte[8]; + + // workitem 8679 + if (_z.AvailableBytesIn != 8) + { + // Make sure we have read to the end of the stream + _z.InputBuffer.AsSpan(_z.NextIn, _z.AvailableBytesIn).CopyTo(trailer); + var bytesNeeded = 8 - _z.AvailableBytesIn; + var bytesRead = await _stream.ReadAsync( + trailer, _z.AvailableBytesIn, bytesNeeded, cancellationToken + ).ConfigureAwait(false); + } + } + else + { + throw new ZlibException("Reading with compression is not supported."); + } + } + } + } + private void end() { if (z is null) @@ -382,6 +533,38 @@ protected override void Dispose(bool disposing) } } +#if !NETFRAMEWORK && !NETSTANDARD2_0 + public override async ValueTask DisposeAsync() + { + if (isDisposed) + { + return; + } + isDisposed = true; +#if DEBUG_STREAMS + this.DebugDispose(typeof(ZlibBaseStream)); +#endif + await base.DisposeAsync().ConfigureAwait(false); + if (_stream is null) + { + return; + } + try + { + await finishAsync().ConfigureAwait(false); + } + finally + { + end(); + if (_stream != null) + { + await _stream.DisposeAsync().ConfigureAwait(false); + _stream = null; + } + } + } +#endif + public override void Flush() { _stream.Flush(); @@ -390,6 +573,14 @@ public override void Flush() z.AvailableBytesIn = 0; } + public override async Task FlushAsync(CancellationToken cancellationToken) + { + await _stream.FlushAsync(cancellationToken).ConfigureAwait(false); + //rewind the buffer + ((IStreamStack)this).Rewind(z.AvailableBytesIn); //unused + z.AvailableBytesIn = 0; + } + public override Int64 Seek(Int64 offset, SeekOrigin origin) => throw new NotSupportedException(); @@ -678,6 +869,42 @@ public override Int32 Read(Byte[] buffer, Int32 offset, Int32 count) return rc; } + public override Task ReadAsync( + byte[] buffer, + int offset, + int count, + CancellationToken cancellationToken + ) + { + // For now, delegate to synchronous Read wrapped in Task + // A full async implementation would require refactoring the complex zlib codec logic + return Task.Run(() => Read(buffer, offset, count), cancellationToken); + } + +#if !NETFRAMEWORK && !NETSTANDARD2_0 + public override ValueTask ReadAsync( + Memory buffer, + CancellationToken cancellationToken = default + ) + { + // For now, delegate to synchronous Read wrapped in ValueTask + return new ValueTask(Task.Run(() => + { + byte[] array = System.Buffers.ArrayPool.Shared.Rent(buffer.Length); + try + { + int read = Read(array, 0, buffer.Length); + array.AsSpan(0, read).CopyTo(buffer.Span); + return read; + } + finally + { + System.Buffers.ArrayPool.Shared.Return(array); + } + }, cancellationToken)); + } +#endif + public override Boolean CanRead => _stream.CanRead; public override Boolean CanSeek => _stream.CanSeek; diff --git a/src/SharpCompress/Compressors/Deflate/ZlibStream.cs b/src/SharpCompress/Compressors/Deflate/ZlibStream.cs index d20516499..8ce52d48a 100644 --- a/src/SharpCompress/Compressors/Deflate/ZlibStream.cs +++ b/src/SharpCompress/Compressors/Deflate/ZlibStream.cs @@ -28,6 +28,8 @@ using System; using System.IO; using System.Text; +using System.Threading; +using System.Threading.Tasks; using SharpCompress.IO; namespace SharpCompress.Compressors.Deflate; @@ -266,6 +268,34 @@ public override void Flush() _baseStream.Flush(); } + public override async Task FlushAsync(CancellationToken cancellationToken) + { + if (_disposed) + { + throw new ObjectDisposedException("ZlibStream"); + } + await _baseStream.FlushAsync(cancellationToken).ConfigureAwait(false); + } + +#if !NETFRAMEWORK && !NETSTANDARD2_0 + public override async ValueTask DisposeAsync() + { + if (_disposed) + { + return; + } + _disposed = true; + if (_baseStream != null) + { + await _baseStream.DisposeAsync().ConfigureAwait(false); + } +#if DEBUG_STREAMS + this.DebugDispose(typeof(ZlibStream)); +#endif + await base.DisposeAsync().ConfigureAwait(false); + } +#endif + /// /// Read data from the stream. /// @@ -301,6 +331,34 @@ public override int Read(byte[] buffer, int offset, int count) return _baseStream.Read(buffer, offset, count); } + public override async Task ReadAsync( + byte[] buffer, + int offset, + int count, + CancellationToken cancellationToken + ) + { + if (_disposed) + { + throw new ObjectDisposedException("ZlibStream"); + } + return await _baseStream.ReadAsync(buffer, offset, count, cancellationToken).ConfigureAwait(false); + } + +#if !NETFRAMEWORK && !NETSTANDARD2_0 + public override async ValueTask ReadAsync( + Memory buffer, + CancellationToken cancellationToken = default + ) + { + if (_disposed) + { + throw new ObjectDisposedException("ZlibStream"); + } + return await _baseStream.ReadAsync(buffer, cancellationToken).ConfigureAwait(false); + } +#endif + public override int ReadByte() { if (_disposed) @@ -355,6 +413,34 @@ public override void Write(byte[] buffer, int offset, int count) _baseStream.Write(buffer, offset, count); } + public override async Task WriteAsync( + byte[] buffer, + int offset, + int count, + CancellationToken cancellationToken + ) + { + if (_disposed) + { + throw new ObjectDisposedException("ZlibStream"); + } + await _baseStream.WriteAsync(buffer, offset, count, cancellationToken).ConfigureAwait(false); + } + +#if !NETFRAMEWORK && !NETSTANDARD2_0 + public override async ValueTask WriteAsync( + ReadOnlyMemory buffer, + CancellationToken cancellationToken = default + ) + { + if (_disposed) + { + throw new ObjectDisposedException("ZlibStream"); + } + await _baseStream.WriteAsync(buffer, cancellationToken).ConfigureAwait(false); + } +#endif + public override void WriteByte(byte value) { if (_disposed) diff --git a/src/SharpCompress/Utility.cs b/src/SharpCompress/Utility.cs index c6cb3e954..fb5021d11 100644 --- a/src/SharpCompress/Utility.cs +++ b/src/SharpCompress/Utility.cs @@ -91,6 +91,28 @@ public static void Skip(this Stream source) while (source.Read(buffer.Memory.Span) > 0) { } } + public static async Task SkipAsync(this Stream source, CancellationToken cancellationToken = default) + { + var array = ArrayPool.Shared.Rent(TEMP_BUFFER_SIZE); + try + { + while (true) + { + var read = await source + .ReadAsync(array, 0, array.Length, cancellationToken) + .ConfigureAwait(false); + if (read <= 0) + { + break; + } + } + } + finally + { + ArrayPool.Shared.Return(array); + } + } + public static DateTime DosDateToDateTime(ushort iDate, ushort iTime) { var year = (iDate / 512) + 1980; From 4f1b61f5bc42f770702af2ed5639c99f2e420d31 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 27 Oct 2025 09:20:37 +0000 Subject: [PATCH 04/12] Add async support to DeflateStream and GZipStream Co-authored-by: adamhathcock <527620+adamhathcock@users.noreply.github.com> --- .../Compressors/Deflate/DeflateStream.cs | 85 ++++++++++++ .../Compressors/Deflate/GZipStream.cs | 128 ++++++++++++++++++ 2 files changed, 213 insertions(+) diff --git a/src/SharpCompress/Compressors/Deflate/DeflateStream.cs b/src/SharpCompress/Compressors/Deflate/DeflateStream.cs index 8f268f004..73df2c73c 100644 --- a/src/SharpCompress/Compressors/Deflate/DeflateStream.cs +++ b/src/SharpCompress/Compressors/Deflate/DeflateStream.cs @@ -28,6 +28,7 @@ using System.IO; using System.Text; using System.Threading; +using System.Threading.Tasks; using SharpCompress.IO; namespace SharpCompress.Compressors.Deflate; @@ -289,6 +290,34 @@ public override void Flush() _baseStream.Flush(); } + public override async Task FlushAsync(CancellationToken cancellationToken) + { + if (_disposed) + { + throw new ObjectDisposedException("DeflateStream"); + } + await _baseStream.FlushAsync(cancellationToken).ConfigureAwait(false); + } + +#if !NETFRAMEWORK && !NETSTANDARD2_0 + public override async ValueTask DisposeAsync() + { + if (_disposed) + { + return; + } + _disposed = true; + if (_baseStream != null) + { + await _baseStream.DisposeAsync().ConfigureAwait(false); + } +#if DEBUG_STREAMS + this.DebugDispose(typeof(DeflateStream)); +#endif + await base.DisposeAsync().ConfigureAwait(false); + } +#endif + /// /// Read data from the stream. /// @@ -325,6 +354,34 @@ public override int Read(byte[] buffer, int offset, int count) return _baseStream.Read(buffer, offset, count); } + public override async Task ReadAsync( + byte[] buffer, + int offset, + int count, + CancellationToken cancellationToken + ) + { + if (_disposed) + { + throw new ObjectDisposedException("DeflateStream"); + } + return await _baseStream.ReadAsync(buffer, offset, count, cancellationToken).ConfigureAwait(false); + } + +#if !NETFRAMEWORK && !NETSTANDARD2_0 + public override async ValueTask ReadAsync( + Memory buffer, + CancellationToken cancellationToken = default + ) + { + if (_disposed) + { + throw new ObjectDisposedException("DeflateStream"); + } + return await _baseStream.ReadAsync(buffer, cancellationToken).ConfigureAwait(false); + } +#endif + public override int ReadByte() { if (_disposed) @@ -386,6 +443,34 @@ public override void Write(byte[] buffer, int offset, int count) _baseStream.Write(buffer, offset, count); } + public override async Task WriteAsync( + byte[] buffer, + int offset, + int count, + CancellationToken cancellationToken + ) + { + if (_disposed) + { + throw new ObjectDisposedException("DeflateStream"); + } + await _baseStream.WriteAsync(buffer, offset, count, cancellationToken).ConfigureAwait(false); + } + +#if !NETFRAMEWORK && !NETSTANDARD2_0 + public override async ValueTask WriteAsync( + ReadOnlyMemory buffer, + CancellationToken cancellationToken = default + ) + { + if (_disposed) + { + throw new ObjectDisposedException("DeflateStream"); + } + await _baseStream.WriteAsync(buffer, cancellationToken).ConfigureAwait(false); + } +#endif + public override void WriteByte(byte value) { if (_disposed) diff --git a/src/SharpCompress/Compressors/Deflate/GZipStream.cs b/src/SharpCompress/Compressors/Deflate/GZipStream.cs index 2f6510408..e830da619 100644 --- a/src/SharpCompress/Compressors/Deflate/GZipStream.cs +++ b/src/SharpCompress/Compressors/Deflate/GZipStream.cs @@ -30,6 +30,8 @@ using System.Buffers.Binary; using System.IO; using System.Text; +using System.Threading; +using System.Threading.Tasks; using SharpCompress.IO; namespace SharpCompress.Compressors.Deflate; @@ -257,6 +259,15 @@ public override void Flush() BaseStream.Flush(); } + public override async Task FlushAsync(CancellationToken cancellationToken) + { + if (_disposed) + { + throw new ObjectDisposedException("GZipStream"); + } + await BaseStream.FlushAsync(cancellationToken).ConfigureAwait(false); + } + /// /// Read and decompress data from the source stream. /// @@ -309,6 +320,52 @@ public override int Read(byte[] buffer, int offset, int count) return n; } + public override async Task ReadAsync( + byte[] buffer, + int offset, + int count, + CancellationToken cancellationToken + ) + { + if (_disposed) + { + throw new ObjectDisposedException("GZipStream"); + } + var n = await BaseStream.ReadAsync(buffer, offset, count, cancellationToken).ConfigureAwait(false); + + if (!_firstReadDone) + { + _firstReadDone = true; + FileName = BaseStream._GzipFileName; + Comment = BaseStream._GzipComment; + LastModified = BaseStream._GzipMtime; + } + return n; + } + +#if !NETFRAMEWORK && !NETSTANDARD2_0 + public override async ValueTask ReadAsync( + Memory buffer, + CancellationToken cancellationToken = default + ) + { + if (_disposed) + { + throw new ObjectDisposedException("GZipStream"); + } + var n = await BaseStream.ReadAsync(buffer, cancellationToken).ConfigureAwait(false); + + if (!_firstReadDone) + { + _firstReadDone = true; + FileName = BaseStream._GzipFileName; + Comment = BaseStream._GzipComment; + LastModified = BaseStream._GzipMtime; + } + return n; + } +#endif + /// /// Calling this method always throws a . /// @@ -368,6 +425,77 @@ public override void Write(byte[] buffer, int offset, int count) BaseStream.Write(buffer, offset, count); } + public override async Task WriteAsync( + byte[] buffer, + int offset, + int count, + CancellationToken cancellationToken + ) + { + if (_disposed) + { + throw new ObjectDisposedException("GZipStream"); + } + if (BaseStream._streamMode == ZlibBaseStream.StreamMode.Undefined) + { + if (BaseStream._wantCompress) + { + // first write in compression, therefore, emit the GZIP header + _headerByteCount = EmitHeader(); + } + else + { + throw new InvalidOperationException(); + } + } + + await BaseStream.WriteAsync(buffer, offset, count, cancellationToken).ConfigureAwait(false); + } + +#if !NETFRAMEWORK && !NETSTANDARD2_0 + public override async ValueTask WriteAsync( + ReadOnlyMemory buffer, + CancellationToken cancellationToken = default + ) + { + if (_disposed) + { + throw new ObjectDisposedException("GZipStream"); + } + if (BaseStream._streamMode == ZlibBaseStream.StreamMode.Undefined) + { + if (BaseStream._wantCompress) + { + // first write in compression, therefore, emit the GZIP header + _headerByteCount = EmitHeader(); + } + else + { + throw new InvalidOperationException(); + } + } + + await BaseStream.WriteAsync(buffer, cancellationToken).ConfigureAwait(false); + } + + public override async ValueTask DisposeAsync() + { + if (_disposed) + { + return; + } + _disposed = true; + if (BaseStream != null) + { + await BaseStream.DisposeAsync().ConfigureAwait(false); + } +#if DEBUG_STREAMS + this.DebugDispose(typeof(GZipStream)); +#endif + await base.DisposeAsync().ConfigureAwait(false); + } +#endif + #endregion Stream methods public string? Comment From b3975b7bbd0deb2ec2fffaeb0f635f18c016b13a Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 27 Oct 2025 09:28:34 +0000 Subject: [PATCH 05/12] Add async tests for EntryStream and compression streams Co-authored-by: adamhathcock <527620+adamhathcock@users.noreply.github.com> --- tests/SharpCompress.Test/AsyncTests.cs | 69 ++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/tests/SharpCompress.Test/AsyncTests.cs b/tests/SharpCompress.Test/AsyncTests.cs index a67f8f00d..6cab92d2e 100644 --- a/tests/SharpCompress.Test/AsyncTests.cs +++ b/tests/SharpCompress.Test/AsyncTests.cs @@ -130,4 +130,73 @@ public async Task Stream_Extensions_Async() Assert.True(File.Exists(outputPath)); Assert.True(new FileInfo(outputPath).Length > 0); } + + [Fact] + public async Task EntryStream_ReadAsync_Works() + { + var testArchive = Path.Combine(TEST_ARCHIVES_PATH, "Tar.tar.gz"); + using var stream = File.OpenRead(testArchive); + using var reader = ReaderFactory.Open(stream); + + while (reader.MoveToNextEntry()) + { + if (!reader.Entry.IsDirectory) + { + using var entryStream = reader.OpenEntryStream(); + var buffer = new byte[4096]; + var totalRead = 0; + int bytesRead; + + // Test ReadAsync on EntryStream + while ((bytesRead = await entryStream.ReadAsync(buffer, 0, buffer.Length)) > 0) + { + totalRead += bytesRead; + } + + Assert.True(totalRead > 0, "Should have read some data from entry stream"); + break; // Test just one entry + } + } + } + + [Fact] + public async Task CompressionStream_Async_ReadWrite() + { + var testData = new byte[1024]; + new Random(42).NextBytes(testData); + + var compressedPath = Path.Combine(SCRATCH_FILES_PATH, "async_compressed.gz"); + + // Test async write with GZipStream + using (var fileStream = File.Create(compressedPath)) + using (var gzipStream = new Compressors.Deflate.GZipStream( + fileStream, + Compressors.CompressionMode.Compress)) + { + await gzipStream.WriteAsync(testData, 0, testData.Length); + await gzipStream.FlushAsync(); + } + + Assert.True(File.Exists(compressedPath)); + Assert.True(new FileInfo(compressedPath).Length > 0); + + // Test async read with GZipStream + using (var fileStream = File.OpenRead(compressedPath)) + using (var gzipStream = new Compressors.Deflate.GZipStream( + fileStream, + Compressors.CompressionMode.Decompress)) + { + var decompressed = new byte[testData.Length]; + var totalRead = 0; + int bytesRead; + while (totalRead < decompressed.Length && + (bytesRead = await gzipStream.ReadAsync(decompressed, totalRead, decompressed.Length - totalRead)) > 0) + { + totalRead += bytesRead; + } + + Assert.Equal(testData.Length, totalRead); + Assert.Equal(testData, decompressed); + } + } } From f8cc4ade8ad5b8107120608577d345a4bdcf5e99 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Mon, 27 Oct 2025 09:37:00 +0000 Subject: [PATCH 06/12] format --- .../Archives/Tar/TarArchiveEntry.cs | 5 +- .../Archives/Zip/ZipArchiveEntry.cs | 5 +- src/SharpCompress/Common/EntryStream.cs | 4 +- .../Compressors/Deflate/DeflateStream.cs | 8 ++- .../Compressors/Deflate/GZipStream.cs | 4 +- .../Compressors/Deflate/ZlibBaseStream.cs | 64 +++++++++++++------ .../Compressors/Deflate/ZlibStream.cs | 8 ++- src/SharpCompress/IO/SourceStream.cs | 1 - src/SharpCompress/Readers/AbstractReader.cs | 3 +- src/SharpCompress/Utility.cs | 7 +- tests/SharpCompress.Test/AsyncTests.cs | 46 +++++++++---- 11 files changed, 108 insertions(+), 47 deletions(-) diff --git a/src/SharpCompress/Archives/Tar/TarArchiveEntry.cs b/src/SharpCompress/Archives/Tar/TarArchiveEntry.cs index e3d2219bb..8c0827917 100644 --- a/src/SharpCompress/Archives/Tar/TarArchiveEntry.cs +++ b/src/SharpCompress/Archives/Tar/TarArchiveEntry.cs @@ -14,8 +14,9 @@ internal TarArchiveEntry(TarArchive archive, TarFilePart? part, CompressionType public virtual Stream OpenEntryStream() => Parts.Single().GetCompressedStream().NotNull(); - public virtual Task OpenEntryStreamAsync(CancellationToken cancellationToken = default) => - Task.FromResult(OpenEntryStream()); + public virtual Task OpenEntryStreamAsync( + CancellationToken cancellationToken = default + ) => Task.FromResult(OpenEntryStream()); #region IArchiveEntry Members diff --git a/src/SharpCompress/Archives/Zip/ZipArchiveEntry.cs b/src/SharpCompress/Archives/Zip/ZipArchiveEntry.cs index 5ab93836a..a6baf34b3 100644 --- a/src/SharpCompress/Archives/Zip/ZipArchiveEntry.cs +++ b/src/SharpCompress/Archives/Zip/ZipArchiveEntry.cs @@ -13,8 +13,9 @@ internal ZipArchiveEntry(ZipArchive archive, SeekableZipFilePart? part) public virtual Stream OpenEntryStream() => Parts.Single().GetCompressedStream().NotNull(); - public virtual Task OpenEntryStreamAsync(CancellationToken cancellationToken = default) => - Task.FromResult(OpenEntryStream()); + public virtual Task OpenEntryStreamAsync( + CancellationToken cancellationToken = default + ) => Task.FromResult(OpenEntryStream()); #region IArchiveEntry Members diff --git a/src/SharpCompress/Common/EntryStream.cs b/src/SharpCompress/Common/EntryStream.cs index cfaef0aa1..97465e2b6 100644 --- a/src/SharpCompress/Common/EntryStream.cs +++ b/src/SharpCompress/Common/EntryStream.cs @@ -163,7 +163,9 @@ public override async Task ReadAsync( CancellationToken cancellationToken ) { - var read = await _stream.ReadAsync(buffer, offset, count, cancellationToken).ConfigureAwait(false); + var read = await _stream + .ReadAsync(buffer, offset, count, cancellationToken) + .ConfigureAwait(false); if (read <= 0) { _completed = true; diff --git a/src/SharpCompress/Compressors/Deflate/DeflateStream.cs b/src/SharpCompress/Compressors/Deflate/DeflateStream.cs index 73df2c73c..55c24813d 100644 --- a/src/SharpCompress/Compressors/Deflate/DeflateStream.cs +++ b/src/SharpCompress/Compressors/Deflate/DeflateStream.cs @@ -365,7 +365,9 @@ CancellationToken cancellationToken { throw new ObjectDisposedException("DeflateStream"); } - return await _baseStream.ReadAsync(buffer, offset, count, cancellationToken).ConfigureAwait(false); + return await _baseStream + .ReadAsync(buffer, offset, count, cancellationToken) + .ConfigureAwait(false); } #if !NETFRAMEWORK && !NETSTANDARD2_0 @@ -454,7 +456,9 @@ CancellationToken cancellationToken { throw new ObjectDisposedException("DeflateStream"); } - await _baseStream.WriteAsync(buffer, offset, count, cancellationToken).ConfigureAwait(false); + await _baseStream + .WriteAsync(buffer, offset, count, cancellationToken) + .ConfigureAwait(false); } #if !NETFRAMEWORK && !NETSTANDARD2_0 diff --git a/src/SharpCompress/Compressors/Deflate/GZipStream.cs b/src/SharpCompress/Compressors/Deflate/GZipStream.cs index e830da619..d9af42841 100644 --- a/src/SharpCompress/Compressors/Deflate/GZipStream.cs +++ b/src/SharpCompress/Compressors/Deflate/GZipStream.cs @@ -331,7 +331,9 @@ CancellationToken cancellationToken { throw new ObjectDisposedException("GZipStream"); } - var n = await BaseStream.ReadAsync(buffer, offset, count, cancellationToken).ConfigureAwait(false); + var n = await BaseStream + .ReadAsync(buffer, offset, count, cancellationToken) + .ConfigureAwait(false); if (!_firstReadDone) { diff --git a/src/SharpCompress/Compressors/Deflate/ZlibBaseStream.cs b/src/SharpCompress/Compressors/Deflate/ZlibBaseStream.cs index 52442f907..bc7bbf7b0 100644 --- a/src/SharpCompress/Compressors/Deflate/ZlibBaseStream.cs +++ b/src/SharpCompress/Compressors/Deflate/ZlibBaseStream.cs @@ -199,7 +199,12 @@ public override void Write(byte[] buffer, int offset, int count) } while (!done); } - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + public override async Task WriteAsync( + byte[] buffer, + int offset, + int count, + CancellationToken cancellationToken + ) { // workitem 7159 // calculate the CRC on the unccompressed data (before writing) @@ -238,7 +243,14 @@ public override async Task WriteAsync(byte[] buffer, int offset, int count, Canc throw new ZlibException((_wantCompress ? "de" : "in") + "flating: " + _z.Message); } - await _stream.WriteAsync(_workingBuffer, 0, _workingBuffer.Length - _z.AvailableBytesOut, cancellationToken).ConfigureAwait(false); + await _stream + .WriteAsync( + _workingBuffer, + 0, + _workingBuffer.Length - _z.AvailableBytesOut, + cancellationToken + ) + .ConfigureAwait(false); done = _z.AvailableBytesIn == 0 && _z.AvailableBytesOut != 0; @@ -418,7 +430,14 @@ private async Task finishAsync(CancellationToken cancellationToken = default) if (_workingBuffer.Length - _z.AvailableBytesOut > 0) { - await _stream.WriteAsync(_workingBuffer, 0, _workingBuffer.Length - _z.AvailableBytesOut, cancellationToken).ConfigureAwait(false); + await _stream + .WriteAsync( + _workingBuffer, + 0, + _workingBuffer.Length - _z.AvailableBytesOut, + cancellationToken + ) + .ConfigureAwait(false); } done = _z.AvailableBytesIn == 0 && _z.AvailableBytesOut != 0; @@ -473,9 +492,9 @@ private async Task finishAsync(CancellationToken cancellationToken = default) // Make sure we have read to the end of the stream _z.InputBuffer.AsSpan(_z.NextIn, _z.AvailableBytesIn).CopyTo(trailer); var bytesNeeded = 8 - _z.AvailableBytesIn; - var bytesRead = await _stream.ReadAsync( - trailer, _z.AvailableBytesIn, bytesNeeded, cancellationToken - ).ConfigureAwait(false); + var bytesRead = await _stream + .ReadAsync(trailer, _z.AvailableBytesIn, bytesNeeded, cancellationToken) + .ConfigureAwait(false); } } else @@ -888,20 +907,25 @@ public override ValueTask ReadAsync( ) { // For now, delegate to synchronous Read wrapped in ValueTask - return new ValueTask(Task.Run(() => - { - byte[] array = System.Buffers.ArrayPool.Shared.Rent(buffer.Length); - try - { - int read = Read(array, 0, buffer.Length); - array.AsSpan(0, read).CopyTo(buffer.Span); - return read; - } - finally - { - System.Buffers.ArrayPool.Shared.Return(array); - } - }, cancellationToken)); + return new ValueTask( + Task.Run( + () => + { + byte[] array = System.Buffers.ArrayPool.Shared.Rent(buffer.Length); + try + { + int read = Read(array, 0, buffer.Length); + array.AsSpan(0, read).CopyTo(buffer.Span); + return read; + } + finally + { + System.Buffers.ArrayPool.Shared.Return(array); + } + }, + cancellationToken + ) + ); } #endif diff --git a/src/SharpCompress/Compressors/Deflate/ZlibStream.cs b/src/SharpCompress/Compressors/Deflate/ZlibStream.cs index 8ce52d48a..3365832cc 100644 --- a/src/SharpCompress/Compressors/Deflate/ZlibStream.cs +++ b/src/SharpCompress/Compressors/Deflate/ZlibStream.cs @@ -342,7 +342,9 @@ CancellationToken cancellationToken { throw new ObjectDisposedException("ZlibStream"); } - return await _baseStream.ReadAsync(buffer, offset, count, cancellationToken).ConfigureAwait(false); + return await _baseStream + .ReadAsync(buffer, offset, count, cancellationToken) + .ConfigureAwait(false); } #if !NETFRAMEWORK && !NETSTANDARD2_0 @@ -424,7 +426,9 @@ CancellationToken cancellationToken { throw new ObjectDisposedException("ZlibStream"); } - await _baseStream.WriteAsync(buffer, offset, count, cancellationToken).ConfigureAwait(false); + await _baseStream + .WriteAsync(buffer, offset, count, cancellationToken) + .ConfigureAwait(false); } #if !NETFRAMEWORK && !NETSTANDARD2_0 diff --git a/src/SharpCompress/IO/SourceStream.cs b/src/SharpCompress/IO/SourceStream.cs index 2c152917d..0712d915c 100644 --- a/src/SharpCompress/IO/SourceStream.cs +++ b/src/SharpCompress/IO/SourceStream.cs @@ -337,7 +337,6 @@ public override async ValueTask ReadAsync( return total - count; } - #endif public override void Close() diff --git a/src/SharpCompress/Readers/AbstractReader.cs b/src/SharpCompress/Readers/AbstractReader.cs index 7d3be5a4f..370373736 100644 --- a/src/SharpCompress/Readers/AbstractReader.cs +++ b/src/SharpCompress/Readers/AbstractReader.cs @@ -211,8 +211,7 @@ internal async Task WriteAsync(Stream writeStream, CancellationToken cancellatio { var streamListener = this as IReaderExtractionListener; using Stream s = OpenEntryStream(); - await s - .TransferToAsync(writeStream, Entry, streamListener, cancellationToken) + await s.TransferToAsync(writeStream, Entry, streamListener, cancellationToken) .ConfigureAwait(false); } diff --git a/src/SharpCompress/Utility.cs b/src/SharpCompress/Utility.cs index fb5021d11..05ee7876a 100644 --- a/src/SharpCompress/Utility.cs +++ b/src/SharpCompress/Utility.cs @@ -91,7 +91,10 @@ public static void Skip(this Stream source) while (source.Read(buffer.Memory.Span) > 0) { } } - public static async Task SkipAsync(this Stream source, CancellationToken cancellationToken = default) + public static async Task SkipAsync( + this Stream source, + CancellationToken cancellationToken = default + ) { var array = ArrayPool.Shared.Rent(TEMP_BUFFER_SIZE); try @@ -261,7 +264,7 @@ public static async Task TransferToAsync( while ( await ReadTransferBlockAsync(source, array, maxReadSize, cancellationToken) .ConfigureAwait(false) - is var (success, count) + is var (success, count) && success ) { diff --git a/tests/SharpCompress.Test/AsyncTests.cs b/tests/SharpCompress.Test/AsyncTests.cs index 6cab92d2e..968d17262 100644 --- a/tests/SharpCompress.Test/AsyncTests.cs +++ b/tests/SharpCompress.Test/AsyncTests.cs @@ -28,7 +28,11 @@ await reader.WriteAllToDirectoryAsync( ); // Just verify some files were extracted - var extractedFiles = Directory.GetFiles(SCRATCH_FILES_PATH, "*", SearchOption.AllDirectories); + var extractedFiles = Directory.GetFiles( + SCRATCH_FILES_PATH, + "*", + SearchOption.AllDirectories + ); Assert.True(extractedFiles.Length > 0, "No files were extracted"); } @@ -107,7 +111,11 @@ await reader.WriteAllToDirectoryAsync( ); // Just verify some files were extracted - var extractedFiles = Directory.GetFiles(SCRATCH_FILES_PATH, "*", SearchOption.AllDirectories); + var extractedFiles = Directory.GetFiles( + SCRATCH_FILES_PATH, + "*", + SearchOption.AllDirectories + ); Assert.True(extractedFiles.Length > 0, "No files were extracted"); } @@ -146,7 +154,7 @@ public async Task EntryStream_ReadAsync_Works() var buffer = new byte[4096]; var totalRead = 0; int bytesRead; - + // Test ReadAsync on EntryStream while ((bytesRead = await entryStream.ReadAsync(buffer, 0, buffer.Length)) > 0) { @@ -166,12 +174,15 @@ public async Task CompressionStream_Async_ReadWrite() new Random(42).NextBytes(testData); var compressedPath = Path.Combine(SCRATCH_FILES_PATH, "async_compressed.gz"); - + // Test async write with GZipStream using (var fileStream = File.Create(compressedPath)) - using (var gzipStream = new Compressors.Deflate.GZipStream( - fileStream, - Compressors.CompressionMode.Compress)) + using ( + var gzipStream = new Compressors.Deflate.GZipStream( + fileStream, + Compressors.CompressionMode.Compress + ) + ) { await gzipStream.WriteAsync(testData, 0, testData.Length); await gzipStream.FlushAsync(); @@ -182,15 +193,26 @@ public async Task CompressionStream_Async_ReadWrite() // Test async read with GZipStream using (var fileStream = File.OpenRead(compressedPath)) - using (var gzipStream = new Compressors.Deflate.GZipStream( - fileStream, - Compressors.CompressionMode.Decompress)) + using ( + var gzipStream = new Compressors.Deflate.GZipStream( + fileStream, + Compressors.CompressionMode.Decompress + ) + ) { var decompressed = new byte[testData.Length]; var totalRead = 0; int bytesRead; - while (totalRead < decompressed.Length && - (bytesRead = await gzipStream.ReadAsync(decompressed, totalRead, decompressed.Length - totalRead)) > 0) + while ( + totalRead < decompressed.Length + && ( + bytesRead = await gzipStream.ReadAsync( + decompressed, + totalRead, + decompressed.Length - totalRead + ) + ) > 0 + ) { totalRead += bytesRead; } From f3d3ac30a60c015de7cb47bdef7f385ad4d622b7 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Mon, 27 Oct 2025 09:39:08 +0000 Subject: [PATCH 07/12] add gubbins --- SharpCompress.sln | 4 ++++ src/SharpCompress/packages.lock.json | 6 +++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/SharpCompress.sln b/SharpCompress.sln index bbd08dae2..fc9b64b8c 100644 --- a/SharpCompress.sln +++ b/SharpCompress.sln @@ -21,6 +21,10 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Config", "Config", "{CDB425 Directory.Packages.props = Directory.Packages.props NuGet.config = NuGet.config .github\workflows\dotnetcore.yml = .github\workflows\dotnetcore.yml + USAGE.md = USAGE.md + README.md = README.md + FORMATS.md = FORMATS.md + AGENTS.md = AGENTS.md EndProjectSection EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharpCompress.Performance", "tests\SharpCompress.Performance\SharpCompress.Performance.csproj", "{5BDE6DBC-9E5F-4E21-AB71-F138A3E72B17}" diff --git a/src/SharpCompress/packages.lock.json b/src/SharpCompress/packages.lock.json index b85a38f73..9ebb16774 100644 --- a/src/SharpCompress/packages.lock.json +++ b/src/SharpCompress/packages.lock.json @@ -335,9 +335,9 @@ "net8.0": { "Microsoft.NET.ILLink.Tasks": { "type": "Direct", - "requested": "[8.0.20, )", - "resolved": "8.0.20", - "contentHash": "Rhcto2AjGvTO62+/VTmBpumBOmqIGp7nYEbTbmEXkCq4yPGxV8whju3/HsIA/bKyo2+DggaYk5+/8sxb1AbPTw==" + "requested": "[8.0.17, )", + "resolved": "8.0.17", + "contentHash": "x5/y4l8AtshpBOrCZdlE4txw8K3e3s9meBFeZeR3l8hbbku2V7kK6ojhXvrbjg1rk3G+JqL1BI26gtgc1ZrdUw==" }, "Microsoft.SourceLink.GitHub": { "type": "Direct", From fdca728fdc2d475a1c02c09af4d4aed0bfb691c3 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Mon, 27 Oct 2025 09:47:15 +0000 Subject: [PATCH 08/12] add some dispose async --- .../{ => GZip}/AsyncTests.cs | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) rename tests/SharpCompress.Test/{ => GZip}/AsyncTests.cs (94%) diff --git a/tests/SharpCompress.Test/AsyncTests.cs b/tests/SharpCompress.Test/GZip/AsyncTests.cs similarity index 94% rename from tests/SharpCompress.Test/AsyncTests.cs rename to tests/SharpCompress.Test/GZip/AsyncTests.cs index 968d17262..d0895067e 100644 --- a/tests/SharpCompress.Test/AsyncTests.cs +++ b/tests/SharpCompress.Test/GZip/AsyncTests.cs @@ -6,12 +6,11 @@ using SharpCompress.Archives; using SharpCompress.Archives.Zip; using SharpCompress.Common; -using SharpCompress.IO; using SharpCompress.Readers; using SharpCompress.Writers; using Xunit; -namespace SharpCompress.Test; +namespace SharpCompress.Test.GZip; public class AsyncTests : TestBase { @@ -19,7 +18,11 @@ public class AsyncTests : TestBase public async Task Reader_Async_Extract_All() { var testArchive = Path.Combine(TEST_ARCHIVES_PATH, "Tar.tar.gz"); + #if NETFRAMEWORK using var stream = File.OpenRead(testArchive); + #else + await using var stream = File.OpenRead(testArchive); +#endif using var reader = ReaderFactory.Open(stream); await reader.WriteAllToDirectoryAsync( @@ -40,7 +43,11 @@ await reader.WriteAllToDirectoryAsync( public async Task Reader_Async_Extract_Single_Entry() { var testArchive = Path.Combine(TEST_ARCHIVES_PATH, "Tar.tar.gz"); +#if NETFRAMEWORK using var stream = File.OpenRead(testArchive); +#else + await using var stream = File.OpenRead(testArchive); +#endif using var reader = ReaderFactory.Open(stream); while (reader.MoveToNextEntry()) @@ -49,8 +56,11 @@ public async Task Reader_Async_Extract_Single_Entry() { var outputPath = Path.Combine(SCRATCH_FILES_PATH, reader.Entry.Key!); Directory.CreateDirectory(Path.GetDirectoryName(outputPath)!); - +#if NETFRAMEWORK using var outputStream = File.Create(outputPath); +#else + await using var outputStream = File.Create(outputPath); +#endif await reader.WriteEntryToAsync(outputStream); break; // Just test one entry } @@ -65,7 +75,11 @@ public async Task Archive_Entry_Async_Open_Stream() foreach (var entry in archive.Entries.Where(e => !e.IsDirectory).Take(1)) { +#if NETFRAMEWORK using var entryStream = await entry.OpenEntryStreamAsync(); +#else + await using var entryStream = await entry.OpenEntryStreamAsync(); +#endif Assert.NotNull(entryStream); Assert.True(entryStream.CanRead); From a4cc7eaf9b95d73975db0ec738248826078a2410 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Mon, 27 Oct 2025 09:51:39 +0000 Subject: [PATCH 09/12] fully use async for zlibbase --- .../Compressors/Deflate/ZlibBaseStream.cs | 302 ++++++++++++++++-- 1 file changed, 277 insertions(+), 25 deletions(-) diff --git a/src/SharpCompress/Compressors/Deflate/ZlibBaseStream.cs b/src/SharpCompress/Compressors/Deflate/ZlibBaseStream.cs index bc7bbf7b0..96651edb4 100644 --- a/src/SharpCompress/Compressors/Deflate/ZlibBaseStream.cs +++ b/src/SharpCompress/Compressors/Deflate/ZlibBaseStream.cs @@ -646,6 +646,31 @@ private string ReadZeroTerminatedString() return _encoding.GetString(buffer, 0, buffer.Length); } + private async Task ReadZeroTerminatedStringAsync(CancellationToken cancellationToken) + { + var list = new List(); + var done = false; + do + { + // workitem 7740 + var n = await _stream.ReadAsync(_buf1, 0, 1, cancellationToken).ConfigureAwait(false); + if (n != 1) + { + throw new ZlibException("Unexpected EOF reading GZIP header."); + } + if (_buf1[0] == 0) + { + done = true; + } + else + { + list.Add(_buf1[0]); + } + } while (!done); + var buffer = list.ToArray(); + return _encoding.GetString(buffer, 0, buffer.Length); + } + private int _ReadAndValidateGzipHeader() { var totalBytesRead = 0; @@ -704,6 +729,64 @@ private int _ReadAndValidateGzipHeader() return totalBytesRead; } + private async Task _ReadAndValidateGzipHeaderAsync(CancellationToken cancellationToken) + { + var totalBytesRead = 0; + + // read the header on the first read + byte[] header = new byte[10]; + var n = await _stream.ReadAsync(header, 0, 10, cancellationToken).ConfigureAwait(false); + + // workitem 8501: handle edge case (decompress empty stream) + if (n == 0) + { + return 0; + } + + if (n != 10) + { + throw new ZlibException("Not a valid GZIP stream."); + } + + if (header[0] != 0x1F || header[1] != 0x8B || header[2] != 8) + { + throw new ZlibException("Bad GZIP header."); + } + + var timet = BinaryPrimitives.ReadInt32LittleEndian(header.AsSpan(4)); + _GzipMtime = TarHeader.EPOCH.AddSeconds(timet); + totalBytesRead += n; + if ((header[3] & 0x04) == 0x04) + { + // read and discard extra field + n = await _stream.ReadAsync(header, 0, 2, cancellationToken).ConfigureAwait(false); // 2-byte length field + totalBytesRead += n; + + var extraLength = (short)(header[0] + header[1] * 256); + var extra = new byte[extraLength]; + n = await _stream.ReadAsync(extra, 0, extra.Length, cancellationToken).ConfigureAwait(false); + if (n != extraLength) + { + throw new ZlibException("Unexpected end-of-file reading GZIP header."); + } + totalBytesRead += n; + } + if ((header[3] & 0x08) == 0x08) + { + _GzipFileName = await ReadZeroTerminatedStringAsync(cancellationToken).ConfigureAwait(false); + } + if ((header[3] & 0x10) == 0x010) + { + _GzipComment = await ReadZeroTerminatedStringAsync(cancellationToken).ConfigureAwait(false); + } + if ((header[3] & 0x02) == 0x02) + { + await _stream.ReadAsync(_buf1, 0, 1, cancellationToken).ConfigureAwait(false); // CRC16, ignore + } + + return totalBytesRead; + } + public override Int32 Read(Byte[] buffer, Int32 offset, Int32 count) { // According to MS documentation, any implementation of the IO.Stream.Read function must: @@ -888,44 +971,213 @@ public override Int32 Read(Byte[] buffer, Int32 offset, Int32 count) return rc; } - public override Task ReadAsync( + public override async Task ReadAsync( byte[] buffer, int offset, int count, CancellationToken cancellationToken ) { - // For now, delegate to synchronous Read wrapped in Task - // A full async implementation would require refactoring the complex zlib codec logic - return Task.Run(() => Read(buffer, offset, count), cancellationToken); + // According to MS documentation, any implementation of the IO.Stream.Read function must: + // (a) throw an exception if offset & count reference an invalid part of the buffer, + // or if count < 0, or if buffer is null + // (b) return 0 only upon EOF, or if count = 0 + // (c) if not EOF, then return at least 1 byte, up to bytes + + if (_streamMode == StreamMode.Undefined) + { + if (!_stream.CanRead) + { + throw new ZlibException("The stream is not readable."); + } + + // for the first read, set up some controls. + _streamMode = StreamMode.Reader; + + // (The first reference to _z goes through the private accessor which + // may initialize it.) + z.AvailableBytesIn = 0; + if (_flavor == ZlibStreamFlavor.GZIP) + { + _gzipHeaderByteCount = await _ReadAndValidateGzipHeaderAsync(cancellationToken).ConfigureAwait(false); + + // workitem 8501: handle edge case (decompress empty stream) + if (_gzipHeaderByteCount == 0) + { + return 0; + } + } + } + + if (_streamMode != StreamMode.Reader) + { + throw new ZlibException("Cannot Read after Writing."); + } + + var rc = 0; + + // set up the output of the deflate/inflate codec: + _z.OutputBuffer = buffer; + _z.NextOut = offset; + _z.AvailableBytesOut = count; + + if (count == 0) + { + return 0; + } + if (nomoreinput && _wantCompress) + { + // no more input data available; therefore we flush to + // try to complete the read + rc = _z.Deflate(FlushType.Finish); + + if (rc != ZlibConstants.Z_OK && rc != ZlibConstants.Z_STREAM_END) + { + throw new ZlibException( + String.Format("Deflating: rc={0} msg={1}", rc, _z.Message) + ); + } + + rc = (count - _z.AvailableBytesOut); + + // calculate CRC after reading + if (crc != null) + { + crc.SlurpBlock(buffer, offset, rc); + } + + return rc; + } + if (buffer is null) + { + throw new ArgumentNullException(nameof(buffer)); + } + if (count < 0) + { + throw new ArgumentOutOfRangeException(nameof(count)); + } + if (offset < buffer.GetLowerBound(0)) + { + throw new ArgumentOutOfRangeException(nameof(offset)); + } + if ((offset + count) > buffer.GetLength(0)) + { + throw new ArgumentOutOfRangeException(nameof(count)); + } + + // This is necessary in case _workingBuffer has been resized. (new byte[]) + // (The first reference to _workingBuffer goes through the private accessor which + // may initialize it.) + _z.InputBuffer = workingBuffer; + + do + { + // need data in _workingBuffer in order to deflate/inflate. Here, we check if we have any. + if ((_z.AvailableBytesIn == 0) && (!nomoreinput)) + { + // No data available, so try to Read data from the captive stream. + _z.NextIn = 0; + _z.AvailableBytesIn = await _stream.ReadAsync(_workingBuffer, 0, _workingBuffer.Length, cancellationToken).ConfigureAwait(false); + if (_z.AvailableBytesIn == 0) + { + nomoreinput = true; + } + } + + // we have data in InputBuffer; now compress or decompress as appropriate + rc = (_wantCompress) ? _z.Deflate(_flushMode) : _z.Inflate(_flushMode); + + if (nomoreinput && (rc == ZlibConstants.Z_BUF_ERROR)) + { + return 0; + } + + if (rc != ZlibConstants.Z_OK && rc != ZlibConstants.Z_STREAM_END) + { + throw new ZlibException( + String.Format( + "{0}flating: rc={1} msg={2}", + (_wantCompress ? "de" : "in"), + rc, + _z.Message + ) + ); + } + + if ( + (nomoreinput || rc == ZlibConstants.Z_STREAM_END) && (_z.AvailableBytesOut == count) + ) + { + break; // nothing more to read + } + } //while (_z.AvailableBytesOut == count && rc == ZlibConstants.Z_OK); + while (_z.AvailableBytesOut > 0 && !nomoreinput && rc == ZlibConstants.Z_OK); + + // workitem 8557 + // is there more room in output? + if (_z.AvailableBytesOut > 0) + { + if (rc == ZlibConstants.Z_OK && _z.AvailableBytesIn == 0) + { + // deferred + } + + // are we completely done reading? + if (nomoreinput) + { + // and in compression? + if (_wantCompress) + { + // no more input data available; therefore we flush to + // try to complete the read + rc = _z.Deflate(FlushType.Finish); + + if (rc != ZlibConstants.Z_OK && rc != ZlibConstants.Z_STREAM_END) + { + throw new ZlibException( + String.Format("Deflating: rc={0} msg={1}", rc, _z.Message) + ); + } + } + } + } + + rc = (count - _z.AvailableBytesOut); + + // calculate CRC after reading + if (crc != null) + { + crc.SlurpBlock(buffer, offset, rc); + } + + if (rc == ZlibConstants.Z_STREAM_END && z.AvailableBytesIn != 0 && !_wantCompress) + { + //rewind the buffer + ((IStreamStack)this).Rewind(z.AvailableBytesIn); //unused + z.AvailableBytesIn = 0; + } + + return rc; } #if !NETFRAMEWORK && !NETSTANDARD2_0 - public override ValueTask ReadAsync( + public override async ValueTask ReadAsync( Memory buffer, CancellationToken cancellationToken = default ) { - // For now, delegate to synchronous Read wrapped in ValueTask - return new ValueTask( - Task.Run( - () => - { - byte[] array = System.Buffers.ArrayPool.Shared.Rent(buffer.Length); - try - { - int read = Read(array, 0, buffer.Length); - array.AsSpan(0, read).CopyTo(buffer.Span); - return read; - } - finally - { - System.Buffers.ArrayPool.Shared.Return(array); - } - }, - cancellationToken - ) - ); + // Use ArrayPool to rent a buffer and delegate to byte[] ReadAsync + byte[] array = System.Buffers.ArrayPool.Shared.Rent(buffer.Length); + try + { + int read = await ReadAsync(array, 0, buffer.Length, cancellationToken).ConfigureAwait(false); + array.AsSpan(0, read).CopyTo(buffer.Span); + return read; + } + finally + { + System.Buffers.ArrayPool.Shared.Return(array); + } } #endif From 738a72228b6402089b81292ab959347cba0dc645 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Mon, 27 Oct 2025 10:15:06 +0000 Subject: [PATCH 10/12] added fixes and more async tests --- .../Common/Tar/TarReadOnlySubStream.cs | 70 ++++++++++++ src/SharpCompress/Readers/AbstractReader.cs | 6 ++ .../GZip/GZipReaderAsyncTests.cs | 100 ++++++++++++++++++ 3 files changed, 176 insertions(+) create mode 100644 tests/SharpCompress.Test/GZip/GZipReaderAsyncTests.cs diff --git a/src/SharpCompress/Common/Tar/TarReadOnlySubStream.cs b/src/SharpCompress/Common/Tar/TarReadOnlySubStream.cs index 46e8120a8..ceb269bd8 100644 --- a/src/SharpCompress/Common/Tar/TarReadOnlySubStream.cs +++ b/src/SharpCompress/Common/Tar/TarReadOnlySubStream.cs @@ -66,6 +66,36 @@ protected override void Dispose(bool disposing) base.Dispose(disposing); } +#if !NETFRAMEWORK && !NETSTANDARD2_0 + public override async System.Threading.Tasks.ValueTask DisposeAsync() + { + if (_isDisposed) + { + return; + } + + _isDisposed = true; +#if DEBUG_STREAMS + this.DebugDispose(typeof(TarReadOnlySubStream)); +#endif + // Ensure we read all remaining blocks for this entry. + await Stream.SkipAsync(BytesLeftToRead).ConfigureAwait(false); + _amountRead += BytesLeftToRead; + + // If the last block wasn't a full 512 bytes, skip the remaining padding bytes. + var bytesInLastBlock = _amountRead % 512; + + if (bytesInLastBlock != 0) + { + await Stream.SkipAsync(512 - bytesInLastBlock).ConfigureAwait(false); + } + + // Call base Dispose instead of base DisposeAsync to avoid double disposal + base.Dispose(true); + GC.SuppressFinalize(this); + } +#endif + private long BytesLeftToRead { get; set; } public override bool CanRead => true; @@ -114,6 +144,46 @@ public override int ReadByte() return value; } + public override async System.Threading.Tasks.Task ReadAsync( + byte[] buffer, + int offset, + int count, + System.Threading.CancellationToken cancellationToken + ) + { + if (BytesLeftToRead < count) + { + count = (int)BytesLeftToRead; + } + var read = await Stream.ReadAsync(buffer, offset, count, cancellationToken).ConfigureAwait(false); + if (read > 0) + { + BytesLeftToRead -= read; + _amountRead += read; + } + return read; + } + +#if !NETFRAMEWORK && !NETSTANDARD2_0 + public override async System.Threading.Tasks.ValueTask ReadAsync( + System.Memory buffer, + System.Threading.CancellationToken cancellationToken = default + ) + { + if (BytesLeftToRead < buffer.Length) + { + buffer = buffer.Slice(0, (int)BytesLeftToRead); + } + var read = await Stream.ReadAsync(buffer, cancellationToken).ConfigureAwait(false); + if (read > 0) + { + BytesLeftToRead -= read; + _amountRead += read; + } + return read; + } +#endif + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); public override void SetLength(long value) => throw new NotSupportedException(); diff --git a/src/SharpCompress/Readers/AbstractReader.cs b/src/SharpCompress/Readers/AbstractReader.cs index 370373736..4bc71031f 100644 --- a/src/SharpCompress/Readers/AbstractReader.cs +++ b/src/SharpCompress/Readers/AbstractReader.cs @@ -210,9 +210,15 @@ internal void Write(Stream writeStream) internal async Task WriteAsync(Stream writeStream, CancellationToken cancellationToken) { var streamListener = this as IReaderExtractionListener; +#if NETFRAMEWORK || NETSTANDARD2_0 using Stream s = OpenEntryStream(); await s.TransferToAsync(writeStream, Entry, streamListener, cancellationToken) .ConfigureAwait(false); +#else + await using Stream s = OpenEntryStream(); + await s.TransferToAsync(writeStream, Entry, streamListener, cancellationToken) + .ConfigureAwait(false); +#endif } public EntryStream OpenEntryStream() diff --git a/tests/SharpCompress.Test/GZip/GZipReaderAsyncTests.cs b/tests/SharpCompress.Test/GZip/GZipReaderAsyncTests.cs new file mode 100644 index 000000000..70771ab1a --- /dev/null +++ b/tests/SharpCompress.Test/GZip/GZipReaderAsyncTests.cs @@ -0,0 +1,100 @@ +using System.IO; +using System.Threading.Tasks; +using SharpCompress.Common; +using SharpCompress.IO; +using SharpCompress.Readers; +using SharpCompress.Readers.GZip; +using SharpCompress.Test.Mocks; +using Xunit; + +namespace SharpCompress.Test.GZip; + +public class GZipReaderAsyncTests : ReaderTests +{ + public GZipReaderAsyncTests() => UseExtensionInsteadOfNameToVerify = true; + + [Fact] + public async Task GZip_Reader_Generic_Async() => + await ReadAsync("Tar.tar.gz", CompressionType.GZip); + + [Fact] + public async Task GZip_Reader_Generic2_Async() + { + //read only as GZip item + using Stream stream = File.OpenRead(Path.Combine(TEST_ARCHIVES_PATH, "Tar.tar.gz")); + using var reader = GZipReader.Open(new SharpCompressStream(stream)); + while (reader.MoveToNextEntry()) + { + Assert.NotEqual(0, reader.Entry.Size); + Assert.NotEqual(0, reader.Entry.Crc); + + // Use async overload for reading the entry + if (!reader.Entry.IsDirectory) + { + using var entryStream = reader.OpenEntryStream(); + using var ms = new MemoryStream(); + await entryStream.CopyToAsync(ms); + } + } + } + + protected async Task ReadAsync( + string testArchive, + CompressionType expectedCompression, + ReaderOptions? options = null + ) + { + testArchive = Path.Combine(TEST_ARCHIVES_PATH, testArchive); + + options ??= new ReaderOptions() { BufferSize = 0x20000 }; + + options.LeaveStreamOpen = true; + await ReadImplAsync(testArchive, expectedCompression, options); + + options.LeaveStreamOpen = false; + await ReadImplAsync(testArchive, expectedCompression, options); + VerifyFiles(); + } + + private async Task ReadImplAsync( + string testArchive, + CompressionType expectedCompression, + ReaderOptions options + ) + { + using var file = File.OpenRead(testArchive); + using var protectedStream = SharpCompressStream.Create( + new ForwardOnlyStream(file, options.BufferSize), + leaveOpen: true, + throwOnDispose: true, + bufferSize: options.BufferSize + ); + using var testStream = new TestStream(protectedStream); + using (var reader = ReaderFactory.Open(testStream, options)) + { + await UseReaderAsync(reader, expectedCompression); + protectedStream.ThrowOnDispose = false; + Assert.False(testStream.IsDisposed, $"{nameof(testStream)} prematurely closed"); + } + + var message = + $"{nameof(options.LeaveStreamOpen)} is set to '{options.LeaveStreamOpen}', so {nameof(testStream.IsDisposed)} should be set to '{!testStream.IsDisposed}', but is set to {testStream.IsDisposed}"; + Assert.True(options.LeaveStreamOpen != testStream.IsDisposed, message); + } + + private async Task UseReaderAsync(IReader reader, CompressionType expectedCompression) + { + while (reader.MoveToNextEntry()) + { + if (!reader.Entry.IsDirectory) + { + Assert.Equal(expectedCompression, reader.Entry.CompressionType); + await reader.WriteEntryToDirectoryAsync( + SCRATCH_FILES_PATH, + new ExtractionOptions { ExtractFullPath = true, Overwrite = true } + ); + } + } + } +} + From c696197b03da5d4e3d55bf2e475e1a9eaa2a4c79 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Mon, 27 Oct 2025 10:19:24 +0000 Subject: [PATCH 11/12] formatting --- .../Common/Tar/TarReadOnlySubStream.cs | 8 +++++++- .../Compressors/Deflate/ZlibBaseStream.cs | 20 +++++++++++++------ tests/SharpCompress.Test/GZip/AsyncTests.cs | 4 ++-- .../GZip/GZipReaderAsyncTests.cs | 1 - 4 files changed, 23 insertions(+), 10 deletions(-) diff --git a/src/SharpCompress/Common/Tar/TarReadOnlySubStream.cs b/src/SharpCompress/Common/Tar/TarReadOnlySubStream.cs index ceb269bd8..4e6ddb700 100644 --- a/src/SharpCompress/Common/Tar/TarReadOnlySubStream.cs +++ b/src/SharpCompress/Common/Tar/TarReadOnlySubStream.cs @@ -106,6 +106,10 @@ public override async System.Threading.Tasks.ValueTask DisposeAsync() public override void Flush() { } + public override System.Threading.Tasks.Task FlushAsync( + System.Threading.CancellationToken cancellationToken + ) => System.Threading.Tasks.Task.CompletedTask; + public override long Length => throw new NotSupportedException(); public override long Position @@ -155,7 +159,9 @@ System.Threading.CancellationToken cancellationToken { count = (int)BytesLeftToRead; } - var read = await Stream.ReadAsync(buffer, offset, count, cancellationToken).ConfigureAwait(false); + var read = await Stream + .ReadAsync(buffer, offset, count, cancellationToken) + .ConfigureAwait(false); if (read > 0) { BytesLeftToRead -= read; diff --git a/src/SharpCompress/Compressors/Deflate/ZlibBaseStream.cs b/src/SharpCompress/Compressors/Deflate/ZlibBaseStream.cs index 96651edb4..e2a757c62 100644 --- a/src/SharpCompress/Compressors/Deflate/ZlibBaseStream.cs +++ b/src/SharpCompress/Compressors/Deflate/ZlibBaseStream.cs @@ -764,7 +764,9 @@ private async Task _ReadAndValidateGzipHeaderAsync(CancellationToken cancel var extraLength = (short)(header[0] + header[1] * 256); var extra = new byte[extraLength]; - n = await _stream.ReadAsync(extra, 0, extra.Length, cancellationToken).ConfigureAwait(false); + n = await _stream + .ReadAsync(extra, 0, extra.Length, cancellationToken) + .ConfigureAwait(false); if (n != extraLength) { throw new ZlibException("Unexpected end-of-file reading GZIP header."); @@ -773,11 +775,13 @@ private async Task _ReadAndValidateGzipHeaderAsync(CancellationToken cancel } if ((header[3] & 0x08) == 0x08) { - _GzipFileName = await ReadZeroTerminatedStringAsync(cancellationToken).ConfigureAwait(false); + _GzipFileName = await ReadZeroTerminatedStringAsync(cancellationToken) + .ConfigureAwait(false); } if ((header[3] & 0x10) == 0x010) { - _GzipComment = await ReadZeroTerminatedStringAsync(cancellationToken).ConfigureAwait(false); + _GzipComment = await ReadZeroTerminatedStringAsync(cancellationToken) + .ConfigureAwait(false); } if ((header[3] & 0x02) == 0x02) { @@ -999,7 +1003,8 @@ CancellationToken cancellationToken z.AvailableBytesIn = 0; if (_flavor == ZlibStreamFlavor.GZIP) { - _gzipHeaderByteCount = await _ReadAndValidateGzipHeaderAsync(cancellationToken).ConfigureAwait(false); + _gzipHeaderByteCount = await _ReadAndValidateGzipHeaderAsync(cancellationToken) + .ConfigureAwait(false); // workitem 8501: handle edge case (decompress empty stream) if (_gzipHeaderByteCount == 0) @@ -1077,7 +1082,9 @@ CancellationToken cancellationToken { // No data available, so try to Read data from the captive stream. _z.NextIn = 0; - _z.AvailableBytesIn = await _stream.ReadAsync(_workingBuffer, 0, _workingBuffer.Length, cancellationToken).ConfigureAwait(false); + _z.AvailableBytesIn = await _stream + .ReadAsync(_workingBuffer, 0, _workingBuffer.Length, cancellationToken) + .ConfigureAwait(false); if (_z.AvailableBytesIn == 0) { nomoreinput = true; @@ -1170,7 +1177,8 @@ public override async ValueTask ReadAsync( byte[] array = System.Buffers.ArrayPool.Shared.Rent(buffer.Length); try { - int read = await ReadAsync(array, 0, buffer.Length, cancellationToken).ConfigureAwait(false); + int read = await ReadAsync(array, 0, buffer.Length, cancellationToken) + .ConfigureAwait(false); array.AsSpan(0, read).CopyTo(buffer.Span); return read; } diff --git a/tests/SharpCompress.Test/GZip/AsyncTests.cs b/tests/SharpCompress.Test/GZip/AsyncTests.cs index d0895067e..5f2ed4fe4 100644 --- a/tests/SharpCompress.Test/GZip/AsyncTests.cs +++ b/tests/SharpCompress.Test/GZip/AsyncTests.cs @@ -18,9 +18,9 @@ public class AsyncTests : TestBase public async Task Reader_Async_Extract_All() { var testArchive = Path.Combine(TEST_ARCHIVES_PATH, "Tar.tar.gz"); - #if NETFRAMEWORK +#if NETFRAMEWORK using var stream = File.OpenRead(testArchive); - #else +#else await using var stream = File.OpenRead(testArchive); #endif using var reader = ReaderFactory.Open(stream); diff --git a/tests/SharpCompress.Test/GZip/GZipReaderAsyncTests.cs b/tests/SharpCompress.Test/GZip/GZipReaderAsyncTests.cs index 70771ab1a..20e9a34fb 100644 --- a/tests/SharpCompress.Test/GZip/GZipReaderAsyncTests.cs +++ b/tests/SharpCompress.Test/GZip/GZipReaderAsyncTests.cs @@ -97,4 +97,3 @@ await reader.WriteEntryToDirectoryAsync( } } } - From 63d08ebfd220a09990d58659853fa15d05ea3904 Mon Sep 17 00:00:00 2001 From: Adam Hathcock Date: Mon, 27 Oct 2025 10:19:57 +0000 Subject: [PATCH 12/12] update agents --- AGENTS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/AGENTS.md b/AGENTS.md index a47fa0297..62c8d4c10 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -26,6 +26,7 @@ applyTo: '**/*.cs' - Use CSharpier for all code formatting to ensure consistent style across the project. - Install CSharpier globally: `dotnet tool install -g csharpier` - Format files with: `dotnet csharpier format .` +- **ALWAYS run `dotnet csharpier format .` after making code changes before committing.** - Configure your IDE to format on save using CSharpier. - CSharpier configuration can be customized via `.csharpierrc` file in the project root. - Trust CSharpier's opinionated formatting decisions to maintain consistency.