Skip to content

Zip: fix Zip64 streaming reader corrupting entry size/CRC and failing on non-seekable streams #1362

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
adamhathcock merged 1 commit into from
Jul 2, 2026
+244 −85
Merged

Zip: fix Zip64 streaming reader corrupting entry size/CRC and failing on non-seekable streams #1362

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
109 changes: 61 additions & 48 deletions src/SharpCompress/Common/Zip/StreamingZipHeaderFactory.Async.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -189,64 +189,77 @@ public async ValueTask<bool> MoveNextAsync()
.ReadUInt32Async(_cancellationToken)
.ConfigureAwait(false);

_ = await _reader.ReadUInt16Async(_cancellationToken).ConfigureAwait(false); // version
_ = await _reader.ReadUInt16Async(_cancellationToken).ConfigureAwait(false); // flags
_ = await _reader.ReadUInt16Async(_cancellationToken).ConfigureAwait(false); // compressionMethod
_ = await _reader.ReadUInt16Async(_cancellationToken).ConfigureAwait(false); // lastModifiedDate
_ = await _reader.ReadUInt16Async(_cancellationToken).ConfigureAwait(false); // lastModifiedTime

var crc = await _reader
.ReadUInt32Async(_cancellationToken)
.ConfigureAwait(false);

if (crc == POST_DATA_DESCRIPTOR)
// A Zip64 entry that does not use a post-data descriptor stores its real CRC
// and sizes in the local header's Zip64 extra field, so a header signature
// (the next local entry, or the central directory) follows the data directly.
// In that case the entry's metadata is already correct and must not be
// overwritten with bytes read from the following header. We have only consumed
// the 4-byte signature, so fall through and parse this header normally. Because
// no seek-back is required here, this also works for non-seekable streams.
if (headerBytes == 0x04034b50 || headerBytes == 0x02014b50)
{
crc = await _reader
if (pos.HasValue)
{
lastEntryHeader.DataStartPosition =
pos - lastEntryHeader.CompressedSize;
}
}
else
{
// A data descriptor follows. Recover the CRC and sizes from it; the
// descriptor can carry either 32-bit or 64-bit sizes.
_ = await _reader.ReadUInt16Async(_cancellationToken).ConfigureAwait(false); // version
_ = await _reader.ReadUInt16Async(_cancellationToken).ConfigureAwait(false); // flags
_ = await _reader.ReadUInt16Async(_cancellationToken).ConfigureAwait(false); // compressionMethod
_ = await _reader.ReadUInt16Async(_cancellationToken).ConfigureAwait(false); // lastModifiedDate
_ = await _reader.ReadUInt16Async(_cancellationToken).ConfigureAwait(false); // lastModifiedTime

var crc = await _reader
.ReadUInt32Async(_cancellationToken)
.ConfigureAwait(false);
}
lastEntryHeader.Crc = crc;
lastEntryHeader.IsCrcAvailable = true;

// The DataDescriptor can be either 64bit or 32bit
var compressedSize = await _reader
.ReadUInt32Async(_cancellationToken)
.ConfigureAwait(false);
var uncompressedSize = await _reader
.ReadUInt32Async(_cancellationToken)
.ConfigureAwait(false);

// Check if we have header or 64bit DataDescriptor
var testHeader = !(headerBytes == 0x04034b50 || headerBytes == 0x02014b50);
if (crc == POST_DATA_DESCRIPTOR)
{
crc = await _reader
.ReadUInt32Async(_cancellationToken)
.ConfigureAwait(false);
}
lastEntryHeader.Crc = crc;
lastEntryHeader.IsCrcAvailable = true;

var test64Bit = ((long)uncompressedSize << 32) | compressedSize;
if (test64Bit == lastEntryHeader.CompressedSize && testHeader)
{
lastEntryHeader.UncompressedSize =
(
(long)
await _reader
.ReadUInt32Async(_cancellationToken)
.ConfigureAwait(false) << 32
) | headerBytes;
headerBytes = await _reader
// The DataDescriptor can be either 64bit or 32bit
var compressedSize = await _reader
.ReadUInt32Async(_cancellationToken)
.ConfigureAwait(false);
var uncompressedSize = await _reader
.ReadUInt32Async(_cancellationToken)
.ConfigureAwait(false);
}
else
{
lastEntryHeader.UncompressedSize = uncompressedSize;
}

if (pos.HasValue)
{
lastEntryHeader.DataStartPosition = pos - lastEntryHeader.CompressedSize;
var test64Bit = ((long)uncompressedSize << 32) | compressedSize;
if (test64Bit == lastEntryHeader.CompressedSize)
{
lastEntryHeader.UncompressedSize =
(
(long)
await _reader
.ReadUInt32Async(_cancellationToken)
.ConfigureAwait(false) << 32
) | headerBytes;
headerBytes = await _reader
.ReadUInt32Async(_cancellationToken)
.ConfigureAwait(false);
}
else
{
lastEntryHeader.UncompressedSize = uncompressedSize;
}

// For SeekableSharpCompressStream, seek back to just after the local header signature.
// Plain SharpCompressStream cannot seek to arbitrary positions, so we skip this.
// 4 = First 4 bytes of the entry header (i.e. 50 4B 03 04)
if (_sharpCompressStream is SeekableSharpCompressStream)
if (pos.HasValue)
{
lastEntryHeader.DataStartPosition =
pos - lastEntryHeader.CompressedSize;

// 4 = First 4 bytes of the entry header (i.e. 50 4B 03 04)
_sharpCompressStream.Position = pos.Value + 4;
}
}
Expand Down
91 changes: 54 additions & 37 deletions src/SharpCompress/Common/Zip/StreamingZipHeaderFactory.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,46 +109,63 @@ internal IEnumerable<ZipHeader> ReadStreamHeader(Stream stream)

headerBytes = reader.ReadUInt32();

var version = reader.ReadUInt16();
var flags = (HeaderFlags)reader.ReadUInt16();
var compressionMethod = (ZipCompressionMethod)reader.ReadUInt16();
var lastModifiedDate = reader.ReadUInt16();
var lastModifiedTime = reader.ReadUInt16();

var crc = reader.ReadUInt32();

if (crc == POST_DATA_DESCRIPTOR)
{
crc = reader.ReadUInt32();
}
_lastEntryHeader.Crc = crc;
_lastEntryHeader.IsCrcAvailable = true;

// The DataDescriptor can be either 64bit or 32bit
var compressed_size = reader.ReadUInt32();
var uncompressed_size = reader.ReadUInt32();

// Check if we have header or 64bit DataDescriptor
var test_header = !(headerBytes == 0x04034b50 || headerBytes == 0x02014b50);

var test_64bit = ((long)uncompressed_size << 32) | compressed_size;
if (test_64bit == _lastEntryHeader.CompressedSize && test_header)
{
_lastEntryHeader.UncompressedSize =
((long)reader.ReadUInt32() << 32) | headerBytes;
headerBytes = reader.ReadUInt32();
// A Zip64 entry that does not use a post-data descriptor stores its real CRC
// and sizes in the local header's Zip64 extra field, so a header signature
// (the next local entry, or the central directory) follows the data directly.
// In that case the entry's metadata is already correct and must not be
// overwritten with bytes read from the following header. We have only consumed
// the 4-byte signature, so fall through and parse this header normally.
if (headerBytes == 0x04034b50 || headerBytes == 0x02014b50)
{
if (pos.HasValue)
{
_lastEntryHeader.DataStartPosition =
pos - _lastEntryHeader.CompressedSize;
}
}
else
{
_lastEntryHeader.UncompressedSize = uncompressed_size;
}

if (pos.HasValue)
{
_lastEntryHeader.DataStartPosition = pos - _lastEntryHeader.CompressedSize;

// 4 = First 4 bytes of the entry header (i.e. 50 4B 03 04)
sharpCompressStream.Position = pos.Value + 4;
// A data descriptor follows. Recover the CRC and sizes from it; the
// descriptor can carry either 32-bit or 64-bit sizes.
_ = reader.ReadUInt16(); // version
_ = reader.ReadUInt16(); // flags
_ = reader.ReadUInt16(); // compressionMethod
_ = reader.ReadUInt16(); // lastModifiedDate
_ = reader.ReadUInt16(); // lastModifiedTime

var crc = reader.ReadUInt32();

if (crc == POST_DATA_DESCRIPTOR)
{
crc = reader.ReadUInt32();
}
_lastEntryHeader.Crc = crc;
_lastEntryHeader.IsCrcAvailable = true;

// The DataDescriptor can be either 64bit or 32bit
var compressed_size = reader.ReadUInt32();
var uncompressed_size = reader.ReadUInt32();

var test_64bit = ((long)uncompressed_size << 32) | compressed_size;
if (test_64bit == _lastEntryHeader.CompressedSize)
{
_lastEntryHeader.UncompressedSize =
((long)reader.ReadUInt32() << 32) | headerBytes;
headerBytes = reader.ReadUInt32();
}
else
{
_lastEntryHeader.UncompressedSize = uncompressed_size;
}

if (pos.HasValue)
{
_lastEntryHeader.DataStartPosition =
pos - _lastEntryHeader.CompressedSize;

// 4 = First 4 bytes of the entry header (i.e. 50 4B 03 04)
sharpCompressStream.Position = pos.Value + 4;
}
}
}
else
Expand Down
129 changes: 129 additions & 0 deletions tests/SharpCompress.Test/Zip/Zip64AsyncTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,135 @@ public async ValueTask Zip64_Single_Large_File_Streaming_Fail_Async()
catch (NotSupportedException) { }
}

// Regression test for reading a Zip64 archive over a *non-seekable* async stream, as
// happens when extracting directly from a network download. When a >=4GB (Zip64) entry
// is followed by another entry, the streaming reader probes a few bytes past the big
// entry's data to locate the next header and must rewind them. For non-seekable streams
// it previously failed to do so (the rewind was gated on SeekableSharpCompressStream),
// leaving the reader misaligned so the *following* local header was parsed from garbage
// and extraction threw near the very end. A seekable stream rewinds correctly and works,
// which is exactly the "works seekable, fails non-seekable" symptom that was reported.
//
// NOTE: heavy (~4GB) like the other Zip64 large-file tests in this file, hence disabled
// by default. Enable to verify the fix.
//[Fact]
[Trait("format", "zip64")]
public async ValueTask Zip64_Large_File_Then_Small_File_NonSeekable_Async()
{
var filename = Path.Combine(SCRATCH2_FILES_PATH, "zip64-nonseekable-async.zip");

// A small trailing entry with recognizable content. Its bytes can only be read back
// correctly if the reader stays byte-aligned after the preceding >=4GB Zip64 entry.
var smallContent = new byte[64 * 1024];
for (var i = 0; i < smallContent.Length; i++)
{
smallContent[i] = (byte)(i % 251);
}

try
{
if (File.Exists(filename))
{
File.Delete(filename);
}

CreateLargeThenSmallZip(filename, FOUR_GB_LIMIT, smallContent);

var (count, lastKey, lastContent) = await ReadLargeThenSmallNonSeekableAsync(filename);

// The reader must reach the second (small) entry without throwing, identify it
// correctly, and read its bytes verbatim.
Assert.Equal(2, count);
Assert.Equal("small", lastKey);
Assert.NotNull(lastContent);
Assert.Equal(smallContent, lastContent!);
}
finally
{
if (File.Exists(filename))
{
File.Delete(filename);
}
}
}

private void CreateLargeThenSmallZip(string filename, long largeSize, byte[] smallContent)
{
var chunk = new byte[1024 * 1024];

// Force Zip64 and store (level 0) so the large entry's compressed size also exceeds
// 4GiB, which is what marks the entry as Zip64 for the streaming reader.
var opts = new ZipWriterOptions(CompressionType.Deflate) { UseZip64 = true };
var eo = new ZipWriterEntryOptions { CompressionLevel = 0 };

using var zip = File.OpenWrite(filename);
using var zipWriter = (ZipWriter)WriterFactory.OpenWriter(zip, ArchiveType.Zip, opts);

using (var str = zipWriter.WriteToStream("large", eo))
{
var left = largeSize;
while (left > 0)
{
var b = (int)Math.Min(left, chunk.Length);
str.Write(chunk, 0, b);
left -= b;
}
}

using (var str = zipWriter.WriteToStream("small", eo))
{
str.Write(smallContent, 0, smallContent.Length);
}
}

private async ValueTask<(
long Count,
string? LastKey,
byte[]? LastContent
)> ReadLargeThenSmallNonSeekableAsync(string filename)
{
long count = 0;
string? lastKey = null;
byte[]? lastContent = null;

using var fs = File.OpenRead(filename);
// ForwardOnlyStream reports CanSeek == false; AsyncOnlyStream forces async reads.
// Together they emulate a non-seekable, async-only source (e.g. a network download).
//
// IMPORTANT: use default ReaderOptions (LeaveStreamOpen == false), exactly as the
// reporting user did. With LeaveStreamOpen == true the Volume wraps the stream in a
// passthrough that Create() later unwraps into a SeekableSharpCompressStream, which
// happens to take the working seek-back path and hides the bug. The default keeps a
// plain ring-buffer SharpCompressStream, which is where the streaming reader fails.
await using var rd = await ReaderFactory.OpenAsyncReader(
new AsyncOnlyStream(new ForwardOnlyStream(fs)),
new ReaderOptions { LookForHeader = false }
);
while (await rd.MoveToNextEntryAsync())
{
count++;
lastKey = rd.Entry.Key;

#if LEGACY_DOTNET
using var entryStream = await rd.OpenEntryStreamAsync();
#else
await using var entryStream = await rd.OpenEntryStreamAsync();
#endif
if (rd.Entry.Key == "small")
{
using var ms = new MemoryStream();
await entryStream.CopyToAsync(ms);
lastContent = ms.ToArray();
}
else
{
await entryStream.SkipEntryAsync();
}
}

return (count, lastKey, lastContent);
}

public async ValueTask RunSingleTestAsync(
long files,
long filesize,
Expand Down