Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Switch to SharpCompress #803

Merged
merged 14 commits into from
May 30, 2017
Binary file removed ext/ICSharpCode.SharpZipLib.dll
Binary file not shown.
9,612 changes: 0 additions & 9,612 deletions ext/ICSharpCode.SharpZipLib.xml

This file was deleted.

18 changes: 11 additions & 7 deletions src/Squirrel/BinaryPatchUtility.cs
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@
using System.Collections.Generic;
using System.IO;
using System.Threading;
using ICSharpCode.SharpZipLib.BZip2;
using SharpCompress.Compressors;
using SharpCompress.Compressors.BZip2;

// Adapted from https://github.com/LogosBible/bsdiff.net/blob/master/src/bsdiff/BinaryPatchUtility.cs

Expand Down Expand Up @@ -111,7 +112,8 @@ 0 32 Header
int dblen = 0;
int eblen = 0;

using (BZip2OutputStream bz2Stream = new BZip2OutputStream(new WrappingStream(output, Ownership.None)))
using (WrappingStream wrappingStream = new WrappingStream(output, Ownership.None))
using (var bz2Stream = new BZip2Stream(wrappingStream, CompressionMode.Compress))
{
// compute the differences, writing ctrl as we go
int scan = 0;
Expand Down Expand Up @@ -228,7 +230,8 @@ 0 32 Header
WriteInt64(controlEndPosition - startPosition - c_headerSize, header, 8);

// write compressed diff data
using (BZip2OutputStream bz2Stream = new BZip2OutputStream(new WrappingStream(output, Ownership.None)))
using (WrappingStream wrappingStream = new WrappingStream(output, Ownership.None))
using (var bz2Stream = new BZip2Stream(wrappingStream, CompressionMode.Compress))
{
bz2Stream.Write(db, 0, dblen);
}
Expand All @@ -238,7 +241,8 @@ 0 32 Header
WriteInt64(diffEndPosition - controlEndPosition, header, 16);

// write compressed extra data
using (BZip2OutputStream bz2Stream = new BZip2OutputStream(new WrappingStream(output, Ownership.None)))
using (WrappingStream wrappingStream = new WrappingStream(output, Ownership.None))
using (var bz2Stream = new BZip2Stream(wrappingStream, CompressionMode.Compress))
{
bz2Stream.Write(eb, 0, eblen);
}
Expand Down Expand Up @@ -323,9 +327,9 @@ with control block a set of triples (x,y,z) meaning "add x bytes
compressedExtraStream.Seek(c_headerSize + controlLength + diffLength, SeekOrigin.Current);

// decompress each part (to read it)
using (BZip2InputStream controlStream = new BZip2InputStream(compressedControlStream))
using (BZip2InputStream diffStream = new BZip2InputStream(compressedDiffStream))
using (BZip2InputStream extraStream = new BZip2InputStream(compressedExtraStream))
using (var controlStream = new BZip2Stream(compressedControlStream, CompressionMode.Decompress))
using (var diffStream = new BZip2Stream(compressedDiffStream, CompressionMode.Decompress))
using (var extraStream = new BZip2Stream(compressedExtraStream, CompressionMode.Decompress))
{
long[] control = new long[3];
byte[] buffer = new byte[8];
Expand Down
46 changes: 37 additions & 9 deletions src/Squirrel/DeltaPackage.cs
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,16 @@
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using ICSharpCode.SharpZipLib.Zip;
using Splat;
using DeltaCompressionDotNet.MsDelta;
using System.ComponentModel;
using Squirrel.Bsdiff;
using SharpCompress.Archives;
using SharpCompress.Archives.Zip;
using SharpCompress.Writers;
using SharpCompress.Common;
using SharpCompress.Readers;
using SharpCompress.Compressors.Deflate;

namespace Squirrel
{
Expand Down Expand Up @@ -63,9 +68,17 @@ public ReleasePackage CreateDeltaPackage(ReleasePackage basePackage, ReleasePack
this.Log().Info("Extracting {0} and {1} into {2}",
basePackage.ReleasePackageFile, newPackage.ReleasePackageFile, tempPath);

var fz = new FastZip();
fz.ExtractZip(basePackage.ReleasePackageFile, baseTempInfo.FullName, null);
fz.ExtractZip(newPackage.ReleasePackageFile, tempInfo.FullName, null);
var opts = new ExtractionOptions() { ExtractFullPath = true, Overwrite = true, PreserveFileTime = true };

using (var za = ZipArchive.Open(basePackage.ReleasePackageFile))
using (var reader = za.ExtractAllEntries()) {
reader.WriteAllToDirectory(baseTempInfo.FullName, opts);
}

using (var za = ZipArchive.Open(basePackage.ReleasePackageFile))
using (var reader = za.ExtractAllEntries()) {
reader.WriteAllToDirectory(tempInfo.FullName, opts);
}

// Collect a list of relative paths under 'lib' and map them
// to their full name. We'll use this later to determine in
Expand All @@ -82,7 +95,10 @@ public ReleasePackage CreateDeltaPackage(ReleasePackage basePackage, ReleasePack
}

ReleasePackage.addDeltaFilesToContentTypes(tempInfo.FullName);
fz.CreateZip(outputFile, tempInfo.FullName, true, null);
using (var za = ZipArchive.Create()) {
za.AddAllFromDirectory(tempInfo.FullName);
za.SaveTo(outputFile, CompressionType.Deflate);
}
}

return new ReleasePackage(outputFile);
Expand All @@ -98,9 +114,16 @@ public ReleasePackage ApplyDeltaPackage(ReleasePackage basePackage, ReleasePacka

using (Utility.WithTempDirectory(out deltaPath, localAppDirectory))
using (Utility.WithTempDirectory(out workingPath, localAppDirectory)) {
var fz = new FastZip();
fz.ExtractZip(deltaPackage.InputPackageFile, deltaPath, null);
fz.ExtractZip(basePackage.InputPackageFile, workingPath, null);
var opts = new ExtractionOptions() { ExtractFullPath = true, Overwrite = true, PreserveFileTime = true };

using (var za = ZipArchive.Open(deltaPackage.InputPackageFile))
using (var reader = za.ExtractAllEntries()) {
reader.WriteAllToDirectory(deltaPath, opts);
}
using (var za = ZipArchive.Open(basePackage.InputPackageFile))
using (var reader = za.ExtractAllEntries()) {
reader.WriteAllToDirectory(workingPath, opts);
}

var pathsVisited = new List<string>();

Expand Down Expand Up @@ -139,7 +162,12 @@ public ReleasePackage ApplyDeltaPackage(ReleasePackage basePackage, ReleasePacka
});

this.Log().Info("Repacking into full package: {0}", outputFile);
fz.CreateZip(outputFile, workingPath, true, null);
using (var za = ZipArchive.Create())
using (var tgt = File.OpenWrite(outputFile)) {
za.DeflateCompressionLevel = CompressionLevel.BestSpeed;
za.AddAllFromDirectory(workingPath);
za.SaveTo(tgt);
}
}

return new ReleasePackage(outputFile);
Expand Down
178 changes: 66 additions & 112 deletions src/Squirrel/ReleasePackage.cs
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,16 @@
using System.Text;
using System.Text.RegularExpressions;
using System.Xml;
using ICSharpCode.SharpZipLib.Zip;
using MarkdownSharp;
using NuGet;
using Splat;
using ICSharpCode.SharpZipLib.Core;
using System.Threading.Tasks;
using SharpCompress.Archives.Zip;
using SharpCompress.Readers;
using SharpCompress.Archives;
using SharpCompress.Common;
using SharpCompress.Compressors.Deflate;
using SharpCompress.Writers;

namespace Squirrel
{
Expand Down Expand Up @@ -64,8 +68,6 @@ public static bool Matches(IVersionSpec versionSpec, SemanticVersion version)

public class ReleasePackage : IEnableLogger, IReleasePackage
{
IEnumerable<IPackage> localPackageCache;

public ReleasePackage(string inputPackageFile, bool isReleasePackage = false)
{
InputPackageFile = inputPackageFile;
Expand Down Expand Up @@ -99,7 +101,10 @@ public string CreateReleasePackage(string outputFile, string packagesRootDir = n
var package = new ZipPackage(InputPackageFile);

var dontcare = default(SemanticVersion);
if (!SemanticVersion.TryParseStrict(package.Version.ToString(), out dontcare)) {

// NB: Our test fixtures use packages that aren't SemVer compliant,
// we don't really care that they aren't valid
if (!ModeDetector.InUnitTestRunner() && !SemanticVersion.TryParseStrict(package.Version.ToString(), out dontcare)) {
throw new Exception(
String.Format(
"Your package version is currently {0}, which is *not* SemVer-compatible, change this to be a SemVer version number",
Expand Down Expand Up @@ -130,15 +135,16 @@ public string CreateReleasePackage(string outputFile, string packagesRootDir = n
var dependencies = findAllDependentPackages(
package,
new LocalPackageRepository(packagesRootDir),
frameworkName: targetFramework);
frameworkName: targetFramework)
.ToArray();

string tempPath = null;

using (Utility.WithTempDirectory(out tempPath, null)) {
var tempDir = new DirectoryInfo(tempPath);

ExtractZipDecoded(InputPackageFile, tempPath);

this.Log().Info("Extracting dependent packages: [{0}]", String.Join(",", dependencies.Select(x => x.Id)));
extractDependentPackages(dependencies, tempDir, targetFramework);

Expand Down Expand Up @@ -169,133 +175,81 @@ public string CreateReleasePackage(string outputFile, string packagesRootDir = n
// We must do this, or PathTooLongException may be thrown for some unicode entry names.
public static void ExtractZipDecoded(string zipFilePath, string outFolder)
{
var zf = new ZipFile(zipFilePath);

var buffer = new byte[64*1024];

try {
foreach (ZipEntry zipEntry in zf) {
if (!zipEntry.IsFile) continue;

var entryFileName = Uri.UnescapeDataString(zipEntry.Name);

var fullZipToPath = Path.Combine(outFolder, entryFileName);
var directoryName = Path.GetDirectoryName(fullZipToPath);

if (directoryName.Length > 0) {
Directory.CreateDirectory(directoryName);
}

var zipStream = zf.GetInputStream(zipEntry);

using (FileStream streamWriter = File.Create(fullZipToPath)) {
StreamUtils.Copy(zipStream, streamWriter, buffer);
using (var za = ZipArchive.Open(zipFilePath))
using (var reader = za.ExtractAllEntries()) {
while (reader.MoveToNextEntry()) {
var parts = reader.Entry.Key.Split('\\', '/').Select(x => Uri.UnescapeDataString(x));
var decoded = String.Join(Path.DirectorySeparatorChar.ToString(), parts);

var fullTargetDir = Path.Combine(outFolder, Path.GetDirectoryName(decoded));
Directory.CreateDirectory(fullTargetDir);

if (reader.Entry.IsDirectory) {
Directory.CreateDirectory(Path.Combine(outFolder, decoded));
} else {
reader.WriteEntryToFile(Path.Combine(outFolder, decoded));
}
}
} finally {
zf.Close();
}
}

public static async Task ExtractZipForInstall(string zipFilePath, string outFolder)
public static Task ExtractZipForInstall(string zipFilePath, string outFolder)
{
var zf = new ZipFile(zipFilePath);
var entries = zf.OfType<ZipEntry>().ToArray();
var re = new Regex(@"lib[\\\/][^\\\/]*[\\\/]", RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);

try {
await Utility.ForEachAsync(entries, (zipEntry) => {
if (!zipEntry.IsFile) return;

var entryFileName = Uri.UnescapeDataString(zipEntry.Name);
if (!re.Match(entryFileName).Success) return;
return Task.Run(() => {
using (var za = ZipArchive.Open(zipFilePath))
using (var reader = za.ExtractAllEntries()) {
while (reader.MoveToNextEntry()) {
var parts = reader.Entry.Key.Split('\\', '/').Select(x => Uri.UnescapeDataString(x));
var decoded = String.Join(Path.DirectorySeparatorChar.ToString(), parts);

var fullZipToPath = Path.Combine(outFolder, re.Replace(entryFileName, "", 1));
if (!re.IsMatch(decoded)) continue;
decoded = re.Replace(decoded, "", 1);

var failureIsOkay = false;
if (entryFileName.Contains("_ExecutionStub.exe")) {
// NB: On upgrade, many of these stubs will be in-use, nbd tho.
failureIsOkay = true;
var fullTargetFile = Path.Combine(outFolder, decoded);
var fullTargetDir = Path.GetDirectoryName(fullTargetFile);
Directory.CreateDirectory(fullTargetDir);

fullZipToPath = Path.Combine(
Directory.GetParent(Path.GetDirectoryName(fullZipToPath)).FullName,
Path.GetFileName(fullZipToPath).Replace("_ExecutionStub.exe", ".exe"));

LogHost.Default.Info("Rigging execution stub for {0} to {1}", entryFileName, fullZipToPath);
}
var failureIsOkay = false;
if (!reader.Entry.IsDirectory && decoded.Contains("_ExecutionStub.exe")) {
// NB: On upgrade, many of these stubs will be in-use, nbd tho.
failureIsOkay = true;

var directoryName = Path.GetDirectoryName(fullZipToPath);
fullTargetFile = Path.Combine(
fullTargetDir,
Path.GetFileName(decoded).Replace("_ExecutionStub.exe", ".exe"));

var buffer = new byte[64*1024];

if (directoryName.Length > 0) {
Utility.Retry(() => Directory.CreateDirectory(directoryName), 2);
}
LogHost.Default.Info("Rigging execution stub for {0} to {1}", decoded, fullTargetFile);
}

try {
Utility.Retry(() => {
using (var zipStream = zf.GetInputStream(zipEntry))
using (FileStream streamWriter = File.Create(fullZipToPath)) {
StreamUtils.Copy(zipStream, streamWriter, buffer);
}
}, 5);
} catch (Exception e) {
if (!failureIsOkay) {
try {
Utility.Retry(() => {
if (reader.Entry.IsDirectory) {
Directory.CreateDirectory(Path.Combine(outFolder, decoded));
} else {
reader.WriteEntryToFile(Path.Combine(outFolder, decoded));
}
}, 5);
} catch (Exception e) {
if (!failureIsOkay) throw;
LogHost.Default.WarnException("Can't write execution stub, probably in use", e);
throw e;
}
}
}, 4);
} finally {
zf.Close();
}
}
});
}

// Create zip file with entry names %-encoded, as nupkg file does.
void createZipEncoded(string zipFilePath, string folder)
{
folder = Path.GetFullPath(folder);
var offset = folder.Length + (folder.EndsWith("\\", StringComparison.OrdinalIgnoreCase) ? 1 : 0);

var fsOut = File.Create(zipFilePath);
var zipStream = new ZipOutputStream(fsOut);

zipStream.SetLevel(9);

compressFolderEncoded(folder, zipStream, offset);

zipStream.IsStreamOwner = true;
zipStream.Close();
}

void compressFolderEncoded(string path, ZipOutputStream zipStream, int folderOffset)
{
string[] files = Directory.GetFiles(path);

var buffer = new byte[64 * 1024];
foreach (string filename in files) {
var fi = new FileInfo(filename);

string entryName = filename.Substring(folderOffset);
entryName = ZipEntry.CleanName(entryName);
entryName = Uri.EscapeUriString(entryName);

var newEntry = new ZipEntry(entryName);
newEntry.DateTime = fi.LastWriteTime;
newEntry.Size = fi.Length;
zipStream.PutNextEntry(newEntry);

using (FileStream streamReader = File.OpenRead(filename)) {
StreamUtils.Copy(streamReader, zipStream, buffer);
}

zipStream.CloseEntry();
}

string[] folders = Directory.GetDirectories(path);

foreach (string folder in folders) {
compressFolderEncoded(folder, zipStream, folderOffset);
using (var archive = ZipArchive.Create())
using (var tgt = File.OpenWrite(zipFilePath)) {
archive.DeflateCompressionLevel = CompressionLevel.BestCompression;
archive.AddAllFromDirectory(folder);
archive.SaveTo(
tgt,
new WriterOptions(CompressionType.Deflate));
}
}

Expand Down
Loading