Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix: PUT partsize to 16MiB & COPY partsize to 512 MiB #531

Merged
merged 1 commit into from
Apr 21, 2021
Merged

fix: PUT partsize to 16MiB & COPY partsize to 512 MiB #531

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 23 additions & 8 deletions Minio.Tests/UtilsTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,7 @@ public void TestFileWithExtension()

[TestMethod]
[ExpectedException(typeof(EntityTooLargeException))]
public void TestInvalidPartSize()
public void TestInvalidPUTPartSize()
{
try
{
Expand All @@ -134,6 +134,21 @@ public void TestInvalidPartSize()
}
}

[TestMethod]
[ExpectedException(typeof(EntityTooLargeException))]
public void TestInvalidCOPYPartSize()
{
try
{
Object multiparts = utils.CalculateMultiPartSize(5000000000000000000, true);
}
catch (EntityTooLargeException ex)
{
Assert.AreEqual(ex.ServerMessage, "Your proposed upload size 5000000000000000000 exceeds the maximum allowed object size " + Constants.MaxMultipartPutObjectSize);
throw;
}
}

[TestMethod]
public void TestValidPartSize1()
{
Expand All @@ -142,21 +157,21 @@ public void TestValidPartSize1()
double partSize = partSizeObject.partSize;
double partCount = partSizeObject.partCount;
double lastPartSize = partSizeObject.lastPartSize;
Assert.AreEqual(partSize, 550502400);
Assert.AreEqual(partCount, 9987);
Assert.AreEqual(lastPartSize, 241172480);
Assert.AreEqual(partSize, 553648128);
Assert.AreEqual(partCount, 9930);
Assert.AreEqual(lastPartSize, 385875968);
}

[TestMethod]
public void TestValidPartSize2()
{
dynamic partSizeObject = utils.CalculateMultiPartSize(5000000000);
dynamic partSizeObject = utils.CalculateMultiPartSize(500000000000, true);
double partSize = partSizeObject.partSize;
double partCount = partSizeObject.partCount;
double lastPartSize = partSizeObject.lastPartSize;
Assert.AreEqual(partSize, 5242880);
Assert.AreEqual(partCount, 954);
Assert.AreEqual(lastPartSize, 3535360);
Assert.AreEqual(partSize, 536870912);
Assert.AreEqual(partCount, 932);
Assert.AreEqual(lastPartSize, 173180928);
}

[TestMethod]
Expand Down
4 changes: 2 additions & 2 deletions Minio/ApiEndpoints/ObjectOperations.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -745,7 +745,7 @@ public async Task<string> PresignedPutObjectAsync(PresignedPutObjectArgs args)
/// <param name="cancellationToken">Optional cancellation token to cancel the operation</param>
private async Task MultipartCopyUploadAsync(MultipartCopyUploadArgs args, CancellationToken cancellationToken = default(CancellationToken))
{
dynamic multiPartInfo = utils.CalculateMultiPartSize(args.CopySize);
dynamic multiPartInfo = utils.CalculateMultiPartSize(args.CopySize, copy:true);
double partSize = multiPartInfo.partSize;
double partCount = multiPartInfo.partCount;
double lastPartSize = multiPartInfo.lastPartSize;
Expand Down Expand Up @@ -1509,7 +1509,7 @@ private async Task<object> CopyObjectRequestAsync(string bucketName, string obje
// For all sizes greater than 5GB or if Copy byte range specified in conditions and byte range larger
// than minimum part size (5 MB) do multipart.

dynamic multiPartInfo = utils.CalculateMultiPartSize(copySize);
dynamic multiPartInfo = utils.CalculateMultiPartSize(copySize, copy:true);
double partSize = multiPartInfo.partSize;
double partCount = multiPartInfo.partCount;
double lastPartSize = multiPartInfo.lastPartSize;
Expand Down
10 changes: 10 additions & 0 deletions Minio/Helper/Constants.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,16 @@ internal static class Constants
/// </summary>
public static long MinimumPartSize = 5 * 1024L * 1024L;

/// <summary>
/// Minimum PUT part size
/// </summary>
public static long MinimumPUTPartSize = 16 * 1024L * 1024L;

/// <summary>
/// Minimum COPY part size
/// </summary>
public static long MinimumCOPYPartSize = 512 * 1024L * 1024L;

/// <summary>
/// Maximum part size
/// </summary>
Expand Down
6 changes: 4 additions & 2 deletions Minio/Helper/utils.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -235,8 +235,9 @@ public static bool CaseInsensitiveContains(string text, string value,
/// Calculate part size and number of parts required.
/// </summary>
/// <param name="size"></param>
/// <param name="copy"> If true, use COPY part size, else use PUT part size</param>
/// <returns></returns>
public static object CalculateMultiPartSize(long size)
public static object CalculateMultiPartSize(long size, bool copy = false)
{
if (size == -1)
{
Expand All @@ -249,7 +250,8 @@ public static object CalculateMultiPartSize(long size)
}

double partSize = (double)Math.Ceiling((decimal)size / Constants.MaxParts);
partSize = (double)Math.Ceiling((decimal)partSize / Constants.MinimumPartSize) * Constants.MinimumPartSize;
long minPartSize = copy ? Constants.MinimumCOPYPartSize: Constants.MinimumPUTPartSize;
partSize = (double)Math.Ceiling((decimal)partSize / minPartSize) * minPartSize;
double partCount = Math.Ceiling(size / partSize);
double lastPartSize = size - (partCount - 1) * partSize;
dynamic obj = new ExpandoObject();
Expand Down