Skip to content

Commit

Permalink
Improve put object performance by sending sha256sum always. (#1333)
Browse files Browse the repository at this point in the history 
Previously, `UNSIGNED-PAYLOAD` is sent sha256sum and computed md5sum
of body for HTTPS and computed sha256sum/md5sum of body for HTTP. This
made the API slower. Now computed sha256sum is always sent for both
HTTP and HTTPS.

Fixes #1332

Signed-off-by: Bala.FA <[email protected]>
  • Loading branch information
@balamurugana
balamurugana authored Jun 12, 2022
1 parent 6ecc05f commit 48b6570
Show file tree
Hide file tree
Showing 2 changed files with 23 additions and 39 deletions.
40 changes: 15 additions & 25 deletions api/src/main/java/io/minio/PartReader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@
import java.io.RandomAccessFile;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.Base64;
import java.util.Locale;
import java.util.Objects;
import javax.annotation.Nonnull;
Expand Down Expand Up @@ -72,15 +71,13 @@ public PartReader(@Nonnull InputStream stream, long objectSize, long partSize, i
for (int i = 0; i < this.buffers.length; i++) this.buffers[i] = new ByteBufferStream();
}

private long readStreamChunk(
ByteBufferStream buffer, long size, MessageDigest md5, MessageDigest sha256)
private long readStreamChunk(ByteBufferStream buffer, long size, MessageDigest sha256)
throws IOException {
long totalBytesRead = 0;

if (this.oneByte != null) {
buffer.write(this.oneByte);
md5.update(this.oneByte);
if (sha256 != null) sha256.update(this.oneByte);
sha256.update(this.oneByte);
totalBytesRead++;
this.oneByte = null;
}
Expand All @@ -95,15 +92,14 @@ private long readStreamChunk(
throw new IOException("unexpected EOF");
}
buffer.write(this.buf16k, 0, bytesRead);
md5.update(this.buf16k, 0, bytesRead);
if (sha256 != null) sha256.update(this.buf16k, 0, bytesRead);
sha256.update(this.buf16k, 0, bytesRead);
totalBytesRead += bytesRead;
}

return totalBytesRead;
}

private long readStream(long size, MessageDigest md5, MessageDigest sha256) throws IOException {
private long readStream(long size, MessageDigest sha256) throws IOException {
long count = size / CHUNK_SIZE;
long lastChunkSize = size - (count * CHUNK_SIZE);
if (lastChunkSize > 0) {
Expand All @@ -116,7 +112,7 @@ private long readStream(long size, MessageDigest md5, MessageDigest sha256) thro
for (int i = 0; i < buffers.length; i++) buffers[i].reset();
for (long i = 1; i <= count && !this.eof; i++) {
long chunkSize = (i != count) ? CHUNK_SIZE : lastChunkSize;
long bytesRead = this.readStreamChunk(buffers[(int) (i - 1)], chunkSize, md5, sha256);
long bytesRead = this.readStreamChunk(buffers[(int) (i - 1)], chunkSize, sha256);
totalBytesRead += bytesRead;
}

Expand All @@ -128,7 +124,7 @@ private long readStream(long size, MessageDigest md5, MessageDigest sha256) thro
return totalBytesRead;
}

private long readFile(long size, MessageDigest md5, MessageDigest sha256) throws IOException {
private long readFile(long size, MessageDigest sha256) throws IOException {
long position = this.file.getFilePointer();
long totalBytesRead = 0;

Expand All @@ -137,44 +133,38 @@ private long readFile(long size, MessageDigest md5, MessageDigest sha256) throws
if (bytesToRead > this.buf16k.length) bytesToRead = this.buf16k.length;
int bytesRead = this.file.read(this.buf16k, 0, (int) bytesToRead);
if (bytesRead < 0) throw new IOException("unexpected EOF");
md5.update(this.buf16k, 0, bytesRead);
if (sha256 != null) sha256.update(this.buf16k, 0, bytesRead);
sha256.update(this.buf16k, 0, bytesRead);
totalBytesRead += bytesRead;
}

this.file.seek(position);
return totalBytesRead;
}

private long read(long size, MessageDigest md5, MessageDigest sha256) throws IOException {
return (this.file != null) ? readFile(size, md5, sha256) : readStream(size, md5, sha256);
private long read(long size, MessageDigest sha256) throws IOException {
return (this.file != null) ? readFile(size, sha256) : readStream(size, sha256);
}

public PartSource getPart(boolean computeSha256) throws NoSuchAlgorithmException, IOException {
public PartSource getPart() throws NoSuchAlgorithmException, IOException {
if (this.partNumber == this.partCount) return null;

this.partNumber++;

MessageDigest md5 = MessageDigest.getInstance("MD5");
MessageDigest sha256 = computeSha256 ? MessageDigest.getInstance("SHA-256") : null;
MessageDigest sha256 = MessageDigest.getInstance("SHA-256");

long partSize = this.partSize;
if (this.partNumber == this.partCount) partSize = this.objectSize - this.totalDataRead;
long bytesRead = this.read(partSize, md5, sha256);
long bytesRead = this.read(partSize, sha256);
this.totalDataRead += bytesRead;
if (this.objectSize < 0 && this.eof) this.partCount = this.partNumber;

String md5Hash = Base64.getEncoder().encodeToString(md5.digest());
String sha256Hash = null;
if (computeSha256) {
sha256Hash = BaseEncoding.base16().encode(sha256.digest()).toLowerCase(Locale.US);
}
String sha256Hash = BaseEncoding.base16().encode(sha256.digest()).toLowerCase(Locale.US);

if (this.file != null) {
return new PartSource(this.partNumber, this.file, bytesRead, md5Hash, sha256Hash);
return new PartSource(this.partNumber, this.file, bytesRead, null, sha256Hash);
}

return new PartSource(this.partNumber, this.buffers, bytesRead, md5Hash, sha256Hash);
return new PartSource(this.partNumber, this.buffers, bytesRead, null, sha256Hash);
}

public int partCount() {
Expand Down
22 changes: 8 additions & 14 deletions api/src/main/java/io/minio/S3Base.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -397,11 +397,7 @@ protected Request createRequest(

String md5Hash = Digest.ZERO_MD5_HASH;
if (body != null) {
if (body instanceof PartSource) {
md5Hash = ((PartSource) body).md5Hash();
} else if (body instanceof byte[]) {
md5Hash = Digest.md5Hash((byte[]) body, length);
}
md5Hash = (body instanceof byte[]) ? Digest.md5Hash((byte[]) body, length) : null;
}

String sha256Hash = null;
Expand All @@ -418,6 +414,9 @@ protected Request createRequest(
} else {
// Fix issue #415: No need to compute sha256 if endpoint scheme is HTTPS.
sha256Hash = "UNSIGNED-PAYLOAD";
if (body != null && body instanceof PartSource) {
sha256Hash = ((PartSource) body).sha256Hash();
}
}
}

Expand Down Expand Up @@ -2696,7 +2695,7 @@ private Part[] uploadParts(
.get();
parts[partNumber - 1] = new Part(partNumber, response.etag());

partSource = partReader.getPart(!this.baseUrl.isHttps());
partSource = partReader.getPart();
if (partSource == null) break;
}

Expand Down Expand Up @@ -2812,7 +2811,7 @@ protected CompletableFuture<ObjectWriteResponse> putObjectAsync(
return CompletableFuture.supplyAsync(
() -> {
try {
return partReader.getPart(!this.baseUrl.isHttps());
return partReader.getPart();
} catch (NoSuchAlgorithmException | IOException e) {
throw new CompletionException(e);
}
Expand Down Expand Up @@ -2947,12 +2946,7 @@ protected CompletableFuture<ObjectWriteResponse> putObjectAsync(

if (partReader != null) {
return putObjectAsync(
bucketName,
region,
objectName,
partReader.getPart(!this.baseUrl.isHttps()),
headers,
extraQueryParams);
bucketName, region, objectName, partReader.getPart(), headers, extraQueryParams);
}

return getRegionAsync(bucketName, region)
Expand Down Expand Up @@ -3447,7 +3441,7 @@ protected CompletableFuture<UploadPartResponse> uploadPartAsync(
bucketName,
region,
objectName,
partReader.getPart(!this.baseUrl.isHttps()),
partReader.getPart(),
partNumber,
uploadId,
extraHeaders,
Expand Down

0 comments on commit 48b6570

Please sign in to comment.