Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Improve put object performance by sending sha256sum always. #1333

Merged
merged 1 commit into from
Jun 12, 2022
Merged

Improve put object performance by sending sha256sum always. #1333

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 15 additions & 25 deletions api/src/main/java/io/minio/PartReader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@
import java.io.RandomAccessFile;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.Base64;
import java.util.Locale;
import java.util.Objects;
import javax.annotation.Nonnull;
Expand Down Expand Up @@ -72,15 +71,13 @@ public PartReader(@Nonnull InputStream stream, long objectSize, long partSize, i
for (int i = 0; i < this.buffers.length; i++) this.buffers[i] = new ByteBufferStream();
}

private long readStreamChunk(
ByteBufferStream buffer, long size, MessageDigest md5, MessageDigest sha256)
private long readStreamChunk(ByteBufferStream buffer, long size, MessageDigest sha256)
throws IOException {
long totalBytesRead = 0;

if (this.oneByte != null) {
buffer.write(this.oneByte);
md5.update(this.oneByte);
if (sha256 != null) sha256.update(this.oneByte);
sha256.update(this.oneByte);
totalBytesRead++;
this.oneByte = null;
}
Expand All @@ -95,15 +92,14 @@ private long readStreamChunk(
throw new IOException("unexpected EOF");
}
buffer.write(this.buf16k, 0, bytesRead);
md5.update(this.buf16k, 0, bytesRead);
if (sha256 != null) sha256.update(this.buf16k, 0, bytesRead);
sha256.update(this.buf16k, 0, bytesRead);
totalBytesRead += bytesRead;
}

return totalBytesRead;
}

private long readStream(long size, MessageDigest md5, MessageDigest sha256) throws IOException {
private long readStream(long size, MessageDigest sha256) throws IOException {
long count = size / CHUNK_SIZE;
long lastChunkSize = size - (count * CHUNK_SIZE);
if (lastChunkSize > 0) {
Expand All @@ -116,7 +112,7 @@ private long readStream(long size, MessageDigest md5, MessageDigest sha256) thro
for (int i = 0; i < buffers.length; i++) buffers[i].reset();
for (long i = 1; i <= count && !this.eof; i++) {
long chunkSize = (i != count) ? CHUNK_SIZE : lastChunkSize;
long bytesRead = this.readStreamChunk(buffers[(int) (i - 1)], chunkSize, md5, sha256);
long bytesRead = this.readStreamChunk(buffers[(int) (i - 1)], chunkSize, sha256);
totalBytesRead += bytesRead;
}

Expand All @@ -128,7 +124,7 @@ private long readStream(long size, MessageDigest md5, MessageDigest sha256) thro
return totalBytesRead;
}

private long readFile(long size, MessageDigest md5, MessageDigest sha256) throws IOException {
private long readFile(long size, MessageDigest sha256) throws IOException {
long position = this.file.getFilePointer();
long totalBytesRead = 0;

Expand All @@ -137,44 +133,38 @@ private long readFile(long size, MessageDigest md5, MessageDigest sha256) throws
if (bytesToRead > this.buf16k.length) bytesToRead = this.buf16k.length;
int bytesRead = this.file.read(this.buf16k, 0, (int) bytesToRead);
if (bytesRead < 0) throw new IOException("unexpected EOF");
md5.update(this.buf16k, 0, bytesRead);
if (sha256 != null) sha256.update(this.buf16k, 0, bytesRead);
sha256.update(this.buf16k, 0, bytesRead);
totalBytesRead += bytesRead;
}

this.file.seek(position);
return totalBytesRead;
}

private long read(long size, MessageDigest md5, MessageDigest sha256) throws IOException {
return (this.file != null) ? readFile(size, md5, sha256) : readStream(size, md5, sha256);
private long read(long size, MessageDigest sha256) throws IOException {
return (this.file != null) ? readFile(size, sha256) : readStream(size, sha256);
}

public PartSource getPart(boolean computeSha256) throws NoSuchAlgorithmException, IOException {
public PartSource getPart() throws NoSuchAlgorithmException, IOException {
if (this.partNumber == this.partCount) return null;

this.partNumber++;

MessageDigest md5 = MessageDigest.getInstance("MD5");
MessageDigest sha256 = computeSha256 ? MessageDigest.getInstance("SHA-256") : null;
MessageDigest sha256 = MessageDigest.getInstance("SHA-256");

long partSize = this.partSize;
if (this.partNumber == this.partCount) partSize = this.objectSize - this.totalDataRead;
long bytesRead = this.read(partSize, md5, sha256);
long bytesRead = this.read(partSize, sha256);
this.totalDataRead += bytesRead;
if (this.objectSize < 0 && this.eof) this.partCount = this.partNumber;

String md5Hash = Base64.getEncoder().encodeToString(md5.digest());
String sha256Hash = null;
if (computeSha256) {
sha256Hash = BaseEncoding.base16().encode(sha256.digest()).toLowerCase(Locale.US);
}
String sha256Hash = BaseEncoding.base16().encode(sha256.digest()).toLowerCase(Locale.US);

if (this.file != null) {
return new PartSource(this.partNumber, this.file, bytesRead, md5Hash, sha256Hash);
return new PartSource(this.partNumber, this.file, bytesRead, null, sha256Hash);
}

return new PartSource(this.partNumber, this.buffers, bytesRead, md5Hash, sha256Hash);
return new PartSource(this.partNumber, this.buffers, bytesRead, null, sha256Hash);
}

public int partCount() {
Expand Down
22 changes: 8 additions & 14 deletions api/src/main/java/io/minio/S3Base.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -391,11 +391,7 @@ protected Request createRequest(

String md5Hash = Digest.ZERO_MD5_HASH;
if (body != null) {
if (body instanceof PartSource) {
md5Hash = ((PartSource) body).md5Hash();
} else if (body instanceof byte[]) {
md5Hash = Digest.md5Hash((byte[]) body, length);
}
md5Hash = (body instanceof byte[]) ? Digest.md5Hash((byte[]) body, length) : null;
}

String sha256Hash = null;
Expand All @@ -412,6 +408,9 @@ protected Request createRequest(
} else {
// Fix issue #415: No need to compute sha256 if endpoint scheme is HTTPS.
sha256Hash = "UNSIGNED-PAYLOAD";
if (body != null && body instanceof PartSource) {
sha256Hash = ((PartSource) body).sha256Hash();
}
}
}

Expand Down Expand Up @@ -2690,7 +2689,7 @@ private Part[] uploadParts(
.get();
parts[partNumber - 1] = new Part(partNumber, response.etag());

partSource = partReader.getPart(!this.baseUrl.isHttps());
partSource = partReader.getPart();
if (partSource == null) break;
}

Expand Down Expand Up @@ -2806,7 +2805,7 @@ protected CompletableFuture<ObjectWriteResponse> putObjectAsync(
return CompletableFuture.supplyAsync(
() -> {
try {
return partReader.getPart(!this.baseUrl.isHttps());
return partReader.getPart();
} catch (NoSuchAlgorithmException | IOException e) {
throw new CompletionException(e);
}
Expand Down Expand Up @@ -2941,12 +2940,7 @@ protected CompletableFuture<ObjectWriteResponse> putObjectAsync(

if (partReader != null) {
return putObjectAsync(
bucketName,
region,
objectName,
partReader.getPart(!this.baseUrl.isHttps()),
headers,
extraQueryParams);
bucketName, region, objectName, partReader.getPart(), headers, extraQueryParams);
}

return getRegionAsync(bucketName, region)
Expand Down Expand Up @@ -3441,7 +3435,7 @@ protected CompletableFuture<UploadPartResponse> uploadPartAsync(
bucketName,
region,
objectName,
partReader.getPart(!this.baseUrl.isHttps()),
partReader.getPart(),
partNumber,
uploadId,
extraHeaders,
Expand Down