Skip to content

AES GCM Stream changes #9453

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
rdblue merged 2 commits into from
Jan 14, 2024
Merged

AES GCM Stream changes #9453

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d276173
initial commit
ggershinsky Jan 10, 2024
c4edf8e
init final position
ggershinsky Jan 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion core/src/main/java/org/apache/iceberg/encryption/AesGcmInputStream.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ public int read(byte[] b, int off, int len) throws IOException {
}

if (available() <= 0 && len > 0) {
throw new EOFException();
return -1;
Copy link
Contributor Author

@ggershinsky ggershinsky Jan 10, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

https://docs.oracle.com/javase/8/docs/api/java/io/InputStream.html#read-byte:A-int-int-

}

if (len == 0) {
Expand Down
15 changes: 15 additions & 0 deletions core/src/main/java/org/apache/iceberg/encryption/AesGcmOutputStream.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,8 @@ public class AesGcmOutputStream extends PositionOutputStream {
private int currentBlockIndex;
private boolean isHeaderWritten;
private boolean lastBlockWritten;
private boolean isClosed;
private long finalPosition;

AesGcmOutputStream(PositionOutputStream targetStream, byte[] aesKey, byte[] fileAadPrefix) {
this.targetStream = targetStream;
Expand All @@ -56,6 +58,8 @@ public class AesGcmOutputStream extends PositionOutputStream {
this.currentBlockIndex = 0;
this.isHeaderWritten = false;
this.lastBlockWritten = false;
this.isClosed = false;
Copy link
Contributor

@rdblue rdblue Jan 12, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nit: initialize finalPosition as well?

Copy link
Contributor Author

@ggershinsky ggershinsky Jan 12, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done.

this.finalPosition = 0;
}

@Override
Expand All @@ -66,6 +70,10 @@ public void write(int b) throws IOException {

@Override
public void write(byte[] b, int off, int len) throws IOException {
if (isClosed) {
throw new IOException("Writing to closed stream");
}

if (!isHeaderWritten) {
writeHeader();
}
Expand Down Expand Up @@ -95,6 +103,10 @@ public void write(byte[] b, int off, int len) throws IOException {

@Override
public long getPos() throws IOException {
Copy link
Contributor Author

@ggershinsky ggershinsky Jan 10, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Iceberg manifest writer calls getPos after closing the avro stream. Wrong value was returned.

if (isClosed) {
return finalPosition;
}

return (long) currentBlockIndex * Ciphers.PLAIN_BLOCK_SIZE + positionInPlainBlock;
}

Expand All @@ -109,6 +121,9 @@ public void close() throws IOException {
writeHeader();
}

finalPosition = getPos();
isClosed = true;

encryptAndWriteBlock();

targetStream.close();
Expand Down
6 changes: 3 additions & 3 deletions core/src/test/java/org/apache/iceberg/encryption/TestGcmStreams.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
*/
package org.apache.iceberg.encryption;

import java.io.EOFException;
import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
Expand Down Expand Up @@ -60,8 +59,7 @@ public void testEmptyFile() throws IOException {
Assert.assertEquals("File size", 0, decryptedFile.getLength());

try (SeekableInputStream decryptedStream = decryptedFile.newStream()) {
Assertions.assertThatThrownBy(() -> decryptedStream.read(readBytes))
.isInstanceOf(EOFException.class);
Assert.assertEquals("Read empty stream", -1, decryptedStream.read(readBytes));
}

// check that the AAD is still verified, even for an empty file
Expand Down Expand Up @@ -284,6 +282,7 @@ public void testRandomWriteRead() throws IOException {
}

encryptedStream.close();
Assert.assertEquals("Final position in closed stream", offset, encryptedStream.getPos());

AesGcmInputFile decryptedFile =
new AesGcmInputFile(Files.localInput(testFile), key, aadPrefix);
Expand Down Expand Up @@ -380,6 +379,7 @@ public void testAlignedWriteRead() throws IOException {
}

encryptedStream.close();
Assert.assertEquals("Final position in closed stream", offset, encryptedStream.getPos());

AesGcmInputFile decryptedFile =
new AesGcmInputFile(Files.localInput(testFile), key, aadPrefix);
Expand Down