Skip to content

Commit 8262e35

Browse files
committed
Optimize PagesSerde
Refactor PagesSerde to avoid: - Unnecessary memory copy - Unnecessary Cipher initialization - Unnecessary allocations when jumbo pages (>4MB) are serialized This is done by implementing block based encryption in compression. Instead of trying to encrypt/compress an entire page a page in serialized form is split into multiple fixed size blocks (64kb by default) Benchmark results Before: ``` Benchmark (compressed) (encrypted) (randomSeed) Mode Cnt Score Error Units BenchmarkPagesSerde.deserialize true true 1000 thrpt 10 1194.138 ± 7.442 ops/s BenchmarkPagesSerde.deserialize true false 1000 thrpt 10 2281.985 ± 8.634 ops/s BenchmarkPagesSerde.deserialize false true 1000 thrpt 10 1416.509 ± 2.928 ops/s BenchmarkPagesSerde.deserialize false false 1000 thrpt 10 5891.232 ± 14.932 ops/s BenchmarkPagesSerde.serialize true true 1000 thrpt 10 312.647 ± 0.800 ops/s BenchmarkPagesSerde.serialize true false 1000 thrpt 10 601.053 ± 1.522 ops/s BenchmarkPagesSerde.serialize false true 1000 thrpt 10 451.260 ± 0.996 ops/s BenchmarkPagesSerde.serialize false false 1000 thrpt 10 3446.756 ± 6.949 ops/s ``` After: ``` Benchmark (compressed) (encrypted) (randomSeed) Mode Cnt Score Error Units BenchmarkPagesSerde.deserialize true true 1000 thrpt 10 1732.440 ± 4.044 ops/s BenchmarkPagesSerde.deserialize true false 1000 thrpt 10 3059.334 ± 5.412 ops/s BenchmarkPagesSerde.deserialize false true 1000 thrpt 10 2093.146 ± 3.505 ops/s BenchmarkPagesSerde.deserialize false false 1000 thrpt 10 5906.148 ± 13.923 ops/s BenchmarkPagesSerde.serialize true true 1000 thrpt 10 329.150 ± 1.269 ops/s BenchmarkPagesSerde.serialize true false 1000 thrpt 10 617.507 ± 1.808 ops/s BenchmarkPagesSerde.serialize false true 1000 thrpt 10 493.182 ± 0.687 ops/s BenchmarkPagesSerde.serialize false false 1000 thrpt 10 3816.939 ± 7.984 ops/s ```
1 parent 3e31aaa commit 8262e35

26 files changed

+1291
-755
lines changed

core/trino-main/src/main/java/io/trino/execution/QueryStateMachine.java

Lines changed: 3 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@
2323
import com.google.common.util.concurrent.Futures;
2424
import com.google.common.util.concurrent.ListenableFuture;
2525
import io.airlift.log.Logger;
26-
import io.airlift.slice.Slices;
2726
import io.airlift.units.Duration;
2827
import io.trino.Session;
2928
import io.trino.exchange.ExchangeInput;
@@ -57,10 +56,8 @@
5756
import javax.annotation.Nullable;
5857
import javax.annotation.concurrent.GuardedBy;
5958
import javax.annotation.concurrent.ThreadSafe;
60-
import javax.crypto.KeyGenerator;
6159

6260
import java.net.URI;
63-
import java.security.NoSuchAlgorithmException;
6461
import java.util.ArrayList;
6562
import java.util.HashMap;
6663
import java.util.HashSet;
@@ -97,9 +94,10 @@
9794
import static io.trino.execution.StageInfo.getAllStages;
9895
import static io.trino.operator.RetryPolicy.TASK;
9996
import static io.trino.server.DynamicFilterService.DynamicFiltersStats;
100-
import static io.trino.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR;
10197
import static io.trino.spi.StandardErrorCode.NOT_FOUND;
10298
import static io.trino.spi.StandardErrorCode.USER_CANCELED;
99+
import static io.trino.util.Ciphers.createRandomAesEncryptionKey;
100+
import static io.trino.util.Ciphers.serializeAesEncryptionKey;
103101
import static io.trino.util.Failures.toFailure;
104102
import static java.util.Objects.requireNonNull;
105103
import static java.util.concurrent.TimeUnit.MILLISECONDS;
@@ -280,14 +278,7 @@ static QueryStateMachine beginWithTicker(
280278

281279
if (getRetryPolicy(session) == TASK && faultTolerantExecutionExchangeEncryptionEnabled) {
282280
// encryption is mandatory for fault tolerant execution as it relies on an external storage to store intermediate data generated during an exchange
283-
try {
284-
KeyGenerator keyGenerator = KeyGenerator.getInstance("AES");
285-
keyGenerator.init(256);
286-
session = session.withExchangeEncryption(Slices.wrappedBuffer(keyGenerator.generateKey().getEncoded()));
287-
}
288-
catch (NoSuchAlgorithmException e) {
289-
throw new TrinoException(GENERIC_INTERNAL_ERROR, "Failed to generate new secret key: " + e.getMessage(), e);
290-
}
281+
session = session.withExchangeEncryption(serializeAesEncryptionKey(createRandomAesEncryptionKey()));
291282
}
292283

293284
QueryStateMachine queryStateMachine = new QueryStateMachine(

0 commit comments

Comments
 (0)