From b1d0847a5a9433f99d3d6ecd0607672e492715fa Mon Sep 17 00:00:00 2001 From: abmdocrt Date: Thu, 26 Sep 2024 22:51:12 +0800 Subject: [PATCH] [Fix](LZ4 compression) Fix wrong LZ4 compression max input size limit (#41239) ## Proposed changes LZ4 compression max supported value is LZ4_MAX_INPUT_SIZE, which is 0x7E000000(2,113,929,216 bytes). Doris use wrong max size INT_MAX, which is 2,147,483,647, to check. If input data size is between this two size, then it can pass the check but LZ4 compression will fail. This PR fix it. --- be/src/util/block_compression.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/be/src/util/block_compression.cpp b/be/src/util/block_compression.cpp index 7afbafa9790a68..1c2a65b5337749 100644 --- a/be/src/util/block_compression.cpp +++ b/be/src/util/block_compression.cpp @@ -108,11 +108,13 @@ class Lz4BlockCompression : public BlockCompressionCodec { } Status compress(const Slice& input, faststring* output) override { - if (input.size > INT_MAX) { + if (input.size > LZ4_MAX_INPUT_SIZE) { return Status::InvalidArgument( - "LZ4 not support those case(input.size>INT_MAX), maybe you should change " - "fragment_transmission_compression_codec to snappy, size={}", - input.size); + "LZ4 not support those case(input.size>LZ4_MAX_INPUT_SIZE), maybe you should " + "change " + "fragment_transmission_compression_codec to snappy, input.size={}, " + "LZ4_MAX_INPUT_SIZE={}", + input.size, LZ4_MAX_INPUT_SIZE); } Context* context;