From 8e1351f76b030c731a56ca50c051a9d43ee83e89 Mon Sep 17 00:00:00 2001 From: seonwoo_jung <79202163+seonwooj0810@users.noreply.github.com> Date: Mon, 22 Jun 2026 21:17:05 +0900 Subject: [PATCH 1/4] Fix #1622: size base64 encoding buffer from binary length hint When writing binary content from an InputStream of known length via `writeBinary(Base64Variant, InputStream, int)`, the read/encoding buffer was always the small default (2000 bytes), forcing many InputStream reads for large content (and, for sources like protobuf streams, an extra throwaway in-memory copy when the supplied buffer is smaller than the serialized size). Now, when the length is known (> 0), the buffer is sized from that hint, capped at 64kB to bound retention of ThreadLocal-recycled buffers. Applied to both UTF8JsonGenerator and WriterBasedJsonGenerator. The default base64 codec buffer is also bumped from 2000 to 16000 bytes to help the unknown-length path. Per maintainer guidance on the issue. Verification: added BinaryWriteBufferSize1622Test asserting the read buffer is sized to the (capped) hint and that output still round-trips; existing base64 and buffer-recycler tests pass. --- release-notes/VERSION-2.x | 4 +- .../jackson/core/json/JsonGeneratorImpl.java | 12 ++ .../jackson/core/json/UTF8JsonGenerator.java | 6 +- .../core/json/WriterBasedJsonGenerator.java | 6 +- .../jackson/core/util/BufferRecycler.java | 5 +- .../base64/BinaryWriteBufferSize1622Test.java | 115 ++++++++++++++++++ 6 files changed, 144 insertions(+), 4 deletions(-) create mode 100644 src/test/java/com/fasterxml/jackson/core/base64/BinaryWriteBufferSize1622Test.java diff --git a/release-notes/VERSION-2.x b/release-notes/VERSION-2.x index eca204ba8b..46783c038e 100644 --- a/release-notes/VERSION-2.x +++ b/release-notes/VERSION-2.x @@ -16,7 +16,9 @@ a pure JSON library. 2.23.0 (not yet released) -No changes since 2.22 +#1622: `UTF8JsonGenerator.writeBinary()` could allocate encoding buffer + based on supplied length + (requested by @kilink) 2.22.0 (03-Jun-2026) diff --git a/src/main/java/com/fasterxml/jackson/core/json/JsonGeneratorImpl.java b/src/main/java/com/fasterxml/jackson/core/json/JsonGeneratorImpl.java index 80f9bcdafd..97f43a9da2 100644 --- a/src/main/java/com/fasterxml/jackson/core/json/JsonGeneratorImpl.java +++ b/src/main/java/com/fasterxml/jackson/core/json/JsonGeneratorImpl.java @@ -30,6 +30,18 @@ public abstract class JsonGeneratorImpl extends GeneratorBase */ protected final static int[] sOutputEscapes = CharTypes.get7BitOutputEscapes(); + /** + * Maximum size, in bytes, of the recyclable base64 encoding buffer to + * allocate when a binary content length hint is available (see + * {@code writeBinary(Base64Variant, InputStream, int)}). Allocating a + * larger buffer for big content reduces the number of + * {@link java.io.InputStream} reads required, but the size is capped to + * limit retention of large {@code ThreadLocal}-recycled buffers. + * + * @since 2.23 + */ + protected final static int MAX_BASE64_ENCODE_BUFFER_LENGTH = 64 * 1024; + /** * Default capabilities for JSON generator implementations which do not * different from "general textual" defaults diff --git a/src/main/java/com/fasterxml/jackson/core/json/UTF8JsonGenerator.java b/src/main/java/com/fasterxml/jackson/core/json/UTF8JsonGenerator.java index 268aabc4ca..f84fe8804d 100644 --- a/src/main/java/com/fasterxml/jackson/core/json/UTF8JsonGenerator.java +++ b/src/main/java/com/fasterxml/jackson/core/json/UTF8JsonGenerator.java @@ -917,7 +917,11 @@ public int writeBinary(Base64Variant b64variant, _flushBuffer(); } _outputBuffer[_outputTail++] = _quoteChar; - byte[] encodingBuffer = _ioContext.allocBase64Buffer(); + // [core#1622]: when length is known, size the read buffer accordingly + // (capped) so large content needs fewer InputStream reads + byte[] encodingBuffer = (dataLength > 0) + ? _ioContext.allocBase64Buffer(Math.min(dataLength, MAX_BASE64_ENCODE_BUFFER_LENGTH)) + : _ioContext.allocBase64Buffer(); int bytes; try { if (dataLength < 0) { // length unknown diff --git a/src/main/java/com/fasterxml/jackson/core/json/WriterBasedJsonGenerator.java b/src/main/java/com/fasterxml/jackson/core/json/WriterBasedJsonGenerator.java index 6bffd40e6b..e1cba91d90 100644 --- a/src/main/java/com/fasterxml/jackson/core/json/WriterBasedJsonGenerator.java +++ b/src/main/java/com/fasterxml/jackson/core/json/WriterBasedJsonGenerator.java @@ -695,7 +695,11 @@ public int writeBinary(Base64Variant b64variant, _flushBuffer(); } _outputBuffer[_outputTail++] = _quoteChar; - byte[] encodingBuffer = _ioContext.allocBase64Buffer(); + // [core#1622]: when length is known, size the read buffer accordingly + // (capped) so large content needs fewer InputStream reads + byte[] encodingBuffer = (dataLength > 0) + ? _ioContext.allocBase64Buffer(Math.min(dataLength, MAX_BASE64_ENCODE_BUFFER_LENGTH)) + : _ioContext.allocBase64Buffer(); int bytes; try { if (dataLength < 0) { // length unknown diff --git a/src/main/java/com/fasterxml/jackson/core/util/BufferRecycler.java b/src/main/java/com/fasterxml/jackson/core/util/BufferRecycler.java index a76167f4a0..f8037d1a00 100644 --- a/src/main/java/com/fasterxml/jackson/core/util/BufferRecycler.java +++ b/src/main/java/com/fasterxml/jackson/core/util/BufferRecycler.java @@ -91,8 +91,11 @@ public interface Gettable { public final static int CHAR_NAME_COPY_BUFFER = 3; // Buffer lengths + // 22-Jun-2026, [core#1622]: bumped default base64 codec buffer (index + // BYTE_BASE64_CODEC_BUFFER) from 2000 to 16000 to reduce InputStream + // reads when encoding binary content of unknown/large length. - private final static int[] BYTE_BUFFER_LENGTHS = new int[] { 8000, 8000, 2000, 2000 }; + private final static int[] BYTE_BUFFER_LENGTHS = new int[] { 8000, 8000, 2000, 16000 }; private final static int[] CHAR_BUFFER_LENGTHS = new int[] { 4000, 4000, 200, 200 }; // Note: changed from simple array in 2.10: diff --git a/src/test/java/com/fasterxml/jackson/core/base64/BinaryWriteBufferSize1622Test.java b/src/test/java/com/fasterxml/jackson/core/base64/BinaryWriteBufferSize1622Test.java new file mode 100644 index 0000000000..3703d16506 --- /dev/null +++ b/src/test/java/com/fasterxml/jackson/core/base64/BinaryWriteBufferSize1622Test.java @@ -0,0 +1,115 @@ +package com.fasterxml.jackson.core.base64; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.StringWriter; + +import org.junit.jupiter.api.Test; + +import com.fasterxml.jackson.core.Base64Variant; +import com.fasterxml.jackson.core.Base64Variants; +import com.fasterxml.jackson.core.JsonEncoding; +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonToken; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +// [core#1622]: When a binary content length is known, the encoding/read buffer +// should be sized from that hint (capped) so large content needs far fewer +// InputStream reads than the small default buffer would require. +class BinaryWriteBufferSize1622Test + extends com.fasterxml.jackson.core.JUnit5TestBase +{ + // Cap mirrored from JsonGeneratorImpl.MAX_BASE64_ENCODE_BUFFER_LENGTH + private final static int MAX_BUFFER = 64 * 1024; + + private final JsonFactory JSON_F = new JsonFactory(); + + private final Base64Variant VARIANT = Base64Variants.MIME; + + /** + * {@link ByteArrayInputStream} that records the largest {@code len} ever + * requested via {@link #read(byte[], int, int)}, which equals the size of + * the read buffer the generator allocated. + */ + static class ReadSizeRecordingInputStream extends ByteArrayInputStream { + int maxRequestedRead = 0; + + ReadSizeRecordingInputStream(byte[] buf) { + super(buf); + } + + @Override + public synchronized int read(byte[] b, int off, int len) { + if (len > maxRequestedRead) { + maxRequestedRead = len; + } + return super.read(b, off, len); + } + } + + @Test + void sizeHintAppliedByteBacked() throws Exception { + // 50_000 is below the 64kB cap, so the read buffer should be sized to it + _testSizeHint(true, 50_000, 50_000); + } + + @Test + void sizeHintAppliedCharBacked() throws Exception { + _testSizeHint(false, 50_000, 50_000); + } + + @Test + void sizeHintCappedByteBacked() throws Exception { + // 200_000 exceeds the cap, so the read buffer should be limited to it + _testSizeHint(true, 200_000, MAX_BUFFER); + } + + @Test + void sizeHintCappedCharBacked() throws Exception { + _testSizeHint(false, 200_000, MAX_BUFFER); + } + + private void _testSizeHint(boolean useBytes, int dataLength, int expectedMaxRead) + throws Exception + { + byte[] input = new byte[dataLength]; + for (int i = 0; i < input.length; ++i) { + input[i] = (byte) (i * 31 + 7); + } + ReadSizeRecordingInputStream in = new ReadSizeRecordingInputStream(input); + + byte[] rawJson; + if (useBytes) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + try (JsonGenerator g = JSON_F.createGenerator(out, JsonEncoding.UTF8)) { + g.writeBinary(VARIANT, in, dataLength); + } + rawJson = out.toByteArray(); + } else { + StringWriter sw = new StringWriter(); + try (JsonGenerator g = JSON_F.createGenerator(sw)) { + g.writeBinary(VARIANT, in, dataLength); + } + rawJson = sw.toString().getBytes("UTF-8"); + } + + // The generator should have requested reads as large as the (capped) hint, + // which is much bigger than the small default buffer used before the fix. + assertEquals(expectedMaxRead, in.maxRequestedRead, + "read buffer should be sized from the length hint (capped)"); + assertTrue(in.maxRequestedRead <= MAX_BUFFER, + "read buffer must never exceed the cap"); + + // ...and the produced base64 must still decode back to the original bytes. + try (JsonParser p = JSON_F.createParser(rawJson)) { + assertEquals(JsonToken.VALUE_STRING, p.nextToken()); + byte[] decoded = p.getBinaryValue(VARIANT); + assertArrayEquals(input, decoded); + } + } +} From ec8c5c5441fda2f7467f27f9a49792b0a967ca20 Mon Sep 17 00:00:00 2001 From: Tatu Saloranta Date: Thu, 25 Jun 2026 20:29:27 -0700 Subject: [PATCH 2/4] Add released notes --- release-notes/CREDITS-2.x | 11 +++++++++++ release-notes/VERSION-2.x | 3 ++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/release-notes/CREDITS-2.x b/release-notes/CREDITS-2.x index 67ffbc2a3d..91e64cc21a 100644 --- a/release-notes/CREDITS-2.x +++ b/release-notes/CREDITS-2.x @@ -520,3 +520,14 @@ Mike Pedersen (@mpdncrwd) * Reported #1581: `NonBlockingByteBufferParser`: Unexpected Illegal surrogate character when parsing field names (2.21.3) + +Patrick Strawderman (@kilink) + * Requested #1622: `UTF8JsonGenerator.writeBinary()` should allocate buffer + based on supplied length + (2.23.0) + +@seonwooj0810 + * Contributed #1622: `UTF8JsonGenerator.writeBinary()` should allocate buffer + based on supplied length + (2.23.0) + diff --git a/release-notes/VERSION-2.x b/release-notes/VERSION-2.x index 46783c038e..854ecd2697 100644 --- a/release-notes/VERSION-2.x +++ b/release-notes/VERSION-2.x @@ -16,9 +16,10 @@ a pure JSON library. 2.23.0 (not yet released) -#1622: `UTF8JsonGenerator.writeBinary()` could allocate encoding buffer +#1622: `UTF8JsonGenerator.writeBinary()` should allocate buffer based on supplied length (requested by @kilink) + (contributed by @seonwooj0810) 2.22.0 (03-Jun-2026) From e2a1ee518d63fe84f73257aadf417998cf24ed55 Mon Sep 17 00:00:00 2001 From: Tatu Saloranta Date: Thu, 25 Jun 2026 20:30:40 -0700 Subject: [PATCH 3/4] Reduce default buffer size change back to 4000 from initial 16000 (wrt older 2000) --- .../java/com/fasterxml/jackson/core/util/BufferRecycler.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/fasterxml/jackson/core/util/BufferRecycler.java b/src/main/java/com/fasterxml/jackson/core/util/BufferRecycler.java index f8037d1a00..fe99f7b4ff 100644 --- a/src/main/java/com/fasterxml/jackson/core/util/BufferRecycler.java +++ b/src/main/java/com/fasterxml/jackson/core/util/BufferRecycler.java @@ -92,10 +92,10 @@ public interface Gettable { // Buffer lengths // 22-Jun-2026, [core#1622]: bumped default base64 codec buffer (index - // BYTE_BASE64_CODEC_BUFFER) from 2000 to 16000 to reduce InputStream + // BYTE_BASE64_CODEC_BUFFER) from 2000 to 4000 to reduce InputStream // reads when encoding binary content of unknown/large length. - private final static int[] BYTE_BUFFER_LENGTHS = new int[] { 8000, 8000, 2000, 16000 }; + private final static int[] BYTE_BUFFER_LENGTHS = new int[] { 8000, 8000, 2000, 4000 }; private final static int[] CHAR_BUFFER_LENGTHS = new int[] { 4000, 4000, 200, 200 }; // Note: changed from simple array in 2.10: From dae6a639afe077cdadf34f7b8b7420e3dba852c1 Mon Sep 17 00:00:00 2001 From: Tatu Saloranta Date: Thu, 25 Jun 2026 20:40:49 -0700 Subject: [PATCH 4/4] ... --- release-notes/CREDITS-2.x | 1 - 1 file changed, 1 deletion(-) diff --git a/release-notes/CREDITS-2.x b/release-notes/CREDITS-2.x index 91e64cc21a..de11837d57 100644 --- a/release-notes/CREDITS-2.x +++ b/release-notes/CREDITS-2.x @@ -530,4 +530,3 @@ Patrick Strawderman (@kilink) * Contributed #1622: `UTF8JsonGenerator.writeBinary()` should allocate buffer based on supplied length (2.23.0) -