Skip to content

Commit 8e253a3

Browse files
poutsmapoutsma
committed
Support top-level scalar values in Jackson2Tokenizer
Issue: SPR-16166
1 parent 51aa16f commit 8e253a3

File tree

3 files changed

+118
-82
lines changed

3 files changed

+118
-82
lines changed

‎spring-web/src/main/java/org/springframework/http/codec/json/AbstractJackson2Decoder.java‎

Lines changed: 3 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -17,12 +17,10 @@
1717
package org.springframework.http.codec.json;
1818

1919
import java.io.IOException;
20-
import java.io.UncheckedIOException;
2120
import java.lang.annotation.Annotation;
2221
import java.util.Map;
2322

2423
import com.fasterxml.jackson.core.JsonFactory;
25-
import com.fasterxml.jackson.core.JsonParser;
2624
import com.fasterxml.jackson.core.JsonProcessingException;
2725
import com.fasterxml.jackson.databind.JavaType;
2826
import com.fasterxml.jackson.databind.ObjectMapper;
@@ -89,15 +87,9 @@ public Mono<Object> decodeToMono(Publisher<DataBuffer> input, ResolvableType ele
8987
}
9088

9189
private Flux<TokenBuffer> tokenize(Publisher<DataBuffer> input, boolean tokenizeArrayElements) {
92-
try {
93-
JsonFactory factory = getObjectMapper().getFactory();
94-
JsonParser parser = factory.createNonBlockingByteArrayParser();
95-
Jackson2Tokenizer tokenizer = new Jackson2Tokenizer(parser, tokenizeArrayElements);
96-
return Flux.from(input).flatMap(tokenizer).doFinally(t -> tokenizer.endOfInput());
97-
}
98-
catch (IOException ex) {
99-
return Flux.error(new UncheckedIOException(ex));
100-
}
90+
Flux<DataBuffer> inputFlux = Flux.from(input);
91+
JsonFactory factory = getObjectMapper().getFactory();
92+
return Jackson2Tokenizer.tokenize(inputFlux, factory, tokenizeArrayElements);
10193
}
10294

10395
private Flux<Object> decodeInternal(Flux<TokenBuffer> tokens, ResolvableType elementType,

‎spring-web/src/main/java/org/springframework/http/codec/json/Jackson2Tokenizer.java‎

Lines changed: 66 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -21,11 +21,13 @@
2121
import java.util.List;
2222
import java.util.function.Function;
2323

24+
import com.fasterxml.jackson.core.JsonFactory;
2425
import com.fasterxml.jackson.core.JsonParser;
2526
import com.fasterxml.jackson.core.JsonProcessingException;
2627
import com.fasterxml.jackson.core.JsonToken;
2728
import com.fasterxml.jackson.core.async.ByteArrayFeeder;
2829
import com.fasterxml.jackson.databind.util.TokenBuffer;
30+
import org.jetbrains.annotations.NotNull;
2931
import reactor.core.publisher.Flux;
3032

3133
import org.springframework.core.codec.DecodingException;
@@ -41,7 +43,7 @@
4143
* @author Arjen Poutsma
4244
* @since 5.0
4345
*/
44-
class Jackson2Tokenizer implements Function<DataBuffer, Flux<TokenBuffer>> {
46+
class Jackson2Tokenizer {
4547

4648
private final JsonParser parser;
4749

@@ -57,15 +59,7 @@ class Jackson2Tokenizer implements Function<DataBuffer, Flux<TokenBuffer>> {
5759
private final ByteArrayFeeder inputFeeder;
5860

5961

60-
/**
61-
* Create a new instance of the {@code Jackson2Tokenizer}.
62-
* @param parser the non-blocking parser, obtained via
63-
* {@link com.fasterxml.jackson.core.JsonFactory#createNonBlockingByteArrayParser}
64-
* @param tokenizeArrayElements if {@code true} and the "top level" JSON
65-
* object is an array, each element is returned individually, immediately
66-
* after it is received.
67-
*/
68-
public Jackson2Tokenizer(JsonParser parser, boolean tokenizeArrayElements) {
62+
private Jackson2Tokenizer(JsonParser parser, boolean tokenizeArrayElements) {
6963
Assert.notNull(parser, "'parser' must not be null");
7064

7165
this.parser = parser;
@@ -74,42 +68,78 @@ public Jackson2Tokenizer(JsonParser parser, boolean tokenizeArrayElements) {
7468
this.inputFeeder = (ByteArrayFeeder) this.parser.getNonBlockingInputFeeder();
7569
}
7670

71+
/**
72+
* Tokenize the given {@link DataBuffer} flux into a {@link TokenBuffer} flux, given the
73+
* parameters.
74+
* @param dataBuffers the source data buffers
75+
* @param jsonFactory the factory to use
76+
* @param tokenizeArrayElements if {@code true} and the "top level" JSON
77+
* object is an array, each element is returned individually, immediately
78+
* after it is received.
79+
* @return the result token buffers
80+
*/
81+
public static Flux<TokenBuffer> tokenize(Flux<DataBuffer> dataBuffers, JsonFactory jsonFactory,
82+
boolean tokenizeArrayElements) {
83+
try {
84+
Jackson2Tokenizer tokenizer =
85+
new Jackson2Tokenizer(jsonFactory.createNonBlockingByteArrayParser(),
86+
tokenizeArrayElements);
87+
return dataBuffers.flatMap(tokenizer::tokenize, Flux::error, tokenizer::endOfInput);
88+
}
89+
catch (IOException ex) {
90+
return Flux.error(ex);
91+
}
92+
}
7793

78-
@Override
79-
public Flux<TokenBuffer> apply(DataBuffer dataBuffer) {
94+
private Flux<TokenBuffer> tokenize(DataBuffer dataBuffer) {
8095
byte[] bytes = new byte[dataBuffer.readableByteCount()];
8196
dataBuffer.read(bytes);
8297
DataBufferUtils.release(dataBuffer);
8398

8499
try {
85100
this.inputFeeder.feedInput(bytes, 0, bytes.length);
86-
List<TokenBuffer> result = new ArrayList<>();
87-
88-
while (true) {
89-
JsonToken token = this.parser.nextToken();
90-
if (token == JsonToken.NOT_AVAILABLE) {
91-
break;
92-
}
93-
updateDepth(token);
94-
95-
if (!this.tokenizeArrayElements) {
96-
processTokenNormal(token, result);
97-
}
98-
else {
99-
processTokenArray(token, result);
100-
}
101-
}
102-
return Flux.fromIterable(result);
101+
return parseTokenBufferFlux();
103102
}
104103
catch (JsonProcessingException ex) {
105104
return Flux.error(new DecodingException(
106105
"JSON decoding error: " + ex.getOriginalMessage(), ex));
107106
}
108-
catch (Exception ex) {
107+
catch (IOException ex) {
108+
return Flux.error(ex);
109+
}
110+
}
111+
112+
private Flux<TokenBuffer> endOfInput() {
113+
this.inputFeeder.endOfInput();
114+
try {
115+
return parseTokenBufferFlux();
116+
}
117+
catch (IOException ex) {
109118
return Flux.error(ex);
110119
}
111120
}
112121

122+
@NotNull
123+
private Flux<TokenBuffer> parseTokenBufferFlux() throws IOException {
124+
List<TokenBuffer> result = new ArrayList<>();
125+
126+
while (true) {
127+
JsonToken token = this.parser.nextToken();
128+
if (token == null || token == JsonToken.NOT_AVAILABLE) {
129+
break;
130+
}
131+
updateDepth(token);
132+
133+
if (!this.tokenizeArrayElements) {
134+
processTokenNormal(token, result);
135+
}
136+
else {
137+
processTokenArray(token, result);
138+
}
139+
}
140+
return Flux.fromIterable(result);
141+
}
142+
113143
private void updateDepth(JsonToken token) {
114144
switch (token) {
115145
case START_OBJECT:
@@ -130,11 +160,10 @@ private void updateDepth(JsonToken token) {
130160
private void processTokenNormal(JsonToken token, List<TokenBuffer> result) throws IOException {
131161
this.tokenBuffer.copyCurrentEvent(this.parser);
132162

133-
if (token == JsonToken.END_OBJECT || token == JsonToken.END_ARRAY) {
134-
if (this.objectDepth == 0 && this.arrayDepth == 0) {
135-
result.add(this.tokenBuffer);
136-
this.tokenBuffer = new TokenBuffer(this.parser);
137-
}
163+
if ((token.isStructEnd() || token.isScalarValue()) &&
164+
this.objectDepth == 0 && this.arrayDepth == 0) {
165+
result.add(this.tokenBuffer);
166+
this.tokenBuffer = new TokenBuffer(this.parser);
138167
}
139168

140169
}
@@ -144,8 +173,8 @@ private void processTokenArray(JsonToken token, List<TokenBuffer> result) throws
144173
this.tokenBuffer.copyCurrentEvent(this.parser);
145174
}
146175

147-
if (token == JsonToken.END_OBJECT && this.objectDepth == 0 &&
148-
(this.arrayDepth == 1 || this.arrayDepth == 0)) {
176+
if ((token == JsonToken.END_OBJECT && this.objectDepth == 0 && (this.arrayDepth == 1 || this.arrayDepth == 0)) ||
177+
(token.isScalarValue()) && this.objectDepth == 0 && this.arrayDepth == 0) {
149178
result.add(this.tokenBuffer);
150179
this.tokenBuffer = new TokenBuffer(this.parser);
151180
}
@@ -156,8 +185,4 @@ private boolean isTopLevelArrayToken(JsonToken token) {
156185
(token == JsonToken.END_ARRAY && this.arrayDepth == 0));
157186
}
158187

159-
public void endOfInput() {
160-
this.inputFeeder.endOfInput();
161-
}
162-
163188
}

‎spring-web/src/test/java/org/springframework/http/codec/json/Jackson2TokenizerTests.java‎

Lines changed: 49 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,9 @@
2222
import java.util.function.Consumer;
2323

2424
import com.fasterxml.jackson.core.JsonFactory;
25-
import com.fasterxml.jackson.core.JsonParser;
2625
import com.fasterxml.jackson.core.TreeNode;
2726
import com.fasterxml.jackson.databind.ObjectMapper;
27+
import com.fasterxml.jackson.databind.util.TokenBuffer;
2828
import org.json.JSONException;
2929
import org.junit.Before;
3030
import org.junit.Test;
@@ -43,44 +43,39 @@
4343
*/
4444
public class Jackson2TokenizerTests extends AbstractDataBufferAllocatingTestCase {
4545

46-
private JsonParser jsonParser;
47-
48-
private Jackson2Tokenizer tokenizer;
49-
5046
private ObjectMapper objectMapper;
5147

48+
private JsonFactory jsonFactory;
49+
5250
@Before
5351
public void createParser() throws IOException {
54-
JsonFactory factory = new JsonFactory();
55-
this.jsonParser = factory.createNonBlockingByteArrayParser();
56-
this.objectMapper = new ObjectMapper(factory);
52+
jsonFactory = new JsonFactory();
53+
this.objectMapper = new ObjectMapper(jsonFactory);
5754
}
5855

5956
@Test
6057
public void doNotTokenizeArrayElements() {
61-
this.tokenizer = new Jackson2Tokenizer(this.jsonParser, false);
62-
6358
testTokenize(
6459
singletonList("{\"foo\": \"foofoo\", \"bar\": \"barbar\"}"),
65-
singletonList("{\"foo\": \"foofoo\", \"bar\": \"barbar\"}"));
60+
singletonList("{\"foo\": \"foofoo\", \"bar\": \"barbar\"}"), false);
6661

6762
testTokenize(
6863
asList("{\"foo\": \"foofoo\"",
6964
", \"bar\": \"barbar\"}"),
70-
singletonList("{\"foo\":\"foofoo\",\"bar\":\"barbar\"}"));
65+
singletonList("{\"foo\":\"foofoo\",\"bar\":\"barbar\"}"), false);
7166

7267
testTokenize(
7368
singletonList("[{\"foo\": \"foofoo\", \"bar\": \"barbar\"},{\"foo\": \"foofoofoo\", \"bar\": \"barbarbar\"}]"),
74-
singletonList("[{\"foo\": \"foofoo\", \"bar\": \"barbar\"},{\"foo\": \"foofoofoo\", \"bar\": \"barbarbar\"}]"));
69+
singletonList("[{\"foo\": \"foofoo\", \"bar\": \"barbar\"},{\"foo\": \"foofoofoo\", \"bar\": \"barbarbar\"}]"), false);
7570

7671
testTokenize(
7772
singletonList("[{\"foo\": \"bar\"},{\"foo\": \"baz\"}]"),
78-
singletonList("[{\"foo\": \"bar\"},{\"foo\": \"baz\"}]"));
73+
singletonList("[{\"foo\": \"bar\"},{\"foo\": \"baz\"}]"), false);
7974

8075
testTokenize(
8176
asList("[{\"foo\": \"foofoo\", \"bar\"",
8277
": \"barbar\"},{\"foo\": \"foofoofoo\", \"bar\": \"barbarbar\"}]"),
83-
singletonList("[{\"foo\": \"foofoo\", \"bar\": \"barbar\"},{\"foo\": \"foofoofoo\", \"bar\": \"barbarbar\"}]"));
78+
singletonList("[{\"foo\": \"foofoo\", \"bar\": \"barbar\"},{\"foo\": \"foofoofoo\", \"bar\": \"barbarbar\"}]"), false);
8479

8580
testTokenize(
8681
asList("[",
@@ -90,31 +85,43 @@ public void doNotTokenizeArrayElements() {
9085
",",
9186
"{\"id\":3,\"name\":\"Ford\"}",
9287
"]"),
93-
singletonList("[{\"id\":1,\"name\":\"Robert\"},{\"id\":2,\"name\":\"Raide\"},{\"id\":3,\"name\":\"Ford\"}]"));
88+
singletonList("[{\"id\":1,\"name\":\"Robert\"},{\"id\":2,\"name\":\"Raide\"},{\"id\":3,\"name\":\"Ford\"}]"), false);
89+
90+
// SPR-16166: top-level JSON values
91+
testTokenize(asList("\"foo", "bar\"")
92+
,singletonList("\"foobar\""), false);
93+
94+
testTokenize(asList("12", "34")
95+
,singletonList("1234"), false);
96+
97+
testTokenize(asList("12.", "34")
98+
,singletonList("12.34"), false);
99+
100+
// note that we do not test for null, true, or false, which are also valid top-level values,
101+
// but are unsupported by JSONassert
102+
94103
}
95104

96105
@Test
97106
public void tokenizeArrayElements() {
98-
this.tokenizer = new Jackson2Tokenizer(this.jsonParser, true);
99-
100107
testTokenize(
101108
singletonList("{\"foo\": \"foofoo\", \"bar\": \"barbar\"}"),
102-
singletonList("{\"foo\": \"foofoo\", \"bar\": \"barbar\"}"));
109+
singletonList("{\"foo\": \"foofoo\", \"bar\": \"barbar\"}"), true);
103110

104111
testTokenize(
105112
asList("{\"foo\": \"foofoo\"",
106113
", \"bar\": \"barbar\"}"),
107-
singletonList("{\"foo\":\"foofoo\",\"bar\":\"barbar\"}"));
114+
singletonList("{\"foo\":\"foofoo\",\"bar\":\"barbar\"}"), true);
108115

109116
testTokenize(
110117
singletonList("[{\"foo\": \"foofoo\", \"bar\": \"barbar\"},{\"foo\": \"foofoofoo\", \"bar\": \"barbarbar\"}]"),
111118
asList("{\"foo\": \"foofoo\", \"bar\": \"barbar\"}",
112-
"{\"foo\": \"foofoofoo\", \"bar\": \"barbarbar\"}"));
119+
"{\"foo\": \"foofoofoo\", \"bar\": \"barbarbar\"}"), true);
113120

114121
testTokenize(
115122
singletonList("[{\"foo\": \"bar\"},{\"foo\": \"baz\"}]"),
116123
asList("{\"foo\": \"bar\"}",
117-
"{\"foo\": \"baz\"}"));
124+
"{\"foo\": \"baz\"}"), true);
118125

119126
// SPR-15803: nested array
120127
testTokenize(
@@ -126,19 +133,19 @@ public void tokenizeArrayElements() {
126133
asList(
127134
"{\"id\":\"0\",\"start\":[-999999999,1,1],\"end\":[999999999,12,31]}",
128135
"{\"id\":\"1\",\"start\":[-999999999,1,1],\"end\":[999999999,12,31]}",
129-
"{\"id\":\"2\",\"start\":[-999999999,1,1],\"end\":[999999999,12,31]}")
130-
);
136+
"{\"id\":\"2\",\"start\":[-999999999,1,1],\"end\":[999999999,12,31]}"),
137+
true);
131138

132139
// SPR-15803: nested array, no top-level array
133140
testTokenize(
134141
singletonList("{\"speakerIds\":[\"tastapod\"],\"language\":\"ENGLISH\"}"),
135-
singletonList("{\"speakerIds\":[\"tastapod\"],\"language\":\"ENGLISH\"}"));
142+
singletonList("{\"speakerIds\":[\"tastapod\"],\"language\":\"ENGLISH\"}"), true);
136143

137144
testTokenize(
138145
asList("[{\"foo\": \"foofoo\", \"bar\"",
139146
": \"barbar\"},{\"foo\": \"foofoofoo\", \"bar\": \"barbarbar\"}]"),
140147
asList("{\"foo\": \"foofoo\", \"bar\": \"barbar\"}",
141-
"{\"foo\": \"foofoofoo\", \"bar\": \"barbarbar\"}"));
148+
"{\"foo\": \"foofoofoo\", \"bar\": \"barbarbar\"}"), true);
142149

143150
testTokenize(
144151
asList("[",
@@ -150,15 +157,27 @@ public void tokenizeArrayElements() {
150157
"]"),
151158
asList("{\"id\":1,\"name\":\"Robert\"}",
152159
"{\"id\":2,\"name\":\"Raide\"}",
153-
"{\"id\":3,\"name\":\"Ford\"}"));
160+
"{\"id\":3,\"name\":\"Ford\"}"), true);
161+
162+
// SPR-16166: top-level JSON values
163+
testTokenize(asList("\"foo", "bar\"")
164+
,singletonList("\"foobar\""), true);
165+
166+
testTokenize(asList("12", "34")
167+
,singletonList("1234"), true);
168+
169+
testTokenize(asList("12.", "34")
170+
,singletonList("12.34"), true);
154171
}
155172

156-
private void testTokenize(List<String> source, List<String> expected) {
173+
private void testTokenize(List<String> source, List<String> expected, boolean tokenizeArrayElements) {
157174
Flux<DataBuffer> sourceFlux = Flux.fromIterable(source)
158175
.map(this::stringBuffer);
159176

160-
Flux<String> result = sourceFlux
161-
.flatMap(this.tokenizer)
177+
Flux<TokenBuffer> tokenBufferFlux =
178+
Jackson2Tokenizer.tokenize(sourceFlux, jsonFactory, tokenizeArrayElements);
179+
180+
Flux<String> result = tokenBufferFlux
162181
.map(tokenBuffer -> {
163182
try {
164183
TreeNode root = this.objectMapper.readTree(tokenBuffer.asParser());

0 commit comments

Comments
 (0)