Skip to content

Validate the end of input when parsing JSON #17377

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
v-jizhang wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Validate the end of input when parsing JSON #17377

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 6 additions & 3 deletions presto-main/src/main/java/com/facebook/presto/operator/scalar/JsonFunctions.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@
import static com.fasterxml.jackson.core.JsonToken.VALUE_STRING;
import static com.fasterxml.jackson.core.JsonToken.VALUE_TRUE;
import static com.fasterxml.jackson.databind.SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS;
import static com.google.common.base.Preconditions.checkState;
import static io.airlift.slice.Slices.utf8Slice;
import static java.lang.String.format;

Expand Down Expand Up @@ -145,9 +146,11 @@ public static Slice jsonParse(@SqlType("varchar(x)") Slice slice)
try (JsonParser parser = createJsonParser(JSON_FACTORY, slice)) {
SliceOutput dynamicSliceOutput = new DynamicSliceOutput(slice.length());
SORTED_MAPPER.writeValue((OutputStream) dynamicSliceOutput, SORTED_MAPPER.readValue(parser, Object.class));
// nextToken() returns null if the input is parsed correctly,
// but will throw an exception if there are trailing characters.
parser.nextToken();
// At this point, the end of input should be reached. nextToken() has three possible results:
// - null, if the end of the input was reached
// - token, if a correct JSON token is found (e.g. '{', 'null', '1')
// - exception, if there are characters which do not form a valid JSON token (e.g. 'abc')
checkState(parser.nextToken() == null, "Found characters after the expected end of input");
return dynamicSliceOutput.slice();
}
catch (Exception e) {
Expand Down
5 changes: 5 additions & 0 deletions presto-main/src/test/java/com/facebook/presto/operator/scalar/TestJsonFunctions.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -278,6 +278,11 @@ public void testInvalidJsonParse()
assertInvalidFunction("JSON 'INVALID'", INVALID_FUNCTION_ARGUMENT);
assertInvalidFunction("JSON_PARSE('INVALID')", INVALID_FUNCTION_ARGUMENT);
assertInvalidFunction("JSON_PARSE('\"x\": 1')", INVALID_FUNCTION_ARGUMENT);
assertInvalidFunction("JSON_PARSE('{}{')", INVALID_FUNCTION_ARGUMENT);
assertInvalidFunction("JSON_PARSE('{} \"a\"')", INVALID_FUNCTION_ARGUMENT);
assertInvalidFunction("JSON_PARSE('{}{abc')", INVALID_FUNCTION_ARGUMENT);
assertInvalidFunction("JSON_PARSE('{}abc')", INVALID_FUNCTION_ARGUMENT);
assertInvalidFunction("JSON_PARSE('')", INVALID_FUNCTION_ARGUMENT);
}

@Test
Expand Down
11 changes: 11 additions & 0 deletions presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestAnalyzer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1322,6 +1322,17 @@ public void testLiteral()
assertFails(INVALID_LITERAL, "SELECT TIMESTAMP '2012-10-31 01:00:00 PT'");
}

@Test
public void testJsonLiteral()
{
// TODO All the below should fail. Literals should be validated during analysis
analyze("SELECT JSON '{}{'");
analyze("SELECT JSON '{} \"a\"'");
analyze("SELECT JSON '{}{abc'");
analyze("SELECT JSON '{}abc'");
analyze("SELECT JSON ''");
}

@Test
public void testLambda()
{
Expand Down
6 changes: 6 additions & 0 deletions presto-main/src/test/java/com/facebook/presto/type/TestJsonOperators.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
import static com.facebook.presto.common.type.TinyintType.TINYINT;
import static com.facebook.presto.common.type.VarcharType.VARCHAR;
import static com.facebook.presto.spi.StandardErrorCode.INVALID_CAST_ARGUMENT;
import static com.facebook.presto.spi.StandardErrorCode.INVALID_FUNCTION_ARGUMENT;
import static com.facebook.presto.testing.DateTimeTestingUtils.sqlTimestampOf;
import static com.facebook.presto.util.StructuralTestUtil.mapType;
import static java.lang.Double.NEGATIVE_INFINITY;
Expand Down Expand Up @@ -178,6 +179,11 @@ public void testTypeConstructor()
assertFunction("JSON '[null]'", JSON, "[null]");
assertFunction("JSON '[13,null,42]'", JSON, "[13,null,42]");
assertFunction("JSON '{\"x\": null}'", JSON, "{\"x\":null}");
assertInvalidFunction("JSON '{}{'", INVALID_FUNCTION_ARGUMENT);
assertInvalidFunction("JSON '{} \"a\"'", INVALID_FUNCTION_ARGUMENT);
assertInvalidFunction("JSON '{}{abc'", INVALID_FUNCTION_ARGUMENT);
assertInvalidFunction("JSON '{}abc'", INVALID_FUNCTION_ARGUMENT);
assertInvalidFunction("JSON ''", INVALID_FUNCTION_ARGUMENT);
}

@Test
Expand Down