Skip to content

Commit 0ca9d10

Browse files
bjornjorgensenLuciferYang
authored andcommitted
[SPARK-43263][BUILD] Upgrade FasterXML jackson to 2.15.0
### What changes were proposed in this pull request? Upgrade FasterXML jackson from 2.14.2 to 2.15.0 ### Why are the changes needed? Upgrade Snakeyaml to 2.0 (resolves CVE-2022-1471 [CVE-2022-1471 at nist](https://nvd.nist.gov/vuln/detail/CVE-2022-1471) ### Does this PR introduce _any_ user-facing change? This PR introduces user-facing changes by implementing streaming read constraints in the JSONOptions class. The constraints limit the size of input constructs, improving security and efficiency when processing input data. Users working with JSON data larger than the following default settings may need to adjust the constraints accordingly: Maximum Number value length: 1000 characters (`DEFAULT_MAX_NUM_LEN`) Maximum String value length: 5,000,000 characters (`DEFAULT_MAX_STRING_LEN`) Maximum Nesting depth: 1000 levels (`DEFAULT_MAX_DEPTH`) Additionally, the maximum magnitude of scale for BigDecimal to BigInteger conversion is set to 100,000 digits (`MAX_BIGINT_SCALE_MAGNITUDE`) and cannot be changed. Users can customize the constraints as needed by providing the corresponding options in the parameters object. If not explicitly specified, default settings will be applied. ### How was this patch tested? Pass GA Closes apache#40933 from bjornjorgensen/test_jacon. Authored-by: bjornjorgensen <[email protected]> Signed-off-by: Sean Owen <[email protected]>
1 parent 3d5b642 commit 0ca9d10

File tree

3 files changed

+34
-11
lines changed

3 files changed

+34
-11
lines changed

dev/deps/spark-deps-hadoop-3-hive-2.3

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -97,13 +97,13 @@ httpcore/4.4.16//httpcore-4.4.16.jar
9797
ini4j/0.5.4//ini4j-0.5.4.jar
9898
istack-commons-runtime/3.0.8//istack-commons-runtime-3.0.8.jar
9999
ivy/2.5.1//ivy-2.5.1.jar
100-
jackson-annotations/2.14.2//jackson-annotations-2.14.2.jar
101-
jackson-core/2.14.2//jackson-core-2.14.2.jar
102-
jackson-databind/2.14.2//jackson-databind-2.14.2.jar
103-
jackson-dataformat-cbor/2.14.2//jackson-dataformat-cbor-2.14.2.jar
104-
jackson-dataformat-yaml/2.14.2//jackson-dataformat-yaml-2.14.2.jar
105-
jackson-datatype-jsr310/2.14.2//jackson-datatype-jsr310-2.14.2.jar
106-
jackson-module-scala_2.12/2.14.2//jackson-module-scala_2.12-2.14.2.jar
100+
jackson-annotations/2.15.0//jackson-annotations-2.15.0.jar
101+
jackson-core/2.15.0//jackson-core-2.15.0.jar
102+
jackson-databind/2.15.0//jackson-databind-2.15.0.jar
103+
jackson-dataformat-cbor/2.15.0//jackson-dataformat-cbor-2.15.0.jar
104+
jackson-dataformat-yaml/2.15.0//jackson-dataformat-yaml-2.15.0.jar
105+
jackson-datatype-jsr310/2.15.0//jackson-datatype-jsr310-2.15.0.jar
106+
jackson-module-scala_2.12/2.15.0//jackson-module-scala_2.12-2.15.0.jar
107107
jakarta.annotation-api/1.3.5//jakarta.annotation-api-1.3.5.jar
108108
jakarta.inject/2.6.1//jakarta.inject-2.6.1.jar
109109
jakarta.servlet-api/4.0.3//jakarta.servlet-api-4.0.3.jar
@@ -233,7 +233,7 @@ scala-xml_2.12/2.1.0//scala-xml_2.12-2.1.0.jar
233233
shims/0.9.39//shims-0.9.39.jar
234234
slf4j-api/2.0.7//slf4j-api-2.0.7.jar
235235
snakeyaml-engine/2.6//snakeyaml-engine-2.6.jar
236-
snakeyaml/1.33//snakeyaml-1.33.jar
236+
snakeyaml/2.0//snakeyaml-2.0.jar
237237
snappy-java/1.1.9.1//snappy-java-1.1.9.1.jar
238238
spire-macros_2.12/0.17.0//spire-macros_2.12-0.17.0.jar
239239
spire-platform_2.12/0.17.0//spire-platform_2.12-0.17.0.jar

pom.xml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -184,8 +184,8 @@
184184
<scalafmt.validateOnly>true</scalafmt.validateOnly>
185185
<scalafmt.changedOnly>true</scalafmt.changedOnly>
186186
<codehaus.jackson.version>1.9.13</codehaus.jackson.version>
187-
<fasterxml.jackson.version>2.14.2</fasterxml.jackson.version>
188-
<fasterxml.jackson.databind.version>2.14.2</fasterxml.jackson.databind.version>
187+
<fasterxml.jackson.version>2.15.0</fasterxml.jackson.version>
188+
<fasterxml.jackson.databind.version>2.15.0</fasterxml.jackson.databind.version>
189189
<snappy.version>1.1.9.1</snappy.version>
190190
<netlib.ludovic.dev.version>3.0.3</netlib.ludovic.dev.version>
191191
<commons-codec.version>1.15</commons-codec.version>

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JSONOptions.scala

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ import java.nio.charset.{Charset, StandardCharsets}
2121
import java.time.ZoneId
2222
import java.util.Locale
2323

24-
import com.fasterxml.jackson.core.{JsonFactory, JsonFactoryBuilder}
24+
import com.fasterxml.jackson.core.{JsonFactory, JsonFactoryBuilder, StreamReadConstraints}
2525
import com.fasterxml.jackson.core.json.JsonReadFeature
2626

2727
import org.apache.spark.internal.Logging
@@ -43,6 +43,21 @@ private[sql] class JSONOptions(
4343

4444
import JSONOptions._
4545

46+
private val maxNestingDepth: Int = parameters
47+
.get("maxNestingDepth")
48+
.map(_.toInt)
49+
.getOrElse(StreamReadConstraints.DEFAULT_MAX_DEPTH)
50+
51+
private val maxNumLen: Int = parameters
52+
.get("maxNumLen")
53+
.map(_.toInt)
54+
.getOrElse(StreamReadConstraints.DEFAULT_MAX_NUM_LEN)
55+
56+
private val maxStringLen: Int = parameters
57+
.get("maxStringLen")
58+
.map(_.toInt)
59+
.getOrElse(StreamReadConstraints.DEFAULT_MAX_STRING_LEN)
60+
4661
def this(
4762
parameters: Map[String, String],
4863
defaultTimeZoneId: String,
@@ -176,6 +191,13 @@ private[sql] class JSONOptions(
176191

177192
/** Build a Jackson [[JsonFactory]] using JSON options. */
178193
def buildJsonFactory(): JsonFactory = {
194+
val streamReadConstraints = StreamReadConstraints
195+
.builder()
196+
.maxNestingDepth(maxNestingDepth)
197+
.maxNumberLength(maxNumLen)
198+
.maxStringLength(maxStringLen)
199+
.build()
200+
179201
new JsonFactoryBuilder()
180202
.configure(JsonReadFeature.ALLOW_JAVA_COMMENTS, allowComments)
181203
.configure(JsonReadFeature.ALLOW_UNQUOTED_FIELD_NAMES, allowUnquotedFieldNames)
@@ -186,6 +208,7 @@ private[sql] class JSONOptions(
186208
JsonReadFeature.ALLOW_BACKSLASH_ESCAPING_ANY_CHARACTER,
187209
allowBackslashEscapingAnyCharacter)
188210
.configure(JsonReadFeature.ALLOW_UNESCAPED_CONTROL_CHARS, allowUnquotedControlChars)
211+
.streamReadConstraints(streamReadConstraints)
189212
.build()
190213
}
191214
}

0 commit comments

Comments
 (0)