diff --git a/assembly/pom.xml b/assembly/pom.xml
index ec243eaebaea..53f18796e601 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -21,7 +21,7 @@
org.apache.spark
spark-parent_2.11
- 2.1.0-SNAPSHOT
+ 2.2.0-SNAPSHOT
../pom.xml
diff --git a/circle.yml b/circle.yml
index 36d076002bd7..53f7c3172a96 100644
--- a/circle.yml
+++ b/circle.yml
@@ -3,6 +3,12 @@ machine:
version: oraclejdk8
environment:
TERM: dumb
+ post:
+ - pyenv global 2.7.11 3.4.4 pypy-4.0.1
+ # turn off circle's bundled services to use less memory
+ - sudo service postgresql stop
+ - sudo service mongod stop
+
checkout:
post:
@@ -18,6 +24,9 @@ dependencies:
test:
override:
+ - HADOOP_PROFILE=hadoop2.7 ./dev/run-tests --container $CIRCLE_NODE_INDEX --total-containers $CIRCLE_NODE_TOTAL:
+ parallel: true
+ timeout: 1200
- ? |
set -euo pipefail
version=$(git describe --tags)
diff --git a/common/network-common/pom.xml b/common/network-common/pom.xml
index ca99fa89ebe1..b63c0ca17332 100644
--- a/common/network-common/pom.xml
+++ b/common/network-common/pom.xml
@@ -22,7 +22,7 @@
org.apache.spark
spark-parent_2.11
- 2.1.0-SNAPSHOT
+ 2.2.0-SNAPSHOT
../../pom.xml
diff --git a/common/network-shuffle/pom.xml b/common/network-shuffle/pom.xml
index 511e1f29de36..5fc92afccbfb 100644
--- a/common/network-shuffle/pom.xml
+++ b/common/network-shuffle/pom.xml
@@ -22,7 +22,7 @@
org.apache.spark
spark-parent_2.11
- 2.1.0-SNAPSHOT
+ 2.2.0-SNAPSHOT
../../pom.xml
diff --git a/common/network-yarn/pom.xml b/common/network-yarn/pom.xml
index 606ad1573961..9fcd636c8882 100644
--- a/common/network-yarn/pom.xml
+++ b/common/network-yarn/pom.xml
@@ -22,7 +22,7 @@
org.apache.spark
spark-parent_2.11
- 2.1.0-SNAPSHOT
+ 2.2.0-SNAPSHOT
../../pom.xml
diff --git a/common/sketch/pom.xml b/common/sketch/pom.xml
index 626f023a5b99..7f57108584c2 100644
--- a/common/sketch/pom.xml
+++ b/common/sketch/pom.xml
@@ -22,7 +22,7 @@
org.apache.spark
spark-parent_2.11
- 2.1.0-SNAPSHOT
+ 2.2.0-SNAPSHOT
../../pom.xml
diff --git a/common/tags/pom.xml b/common/tags/pom.xml
index 1c60d510e570..81191cb8f4a5 100644
--- a/common/tags/pom.xml
+++ b/common/tags/pom.xml
@@ -22,7 +22,7 @@
org.apache.spark
spark-parent_2.11
- 2.1.0-SNAPSHOT
+ 2.2.0-SNAPSHOT
../../pom.xml
diff --git a/common/unsafe/pom.xml b/common/unsafe/pom.xml
index 45af98d94ef9..d00bddf5515c 100644
--- a/common/unsafe/pom.xml
+++ b/common/unsafe/pom.xml
@@ -22,7 +22,7 @@
org.apache.spark
spark-parent_2.11
- 2.1.0-SNAPSHOT
+ 2.2.0-SNAPSHOT
../../pom.xml
diff --git a/common/unsafe/src/main/java/org/apache/spark/unsafe/types/CalendarInterval.java b/common/unsafe/src/main/java/org/apache/spark/unsafe/types/CalendarInterval.java
index a7b0e6f80c2b..fd6e95c3e0a3 100644
--- a/common/unsafe/src/main/java/org/apache/spark/unsafe/types/CalendarInterval.java
+++ b/common/unsafe/src/main/java/org/apache/spark/unsafe/types/CalendarInterval.java
@@ -252,7 +252,7 @@ public static long parseSecondNano(String secondNano) throws IllegalArgumentExce
public final int months;
public final long microseconds;
- public final long milliseconds() {
+ public long milliseconds() {
return this.microseconds / MICROS_PER_MILLI;
}
diff --git a/core/pom.xml b/core/pom.xml
index eac99ab82a2e..8ee88cc730ab 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -21,7 +21,7 @@
org.apache.spark
spark-parent_2.11
- 2.1.0-SNAPSHOT
+ 2.2.0-SNAPSHOT
../pom.xml
diff --git a/dev/deps/spark-deps-hadoop-2.7 b/dev/deps/spark-deps-hadoop-2.7
index c122043f70b7..e0ae2e311210 100644
--- a/dev/deps/spark-deps-hadoop-2.7
+++ b/dev/deps/spark-deps-hadoop-2.7
@@ -146,13 +146,13 @@ opencsv-2.3.jar
oro-2.0.8.jar
osgi-resource-locator-1.0.1.jar
paranamer-2.3.jar
-parquet-column-1.9.0-palantir2.jar
-parquet-common-1.9.0-palantir2.jar
-parquet-encoding-1.9.0-palantir2.jar
+parquet-column-1.9.0-palantir4.jar
+parquet-common-1.9.0-palantir4.jar
+parquet-encoding-1.9.0-palantir4.jar
parquet-format-2.3.1.jar
-parquet-hadoop-1.9.0-palantir2.jar
+parquet-hadoop-1.9.0-palantir4.jar
parquet-hadoop-bundle-1.6.0.jar
-parquet-jackson-1.9.0-palantir2.jar
+parquet-jackson-1.9.0-palantir4.jar
pmml-model-1.2.15.jar
pmml-schema-1.2.15.jar
protobuf-java-2.5.0.jar
diff --git a/dev/run-tests.py b/dev/run-tests.py
index ab285ac96af7..8f07f2b6f1e1 100755
--- a/dev/run-tests.py
+++ b/dev/run-tests.py
@@ -102,10 +102,10 @@ def determine_modules_to_test(changed_modules):
Returns a topologically-sorted list of modules (ties are broken by sorting on module names).
- >>> [x.name for x in determine_modules_to_test([modules.root])]
- ['root']
- >>> [x.name for x in determine_modules_to_test([modules.build])]
- ['root']
+ #>>> [x.name for x in determine_modules_to_test([modules.root])]
+ #['root']
+ #>>> [x.name for x in determine_modules_to_test([modules.build])]
+ #['root']
>>> [x.name for x in determine_modules_to_test([modules.graphx])]
['graphx', 'examples']
>>> x = [x.name for x in determine_modules_to_test([modules.sql])]
@@ -117,9 +117,13 @@ def determine_modules_to_test(changed_modules):
for module in changed_modules:
modules_to_test = modules_to_test.union(determine_modules_to_test(module.dependent_modules))
modules_to_test = modules_to_test.union(set(changed_modules))
- # If we need to run all of the tests, then we should short-circuit and return 'root'
+
+ # want to return a fine-grained list of modules rather than coalescing to
+ # modules.root for better container parallelism
if modules.root in modules_to_test:
- return [modules.root]
+ if len(modules_to_test) == 1:
+ modules_to_test = modules.all_modules
+ modules_to_test = [m for m in modules_to_test if m != modules.root]
return toposort_flatten(
{m: set(m.dependencies).intersection(modules_to_test) for m in modules_to_test}, sort=True)
@@ -403,12 +407,19 @@ def run_scala_tests_sbt(test_modules, test_profiles):
exec_sbt(profiles_and_goals)
-def run_scala_tests(build_tool, hadoop_version, test_modules, excluded_tags):
+def run_scala_tests(build_tool, hadoop_version, test_modules, excluded_tags,
+ container, total_containers):
"""Function to properly execute all tests passed in as a set from the
`determine_test_suites` function"""
set_title_and_block("Running Spark unit tests", "BLOCK_SPARK_UNIT_TESTS")
- test_modules = set(test_modules)
+ this_containers_test_modules = [m for (i, m) in enumerate(test_modules) if
+ i % total_containers == container]
+
+ print("[info] Running tests for container index %d of %d on modules %s of all modules %s"
+ % (container, total_containers, this_containers_test_modules, test_modules))
+
+ test_modules = set(this_containers_test_modules)
test_profiles = get_hadoop_profiles(hadoop_version) + \
list(set(itertools.chain.from_iterable(m.build_profile_flags for m in test_modules)))
@@ -461,12 +472,29 @@ def parse_opts():
"-p", "--parallelism", type="int", default=4,
help="The number of suites to test in parallel (default %default)"
)
+ parser.add_option(
+ "--container", type="int", default=0,
+ help="The index of this container, starting from 0 (default %default)"
+ )
+ parser.add_option(
+ "--total-containers", type="int", default=1,
+ help="The count of test containers (default %default)"
+ )
(opts, args) = parser.parse_args()
if args:
parser.error("Unsupported arguments: %s" % ' '.join(args))
if opts.parallelism < 1:
parser.error("Parallelism cannot be less than 1")
+ if opts.container < 0:
+ parser.error("Container index %d cannot be less than 0" %
+ opts.container)
+ if opts.total_containers < 1:
+ parser.error("Must have at least one container, found %d" %
+ opts.total_containers)
+ if opts.container >= opts.total_containers:
+ parser.error("Container index %d cannot be at or above total_containers %d" %
+ (opts.container, opts.total_containers))
return opts
@@ -529,7 +557,7 @@ def main():
changed_modules = determine_modules_for_files(changed_files)
excluded_tags = determine_tags_to_exclude(changed_modules)
if not changed_modules:
- changed_modules = [modules.root]
+ changed_modules = modules.all_modules
excluded_tags = []
print("[info] Found the following changed modules:",
", ".join(x.name for x in changed_modules))
@@ -584,14 +612,16 @@ def main():
build_spark_assembly_sbt(hadoop_version)
# run the test suites
- run_scala_tests(build_tool, hadoop_version, test_modules, excluded_tags)
-
- modules_with_python_tests = [m for m in test_modules if m.python_test_goals]
- if modules_with_python_tests:
- run_python_tests(modules_with_python_tests, opts.parallelism)
- run_python_packaging_tests()
- if any(m.should_run_r_tests for m in test_modules):
- run_sparkr_tests()
+ run_scala_tests(build_tool, hadoop_version, test_modules, excluded_tags,
+ opts.container, opts.total_containers)
+
+ # TODO: re-enable Python and R tests
+ # modules_with_python_tests = [m for m in test_modules if m.python_test_goals]
+ # if modules_with_python_tests:
+ # run_python_tests(modules_with_python_tests, opts.parallelism)
+ # run_python_packaging_tests()
+ # if any(m.should_run_r_tests for m in test_modules):
+ # run_sparkr_tests()
def _test():
diff --git a/docs/_config.yml b/docs/_config.yml
index e4fc093fe733..83bb30598d15 100644
--- a/docs/_config.yml
+++ b/docs/_config.yml
@@ -14,8 +14,8 @@ include:
# These allow the documentation to be updated with newer releases
# of Spark, Scala, and Mesos.
-SPARK_VERSION: 2.1.0-SNAPSHOT
-SPARK_VERSION_SHORT: 2.1.0
+SPARK_VERSION: 2.2.0-SNAPSHOT
+SPARK_VERSION_SHORT: 2.2.0
SCALA_BINARY_VERSION: "2.11"
SCALA_VERSION: "2.11.7"
MESOS_VERSION: 1.0.0
diff --git a/examples/pom.xml b/examples/pom.xml
index 8d15b42a48e6..5a869d9ed921 100644
--- a/examples/pom.xml
+++ b/examples/pom.xml
@@ -21,7 +21,7 @@
org.apache.spark
spark-parent_2.11
- 2.1.0-SNAPSHOT
+ 2.2.0-SNAPSHOT
../pom.xml
diff --git a/external/docker-integration-tests/pom.xml b/external/docker-integration-tests/pom.xml
index 57d553b75b87..6741932f3ddb 100644
--- a/external/docker-integration-tests/pom.xml
+++ b/external/docker-integration-tests/pom.xml
@@ -22,7 +22,7 @@
org.apache.spark
spark-parent_2.11
- 2.1.0-SNAPSHOT
+ 2.2.0-SNAPSHOT
../../pom.xml
diff --git a/external/flume-assembly/pom.xml b/external/flume-assembly/pom.xml
index fb0292a5f11e..f8ef8a991316 100644
--- a/external/flume-assembly/pom.xml
+++ b/external/flume-assembly/pom.xml
@@ -21,7 +21,7 @@
org.apache.spark
spark-parent_2.11
- 2.1.0-SNAPSHOT
+ 2.2.0-SNAPSHOT
../../pom.xml
diff --git a/external/flume-sink/pom.xml b/external/flume-sink/pom.xml
index 5e9275c8e66d..83619ccf258c 100644
--- a/external/flume-sink/pom.xml
+++ b/external/flume-sink/pom.xml
@@ -21,7 +21,7 @@
org.apache.spark
spark-parent_2.11
- 2.1.0-SNAPSHOT
+ 2.2.0-SNAPSHOT
../../pom.xml
diff --git a/external/flume/pom.xml b/external/flume/pom.xml
index 7b68ca7373fe..4a40f8ba28b7 100644
--- a/external/flume/pom.xml
+++ b/external/flume/pom.xml
@@ -21,7 +21,7 @@
org.apache.spark
spark-parent_2.11
- 2.1.0-SNAPSHOT
+ 2.2.0-SNAPSHOT
../../pom.xml
diff --git a/external/java8-tests/pom.xml b/external/java8-tests/pom.xml
index 1bc206e8675f..98467f2cc5c2 100644
--- a/external/java8-tests/pom.xml
+++ b/external/java8-tests/pom.xml
@@ -20,7 +20,7 @@
org.apache.spark
spark-parent_2.11
- 2.1.0-SNAPSHOT
+ 2.2.0-SNAPSHOT
../../pom.xml
diff --git a/external/kafka-0-10-assembly/pom.xml b/external/kafka-0-10-assembly/pom.xml
index 4f5045326a00..295142cbfdff 100644
--- a/external/kafka-0-10-assembly/pom.xml
+++ b/external/kafka-0-10-assembly/pom.xml
@@ -21,7 +21,7 @@
org.apache.spark
spark-parent_2.11
- 2.1.0-SNAPSHOT
+ 2.2.0-SNAPSHOT
../../pom.xml
diff --git a/external/kafka-0-10-sql/pom.xml b/external/kafka-0-10-sql/pom.xml
index ebff5fd07a9b..29abd88fcff2 100644
--- a/external/kafka-0-10-sql/pom.xml
+++ b/external/kafka-0-10-sql/pom.xml
@@ -21,7 +21,7 @@
org.apache.spark
spark-parent_2.11
- 2.1.0-SNAPSHOT
+ 2.2.0-SNAPSHOT
../../pom.xml
diff --git a/external/kafka-0-10/pom.xml b/external/kafka-0-10/pom.xml
index c36d47900709..b9ab57624c74 100644
--- a/external/kafka-0-10/pom.xml
+++ b/external/kafka-0-10/pom.xml
@@ -21,7 +21,7 @@
org.apache.spark
spark-parent_2.11
- 2.1.0-SNAPSHOT
+ 2.2.0-SNAPSHOT
../../pom.xml
diff --git a/external/kafka-0-8-assembly/pom.xml b/external/kafka-0-8-assembly/pom.xml
index bc02b8a66246..3fedd9eda195 100644
--- a/external/kafka-0-8-assembly/pom.xml
+++ b/external/kafka-0-8-assembly/pom.xml
@@ -21,7 +21,7 @@
org.apache.spark
spark-parent_2.11
- 2.1.0-SNAPSHOT
+ 2.2.0-SNAPSHOT
../../pom.xml
diff --git a/external/kafka-0-8/pom.xml b/external/kafka-0-8/pom.xml
index 91ccd4a927e9..bdb3b4d2f56e 100644
--- a/external/kafka-0-8/pom.xml
+++ b/external/kafka-0-8/pom.xml
@@ -21,7 +21,7 @@
org.apache.spark
spark-parent_2.11
- 2.1.0-SNAPSHOT
+ 2.2.0-SNAPSHOT
../../pom.xml
diff --git a/external/kinesis-asl-assembly/pom.xml b/external/kinesis-asl-assembly/pom.xml
index f7cb76446339..90bb0e4987c8 100644
--- a/external/kinesis-asl-assembly/pom.xml
+++ b/external/kinesis-asl-assembly/pom.xml
@@ -21,7 +21,7 @@
org.apache.spark
spark-parent_2.11
- 2.1.0-SNAPSHOT
+ 2.2.0-SNAPSHOT
../../pom.xml
diff --git a/external/kinesis-asl/pom.xml b/external/kinesis-asl/pom.xml
index 57809ff692c2..32ea114856d8 100644
--- a/external/kinesis-asl/pom.xml
+++ b/external/kinesis-asl/pom.xml
@@ -20,7 +20,7 @@
org.apache.spark
spark-parent_2.11
- 2.1.0-SNAPSHOT
+ 2.2.0-SNAPSHOT
../../pom.xml
diff --git a/external/spark-ganglia-lgpl/pom.xml b/external/spark-ganglia-lgpl/pom.xml
index fab409d3e9f9..7da27817ebaf 100644
--- a/external/spark-ganglia-lgpl/pom.xml
+++ b/external/spark-ganglia-lgpl/pom.xml
@@ -20,7 +20,7 @@
org.apache.spark
spark-parent_2.11
- 2.1.0-SNAPSHOT
+ 2.2.0-SNAPSHOT
../../pom.xml
diff --git a/graphx/pom.xml b/graphx/pom.xml
index 10d5ba93ebb8..a9f7839917f3 100644
--- a/graphx/pom.xml
+++ b/graphx/pom.xml
@@ -21,7 +21,7 @@
org.apache.spark
spark-parent_2.11
- 2.1.0-SNAPSHOT
+ 2.2.0-SNAPSHOT
../pom.xml
diff --git a/launcher/pom.xml b/launcher/pom.xml
index 6023cf077186..899957f62778 100644
--- a/launcher/pom.xml
+++ b/launcher/pom.xml
@@ -22,7 +22,7 @@
org.apache.spark
spark-parent_2.11
- 2.1.0-SNAPSHOT
+ 2.2.0-SNAPSHOT
../pom.xml
diff --git a/mesos/pom.xml b/mesos/pom.xml
index 57cc26a4ccef..60f23596bcc7 100644
--- a/mesos/pom.xml
+++ b/mesos/pom.xml
@@ -20,7 +20,7 @@
org.apache.spark
spark-parent_2.11
- 2.1.0-SNAPSHOT
+ 2.2.0-SNAPSHOT
../pom.xml
diff --git a/mllib-local/pom.xml b/mllib-local/pom.xml
index 8c985fd13ac0..201446efb9ab 100644
--- a/mllib-local/pom.xml
+++ b/mllib-local/pom.xml
@@ -21,7 +21,7 @@
org.apache.spark
spark-parent_2.11
- 2.1.0-SNAPSHOT
+ 2.2.0-SNAPSHOT
../pom.xml
diff --git a/mllib/pom.xml b/mllib/pom.xml
index 4484998a49c8..aad69742770a 100644
--- a/mllib/pom.xml
+++ b/mllib/pom.xml
@@ -21,7 +21,7 @@
org.apache.spark
spark-parent_2.11
- 2.1.0-SNAPSHOT
+ 2.2.0-SNAPSHOT
../pom.xml
diff --git a/pom.xml b/pom.xml
index 9a4fe4c3ca97..d87df4726fe0 100644
--- a/pom.xml
+++ b/pom.xml
@@ -26,7 +26,7 @@
org.apache.spark
spark-parent_2.11
- 2.1.0-SNAPSHOT
+ 2.2.0-SNAPSHOT
pom
Spark Project Parent POM
http://spark.apache.org/
diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala
index b113bbf803d9..7fed8cb0080c 100644
--- a/project/MimaExcludes.scala
+++ b/project/MimaExcludes.scala
@@ -34,6 +34,7 @@ import com.typesafe.tools.mima.core.ProblemFilters._
*/
object MimaExcludes {
+ // Exclude rules for 2.2.x
lazy val v22excludes = v21excludes ++ Seq(
// [SPARK-18663][SQL] Simplify CountMinSketch aggregate implementation
ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.util.sketch.CountMinSketch.toByteArray")
@@ -918,7 +919,7 @@ object MimaExcludes {
def excludes(version: String) = version match {
case v if v.startsWith("2.2") => v22excludes
- case v if v.startsWith("2.1") => v22excludes // TODO: Update this when we bump version to 2.2
+ case v if v.startsWith("2.1") => v21excludes
case v if v.startsWith("2.0") => v20excludes
case _ => Seq()
}
diff --git a/repl/pom.xml b/repl/pom.xml
index 73493e600e54..1baf1340cfb3 100644
--- a/repl/pom.xml
+++ b/repl/pom.xml
@@ -21,7 +21,7 @@
org.apache.spark
spark-parent_2.11
- 2.1.0-SNAPSHOT
+ 2.2.0-SNAPSHOT
../pom.xml
diff --git a/sql/catalyst/pom.xml b/sql/catalyst/pom.xml
index 82a5a8531761..ba5b5c965f6d 100644
--- a/sql/catalyst/pom.xml
+++ b/sql/catalyst/pom.xml
@@ -22,7 +22,7 @@
org.apache.spark
spark-parent_2.11
- 2.1.0-SNAPSHOT
+ 2.2.0-SNAPSHOT
../../pom.xml
diff --git a/sql/core/pom.xml b/sql/core/pom.xml
index 7da77158ff07..4178b0a1fcf1 100644
--- a/sql/core/pom.xml
+++ b/sql/core/pom.xml
@@ -22,7 +22,7 @@
org.apache.spark
spark-parent_2.11
- 2.1.0-SNAPSHOT
+ 2.2.0-SNAPSHOT
../../pom.xml
diff --git a/sql/hive-thriftserver/pom.xml b/sql/hive-thriftserver/pom.xml
index 819897cd4685..7a6e966f960d 100644
--- a/sql/hive-thriftserver/pom.xml
+++ b/sql/hive-thriftserver/pom.xml
@@ -22,7 +22,7 @@
org.apache.spark
spark-parent_2.11
- 2.1.0-SNAPSHOT
+ 2.2.0-SNAPSHOT
../../pom.xml
diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml
index 59f26e1660ed..c6333b048a8f 100644
--- a/sql/hive/pom.xml
+++ b/sql/hive/pom.xml
@@ -22,7 +22,7 @@
org.apache.spark
spark-parent_2.11
- 2.1.0-SNAPSHOT
+ 2.2.0-SNAPSHOT
../../pom.xml
diff --git a/streaming/pom.xml b/streaming/pom.xml
index 07a0dab0ee04..7168999898b1 100644
--- a/streaming/pom.xml
+++ b/streaming/pom.xml
@@ -21,7 +21,7 @@
org.apache.spark
spark-parent_2.11
- 2.1.0-SNAPSHOT
+ 2.2.0-SNAPSHOT
../pom.xml
diff --git a/tools/pom.xml b/tools/pom.xml
index b9be8db684a9..938ba2f6ac20 100644
--- a/tools/pom.xml
+++ b/tools/pom.xml
@@ -20,7 +20,7 @@
org.apache.spark
spark-parent_2.11
- 2.1.0-SNAPSHOT
+ 2.2.0-SNAPSHOT
../pom.xml
diff --git a/yarn/pom.xml b/yarn/pom.xml
index 64ff845b5ae9..91e3437749c3 100644
--- a/yarn/pom.xml
+++ b/yarn/pom.xml
@@ -20,7 +20,7 @@
org.apache.spark
spark-parent_2.11
- 2.1.0-SNAPSHOT
+ 2.2.0-SNAPSHOT
../pom.xml