Parallelize tests across circle containers

assembly/pom.xml

@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.1.0-SNAPSHOT</version>
+    <version>2.2.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

circle.yml

@@ -3,6 +3,12 @@ machine:
     version: oraclejdk8
   environment:
     TERM: dumb
+  post:
+   - pyenv global 2.7.11 3.4.4 pypy-4.0.1
+   # turn off circle's bundled services to use less memory
+   - sudo service postgresql stop
+   - sudo service mongod stop
+
 
 checkout:
   post:
@@ -18,6 +24,9 @@ dependencies:
 
 test:
   override:
+    - HADOOP_PROFILE=hadoop2.7 ./dev/run-tests --container $CIRCLE_NODE_INDEX --total-containers $CIRCLE_NODE_TOTAL:
+        parallel: true
+        timeout: 1200
     - ? |
           set -euo pipefail
           version=$(git describe --tags)

common/network-common/pom.xml

@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.1.0-SNAPSHOT</version>
+    <version>2.2.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

common/network-shuffle/pom.xml

@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.1.0-SNAPSHOT</version>
+    <version>2.2.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

common/network-yarn/pom.xml

@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.1.0-SNAPSHOT</version>
+    <version>2.2.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

common/sketch/pom.xml

@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.1.0-SNAPSHOT</version>
+    <version>2.2.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

common/tags/pom.xml

@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.1.0-SNAPSHOT</version>
+    <version>2.2.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

common/unsafe/pom.xml

@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.1.0-SNAPSHOT</version>
+    <version>2.2.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

common/unsafe/src/main/java/org/apache/spark/unsafe/types/CalendarInterval.java

@@ -252,7 +252,7 @@ public static long parseSecondNano(String secondNano) throws IllegalArgumentExce
   public final int months;
   public final long microseconds;
 
-  public final long milliseconds() {
+  public long milliseconds() {
     return this.microseconds / MICROS_PER_MILLI;
   }
 

core/pom.xml

@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.1.0-SNAPSHOT</version>
+    <version>2.2.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

dev/deps/spark-deps-hadoop-2.7

@@ -146,13 +146,13 @@ opencsv-2.3.jar
 oro-2.0.8.jar
 osgi-resource-locator-1.0.1.jar
 paranamer-2.3.jar
-parquet-column-1.9.0-palantir2.jar
-parquet-common-1.9.0-palantir2.jar
-parquet-encoding-1.9.0-palantir2.jar
+parquet-column-1.9.0-palantir4.jar
+parquet-common-1.9.0-palantir4.jar
+parquet-encoding-1.9.0-palantir4.jar
 parquet-format-2.3.1.jar
-parquet-hadoop-1.9.0-palantir2.jar
+parquet-hadoop-1.9.0-palantir4.jar
 parquet-hadoop-bundle-1.6.0.jar
-parquet-jackson-1.9.0-palantir2.jar
+parquet-jackson-1.9.0-palantir4.jar
 pmml-model-1.2.15.jar
 pmml-schema-1.2.15.jar
 protobuf-java-2.5.0.jar

dev/run-tests.py

@@ -102,10 +102,10 @@ def determine_modules_to_test(changed_modules):
 
     Returns a topologically-sorted list of modules (ties are broken by sorting on module names).
 
-    >>> [x.name for x in determine_modules_to_test([modules.root])]
-    ['root']
-    >>> [x.name for x in determine_modules_to_test([modules.build])]
-    ['root']
+    #>>> [x.name for x in determine_modules_to_test([modules.root])]
+    #['root']
+    #>>> [x.name for x in determine_modules_to_test([modules.build])]
+    #['root']
     >>> [x.name for x in determine_modules_to_test([modules.graphx])]
     ['graphx', 'examples']
     >>> x = [x.name for x in determine_modules_to_test([modules.sql])]
@@ -117,9 +117,13 @@ def determine_modules_to_test(changed_modules):
     for module in changed_modules:
         modules_to_test = modules_to_test.union(determine_modules_to_test(module.dependent_modules))
     modules_to_test = modules_to_test.union(set(changed_modules))
-    # If we need to run all of the tests, then we should short-circuit and return 'root'
+
+    # want to return a fine-grained list of modules rather than coalescing to
+    # modules.root for better container parallelism
     if modules.root in modules_to_test:
-        return [modules.root]
+        if len(modules_to_test) == 1:
+            modules_to_test = modules.all_modules
+        modules_to_test = [m for m in modules_to_test if m != modules.root]
     return toposort_flatten(
         {m: set(m.dependencies).intersection(modules_to_test) for m in modules_to_test}, sort=True)
 
@@ -403,12 +407,19 @@ def run_scala_tests_sbt(test_modules, test_profiles):
     exec_sbt(profiles_and_goals)
 
 
-def run_scala_tests(build_tool, hadoop_version, test_modules, excluded_tags):
+def run_scala_tests(build_tool, hadoop_version, test_modules, excluded_tags,
+                    container, total_containers):
     """Function to properly execute all tests passed in as a set from the
     `determine_test_suites` function"""
     set_title_and_block("Running Spark unit tests", "BLOCK_SPARK_UNIT_TESTS")
 
-    test_modules = set(test_modules)
+    this_containers_test_modules = [m for (i, m) in enumerate(test_modules) if
+                                    i % total_containers == container]
+
+    print("[info] Running tests for container index %d of %d on modules %s of all modules %s"
+          % (container, total_containers, this_containers_test_modules, test_modules))
+
+    test_modules = set(this_containers_test_modules)
 
     test_profiles = get_hadoop_profiles(hadoop_version) + \
         list(set(itertools.chain.from_iterable(m.build_profile_flags for m in test_modules)))
@@ -461,12 +472,29 @@ def parse_opts():
         "-p", "--parallelism", type="int", default=4,
         help="The number of suites to test in parallel (default %default)"
     )
+    parser.add_option(
+        "--container", type="int", default=0,
+        help="The index of this container, starting from 0 (default %default)"
+    )
+    parser.add_option(
+        "--total-containers", type="int", default=1,
+        help="The count of test containers (default %default)"
+    )
 
     (opts, args) = parser.parse_args()
     if args:
         parser.error("Unsupported arguments: %s" % ' '.join(args))
     if opts.parallelism < 1:
         parser.error("Parallelism cannot be less than 1")
+    if opts.container < 0:
+        parser.error("Container index %d cannot be less than 0" %
+                     opts.container)
+    if opts.total_containers < 1:
+        parser.error("Must have at least one container, found %d" %
+                     opts.total_containers)
+    if opts.container >= opts.total_containers:
+        parser.error("Container index %d cannot be at or above total_containers %d" %
+                     (opts.container, opts.total_containers))
     return opts
 
 
@@ -529,7 +557,7 @@ def main():
         changed_modules = determine_modules_for_files(changed_files)
         excluded_tags = determine_tags_to_exclude(changed_modules)
     if not changed_modules:
-        changed_modules = [modules.root]
+        changed_modules = modules.all_modules
         excluded_tags = []
     print("[info] Found the following changed modules:",
           ", ".join(x.name for x in changed_modules))
@@ -584,14 +612,16 @@ def main():
         build_spark_assembly_sbt(hadoop_version)
 
     # run the test suites
-    run_scala_tests(build_tool, hadoop_version, test_modules, excluded_tags)
-
-    modules_with_python_tests = [m for m in test_modules if m.python_test_goals]
-    if modules_with_python_tests:
-        run_python_tests(modules_with_python_tests, opts.parallelism)
-        run_python_packaging_tests()
-    if any(m.should_run_r_tests for m in test_modules):
-        run_sparkr_tests()
+    run_scala_tests(build_tool, hadoop_version, test_modules, excluded_tags,
+                    opts.container, opts.total_containers)
+
+    # TODO: re-enable Python and R tests
+    # modules_with_python_tests = [m for m in test_modules if m.python_test_goals]
+    # if modules_with_python_tests:
+    #     run_python_tests(modules_with_python_tests, opts.parallelism)
+    #     run_python_packaging_tests()
+    # if any(m.should_run_r_tests for m in test_modules):
+    #     run_sparkr_tests()
 
 
 def _test():

docs/_config.yml

@@ -14,8 +14,8 @@ include:
 
 # These allow the documentation to be updated with newer releases
 # of Spark, Scala, and Mesos.
-SPARK_VERSION: 2.1.0-SNAPSHOT
-SPARK_VERSION_SHORT: 2.1.0
+SPARK_VERSION: 2.2.0-SNAPSHOT
+SPARK_VERSION_SHORT: 2.2.0
 SCALA_BINARY_VERSION: "2.11"
 SCALA_VERSION: "2.11.7"
 MESOS_VERSION: 1.0.0

examples/pom.xml

@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.1.0-SNAPSHOT</version>
+    <version>2.2.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

external/docker-integration-tests/pom.xml

@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.1.0-SNAPSHOT</version>
+    <version>2.2.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

external/flume-assembly/pom.xml

@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.1.0-SNAPSHOT</version>
+    <version>2.2.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

external/flume-sink/pom.xml

@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.1.0-SNAPSHOT</version>
+    <version>2.2.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

external/flume/pom.xml

@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.1.0-SNAPSHOT</version>
+    <version>2.2.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

external/java8-tests/pom.xml

@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.1.0-SNAPSHOT</version>
+    <version>2.2.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

external/kafka-0-10-assembly/pom.xml

@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.1.0-SNAPSHOT</version>
+    <version>2.2.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

external/kafka-0-10-sql/pom.xml

@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.1.0-SNAPSHOT</version>
+    <version>2.2.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

external/kafka-0-10/pom.xml

@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.1.0-SNAPSHOT</version>
+    <version>2.2.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

external/kafka-0-8-assembly/pom.xml

@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.1.0-SNAPSHOT</version>
+    <version>2.2.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

external/kafka-0-8/pom.xml

@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.1.0-SNAPSHOT</version>
+    <version>2.2.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

external/kinesis-asl-assembly/pom.xml

@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.1.0-SNAPSHOT</version>
+    <version>2.2.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

external/kinesis-asl/pom.xml

@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.1.0-SNAPSHOT</version>
+    <version>2.2.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

external/spark-ganglia-lgpl/pom.xml

@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.1.0-SNAPSHOT</version>
+    <version>2.2.0-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

graphx/pom.xml

@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.1.0-SNAPSHOT</version>
+    <version>2.2.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

launcher/pom.xml

@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.1.0-SNAPSHOT</version>
+    <version>2.2.0-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>