Re-enable tests

circle.yml

@@ -14,55 +14,19 @@ checkout:
 
 dependencies:
   override:
-    - ./build/mvn -DskipTests -Phadoop-2.7 -Pmesos -Pkinesis-asl -Pyarn -Phive-thriftserver -Phive dependency:go-offline
+    - ./build/mvn -Phadoop-2.7 -DskipTests -Pmesos -Pkinesis-asl -Pyarn -Phive-thriftserver -Phive dependency:go-offline
 
 test:
   override:
-    - ? |
-          set -euo pipefail
-          version=$(git describe --tags)
-
-          publish_artifacts() {
-            tmp_settings="tmp-settings.xml"
-            echo "<settings><servers><server>" > $tmp_settings
-            echo "<id>bintray-palantir-release</id><username>$BINTRAY_USERNAME</username>" >> $tmp_settings
-            echo "<password>$BINTRAY_PASSWORD</password>" >> $tmp_settings
-            echo "</server></servers></settings>" >> $tmp_settings
-
-            ./build/mvn versions:set -DnewVersion=$version
-            ./build/mvn --settings $tmp_settings -DskipTests -Phadoop-2.7 -Pmesos -Pkinesis-asl -Pyarn -Phive-thriftserver clean deploy
-          }
-
-          make_dist() {
-            dist_name="$1"
-            build_flags="$2"
-            file_name="spark-${version}-bin-${dist_name}.tgz"
-            sudo apt-get --assume-yes install r-base r-base-dev 
-            ./dev/make-distribution.sh --name $dist_name --tgz $build_flags
-            curl -u $BINTRAY_USERNAME:$BINTRAY_PASSWORD -T $file_name "https://api.bintray.com/content/palantir/releases/spark/${version}/org/apache/spark/dist/${file_name}"
-          }
-
-          case $CIRCLE_NODE_INDEX in
-          0)
-            publish_artifacts
-            ;;
-          1)
-            make_dist hadoop-2.7 "-Phadoop-2.7 -Pmesos -Pkinesis-asl -Pyarn -Phive-thriftserver -Phive"
-            ;;
-          2)
-            make_dist without-hadoop "-Psparkr -Phadoop-provided -Pyarn -Pmesos"
-            ;;
-          esac
-      :
-        parallel: true
-        timeout: 1200
+    - HADOOP_PROFILE="hadoop2.7" ./dev/run-tests
 
 deployment:
   release:
     tag: /[0-9]+(?:\.[0-9]+)+-palantir[0-9]+/
     commands:
-      - curl -u $BINTRAY_USERNAME:$BINTRAY_PASSWORD -X POST https://api.bintray.com/content/palantir/releases/spark/$(git describe --tags)/publish
+      - ./dev/publish-palantir.sh
+
   snapshot:
     branch: master
     commands:
-      - curl -u $BINTRAY_USERNAME:$BINTRAY_PASSWORD -X POST https://api.bintray.com/content/palantir/releases/spark/$(git describe --tags)/publish
+      - ./dev/publish-palantir.sh

dev/deps/spark-deps-hadoop-2.7

@@ -55,6 +55,7 @@ datanucleus-core-3.2.10.jar
 datanucleus-rdbms-3.2.9.jar
 derby-10.12.1.1.jar
 eigenbase-properties-1.1.5.jar
+fastutil-6.5.7.jar
 gson-2.2.4.jar
 guava-14.0.1.jar
 guice-3.0.jar
@@ -146,13 +147,13 @@ opencsv-2.3.jar
 oro-2.0.8.jar
 osgi-resource-locator-1.0.1.jar
 paranamer-2.3.jar
-parquet-column-1.8.1.jar
-parquet-common-1.8.1.jar
-parquet-encoding-1.8.1.jar
-parquet-format-2.3.0-incubating.jar
-parquet-hadoop-1.8.1.jar
+parquet-column-1.8.1-palantir7.jar
+parquet-common-1.8.1-palantir7.jar
+parquet-encoding-1.8.1-palantir7.jar
+parquet-format-2.3.1.jar
+parquet-hadoop-1.8.1-palantir7.jar
 parquet-hadoop-bundle-1.6.0.jar
-parquet-jackson-1.8.1.jar
+parquet-jackson-1.8.1-palantir7.jar
 pmml-model-1.2.15.jar
 pmml-schema-1.2.15.jar
 protobuf-java-2.5.0.jar

dev/publish-palantir.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -euo pipefail
+version=$(git describe --tags)
+
+make_dist() {
+  dist_name="$1"
+  build_flags="$2"
+  file_name="spark-${version}-bin-${dist_name}.tgz"
+  sudo apt-get --assume-yes install r-base r-base-dev
+  ./dev/make-distribution.sh --name $dist_name --tgz $build_flags
+  curl -u $BINTRAY_USERNAME:$BINTRAY_PASSWORD -T $file_name "https://api.bintray.com/content/palantir/releases/spark/${version}/org/apache/spark/dist/${file_name}"
+}
+
+publish_artifacts() {
+  tmp_settings="tmp-settings.xml"
+  echo "<settings><servers><server>" > $tmp_settings
+  echo "<id>bintray-palantir-release</id><username>$BINTRAY_USERNAME</username>" >> $tmp_settings
+  echo "<password>$BINTRAY_PASSWORD</password>" >> $tmp_settings
+  echo "</server></servers></settings>" >> $tmp_settings
+
+  ./build/mvn versions:set -DnewVersion=$version
+  ./build/mvn --settings $tmp_settings -DskipTests -Phadoop-2.7 -Pmesos -Pkinesis-asl -Pyarn -Phive-thriftserver clean deploy
+}
+
+make_dist hadoop-2.7 "-Phadoop-2.7 -Pmesos -Pkinesis-asl -Pyarn -Phive-thriftserver -Phive"
+make_dist without-hadoop "-Psparkr -Phadoop-provided -Pyarn -Pmesos"
+
+publish_artifacts
+
+curl -u $BINTRAY_USERNAME:$BINTRAY_PASSWORD -X POST https://api.bintray.com/content/palantir/releases/spark/$(git describe --tags)/publish