sbt build (apache#34)

Author: robert3005

circle.yml

@@ -1,21 +1,9 @@
-general:
-  artifacts:
-    - target
-    - binary-release-hadoop2.7.log
-    - binary-release-without-hadoop.log
-
 machine:
-  pre:
-    - sudo pkill -u postgres
   java:
     version: oraclejdk8
   environment:
     TERM: dumb
-    HADOOP_PROFILE: "hadoop2.7"
-  post:
-    - sudo pkill mongod || true
-    - sudo pkill mysqld || true
-    - sudo pkill postgres || true
+    HADOOP_PROFILE: "hadoop-2.7"
 
 checkout:
   post:
@@ -31,28 +19,27 @@ dependencies:
 
 test:
   override:
-    - /bin/true
-    #- ./build/sbt -Phadoop-2.7 -Pmesos -Pkinesis-asl -Pyarn -Phive-thriftserver -Phive package
+    - ./build/sbt -Phadoop-2.7 -Pmesos -Pkinesis-asl -Pyarn -Phive-thriftserver -Phive package
     #- ./dev/run-tests -p 1
 
 deployment:
   release:
     tag: /[0-9]+(?:\.[0-9]+)+-palantir[0-9]+/
     commands:
       - ./build/sbt -Phadoop-2.7 -Pmesos -Pkinesis-asl -Pyarn -Phive-thriftserver -Phive publish
-      - ./dev/make-distribution.sh --name hadoop2.7 --tgz "-Phadoop2.7 -Psparkr -Phive -Phive-thriftserver -Pyarn -Pmesos" 2>&1 > binary-release-hadoop2.7.log
-      - curl -u $BINTRAY_USERNAME:$BINTRAY_PASSWORD -T "spark-$(git describe --tags)-bin-hadoop2.7.tgz" "https://api.bintray.com/content/palantir/releases/spark/$(git describe --tags)/org/apache/spark/dist/spark-$(git describe --tags)-bin-hadoop2.7.tgz"
+      - ./dev/make-distribution.sh --name hadoop-2.7 --tgz -Phadoop-2.7 -Psparkr -Phive -Phive-thriftserver -Pyarn -Pmesos
+      - curl -u $BINTRAY_USERNAME:$BINTRAY_PASSWORD -T "spark-$(git describe --tags)-bin-hadoop-2.7.tgz" "https://api.bintray.com/content/palantir/releases/spark/$(git describe --tags)/org/apache/spark/dist/spark-$(git describe --tags)-bin-hadoop2.7.tgz"
       - ./build/sbt -Phadoop-2.7 -Pmesos -Pkinesis-asl -Pyarn -Phive-thriftserver -Phive clean
-      - ./dev/make-distribution.sh --name without-hadoop --tgz "-Psparkr -Phadoop-provided -Pyarn -Pmesos" 2>&1 > binary-release-without-hadoop.log
+      - ./dev/make-distribution.sh --name without-hadoop --tgz -Psparkr -Phadoop-provided -Pyarn -Pmesos
       - curl -u $BINTRAY_USERNAME:$BINTRAY_PASSWORD -T "spark-$(git describe --tags)-bin-without-hadoop.tgz" "https://api.bintray.com/content/palantir/releases/spark/$(git describe --tags)/org/apache/spark/dist/spark-$(git describe --tags)-bin-without-hadoop.tgz"
       - curl -u $BINTRAY_USERNAME:$BINTRAY_PASSWORD -X POST https://api.bintray.com/content/palantir/releases/spark/$(git describe --tags)/publish
   snapshot:
     branch: master
     commands:
       - ./build/sbt -Phadoop-2.7 -Pmesos -Pkinesis-asl -Pyarn -Phive-thriftserver -Phive publish
-      - ./dev/make-distribution.sh --name hadoop-2.7 --tgz "-Phadoop-2.7 -Psparkr -Phive -Phive-thriftserver -Pyarn -Pmesos" 2>&1 > binary-release-hadoop-2.7.log
+      - ./dev/make-distribution.sh --name hadoop-2.7 --tgz -Phadoop-2.7 -Psparkr -Phive -Phive-thriftserver -Pyarn -Pmesos
       - curl -u $BINTRAY_USERNAME:$BINTRAY_PASSWORD -T "spark-$(git describe --tags)-bin-hadoop-2.7.tgz" "https://api.bintray.com/content/palantir/releases/spark/$(git describe --tags)/org/apache/spark/dist/spark-$(git describe --tags)-bin-hadoop-2.7.tgz"
       - ./build/sbt -Phadoop-2.7 -Pmesos -Pkinesis-asl -Pyarn -Phive-thriftserver -Phive clean
-      - ./dev/make-distribution.sh --name without-hadoop --tgz "-Psparkr -Phadoop-provided -Pyarn -Pmesos" 2>&1 > binary-release-without-hadoop.log
+      - ./dev/make-distribution.sh --name without-hadoop --tgz -Psparkr -Phadoop-provided -Pyarn -Pmesos
       - curl -u $BINTRAY_USERNAME:$BINTRAY_PASSWORD -T "spark-$(git describe --tags)-bin-without-hadoop.tgz" "https://api.bintray.com/content/palantir/releases/spark/$(git describe --tags)/org/apache/spark/dist/spark-$(git describe --tags)-bin-without-hadoop.tgz"
       - curl -u $BINTRAY_USERNAME:$BINTRAY_PASSWORD -X POST https://api.bintray.com/content/palantir/releases/spark/$(git describe --tags)/publish

dev/make-distribution.sh

@@ -35,6 +35,7 @@ DISTDIR="$SPARK_HOME/dist"
 MAKE_TGZ=false
 NAME=none
 MVN="$SPARK_HOME/build/mvn"
+SBT="$SPARK_HOME/build/sbt"
 
 function exit_with_usage {
   echo "make-distribution.sh - tool for making binary distributions of Spark"
@@ -150,7 +151,7 @@ export MAVEN_OPTS="${MAVEN_OPTS:--Xmx2g -XX:MaxPermSize=512M -XX:ReservedCodeCac
 # Store the command as an array because $MVN variable might have spaces in it.
 # Normal quoting tricks don't work.
 # See: http://mywiki.wooledge.org/BashFAQ/050
-BUILD_COMMAND=("$MVN" -T 1C package -DskipTests $@)
+BUILD_COMMAND=("$SBT" assembly examples/package network-yarn/assembly $@)
 
 # Actually build the jar
 echo -e "\nBuilding with..."
@@ -168,9 +169,9 @@ echo "Build flags: $@" >> "$DISTDIR/RELEASE"
 cp "$SPARK_HOME"/assembly/target/scala*/jars/* "$DISTDIR/jars/"
 
 # Only create the yarn directory if the yarn artifacts were build.
-if [ -f "$SPARK_HOME"/common/network-yarn/target/scala*/spark-*-yarn-shuffle.jar ]; then
+if [ -f "$SPARK_HOME"/common/network-yarn/target/scala*/spark-network-yarn-*.jar ]; then
   mkdir "$DISTDIR"/yarn
-  cp "$SPARK_HOME"/common/network-yarn/target/scala*/spark-*-yarn-shuffle.jar "$DISTDIR/yarn"
+  cp "$SPARK_HOME"/common/network-yarn/target/scala*/spark-network-yarn-*.jar "$DISTDIR/yarn/spark-${VERSION}-yarn-shuffle.jar"
 fi
 
 # Copy examples and dependencies

project/SparkBuild.scala

@@ -234,7 +234,7 @@ object SparkBuild extends PomBuild {
     }
   )
 
-  lazy val sharedSettings = versionWithGit ++ // sparkGenjavadocSettings ++
+  lazy val sharedSettings = versionWithGit ++ sparkGenjavadocSettings ++
       (if (sys.env.contains("NOLINT_ON_COMPILE")) Nil else enableScalaStyle) ++ Seq(
     exportJars in Compile := true,
     exportJars in Test := false,
@@ -403,8 +403,8 @@ object SparkBuild extends PomBuild {
   /* Package pyspark artifacts in a separate zip file for YARN. */
   enable(PySparkAssembly.settings)(assembly)
 
-//  /* Enable unidoc only for the root spark project */
-//  enable(Unidoc.settings)(spark)
+  /* Enable unidoc only for the root spark project */
+  enable(Unidoc.settings)(spark)
 
   /* Catalyst ANTLR generation settings */
   enable(Catalyst.settings)(catalyst)