From 5f6ee2b3cf1fa11b96056282c492c1e26d5dbe44 Mon Sep 17 00:00:00 2001
From: Katsunori Kanda <potix2@gmail.com>
Date: Thu, 29 Jan 2015 22:38:43 +0900
Subject: [PATCH 1/6] [SPARK-5482][PySpark] Allow individual test suites in
 python/run-tests

---
 python/run-tests | 87 +++++++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 79 insertions(+), 8 deletions(-)

diff --git a/python/run-tests b/python/run-tests
index 4468fdb3f267e..e99476abb6cd0 100755
--- a/python/run-tests
+++ b/python/run-tests
@@ -17,6 +17,55 @@
 # limitations under the License.
 #
 
+# Run test suites and indivisual test suite.
+#
+# Usage: run-tests [core|sql|mllib|ml|streaming]
+#
+# When you select none, all test suites are run. You can also select
+# multiple test suites.
+
+function usage() {
+    echo "Usage: run-tests [core|sql|mllib|ml|streaming]"
+}
+
+if [ $# == 0 ]; then
+    DO_CORE_TESTS=1
+    DO_SQL_TESTS=1
+    DO_MLLIB_TESTS=1
+    DO_ML_TESTS=1
+    DO_STREAMING_TESTS=1
+else
+    DO_CORE_TESTS=0
+    DO_SQL_TESTS=0
+    DO_MLLIB_TESTS=0
+    DO_ML_TESTS=0
+    DO_STREAMING_TESTS=0
+
+    while (("$#")); do
+        case $1 in
+            core)
+                DO_CORE_TESTS=1
+                ;;
+            sql)
+                DO_SQL_TESTS=1
+                ;;
+            mllib)
+                DO_MLLIB_TESTS=1
+                ;;
+            ml)
+                DO_ML_TESTS=1
+                ;;
+            streaming)
+                DO_STREAMING_TESTS=1
+                ;;
+            *)
+                usage
+                exit 1
+                ;;
+        esac
+        shift
+    done
+fi
 
 # Figure out where the Spark framework is installed
 FWDIR="$(cd "`dirname "$0"`"; cd ../; pwd)"
@@ -143,11 +192,25 @@ fi
 echo "Testing with Python version:"
 $PYSPARK_PYTHON --version
 
-run_core_tests
-run_sql_tests
-run_mllib_tests
-run_ml_tests
-run_streaming_tests
+if [ $DO_CORE_TESTS == 1 ]; then
+    run_core_tests
+fi
+
+if [ $DO_SQL_TESTS == 1 ]; then
+    run_sql_tests
+fi
+
+if [ $DO_MLLIB_TESTS == 1 ]; then
+    run_mllib_tests
+fi
+
+if [ $DO_ML_TESTS == 1 ]; then
+    run_ml_tests
+fi
+
+if [ $DO_STREAMING_TESTS == 1 ]; then
+    run_streaming_tests
+fi
 
 # Try to test with Python 3
 if [ $(which python3.4) ]; then
@@ -168,9 +231,17 @@ if [ $(which pypy) ]; then
     echo "Testing with PyPy version:"
     $PYSPARK_PYTHON --version
 
-    run_core_tests
-    run_sql_tests
-    run_streaming_tests
+    if [ $DO_CORE_TESTS == 1 ]; then
+        run_core_tests
+    fi
+
+    if [ $DO_SQL_TESTS == 1 ]; then
+        run_sql_tests
+    fi
+
+    if [ $DO_STREAMING_TESTS == 1 ]; then
+        run_streaming_tests
+    fi
 fi
 
 if [[ $FAILED == 0 ]]; then

From 0421f609434d83affcb9fcfdd79720075f4b9892 Mon Sep 17 00:00:00 2001
From: Katsunori Kanda <potix2@gmail.com>
Date: Mon, 23 Feb 2015 23:50:12 +0900
Subject: [PATCH 2/6] Move conditional check into the test functions

---
 python/run-tests | 58 +++++++++++++++++++++++-------------------------
 1 file changed, 28 insertions(+), 30 deletions(-)

diff --git a/python/run-tests b/python/run-tests
index e99476abb6cd0..3ee2c015f5dcf 100755
--- a/python/run-tests
+++ b/python/run-tests
@@ -105,6 +105,10 @@ function run_test() {
 }
 
 function run_core_tests() {
+    if [ $DO_CORE_TESTS == 0 ]; then
+        return 0
+    fi
+
     echo "Run core tests ..."
     run_test "pyspark.rdd"
     run_test "pyspark.context"
@@ -118,6 +122,10 @@ function run_core_tests() {
 }
 
 function run_sql_tests() {
+    if [ $DO_SQL_TESTS == 0 ]; then
+        return 0
+    fi
+
     echo "Run sql tests ..."
     run_test "pyspark.sql.types"
     run_test "pyspark.sql.context"
@@ -131,6 +139,10 @@ function run_sql_tests() {
 }
 
 function run_mllib_tests() {
+    if [ $DO_MLLIB_TESTS == 0 ]; then
+        return 0
+    fi
+
     echo "Run mllib tests ..."
     run_test "pyspark.mllib.classification"
     run_test "pyspark.mllib.clustering"
@@ -149,6 +161,10 @@ function run_mllib_tests() {
 }
 
 function run_ml_tests() {
+    if [ $DO_ML_TESTS == 0 ]; then
+        return 0
+    fi
+
     echo "Run ml tests ..."
     run_test "pyspark.ml.feature"
     run_test "pyspark.ml.classification"
@@ -160,6 +176,10 @@ function run_ml_tests() {
 }
 
 function run_streaming_tests() {
+    if [ $DO_STREAMING_TESTS == 0 ]; then
+        return 0
+    fi
+
     echo "Run streaming tests ..."
 
     KAFKA_ASSEMBLY_DIR="$FWDIR"/external/kafka-assembly
@@ -192,25 +212,11 @@ fi
 echo "Testing with Python version:"
 $PYSPARK_PYTHON --version
 
-if [ $DO_CORE_TESTS == 1 ]; then
-    run_core_tests
-fi
-
-if [ $DO_SQL_TESTS == 1 ]; then
-    run_sql_tests
-fi
-
-if [ $DO_MLLIB_TESTS == 1 ]; then
-    run_mllib_tests
-fi
-
-if [ $DO_ML_TESTS == 1 ]; then
-    run_ml_tests
-fi
-
-if [ $DO_STREAMING_TESTS == 1 ]; then
-    run_streaming_tests
-fi
+run_core_tests
+run_sql_tests
+run_mllib_tests
+run_ml_tests
+run_streaming_tests
 
 # Try to test with Python 3
 if [ $(which python3.4) ]; then
@@ -231,17 +237,9 @@ if [ $(which pypy) ]; then
     echo "Testing with PyPy version:"
     $PYSPARK_PYTHON --version
 
-    if [ $DO_CORE_TESTS == 1 ]; then
-        run_core_tests
-    fi
-
-    if [ $DO_SQL_TESTS == 1 ]; then
-        run_sql_tests
-    fi
-
-    if [ $DO_STREAMING_TESTS == 1 ]; then
-        run_streaming_tests
-    fi
+    run_core_tests
+    run_sql_tests
+    run_streaming_tests
 fi
 
 if [[ $FAILED == 0 ]]; then

From 5aa5aa5e39312e81fceb56f201bf314a67f79757 Mon Sep 17 00:00:00 2001
From: Katsunori Kanda <potix2@gmail.com>
Date: Wed, 17 Jun 2015 04:41:27 -0700
Subject: [PATCH 3/6] Split run-tests into the main script and runner scripts
 for a particular python version

---
 python/run-tests     | 197 ++++++++-----------------------------------
 python/tests/common  | 133 +++++++++++++++++++++++++++++
 python/tests/default |  38 +++++++++
 python/tests/pypy.sh |  33 ++++++++
 4 files changed, 238 insertions(+), 163 deletions(-)
 create mode 100644 python/tests/common
 create mode 100644 python/tests/default
 create mode 100644 python/tests/pypy.sh

diff --git a/python/run-tests b/python/run-tests
index 3ee2c015f5dcf..79c60b58f8583 100755
--- a/python/run-tests
+++ b/python/run-tests
@@ -19,15 +19,23 @@
 
 # Run test suites and indivisual test suite.
 #
-# Usage: run-tests [core|sql|mllib|ml|streaming]
+# Usage: run-tests [-v python version] [core|sql|mllib|ml|streaming]
 #
 # When you select none, all test suites are run. You can also select
 # multiple test suites.
 
 function usage() {
-    echo "Usage: run-tests [core|sql|mllib|ml|streaming]"
+    echo "Usage: run-tests [-v python version] [core|sql|mllib|ml|streaming]"
 }
 
+SUPPORT_PYTHON_VERSIONS="2.6 3.4 pypy"
+TARGET_PYTHON_VERSION=
+if [ $# != 0 -a $1 == "-v" ]; then
+    TARGET_PYTHON_VERSION=$2
+    shift
+    shift
+fi
+
 if [ $# == 0 ]; then
     DO_CORE_TESTS=1
     DO_SQL_TESTS=1
@@ -67,6 +75,9 @@ else
     done
 fi
 
+# set the directory that this script stored in.
+SCRIPT_DIR="$(cd "`dirname "$0"`"; pwd)"
+
 # Figure out where the Spark framework is installed
 FWDIR="$(cd "`dirname "$0"`"; cd ../; pwd)"
 
@@ -75,177 +86,37 @@ FWDIR="$(cd "`dirname "$0"`"; cd ../; pwd)"
 # CD into the python directory to find things on the right path
 cd "$FWDIR/python"
 
-FAILED=0
 LOG_FILE=unit-tests.log
-START=$(date +"%s")
 
 rm -f $LOG_FILE
 
-# Remove the metastore and warehouse directory created by the HiveContext tests in Spark SQL
-rm -rf metastore warehouse
-
-function run_test() {
-    echo -en "Running test: $1 ... " | tee -a $LOG_FILE
-    start=$(date +"%s")
-    SPARK_TESTING=1 time "$FWDIR"/bin/pyspark $1 > $LOG_FILE 2>&1
-
-    FAILED=$((PIPESTATUS[0]||$FAILED))
-
-    # Fail and exit on the first test failure.
-    if [[ $FAILED != 0 ]]; then
-        cat $LOG_FILE | grep -v "^[0-9][0-9]*" # filter all lines starting with a number.
-        echo -en "\033[31m"  # Red
-        echo "Had test failures; see logs."
-        echo -en "\033[0m"  # No color
-        exit -1
-    else
-        now=$(date +"%s")
-        echo "ok ($(($now - $start))s)"
-    fi
-}
-
-function run_core_tests() {
-    if [ $DO_CORE_TESTS == 0 ]; then
-        return 0
-    fi
-
-    echo "Run core tests ..."
-    run_test "pyspark.rdd"
-    run_test "pyspark.context"
-    run_test "pyspark.conf"
-    run_test "pyspark.broadcast"
-    run_test "pyspark.accumulators"
-    run_test "pyspark.serializers"
-    run_test "pyspark.profiler"
-    run_test "pyspark.shuffle"
-    run_test "pyspark.tests"
-}
+echo "Running PySpark tests. Output is in python/$LOG_FILE."
 
-function run_sql_tests() {
-    if [ $DO_SQL_TESTS == 0 ]; then
-        return 0
+if [ -n "$TARGET_PYTHON_VERSION" ]; then
+    if [[ ! "$SUPPORT_PYTHON_VERSIONS" =~ "$TARGET_PYTHON_VERSION" ]]; then
+        echo "python $TARGET_PYTHON_VERSION is not supported."
+        exit 1
     fi
 
-    echo "Run sql tests ..."
-    run_test "pyspark.sql.types"
-    run_test "pyspark.sql.context"
-    run_test "pyspark.sql.column"
-    run_test "pyspark.sql.dataframe"
-    run_test "pyspark.sql.group"
-    run_test "pyspark.sql.functions"
-    run_test "pyspark.sql.readwriter"
-    run_test "pyspark.sql.window"
-    run_test "pyspark.sql.tests"
-}
+    SUPPORT_PYTHON_VERSIONS=$TARGET_PYTHON_VERSION
+fi
 
-function run_mllib_tests() {
-    if [ $DO_MLLIB_TESTS == 0 ]; then
-        return 0
-    fi
+for PYVERSION in $SUPPORT_PYTHON_VERSIONS;
+do
+    # Remove the metastore and warehouse directory created by the HiveContext tests in Spark SQL
+    rm -rf metastore warehouse
 
-    echo "Run mllib tests ..."
-    run_test "pyspark.mllib.classification"
-    run_test "pyspark.mllib.clustering"
-    run_test "pyspark.mllib.evaluation"
-    run_test "pyspark.mllib.feature"
-    run_test "pyspark.mllib.fpm"
-    run_test "pyspark.mllib.linalg"
-    run_test "pyspark.mllib.random"
-    run_test "pyspark.mllib.recommendation"
-    run_test "pyspark.mllib.regression"
-    run_test "pyspark.mllib.stat._statistics"
-    run_test "pyspark.mllib.stat.KernelDensity"
-    run_test "pyspark.mllib.tree"
-    run_test "pyspark.mllib.util"
-    run_test "pyspark.mllib.tests"
-}
+    START=$(date +"%s")
+    FAILED=0
 
-function run_ml_tests() {
-    if [ $DO_ML_TESTS == 0 ]; then
-        return 0
+    if [ -f "${SCRIPT_DIR}/tests/$PYVERSION.sh" ]; then
+        . "${SCRIPT_DIR}/tests/$PYVERSION.sh"
+    else
+        . "${SCRIPT_DIR}/tests/default" $PYVERSION
     fi
 
-    echo "Run ml tests ..."
-    run_test "pyspark.ml.feature"
-    run_test "pyspark.ml.classification"
-    run_test "pyspark.ml.recommendation"
-    run_test "pyspark.ml.regression"
-    run_test "pyspark.ml.tuning"
-    run_test "pyspark.ml.tests"
-    run_test "pyspark.ml.evaluation"
-}
-
-function run_streaming_tests() {
-    if [ $DO_STREAMING_TESTS == 0 ]; then
-        return 0
+    if [[ $FAILED == 0 ]]; then
+        now=$(date +"%s")
+        echo -e "\033[32mTests passed \033[0min $(($now - $START)) seconds"
     fi
-
-    echo "Run streaming tests ..."
-
-    KAFKA_ASSEMBLY_DIR="$FWDIR"/external/kafka-assembly
-    JAR_PATH="${KAFKA_ASSEMBLY_DIR}/target/scala-${SPARK_SCALA_VERSION}"
-    for f in "${JAR_PATH}"/spark-streaming-kafka-assembly-*.jar; do
-      if [[ ! -e "$f" ]]; then
-        echo "Failed to find Spark Streaming Kafka assembly jar in $KAFKA_ASSEMBLY_DIR" 1>&2
-        echo "You need to build Spark with " \
-             "'build/sbt assembly/assembly streaming-kafka-assembly/assembly' or" \
-             "'build/mvn package' before running this program" 1>&2
-        exit 1
-      fi
-      KAFKA_ASSEMBLY_JAR="$f"
-    done
-
-    export PYSPARK_SUBMIT_ARGS="--jars ${KAFKA_ASSEMBLY_JAR} pyspark-shell"
-    run_test "pyspark.streaming.util"
-    run_test "pyspark.streaming.tests"
-}
-
-echo "Running PySpark tests. Output is in python/$LOG_FILE."
-
-export PYSPARK_PYTHON="python"
-
-# Try to test with Python 2.6, since that's the minimum version that we support:
-if [ $(which python2.6) ]; then
-    export PYSPARK_PYTHON="python2.6"
-fi
-
-echo "Testing with Python version:"
-$PYSPARK_PYTHON --version
-
-run_core_tests
-run_sql_tests
-run_mllib_tests
-run_ml_tests
-run_streaming_tests
-
-# Try to test with Python 3
-if [ $(which python3.4) ]; then
-    export PYSPARK_PYTHON="python3.4"
-    echo "Testing with Python3.4 version:"
-    $PYSPARK_PYTHON --version
-
-    run_core_tests
-    run_sql_tests
-    run_mllib_tests
-    run_ml_tests
-    run_streaming_tests
-fi
-
-# Try to test with PyPy
-if [ $(which pypy) ]; then
-    export PYSPARK_PYTHON="pypy"
-    echo "Testing with PyPy version:"
-    $PYSPARK_PYTHON --version
-
-    run_core_tests
-    run_sql_tests
-    run_streaming_tests
-fi
-
-if [[ $FAILED == 0 ]]; then
-    now=$(date +"%s")
-    echo -e "\033[32mTests passed \033[0min $(($now - $START)) seconds"
-fi
-
-# TODO: in the long-run, it would be nice to use a test runner like `nose`.
-# The doctest fixtures are the current barrier to doing this.
+done
diff --git a/python/tests/common b/python/tests/common
new file mode 100644
index 0000000000000..a611b1475a9dd
--- /dev/null
+++ b/python/tests/common
@@ -0,0 +1,133 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+function run_test() {
+    echo -en "Running test: $1 ... " | tee -a $LOG_FILE
+    start=$(date +"%s")
+    SPARK_TESTING=1 time "$FWDIR"/bin/pyspark $1 > $LOG_FILE 2>&1
+
+    FAILED=$((PIPESTATUS[0]||$FAILED))
+
+    # Fail and exit on the first test failure.
+    if [[ $FAILED != 0 ]]; then
+        cat $LOG_FILE | grep -v "^[0-9][0-9]*" # filter all lines starting with a number.
+        echo -en "\033[31m"  # Red
+        echo "Had test failures; see logs."
+        echo -en "\033[0m"  # No color
+        exit -1
+    else
+        now=$(date +"%s")
+        echo "ok ($(($now - $start))s)"
+    fi
+}
+
+function run_core_tests() {
+    if [ $DO_CORE_TESTS == 0 ]; then
+        return 0
+    fi
+
+    echo "Run core tests ..."
+    run_test "pyspark.rdd"
+    run_test "pyspark.context"
+    run_test "pyspark.conf"
+    run_test "pyspark.broadcast"
+    run_test "pyspark.accumulators"
+    run_test "pyspark.serializers"
+    run_test "pyspark.profiler"
+    run_test "pyspark.shuffle"
+    run_test "pyspark.tests"
+}
+
+function run_sql_tests() {
+    if [ $DO_SQL_TESTS == 0 ]; then
+        return 0
+    fi
+
+    echo "Run sql tests ..."
+    run_test "pyspark.sql.types"
+    run_test "pyspark.sql.context"
+    run_test "pyspark.sql.column"
+    run_test "pyspark.sql.dataframe"
+    run_test "pyspark.sql.group"
+    run_test "pyspark.sql.functions"
+    run_test "pyspark.sql.readwriter"
+    run_test "pyspark.sql.window"
+    run_test "pyspark.sql.tests"
+}
+
+function run_mllib_tests() {
+    if [ $DO_MLLIB_TESTS == 0 ]; then
+        return 0
+    fi
+
+    echo "Run mllib tests ..."
+    run_test "pyspark.mllib.classification"
+    run_test "pyspark.mllib.clustering"
+    run_test "pyspark.mllib.evaluation"
+    run_test "pyspark.mllib.feature"
+    run_test "pyspark.mllib.fpm"
+    run_test "pyspark.mllib.linalg"
+    run_test "pyspark.mllib.random"
+    run_test "pyspark.mllib.recommendation"
+    run_test "pyspark.mllib.regression"
+    run_test "pyspark.mllib.stat._statistics"
+    run_test "pyspark.mllib.stat.KernelDensity"
+    run_test "pyspark.mllib.tree"
+    run_test "pyspark.mllib.util"
+    run_test "pyspark.mllib.tests"
+}
+
+function run_ml_tests() {
+    if [ $DO_ML_TESTS == 0 ]; then
+        return 0
+    fi
+
+    echo "Run ml tests ..."
+    run_test "pyspark.ml.feature"
+    run_test "pyspark.ml.classification"
+    run_test "pyspark.ml.recommendation"
+    run_test "pyspark.ml.regression"
+    run_test "pyspark.ml.tuning"
+    run_test "pyspark.ml.tests"
+    run_test "pyspark.ml.evaluation"
+}
+
+function run_streaming_tests() {
+    if [ $DO_STREAMING_TESTS == 0 ]; then
+        return 0
+    fi
+
+    echo "Run streaming tests ..."
+
+    KAFKA_ASSEMBLY_DIR="$FWDIR"/external/kafka-assembly
+    JAR_PATH="${KAFKA_ASSEMBLY_DIR}/target/scala-${SPARK_SCALA_VERSION}"
+    for f in "${JAR_PATH}"/spark-streaming-kafka-assembly-*.jar; do
+      if [[ ! -e "$f" ]]; then
+        echo "Failed to find Spark Streaming Kafka assembly jar in $KAFKA_ASSEMBLY_DIR" 1>&2
+        echo "You need to build Spark with " \
+             "'build/sbt assembly/assembly streaming-kafka-assembly/assembly' or" \
+             "'build/mvn package' before running this program" 1>&2
+        exit 1
+      fi
+      KAFKA_ASSEMBLY_JAR="$f"
+    done
+
+    export PYSPARK_SUBMIT_ARGS="--jars ${KAFKA_ASSEMBLY_JAR} pyspark-shell"
+    run_test "pyspark.streaming.util"
+    run_test "pyspark.streaming.tests"
+}
diff --git a/python/tests/default b/python/tests/default
new file mode 100644
index 0000000000000..181a0546b46dd
--- /dev/null
+++ b/python/tests/default
@@ -0,0 +1,38 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+. tests/common
+
+PYVERSION=$1
+PYBIN_NAME="python${PYVERSION}"
+
+# Try to test with particular version of Python, since that's the minimum version that we support:
+if [ $(which $PYBIN_NAME) ]; then
+    export PYSPARK_PYTHON=$PYBIN_NAME
+    echo "Testing with Python version:"
+    $PYSPARK_PYTHON --version
+
+    run_core_tests
+    run_sql_tests
+    run_mllib_tests
+    run_ml_tests
+    run_streaming_tests
+else
+    echo "Skipping tests with Python version: $PYVERSION"
+    FAILED=1
+fi
diff --git a/python/tests/pypy.sh b/python/tests/pypy.sh
new file mode 100644
index 0000000000000..bcd00168890c4
--- /dev/null
+++ b/python/tests/pypy.sh
@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+. tests/common
+
+# Try to test with PyPy
+if [ $(which pypy) ]; then
+    export PYSPARK_PYTHON="pypy"
+    echo "Testing with PyPy version:"
+    $PYSPARK_PYTHON --version
+
+    run_core_tests
+    run_sql_tests
+    run_streaming_tests
+else
+    echo "Skipping tests with PyPy"
+    FAILED=1
+fi

From 36153f677ff8e37089deb26bee2d352bff926879 Mon Sep 17 00:00:00 2001
From: Katsunori Kanda <potix2@gmail.com>
Date: Fri, 19 Jun 2015 09:57:09 +0900
Subject: [PATCH 4/6] Remove unnecessary comments

---
 python/tests/common  | 18 ------------------
 python/tests/default | 18 ------------------
 python/tests/pypy.sh | 18 ------------------
 3 files changed, 54 deletions(-)

diff --git a/python/tests/common b/python/tests/common
index a611b1475a9dd..b9523808dabd5 100644
--- a/python/tests/common
+++ b/python/tests/common
@@ -1,21 +1,3 @@
-# -*- coding: utf-8 -*-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
 function run_test() {
     echo -en "Running test: $1 ... " | tee -a $LOG_FILE
     start=$(date +"%s")
diff --git a/python/tests/default b/python/tests/default
index 181a0546b46dd..6235868c2bf32 100644
--- a/python/tests/default
+++ b/python/tests/default
@@ -1,21 +1,3 @@
-# -*- coding: utf-8 -*-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
 . tests/common
 
 PYVERSION=$1
diff --git a/python/tests/pypy.sh b/python/tests/pypy.sh
index bcd00168890c4..5727d5ffdf803 100644
--- a/python/tests/pypy.sh
+++ b/python/tests/pypy.sh
@@ -1,21 +1,3 @@
-# -*- coding: utf-8 -*-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
 . tests/common
 
 # Try to test with PyPy

From 6dec2468f9df040f4bbec72298ce51177c2d2039 Mon Sep 17 00:00:00 2001
From: Katsunori Kanda <potix2@gmail.com>
Date: Fri, 19 Jun 2015 10:19:23 +0900
Subject: [PATCH 5/6] Add license headers.

---
 python/tests/common  | 17 +++++++++++++++++
 python/tests/default | 17 +++++++++++++++++
 python/tests/pypy.sh | 17 +++++++++++++++++
 3 files changed, 51 insertions(+)

diff --git a/python/tests/common b/python/tests/common
index b9523808dabd5..6f5000d59eda0 100644
--- a/python/tests/common
+++ b/python/tests/common
@@ -1,3 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
 function run_test() {
     echo -en "Running test: $1 ... " | tee -a $LOG_FILE
     start=$(date +"%s")
diff --git a/python/tests/default b/python/tests/default
index 6235868c2bf32..79eb0748405a4 100644
--- a/python/tests/default
+++ b/python/tests/default
@@ -1,3 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
 . tests/common
 
 PYVERSION=$1
diff --git a/python/tests/pypy.sh b/python/tests/pypy.sh
index 5727d5ffdf803..35ee1857cc0f7 100644
--- a/python/tests/pypy.sh
+++ b/python/tests/pypy.sh
@@ -1,3 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
 . tests/common
 
 # Try to test with PyPy

From e08f602c5f510132b313505fc4b7e704c901a8a8 Mon Sep 17 00:00:00 2001
From: Katsunori Kanda <potix2@gmail.com>
Date: Fri, 19 Jun 2015 10:31:32 +0900
Subject: [PATCH 6/6] Remove redundant shift

---
 python/run-tests | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/python/run-tests b/python/run-tests
index 79c60b58f8583..da6d37a1cd8e5 100755
--- a/python/run-tests
+++ b/python/run-tests
@@ -32,8 +32,7 @@ SUPPORT_PYTHON_VERSIONS="2.6 3.4 pypy"
 TARGET_PYTHON_VERSION=
 if [ $# != 0 -a $1 == "-v" ]; then
     TARGET_PYTHON_VERSION=$2
-    shift
-    shift
+    shift 2
 fi
 
 if [ $# == 0 ]; then