From 9cb7ab88fb2d8c8cc2bcbfb4f30647348b9b5aab Mon Sep 17 00:00:00 2001 From: Shay Rojansky Date: Sun, 10 Aug 2014 11:04:45 +0300 Subject: [PATCH 1/9] Support executing Spark from symlinks The current scripts (e.g. pyspark) fail to run when they are executed via symlinks. A common Linux scenario would be to have Spark installed somewhere (e.g. /opt) and have a symlink to it in /usr/bin. Fixed the scripts to traverse symlinks until reaching the actual binary. --- bin/beeline | 8 +++++++- bin/load-spark-env.sh | 2 +- bin/run-example | 12 +++++++++++- bin/spark-class | 11 ++++++++++- bin/spark-shell | 9 ++++++++- bin/spark-submit | 12 ++++++++++-- sbin/slaves.sh | 14 ++++++++++---- sbin/spark-config.sh | 2 +- sbin/spark-daemon.sh | 20 +++++++++++++------- sbin/spark-daemons.sh | 14 ++++++++++---- sbin/start-all.sh | 16 +++++++++++----- sbin/start-history-server.sh | 14 ++++++++++---- sbin/start-master.sh | 24 ++++++++++++++---------- sbin/start-slave.sh | 7 +++---- sbin/start-slaves.sh | 20 ++++++++++++-------- sbin/start-thriftserver.sh | 8 +++++++- sbin/stop-all.sh | 13 +++++++++---- sbin/stop-history-server.sh | 12 +++++++++--- sbin/stop-master.sh | 18 ++++++++++++------ sbin/stop-slaves.sh | 19 ++++++++++++------- 20 files changed, 180 insertions(+), 75 deletions(-) diff --git a/bin/beeline b/bin/beeline index 3fcb6df34339..a58513bbc830 100755 --- a/bin/beeline +++ b/bin/beeline @@ -24,7 +24,13 @@ set -o posix # Figure out where Spark is installed -FWDIR="$(cd "`dirname "$0"`"/..; pwd)" +SOURCE=$0 +while [ -h "$SOURCE" ] +do + SOURCE="$(readlink "$SOURCE")" + [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" +done +FWDIR="$(cd `dirname $SOURCE`/..; pwd)" CLASS="org.apache.hive.beeline.BeeLine" exec "$FWDIR/bin/spark-class" $CLASS "$@" diff --git a/bin/load-spark-env.sh b/bin/load-spark-env.sh index 95779e9ddbb1..29e39dd8359f 100644 --- a/bin/load-spark-env.sh +++ b/bin/load-spark-env.sh @@ -18,7 +18,7 @@ # # This script loads spark-env.sh if it exists, and ensures it is only loaded once. -# spark-env.sh is loaded from SPARK_CONF_DIR if set, or within the current directory's +# spark-env.sh is loaded from SPARK_CONF_DIR if set, or within SPARK_HOME's # conf/ subdirectory. FWDIR="$(cd "`dirname "$0"`"/..; pwd)" diff --git a/bin/run-example b/bin/run-example index 798e2caeb88c..6e7bc22187eb 100755 --- a/bin/run-example +++ b/bin/run-example @@ -17,7 +17,17 @@ # limitations under the License. # -FWDIR="$(cd "`dirname "$0"`"/..; pwd)" +SCALA_VERSION=2.10 + +# Figure out where Spark is installed +SOURCE=$0 +while [ -h "$SOURCE" ] +do + SOURCE="$(readlink "$SOURCE")" + [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" +done +FWDIR="$(cd `dirname $SOURCE`/..; pwd)" + export SPARK_HOME="$FWDIR" EXAMPLES_DIR="$FWDIR"/examples diff --git a/bin/spark-class b/bin/spark-class index 8cae6ccbabe7..78e7cb81a632 100755 --- a/bin/spark-class +++ b/bin/spark-class @@ -18,7 +18,16 @@ # # Figure out where Spark is installed -export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" +SOURCE=$0 +while [ -h "$SOURCE" ] +do + SOURCE="$(readlink "$SOURCE")" + [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" +done +FWDIR="$(cd `dirname $SOURCE`/..; pwd)" + +# Export this as SPARK_HOME +export SPARK_HOME="$FWDIR" . "$SPARK_HOME"/bin/load-spark-env.sh diff --git a/bin/spark-shell b/bin/spark-shell index 00ab7afd118b..98c0704d0516 100755 --- a/bin/spark-shell +++ b/bin/spark-shell @@ -28,8 +28,15 @@ esac # Enter posix mode for bash set -o posix -export FWDIR="$(cd "`dirname "$0"`"/..; pwd)" export _SPARK_CMD_USAGE="Usage: ./bin/spark-shell [options]" +# Figure out where Spark is installed +SOURCE=$0 +while [ -h "$SOURCE" ] +do + SOURCE="$(readlink "$SOURCE")" + [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" +done +FWDIR="$(cd `dirname $SOURCE`/..; pwd)" # SPARK-4161: scala does not assume use of the java classpath, # so we need to add the "-Dscala.usejavacp=true" flag manually. We diff --git a/bin/spark-submit b/bin/spark-submit index 255378b0f077..ea02b9af0194 100755 --- a/bin/spark-submit +++ b/bin/spark-submit @@ -17,9 +17,17 @@ # limitations under the License. # -SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" - # disable randomized hash for string in Python 3.3+ export PYTHONHASHSEED=0 +# Figure out where Spark is installed +SOURCE=$0 +while [ -h "$SOURCE" ] +do + SOURCE="$(readlink "$SOURCE")" + [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" +done +FWDIR="$(cd `dirname $SOURCE`/..; pwd)" +# Export this as SPARK_HOME +export SPARK_HOME="$FWDIR" exec "$SPARK_HOME"/bin/spark-class org.apache.spark.deploy.SparkSubmit "$@" diff --git a/sbin/slaves.sh b/sbin/slaves.sh index cdad47ee2e59..80d7f7b1e664 100755 --- a/sbin/slaves.sh +++ b/sbin/slaves.sh @@ -36,10 +36,16 @@ if [ $# -le 0 ]; then exit 1 fi -sbin="`dirname "$0"`" -sbin="`cd "$sbin"; pwd`" +# Figure out where Spark is installed +SOURCE=$0 +while [ -h "$SOURCE" ] +do + SOURCE="$(readlink "$SOURCE")" + [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" +done +export SPARK_HOME="$(cd `dirname $SOURCE`/..; pwd)" -. "$sbin/spark-config.sh" +. "$SPARK_HOME/sbin/spark-config.sh" # If the slaves file is specified in the command line, # then it takes precedence over the definition in @@ -65,7 +71,7 @@ then shift fi -. "$SPARK_PREFIX/bin/load-spark-env.sh" +. "$SPARK_HOME/bin/load-spark-env.sh" if [ "$HOSTLIST" = "" ]; then if [ "$SPARK_SLAVES" = "" ]; then diff --git a/sbin/spark-config.sh b/sbin/spark-config.sh index e6bf544c1479..36c467f7e218 100755 --- a/sbin/spark-config.sh +++ b/sbin/spark-config.sh @@ -17,7 +17,7 @@ # included in all the spark scripts with source command # should not be executable directly -# also should not be passed any arguments, since we need original $* +# depends on SPARK_HOME being properly set in the sourcing script # resolve links - $0 may be a softlink this="${BASH_SOURCE:-$0}" diff --git a/sbin/spark-daemon.sh b/sbin/spark-daemon.sh index 0fbe795822fb..fe6d147def0f 100755 --- a/sbin/spark-daemon.sh +++ b/sbin/spark-daemon.sh @@ -37,10 +37,16 @@ if [ $# -le 1 ]; then exit 1 fi -sbin="`dirname "$0"`" -sbin="`cd "$sbin"; pwd`" +# Figure out where Spark is installed +SOURCE=$0 +while [ -h "$SOURCE" ] +do + SOURCE="$(readlink "$SOURCE")" + [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" +done +export SPARK_HOME="$(cd `dirname $SOURCE`/..; pwd)" -. "$sbin/spark-config.sh" +. "$SPARK_HOME/sbin/spark-config.sh" # get arguments @@ -86,7 +92,7 @@ spark_rotate_log () fi } -. "$SPARK_PREFIX/bin/load-spark-env.sh" +. "$SPARK_HOME/bin/load-spark-env.sh" if [ "$SPARK_IDENT_STRING" = "" ]; then export SPARK_IDENT_STRING="$USER" @@ -205,13 +211,13 @@ case $option in else echo $pid file is present but $command not running exit 1 - fi + fi else echo $command not running. exit 2 - fi + fi ;; - + (*) echo $usage exit 1 diff --git a/sbin/spark-daemons.sh b/sbin/spark-daemons.sh index 5d9f2bb51cae..d213eb82981f 100755 --- a/sbin/spark-daemons.sh +++ b/sbin/spark-daemons.sh @@ -27,9 +27,15 @@ if [ $# -le 1 ]; then exit 1 fi -sbin=`dirname "$0"` -sbin=`cd "$sbin"; pwd` +# Figure out where Spark is installed +SOURCE=$0 +while [ -h "$SOURCE" ] +do + SOURCE="$(readlink "$SOURCE")" + [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" +done +export SPARK_HOME="$(cd `dirname $SOURCE`/..; pwd)" -. "$sbin/spark-config.sh" +. "$SPARK_HOME/sbin/spark-config.sh" -exec "$sbin/slaves.sh" cd "$SPARK_HOME" \; "$sbin/spark-daemon.sh" "$@" +exec "$SPARK_HOME/sbin/slaves.sh" cd "$SPARK_HOME" \; "$SPARK_HOME/sbin/spark-daemon.sh" "$@" diff --git a/sbin/start-all.sh b/sbin/start-all.sh index 1baf57cea09e..39eb52349e5f 100755 --- a/sbin/start-all.sh +++ b/sbin/start-all.sh @@ -21,8 +21,14 @@ # Starts the master on this node. # Starts a worker on each node specified in conf/slaves -sbin="`dirname "$0"`" -sbin="`cd "$sbin"; pwd`" +# Figure out where Spark is installed +SOURCE=$0 +while [ -h "$SOURCE" ] +do + SOURCE="$(readlink "$SOURCE")" + [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" +done +export SPARK_HOME="$(cd `dirname $SOURCE`/..; pwd)" TACHYON_STR="" @@ -36,10 +42,10 @@ shift done # Load the Spark configuration -. "$sbin/spark-config.sh" +. "$SPARK_HOME/sbin/spark-config.sh" # Start Master -"$sbin"/start-master.sh $TACHYON_STR +"$SPARK_HOME/sbin/start-master.sh" $TACHYON_STR # Start Workers -"$sbin"/start-slaves.sh $TACHYON_STR +"$SPARK_HOME/sbin/start-slaves.sh" $TACHYON_STR diff --git a/sbin/start-history-server.sh b/sbin/start-history-server.sh index 9034e5715cc8..a9a308a7f55f 100755 --- a/sbin/start-history-server.sh +++ b/sbin/start-history-server.sh @@ -24,10 +24,16 @@ # Use the SPARK_HISTORY_OPTS environment variable to set history server configuration. # -sbin="`dirname "$0"`" -sbin="`cd "$sbin"; pwd`" +# Figure out where Spark is installed +SOURCE=$0 +while [ -h "$SOURCE" ] +do + SOURCE="$(readlink "$SOURCE")" + [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" +done +export SPARK_HOME="$(cd `dirname $SOURCE`/..; pwd)" -. "$sbin/spark-config.sh" -. "$SPARK_PREFIX/bin/load-spark-env.sh" +. "$SPARK_HOME/sbin/spark-config.sh" +. "$SPARK_HOME/bin/load-spark-env.sh" exec "$sbin"/spark-daemon.sh start org.apache.spark.deploy.history.HistoryServer 1 $@ diff --git a/sbin/start-master.sh b/sbin/start-master.sh index a7f5d5702fd8..4c62690ebdc6 100755 --- a/sbin/start-master.sh +++ b/sbin/start-master.sh @@ -19,17 +19,22 @@ # Starts the master on the machine this script is executed on. -sbin="`dirname "$0"`" -sbin="`cd "$sbin"; pwd`" - ORIGINAL_ARGS="$@" +# Figure out where Spark is installed +SOURCE=$0 +while [ -h "$SOURCE" ] +do + SOURCE="$(readlink "$SOURCE")" + [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" +done +export SPARK_HOME="$(cd `dirname $SOURCE`/..; pwd)" START_TACHYON=false while (( "$#" )); do case $1 in --with-tachyon) - if [ ! -e "$sbin"/../tachyon/bin/tachyon ]; then + if [ ! -e "$SPARK_HOME/tachyon/bin/tachyon" ]; then echo "Error: --with-tachyon specified, but tachyon not found." exit -1 fi @@ -39,9 +44,8 @@ case $1 in shift done -. "$sbin/spark-config.sh" - -. "$SPARK_PREFIX/bin/load-spark-env.sh" +. "$SPARK_HOME/sbin/spark-config.sh" +. "$SPARK_HOME/bin/load-spark-env.sh" if [ "$SPARK_MASTER_PORT" = "" ]; then SPARK_MASTER_PORT=7077 @@ -60,7 +64,7 @@ fi $ORIGINAL_ARGS if [ "$START_TACHYON" == "true" ]; then - "$sbin"/../tachyon/bin/tachyon bootstrap-conf $SPARK_MASTER_IP - "$sbin"/../tachyon/bin/tachyon format -s - "$sbin"/../tachyon/bin/tachyon-start.sh master + "$SPARK_HOME/tachyon/bin/tachyon" bootstrap-conf $SPARK_MASTER_IP + "$SPARK_HOME/tachyon/bin/tachyon" format -s + "$SPARK_HOME/tachyon/bin/tachyon-start.sh" master fi diff --git a/sbin/start-slave.sh b/sbin/start-slave.sh index 4c919ff76a8f..d15fba28ec82 100755 --- a/sbin/start-slave.sh +++ b/sbin/start-slave.sh @@ -21,14 +21,14 @@ # # Environment Variables # -# SPARK_WORKER_INSTANCES The number of worker instances to run on this +# SPARK_WORKER_INSTANCES The number of worker instances to run on this # slave. Default is 1. -# SPARK_WORKER_PORT The base port number for the first worker. If set, +# SPARK_WORKER_PORT The base port number for the first worker. If set, # subsequent workers will increment this number. If # unset, Spark will find a valid port number, but # with no guarantee of a predictable pattern. # SPARK_WORKER_WEBUI_PORT The base port for the web interface of the first -# worker. Subsequent workers will increment this +# worker. Subsequent workers will increment this # number. Default is 8081. usage="Usage: start-slave.sh where is like spark://localhost:7077" @@ -82,4 +82,3 @@ else start_instance $(( 1 + $i )) "$@" done fi - diff --git a/sbin/start-slaves.sh b/sbin/start-slaves.sh index 24d6268815ed..74a6bd15d6d2 100755 --- a/sbin/start-slaves.sh +++ b/sbin/start-slaves.sh @@ -19,16 +19,21 @@ # Starts a slave instance on each machine specified in the conf/slaves file. -sbin="`dirname "$0"`" -sbin="`cd "$sbin"; pwd`" - +# Figure out where Spark is installed +SOURCE=$0 +while [ -h "$SOURCE" ] +do + SOURCE="$(readlink "$SOURCE")" + [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" +done +export SPARK_HOME="$(cd `dirname $SOURCE`/..; pwd)" START_TACHYON=false while (( "$#" )); do case $1 in --with-tachyon) - if [ ! -e "$sbin"/../tachyon/bin/tachyon ]; then + if [ ! -e "$SPARK_HOME/tachyon/bin/tachyon" ]; then echo "Error: --with-tachyon specified, but tachyon not found." exit -1 fi @@ -38,9 +43,8 @@ case $1 in shift done -. "$sbin/spark-config.sh" - -. "$SPARK_PREFIX/bin/load-spark-env.sh" +. "$SPARK_HOME/sbin/spark-config.sh" +. "$SPARK_HOME/bin/load-spark-env.sh" # Find the port number for the master if [ "$SPARK_MASTER_PORT" = "" ]; then @@ -55,7 +59,7 @@ if [ "$START_TACHYON" == "true" ]; then "$sbin/slaves.sh" cd "$SPARK_HOME" \; "$sbin"/../tachyon/bin/tachyon bootstrap-conf "$SPARK_MASTER_IP" # set -t so we can call sudo - SPARK_SSH_OPTS="-o StrictHostKeyChecking=no -t" "$sbin/slaves.sh" cd "$SPARK_HOME" \; "$sbin/../tachyon/bin/tachyon-start.sh" worker SudoMount \; sleep 1 + SPARK_SSH_OPTS="-o StrictHostKeyChecking=no -t" "$SPARK_HOME/sbin/slaves.sh" cd "$SPARK_HOME" \; "$SPARK_HOME/tachyon/bin/tachyon-start.sh" worker SudoMount \; sleep 1 fi # Launch the slaves diff --git a/sbin/start-thriftserver.sh b/sbin/start-thriftserver.sh index 5b0aeb177fff..ec3e421190e0 100755 --- a/sbin/start-thriftserver.sh +++ b/sbin/start-thriftserver.sh @@ -24,7 +24,13 @@ set -o posix # Figure out where Spark is installed -FWDIR="$(cd "`dirname "$0"`"/..; pwd)" +SOURCE=$0 +while [ -h "$SOURCE" ] +do + SOURCE="$(readlink "$SOURCE")" + [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" +done +export SPARK_HOME="$(cd `dirname $SOURCE`/..; pwd)" # NOTE: This exact class name is matched downstream by SparkSubmit. # Any changes need to be reflected there. diff --git a/sbin/stop-all.sh b/sbin/stop-all.sh index 1a9abe07db84..72d4881c1307 100755 --- a/sbin/stop-all.sh +++ b/sbin/stop-all.sh @@ -20,12 +20,17 @@ # Stop all spark daemons. # Run this on the master node. - -sbin="`dirname "$0"`" -sbin="`cd "$sbin"; pwd`" +# Figure out where Spark is installed +SOURCE=$0 +while [ -h "$SOURCE" ] +do + SOURCE="$(readlink "$SOURCE")" + [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" +done +export SPARK_HOME="$(cd `dirname $SOURCE`/..; pwd)" # Load the Spark configuration -. "$sbin/spark-config.sh" +. "$SPARK_HOME/sbin/spark-config.sh" # Stop the slaves, then the master "$sbin"/stop-slaves.sh diff --git a/sbin/stop-history-server.sh b/sbin/stop-history-server.sh index 6e6056359510..0960e13a29d7 100755 --- a/sbin/stop-history-server.sh +++ b/sbin/stop-history-server.sh @@ -19,7 +19,13 @@ # Stops the history server on the machine this script is executed on. -sbin="`dirname "$0"`" -sbin="`cd "$sbin"; pwd`" +# Figure out where Spark is installed +SOURCE=$0 +while [ -h "$SOURCE" ] +do + SOURCE="$(readlink "$SOURCE")" + [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" +done +export SPARK_HOME="$(cd `dirname $SOURCE`/..; pwd)" -"$sbin"/spark-daemon.sh stop org.apache.spark.deploy.history.HistoryServer 1 +"$SPARK_HOME/sbin/spark-daemon.sh" stop org.apache.spark.deploy.history.HistoryServer 1 diff --git a/sbin/stop-master.sh b/sbin/stop-master.sh index 729702d92191..b0397dae59df 100755 --- a/sbin/stop-master.sh +++ b/sbin/stop-master.sh @@ -19,13 +19,19 @@ # Stops the master on the machine this script is executed on. -sbin=`dirname "$0"` -sbin=`cd "$sbin"; pwd` +# Figure out where Spark is installed +SOURCE=$0 +while [ -h "$SOURCE" ] +do + SOURCE="$(readlink "$SOURCE")" + [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" +done +export SPARK_HOME="$(cd `dirname $SOURCE`/..; pwd)" -. "$sbin/spark-config.sh" +. "$SPARK_HOME/sbin/spark-config.sh" -"$sbin"/spark-daemon.sh stop org.apache.spark.deploy.master.Master 1 +"$SPARK_HOME/sbin/spark-daemon.sh" stop org.apache.spark.deploy.master.Master 1 -if [ -e "$sbin"/../tachyon/bin/tachyon ]; then - "$sbin"/../tachyon/bin/tachyon killAll tachyon.master.Master +if [ -e "$SPARK_HOME/tachyon/bin/tachyon" ]; then + "$SPARK_HOME/tachyon/bin/tachyon" killAll tachyon.master.Master fi diff --git a/sbin/stop-slaves.sh b/sbin/stop-slaves.sh index 54c9bd46803a..253e468e8bed 100755 --- a/sbin/stop-slaves.sh +++ b/sbin/stop-slaves.sh @@ -17,16 +17,21 @@ # limitations under the License. # -sbin="`dirname "$0"`" -sbin="`cd "$sbin"; pwd`" +# Figure out where Spark is installed +SOURCE=$0 +while [ -h "$SOURCE" ] +do + SOURCE="$(readlink "$SOURCE")" + [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" +done +export SPARK_HOME="$(cd `dirname $SOURCE`/..; pwd)" -. "$sbin/spark-config.sh" - -. "$SPARK_PREFIX/bin/load-spark-env.sh" +. "$SPARK_HOME/sbin/spark-config.sh" +. "$SPARK_HOME/bin/load-spark-env.sh" # do before the below calls as they exec -if [ -e "$sbin"/../tachyon/bin/tachyon ]; then - "$sbin/slaves.sh" cd "$SPARK_HOME" \; "$sbin"/../tachyon/bin/tachyon killAll tachyon.worker.Worker +if [ -e "$SPARK_HOME/tachyon/bin/tachyon" ]; then + "$SPARK_HOME/sbin/slaves.sh" cd "$SPARK_HOME" \; "$SPARK_HOME"/tachyon/bin/tachyon killAll tachyon.worker.Worker fi "$sbin/slaves.sh" cd "$SPARK_HOME" \; "$sbin"/stop-slave.sh From 1c08d804d87cbd4d824113693be953b5d3ff3376 Mon Sep 17 00:00:00 2001 From: jerryshao Date: Wed, 9 Sep 2015 17:08:40 +0800 Subject: [PATCH 2/9] :Make Spark script work with symlink --- bin/beeline | 29 ++++++++++++++++++++++------- bin/load-spark-env.sh | 24 +++++++++++++++++++++++- bin/pyspark | 25 ++++++++++++++++++++++++- bin/run-example | 31 +++++++++++++++++++++---------- bin/spark-class | 32 ++++++++++++++++++++++---------- bin/spark-shell | 32 ++++++++++++++++++++++++-------- bin/spark-sql | 25 ++++++++++++++++++++++++- bin/spark-submit | 36 +++++++++++++++++++++++++----------- bin/sparkR | 25 ++++++++++++++++++++++++- sbin/slaves.sh | 14 ++++---------- sbin/spark-config.sh | 2 +- sbin/spark-daemon.sh | 20 +++++++------------- sbin/spark-daemons.sh | 14 ++++---------- sbin/start-all.sh | 16 +++++----------- sbin/start-history-server.sh | 14 ++++---------- sbin/start-master.sh | 24 ++++++++++-------------- sbin/start-slave.sh | 7 ++++--- sbin/start-slaves.sh | 20 ++++++++------------ sbin/start-thriftserver.sh | 8 +------- sbin/stop-all.sh | 13 ++++--------- sbin/stop-history-server.sh | 12 +++--------- sbin/stop-master.sh | 18 ++++++------------ sbin/stop-slaves.sh | 19 +++++++------------ 23 files changed, 277 insertions(+), 183 deletions(-) diff --git a/bin/beeline b/bin/beeline index a58513bbc830..ed4d568dcb34 100755 --- a/bin/beeline +++ b/bin/beeline @@ -23,14 +23,29 @@ # Enter posix mode for bash set -o posix +realpath () { +( + TARGET_FILE="$1" + + cd "$(dirname "$TARGET_FILE")" + TARGET_FILE="$(basename "$TARGET_FILE")" + + COUNT=0 + while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ] + do + TARGET_FILE="$(readlink "$TARGET_FILE")" + cd $(dirname "$TARGET_FILE") + TARGET_FILE="$(basename $TARGET_FILE)" + COUNT=$(($COUNT + 1)) + done + + echo "$(pwd -P)/"$TARGET_FILE"" +) +} + # Figure out where Spark is installed -SOURCE=$0 -while [ -h "$SOURCE" ] -do - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" -done -FWDIR="$(cd `dirname $SOURCE`/..; pwd)" +DIR="$(dirname "$(realpath "$0")")" +FWDIR="$(cd "$DIR/.."; pwd)" CLASS="org.apache.hive.beeline.BeeLine" exec "$FWDIR/bin/spark-class" $CLASS "$@" diff --git a/bin/load-spark-env.sh b/bin/load-spark-env.sh index 29e39dd8359f..7280d7fdb79c 100644 --- a/bin/load-spark-env.sh +++ b/bin/load-spark-env.sh @@ -20,7 +20,29 @@ # This script loads spark-env.sh if it exists, and ensures it is only loaded once. # spark-env.sh is loaded from SPARK_CONF_DIR if set, or within SPARK_HOME's # conf/ subdirectory. -FWDIR="$(cd "`dirname "$0"`"/..; pwd)" +realpath () { +( + TARGET_FILE="$1" + + cd "$(dirname "$TARGET_FILE")" + TARGET_FILE="$(basename "$TARGET_FILE")" + + COUNT=0 + while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ] + do + TARGET_FILE="$(readlink "$TARGET_FILE")" + cd $(dirname "$TARGET_FILE") + TARGET_FILE="$(basename $TARGET_FILE)" + COUNT=$(($COUNT + 1)) + done + + echo "$(pwd -P)/"$TARGET_FILE"" +) +} + +# Figure out where Spark is installed +DIR="$(dirname "$(realpath "$0")")" +FWDIR="$(cd "$DIR/.."; pwd)" if [ -z "$SPARK_ENV_LOADED" ]; then export SPARK_ENV_LOADED=1 diff --git a/bin/pyspark b/bin/pyspark index 18012ee4a0b4..88fd127c3d40 100755 --- a/bin/pyspark +++ b/bin/pyspark @@ -17,7 +17,30 @@ # limitations under the License. # -export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" +realpath () { +( + TARGET_FILE="$1" + + cd "$(dirname "$TARGET_FILE")" + TARGET_FILE="$(basename "$TARGET_FILE")" + + COUNT=0 + while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ] + do + TARGET_FILE="$(readlink "$TARGET_FILE")" + cd $(dirname "$TARGET_FILE") + TARGET_FILE="$(basename $TARGET_FILE)" + COUNT=$(($COUNT + 1)) + done + + echo "$(pwd -P)/"$TARGET_FILE"" +) +} + +# Figure out where Spark is installed +DIR="$(dirname "$(realpath "$0")")" + +export SPARK_HOME="$(cd "$DIR/.."; pwd)" source "$SPARK_HOME"/bin/load-spark-env.sh export _SPARK_CMD_USAGE="Usage: ./bin/pyspark [options]" diff --git a/bin/run-example b/bin/run-example index 6e7bc22187eb..144b3a5a9c99 100755 --- a/bin/run-example +++ b/bin/run-example @@ -17,17 +17,28 @@ # limitations under the License. # -SCALA_VERSION=2.10 - -# Figure out where Spark is installed -SOURCE=$0 -while [ -h "$SOURCE" ] -do - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" -done -FWDIR="$(cd `dirname $SOURCE`/..; pwd)" +realpath () { +( + TARGET_FILE="$1" + + cd "$(dirname "$TARGET_FILE")" + TARGET_FILE="$(basename "$TARGET_FILE")" + + COUNT=0 + while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ] + do + TARGET_FILE="$(readlink "$TARGET_FILE")" + cd $(dirname "$TARGET_FILE") + TARGET_FILE="$(basename $TARGET_FILE)" + COUNT=$(($COUNT + 1)) + done + + echo "$(pwd -P)/"$TARGET_FILE"" +) +} +DIR="$(dirname "$(realpath "$0")")" +FWDIR="$(cd "$DIR/.."; pwd)" export SPARK_HOME="$FWDIR" EXAMPLES_DIR="$FWDIR"/examples diff --git a/bin/spark-class b/bin/spark-class index 78e7cb81a632..a810a097e5f9 100755 --- a/bin/spark-class +++ b/bin/spark-class @@ -17,17 +17,29 @@ # limitations under the License. # -# Figure out where Spark is installed -SOURCE=$0 -while [ -h "$SOURCE" ] -do - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" -done -FWDIR="$(cd `dirname $SOURCE`/..; pwd)" +realpath () { +( + TARGET_FILE="$1" + + cd "$(dirname "$TARGET_FILE")" + TARGET_FILE="$(basename "$TARGET_FILE")" + + COUNT=0 + while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ] + do + TARGET_FILE="$(readlink "$TARGET_FILE")" + cd $(dirname "$TARGET_FILE") + TARGET_FILE="$(basename $TARGET_FILE)" + COUNT=$(($COUNT + 1)) + done -# Export this as SPARK_HOME -export SPARK_HOME="$FWDIR" + echo "$(pwd -P)/"$TARGET_FILE"" +) +} + +# Figure out where Spark is installed +DIR="$(dirname "$(realpath "$0")")" +export SPARK_HOME="$(cd "$DIR/.."; pwd)" . "$SPARK_HOME"/bin/load-spark-env.sh diff --git a/bin/spark-shell b/bin/spark-shell index 98c0704d0516..885bdf63ec8b 100755 --- a/bin/spark-shell +++ b/bin/spark-shell @@ -28,15 +28,31 @@ esac # Enter posix mode for bash set -o posix -export _SPARK_CMD_USAGE="Usage: ./bin/spark-shell [options]" +realpath () { +( + TARGET_FILE="$1" + + cd "$(dirname "$TARGET_FILE")" + TARGET_FILE="$(basename "$TARGET_FILE")" + + COUNT=0 + while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ] + do + TARGET_FILE="$(readlink "$TARGET_FILE")" + cd $(dirname "$TARGET_FILE") + TARGET_FILE="$(basename $TARGET_FILE)" + COUNT=$(($COUNT + 1)) + done + + echo "$(pwd -P)/"$TARGET_FILE"" +) +} + # Figure out where Spark is installed -SOURCE=$0 -while [ -h "$SOURCE" ] -do - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" -done -FWDIR="$(cd `dirname $SOURCE`/..; pwd)" +DIR="$(dirname "$(realpath "$0")")" + +export FWDIR="$(cd "$DIR/.."; pwd)" +export _SPARK_CMD_USAGE="Usage: ./bin/spark-shell [options]" # SPARK-4161: scala does not assume use of the java classpath, # so we need to add the "-Dscala.usejavacp=true" flag manually. We diff --git a/bin/spark-sql b/bin/spark-sql index 4ea7bc6e39c0..8973e41d2bb7 100755 --- a/bin/spark-sql +++ b/bin/spark-sql @@ -17,6 +17,29 @@ # limitations under the License. # -export FWDIR="$(cd "`dirname "$0"`"/..; pwd)" +realpath () { +( + TARGET_FILE="$1" + + cd "$(dirname "$TARGET_FILE")" + TARGET_FILE="$(basename "$TARGET_FILE")" + + COUNT=0 + while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ] + do + TARGET_FILE="$(readlink "$TARGET_FILE")" + cd $(dirname "$TARGET_FILE") + TARGET_FILE="$(basename $TARGET_FILE)" + COUNT=$(($COUNT + 1)) + done + + echo "$(pwd -P)/"$TARGET_FILE"" +) +} + +# Figure out where Spark is installed +DIR="$(dirname "$(realpath "$0")")" + +FWDIR="$(cd "$DIR/.."; pwd)" export _SPARK_CMD_USAGE="Usage: ./bin/spark-sql [options] [cli option]" exec "$FWDIR"/bin/spark-submit --class org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver "$@" diff --git a/bin/spark-submit b/bin/spark-submit index ea02b9af0194..efed9b1ad580 100755 --- a/bin/spark-submit +++ b/bin/spark-submit @@ -17,17 +17,31 @@ # limitations under the License. # +realpath () { +( + TARGET_FILE="$1" + + cd "$(dirname "$TARGET_FILE")" + TARGET_FILE="$(basename "$TARGET_FILE")" + + COUNT=0 + while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ] + do + TARGET_FILE="$(readlink "$TARGET_FILE")" + cd $(dirname "$TARGET_FILE") + TARGET_FILE="$(basename $TARGET_FILE)" + COUNT=$(($COUNT + 1)) + done + + echo "$(pwd -P)/"$TARGET_FILE"" +) +} + +# Figure out where Spark is installed +DIR="$(dirname "$(realpath "$0")")" +SPARK_HOME="$(cd "$DIR/.."; pwd)" + # disable randomized hash for string in Python 3.3+ export PYTHONHASHSEED=0 -# Figure out where Spark is installed -SOURCE=$0 -while [ -h "$SOURCE" ] -do - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" -done -FWDIR="$(cd `dirname $SOURCE`/..; pwd)" - -# Export this as SPARK_HOME -export SPARK_HOME="$FWDIR" + exec "$SPARK_HOME"/bin/spark-class org.apache.spark.deploy.SparkSubmit "$@" diff --git a/bin/sparkR b/bin/sparkR index 464c29f36942..7b5123134a09 100755 --- a/bin/sparkR +++ b/bin/sparkR @@ -17,7 +17,30 @@ # limitations under the License. # -export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" +realpath () { +( + TARGET_FILE="$1" + + cd "$(dirname "$TARGET_FILE")" + TARGET_FILE="$(basename "$TARGET_FILE")" + + COUNT=0 + while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ] + do + TARGET_FILE="$(readlink "$TARGET_FILE")" + cd $(dirname "$TARGET_FILE") + TARGET_FILE="$(basename $TARGET_FILE)" + COUNT=$(($COUNT + 1)) + done + + echo "$(pwd -P)/"$TARGET_FILE"" +) +} + +# Figure out where Spark is installed +DIR="$(dirname "$(realpath "$0")")" + +export SPARK_HOME="$(cd "$DIR/.."; pwd)" source "$SPARK_HOME"/bin/load-spark-env.sh export _SPARK_CMD_USAGE="Usage: ./bin/sparkR [options]" exec "$SPARK_HOME"/bin/spark-submit sparkr-shell-main "$@" diff --git a/sbin/slaves.sh b/sbin/slaves.sh index 80d7f7b1e664..cdad47ee2e59 100755 --- a/sbin/slaves.sh +++ b/sbin/slaves.sh @@ -36,16 +36,10 @@ if [ $# -le 0 ]; then exit 1 fi -# Figure out where Spark is installed -SOURCE=$0 -while [ -h "$SOURCE" ] -do - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" -done -export SPARK_HOME="$(cd `dirname $SOURCE`/..; pwd)" +sbin="`dirname "$0"`" +sbin="`cd "$sbin"; pwd`" -. "$SPARK_HOME/sbin/spark-config.sh" +. "$sbin/spark-config.sh" # If the slaves file is specified in the command line, # then it takes precedence over the definition in @@ -71,7 +65,7 @@ then shift fi -. "$SPARK_HOME/bin/load-spark-env.sh" +. "$SPARK_PREFIX/bin/load-spark-env.sh" if [ "$HOSTLIST" = "" ]; then if [ "$SPARK_SLAVES" = "" ]; then diff --git a/sbin/spark-config.sh b/sbin/spark-config.sh index 36c467f7e218..e6bf544c1479 100755 --- a/sbin/spark-config.sh +++ b/sbin/spark-config.sh @@ -17,7 +17,7 @@ # included in all the spark scripts with source command # should not be executable directly -# depends on SPARK_HOME being properly set in the sourcing script +# also should not be passed any arguments, since we need original $* # resolve links - $0 may be a softlink this="${BASH_SOURCE:-$0}" diff --git a/sbin/spark-daemon.sh b/sbin/spark-daemon.sh index fe6d147def0f..0fbe795822fb 100755 --- a/sbin/spark-daemon.sh +++ b/sbin/spark-daemon.sh @@ -37,16 +37,10 @@ if [ $# -le 1 ]; then exit 1 fi -# Figure out where Spark is installed -SOURCE=$0 -while [ -h "$SOURCE" ] -do - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" -done -export SPARK_HOME="$(cd `dirname $SOURCE`/..; pwd)" +sbin="`dirname "$0"`" +sbin="`cd "$sbin"; pwd`" -. "$SPARK_HOME/sbin/spark-config.sh" +. "$sbin/spark-config.sh" # get arguments @@ -92,7 +86,7 @@ spark_rotate_log () fi } -. "$SPARK_HOME/bin/load-spark-env.sh" +. "$SPARK_PREFIX/bin/load-spark-env.sh" if [ "$SPARK_IDENT_STRING" = "" ]; then export SPARK_IDENT_STRING="$USER" @@ -211,13 +205,13 @@ case $option in else echo $pid file is present but $command not running exit 1 - fi + fi else echo $command not running. exit 2 - fi + fi ;; - + (*) echo $usage exit 1 diff --git a/sbin/spark-daemons.sh b/sbin/spark-daemons.sh index d213eb82981f..5d9f2bb51cae 100755 --- a/sbin/spark-daemons.sh +++ b/sbin/spark-daemons.sh @@ -27,15 +27,9 @@ if [ $# -le 1 ]; then exit 1 fi -# Figure out where Spark is installed -SOURCE=$0 -while [ -h "$SOURCE" ] -do - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" -done -export SPARK_HOME="$(cd `dirname $SOURCE`/..; pwd)" +sbin=`dirname "$0"` +sbin=`cd "$sbin"; pwd` -. "$SPARK_HOME/sbin/spark-config.sh" +. "$sbin/spark-config.sh" -exec "$SPARK_HOME/sbin/slaves.sh" cd "$SPARK_HOME" \; "$SPARK_HOME/sbin/spark-daemon.sh" "$@" +exec "$sbin/slaves.sh" cd "$SPARK_HOME" \; "$sbin/spark-daemon.sh" "$@" diff --git a/sbin/start-all.sh b/sbin/start-all.sh index 39eb52349e5f..1baf57cea09e 100755 --- a/sbin/start-all.sh +++ b/sbin/start-all.sh @@ -21,14 +21,8 @@ # Starts the master on this node. # Starts a worker on each node specified in conf/slaves -# Figure out where Spark is installed -SOURCE=$0 -while [ -h "$SOURCE" ] -do - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" -done -export SPARK_HOME="$(cd `dirname $SOURCE`/..; pwd)" +sbin="`dirname "$0"`" +sbin="`cd "$sbin"; pwd`" TACHYON_STR="" @@ -42,10 +36,10 @@ shift done # Load the Spark configuration -. "$SPARK_HOME/sbin/spark-config.sh" +. "$sbin/spark-config.sh" # Start Master -"$SPARK_HOME/sbin/start-master.sh" $TACHYON_STR +"$sbin"/start-master.sh $TACHYON_STR # Start Workers -"$SPARK_HOME/sbin/start-slaves.sh" $TACHYON_STR +"$sbin"/start-slaves.sh $TACHYON_STR diff --git a/sbin/start-history-server.sh b/sbin/start-history-server.sh index a9a308a7f55f..9034e5715cc8 100755 --- a/sbin/start-history-server.sh +++ b/sbin/start-history-server.sh @@ -24,16 +24,10 @@ # Use the SPARK_HISTORY_OPTS environment variable to set history server configuration. # -# Figure out where Spark is installed -SOURCE=$0 -while [ -h "$SOURCE" ] -do - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" -done -export SPARK_HOME="$(cd `dirname $SOURCE`/..; pwd)" +sbin="`dirname "$0"`" +sbin="`cd "$sbin"; pwd`" -. "$SPARK_HOME/sbin/spark-config.sh" -. "$SPARK_HOME/bin/load-spark-env.sh" +. "$sbin/spark-config.sh" +. "$SPARK_PREFIX/bin/load-spark-env.sh" exec "$sbin"/spark-daemon.sh start org.apache.spark.deploy.history.HistoryServer 1 $@ diff --git a/sbin/start-master.sh b/sbin/start-master.sh index 4c62690ebdc6..a7f5d5702fd8 100755 --- a/sbin/start-master.sh +++ b/sbin/start-master.sh @@ -19,22 +19,17 @@ # Starts the master on the machine this script is executed on. +sbin="`dirname "$0"`" +sbin="`cd "$sbin"; pwd`" + ORIGINAL_ARGS="$@" -# Figure out where Spark is installed -SOURCE=$0 -while [ -h "$SOURCE" ] -do - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" -done -export SPARK_HOME="$(cd `dirname $SOURCE`/..; pwd)" START_TACHYON=false while (( "$#" )); do case $1 in --with-tachyon) - if [ ! -e "$SPARK_HOME/tachyon/bin/tachyon" ]; then + if [ ! -e "$sbin"/../tachyon/bin/tachyon ]; then echo "Error: --with-tachyon specified, but tachyon not found." exit -1 fi @@ -44,8 +39,9 @@ case $1 in shift done -. "$SPARK_HOME/sbin/spark-config.sh" -. "$SPARK_HOME/bin/load-spark-env.sh" +. "$sbin/spark-config.sh" + +. "$SPARK_PREFIX/bin/load-spark-env.sh" if [ "$SPARK_MASTER_PORT" = "" ]; then SPARK_MASTER_PORT=7077 @@ -64,7 +60,7 @@ fi $ORIGINAL_ARGS if [ "$START_TACHYON" == "true" ]; then - "$SPARK_HOME/tachyon/bin/tachyon" bootstrap-conf $SPARK_MASTER_IP - "$SPARK_HOME/tachyon/bin/tachyon" format -s - "$SPARK_HOME/tachyon/bin/tachyon-start.sh" master + "$sbin"/../tachyon/bin/tachyon bootstrap-conf $SPARK_MASTER_IP + "$sbin"/../tachyon/bin/tachyon format -s + "$sbin"/../tachyon/bin/tachyon-start.sh master fi diff --git a/sbin/start-slave.sh b/sbin/start-slave.sh index d15fba28ec82..4c919ff76a8f 100755 --- a/sbin/start-slave.sh +++ b/sbin/start-slave.sh @@ -21,14 +21,14 @@ # # Environment Variables # -# SPARK_WORKER_INSTANCES The number of worker instances to run on this +# SPARK_WORKER_INSTANCES The number of worker instances to run on this # slave. Default is 1. -# SPARK_WORKER_PORT The base port number for the first worker. If set, +# SPARK_WORKER_PORT The base port number for the first worker. If set, # subsequent workers will increment this number. If # unset, Spark will find a valid port number, but # with no guarantee of a predictable pattern. # SPARK_WORKER_WEBUI_PORT The base port for the web interface of the first -# worker. Subsequent workers will increment this +# worker. Subsequent workers will increment this # number. Default is 8081. usage="Usage: start-slave.sh where is like spark://localhost:7077" @@ -82,3 +82,4 @@ else start_instance $(( 1 + $i )) "$@" done fi + diff --git a/sbin/start-slaves.sh b/sbin/start-slaves.sh index 74a6bd15d6d2..24d6268815ed 100755 --- a/sbin/start-slaves.sh +++ b/sbin/start-slaves.sh @@ -19,21 +19,16 @@ # Starts a slave instance on each machine specified in the conf/slaves file. -# Figure out where Spark is installed -SOURCE=$0 -while [ -h "$SOURCE" ] -do - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" -done -export SPARK_HOME="$(cd `dirname $SOURCE`/..; pwd)" +sbin="`dirname "$0"`" +sbin="`cd "$sbin"; pwd`" + START_TACHYON=false while (( "$#" )); do case $1 in --with-tachyon) - if [ ! -e "$SPARK_HOME/tachyon/bin/tachyon" ]; then + if [ ! -e "$sbin"/../tachyon/bin/tachyon ]; then echo "Error: --with-tachyon specified, but tachyon not found." exit -1 fi @@ -43,8 +38,9 @@ case $1 in shift done -. "$SPARK_HOME/sbin/spark-config.sh" -. "$SPARK_HOME/bin/load-spark-env.sh" +. "$sbin/spark-config.sh" + +. "$SPARK_PREFIX/bin/load-spark-env.sh" # Find the port number for the master if [ "$SPARK_MASTER_PORT" = "" ]; then @@ -59,7 +55,7 @@ if [ "$START_TACHYON" == "true" ]; then "$sbin/slaves.sh" cd "$SPARK_HOME" \; "$sbin"/../tachyon/bin/tachyon bootstrap-conf "$SPARK_MASTER_IP" # set -t so we can call sudo - SPARK_SSH_OPTS="-o StrictHostKeyChecking=no -t" "$SPARK_HOME/sbin/slaves.sh" cd "$SPARK_HOME" \; "$SPARK_HOME/tachyon/bin/tachyon-start.sh" worker SudoMount \; sleep 1 + SPARK_SSH_OPTS="-o StrictHostKeyChecking=no -t" "$sbin/slaves.sh" cd "$SPARK_HOME" \; "$sbin/../tachyon/bin/tachyon-start.sh" worker SudoMount \; sleep 1 fi # Launch the slaves diff --git a/sbin/start-thriftserver.sh b/sbin/start-thriftserver.sh index ec3e421190e0..5b0aeb177fff 100755 --- a/sbin/start-thriftserver.sh +++ b/sbin/start-thriftserver.sh @@ -24,13 +24,7 @@ set -o posix # Figure out where Spark is installed -SOURCE=$0 -while [ -h "$SOURCE" ] -do - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" -done -export SPARK_HOME="$(cd `dirname $SOURCE`/..; pwd)" +FWDIR="$(cd "`dirname "$0"`"/..; pwd)" # NOTE: This exact class name is matched downstream by SparkSubmit. # Any changes need to be reflected there. diff --git a/sbin/stop-all.sh b/sbin/stop-all.sh index 72d4881c1307..1a9abe07db84 100755 --- a/sbin/stop-all.sh +++ b/sbin/stop-all.sh @@ -20,17 +20,12 @@ # Stop all spark daemons. # Run this on the master node. -# Figure out where Spark is installed -SOURCE=$0 -while [ -h "$SOURCE" ] -do - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" -done -export SPARK_HOME="$(cd `dirname $SOURCE`/..; pwd)" + +sbin="`dirname "$0"`" +sbin="`cd "$sbin"; pwd`" # Load the Spark configuration -. "$SPARK_HOME/sbin/spark-config.sh" +. "$sbin/spark-config.sh" # Stop the slaves, then the master "$sbin"/stop-slaves.sh diff --git a/sbin/stop-history-server.sh b/sbin/stop-history-server.sh index 0960e13a29d7..6e6056359510 100755 --- a/sbin/stop-history-server.sh +++ b/sbin/stop-history-server.sh @@ -19,13 +19,7 @@ # Stops the history server on the machine this script is executed on. -# Figure out where Spark is installed -SOURCE=$0 -while [ -h "$SOURCE" ] -do - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" -done -export SPARK_HOME="$(cd `dirname $SOURCE`/..; pwd)" +sbin="`dirname "$0"`" +sbin="`cd "$sbin"; pwd`" -"$SPARK_HOME/sbin/spark-daemon.sh" stop org.apache.spark.deploy.history.HistoryServer 1 +"$sbin"/spark-daemon.sh stop org.apache.spark.deploy.history.HistoryServer 1 diff --git a/sbin/stop-master.sh b/sbin/stop-master.sh index b0397dae59df..729702d92191 100755 --- a/sbin/stop-master.sh +++ b/sbin/stop-master.sh @@ -19,19 +19,13 @@ # Stops the master on the machine this script is executed on. -# Figure out where Spark is installed -SOURCE=$0 -while [ -h "$SOURCE" ] -do - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" -done -export SPARK_HOME="$(cd `dirname $SOURCE`/..; pwd)" +sbin=`dirname "$0"` +sbin=`cd "$sbin"; pwd` -. "$SPARK_HOME/sbin/spark-config.sh" +. "$sbin/spark-config.sh" -"$SPARK_HOME/sbin/spark-daemon.sh" stop org.apache.spark.deploy.master.Master 1 +"$sbin"/spark-daemon.sh stop org.apache.spark.deploy.master.Master 1 -if [ -e "$SPARK_HOME/tachyon/bin/tachyon" ]; then - "$SPARK_HOME/tachyon/bin/tachyon" killAll tachyon.master.Master +if [ -e "$sbin"/../tachyon/bin/tachyon ]; then + "$sbin"/../tachyon/bin/tachyon killAll tachyon.master.Master fi diff --git a/sbin/stop-slaves.sh b/sbin/stop-slaves.sh index 253e468e8bed..54c9bd46803a 100755 --- a/sbin/stop-slaves.sh +++ b/sbin/stop-slaves.sh @@ -17,21 +17,16 @@ # limitations under the License. # -# Figure out where Spark is installed -SOURCE=$0 -while [ -h "$SOURCE" ] -do - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" -done -export SPARK_HOME="$(cd `dirname $SOURCE`/..; pwd)" +sbin="`dirname "$0"`" +sbin="`cd "$sbin"; pwd`" -. "$SPARK_HOME/sbin/spark-config.sh" -. "$SPARK_HOME/bin/load-spark-env.sh" +. "$sbin/spark-config.sh" + +. "$SPARK_PREFIX/bin/load-spark-env.sh" # do before the below calls as they exec -if [ -e "$SPARK_HOME/tachyon/bin/tachyon" ]; then - "$SPARK_HOME/sbin/slaves.sh" cd "$SPARK_HOME" \; "$SPARK_HOME"/tachyon/bin/tachyon killAll tachyon.worker.Worker +if [ -e "$sbin"/../tachyon/bin/tachyon ]; then + "$sbin/slaves.sh" cd "$SPARK_HOME" \; "$sbin"/../tachyon/bin/tachyon killAll tachyon.worker.Worker fi "$sbin/slaves.sh" cd "$SPARK_HOME" \; "$sbin"/stop-slave.sh From 350bec938a6dafdb796e632b3db6558b18d8be5d Mon Sep 17 00:00:00 2001 From: jerryshao Date: Wed, 9 Sep 2015 21:53:22 +0800 Subject: [PATCH 3/9] recover the comment --- bin/load-spark-env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/load-spark-env.sh b/bin/load-spark-env.sh index 7280d7fdb79c..1cd7ebc05b8c 100644 --- a/bin/load-spark-env.sh +++ b/bin/load-spark-env.sh @@ -18,7 +18,7 @@ # # This script loads spark-env.sh if it exists, and ensures it is only loaded once. -# spark-env.sh is loaded from SPARK_CONF_DIR if set, or within SPARK_HOME's +# spark-env.sh is loaded from SPARK_CONF_DIR if set, or within the current directory's # conf/ subdirectory. realpath () { ( From aa470fe472b05187007804a5ff4c5726caac4f97 Mon Sep 17 00:00:00 2001 From: jerryshao Date: Thu, 10 Sep 2015 15:02:27 +0800 Subject: [PATCH 4/9] Address the same issue for sbin scripts --- sbin/slaves.sh | 25 +++++++++++++++++++++-- sbin/spark-daemon.sh | 30 +++++++++++++++++++++++----- sbin/spark-daemons.sh | 24 ++++++++++++++++++++-- sbin/start-all.sh | 24 ++++++++++++++++++++-- sbin/start-history-server.sh | 24 ++++++++++++++++++++-- sbin/start-master.sh | 24 ++++++++++++++++++++-- sbin/start-mesos-dispatcher.sh | 24 ++++++++++++++++++++-- sbin/start-mesos-shuffle-service.sh | 24 ++++++++++++++++++++-- sbin/start-shuffle-service.sh | 24 ++++++++++++++++++++-- sbin/start-slave.sh | 31 +++++++++++++++++++++++------ sbin/start-slaves.sh | 27 +++++++++++++++++++++---- sbin/start-thriftserver.sh | 25 +++++++++++++++++++++-- sbin/stop-all.sh | 24 ++++++++++++++++++++-- sbin/stop-history-server.sh | 26 +++++++++++++++++++++--- sbin/stop-master.sh | 25 +++++++++++++++++++++-- sbin/stop-mesos-dispatcher.sh | 24 ++++++++++++++++++++-- sbin/stop-mesos-shuffle-service.sh | 24 ++++++++++++++++++++-- sbin/stop-shuffle-service.sh | 24 ++++++++++++++++++++-- sbin/stop-slave.sh | 26 +++++++++++++++++++++--- sbin/stop-slaves.sh | 24 ++++++++++++++++++++-- sbin/stop-thriftserver.sh | 24 ++++++++++++++++++++-- 21 files changed, 474 insertions(+), 53 deletions(-) diff --git a/sbin/slaves.sh b/sbin/slaves.sh index cdad47ee2e59..2b1df527ae98 100755 --- a/sbin/slaves.sh +++ b/sbin/slaves.sh @@ -36,8 +36,29 @@ if [ $# -le 0 ]; then exit 1 fi -sbin="`dirname "$0"`" -sbin="`cd "$sbin"; pwd`" +realpath () { +( + TARGET_FILE="$1" + + cd "$(dirname "$TARGET_FILE")" + TARGET_FILE="$(basename "$TARGET_FILE")" + + COUNT=0 + while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ] + do + TARGET_FILE="$(readlink "$TARGET_FILE")" + cd $(dirname "$TARGET_FILE") + TARGET_FILE="$(basename $TARGET_FILE)" + COUNT=$(($COUNT + 1)) + done + + echo "$(pwd -P)/"$TARGET_FILE"" +) +} + +# Figure out where Spark is installed +sbin="$(dirname "$(realpath "$0")")" +sbin="$(cd "$sbin"; pwd)" . "$sbin/spark-config.sh" diff --git a/sbin/spark-daemon.sh b/sbin/spark-daemon.sh index 0fbe795822fb..a02aac4467f6 100755 --- a/sbin/spark-daemon.sh +++ b/sbin/spark-daemon.sh @@ -37,8 +37,28 @@ if [ $# -le 1 ]; then exit 1 fi -sbin="`dirname "$0"`" -sbin="`cd "$sbin"; pwd`" +realpath () { +( + TARGET_FILE="$1" + + cd "$(dirname "$TARGET_FILE")" + TARGET_FILE="$(basename "$TARGET_FILE")" + + COUNT=0 + while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ] + do + TARGET_FILE="$(readlink "$TARGET_FILE")" + cd $(dirname "$TARGET_FILE") + TARGET_FILE="$(basename $TARGET_FILE)" + COUNT=$(($COUNT + 1)) + done + + echo "$(pwd -P)/"$TARGET_FILE"" +) +} + +sbin="$(dirname "$(realpath "$0")")" +sbin="$(cd "$sbin"; pwd)" . "$sbin/spark-config.sh" @@ -205,13 +225,13 @@ case $option in else echo $pid file is present but $command not running exit 1 - fi + fi else echo $command not running. exit 2 - fi + fi ;; - + (*) echo $usage exit 1 diff --git a/sbin/spark-daemons.sh b/sbin/spark-daemons.sh index 5d9f2bb51cae..afb86375e49d 100755 --- a/sbin/spark-daemons.sh +++ b/sbin/spark-daemons.sh @@ -27,8 +27,28 @@ if [ $# -le 1 ]; then exit 1 fi -sbin=`dirname "$0"` -sbin=`cd "$sbin"; pwd` +realpath () { +( + TARGET_FILE="$1" + + cd "$(dirname "$TARGET_FILE")" + TARGET_FILE="$(basename "$TARGET_FILE")" + + COUNT=0 + while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ] + do + TARGET_FILE="$(readlink "$TARGET_FILE")" + cd $(dirname "$TARGET_FILE") + TARGET_FILE="$(basename $TARGET_FILE)" + COUNT=$(($COUNT + 1)) + done + + echo "$(pwd -P)/"$TARGET_FILE"" +) +} + +sbin="$(dirname "$(realpath "$0")")" +sbin="$(cd "$sbin"; pwd)" . "$sbin/spark-config.sh" diff --git a/sbin/start-all.sh b/sbin/start-all.sh index 1baf57cea09e..cce3110c5b4f 100755 --- a/sbin/start-all.sh +++ b/sbin/start-all.sh @@ -21,8 +21,28 @@ # Starts the master on this node. # Starts a worker on each node specified in conf/slaves -sbin="`dirname "$0"`" -sbin="`cd "$sbin"; pwd`" +realpath () { +( + TARGET_FILE="$1" + + cd "$(dirname "$TARGET_FILE")" + TARGET_FILE="$(basename "$TARGET_FILE")" + + COUNT=0 + while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ] + do + TARGET_FILE="$(readlink "$TARGET_FILE")" + cd $(dirname "$TARGET_FILE") + TARGET_FILE="$(basename $TARGET_FILE)" + COUNT=$(($COUNT + 1)) + done + + echo "$(pwd -P)/"$TARGET_FILE"" +) +} + +sbin="$(dirname "$(realpath "$0")")" +sbin="$(cd "$sbin"; pwd)" TACHYON_STR="" diff --git a/sbin/start-history-server.sh b/sbin/start-history-server.sh index 9034e5715cc8..d9da02c325e8 100755 --- a/sbin/start-history-server.sh +++ b/sbin/start-history-server.sh @@ -24,8 +24,28 @@ # Use the SPARK_HISTORY_OPTS environment variable to set history server configuration. # -sbin="`dirname "$0"`" -sbin="`cd "$sbin"; pwd`" +realpath () { +( + TARGET_FILE="$1" + + cd "$(dirname "$TARGET_FILE")" + TARGET_FILE="$(basename "$TARGET_FILE")" + + COUNT=0 + while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ] + do + TARGET_FILE="$(readlink "$TARGET_FILE")" + cd $(dirname "$TARGET_FILE") + TARGET_FILE="$(basename $TARGET_FILE)" + COUNT=$(($COUNT + 1)) + done + + echo "$(pwd -P)/"$TARGET_FILE"" +) +} + +sbin="$(dirname "$(realpath "$0")")" +sbin="$(cd "$sbin"; pwd)" . "$sbin/spark-config.sh" . "$SPARK_PREFIX/bin/load-spark-env.sh" diff --git a/sbin/start-master.sh b/sbin/start-master.sh index a7f5d5702fd8..bb7f3abcff10 100755 --- a/sbin/start-master.sh +++ b/sbin/start-master.sh @@ -19,8 +19,28 @@ # Starts the master on the machine this script is executed on. -sbin="`dirname "$0"`" -sbin="`cd "$sbin"; pwd`" +realpath () { +( + TARGET_FILE="$1" + + cd "$(dirname "$TARGET_FILE")" + TARGET_FILE="$(basename "$TARGET_FILE")" + + COUNT=0 + while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ] + do + TARGET_FILE="$(readlink "$TARGET_FILE")" + cd $(dirname "$TARGET_FILE") + TARGET_FILE="$(basename $TARGET_FILE)" + COUNT=$(($COUNT + 1)) + done + + echo "$(pwd -P)/"$TARGET_FILE"" +) +} + +sbin="$(dirname "$(realpath "$0")")" +sbin="$(cd "$sbin"; pwd)" ORIGINAL_ARGS="$@" diff --git a/sbin/start-mesos-dispatcher.sh b/sbin/start-mesos-dispatcher.sh index ef1fc573d5c6..55eb737ec450 100755 --- a/sbin/start-mesos-dispatcher.sh +++ b/sbin/start-mesos-dispatcher.sh @@ -21,8 +21,28 @@ # Rest server to handle driver requests for Mesos cluster mode. # Only one cluster dispatcher is needed per Mesos cluster. -sbin="`dirname "$0"`" -sbin="`cd "$sbin"; pwd`" +realpath () { +( + TARGET_FILE="$1" + + cd "$(dirname "$TARGET_FILE")" + TARGET_FILE="$(basename "$TARGET_FILE")" + + COUNT=0 + while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ] + do + TARGET_FILE="$(readlink "$TARGET_FILE")" + cd $(dirname "$TARGET_FILE") + TARGET_FILE="$(basename $TARGET_FILE)" + COUNT=$(($COUNT + 1)) + done + + echo "$(pwd -P)/"$TARGET_FILE"" +) +} + +sbin="$(dirname "$(realpath "$0")")" +sbin="$(cd "$sbin"; pwd)" . "$sbin/spark-config.sh" diff --git a/sbin/start-mesos-shuffle-service.sh b/sbin/start-mesos-shuffle-service.sh index 64580762c5dc..a636d32d0a94 100755 --- a/sbin/start-mesos-shuffle-service.sh +++ b/sbin/start-mesos-shuffle-service.sh @@ -26,8 +26,28 @@ # Use the SPARK_SHUFFLE_OPTS environment variable to set shuffle service configuration. # -sbin="`dirname "$0"`" -sbin="`cd "$sbin"; pwd`" +realpath () { +( + TARGET_FILE="$1" + + cd "$(dirname "$TARGET_FILE")" + TARGET_FILE="$(basename "$TARGET_FILE")" + + COUNT=0 + while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ] + do + TARGET_FILE="$(readlink "$TARGET_FILE")" + cd $(dirname "$TARGET_FILE") + TARGET_FILE="$(basename $TARGET_FILE)" + COUNT=$(($COUNT + 1)) + done + + echo "$(pwd -P)/"$TARGET_FILE"" +) +} + +sbin="$(dirname "$(realpath "$0")")" +sbin="$(cd "$sbin"; pwd)" . "$sbin/spark-config.sh" . "$SPARK_PREFIX/bin/load-spark-env.sh" diff --git a/sbin/start-shuffle-service.sh b/sbin/start-shuffle-service.sh index 4fddcf7f95d4..47043bdfb351 100755 --- a/sbin/start-shuffle-service.sh +++ b/sbin/start-shuffle-service.sh @@ -24,8 +24,28 @@ # Use the SPARK_SHUFFLE_OPTS environment variable to set shuffle server configuration. # -sbin="`dirname "$0"`" -sbin="`cd "$sbin"; pwd`" +realpath () { +( + TARGET_FILE="$1" + + cd "$(dirname "$TARGET_FILE")" + TARGET_FILE="$(basename "$TARGET_FILE")" + + COUNT=0 + while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ] + do + TARGET_FILE="$(readlink "$TARGET_FILE")" + cd $(dirname "$TARGET_FILE") + TARGET_FILE="$(basename $TARGET_FILE)" + COUNT=$(($COUNT + 1)) + done + + echo "$(pwd -P)/"$TARGET_FILE"" +) +} + +sbin="$(dirname "$(realpath "$0")")" +sbin="$(cd "$sbin"; pwd)" . "$sbin/spark-config.sh" . "$SPARK_PREFIX/bin/load-spark-env.sh" diff --git a/sbin/start-slave.sh b/sbin/start-slave.sh index 4c919ff76a8f..4292d11f9435 100755 --- a/sbin/start-slave.sh +++ b/sbin/start-slave.sh @@ -21,14 +21,14 @@ # # Environment Variables # -# SPARK_WORKER_INSTANCES The number of worker instances to run on this +# SPARK_WORKER_INSTANCES The number of worker instances to run on this # slave. Default is 1. -# SPARK_WORKER_PORT The base port number for the first worker. If set, +# SPARK_WORKER_PORT The base port number for the first worker. If set, # subsequent workers will increment this number. If # unset, Spark will find a valid port number, but # with no guarantee of a predictable pattern. # SPARK_WORKER_WEBUI_PORT The base port for the web interface of the first -# worker. Subsequent workers will increment this +# worker. Subsequent workers will increment this # number. Default is 8081. usage="Usage: start-slave.sh where is like spark://localhost:7077" @@ -39,8 +39,28 @@ if [ $# -lt 1 ]; then exit 1 fi -sbin="`dirname "$0"`" -sbin="`cd "$sbin"; pwd`" +realpath () { +( + TARGET_FILE="$1" + + cd "$(dirname "$TARGET_FILE")" + TARGET_FILE="$(basename "$TARGET_FILE")" + + COUNT=0 + while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ] + do + TARGET_FILE="$(readlink "$TARGET_FILE")" + cd $(dirname "$TARGET_FILE") + TARGET_FILE="$(basename $TARGET_FILE)" + COUNT=$(($COUNT + 1)) + done + + echo "$(pwd -P)/"$TARGET_FILE"" +) +} + +sbin="$(dirname "$(realpath "$0")")" +sbin="$(cd "$sbin"; pwd)" . "$sbin/spark-config.sh" @@ -82,4 +102,3 @@ else start_instance $(( 1 + $i )) "$@" done fi - diff --git a/sbin/start-slaves.sh b/sbin/start-slaves.sh index 24d6268815ed..548b604ed625 100755 --- a/sbin/start-slaves.sh +++ b/sbin/start-slaves.sh @@ -19,8 +19,28 @@ # Starts a slave instance on each machine specified in the conf/slaves file. -sbin="`dirname "$0"`" -sbin="`cd "$sbin"; pwd`" +realpath () { +( + TARGET_FILE="$1" + + cd "$(dirname "$TARGET_FILE")" + TARGET_FILE="$(basename "$TARGET_FILE")" + + COUNT=0 + while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ] + do + TARGET_FILE="$(readlink "$TARGET_FILE")" + cd $(dirname "$TARGET_FILE") + TARGET_FILE="$(basename $TARGET_FILE)" + COUNT=$(($COUNT + 1)) + done + + echo "$(pwd -P)/"$TARGET_FILE"" +) +} + +sbin="$(dirname "$(realpath "$0")")" +sbin="$(cd "$sbin"; pwd)" START_TACHYON=false @@ -39,7 +59,6 @@ shift done . "$sbin/spark-config.sh" - . "$SPARK_PREFIX/bin/load-spark-env.sh" # Find the port number for the master @@ -55,7 +74,7 @@ if [ "$START_TACHYON" == "true" ]; then "$sbin/slaves.sh" cd "$SPARK_HOME" \; "$sbin"/../tachyon/bin/tachyon bootstrap-conf "$SPARK_MASTER_IP" # set -t so we can call sudo - SPARK_SSH_OPTS="-o StrictHostKeyChecking=no -t" "$sbin/slaves.sh" cd "$SPARK_HOME" \; "$sbin/../tachyon/bin/tachyon-start.sh" worker SudoMount \; sleep 1 + SPARK_SSH_OPTS="-o StrictHostKeyChecking=no -t" "$SPARK_HOME/sbin/slaves.sh" cd "$SPARK_HOME" \; "$SPARK_HOME/tachyon/bin/tachyon-start.sh" worker SudoMount \; sleep 1 fi # Launch the slaves diff --git a/sbin/start-thriftserver.sh b/sbin/start-thriftserver.sh index 5b0aeb177fff..1a260b7ffa52 100755 --- a/sbin/start-thriftserver.sh +++ b/sbin/start-thriftserver.sh @@ -23,8 +23,29 @@ # Enter posix mode for bash set -o posix -# Figure out where Spark is installed -FWDIR="$(cd "`dirname "$0"`"/..; pwd)" +realpath () { +( + TARGET_FILE="$1" + + cd "$(dirname "$TARGET_FILE")" + TARGET_FILE="$(basename "$TARGET_FILE")" + + COUNT=0 + while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ] + do + TARGET_FILE="$(readlink "$TARGET_FILE")" + cd $(dirname "$TARGET_FILE") + TARGET_FILE="$(basename $TARGET_FILE)" + COUNT=$(($COUNT + 1)) + done + + echo "$(pwd -P)/"$TARGET_FILE"" +) +} + +#Figure out where Spark is installed +DIR="$(dirname "$(realpath "$0")")" +FWDIR="$(cd "$DIR/.."; pwd)" # NOTE: This exact class name is matched downstream by SparkSubmit. # Any changes need to be reflected there. diff --git a/sbin/stop-all.sh b/sbin/stop-all.sh index 1a9abe07db84..998129819864 100755 --- a/sbin/stop-all.sh +++ b/sbin/stop-all.sh @@ -21,8 +21,28 @@ # Run this on the master node. -sbin="`dirname "$0"`" -sbin="`cd "$sbin"; pwd`" +realpath () { +( + TARGET_FILE="$1" + + cd "$(dirname "$TARGET_FILE")" + TARGET_FILE="$(basename "$TARGET_FILE")" + + COUNT=0 + while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ] + do + TARGET_FILE="$(readlink "$TARGET_FILE")" + cd $(dirname "$TARGET_FILE") + TARGET_FILE="$(basename $TARGET_FILE)" + COUNT=$(($COUNT + 1)) + done + + echo "$(pwd -P)/"$TARGET_FILE"" +) +} + +sbin="$(dirname "$(realpath "$0")")" +sbin="$(cd "$sbin"; pwd)" # Load the Spark configuration . "$sbin/spark-config.sh" diff --git a/sbin/stop-history-server.sh b/sbin/stop-history-server.sh index 6e6056359510..9122c9c20c33 100755 --- a/sbin/stop-history-server.sh +++ b/sbin/stop-history-server.sh @@ -19,7 +19,27 @@ # Stops the history server on the machine this script is executed on. -sbin="`dirname "$0"`" -sbin="`cd "$sbin"; pwd`" +realpath () { +( + TARGET_FILE="$1" -"$sbin"/spark-daemon.sh stop org.apache.spark.deploy.history.HistoryServer 1 + cd "$(dirname "$TARGET_FILE")" + TARGET_FILE="$(basename "$TARGET_FILE")" + + COUNT=0 + while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ] + do + TARGET_FILE="$(readlink "$TARGET_FILE")" + cd $(dirname "$TARGET_FILE") + TARGET_FILE="$(basename $TARGET_FILE)" + COUNT=$(($COUNT + 1)) + done + + echo "$(pwd -P)/"$TARGET_FILE"" +) +} + +sbin="$(dirname "$(realpath "$0")")" +sbin="$(cd "$sbin"; pwd)" + +"$sbin/spark-daemon.sh" stop org.apache.spark.deploy.history.HistoryServer 1 diff --git a/sbin/stop-master.sh b/sbin/stop-master.sh index 729702d92191..28edb9bf0715 100755 --- a/sbin/stop-master.sh +++ b/sbin/stop-master.sh @@ -19,8 +19,29 @@ # Stops the master on the machine this script is executed on. -sbin=`dirname "$0"` -sbin=`cd "$sbin"; pwd` +realpath () { +( + TARGET_FILE="$1" + + cd "$(dirname "$TARGET_FILE")" + TARGET_FILE="$(basename "$TARGET_FILE")" + + COUNT=0 + while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ] + do + TARGET_FILE="$(readlink "$TARGET_FILE")" + cd $(dirname "$TARGET_FILE") + TARGET_FILE="$(basename $TARGET_FILE)" + COUNT=$(($COUNT + 1)) + done + + echo "$(pwd -P)/"$TARGET_FILE"" +) +} + +sbin="$(dirname "$(realpath "$0")")" +sbin="$(cd "$sbin"; pwd)" + . "$sbin/spark-config.sh" diff --git a/sbin/stop-mesos-dispatcher.sh b/sbin/stop-mesos-dispatcher.sh index cb65d95b5e52..fe54e36c7a05 100755 --- a/sbin/stop-mesos-dispatcher.sh +++ b/sbin/stop-mesos-dispatcher.sh @@ -18,8 +18,28 @@ # # Stop the Mesos Cluster dispatcher on the machine this script is executed on. -sbin=`dirname "$0"` -sbin=`cd "$sbin"; pwd` +realpath () { +( + TARGET_FILE="$1" + + cd "$(dirname "$TARGET_FILE")" + TARGET_FILE="$(basename "$TARGET_FILE")" + + COUNT=0 + while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ] + do + TARGET_FILE="$(readlink "$TARGET_FILE")" + cd $(dirname "$TARGET_FILE") + TARGET_FILE="$(basename $TARGET_FILE)" + COUNT=$(($COUNT + 1)) + done + + echo "$(pwd -P)/"$TARGET_FILE"" +) +} + +sbin="$(dirname "$(realpath "$0")")" +sbin="$(cd "$sbin"; pwd)" . "$sbin/spark-config.sh" diff --git a/sbin/stop-mesos-shuffle-service.sh b/sbin/stop-mesos-shuffle-service.sh index 0e965d5ec588..02bdb6235342 100755 --- a/sbin/stop-mesos-shuffle-service.sh +++ b/sbin/stop-mesos-shuffle-service.sh @@ -19,7 +19,27 @@ # Stops the Mesos external shuffle service on the machine this script is executed on. -sbin="`dirname "$0"`" -sbin="`cd "$sbin"; pwd`" +realpath () { +( + TARGET_FILE="$1" + + cd "$(dirname "$TARGET_FILE")" + TARGET_FILE="$(basename "$TARGET_FILE")" + + COUNT=0 + while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ] + do + TARGET_FILE="$(readlink "$TARGET_FILE")" + cd $(dirname "$TARGET_FILE") + TARGET_FILE="$(basename $TARGET_FILE)" + COUNT=$(($COUNT + 1)) + done + + echo "$(pwd -P)/"$TARGET_FILE"" +) +} + +sbin="$(dirname "$(realpath "$0")")" +sbin="$(cd "$sbin"; pwd)" "$sbin"/spark-daemon.sh stop org.apache.spark.deploy.mesos.MesosExternalShuffleService 1 diff --git a/sbin/stop-shuffle-service.sh b/sbin/stop-shuffle-service.sh index 4cb6891ae27f..6d229a830326 100755 --- a/sbin/stop-shuffle-service.sh +++ b/sbin/stop-shuffle-service.sh @@ -19,7 +19,27 @@ # Stops the external shuffle service on the machine this script is executed on. -sbin="`dirname "$0"`" -sbin="`cd "$sbin"; pwd`" +realpath () { +( + TARGET_FILE="$1" + + cd "$(dirname "$TARGET_FILE")" + TARGET_FILE="$(basename "$TARGET_FILE")" + + COUNT=0 + while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ] + do + TARGET_FILE="$(readlink "$TARGET_FILE")" + cd $(dirname "$TARGET_FILE") + TARGET_FILE="$(basename $TARGET_FILE)" + COUNT=$(($COUNT + 1)) + done + + echo "$(pwd -P)/"$TARGET_FILE"" +) +} + +sbin="$(dirname "$(realpath "$0")")" +sbin="$(cd "$sbin"; pwd)" "$sbin"/spark-daemon.sh stop org.apache.spark.deploy.ExternalShuffleService 1 diff --git a/sbin/stop-slave.sh b/sbin/stop-slave.sh index 3d1da5b254f2..204b00115552 100755 --- a/sbin/stop-slave.sh +++ b/sbin/stop-slave.sh @@ -21,14 +21,34 @@ # # Environment variables # -# SPARK_WORKER_INSTANCES The number of worker instances that should be +# SPARK_WORKER_INSTANCES The number of worker instances that should be # running on this slave. Default is 1. # Usage: stop-slave.sh # Stops all slaves on this worker machine -sbin="`dirname "$0"`" -sbin="`cd "$sbin"; pwd`" +realpath () { +( + TARGET_FILE="$1" + + cd "$(dirname "$TARGET_FILE")" + TARGET_FILE="$(basename "$TARGET_FILE")" + + COUNT=0 + while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ] + do + TARGET_FILE="$(readlink "$TARGET_FILE")" + cd $(dirname "$TARGET_FILE") + TARGET_FILE="$(basename $TARGET_FILE)" + COUNT=$(($COUNT + 1)) + done + + echo "$(pwd -P)/"$TARGET_FILE"" +) +} + +sbin="$(dirname "$(realpath "$0")")" +sbin="$(cd "$sbin"; pwd)" . "$sbin/spark-config.sh" diff --git a/sbin/stop-slaves.sh b/sbin/stop-slaves.sh index 54c9bd46803a..38044d4fd7df 100755 --- a/sbin/stop-slaves.sh +++ b/sbin/stop-slaves.sh @@ -17,8 +17,28 @@ # limitations under the License. # -sbin="`dirname "$0"`" -sbin="`cd "$sbin"; pwd`" +realpath () { +( + TARGET_FILE="$1" + + cd "$(dirname "$TARGET_FILE")" + TARGET_FILE="$(basename "$TARGET_FILE")" + + COUNT=0 + while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ] + do + TARGET_FILE="$(readlink "$TARGET_FILE")" + cd $(dirname "$TARGET_FILE") + TARGET_FILE="$(basename $TARGET_FILE)" + COUNT=$(($COUNT + 1)) + done + + echo "$(pwd -P)/"$TARGET_FILE"" +) +} + +sbin="$(dirname "$(realpath "$0")")" +sbin="$(cd "$sbin"; pwd)" . "$sbin/spark-config.sh" diff --git a/sbin/stop-thriftserver.sh b/sbin/stop-thriftserver.sh index 4031a00d4a68..e758884e210c 100755 --- a/sbin/stop-thriftserver.sh +++ b/sbin/stop-thriftserver.sh @@ -19,7 +19,27 @@ # Stops the thrift server on the machine this script is executed on. -sbin="`dirname "$0"`" -sbin="`cd "$sbin"; pwd`" +realpath () { +( + TARGET_FILE="$1" + + cd "$(dirname "$TARGET_FILE")" + TARGET_FILE="$(basename "$TARGET_FILE")" + + COUNT=0 + while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ] + do + TARGET_FILE="$(readlink "$TARGET_FILE")" + cd $(dirname "$TARGET_FILE") + TARGET_FILE="$(basename $TARGET_FILE)" + COUNT=$(($COUNT + 1)) + done + + echo "$(pwd -P)/"$TARGET_FILE"" +) +} + +sbin="$(dirname "$(realpath "$0")")" +sbin="$(cd "$sbin"; pwd)" "$sbin"/spark-daemon.sh stop org.apache.spark.sql.hive.thriftserver.HiveThriftServer2 1 From c796397da5159335de3c11e62ecedc427a9c5a64 Mon Sep 17 00:00:00 2001 From: jerryshao Date: Tue, 27 Oct 2015 11:34:19 +0800 Subject: [PATCH 5/9] Changes to honor SPARK_HOME to solve symlink issue --- bin/beeline | 29 ++++------------------ bin/load-spark-env.sh | 30 +++++------------------ bin/pyspark | 37 ++++++---------------------- bin/run-example | 37 +++++++--------------------- bin/spark-class | 36 ++++++--------------------- bin/spark-shell | 30 ++++------------------- bin/spark-sql | 28 +++------------------ bin/spark-submit | 28 +++------------------ bin/sparkR | 30 ++++------------------- sbin/slaves.sh | 30 ++++------------------- sbin/spark-config.sh | 17 +++---------- sbin/spark-daemon.sh | 33 ++++++------------------- sbin/spark-daemons.sh | 29 ++++------------------ sbin/start-all.sh | 31 +++++------------------ sbin/start-history-server.sh | 31 +++++------------------ sbin/start-master.sh | 37 +++++++--------------------- sbin/start-mesos-dispatcher.sh | 31 +++++------------------ sbin/start-mesos-shuffle-service.sh | 31 +++++------------------ sbin/start-shuffle-service.sh | 31 +++++------------------ sbin/start-slave.sh | 31 +++++------------------ sbin/start-slaves.sh | 38 +++++++---------------------- sbin/start-thriftserver.sh | 32 +++++------------------- sbin/stop-all.sh | 34 ++++++-------------------- sbin/stop-history-server.sh | 27 +++----------------- sbin/stop-master.sh | 34 ++++++-------------------- sbin/stop-mesos-dispatcher.sh | 29 ++++------------------ sbin/stop-mesos-shuffle-service.sh | 27 +++----------------- sbin/stop-shuffle-service.sh | 27 +++----------------- sbin/stop-slave.sh | 33 ++++++------------------- sbin/stop-slaves.sh | 35 ++++++-------------------- sbin/stop-thriftserver.sh | 27 +++----------------- 31 files changed, 185 insertions(+), 775 deletions(-) diff --git a/bin/beeline b/bin/beeline index ed4d568dcb34..84a150764567 100755 --- a/bin/beeline +++ b/bin/beeline @@ -23,29 +23,10 @@ # Enter posix mode for bash set -o posix -realpath () { -( - TARGET_FILE="$1" - - cd "$(dirname "$TARGET_FILE")" - TARGET_FILE="$(basename "$TARGET_FILE")" - - COUNT=0 - while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ] - do - TARGET_FILE="$(readlink "$TARGET_FILE")" - cd $(dirname "$TARGET_FILE") - TARGET_FILE="$(basename $TARGET_FILE)" - COUNT=$(($COUNT + 1)) - done - - echo "$(pwd -P)/"$TARGET_FILE"" -) -} - -# Figure out where Spark is installed -DIR="$(dirname "$(realpath "$0")")" -FWDIR="$(cd "$DIR/.."; pwd)" +# Figure out if SPARK_HOME is set +if [ -z "${SPARK_HOME}" ]; then + export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" +fi CLASS="org.apache.hive.beeline.BeeLine" -exec "$FWDIR/bin/spark-class" $CLASS "$@" +exec "${SPARK_HOME}/bin/spark-class" $CLASS "$@" diff --git a/bin/load-spark-env.sh b/bin/load-spark-env.sh index 1cd7ebc05b8c..50528ead7a78 100644 --- a/bin/load-spark-env.sh +++ b/bin/load-spark-env.sh @@ -20,35 +20,17 @@ # This script loads spark-env.sh if it exists, and ensures it is only loaded once. # spark-env.sh is loaded from SPARK_CONF_DIR if set, or within the current directory's # conf/ subdirectory. -realpath () { -( - TARGET_FILE="$1" - - cd "$(dirname "$TARGET_FILE")" - TARGET_FILE="$(basename "$TARGET_FILE")" - - COUNT=0 - while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ] - do - TARGET_FILE="$(readlink "$TARGET_FILE")" - cd $(dirname "$TARGET_FILE") - TARGET_FILE="$(basename $TARGET_FILE)" - COUNT=$(($COUNT + 1)) - done - - echo "$(pwd -P)/"$TARGET_FILE"" -) -} # Figure out where Spark is installed -DIR="$(dirname "$(realpath "$0")")" -FWDIR="$(cd "$DIR/.."; pwd)" +if [ -z "${SPARK_HOME}" ]; then + export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" +fi if [ -z "$SPARK_ENV_LOADED" ]; then export SPARK_ENV_LOADED=1 # Returns the parent of the directory this script lives in. - parent_dir="$(cd "`dirname "$0"`"/..; pwd)" + parent_dir="${SPARK_HOME}" user_conf_dir="${SPARK_CONF_DIR:-"$parent_dir"/conf}" @@ -64,8 +46,8 @@ fi if [ -z "$SPARK_SCALA_VERSION" ]; then - ASSEMBLY_DIR2="$FWDIR/assembly/target/scala-2.11" - ASSEMBLY_DIR1="$FWDIR/assembly/target/scala-2.10" + ASSEMBLY_DIR2="${SPARK_HOME}/assembly/target/scala-2.11" + ASSEMBLY_DIR1="${SPARK_HOME}/assembly/target/scala-2.10" if [[ -d "$ASSEMBLY_DIR2" && -d "$ASSEMBLY_DIR1" ]]; then echo -e "Presence of build for both scala versions(SCALA 2.10 and SCALA 2.11) detected." 1>&2 diff --git a/bin/pyspark b/bin/pyspark index 88fd127c3d40..0f9310cf222b 100755 --- a/bin/pyspark +++ b/bin/pyspark @@ -17,32 +17,11 @@ # limitations under the License. # -realpath () { -( - TARGET_FILE="$1" - - cd "$(dirname "$TARGET_FILE")" - TARGET_FILE="$(basename "$TARGET_FILE")" - - COUNT=0 - while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ] - do - TARGET_FILE="$(readlink "$TARGET_FILE")" - cd $(dirname "$TARGET_FILE") - TARGET_FILE="$(basename $TARGET_FILE)" - COUNT=$(($COUNT + 1)) - done - - echo "$(pwd -P)/"$TARGET_FILE"" -) -} - -# Figure out where Spark is installed -DIR="$(dirname "$(realpath "$0")")" - -export SPARK_HOME="$(cd "$DIR/.."; pwd)" +if [ -z "${SPARK_HOME}" ]; then + export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" +fi -source "$SPARK_HOME"/bin/load-spark-env.sh +source "{$SPARK_HOME}"/bin/load-spark-env.sh export _SPARK_CMD_USAGE="Usage: ./bin/pyspark [options]" # In Spark <= 1.1, setting IPYTHON=1 would cause the driver to be launched using the `ipython` @@ -87,12 +66,12 @@ fi export PYSPARK_PYTHON # Add the PySpark classes to the Python path: -export PYTHONPATH="$SPARK_HOME/python/:$PYTHONPATH" -export PYTHONPATH="$SPARK_HOME/python/lib/py4j-0.9-src.zip:$PYTHONPATH" +export PYTHONPATH="${SPARK_HOME}/python/:$PYTHONPATH" +export PYTHONPATH="${SPARK_HOME}/python/lib/py4j-0.9-src.zip:$PYTHONPATH" # Load the PySpark shell.py script when ./pyspark is used interactively: export OLD_PYTHONSTARTUP="$PYTHONSTARTUP" -export PYTHONSTARTUP="$SPARK_HOME/python/pyspark/shell.py" +export PYTHONSTARTUP="${SPARK_HOME}/python/pyspark/shell.py" # For pyspark tests if [[ -n "$SPARK_TESTING" ]]; then @@ -105,4 +84,4 @@ fi export PYSPARK_DRIVER_PYTHON export PYSPARK_DRIVER_PYTHON_OPTS -exec "$SPARK_HOME"/bin/spark-submit pyspark-shell-main --name "PySparkShell" "$@" +exec "${SPARK_HOME}"/bin/spark-submit pyspark-shell-main --name "PySparkShell" "$@" diff --git a/bin/run-example b/bin/run-example index 144b3a5a9c99..438877049778 100755 --- a/bin/run-example +++ b/bin/run-example @@ -17,32 +17,13 @@ # limitations under the License. # -realpath () { -( - TARGET_FILE="$1" - - cd "$(dirname "$TARGET_FILE")" - TARGET_FILE="$(basename "$TARGET_FILE")" - - COUNT=0 - while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ] - do - TARGET_FILE="$(readlink "$TARGET_FILE")" - cd $(dirname "$TARGET_FILE") - TARGET_FILE="$(basename $TARGET_FILE)" - COUNT=$(($COUNT + 1)) - done - - echo "$(pwd -P)/"$TARGET_FILE"" -) -} +if [ -z "${SPARK_HOME}" ]; then + export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" +fi -DIR="$(dirname "$(realpath "$0")")" -FWDIR="$(cd "$DIR/.."; pwd)" -export SPARK_HOME="$FWDIR" -EXAMPLES_DIR="$FWDIR"/examples +EXAMPLES_DIR="${SPARK_HOME}"/examples -. "$FWDIR"/bin/load-spark-env.sh +. "${SPARK_HOME}"/bin/load-spark-env.sh if [ -n "$1" ]; then EXAMPLE_CLASS="$1" @@ -55,8 +36,8 @@ else exit 1 fi -if [ -f "$FWDIR/RELEASE" ]; then - JAR_PATH="${FWDIR}/lib" +if [ -f "${SPARK_HOME}/RELEASE" ]; then + JAR_PATH="${SPARK_HOME}/lib" else JAR_PATH="${EXAMPLES_DIR}/target/scala-${SPARK_SCALA_VERSION}" fi @@ -65,7 +46,7 @@ JAR_COUNT=0 for f in "${JAR_PATH}"/spark-examples-*hadoop*.jar; do if [[ ! -e "$f" ]]; then - echo "Failed to find Spark examples assembly in $FWDIR/lib or $FWDIR/examples/target" 1>&2 + echo "Failed to find Spark examples assembly in ${SPARK_HOME}/lib or $SPARK_HOME/examples/target" 1>&2 echo "You need to build Spark before running this program" 1>&2 exit 1 fi @@ -88,7 +69,7 @@ if [[ ! $EXAMPLE_CLASS == org.apache.spark.examples* ]]; then EXAMPLE_CLASS="org.apache.spark.examples.$EXAMPLE_CLASS" fi -exec "$FWDIR"/bin/spark-submit \ +exec "${SPARK_HOME}"/bin/spark-submit \ --master $EXAMPLE_MASTER \ --class $EXAMPLE_CLASS \ "$SPARK_EXAMPLES_JAR" \ diff --git a/bin/spark-class b/bin/spark-class index a810a097e5f9..f6aad0bba8fc 100755 --- a/bin/spark-class +++ b/bin/spark-class @@ -17,31 +17,11 @@ # limitations under the License. # -realpath () { -( - TARGET_FILE="$1" - - cd "$(dirname "$TARGET_FILE")" - TARGET_FILE="$(basename "$TARGET_FILE")" - - COUNT=0 - while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ] - do - TARGET_FILE="$(readlink "$TARGET_FILE")" - cd $(dirname "$TARGET_FILE") - TARGET_FILE="$(basename $TARGET_FILE)" - COUNT=$(($COUNT + 1)) - done - - echo "$(pwd -P)/"$TARGET_FILE"" -) -} - -# Figure out where Spark is installed -DIR="$(dirname "$(realpath "$0")")" -export SPARK_HOME="$(cd "$DIR/.."; pwd)" +if [ -z "${SPARK_HOME}" ]; then + export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" +fi -. "$SPARK_HOME"/bin/load-spark-env.sh +. "${SPARK_HOME}"/bin/load-spark-env.sh # Find the java binary if [ -n "${JAVA_HOME}" ]; then @@ -57,10 +37,10 @@ fi # Find assembly jar SPARK_ASSEMBLY_JAR= -if [ -f "$SPARK_HOME/RELEASE" ]; then - ASSEMBLY_DIR="$SPARK_HOME/lib" +if [ -f "${SPARK_HOME}/RELEASE" ]; then + ASSEMBLY_DIR="${SPARK_HOME}/lib" else - ASSEMBLY_DIR="$SPARK_HOME/assembly/target/scala-$SPARK_SCALA_VERSION" + ASSEMBLY_DIR="${SPARK_HOME}/assembly/target/scala-$SPARK_SCALA_VERSION" fi GREP_OPTIONS= @@ -86,7 +66,7 @@ LAUNCH_CLASSPATH="$SPARK_ASSEMBLY_JAR" # Add the launcher build dir to the classpath if requested. if [ -n "$SPARK_PREPEND_CLASSES" ]; then - LAUNCH_CLASSPATH="$SPARK_HOME/launcher/target/scala-$SPARK_SCALA_VERSION/classes:$LAUNCH_CLASSPATH" + LAUNCH_CLASSPATH="${SPARK_HOME}/launcher/target/scala-$SPARK_SCALA_VERSION/classes:$LAUNCH_CLASSPATH" fi export _SPARK_ASSEMBLY="$SPARK_ASSEMBLY_JAR" diff --git a/bin/spark-shell b/bin/spark-shell index 885bdf63ec8b..1ac6e6b86223 100755 --- a/bin/spark-shell +++ b/bin/spark-shell @@ -28,30 +28,10 @@ esac # Enter posix mode for bash set -o posix -realpath () { -( - TARGET_FILE="$1" - - cd "$(dirname "$TARGET_FILE")" - TARGET_FILE="$(basename "$TARGET_FILE")" - - COUNT=0 - while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ] - do - TARGET_FILE="$(readlink "$TARGET_FILE")" - cd $(dirname "$TARGET_FILE") - TARGET_FILE="$(basename $TARGET_FILE)" - COUNT=$(($COUNT + 1)) - done - - echo "$(pwd -P)/"$TARGET_FILE"" -) -} - -# Figure out where Spark is installed -DIR="$(dirname "$(realpath "$0")")" +if [ -z "${SPARK_HOME}" ]; then + export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" +fi -export FWDIR="$(cd "$DIR/.."; pwd)" export _SPARK_CMD_USAGE="Usage: ./bin/spark-shell [options]" # SPARK-4161: scala does not assume use of the java classpath, @@ -70,11 +50,11 @@ function main() { # (see https://github.com/sbt/sbt/issues/562). stty -icanon min 1 -echo > /dev/null 2>&1 export SPARK_SUBMIT_OPTS="$SPARK_SUBMIT_OPTS -Djline.terminal=unix" - "$FWDIR"/bin/spark-submit --class org.apache.spark.repl.Main --name "Spark shell" "$@" + "${SPARK_HOME}"/bin/spark-submit --class org.apache.spark.repl.Main --name "Spark shell" "$@" stty icanon echo > /dev/null 2>&1 else export SPARK_SUBMIT_OPTS - "$FWDIR"/bin/spark-submit --class org.apache.spark.repl.Main --name "Spark shell" "$@" + "${SPARK_HOME}"/bin/spark-submit --class org.apache.spark.repl.Main --name "Spark shell" "$@" fi } diff --git a/bin/spark-sql b/bin/spark-sql index 8973e41d2bb7..fa8433f35a13 100755 --- a/bin/spark-sql +++ b/bin/spark-sql @@ -17,29 +17,9 @@ # limitations under the License. # -realpath () { -( - TARGET_FILE="$1" +if [ -z "${SPARK_HOME}" ]; then + export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" +fi - cd "$(dirname "$TARGET_FILE")" - TARGET_FILE="$(basename "$TARGET_FILE")" - - COUNT=0 - while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ] - do - TARGET_FILE="$(readlink "$TARGET_FILE")" - cd $(dirname "$TARGET_FILE") - TARGET_FILE="$(basename $TARGET_FILE)" - COUNT=$(($COUNT + 1)) - done - - echo "$(pwd -P)/"$TARGET_FILE"" -) -} - -# Figure out where Spark is installed -DIR="$(dirname "$(realpath "$0")")" - -FWDIR="$(cd "$DIR/.."; pwd)" export _SPARK_CMD_USAGE="Usage: ./bin/spark-sql [options] [cli option]" -exec "$FWDIR"/bin/spark-submit --class org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver "$@" +exec "${SPARK_HOME}"/bin/spark-submit --class org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver "$@" diff --git a/bin/spark-submit b/bin/spark-submit index efed9b1ad580..8953167a574d 100755 --- a/bin/spark-submit +++ b/bin/spark-submit @@ -17,31 +17,11 @@ # limitations under the License. # -realpath () { -( - TARGET_FILE="$1" - - cd "$(dirname "$TARGET_FILE")" - TARGET_FILE="$(basename "$TARGET_FILE")" - - COUNT=0 - while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ] - do - TARGET_FILE="$(readlink "$TARGET_FILE")" - cd $(dirname "$TARGET_FILE") - TARGET_FILE="$(basename $TARGET_FILE)" - COUNT=$(($COUNT + 1)) - done - - echo "$(pwd -P)/"$TARGET_FILE"" -) -} - -# Figure out where Spark is installed -DIR="$(dirname "$(realpath "$0")")" -SPARK_HOME="$(cd "$DIR/.."; pwd)" +if [ -z "${SPARK_HOME}" ]; then + export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" +fi # disable randomized hash for string in Python 3.3+ export PYTHONHASHSEED=0 -exec "$SPARK_HOME"/bin/spark-class org.apache.spark.deploy.SparkSubmit "$@" +exec "${SPARK_HOME}"/bin/spark-class org.apache.spark.deploy.SparkSubmit "$@" diff --git a/bin/sparkR b/bin/sparkR index 7b5123134a09..6dec68934852 100755 --- a/bin/sparkR +++ b/bin/sparkR @@ -17,30 +17,10 @@ # limitations under the License. # -realpath () { -( - TARGET_FILE="$1" +if [ -z "${SPARK_HOME}" ]; then + export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" +fi - cd "$(dirname "$TARGET_FILE")" - TARGET_FILE="$(basename "$TARGET_FILE")" - - COUNT=0 - while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ] - do - TARGET_FILE="$(readlink "$TARGET_FILE")" - cd $(dirname "$TARGET_FILE") - TARGET_FILE="$(basename $TARGET_FILE)" - COUNT=$(($COUNT + 1)) - done - - echo "$(pwd -P)/"$TARGET_FILE"" -) -} - -# Figure out where Spark is installed -DIR="$(dirname "$(realpath "$0")")" - -export SPARK_HOME="$(cd "$DIR/.."; pwd)" -source "$SPARK_HOME"/bin/load-spark-env.sh +source "${SPARK_HOME}"/bin/load-spark-env.sh export _SPARK_CMD_USAGE="Usage: ./bin/sparkR [options]" -exec "$SPARK_HOME"/bin/spark-submit sparkr-shell-main "$@" +exec "${SPARK_HOME}"/bin/spark-submit sparkr-shell-main "$@" diff --git a/sbin/slaves.sh b/sbin/slaves.sh index 2b1df527ae98..110fa552c231 100755 --- a/sbin/slaves.sh +++ b/sbin/slaves.sh @@ -36,31 +36,11 @@ if [ $# -le 0 ]; then exit 1 fi -realpath () { -( - TARGET_FILE="$1" - - cd "$(dirname "$TARGET_FILE")" - TARGET_FILE="$(basename "$TARGET_FILE")" - - COUNT=0 - while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ] - do - TARGET_FILE="$(readlink "$TARGET_FILE")" - cd $(dirname "$TARGET_FILE") - TARGET_FILE="$(basename $TARGET_FILE)" - COUNT=$(($COUNT + 1)) - done - - echo "$(pwd -P)/"$TARGET_FILE"" -) -} - -# Figure out where Spark is installed -sbin="$(dirname "$(realpath "$0")")" -sbin="$(cd "$sbin"; pwd)" +if [ -z "${SPARK_HOME}" ]; then + export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" +fi -. "$sbin/spark-config.sh" +. "${SPARK_HOME}/sbin/spark-config.sh" # If the slaves file is specified in the command line, # then it takes precedence over the definition in @@ -86,7 +66,7 @@ then shift fi -. "$SPARK_PREFIX/bin/load-spark-env.sh" +. "${SPARK_HOME}/bin/load-spark-env.sh" if [ "$HOSTLIST" = "" ]; then if [ "$SPARK_SLAVES" = "" ]; then diff --git a/sbin/spark-config.sh b/sbin/spark-config.sh index e6bf544c1479..0bc2334c8d05 100755 --- a/sbin/spark-config.sh +++ b/sbin/spark-config.sh @@ -19,20 +19,11 @@ # should not be executable directly # also should not be passed any arguments, since we need original $* -# resolve links - $0 may be a softlink -this="${BASH_SOURCE:-$0}" -common_bin="$(cd -P -- "$(dirname -- "$this")" && pwd -P)" -script="$(basename -- "$this")" -this="$common_bin/$script" +# symlink and absolute path should rely on SPARK_HOME to resolve +if [ -z "${SPARK_HOME}" ]; then + export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" +fi -# convert relative path to absolute path -config_bin="`dirname "$this"`" -script="`basename "$this"`" -config_bin="`cd "$config_bin"; pwd`" -this="$config_bin/$script" - -export SPARK_PREFIX="`dirname "$this"`"/.. -export SPARK_HOME="${SPARK_PREFIX}" export SPARK_CONF_DIR="${SPARK_CONF_DIR:-"$SPARK_HOME/conf"}" # Add the PySpark classes to the PYTHONPATH: export PYTHONPATH="$SPARK_HOME/python:$PYTHONPATH" diff --git a/sbin/spark-daemon.sh b/sbin/spark-daemon.sh index a02aac4467f6..c981afdab558 100755 --- a/sbin/spark-daemon.sh +++ b/sbin/spark-daemon.sh @@ -37,30 +37,11 @@ if [ $# -le 1 ]; then exit 1 fi -realpath () { -( - TARGET_FILE="$1" - - cd "$(dirname "$TARGET_FILE")" - TARGET_FILE="$(basename "$TARGET_FILE")" - - COUNT=0 - while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ] - do - TARGET_FILE="$(readlink "$TARGET_FILE")" - cd $(dirname "$TARGET_FILE") - TARGET_FILE="$(basename $TARGET_FILE)" - COUNT=$(($COUNT + 1)) - done - - echo "$(pwd -P)/"$TARGET_FILE"" -) -} - -sbin="$(dirname "$(realpath "$0")")" -sbin="$(cd "$sbin"; pwd)" +if [ -z "${SPARK_HOME}" ]; then + export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" +fi -. "$sbin/spark-config.sh" +. "${SPARK_HOME}/sbin/spark-config.sh" # get arguments @@ -106,7 +87,7 @@ spark_rotate_log () fi } -. "$SPARK_PREFIX/bin/load-spark-env.sh" +. "${SPARK_HOME}/bin/load-spark-env.sh" if [ "$SPARK_IDENT_STRING" = "" ]; then export SPARK_IDENT_STRING="$USER" @@ -165,12 +146,12 @@ run_command() { case "$mode" in (class) - nohup nice -n "$SPARK_NICENESS" "$SPARK_PREFIX"/bin/spark-class $command "$@" >> "$log" 2>&1 < /dev/null & + nohup nice -n "$SPARK_NICENESS" "${SPARK_HOME}"/bin/spark-class $command "$@" >> "$log" 2>&1 < /dev/null & newpid="$!" ;; (submit) - nohup nice -n "$SPARK_NICENESS" "$SPARK_PREFIX"/bin/spark-submit --class $command "$@" >> "$log" 2>&1 < /dev/null & + nohup nice -n "$SPARK_NICENESS" "${SPARK_HOME}"/bin/spark-submit --class $command "$@" >> "$log" 2>&1 < /dev/null & newpid="$!" ;; diff --git a/sbin/spark-daemons.sh b/sbin/spark-daemons.sh index afb86375e49d..732a933ffe3b 100755 --- a/sbin/spark-daemons.sh +++ b/sbin/spark-daemons.sh @@ -27,29 +27,10 @@ if [ $# -le 1 ]; then exit 1 fi -realpath () { -( - TARGET_FILE="$1" - - cd "$(dirname "$TARGET_FILE")" - TARGET_FILE="$(basename "$TARGET_FILE")" - - COUNT=0 - while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ] - do - TARGET_FILE="$(readlink "$TARGET_FILE")" - cd $(dirname "$TARGET_FILE") - TARGET_FILE="$(basename $TARGET_FILE)" - COUNT=$(($COUNT + 1)) - done - - echo "$(pwd -P)/"$TARGET_FILE"" -) -} - -sbin="$(dirname "$(realpath "$0")")" -sbin="$(cd "$sbin"; pwd)" +if [ -z "${SPARK_HOME}" ]; then + export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" +fi -. "$sbin/spark-config.sh" +. "${SPARK_HOME}/sbin/spark-config.sh" -exec "$sbin/slaves.sh" cd "$SPARK_HOME" \; "$sbin/spark-daemon.sh" "$@" +exec "${SPARK_HOME}/sbin/slaves.sh" cd "${SPARK_HOME}" \; "${SPARK_HOME}/sbin/spark-daemon.sh" "$@" diff --git a/sbin/start-all.sh b/sbin/start-all.sh index cce3110c5b4f..9e86832b1eaa 100755 --- a/sbin/start-all.sh +++ b/sbin/start-all.sh @@ -21,28 +21,9 @@ # Starts the master on this node. # Starts a worker on each node specified in conf/slaves -realpath () { -( - TARGET_FILE="$1" - - cd "$(dirname "$TARGET_FILE")" - TARGET_FILE="$(basename "$TARGET_FILE")" - - COUNT=0 - while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ] - do - TARGET_FILE="$(readlink "$TARGET_FILE")" - cd $(dirname "$TARGET_FILE") - TARGET_FILE="$(basename $TARGET_FILE)" - COUNT=$(($COUNT + 1)) - done - - echo "$(pwd -P)/"$TARGET_FILE"" -) -} - -sbin="$(dirname "$(realpath "$0")")" -sbin="$(cd "$sbin"; pwd)" +if [ -z "${SPARK_HOME}" ]; then + export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" +fi TACHYON_STR="" @@ -56,10 +37,10 @@ shift done # Load the Spark configuration -. "$sbin/spark-config.sh" +. "${SPARK_HOME}/sbin/spark-config.sh" # Start Master -"$sbin"/start-master.sh $TACHYON_STR +"${SPARK_HOME}sbin"/start-master.sh $TACHYON_STR # Start Workers -"$sbin"/start-slaves.sh $TACHYON_STR +"${SPARK_HOME}/sbin"/start-slaves.sh $TACHYON_STR diff --git a/sbin/start-history-server.sh b/sbin/start-history-server.sh index d9da02c325e8..96e52ef3e3b0 100755 --- a/sbin/start-history-server.sh +++ b/sbin/start-history-server.sh @@ -24,30 +24,11 @@ # Use the SPARK_HISTORY_OPTS environment variable to set history server configuration. # -realpath () { -( - TARGET_FILE="$1" +if [ -z "${SPARK_HOME}" ]; then + export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" +fi - cd "$(dirname "$TARGET_FILE")" - TARGET_FILE="$(basename "$TARGET_FILE")" +. "${SPARK_HOME}/sbin/spark-config.sh" +. "${SPARK_HOME}/bin/load-spark-env.sh" - COUNT=0 - while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ] - do - TARGET_FILE="$(readlink "$TARGET_FILE")" - cd $(dirname "$TARGET_FILE") - TARGET_FILE="$(basename $TARGET_FILE)" - COUNT=$(($COUNT + 1)) - done - - echo "$(pwd -P)/"$TARGET_FILE"" -) -} - -sbin="$(dirname "$(realpath "$0")")" -sbin="$(cd "$sbin"; pwd)" - -. "$sbin/spark-config.sh" -. "$SPARK_PREFIX/bin/load-spark-env.sh" - -exec "$sbin"/spark-daemon.sh start org.apache.spark.deploy.history.HistoryServer 1 $@ +exec "${SPARK_HOME}/sbin"/spark-daemon.sh start org.apache.spark.deploy.history.HistoryServer 1 $@ diff --git a/sbin/start-master.sh b/sbin/start-master.sh index bb7f3abcff10..2a81473a5219 100755 --- a/sbin/start-master.sh +++ b/sbin/start-master.sh @@ -19,28 +19,9 @@ # Starts the master on the machine this script is executed on. -realpath () { -( - TARGET_FILE="$1" - - cd "$(dirname "$TARGET_FILE")" - TARGET_FILE="$(basename "$TARGET_FILE")" - - COUNT=0 - while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ] - do - TARGET_FILE="$(readlink "$TARGET_FILE")" - cd $(dirname "$TARGET_FILE") - TARGET_FILE="$(basename $TARGET_FILE)" - COUNT=$(($COUNT + 1)) - done - - echo "$(pwd -P)/"$TARGET_FILE"" -) -} - -sbin="$(dirname "$(realpath "$0")")" -sbin="$(cd "$sbin"; pwd)" +if [ -z "${SPARK_HOME}" ]; then + export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" +fi ORIGINAL_ARGS="$@" @@ -59,9 +40,9 @@ case $1 in shift done -. "$sbin/spark-config.sh" +. "${SPARK_HOME}/sbin/spark-config.sh" -. "$SPARK_PREFIX/bin/load-spark-env.sh" +. "${SPARK_HOME}/bin/load-spark-env.sh" if [ "$SPARK_MASTER_PORT" = "" ]; then SPARK_MASTER_PORT=7077 @@ -75,12 +56,12 @@ if [ "$SPARK_MASTER_WEBUI_PORT" = "" ]; then SPARK_MASTER_WEBUI_PORT=8080 fi -"$sbin"/spark-daemon.sh start org.apache.spark.deploy.master.Master 1 \ +"${SPARK_HOME}/sbin"/spark-daemon.sh start org.apache.spark.deploy.master.Master 1 \ --ip $SPARK_MASTER_IP --port $SPARK_MASTER_PORT --webui-port $SPARK_MASTER_WEBUI_PORT \ $ORIGINAL_ARGS if [ "$START_TACHYON" == "true" ]; then - "$sbin"/../tachyon/bin/tachyon bootstrap-conf $SPARK_MASTER_IP - "$sbin"/../tachyon/bin/tachyon format -s - "$sbin"/../tachyon/bin/tachyon-start.sh master + "${SPARK_HOME}/sbin"/../tachyon/bin/tachyon bootstrap-conf $SPARK_MASTER_IP + "${SPARK_HOME}/sbin"/../tachyon/bin/tachyon format -s + "${SPARK_HOME}/sbin"/../tachyon/bin/tachyon-start.sh master fi diff --git a/sbin/start-mesos-dispatcher.sh b/sbin/start-mesos-dispatcher.sh index 55eb737ec450..a43d162d5fa9 100755 --- a/sbin/start-mesos-dispatcher.sh +++ b/sbin/start-mesos-dispatcher.sh @@ -21,32 +21,13 @@ # Rest server to handle driver requests for Mesos cluster mode. # Only one cluster dispatcher is needed per Mesos cluster. -realpath () { -( - TARGET_FILE="$1" - - cd "$(dirname "$TARGET_FILE")" - TARGET_FILE="$(basename "$TARGET_FILE")" - - COUNT=0 - while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ] - do - TARGET_FILE="$(readlink "$TARGET_FILE")" - cd $(dirname "$TARGET_FILE") - TARGET_FILE="$(basename $TARGET_FILE)" - COUNT=$(($COUNT + 1)) - done - - echo "$(pwd -P)/"$TARGET_FILE"" -) -} - -sbin="$(dirname "$(realpath "$0")")" -sbin="$(cd "$sbin"; pwd)" +if [ -z "${SPARK_HOME}" ]; then + export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" +fi -. "$sbin/spark-config.sh" +. "${SPARK_HOME}/sbin/spark-config.sh" -. "$SPARK_PREFIX/bin/load-spark-env.sh" +. "${SPARK_HOME}/bin/load-spark-env.sh" if [ "$SPARK_MESOS_DISPATCHER_PORT" = "" ]; then SPARK_MESOS_DISPATCHER_PORT=7077 @@ -57,4 +38,4 @@ if [ "$SPARK_MESOS_DISPATCHER_HOST" = "" ]; then fi -"$sbin"/spark-daemon.sh start org.apache.spark.deploy.mesos.MesosClusterDispatcher 1 --host $SPARK_MESOS_DISPATCHER_HOST --port $SPARK_MESOS_DISPATCHER_PORT "$@" +"${SPARK_HOME}/sbin"/spark-daemon.sh start org.apache.spark.deploy.mesos.MesosClusterDispatcher 1 --host $SPARK_MESOS_DISPATCHER_HOST --port $SPARK_MESOS_DISPATCHER_PORT "$@" diff --git a/sbin/start-mesos-shuffle-service.sh b/sbin/start-mesos-shuffle-service.sh index a636d32d0a94..29442bfcbb8a 100755 --- a/sbin/start-mesos-shuffle-service.sh +++ b/sbin/start-mesos-shuffle-service.sh @@ -26,30 +26,11 @@ # Use the SPARK_SHUFFLE_OPTS environment variable to set shuffle service configuration. # -realpath () { -( - TARGET_FILE="$1" +if [ -z "${SPARK_HOME}" ]; then + export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" +fi - cd "$(dirname "$TARGET_FILE")" - TARGET_FILE="$(basename "$TARGET_FILE")" +. "${SPARK_HOME}/sbin/spark-config.sh" +. "${SPARK_HOME}/bin/load-spark-env.sh" - COUNT=0 - while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ] - do - TARGET_FILE="$(readlink "$TARGET_FILE")" - cd $(dirname "$TARGET_FILE") - TARGET_FILE="$(basename $TARGET_FILE)" - COUNT=$(($COUNT + 1)) - done - - echo "$(pwd -P)/"$TARGET_FILE"" -) -} - -sbin="$(dirname "$(realpath "$0")")" -sbin="$(cd "$sbin"; pwd)" - -. "$sbin/spark-config.sh" -. "$SPARK_PREFIX/bin/load-spark-env.sh" - -exec "$sbin"/spark-daemon.sh start org.apache.spark.deploy.mesos.MesosExternalShuffleService 1 +exec "${SPARK_HOME}/sbin"/spark-daemon.sh start org.apache.spark.deploy.mesos.MesosExternalShuffleService 1 diff --git a/sbin/start-shuffle-service.sh b/sbin/start-shuffle-service.sh index 47043bdfb351..8f6243831c04 100755 --- a/sbin/start-shuffle-service.sh +++ b/sbin/start-shuffle-service.sh @@ -24,30 +24,11 @@ # Use the SPARK_SHUFFLE_OPTS environment variable to set shuffle server configuration. # -realpath () { -( - TARGET_FILE="$1" +if [ -z "${SPARK_HOME}" ]; then + export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" +fi - cd "$(dirname "$TARGET_FILE")" - TARGET_FILE="$(basename "$TARGET_FILE")" +. "${SPARK_HOME}/sbin/spark-config.sh" +. "${SPARK_HOME}/bin/load-spark-env.sh" - COUNT=0 - while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ] - do - TARGET_FILE="$(readlink "$TARGET_FILE")" - cd $(dirname "$TARGET_FILE") - TARGET_FILE="$(basename $TARGET_FILE)" - COUNT=$(($COUNT + 1)) - done - - echo "$(pwd -P)/"$TARGET_FILE"" -) -} - -sbin="$(dirname "$(realpath "$0")")" -sbin="$(cd "$sbin"; pwd)" - -. "$sbin/spark-config.sh" -. "$SPARK_PREFIX/bin/load-spark-env.sh" - -exec "$sbin"/spark-daemon.sh start org.apache.spark.deploy.ExternalShuffleService 1 +exec "${SPARK_HOME}/sbin"/spark-daemon.sh start org.apache.spark.deploy.ExternalShuffleService 1 diff --git a/sbin/start-slave.sh b/sbin/start-slave.sh index 4292d11f9435..178749dbf67c 100755 --- a/sbin/start-slave.sh +++ b/sbin/start-slave.sh @@ -39,32 +39,13 @@ if [ $# -lt 1 ]; then exit 1 fi -realpath () { -( - TARGET_FILE="$1" - - cd "$(dirname "$TARGET_FILE")" - TARGET_FILE="$(basename "$TARGET_FILE")" - - COUNT=0 - while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ] - do - TARGET_FILE="$(readlink "$TARGET_FILE")" - cd $(dirname "$TARGET_FILE") - TARGET_FILE="$(basename $TARGET_FILE)" - COUNT=$(($COUNT + 1)) - done - - echo "$(pwd -P)/"$TARGET_FILE"" -) -} - -sbin="$(dirname "$(realpath "$0")")" -sbin="$(cd "$sbin"; pwd)" +if [ -z "${SPARK_HOME}" ]; then + export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" +fi -. "$sbin/spark-config.sh" +. "${SPARK_HOME}/sbin/spark-config.sh" -. "$SPARK_PREFIX/bin/load-spark-env.sh" +. "${SPARK_HOME}/bin/load-spark-env.sh" # First argument should be the master; we need to store it aside because we may # need to insert arguments between it and the other arguments @@ -91,7 +72,7 @@ function start_instance { fi WEBUI_PORT=$(( $SPARK_WORKER_WEBUI_PORT + $WORKER_NUM - 1 )) - "$sbin"/spark-daemon.sh start org.apache.spark.deploy.worker.Worker $WORKER_NUM \ + "${SPARK_HOME}/sbin"/spark-daemon.sh start org.apache.spark.deploy.worker.Worker $WORKER_NUM \ --webui-port "$WEBUI_PORT" $PORT_FLAG $PORT_NUM $MASTER "$@" } diff --git a/sbin/start-slaves.sh b/sbin/start-slaves.sh index 548b604ed625..5fb76294a117 100755 --- a/sbin/start-slaves.sh +++ b/sbin/start-slaves.sh @@ -19,36 +19,16 @@ # Starts a slave instance on each machine specified in the conf/slaves file. -realpath () { -( - TARGET_FILE="$1" - - cd "$(dirname "$TARGET_FILE")" - TARGET_FILE="$(basename "$TARGET_FILE")" - - COUNT=0 - while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ] - do - TARGET_FILE="$(readlink "$TARGET_FILE")" - cd $(dirname "$TARGET_FILE") - TARGET_FILE="$(basename $TARGET_FILE)" - COUNT=$(($COUNT + 1)) - done - - echo "$(pwd -P)/"$TARGET_FILE"" -) -} - -sbin="$(dirname "$(realpath "$0")")" -sbin="$(cd "$sbin"; pwd)" - +if [ -z "${SPARK_HOME}" ]; then + export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" +fi START_TACHYON=false while (( "$#" )); do case $1 in --with-tachyon) - if [ ! -e "$sbin"/../tachyon/bin/tachyon ]; then + if [ ! -e "${SPARK_HOME}/sbin"/../tachyon/bin/tachyon ]; then echo "Error: --with-tachyon specified, but tachyon not found." exit -1 fi @@ -58,8 +38,8 @@ case $1 in shift done -. "$sbin/spark-config.sh" -. "$SPARK_PREFIX/bin/load-spark-env.sh" +. "${SPARK_HOME}/sbin/spark-config.sh" +. "${SPARK_HOME}/bin/load-spark-env.sh" # Find the port number for the master if [ "$SPARK_MASTER_PORT" = "" ]; then @@ -71,11 +51,11 @@ if [ "$SPARK_MASTER_IP" = "" ]; then fi if [ "$START_TACHYON" == "true" ]; then - "$sbin/slaves.sh" cd "$SPARK_HOME" \; "$sbin"/../tachyon/bin/tachyon bootstrap-conf "$SPARK_MASTER_IP" + "${SPARK_HOME}/sbin/slaves.sh" cd "$SPARK_HOME" \; "${SPARK_HOME}/sbin"/../tachyon/bin/tachyon bootstrap-conf "$SPARK_MASTER_IP" # set -t so we can call sudo - SPARK_SSH_OPTS="-o StrictHostKeyChecking=no -t" "$SPARK_HOME/sbin/slaves.sh" cd "$SPARK_HOME" \; "$SPARK_HOME/tachyon/bin/tachyon-start.sh" worker SudoMount \; sleep 1 + SPARK_SSH_OPTS="-o StrictHostKeyChecking=no -t" "${SPARK_HOME}/sbin/slaves.sh" cd "${SPARK_HOME}" \; "${SPARK_HOME}/tachyon/bin/tachyon-start.sh" worker SudoMount \; sleep 1 fi # Launch the slaves -"$sbin/slaves.sh" cd "$SPARK_HOME" \; "$sbin/start-slave.sh" "spark://$SPARK_MASTER_IP:$SPARK_MASTER_PORT" +"${SPARK_HOME}/sbin/slaves.sh" cd "${SPARK_HOME}" \; "${SPARK_HOME}/sbin/start-slave.sh" "spark://$SPARK_MASTER_IP:$SPARK_MASTER_PORT" diff --git a/sbin/start-thriftserver.sh b/sbin/start-thriftserver.sh index 1a260b7ffa52..fc540a2ba395 100755 --- a/sbin/start-thriftserver.sh +++ b/sbin/start-thriftserver.sh @@ -23,29 +23,9 @@ # Enter posix mode for bash set -o posix -realpath () { -( - TARGET_FILE="$1" - - cd "$(dirname "$TARGET_FILE")" - TARGET_FILE="$(basename "$TARGET_FILE")" - - COUNT=0 - while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ] - do - TARGET_FILE="$(readlink "$TARGET_FILE")" - cd $(dirname "$TARGET_FILE") - TARGET_FILE="$(basename $TARGET_FILE)" - COUNT=$(($COUNT + 1)) - done - - echo "$(pwd -P)/"$TARGET_FILE"" -) -} - -#Figure out where Spark is installed -DIR="$(dirname "$(realpath "$0")")" -FWDIR="$(cd "$DIR/.."; pwd)" +if [ -z "${SPARK_HOME}" ]; then + export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" +fi # NOTE: This exact class name is matched downstream by SparkSubmit. # Any changes need to be reflected there. @@ -60,10 +40,10 @@ function usage { pattern+="\|=======" pattern+="\|--help" - "$FWDIR"/bin/spark-submit --help 2>&1 | grep -v Usage 1>&2 + "${SPARK_HOME}"/bin/spark-submit --help 2>&1 | grep -v Usage 1>&2 echo echo "Thrift server options:" - "$FWDIR"/bin/spark-class $CLASS --help 2>&1 | grep -v "$pattern" 1>&2 + "${SPARK_HOME}"/bin/spark-class $CLASS --help 2>&1 | grep -v "$pattern" 1>&2 } if [[ "$@" = *--help ]] || [[ "$@" = *-h ]]; then @@ -73,4 +53,4 @@ fi export SUBMIT_USAGE_FUNCTION=usage -exec "$FWDIR"/sbin/spark-daemon.sh submit $CLASS 1 "$@" +exec "${SPARK_HOME}"/sbin/spark-daemon.sh submit $CLASS 1 "$@" diff --git a/sbin/stop-all.sh b/sbin/stop-all.sh index 998129819864..3455da0bea69 100755 --- a/sbin/stop-all.sh +++ b/sbin/stop-all.sh @@ -20,43 +20,23 @@ # Stop all spark daemons. # Run this on the master node. - -realpath () { -( - TARGET_FILE="$1" - - cd "$(dirname "$TARGET_FILE")" - TARGET_FILE="$(basename "$TARGET_FILE")" - - COUNT=0 - while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ] - do - TARGET_FILE="$(readlink "$TARGET_FILE")" - cd $(dirname "$TARGET_FILE") - TARGET_FILE="$(basename $TARGET_FILE)" - COUNT=$(($COUNT + 1)) - done - - echo "$(pwd -P)/"$TARGET_FILE"" -) -} - -sbin="$(dirname "$(realpath "$0")")" -sbin="$(cd "$sbin"; pwd)" +if [ -z "${SPARK_HOME}" ]; then + export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" +fi # Load the Spark configuration -. "$sbin/spark-config.sh" +. "${SPARK_HOME}/sbin/spark-config.sh" # Stop the slaves, then the master -"$sbin"/stop-slaves.sh -"$sbin"/stop-master.sh +"${SPARK_HOME}/sbin"/stop-slaves.sh +"${SPARK_HOME}/sbin"/stop-master.sh if [ "$1" == "--wait" ] then printf "Waiting for workers to shut down..." while true do - running=`$sbin/slaves.sh ps -ef | grep -v grep | grep deploy.worker.Worker` + running=`${SPARK_HOME}/sbin/slaves.sh ps -ef | grep -v grep | grep deploy.worker.Worker` if [ -z "$running" ] then printf "\nAll workers successfully shut down.\n" diff --git a/sbin/stop-history-server.sh b/sbin/stop-history-server.sh index 9122c9c20c33..0bbbba0a0762 100755 --- a/sbin/stop-history-server.sh +++ b/sbin/stop-history-server.sh @@ -19,27 +19,8 @@ # Stops the history server on the machine this script is executed on. -realpath () { -( - TARGET_FILE="$1" +if [ -z "${SPARK_HOME}" ]; then + export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" +fi - cd "$(dirname "$TARGET_FILE")" - TARGET_FILE="$(basename "$TARGET_FILE")" - - COUNT=0 - while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ] - do - TARGET_FILE="$(readlink "$TARGET_FILE")" - cd $(dirname "$TARGET_FILE") - TARGET_FILE="$(basename $TARGET_FILE)" - COUNT=$(($COUNT + 1)) - done - - echo "$(pwd -P)/"$TARGET_FILE"" -) -} - -sbin="$(dirname "$(realpath "$0")")" -sbin="$(cd "$sbin"; pwd)" - -"$sbin/spark-daemon.sh" stop org.apache.spark.deploy.history.HistoryServer 1 +"${SPARK_HOME}/sbin/spark-daemon.sh" stop org.apache.spark.deploy.history.HistoryServer 1 diff --git a/sbin/stop-master.sh b/sbin/stop-master.sh index 28edb9bf0715..3f5e4589d0d5 100755 --- a/sbin/stop-master.sh +++ b/sbin/stop-master.sh @@ -19,34 +19,14 @@ # Stops the master on the machine this script is executed on. -realpath () { -( - TARGET_FILE="$1" - - cd "$(dirname "$TARGET_FILE")" - TARGET_FILE="$(basename "$TARGET_FILE")" - - COUNT=0 - while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ] - do - TARGET_FILE="$(readlink "$TARGET_FILE")" - cd $(dirname "$TARGET_FILE") - TARGET_FILE="$(basename $TARGET_FILE)" - COUNT=$(($COUNT + 1)) - done - - echo "$(pwd -P)/"$TARGET_FILE"" -) -} - -sbin="$(dirname "$(realpath "$0")")" -sbin="$(cd "$sbin"; pwd)" - +if [ -z "${SPARK_HOME}" ]; then + export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" +fi -. "$sbin/spark-config.sh" +. "${SPARK_HOME}/sbin/spark-config.sh" -"$sbin"/spark-daemon.sh stop org.apache.spark.deploy.master.Master 1 +"${SPARK_HOME}/sbin"/spark-daemon.sh stop org.apache.spark.deploy.master.Master 1 -if [ -e "$sbin"/../tachyon/bin/tachyon ]; then - "$sbin"/../tachyon/bin/tachyon killAll tachyon.master.Master +if [ -e "${SPARK_HOME}/sbin"/../tachyon/bin/tachyon ]; then + "${SPARK_HOME}/sbin"/../tachyon/bin/tachyon killAll tachyon.master.Master fi diff --git a/sbin/stop-mesos-dispatcher.sh b/sbin/stop-mesos-dispatcher.sh index fe54e36c7a05..708d357a28a7 100755 --- a/sbin/stop-mesos-dispatcher.sh +++ b/sbin/stop-mesos-dispatcher.sh @@ -18,30 +18,11 @@ # # Stop the Mesos Cluster dispatcher on the machine this script is executed on. -realpath () { -( - TARGET_FILE="$1" +if [ -z "${SPARK_HOME}" ]; then + export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" +fi - cd "$(dirname "$TARGET_FILE")" - TARGET_FILE="$(basename "$TARGET_FILE")" +. "${SPARK_HOME}/sbin/spark-config.sh" - COUNT=0 - while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ] - do - TARGET_FILE="$(readlink "$TARGET_FILE")" - cd $(dirname "$TARGET_FILE") - TARGET_FILE="$(basename $TARGET_FILE)" - COUNT=$(($COUNT + 1)) - done - - echo "$(pwd -P)/"$TARGET_FILE"" -) -} - -sbin="$(dirname "$(realpath "$0")")" -sbin="$(cd "$sbin"; pwd)" - -. "$sbin/spark-config.sh" - -"$sbin"/spark-daemon.sh stop org.apache.spark.deploy.mesos.MesosClusterDispatcher 1 +"${SPARK_HOME}/sbin"/spark-daemon.sh stop org.apache.spark.deploy.mesos.MesosClusterDispatcher 1 diff --git a/sbin/stop-mesos-shuffle-service.sh b/sbin/stop-mesos-shuffle-service.sh index 02bdb6235342..3eb13c6723e8 100755 --- a/sbin/stop-mesos-shuffle-service.sh +++ b/sbin/stop-mesos-shuffle-service.sh @@ -19,27 +19,8 @@ # Stops the Mesos external shuffle service on the machine this script is executed on. -realpath () { -( - TARGET_FILE="$1" +if [ -z "${SPARK_HOME}" ]; then + export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" +fi - cd "$(dirname "$TARGET_FILE")" - TARGET_FILE="$(basename "$TARGET_FILE")" - - COUNT=0 - while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ] - do - TARGET_FILE="$(readlink "$TARGET_FILE")" - cd $(dirname "$TARGET_FILE") - TARGET_FILE="$(basename $TARGET_FILE)" - COUNT=$(($COUNT + 1)) - done - - echo "$(pwd -P)/"$TARGET_FILE"" -) -} - -sbin="$(dirname "$(realpath "$0")")" -sbin="$(cd "$sbin"; pwd)" - -"$sbin"/spark-daemon.sh stop org.apache.spark.deploy.mesos.MesosExternalShuffleService 1 +"${SPARK_HOME}/sbin"/spark-daemon.sh stop org.apache.spark.deploy.mesos.MesosExternalShuffleService 1 diff --git a/sbin/stop-shuffle-service.sh b/sbin/stop-shuffle-service.sh index 6d229a830326..1e8875e86fa6 100755 --- a/sbin/stop-shuffle-service.sh +++ b/sbin/stop-shuffle-service.sh @@ -19,27 +19,8 @@ # Stops the external shuffle service on the machine this script is executed on. -realpath () { -( - TARGET_FILE="$1" +if [ -z "${SPARK_HOME}" ]; then + export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" +fi - cd "$(dirname "$TARGET_FILE")" - TARGET_FILE="$(basename "$TARGET_FILE")" - - COUNT=0 - while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ] - do - TARGET_FILE="$(readlink "$TARGET_FILE")" - cd $(dirname "$TARGET_FILE") - TARGET_FILE="$(basename $TARGET_FILE)" - COUNT=$(($COUNT + 1)) - done - - echo "$(pwd -P)/"$TARGET_FILE"" -) -} - -sbin="$(dirname "$(realpath "$0")")" -sbin="$(cd "$sbin"; pwd)" - -"$sbin"/spark-daemon.sh stop org.apache.spark.deploy.ExternalShuffleService 1 +"${SPARK_HOME}/sbin"/spark-daemon.sh stop org.apache.spark.deploy.ExternalShuffleService 1 diff --git a/sbin/stop-slave.sh b/sbin/stop-slave.sh index 204b00115552..7ea9c015ebf7 100755 --- a/sbin/stop-slave.sh +++ b/sbin/stop-slave.sh @@ -27,37 +27,18 @@ # Usage: stop-slave.sh # Stops all slaves on this worker machine -realpath () { -( - TARGET_FILE="$1" - - cd "$(dirname "$TARGET_FILE")" - TARGET_FILE="$(basename "$TARGET_FILE")" - - COUNT=0 - while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ] - do - TARGET_FILE="$(readlink "$TARGET_FILE")" - cd $(dirname "$TARGET_FILE") - TARGET_FILE="$(basename $TARGET_FILE)" - COUNT=$(($COUNT + 1)) - done - - echo "$(pwd -P)/"$TARGET_FILE"" -) -} - -sbin="$(dirname "$(realpath "$0")")" -sbin="$(cd "$sbin"; pwd)" +if [ -z "${SPARK_HOME}" ]; then + export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" +fi -. "$sbin/spark-config.sh" +. "${SPARK_HOME}/sbin/spark-config.sh" -. "$SPARK_PREFIX/bin/load-spark-env.sh" +. "${SPARK_HOME}/bin/load-spark-env.sh" if [ "$SPARK_WORKER_INSTANCES" = "" ]; then - "$sbin"/spark-daemon.sh stop org.apache.spark.deploy.worker.Worker 1 + "${SPARK_HOME}/sbin"/spark-daemon.sh stop org.apache.spark.deploy.worker.Worker 1 else for ((i=0; i<$SPARK_WORKER_INSTANCES; i++)); do - "$sbin"/spark-daemon.sh stop org.apache.spark.deploy.worker.Worker $(( $i + 1 )) + "${SPARK_HOME}/sbin"/spark-daemon.sh stop org.apache.spark.deploy.worker.Worker $(( $i + 1 )) done fi diff --git a/sbin/stop-slaves.sh b/sbin/stop-slaves.sh index 38044d4fd7df..7a2a74e3f564 100755 --- a/sbin/stop-slaves.sh +++ b/sbin/stop-slaves.sh @@ -17,36 +17,17 @@ # limitations under the License. # -realpath () { -( - TARGET_FILE="$1" - - cd "$(dirname "$TARGET_FILE")" - TARGET_FILE="$(basename "$TARGET_FILE")" - - COUNT=0 - while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ] - do - TARGET_FILE="$(readlink "$TARGET_FILE")" - cd $(dirname "$TARGET_FILE") - TARGET_FILE="$(basename $TARGET_FILE)" - COUNT=$(($COUNT + 1)) - done - - echo "$(pwd -P)/"$TARGET_FILE"" -) -} - -sbin="$(dirname "$(realpath "$0")")" -sbin="$(cd "$sbin"; pwd)" +if [ -z "${SPARK_HOME}" ]; then + export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" +fi -. "$sbin/spark-config.sh" +. "${SPARK_HOME}/sbin/spark-config.sh" -. "$SPARK_PREFIX/bin/load-spark-env.sh" +. "${SPARK_HOME}/bin/load-spark-env.sh" # do before the below calls as they exec -if [ -e "$sbin"/../tachyon/bin/tachyon ]; then - "$sbin/slaves.sh" cd "$SPARK_HOME" \; "$sbin"/../tachyon/bin/tachyon killAll tachyon.worker.Worker +if [ -e "${SPARK_HOME}/sbin"/../tachyon/bin/tachyon ]; then + "${SPARK_HOME}/sbin/slaves.sh" cd "${SPARK_HOME}" \; "${SPARK_HOME}/sbin"/../tachyon/bin/tachyon killAll tachyon.worker.Worker fi -"$sbin/slaves.sh" cd "$SPARK_HOME" \; "$sbin"/stop-slave.sh +"${SPARK_HOME}/sbin/slaves.sh" cd "${SPARK_HOME}" \; "${SPARK_HOME}/sbin"/stop-slave.sh diff --git a/sbin/stop-thriftserver.sh b/sbin/stop-thriftserver.sh index e758884e210c..4c7326fd9ce6 100755 --- a/sbin/stop-thriftserver.sh +++ b/sbin/stop-thriftserver.sh @@ -19,27 +19,8 @@ # Stops the thrift server on the machine this script is executed on. -realpath () { -( - TARGET_FILE="$1" +if [ -z "${SPARK_HOME}" ]; then + export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" +fi - cd "$(dirname "$TARGET_FILE")" - TARGET_FILE="$(basename "$TARGET_FILE")" - - COUNT=0 - while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ] - do - TARGET_FILE="$(readlink "$TARGET_FILE")" - cd $(dirname "$TARGET_FILE") - TARGET_FILE="$(basename $TARGET_FILE)" - COUNT=$(($COUNT + 1)) - done - - echo "$(pwd -P)/"$TARGET_FILE"" -) -} - -sbin="$(dirname "$(realpath "$0")")" -sbin="$(cd "$sbin"; pwd)" - -"$sbin"/spark-daemon.sh stop org.apache.spark.sql.hive.thriftserver.HiveThriftServer2 1 +"${SPARK_HOME}/sbin"/spark-daemon.sh stop org.apache.spark.sql.hive.thriftserver.HiveThriftServer2 1 From 41511b417b1009bfda3f3cf8104ef2025f50227d Mon Sep 17 00:00:00 2001 From: jerryshao Date: Tue, 27 Oct 2015 11:52:54 +0800 Subject: [PATCH 6/9] Fix missing slash --- sbin/start-all.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sbin/start-all.sh b/sbin/start-all.sh index 9e86832b1eaa..c7353212371f 100755 --- a/sbin/start-all.sh +++ b/sbin/start-all.sh @@ -40,7 +40,7 @@ done . "${SPARK_HOME}/sbin/spark-config.sh" # Start Master -"${SPARK_HOME}sbin"/start-master.sh $TACHYON_STR +"${SPARK_HOME}/sbin"/start-master.sh $TACHYON_STR # Start Workers "${SPARK_HOME}/sbin"/start-slaves.sh $TACHYON_STR From 28fd58dae040db2a85a09a2b9246907666eb0d24 Mon Sep 17 00:00:00 2001 From: jerryshao Date: Tue, 27 Oct 2015 18:33:28 +0800 Subject: [PATCH 7/9] Fix 2 space indent --- bin/beeline | 2 +- bin/load-spark-env.sh | 26 +++++++++++++------------- bin/pyspark | 4 ++-- bin/run-example | 2 +- bin/spark-class | 2 +- bin/spark-shell | 2 +- bin/spark-sql | 2 +- bin/spark-submit | 2 +- bin/sparkR | 2 +- sbin/slaves.sh | 2 +- sbin/spark-config.sh | 2 +- sbin/spark-daemon.sh | 2 +- sbin/spark-daemons.sh | 2 +- sbin/start-all.sh | 2 +- sbin/start-history-server.sh | 2 +- sbin/start-master.sh | 2 +- sbin/start-mesos-dispatcher.sh | 2 +- sbin/start-mesos-shuffle-service.sh | 2 +- sbin/start-shuffle-service.sh | 2 +- sbin/start-slave.sh | 2 +- sbin/start-slaves.sh | 2 +- sbin/start-thriftserver.sh | 2 +- sbin/stop-all.sh | 2 +- sbin/stop-history-server.sh | 2 +- sbin/stop-master.sh | 2 +- sbin/stop-mesos-dispatcher.sh | 2 +- sbin/stop-mesos-shuffle-service.sh | 2 +- sbin/stop-shuffle-service.sh | 2 +- sbin/stop-slave.sh | 2 +- sbin/stop-slaves.sh | 2 +- sbin/stop-thriftserver.sh | 2 +- 31 files changed, 44 insertions(+), 44 deletions(-) diff --git a/bin/beeline b/bin/beeline index 84a150764567..1627626941a7 100755 --- a/bin/beeline +++ b/bin/beeline @@ -25,7 +25,7 @@ set -o posix # Figure out if SPARK_HOME is set if [ -z "${SPARK_HOME}" ]; then - export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" + export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" fi CLASS="org.apache.hive.beeline.BeeLine" diff --git a/bin/load-spark-env.sh b/bin/load-spark-env.sh index 50528ead7a78..eaea964ed5b3 100644 --- a/bin/load-spark-env.sh +++ b/bin/load-spark-env.sh @@ -23,7 +23,7 @@ # Figure out where Spark is installed if [ -z "${SPARK_HOME}" ]; then - export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" + export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" fi if [ -z "$SPARK_ENV_LOADED" ]; then @@ -46,18 +46,18 @@ fi if [ -z "$SPARK_SCALA_VERSION" ]; then - ASSEMBLY_DIR2="${SPARK_HOME}/assembly/target/scala-2.11" - ASSEMBLY_DIR1="${SPARK_HOME}/assembly/target/scala-2.10" + ASSEMBLY_DIR2="${SPARK_HOME}/assembly/target/scala-2.11" + ASSEMBLY_DIR1="${SPARK_HOME}/assembly/target/scala-2.10" - if [[ -d "$ASSEMBLY_DIR2" && -d "$ASSEMBLY_DIR1" ]]; then - echo -e "Presence of build for both scala versions(SCALA 2.10 and SCALA 2.11) detected." 1>&2 - echo -e 'Either clean one of them or, export SPARK_SCALA_VERSION=2.11 in spark-env.sh.' 1>&2 - exit 1 - fi + if [[ -d "$ASSEMBLY_DIR2" && -d "$ASSEMBLY_DIR1" ]]; then + echo -e "Presence of build for both scala versions(SCALA 2.10 and SCALA 2.11) detected." 1>&2 + echo -e 'Either clean one of them or, export SPARK_SCALA_VERSION=2.11 in spark-env.sh.' 1>&2 + exit 1 + fi - if [ -d "$ASSEMBLY_DIR2" ]; then - export SPARK_SCALA_VERSION="2.11" - else - export SPARK_SCALA_VERSION="2.10" - fi + if [ -d "$ASSEMBLY_DIR2" ]; then + export SPARK_SCALA_VERSION="2.11" + else + export SPARK_SCALA_VERSION="2.10" + fi fi diff --git a/bin/pyspark b/bin/pyspark index 0f9310cf222b..5eaa17d3c201 100755 --- a/bin/pyspark +++ b/bin/pyspark @@ -18,10 +18,10 @@ # if [ -z "${SPARK_HOME}" ]; then - export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" + export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" fi -source "{$SPARK_HOME}"/bin/load-spark-env.sh +source "${SPARK_HOME}"/bin/load-spark-env.sh export _SPARK_CMD_USAGE="Usage: ./bin/pyspark [options]" # In Spark <= 1.1, setting IPYTHON=1 would cause the driver to be launched using the `ipython` diff --git a/bin/run-example b/bin/run-example index 438877049778..26146d7567c5 100755 --- a/bin/run-example +++ b/bin/run-example @@ -18,7 +18,7 @@ # if [ -z "${SPARK_HOME}" ]; then - export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" + export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" fi EXAMPLES_DIR="${SPARK_HOME}"/examples diff --git a/bin/spark-class b/bin/spark-class index f6aad0bba8fc..87d06693af4f 100755 --- a/bin/spark-class +++ b/bin/spark-class @@ -18,7 +18,7 @@ # if [ -z "${SPARK_HOME}" ]; then - export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" + export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" fi . "${SPARK_HOME}"/bin/load-spark-env.sh diff --git a/bin/spark-shell b/bin/spark-shell index 1ac6e6b86223..6583b5bd880e 100755 --- a/bin/spark-shell +++ b/bin/spark-shell @@ -29,7 +29,7 @@ esac set -o posix if [ -z "${SPARK_HOME}" ]; then - export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" + export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" fi export _SPARK_CMD_USAGE="Usage: ./bin/spark-shell [options]" diff --git a/bin/spark-sql b/bin/spark-sql index fa8433f35a13..970d12cbf51d 100755 --- a/bin/spark-sql +++ b/bin/spark-sql @@ -18,7 +18,7 @@ # if [ -z "${SPARK_HOME}" ]; then - export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" + export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" fi export _SPARK_CMD_USAGE="Usage: ./bin/spark-sql [options] [cli option]" diff --git a/bin/spark-submit b/bin/spark-submit index 8953167a574d..023f9c162f4b 100755 --- a/bin/spark-submit +++ b/bin/spark-submit @@ -18,7 +18,7 @@ # if [ -z "${SPARK_HOME}" ]; then - export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" + export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" fi # disable randomized hash for string in Python 3.3+ diff --git a/bin/sparkR b/bin/sparkR index 6dec68934852..2c07a82e2173 100755 --- a/bin/sparkR +++ b/bin/sparkR @@ -18,7 +18,7 @@ # if [ -z "${SPARK_HOME}" ]; then - export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" + export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" fi source "${SPARK_HOME}"/bin/load-spark-env.sh diff --git a/sbin/slaves.sh b/sbin/slaves.sh index 110fa552c231..c971aa3296b0 100755 --- a/sbin/slaves.sh +++ b/sbin/slaves.sh @@ -37,7 +37,7 @@ if [ $# -le 0 ]; then fi if [ -z "${SPARK_HOME}" ]; then - export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" + export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" fi . "${SPARK_HOME}/sbin/spark-config.sh" diff --git a/sbin/spark-config.sh b/sbin/spark-config.sh index 0bc2334c8d05..bc73ee7d3b1e 100755 --- a/sbin/spark-config.sh +++ b/sbin/spark-config.sh @@ -21,7 +21,7 @@ # symlink and absolute path should rely on SPARK_HOME to resolve if [ -z "${SPARK_HOME}" ]; then - export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" + export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" fi export SPARK_CONF_DIR="${SPARK_CONF_DIR:-"$SPARK_HOME/conf"}" diff --git a/sbin/spark-daemon.sh b/sbin/spark-daemon.sh index c981afdab558..9304f849ebd3 100755 --- a/sbin/spark-daemon.sh +++ b/sbin/spark-daemon.sh @@ -38,7 +38,7 @@ if [ $# -le 1 ]; then fi if [ -z "${SPARK_HOME}" ]; then - export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" + export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" fi . "${SPARK_HOME}/sbin/spark-config.sh" diff --git a/sbin/spark-daemons.sh b/sbin/spark-daemons.sh index 732a933ffe3b..dec2f4432df3 100755 --- a/sbin/spark-daemons.sh +++ b/sbin/spark-daemons.sh @@ -28,7 +28,7 @@ if [ $# -le 1 ]; then fi if [ -z "${SPARK_HOME}" ]; then - export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" + export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" fi . "${SPARK_HOME}/sbin/spark-config.sh" diff --git a/sbin/start-all.sh b/sbin/start-all.sh index c7353212371f..6217f9bf28e3 100755 --- a/sbin/start-all.sh +++ b/sbin/start-all.sh @@ -22,7 +22,7 @@ # Starts a worker on each node specified in conf/slaves if [ -z "${SPARK_HOME}" ]; then - export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" + export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" fi TACHYON_STR="" diff --git a/sbin/start-history-server.sh b/sbin/start-history-server.sh index 96e52ef3e3b0..6851d99b7e8f 100755 --- a/sbin/start-history-server.sh +++ b/sbin/start-history-server.sh @@ -25,7 +25,7 @@ # if [ -z "${SPARK_HOME}" ]; then - export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" + export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" fi . "${SPARK_HOME}/sbin/spark-config.sh" diff --git a/sbin/start-master.sh b/sbin/start-master.sh index 2a81473a5219..c20e19a8412d 100755 --- a/sbin/start-master.sh +++ b/sbin/start-master.sh @@ -20,7 +20,7 @@ # Starts the master on the machine this script is executed on. if [ -z "${SPARK_HOME}" ]; then - export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" + export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" fi ORIGINAL_ARGS="$@" diff --git a/sbin/start-mesos-dispatcher.sh b/sbin/start-mesos-dispatcher.sh index a43d162d5fa9..4777e1668c70 100755 --- a/sbin/start-mesos-dispatcher.sh +++ b/sbin/start-mesos-dispatcher.sh @@ -22,7 +22,7 @@ # Only one cluster dispatcher is needed per Mesos cluster. if [ -z "${SPARK_HOME}" ]; then - export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" + export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" fi . "${SPARK_HOME}/sbin/spark-config.sh" diff --git a/sbin/start-mesos-shuffle-service.sh b/sbin/start-mesos-shuffle-service.sh index 29442bfcbb8a..184584567602 100755 --- a/sbin/start-mesos-shuffle-service.sh +++ b/sbin/start-mesos-shuffle-service.sh @@ -27,7 +27,7 @@ # if [ -z "${SPARK_HOME}" ]; then - export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" + export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" fi . "${SPARK_HOME}/sbin/spark-config.sh" diff --git a/sbin/start-shuffle-service.sh b/sbin/start-shuffle-service.sh index 8f6243831c04..793e165be6c7 100755 --- a/sbin/start-shuffle-service.sh +++ b/sbin/start-shuffle-service.sh @@ -25,7 +25,7 @@ # if [ -z "${SPARK_HOME}" ]; then - export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" + export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" fi . "${SPARK_HOME}/sbin/spark-config.sh" diff --git a/sbin/start-slave.sh b/sbin/start-slave.sh index 178749dbf67c..21455648d1c6 100755 --- a/sbin/start-slave.sh +++ b/sbin/start-slave.sh @@ -40,7 +40,7 @@ if [ $# -lt 1 ]; then fi if [ -z "${SPARK_HOME}" ]; then - export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" + export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" fi . "${SPARK_HOME}/sbin/spark-config.sh" diff --git a/sbin/start-slaves.sh b/sbin/start-slaves.sh index 5fb76294a117..5f3d35305333 100755 --- a/sbin/start-slaves.sh +++ b/sbin/start-slaves.sh @@ -20,7 +20,7 @@ # Starts a slave instance on each machine specified in the conf/slaves file. if [ -z "${SPARK_HOME}" ]; then - export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" + export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" fi START_TACHYON=false diff --git a/sbin/start-thriftserver.sh b/sbin/start-thriftserver.sh index fc540a2ba395..ad7e7c5277eb 100755 --- a/sbin/start-thriftserver.sh +++ b/sbin/start-thriftserver.sh @@ -24,7 +24,7 @@ set -o posix if [ -z "${SPARK_HOME}" ]; then - export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" + export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" fi # NOTE: This exact class name is matched downstream by SparkSubmit. diff --git a/sbin/stop-all.sh b/sbin/stop-all.sh index 3455da0bea69..4e476ca05cb0 100755 --- a/sbin/stop-all.sh +++ b/sbin/stop-all.sh @@ -21,7 +21,7 @@ # Run this on the master node. if [ -z "${SPARK_HOME}" ]; then - export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" + export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" fi # Load the Spark configuration diff --git a/sbin/stop-history-server.sh b/sbin/stop-history-server.sh index 0bbbba0a0762..14e3af4be910 100755 --- a/sbin/stop-history-server.sh +++ b/sbin/stop-history-server.sh @@ -20,7 +20,7 @@ # Stops the history server on the machine this script is executed on. if [ -z "${SPARK_HOME}" ]; then - export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" + export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" fi "${SPARK_HOME}/sbin/spark-daemon.sh" stop org.apache.spark.deploy.history.HistoryServer 1 diff --git a/sbin/stop-master.sh b/sbin/stop-master.sh index 3f5e4589d0d5..e57962bb354d 100755 --- a/sbin/stop-master.sh +++ b/sbin/stop-master.sh @@ -20,7 +20,7 @@ # Stops the master on the machine this script is executed on. if [ -z "${SPARK_HOME}" ]; then - export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" + export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" fi . "${SPARK_HOME}/sbin/spark-config.sh" diff --git a/sbin/stop-mesos-dispatcher.sh b/sbin/stop-mesos-dispatcher.sh index 708d357a28a7..5c0b4e051db3 100755 --- a/sbin/stop-mesos-dispatcher.sh +++ b/sbin/stop-mesos-dispatcher.sh @@ -19,7 +19,7 @@ # Stop the Mesos Cluster dispatcher on the machine this script is executed on. if [ -z "${SPARK_HOME}" ]; then - export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" + export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" fi . "${SPARK_HOME}/sbin/spark-config.sh" diff --git a/sbin/stop-mesos-shuffle-service.sh b/sbin/stop-mesos-shuffle-service.sh index 3eb13c6723e8..d23cad375e1b 100755 --- a/sbin/stop-mesos-shuffle-service.sh +++ b/sbin/stop-mesos-shuffle-service.sh @@ -20,7 +20,7 @@ # Stops the Mesos external shuffle service on the machine this script is executed on. if [ -z "${SPARK_HOME}" ]; then - export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" + export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" fi "${SPARK_HOME}/sbin"/spark-daemon.sh stop org.apache.spark.deploy.mesos.MesosExternalShuffleService 1 diff --git a/sbin/stop-shuffle-service.sh b/sbin/stop-shuffle-service.sh index 1e8875e86fa6..50d69cf34e0a 100755 --- a/sbin/stop-shuffle-service.sh +++ b/sbin/stop-shuffle-service.sh @@ -20,7 +20,7 @@ # Stops the external shuffle service on the machine this script is executed on. if [ -z "${SPARK_HOME}" ]; then - export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" + export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" fi "${SPARK_HOME}/sbin"/spark-daemon.sh stop org.apache.spark.deploy.ExternalShuffleService 1 diff --git a/sbin/stop-slave.sh b/sbin/stop-slave.sh index 7ea9c015ebf7..685bcf59b33a 100755 --- a/sbin/stop-slave.sh +++ b/sbin/stop-slave.sh @@ -28,7 +28,7 @@ # Stops all slaves on this worker machine if [ -z "${SPARK_HOME}" ]; then - export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" + export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" fi . "${SPARK_HOME}/sbin/spark-config.sh" diff --git a/sbin/stop-slaves.sh b/sbin/stop-slaves.sh index 7a2a74e3f564..63956377629d 100755 --- a/sbin/stop-slaves.sh +++ b/sbin/stop-slaves.sh @@ -18,7 +18,7 @@ # if [ -z "${SPARK_HOME}" ]; then - export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" + export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" fi . "${SPARK_HOME}/sbin/spark-config.sh" diff --git a/sbin/stop-thriftserver.sh b/sbin/stop-thriftserver.sh index 4c7326fd9ce6..cf45058f882a 100755 --- a/sbin/stop-thriftserver.sh +++ b/sbin/stop-thriftserver.sh @@ -20,7 +20,7 @@ # Stops the thrift server on the machine this script is executed on. if [ -z "${SPARK_HOME}" ]; then - export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" + export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" fi "${SPARK_HOME}/sbin"/spark-daemon.sh stop org.apache.spark.sql.hive.thriftserver.HiveThriftServer2 1 From 5795800f27efa2c04bfd20794f03c9a344387235 Mon Sep 17 00:00:00 2001 From: jerryshao Date: Wed, 28 Oct 2015 09:41:20 +0800 Subject: [PATCH 8/9] add curly brace for variables --- bin/run-example | 2 +- sbin/spark-config.sh | 6 +++--- sbin/spark-daemon.sh | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/bin/run-example b/bin/run-example index 26146d7567c5..e1b0d5789bed 100755 --- a/bin/run-example +++ b/bin/run-example @@ -46,7 +46,7 @@ JAR_COUNT=0 for f in "${JAR_PATH}"/spark-examples-*hadoop*.jar; do if [[ ! -e "$f" ]]; then - echo "Failed to find Spark examples assembly in ${SPARK_HOME}/lib or $SPARK_HOME/examples/target" 1>&2 + echo "Failed to find Spark examples assembly in ${SPARK_HOME}/lib or ${SPARK_HOME}/examples/target" 1>&2 echo "You need to build Spark before running this program" 1>&2 exit 1 fi diff --git a/sbin/spark-config.sh b/sbin/spark-config.sh index bc73ee7d3b1e..d8d9d00d64eb 100755 --- a/sbin/spark-config.sh +++ b/sbin/spark-config.sh @@ -24,7 +24,7 @@ if [ -z "${SPARK_HOME}" ]; then export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" fi -export SPARK_CONF_DIR="${SPARK_CONF_DIR:-"$SPARK_HOME/conf"}" +export SPARK_CONF_DIR="${SPARK_CONF_DIR:-"${SPARK_HOME}/conf"}" # Add the PySpark classes to the PYTHONPATH: -export PYTHONPATH="$SPARK_HOME/python:$PYTHONPATH" -export PYTHONPATH="$SPARK_HOME/python/lib/py4j-0.9-src.zip:$PYTHONPATH" +export PYTHONPATH="${SPARK_HOME}/python:${PYTHONPATH}" +export PYTHONPATH="${SPARK_HOME}/python/lib/py4j-0.9-src.zip:${PYTHONPATH}" diff --git a/sbin/spark-daemon.sh b/sbin/spark-daemon.sh index 9304f849ebd3..6ab57df40952 100755 --- a/sbin/spark-daemon.sh +++ b/sbin/spark-daemon.sh @@ -98,7 +98,7 @@ export SPARK_PRINT_LAUNCH_COMMAND="1" # get log directory if [ "$SPARK_LOG_DIR" = "" ]; then - export SPARK_LOG_DIR="$SPARK_HOME/logs" + export SPARK_LOG_DIR="${SPARK_HOME}/logs" fi mkdir -p "$SPARK_LOG_DIR" touch "$SPARK_LOG_DIR"/.spark_test > /dev/null 2>&1 @@ -138,7 +138,7 @@ run_command() { if [ "$SPARK_MASTER" != "" ]; then echo rsync from "$SPARK_MASTER" - rsync -a -e ssh --delete --exclude=.svn --exclude='logs/*' --exclude='contrib/hod/logs/*' "$SPARK_MASTER/" "$SPARK_HOME" + rsync -a -e ssh --delete --exclude=.svn --exclude='logs/*' --exclude='contrib/hod/logs/*' "$SPARK_MASTER/" "${SPARK_HOME}" fi spark_rotate_log "$log" From 16ea8840c6b51dc05008167080c41d7cdd455275 Mon Sep 17 00:00:00 2001 From: jerryshao Date: Wed, 4 Nov 2015 10:04:46 +0800 Subject: [PATCH 9/9] Address the comments --- sbin/start-slaves.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sbin/start-slaves.sh b/sbin/start-slaves.sh index 5f3d35305333..51ca81e053b7 100755 --- a/sbin/start-slaves.sh +++ b/sbin/start-slaves.sh @@ -51,7 +51,7 @@ if [ "$SPARK_MASTER_IP" = "" ]; then fi if [ "$START_TACHYON" == "true" ]; then - "${SPARK_HOME}/sbin/slaves.sh" cd "$SPARK_HOME" \; "${SPARK_HOME}/sbin"/../tachyon/bin/tachyon bootstrap-conf "$SPARK_MASTER_IP" + "${SPARK_HOME}/sbin/slaves.sh" cd "${SPARK_HOME}" \; "${SPARK_HOME}/sbin"/../tachyon/bin/tachyon bootstrap-conf "$SPARK_MASTER_IP" # set -t so we can call sudo SPARK_SSH_OPTS="-o StrictHostKeyChecking=no -t" "${SPARK_HOME}/sbin/slaves.sh" cd "${SPARK_HOME}" \; "${SPARK_HOME}/tachyon/bin/tachyon-start.sh" worker SudoMount \; sleep 1