diff --git a/sbin/run-master.sh b/sbin/run-master.sh new file mode 120000 index 000000000000..f5db3ceb345d --- /dev/null +++ b/sbin/run-master.sh @@ -0,0 +1 @@ +start-master.sh \ No newline at end of file diff --git a/sbin/run-slave.sh b/sbin/run-slave.sh new file mode 120000 index 000000000000..4c081cc1d622 --- /dev/null +++ b/sbin/run-slave.sh @@ -0,0 +1 @@ +start-slave.sh \ No newline at end of file diff --git a/sbin/spark-daemon-run.sh b/sbin/spark-daemon-run.sh new file mode 100755 index 000000000000..34e3a80fa37a --- /dev/null +++ b/sbin/spark-daemon-run.sh @@ -0,0 +1,139 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Runs a Spark daemon foreground. +# +# Environment Variables +# +# SPARK_CONF_DIR Alternate conf dir. Default is ${SPARK_HOME}/conf. +# SPARK_LOG_DIR Where log files are stored. ${SPARK_HOME}/logs by default. +# SPARK_MASTER host:path where spark code should be rsync'd from +# SPARK_IDENT_STRING A string representing this instance of spark. $USER by default +# SPARK_NICENESS The scheduling priority for daemons. Defaults to 0. +## + +usage="Usage: spark-daemon-run.sh [--config ] (class|submit) " + +# if no args specified, show usage +if [ $# -le 1 ]; then + echo $usage + exit 1 +fi + +if [ -z "${SPARK_HOME}" ]; then + export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" +fi + +. "${SPARK_HOME}/sbin/spark-config.sh" + +# get arguments + +# Check if --config is passed as an argument. It is an optional parameter. +# Exit if the argument is not a directory. + +if [ "$1" == "--config" ] +then + shift + conf_dir="$1" + if [ ! -d "$conf_dir" ] + then + echo "ERROR : $conf_dir is not a directory" + echo $usage + exit 1 + else + export SPARK_CONF_DIR="$conf_dir" + fi + shift +fi + +mode=$1 +shift +command=$1 +shift +instance=$1 +shift + +spark_rotate_log () +{ + log=$1; + num=5; + if [ -n "$2" ]; then + num=$2 + fi + if [ -f "$log" ]; then # rotate logs + while [ $num -gt 1 ]; do + prev=`expr $num - 1` + [ -f "$log.$prev" ] && mv "$log.$prev" "$log.$num" + num=$prev + done + mv "$log" "$log.$num"; + fi +} + +. "${SPARK_HOME}/bin/load-spark-env.sh" + +if [ "$SPARK_IDENT_STRING" = "" ]; then + export SPARK_IDENT_STRING="$USER" +fi + + + +# get log directory +if [ "$SPARK_LOG_DIR" = "" ]; then + export SPARK_LOG_DIR="${SPARK_HOME}/logs" +fi +mkdir -p "$SPARK_LOG_DIR" +touch "$SPARK_LOG_DIR"/.spark_test > /dev/null 2>&1 +TEST_LOG_DIR=$? +if [ "${TEST_LOG_DIR}" = "0" ]; then + rm -f "$SPARK_LOG_DIR"/.spark_test +else + chown "$SPARK_IDENT_STRING" "$SPARK_LOG_DIR" +fi + +# some variables +log="$SPARK_LOG_DIR/spark-$SPARK_IDENT_STRING-$command-$instance-$HOSTNAME.out" + +# Set default scheduling priority +if [ "$SPARK_NICENESS" = "" ]; then + export SPARK_NICENESS=0 +fi + +if [ "$SPARK_MASTER" != "" ]; then + echo rsync from "$SPARK_MASTER" + rsync -a -e ssh --delete --exclude=.svn --exclude='logs/*' --exclude='contrib/hod/logs/*' "$SPARK_MASTER/" "${SPARK_HOME}" +fi + +spark_rotate_log "$log" +echo "running $command, logging to $log" + +case "$mode" in + (start) + exec nice -n "$SPARK_NICENESS" "${SPARK_HOME}"/bin/spark-class $command "$@" >> "$log" 2>&1 < /dev/null + ;; + + (submit) + exec nice -n "$SPARK_NICENESS" "${SPARK_HOME}"/bin/spark-submit --class $command "$@" >> "$log" 2>&1 < /dev/null + ;; + + (*) + echo "unknown mode: $mode" + exit 1 + ;; +esac diff --git a/sbin/spark-daemon.sh b/sbin/spark-daemon.sh index 59823571124f..75bc6994409d 100755 --- a/sbin/spark-daemon.sh +++ b/sbin/spark-daemon.sh @@ -66,26 +66,7 @@ fi option=$1 shift command=$1 -shift -instance=$1 -shift - -spark_rotate_log () -{ - log=$1; - num=5; - if [ -n "$2" ]; then - num=$2 - fi - if [ -f "$log" ]; then # rotate logs - while [ $num -gt 1 ]; do - prev=`expr $num - 1` - [ -f "$log.$prev" ] && mv "$log.$prev" "$log.$num" - num=$prev - done - mv "$log" "$log.$num"; - fi -} +instance=$2 . "${SPARK_HOME}/bin/load-spark-env.sh" @@ -100,14 +81,6 @@ export SPARK_PRINT_LAUNCH_COMMAND="1" if [ "$SPARK_LOG_DIR" = "" ]; then export SPARK_LOG_DIR="${SPARK_HOME}/logs" fi -mkdir -p "$SPARK_LOG_DIR" -touch "$SPARK_LOG_DIR"/.spark_test > /dev/null 2>&1 -TEST_LOG_DIR=$? -if [ "${TEST_LOG_DIR}" = "0" ]; then - rm -f "$SPARK_LOG_DIR"/.spark_test -else - chown "$SPARK_IDENT_STRING" "$SPARK_LOG_DIR" -fi if [ "$SPARK_PID_DIR" = "" ]; then SPARK_PID_DIR=/tmp @@ -115,16 +88,9 @@ fi # some variables log="$SPARK_LOG_DIR/spark-$SPARK_IDENT_STRING-$command-$instance-$HOSTNAME.out" -pid="$SPARK_PID_DIR/spark-$SPARK_IDENT_STRING-$command-$instance.pid" - -# Set default scheduling priority -if [ "$SPARK_NICENESS" = "" ]; then - export SPARK_NICENESS=0 -fi +pid="$SPARK_PID_DIR/spark-$SPARK_IDENT_STRING-$command-$instance-$HOSTNAME.pid" run_command() { - mode="$1" - shift mkdir -p "$SPARK_PID_DIR" @@ -136,33 +102,12 @@ run_command() { fi fi - if [ "$SPARK_MASTER" != "" ]; then - echo rsync from "$SPARK_MASTER" - rsync -a -e ssh --delete --exclude=.svn --exclude='logs/*' --exclude='contrib/hod/logs/*' "$SPARK_MASTER/" "${SPARK_HOME}" - fi - - spark_rotate_log "$log" - echo "starting $command, logging to $log" - - case "$mode" in - (class) - nohup nice -n "$SPARK_NICENESS" "${SPARK_HOME}"/bin/spark-class $command "$@" >> "$log" 2>&1 < /dev/null & - newpid="$!" - ;; - (submit) - nohup nice -n "$SPARK_NICENESS" "${SPARK_HOME}"/bin/spark-submit --class $command "$@" >> "$log" 2>&1 < /dev/null & - newpid="$!" - ;; - - (*) - echo "unknown mode: $mode" - exit 1 - ;; - esac + nohup "${SPARK_HOME}"/sbin/spark-daemon-run.sh "$@" >/dev/null /dev/null & + newpid="$!" echo "$newpid" > "$pid" - + #Poll for up to 5 seconds for the java process to start for i in {1..10} do @@ -183,12 +128,8 @@ run_command() { case $option in - (submit) - run_command submit "$@" - ;; - - (start) - run_command class "$@" + (submit|start) + run_command $option "$@" ;; (stop) @@ -229,5 +170,3 @@ case $option in ;; esac - - diff --git a/sbin/start-master.sh b/sbin/start-master.sh index 981cb15bc000..3e1f5a0df9a3 100755 --- a/sbin/start-master.sh +++ b/sbin/start-master.sh @@ -55,6 +55,18 @@ if [ "$SPARK_MASTER_WEBUI_PORT" = "" ]; then SPARK_MASTER_WEBUI_PORT=8080 fi -"${SPARK_HOME}/sbin"/spark-daemon.sh start $CLASS 1 \ +case "$(basename $0 .sh)" in + (start-master) + execname=spark-daemon + ;; + (run-master) + execname=spark-daemon-run + ;; + (*) + echo "bad command $0" + exit 1 +esac + +"${SPARK_HOME}/sbin"/$execname.sh start $CLASS 1 \ --host $SPARK_MASTER_HOST --port $SPARK_MASTER_PORT --webui-port $SPARK_MASTER_WEBUI_PORT \ $ORIGINAL_ARGS diff --git a/sbin/start-slave.sh b/sbin/start-slave.sh index 8c268b885915..c958e556918b 100755 --- a/sbin/start-slave.sh +++ b/sbin/start-slave.sh @@ -63,6 +63,19 @@ if [ "$SPARK_WORKER_WEBUI_PORT" = "" ]; then SPARK_WORKER_WEBUI_PORT=8081 fi +# decide which shell to call +case "$(basename $0 .sh)" in + (start-slave) + execname=spark-daemon + ;; + (run-slave) + execname=spark-daemon-run + ;; + (*) + echo "bad command $0" + exit 1 +esac + # Start up the appropriate number of workers on this machine. # quick local function to start a worker function start_instance { @@ -78,7 +91,7 @@ function start_instance { fi WEBUI_PORT=$(( $SPARK_WORKER_WEBUI_PORT + $WORKER_NUM - 1 )) - "${SPARK_HOME}/sbin"/spark-daemon.sh start $CLASS $WORKER_NUM \ + "${SPARK_HOME}/sbin"/$execname.sh start $CLASS $WORKER_NUM \ --webui-port "$WEBUI_PORT" $PORT_FLAG $PORT_NUM $MASTER "$@" }