Skip to content

Commit c796397

Browse files
jerryshaojerryshao
committed
Changes to honor SPARK_HOME to solve symlink issue
1 parent aa470fe commit c796397

31 files changed

+185
-775
lines changed

‎bin/beeline‎

Lines changed: 5 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -23,29 +23,10 @@
2323
# Enter posix mode for bash
2424
set -o posix
2525

26-
realpath () {
27-
(
28-
TARGET_FILE="$1"
29-
30-
cd "$(dirname "$TARGET_FILE")"
31-
TARGET_FILE="$(basename "$TARGET_FILE")"
32-
33-
COUNT=0
34-
while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ]
35-
do
36-
TARGET_FILE="$(readlink "$TARGET_FILE")"
37-
cd $(dirname "$TARGET_FILE")
38-
TARGET_FILE="$(basename $TARGET_FILE)"
39-
COUNT=$(($COUNT + 1))
40-
done
41-
42-
echo "$(pwd -P)/"$TARGET_FILE""
43-
)
44-
}
45-
46-
# Figure out where Spark is installed
47-
DIR="$(dirname "$(realpath "$0")")"
48-
FWDIR="$(cd "$DIR/.."; pwd)"
26+
# Figure out if SPARK_HOME is set
27+
if [ -z "${SPARK_HOME}" ]; then
28+
export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)"
29+
fi
4930

5031
CLASS="org.apache.hive.beeline.BeeLine"
51-
exec "$FWDIR/bin/spark-class" $CLASS "$@"
32+
exec "${SPARK_HOME}/bin/spark-class" $CLASS "$@"

‎bin/load-spark-env.sh‎

Lines changed: 6 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -20,35 +20,17 @@
2020
# This script loads spark-env.sh if it exists, and ensures it is only loaded once.
2121
# spark-env.sh is loaded from SPARK_CONF_DIR if set, or within the current directory's
2222
# conf/ subdirectory.
23-
realpath () {
24-
(
25-
TARGET_FILE="$1"
26-
27-
cd "$(dirname "$TARGET_FILE")"
28-
TARGET_FILE="$(basename "$TARGET_FILE")"
29-
30-
COUNT=0
31-
while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ]
32-
do
33-
TARGET_FILE="$(readlink "$TARGET_FILE")"
34-
cd $(dirname "$TARGET_FILE")
35-
TARGET_FILE="$(basename $TARGET_FILE)"
36-
COUNT=$(($COUNT + 1))
37-
done
38-
39-
echo "$(pwd -P)/"$TARGET_FILE""
40-
)
41-
}
4223

4324
# Figure out where Spark is installed
44-
DIR="$(dirname "$(realpath "$0")")"
45-
FWDIR="$(cd "$DIR/.."; pwd)"
25+
if [ -z "${SPARK_HOME}" ]; then
26+
export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)"
27+
fi
4628

4729
if [ -z "$SPARK_ENV_LOADED" ]; then
4830
export SPARK_ENV_LOADED=1
4931

5032
# Returns the parent of the directory this script lives in.
51-
parent_dir="$(cd "`dirname "$0"`"/..; pwd)"
33+
parent_dir="${SPARK_HOME}"
5234

5335
user_conf_dir="${SPARK_CONF_DIR:-"$parent_dir"/conf}"
5436

@@ -64,8 +46,8 @@ fi
6446

6547
if [ -z "$SPARK_SCALA_VERSION" ]; then
6648

67-
ASSEMBLY_DIR2="$FWDIR/assembly/target/scala-2.11"
68-
ASSEMBLY_DIR1="$FWDIR/assembly/target/scala-2.10"
49+
ASSEMBLY_DIR2="${SPARK_HOME}/assembly/target/scala-2.11"
50+
ASSEMBLY_DIR1="${SPARK_HOME}/assembly/target/scala-2.10"
6951

7052
if [[ -d "$ASSEMBLY_DIR2" && -d "$ASSEMBLY_DIR1" ]]; then
7153
echo -e "Presence of build for both scala versions(SCALA 2.10 and SCALA 2.11) detected." 1>&2

‎bin/pyspark‎

Lines changed: 8 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -17,32 +17,11 @@
1717
# limitations under the License.
1818
#
1919

20-
realpath () {
21-
(
22-
TARGET_FILE="$1"
23-
24-
cd "$(dirname "$TARGET_FILE")"
25-
TARGET_FILE="$(basename "$TARGET_FILE")"
26-
27-
COUNT=0
28-
while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ]
29-
do
30-
TARGET_FILE="$(readlink "$TARGET_FILE")"
31-
cd $(dirname "$TARGET_FILE")
32-
TARGET_FILE="$(basename $TARGET_FILE)"
33-
COUNT=$(($COUNT + 1))
34-
done
35-
36-
echo "$(pwd -P)/"$TARGET_FILE""
37-
)
38-
}
39-
40-
# Figure out where Spark is installed
41-
DIR="$(dirname "$(realpath "$0")")"
42-
43-
export SPARK_HOME="$(cd "$DIR/.."; pwd)"
20+
if [ -z "${SPARK_HOME}" ]; then
21+
export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)"
22+
fi
4423

45-
source "$SPARK_HOME"/bin/load-spark-env.sh
24+
source "{$SPARK_HOME}"/bin/load-spark-env.sh
4625
export _SPARK_CMD_USAGE="Usage: ./bin/pyspark [options]"
4726

4827
# In Spark <= 1.1, setting IPYTHON=1 would cause the driver to be launched using the `ipython`
@@ -87,12 +66,12 @@ fi
8766
export PYSPARK_PYTHON
8867

8968
# Add the PySpark classes to the Python path:
90-
export PYTHONPATH="$SPARK_HOME/python/:$PYTHONPATH"
91-
export PYTHONPATH="$SPARK_HOME/python/lib/py4j-0.9-src.zip:$PYTHONPATH"
69+
export PYTHONPATH="${SPARK_HOME}/python/:$PYTHONPATH"
70+
export PYTHONPATH="${SPARK_HOME}/python/lib/py4j-0.9-src.zip:$PYTHONPATH"
9271

9372
# Load the PySpark shell.py script when ./pyspark is used interactively:
9473
export OLD_PYTHONSTARTUP="$PYTHONSTARTUP"
95-
export PYTHONSTARTUP="$SPARK_HOME/python/pyspark/shell.py"
74+
export PYTHONSTARTUP="${SPARK_HOME}/python/pyspark/shell.py"
9675

9776
# For pyspark tests
9877
if [[ -n "$SPARK_TESTING" ]]; then
@@ -105,4 +84,4 @@ fi
10584

10685
export PYSPARK_DRIVER_PYTHON
10786
export PYSPARK_DRIVER_PYTHON_OPTS
108-
exec "$SPARK_HOME"/bin/spark-submit pyspark-shell-main --name "PySparkShell" "$@"
87+
exec "${SPARK_HOME}"/bin/spark-submit pyspark-shell-main --name "PySparkShell" "$@"

‎bin/run-example‎

Lines changed: 9 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -17,32 +17,13 @@
1717
# limitations under the License.
1818
#
1919

20-
realpath () {
21-
(
22-
TARGET_FILE="$1"
23-
24-
cd "$(dirname "$TARGET_FILE")"
25-
TARGET_FILE="$(basename "$TARGET_FILE")"
26-
27-
COUNT=0
28-
while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ]
29-
do
30-
TARGET_FILE="$(readlink "$TARGET_FILE")"
31-
cd $(dirname "$TARGET_FILE")
32-
TARGET_FILE="$(basename $TARGET_FILE)"
33-
COUNT=$(($COUNT + 1))
34-
done
35-
36-
echo "$(pwd -P)/"$TARGET_FILE""
37-
)
38-
}
20+
if [ -z "${SPARK_HOME}" ]; then
21+
export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)"
22+
fi
3923

40-
DIR="$(dirname "$(realpath "$0")")"
41-
FWDIR="$(cd "$DIR/.."; pwd)"
42-
export SPARK_HOME="$FWDIR"
43-
EXAMPLES_DIR="$FWDIR"/examples
24+
EXAMPLES_DIR="${SPARK_HOME}"/examples
4425

45-
. "$FWDIR"/bin/load-spark-env.sh
26+
. "${SPARK_HOME}"/bin/load-spark-env.sh
4627

4728
if [ -n "$1" ]; then
4829
EXAMPLE_CLASS="$1"
@@ -55,8 +36,8 @@ else
5536
exit 1
5637
fi
5738

58-
if [ -f "$FWDIR/RELEASE" ]; then
59-
JAR_PATH="${FWDIR}/lib"
39+
if [ -f "${SPARK_HOME}/RELEASE" ]; then
40+
JAR_PATH="${SPARK_HOME}/lib"
6041
else
6142
JAR_PATH="${EXAMPLES_DIR}/target/scala-${SPARK_SCALA_VERSION}"
6243
fi
@@ -65,7 +46,7 @@ JAR_COUNT=0
6546

6647
for f in "${JAR_PATH}"/spark-examples-*hadoop*.jar; do
6748
if [[ ! -e "$f" ]]; then
68-
echo "Failed to find Spark examples assembly in $FWDIR/lib or $FWDIR/examples/target" 1>&2
49+
echo "Failed to find Spark examples assembly in ${SPARK_HOME}/lib or $SPARK_HOME/examples/target" 1>&2
6950
echo "You need to build Spark before running this program" 1>&2
7051
exit 1
7152
fi
@@ -88,7 +69,7 @@ if [[ ! $EXAMPLE_CLASS == org.apache.spark.examples* ]]; then
8869
EXAMPLE_CLASS="org.apache.spark.examples.$EXAMPLE_CLASS"
8970
fi
9071

91-
exec "$FWDIR"/bin/spark-submit \
72+
exec "${SPARK_HOME}"/bin/spark-submit \
9273
--master $EXAMPLE_MASTER \
9374
--class $EXAMPLE_CLASS \
9475
"$SPARK_EXAMPLES_JAR" \

‎bin/spark-class‎

Lines changed: 8 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -17,31 +17,11 @@
1717
# limitations under the License.
1818
#
1919

20-
realpath () {
21-
(
22-
TARGET_FILE="$1"
23-
24-
cd "$(dirname "$TARGET_FILE")"
25-
TARGET_FILE="$(basename "$TARGET_FILE")"
26-
27-
COUNT=0
28-
while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ]
29-
do
30-
TARGET_FILE="$(readlink "$TARGET_FILE")"
31-
cd $(dirname "$TARGET_FILE")
32-
TARGET_FILE="$(basename $TARGET_FILE)"
33-
COUNT=$(($COUNT + 1))
34-
done
35-
36-
echo "$(pwd -P)/"$TARGET_FILE""
37-
)
38-
}
39-
40-
# Figure out where Spark is installed
41-
DIR="$(dirname "$(realpath "$0")")"
42-
export SPARK_HOME="$(cd "$DIR/.."; pwd)"
20+
if [ -z "${SPARK_HOME}" ]; then
21+
export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)"
22+
fi
4323

44-
. "$SPARK_HOME"/bin/load-spark-env.sh
24+
. "${SPARK_HOME}"/bin/load-spark-env.sh
4525

4626
# Find the java binary
4727
if [ -n "${JAVA_HOME}" ]; then
@@ -57,10 +37,10 @@ fi
5737

5838
# Find assembly jar
5939
SPARK_ASSEMBLY_JAR=
60-
if [ -f "$SPARK_HOME/RELEASE" ]; then
61-
ASSEMBLY_DIR="$SPARK_HOME/lib"
40+
if [ -f "${SPARK_HOME}/RELEASE" ]; then
41+
ASSEMBLY_DIR="${SPARK_HOME}/lib"
6242
else
63-
ASSEMBLY_DIR="$SPARK_HOME/assembly/target/scala-$SPARK_SCALA_VERSION"
43+
ASSEMBLY_DIR="${SPARK_HOME}/assembly/target/scala-$SPARK_SCALA_VERSION"
6444
fi
6545

6646
GREP_OPTIONS=
@@ -86,7 +66,7 @@ LAUNCH_CLASSPATH="$SPARK_ASSEMBLY_JAR"
8666

8767
# Add the launcher build dir to the classpath if requested.
8868
if [ -n "$SPARK_PREPEND_CLASSES" ]; then
89-
LAUNCH_CLASSPATH="$SPARK_HOME/launcher/target/scala-$SPARK_SCALA_VERSION/classes:$LAUNCH_CLASSPATH"
69+
LAUNCH_CLASSPATH="${SPARK_HOME}/launcher/target/scala-$SPARK_SCALA_VERSION/classes:$LAUNCH_CLASSPATH"
9070
fi
9171

9272
export _SPARK_ASSEMBLY="$SPARK_ASSEMBLY_JAR"

‎bin/spark-shell‎

Lines changed: 5 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -28,30 +28,10 @@ esac
2828
# Enter posix mode for bash
2929
set -o posix
3030

31-
realpath () {
32-
(
33-
TARGET_FILE="$1"
34-
35-
cd "$(dirname "$TARGET_FILE")"
36-
TARGET_FILE="$(basename "$TARGET_FILE")"
37-
38-
COUNT=0
39-
while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ]
40-
do
41-
TARGET_FILE="$(readlink "$TARGET_FILE")"
42-
cd $(dirname "$TARGET_FILE")
43-
TARGET_FILE="$(basename $TARGET_FILE)"
44-
COUNT=$(($COUNT + 1))
45-
done
46-
47-
echo "$(pwd -P)/"$TARGET_FILE""
48-
)
49-
}
50-
51-
# Figure out where Spark is installed
52-
DIR="$(dirname "$(realpath "$0")")"
31+
if [ -z "${SPARK_HOME}" ]; then
32+
export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)"
33+
fi
5334

54-
export FWDIR="$(cd "$DIR/.."; pwd)"
5535
export _SPARK_CMD_USAGE="Usage: ./bin/spark-shell [options]"
5636

5737
# SPARK-4161: scala does not assume use of the java classpath,
@@ -70,11 +50,11 @@ function main() {
7050
# (see https://github.com/sbt/sbt/issues/562).
7151
stty -icanon min 1 -echo > /dev/null 2>&1
7252
export SPARK_SUBMIT_OPTS="$SPARK_SUBMIT_OPTS -Djline.terminal=unix"
73-
"$FWDIR"/bin/spark-submit --class org.apache.spark.repl.Main --name "Spark shell" "$@"
53+
"${SPARK_HOME}"/bin/spark-submit --class org.apache.spark.repl.Main --name "Spark shell" "$@"
7454
stty icanon echo > /dev/null 2>&1
7555
else
7656
export SPARK_SUBMIT_OPTS
77-
"$FWDIR"/bin/spark-submit --class org.apache.spark.repl.Main --name "Spark shell" "$@"
57+
"${SPARK_HOME}"/bin/spark-submit --class org.apache.spark.repl.Main --name "Spark shell" "$@"
7858
fi
7959
}
8060

‎bin/spark-sql‎

Lines changed: 4 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -17,29 +17,9 @@
1717
# limitations under the License.
1818
#
1919

20-
realpath () {
21-
(
22-
TARGET_FILE="$1"
20+
if [ -z "${SPARK_HOME}" ]; then
21+
export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)"
22+
fi
2323

24-
cd "$(dirname "$TARGET_FILE")"
25-
TARGET_FILE="$(basename "$TARGET_FILE")"
26-
27-
COUNT=0
28-
while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ]
29-
do
30-
TARGET_FILE="$(readlink "$TARGET_FILE")"
31-
cd $(dirname "$TARGET_FILE")
32-
TARGET_FILE="$(basename $TARGET_FILE)"
33-
COUNT=$(($COUNT + 1))
34-
done
35-
36-
echo "$(pwd -P)/"$TARGET_FILE""
37-
)
38-
}
39-
40-
# Figure out where Spark is installed
41-
DIR="$(dirname "$(realpath "$0")")"
42-
43-
FWDIR="$(cd "$DIR/.."; pwd)"
4424
export _SPARK_CMD_USAGE="Usage: ./bin/spark-sql [options] [cli option]"
45-
exec "$FWDIR"/bin/spark-submit --class org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver "$@"
25+
exec "${SPARK_HOME}"/bin/spark-submit --class org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver "$@"

‎bin/spark-submit‎

Lines changed: 4 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -17,31 +17,11 @@
1717
# limitations under the License.
1818
#
1919

20-
realpath () {
21-
(
22-
TARGET_FILE="$1"
23-
24-
cd "$(dirname "$TARGET_FILE")"
25-
TARGET_FILE="$(basename "$TARGET_FILE")"
26-
27-
COUNT=0
28-
while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ]
29-
do
30-
TARGET_FILE="$(readlink "$TARGET_FILE")"
31-
cd $(dirname "$TARGET_FILE")
32-
TARGET_FILE="$(basename $TARGET_FILE)"
33-
COUNT=$(($COUNT + 1))
34-
done
35-
36-
echo "$(pwd -P)/"$TARGET_FILE""
37-
)
38-
}
39-
40-
# Figure out where Spark is installed
41-
DIR="$(dirname "$(realpath "$0")")"
42-
SPARK_HOME="$(cd "$DIR/.."; pwd)"
20+
if [ -z "${SPARK_HOME}" ]; then
21+
export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)"
22+
fi
4323

4424
# disable randomized hash for string in Python 3.3+
4525
export PYTHONHASHSEED=0
4626

47-
exec "$SPARK_HOME"/bin/spark-class org.apache.spark.deploy.SparkSubmit "$@"
27+
exec "${SPARK_HOME}"/bin/spark-class org.apache.spark.deploy.SparkSubmit "$@"

0 commit comments

Comments
 (0)