Nullify my own hard work to simplify bash

:(

Author: andrewor14

bin/run-tests

@@ -30,9 +30,6 @@ FWDIR="$(cd `dirname $0`/..; pwd)"
 # Export this as SPARK_HOME
 export SPARK_HOME="$FWDIR"
 
-# Load utility functions
-. "$SPARK_HOME/bin/utils.sh"
-
 . $FWDIR/bin/load-spark-env.sh
 
 if [ -z "$1" ]; then
@@ -112,10 +109,6 @@ if [ -e "$FWDIR/conf/java-opts" ] ; then
   JAVA_OPTS="$JAVA_OPTS `cat $FWDIR/conf/java-opts`"
 fi
 
-# Split JAVA_OPTS properly to handle whitespace, double quotes and backslashes
-# This exports the split java options into SPLIT_JAVA_OPTS
-split_java_options "$JAVA_OPTS"
-
 # Attention: when changing the way the JAVA_OPTS are assembled, the change must be reflected in CommandUtils.scala!
 
 TOOLS_DIR="$FWDIR"/tools
@@ -156,13 +149,27 @@ if $cygwin; then
 fi
 export CLASSPATH
 
-if [ "$SPARK_PRINT_LAUNCH_COMMAND" == "1" ]; then
-  # Put quotes around system properties in case they contain spaces for readability
-  # This exports the resulting list of java opts into QUOTED_JAVA_OPTS
-  quote_java_property "${SPLIT_JAVA_OPTS[@]}"
+if [ -n "$SPARK_PRINT_LAUNCH_COMMAND" ]; then
   echo -n "Spark Command: " 1>&2
-  echo "$RUNNER" -cp "$CLASSPATH" "${QUOTED_JAVA_OPTS[@]}" "$@" 1>&2
+  echo "$RUNNER" -cp "$CLASSPATH" $JAVA_OPTS "$@" 1>&2
   echo -e "========================================\n" 1>&2
 fi
 
-exec "$RUNNER" -cp "$CLASSPATH" "${SPLIT_JAVA_OPTS[@]}" "$@"
+# In Spark submit client mode, the driver is launched in the same JVM as Spark submit itself.
+# Here we must parse the properties file for relevant "spark.driver.*" configs for launching
+# the driver JVM itself.
+
+if [ -n "$SPARK_SUBMIT_CLIENT_MODE" ]; then
+  exec "$RUNNER" org.apache.spark.deploy.SparkClassLauncher \
+    "$PROPERTIES_FILE" \
+    "$RUNNER" \
+    "$CLASSPATH" \
+    "$SPARK_SUBMIT_LIBRARY_PATH" \
+    "$JAVA_OPTS" \
+    "$OUR_JAVA_MEM" \
+    true \
+    "$@"
+else
+  exec "$RUNNER" -cp "$CLASSPATH" $JAVA_OPTS "$@"
+fi
+

bin/spark-class

@@ -20,9 +20,6 @@
 export SPARK_HOME="$(cd `dirname $0`/..; pwd)"
 ORIG_ARGS=("$@")
 
-# Load utility functions
-. "$SPARK_HOME/bin/utils.sh"
-
 while (($#)); do
   if [ "$1" = "--deploy-mode" ]; then
     DEPLOY_MODE=$2
@@ -44,68 +41,17 @@ DEPLOY_MODE=${DEPLOY_MODE:-"client"}
 DEFAULT_PROPERTIES_FILE="$SPARK_HOME/conf/spark-defaults.conf"
 PROPERTIES_FILE=${PROPERTIES_FILE:-"$DEFAULT_PROPERTIES_FILE"}
 
-unset DRIVER_EXTRA_JAVA_OPTIONS
-unset EXECUTOR_EXTRA_JAVA_OPTIONS
+# For client mode, the driver will be launched in the same JVM that launches
+# SparkSubmit, so we to read the properties file for any class paths, library
+# paths, java options and memory early on. Otherwise, it will be too late by
+# the time the JVM has started.
 
-# A few Spark configs must be parsed early on before launching the JVM:
-#
-#   [spark.driver.extra*]
-#     These configs encode java options, class paths, and library paths
-#     needed to launch the JVM if we are running Spark in client mode
-#
-#   [spark.*.extraJavaOptions]
-#     The escaped characters in these configs must be preserved for
-#     splitting the arguments in Java later. For these configs, we
-#     export the raw values as environment variables.
-#
-if [[ -f "$PROPERTIES_FILE" ]]; then
-  echo "Using properties file $PROPERTIES_FILE." 1>&2
-  # Parse the properties file here only if these special configs exist
-  should_parse=$(grep -e "spark.driver.extra*\|spark.*.extraJavaOptions" "$PROPERTIES_FILE")
-  if [[ -n "$should_parse" ]]; then
-    # This exports the value of the given key into JAVA_PROPERTY_VALUE
-    parse_java_property "spark.driver.memory"
-    DRIVER_MEMORY_CONF="$JAVA_PROPERTY_VALUE"
-    parse_java_property "spark.driver.extraLibraryPath"
-    DRIVER_EXTRA_LIBRARY_PATH="$JAVA_PROPERTY_VALUE"
-    parse_java_property "spark.driver.extraClassPath"
-    DRIVER_EXTRA_CLASSPATH="$JAVA_PROPERTY_VALUE"
-    parse_java_property "spark.driver.extraJavaOptions"
-    DRIVER_EXTRA_JAVA_OPTS="$JAVA_PROPERTY_VALUE"
-    parse_java_property "spark.executor.extraJavaOptions"
-    EXECUTOR_EXTRA_JAVA_OPTS="$JAVA_PROPERTY_VALUE"
-    # Export these for SparkSubmitArguments.scala to consume
-    if [[ -n "DRIVER_EXTRA_JAVA_OPTS" ]]; then
-      export DRIVER_EXTRA_JAVA_OPTS
-    fi
-    if [[ -n "EXECUTOR_EXTRA_JAVA_OPTS" ]]; then
-      export EXECUTOR_EXTRA_JAVA_OPTS
-    fi
-  fi
-elif [[ "$PROPERTIES_FILE" != "$DEFAULT_PROPERTIES_FILE" ]]; then
-  echo "Warning: properties file $PROPERTIES_FILE does not exist." 1>&2
-fi
-
-# For client mode, the driver will be launched in the JVM that launches
-# SparkSubmit, so we need to handle the class paths, java options, and
-# memory preemptively in bash. Otherwise, it will be too late by the
-# time the JVM has started.
-
-if [[ $DEPLOY_MODE == "client" ]]; then
-  if [[ -n "$DRIVER_EXTRA_JAVA_OPTS" ]]; then
-    export SPARK_SUBMIT_OPTS="$SPARK_SUBMIT_OPTS $DRIVER_EXTRA_JAVA_OPTS"
-  fi
-  if [[ -n "$DRIVER_EXTRA_CLASSPATH" ]]; then
-    export SPARK_SUBMIT_CLASSPATH="$SPARK_SUBMIT_CLASSPATH:$DRIVER_EXTRA_CLASSPATH"
-  fi
-  if [[ -n "$DRIVER_EXTRA_LIBRARY_PATH" ]]; then
-    export SPARK_SUBMIT_LIBRARY_PATH="$SPARK_SUBMIT_LIBRARY_PATH:$DRIVER_EXTRA_LIBRARY_PATH"
-  fi
-  # Favor command line memory over config memory
-  DRIVER_MEMORY=${DRIVER_MEMORY:-"$DRIVER_MEMORY_CONF"}
-  if [[ -n "$DRIVER_MEMORY" ]]; then
+if [ "$DEPLOY_MODE" == "client" ]; then
+  if [ -n "$DRIVER_MEMORY" ]; then
     export SPARK_DRIVER_MEMORY=$DRIVER_MEMORY
   fi
+  export PROPERTIES_FILE
+  export SPARK_SUBMIT_CLIENT_MODE=1
 fi
 
 exec $SPARK_HOME/bin/spark-class org.apache.spark.deploy.SparkSubmit "${ORIG_ARGS[@]}"