diff --git a/connector/connect/bin/spark-connect b/connector/connect/bin/spark-connect index 62d0d36b4410..772a88a04f3e 100755 --- a/connector/connect/bin/spark-connect +++ b/connector/connect/bin/spark-connect @@ -26,7 +26,14 @@ FWDIR="$(cd "`dirname "$0"`"/../../..; pwd)" cd "$FWDIR" export SPARK_HOME=$FWDIR +# Determine the Scala version used in Spark +SCALA_BINARY_VER=`grep "scala.binary.version" "${SPARK_HOME}/pom.xml" | head -n1 | awk -F '[<>]' '{print $3}'` +SCALA_ARG="-Pscala-${SCALA_BINARY_VER}" + # Build the jars needed for spark submit and spark connect -build/sbt -Phive -Pconnect package +build/sbt "${SCALA_ARG}" -Phive -Pconnect package + +# This jar is already in the classpath, but the submit commands wants a jar as the input. +CONNECT_JAR=`ls "${SPARK_HOME}"/assembly/target/scala-"${SCALA_BINARY_VER}"/jars/spark-connect_*.jar | paste -sd ',' -` -exec "${SPARK_HOME}"/bin/spark-submit --class org.apache.spark.sql.connect.SimpleSparkConnectService "$@" \ No newline at end of file +exec "${SPARK_HOME}"/bin/spark-submit "$@" --class org.apache.spark.sql.connect.SimpleSparkConnectService "$CONNECT_JAR" diff --git a/connector/connect/bin/spark-connect-scala-client b/connector/connect/bin/spark-connect-scala-client index 902091a74de0..8c5e687ef245 100755 --- a/connector/connect/bin/spark-connect-scala-client +++ b/connector/connect/bin/spark-connect-scala-client @@ -34,17 +34,18 @@ FWDIR="$(cd "`dirname "$0"`"/../../..; pwd)" cd "$FWDIR" export SPARK_HOME=$FWDIR -# Build the jars needed for spark connect JVM client -build/sbt "sql/package;connect-client-jvm/assembly" - -CONNECT_CLASSPATH="$(build/sbt -DcopyDependencies=false "export connect-client-jvm/fullClasspath" | grep jar | tail -n1)" -SQL_CLASSPATH="$(build/sbt -DcopyDependencies=false "export sql/fullClasspath" | grep jar | tail -n1)" - -INIT_SCRIPT="${SPARK_HOME}"/connector/connect/bin/spark-connect-scala-client.sc - # Determine the Scala version used in Spark SCALA_BINARY_VER=`grep "scala.binary.version" "${SPARK_HOME}/pom.xml" | head -n1 | awk -F '[<>]' '{print $3}'` SCALA_VER=`grep "scala.version" "${SPARK_HOME}/pom.xml" | grep ${SCALA_BINARY_VER} | head -n1 | awk -F '[<>]' '{print $3}'` SCALA_BIN="${SPARK_HOME}/build/scala-${SCALA_VER}/bin/scala" +SCALA_ARG="-Pscala-${SCALA_BINARY_VER}" + +# Build the jars needed for spark connect JVM client +build/sbt "${SCALA_ARG}" "sql/package;connect-client-jvm/assembly" + +CONNECT_CLASSPATH="$(build/sbt "${SCALA_ARG}" -DcopyDependencies=false "export connect-client-jvm/fullClasspath" | grep jar | tail -n1)" +SQL_CLASSPATH="$(build/sbt "${SCALA_ARG}" -DcopyDependencies=false "export sql/fullClasspath" | grep jar | tail -n1)" + +INIT_SCRIPT="${SPARK_HOME}"/connector/connect/bin/spark-connect-scala-client.sc -exec "${SCALA_BIN}" -cp "$CONNECT_CLASSPATH:$SQL_CLASSPATH" -i $INIT_SCRIPT \ No newline at end of file +exec "${SCALA_BIN}" -cp "$CONNECT_CLASSPATH:$SQL_CLASSPATH" -i $INIT_SCRIPT diff --git a/connector/connect/bin/spark-connect-shell b/connector/connect/bin/spark-connect-shell index b31ba1bf140d..0fcf831e03db 100755 --- a/connector/connect/bin/spark-connect-shell +++ b/connector/connect/bin/spark-connect-shell @@ -26,7 +26,11 @@ FWDIR="$(cd "`dirname "$0"`"/../../..; pwd)" cd "$FWDIR" export SPARK_HOME=$FWDIR -# Build the jars needed for spark shell and spark connect -build/sbt -Phive -Pconnect package +# Determine the Scala version used in Spark +SCALA_BINARY_VER=`grep "scala.binary.version" "${SPARK_HOME}/pom.xml" | head -n1 | awk -F '[<>]' '{print $3}'` +SCALA_ARG="-Pscala-${SCALA_BINARY_VER}" -exec "${SPARK_HOME}"/bin/spark-shell --conf spark.plugins=org.apache.spark.sql.connect.SparkConnectPlugin "$@" \ No newline at end of file +# Build the jars needed for spark submit and spark connect +build/sbt "${SCALA_ARG}" -Phive -Pconnect package + +exec "${SPARK_HOME}"/bin/spark-shell --conf spark.plugins=org.apache.spark.sql.connect.SparkConnectPlugin "$@"