Skip to content

Commit e9aa957

Browse files
ekrivokonmaprekrivokonmapr
committed
MapR [SPARK-1312] Make separate EEP Spark service for spark-connect (apache#1219)
1 parent d3e36e0 commit e9aa957

File tree

10 files changed

+191
-1
lines changed

10 files changed

+191
-1
lines changed

‎bin/configure.sh‎

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,8 @@ declare -a SPARK_CONF_FILES=("$SPARK_CONF/spark-defaults.conf" "$SPARK_CONF/spar
108108
# Spark ports
109109
sparkHSUIPort=18080
110110
isSparkHSUIPortDef=false
111+
sparkCSPort=15002
112+
isSparkCSPortDef=false
111113
sparkTSPort=2304
112114
isSparkTSPortDef=false
113115
sparkTSUIPort=4040
@@ -513,10 +515,27 @@ function registerPortHistoryServer() {
513515
fi
514516
}
515517

518+
function registerPortConnectServer() {
519+
if [ -f $SPARK_HOME/warden/warden.spark-connectserver.conf ] ; then
520+
if checkNetworkPortAvailability $sparkCSPort 2>/dev/null; then
521+
{ set +x; } 2>/dev/null
522+
registerNetworkPort spark_connectServer $sparkCSPort 2>/dev/null
523+
else
524+
{ set +x; } 2>/dev/null
525+
logWarn "Spark-connectServer port already has been taken by $(whoHasNetworkPort $sparkCSPort)"
526+
fi
527+
528+
if [ "$isSparkCSPortDef" = true ] || [ "$IS_FIRST_RUN" = true ] ; then
529+
changeWardenConfig "service.port" "service.port=$sparkCSPort" "connectserver"
530+
fi
531+
fi
532+
}
533+
516534
function registerServicePorts() {
517535
registerPortMaster
518536
registerPortThriftServer
519537
registerPortHistoryServer
538+
registerPortConnectServer
520539
}
521540

522541
function configureHiveWarehouseForNonHiveEnv() {
@@ -543,6 +562,7 @@ function copyWardenConfFiles() {
543562
copyWardenFile master
544563
copyWardenFile historyserver
545564
copyWardenFile thriftserver
565+
copyWardenFile connectserver
546566
}
547567

548568
function mkBackupForOldConfigs() {
@@ -578,6 +598,7 @@ function stopServicesForRestartByWarden() {
578598
stopService master master
579599
stopService historyserver history-server
580600
stopService thriftserver thriftserver
601+
stopService connectserver connect-server
581602
fi
582603
}
583604

@@ -605,7 +626,7 @@ function replaceConfigFromPreviousVersion() {
605626

606627
USAGE="usage: $0 [-s|--secure || -u|--unsecure || -cs|--customSecure] [-R] [--EC <common args>] [-h|--help]]"
607628

608-
{ OPTS=`getopt -n "$0" -a -o suhR --long secure,unsecure,customSecure,help,EC:,sparkHSUIPort:,sparkMasterPort:,sparkTSPort:,sparkMasterUIPort:,sparkTSUIPort: -- "$@"`; } 2>/dev/null
629+
{ OPTS=`getopt -n "$0" -a -o suhR --long secure,unsecure,customSecure,help,EC:,sparkHSUIPort:,sparkCSPort:,sparkMasterPort:,sparkTSPort:,sparkMasterUIPort:,sparkTSUIPort: -- "$@"`; } 2>/dev/null
609630

610631
eval set -- "$OPTS"
611632

@@ -639,6 +660,11 @@ while [ ${#} -gt 0 ] ; do
639660
isSparkHSUIPortDef=true
640661
shift 2
641662
;;
663+
--sparkCSPort)
664+
sparkCSPort=$2
665+
isSparkCSPortDef=true
666+
shift 2
667+
;;
642668
--sparkMasterPort)
643669
sparkMasterPort=$2
644670
isSparkMasterPortDef=true

‎devops/scripts/build.sh‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ build_spark() {
2121
cp ext-conf/warden.spark-historyserver.conf "${BUILD_ROOT}/build/warden/warden.spark-historyserver.conf.template"
2222
cp ext-conf/warden.spark-master.conf "${BUILD_ROOT}/build/warden/warden.spark-master.conf.template"
2323
cp ext-conf/warden.spark-thriftserver.conf "${BUILD_ROOT}/build/warden/warden.spark-thriftserver.conf.template"
24+
cp ext-conf/warden.spark-connectserver.conf "${BUILD_ROOT}/build/warden/warden.spark-connectserver.conf.template"
2425

2526
cp -rp ext-lib/scala "${BUILD_ROOT}/build"
2627
}
@@ -42,6 +43,7 @@ main() {
4243
setup_role "mapr-spark-historyserver"
4344
setup_role "mapr-spark-master"
4445
setup_role "mapr-spark-thriftserver"
46+
setup_role "mapr-spark-connectserver"
4547

4648
setup_package "mapr-spark"
4749

@@ -50,6 +52,7 @@ main() {
5052
build_package "mapr-spark-historyserver"
5153
build_package "mapr-spark-master"
5254
build_package "mapr-spark-thriftserver"
55+
build_package "mapr-spark-connectserver"
5356

5457
echo "Resulting packages:"
5558
find "$DIST_DIR" -exec readlink -f {} \;

‎devops/specs/mapr-spark-connect/deb/control‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
Package: mapr-spark-connectserver
2+
Version: __RELEASE_VERSION__
3+
Section: base
4+
Priority: optional
5+
Architecture: all
6+
Depends: mapr-spark (= __RELEASE_VERSION__)
7+
Maintainer: Hewlett Packard Enterprise <[email protected]>
8+
Description: Ezmeral Ecosystem Pack: Spark Connect Server package
9+
Tag: __RELEASE_BRANCH__
10+
Commit: __GIT_COMMIT__

‎devops/specs/mapr-spark-connect/deb/copyright‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Copyright: Hewlett Packard Enterprise, All rights reserved

‎devops/specs/mapr-spark-connect/deb/postinst‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
#!/bin/sh
2+
3+
set -e
4+
SPARK_VERSION=__VERSION_3DIGIT__
5+
MAPR_SPARK_HOME=$(dir -d -1 __PREFIX__/spark/spark-${SPARK_VERSION} 2> /dev/null | tail -1)
6+
7+
case "$1" in
8+
configure)
9+
echo "postinst called with argument \`$1'" >&2
10+
11+
cp $MAPR_SPARK_HOME/warden/warden.spark-connectserver.conf.template \
12+
$MAPR_SPARK_HOME/warden/warden.spark-connectserver.conf
13+
14+
;;
15+
16+
abort-upgrade|abort-remove|abort-deconfigure)
17+
echo "postinst called with argument \`$1'" >&2
18+
;;
19+
20+
*)
21+
echo "postinst called with unknown argument \`$1'" >&2
22+
exit 1
23+
;;
24+
esac
25+
26+

‎devops/specs/mapr-spark-connect/deb/postrm‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
#!/bin/sh
2+
3+
set -e
4+
5+
case "$1" in
6+
upgrade)
7+
echo "postrm called with argument \`$1'" >&2
8+
;;
9+
10+
remove|purge|disappear)
11+
echo "postrm called with argument \`$1'" >&2
12+
;;
13+
14+
abort-install|abort-upgrade|failed-upgrade)
15+
echo "postrm called with argument \`$1'" >&2
16+
;;
17+
18+
*)
19+
echo "postrm called with unknown argument \`$1'" >&2
20+
exit 1
21+
;;
22+
esac
23+
24+
exit 0

‎devops/specs/mapr-spark-connect/deb/prerm‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
#!/bin/sh
2+
3+
set -e
4+
SPARK_VERSION=__VERSION_3DIGIT__
5+
MAPR_SPARK_HOME=$(dir -d -1 __PREFIX__/spark/spark-${SPARK_VERSION} 2> /dev/null | tail -1)
6+
case "$1" in
7+
remove|upgrade|deconfigure)
8+
echo "prerm called with argument \`$1'" >&2
9+
echo "stopping any alive Spark-ConnectServer daemon on the node"
10+
rm -f __PREFIX__/conf/conf.d/warden.spark-connectserver.conf
11+
rm -f $MAPR_SPARK_HOME/warden/warden.spark-connectserver.conf
12+
if [ -f ${MAPR_SPARK_HOME}/sbin/stop-connect-server.sh ]; then
13+
${MAPR_SPARK_HOME}/sbin/stop-connect-server.sh
14+
fi
15+
;;
16+
17+
abort-upgrade|abort-remove|failed-upgrade)
18+
echo "prerm called with argument \`$1'" >&2
19+
;;
20+
21+
*)
22+
echo "prerm called with unknown argument \`$1'" >&2
23+
exit 1
24+
;;
25+
esac
26+
27+
28+
exit 0

‎devops/specs/mapr-spark-connect/roles/spark-connectserver‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
PKG_HOME_DIR=__INSTALL_3DIGIT__
2+
PKG_CONFIG_COMMAND=

‎devops/specs/mapr-spark-connect/rpm/mapr-spark-connectserver.spec‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
%undefine __check_files
2+
3+
summary: MapR
4+
license: Hewlett Packard Enterprise, CopyRight
5+
Vendor: Hewlett Packard Enterprise, <[email protected]>
6+
name: mapr-spark-connectserver
7+
version: __RELEASE_VERSION__
8+
release: 1
9+
prefix: /
10+
group: MapR
11+
buildarch: noarch
12+
requires: mapr-spark = __RELEASE_VERSION__
13+
AutoReqProv: no
14+
15+
%description
16+
Ezmeral Ecosystem Pack: Spark Connect Server package
17+
Tag: __RELEASE_BRANCH__
18+
Commit: __GIT_COMMIT__
19+
20+
21+
%clean
22+
echo "NOOP"
23+
24+
25+
%files
26+
__PREFIX__/roles
27+
28+
29+
%post
30+
echo "post-install called with argument \`$1'" >&2
31+
32+
SPARK_VERSION=__VERSION_3DIGIT__
33+
MAPR_SPARK_HOME=$(dir -d -1 __PREFIX__/spark/spark-$SPARK_VERSION 2> /dev/null | tail -1)
34+
35+
cp $MAPR_SPARK_HOME/warden/warden.spark-connectserver.conf.template \
36+
$MAPR_SPARK_HOME/warden/warden.spark-connectserver.conf
37+
38+
if [ "$1" = "1" ] ; then
39+
echo $SPARK_VERSION > __PREFIX__/spark/sparkversion
40+
fi
41+
42+
%preun
43+
SPARK_VERSION=__VERSION_3DIGIT__
44+
MAPR_SPARK_HOME=$(dir -d -1 __PREFIX__/spark/spark-$SPARK_VERSION 2> /dev/null | tail -1)
45+
echo "pre-uninstall called with argument \`$1'" >&2
46+
echo "stopping any alive Spark-ConnectServer daemon on the node"
47+
rm -f __PREFIX__/conf/conf.d/warden.spark-connectserver.conf
48+
if [ $1 -eq 0 ]; then
49+
rm -f $MAPR_SPARK_HOME/warden/warden.spark-connectserver.conf
50+
fi
51+
if [ -f "$MAPR_SPARK_HOME/sbin/stop-connect-server.sh" ]; then
52+
$MAPR_SPARK_HOME/sbin/stop-connect-server.sh
53+
fi
54+
55+
%posttrans
56+
SPARK_VERSION=__VERSION_3DIGIT__
57+
echo "post-transaction called with argument \`$1'" >&2
58+
echo $SPARK_VERSION > __PREFIX__/spark/sparkversion

‎ext-conf/warden.spark-connectserver.conf‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
services=spark-connect:1:cldb
2+
service.displayname=SparkConnectServer
3+
service.command.start=/opt/mapr/spark/spark-3.5.2/sbin/start-connect-server.sh
4+
service.command.stop=/opt/mapr/spark/spark-3.5.2/sbin/stop-connect-server.sh
5+
service.command.monitorcommand=/opt/mapr/spark/spark-3.5.2/sbin/spark-daemon.sh status org.apache.spark.sql.connect.service.SparkConnectServer 1
6+
service.command.type=BACKGROUND
7+
service.port=15002
8+
service.uri=/
9+
service.logs.location=/opt/mapr/spark/spark-3.5.2/logs/
10+
service.process.type=JAVA
11+
service.alarm.tersename=nasparkh
12+
service.alarm.label=SPARKCONNECTDOWNALARM

0 commit comments

Comments
 (0)