-
Notifications
You must be signed in to change notification settings - Fork 2
/
Copy pathspark.json
151 lines (151 loc) · 9.45 KB
/
spark.json
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
{
"_copyright": "2016-2022, Frederico Martins",
"_author": "Frederico Martins <https://github.com/fscm>",
"_license": "SPDX-License-Identifier: MIT",
"variables": {
"aws_access_key": "{{env `aws_access_key`}}",
"aws_ami_name": "spark",
"aws_ami_name_prefix": "",
"aws_instance_type": "t2.micro",
"aws_region": "{{env `aws_region`}}",
"aws_secret_key": "{{env `aws_secret_key`}}",
"aws_ssh_username": "admin",
"java_build_number": "12",
"java_major_version": "8",
"java_token": "e758a0de34e24606bca991d704f6dcbf",
"java_update_version": "151",
"os_short_arch": "x64",
"scala_short_version": "2.11",
"scala_version": "2.11.11",
"spark_hadoop_version": "{{env `hadoop_version`}}",
"spark_uid": "2000",
"spark_version": "{{env `spark_version`}}",
"system_locale": "en_US"
},
"builders": [{
"type": "amazon-ebs",
"access_key": "{{user `aws_access_key`}}",
"secret_key": "{{user `aws_secret_key`}}",
"region": "{{user `aws_region`}}",
"instance_type": "{{user `aws_instance_type`}}",
"ssh_username": "{{user `aws_ssh_username`}}",
"associate_public_ip_address": true,
"ami_name": "{{user `aws_ami_name_prefix`}}{{user `aws_ami_name`}}-{{user `spark_version`}}_{{user `hadoop_version`}}-({{isotime \"20060102150405\"}})",
"source_ami_filter": {
"filters": {
"architecture": "x86_64",
"name": "debian-jessie-*",
"root-device-type": "ebs",
"virtualization-type": "hvm"
},
"owners": ["379101102735"],
"most_recent": true
}
}],
"provisioners": [
{
"type": "file",
"source": "files/sysctl/",
"destination": "/tmp"
},
{
"type": "file",
"source": "files/systemd/",
"destination": "/tmp"
},
{
"type": "file",
"source": "files/spark/",
"destination": "/tmp"
},
{
"type": "shell",
"inline_shebang": "/bin/bash -e",
"environment_vars": [
"DEBIAN_FRONTEND=noninteractive"
],
"inline": [
"unset HISTFILE",
"history -cw",
"echo === Waiting for Cloud-Init ===",
"timeout 180 /bin/bash -c 'until stat /var/lib/cloud/instance/boot-finished &>/dev/null; do echo waiting...; sleep 6; done'",
"echo === System Packages ===",
"echo 'deb http://ftp.debian.org/debian jessie-backports main contrib non-free' | sudo tee /etc/apt/sources.list.d/backports.list > /dev/null",
"sudo apt-get -qq update",
"sudo apt-get -y -qq install --no-install-recommends apt-transport-https apt-show-versions bash-completion logrotate ntp ntpdate htop vim wget curl dbus bmon nmon parted wget curl sudo rsyslog ethtool unzip zip telnet tcpdump strace tar libyaml-0-2 lsb-base lsb-release xfsprogs sysfsutils",
"sudo apt-get -y -qq install --no-install-recommends python3 python3-dev python3-pip libzmq3 libzmq3-dev ipython3-notebook python3-zmq ipython3 python3-nose python3-backports-abc python3-jsonschema python3-numpy python3-scipy python3-pandas python3-jinja2 python3-markupsafe python3-tornado python3-setuptools python3-jsonschema python3-yaml",
"sudo apt-get -y -qq --purge autoremove",
"sudo apt-get autoclean",
"sudo apt-get clean",
"echo === Python Settings ===",
"echo -e 'export PYTHONHASHSEED=0\\nexport PYTHONIOENCODING=UTF-8\\nexport PIP_DISABLE_PIP_VERSION_CHECK=1' | sudo tee /etc/profile.d/python.sh > /dev/null",
"sudo sed -i -r -e 's/#DefaultEnvironment/DefaultEnvironment/;/DefaultEnvironment/s/([^=])$/\\1 /;/DefaultEnvironment/s/$/\"PYTHONHASHSEED=0\" \"PYTHONIOENCODING=UTF-8\" \"PIP_DISABLE_PIP_VERSION_CHECK=1\"/' /etc/systemd/system.conf",
"echo === System Settings ===",
"echo 'dash dash/sh boolean false' | sudo debconf-set-selections",
"sudo dpkg-reconfigure -f noninteractive dash",
"sudo update-locale LC_CTYPE={{user `system_locale`}}.UTF-8",
"echo 'export TZ=:/etc/localtime' | sudo tee /etc/profile.d/tz.sh > /dev/null",
"sudo update-alternatives --set editor /usr/bin/vim.basic",
"echo === Sysctl ===",
"sudo cp /tmp/50-spark.conf /etc/sysctl.d/",
"sudo chown root:root /etc/sysctl.d/50-spark.conf",
"sudo chmod 0644 /etc/sysctl.d/50-spark.conf",
"sudo sysctl -p /etc/sysctl.d/50-spark.conf",
"echo === Java ===",
"sudo mkdir /opt/java",
"curl -sL --retry 3 --insecure --header 'Cookie: oraclelicense=accept-securebackup-cookie;' 'http://download.oracle.com/otn-pub/java/jdk/{{user `java_major_version`}}u{{user `java_update_version`}}-b{{user `java_build_number`}}/{{user `java_token`}}/jre-{{user `java_major_version`}}u{{user `java_update_version`}}-linux-{{user `os_short_arch`}}.tar.gz' | sudo tar xz --strip-components=1 -C /opt/java/",
"sudo chown -R root:root /opt/java",
"echo 'export JAVA_HOME=/opt/java' | sudo tee /etc/profile.d/java.sh > /dev/null",
"sudo sed -i -r -e 's/#DefaultEnvironment/DefaultEnvironment/;/DefaultEnvironment/s/([^=])$/\\1 /;/DefaultEnvironment/s/$/\"JAVA_HOME=\\/opt\\/java\"/' /etc/systemd/system.conf",
"sudo gzip -r /opt/java/man/man1",
"for program in /opt/java/bin/*; do name=${program##*/}; manpage=''; [[ -f /opt/java/man/man1/${name}.1.gz ]] && manpage=\"--slave /usr/share/man/man1/${name}.1.gz ${name}.1.gz /opt/java/man/man1/${name}.1.gz\"; [[ -x ${program} && ! -L ${program} ]] && sudo update-alternatives --install /usr/bin/${name} ${name} /opt/java/bin/${name} 1 ${manpage}; done",
"echo === Scala ===",
"sudo mkdir -p /opt/scala",
"curl -sL --retry 3 --insecure 'http://downloads.lightbend.com/scala/{{user `scala_version`}}/scala-{{user `scala_version`}}.tgz' | sudo tar xz --strip-components=1 -C /opt/scala/",
"echo -e 'export SCALA_HOME=/opt/scala' | sudo tee /etc/profile.d/scala.sh > /dev/null",
"sudo sed -i -r -e 's/#DefaultEnvironment/DefaultEnvironment/;/DefaultEnvironment/s/([^=])$/\\1 /;/DefaultEnvironment/s|$|\"SCALA_HOME=/opt/scala\"|' /etc/systemd/system.conf",
"sudo gzip -r /opt/scala/man/man1",
"sudo chown -R root:root /opt/scala",
"for program in fsc scala scalac scaladoc scalap; do sudo update-alternatives --install /usr/bin/${program} ${program} /opt/scala/bin/${program} 1 --slave /usr/share/man/man1/${program}.1.gz ${program}.1.gz /opt/scala/man/man1/${program}.1.gz; done",
"echo === Spark ===",
"sudo groupadd -g {{user `spark_uid`}} spark",
"sudo useradd -m -u {{user `spark_uid`}} -g {{user `spark_uid`}} -c 'Apache Spark' -s /bin/bash -d /srv/spark spark",
"curl -sL --retry 3 --insecure 'http://d3kbcqa49mib13.cloudfront.net/spark-{{user `spark_version`}}-bin-hadoop{{user `spark_hadoop_version`}}.tgz' | sudo tar xz --strip-components=1 -C /srv/spark/",
"sudo mkdir -p /srv/spark/{work,tmp}",
"sudo mkdir -p /var/{log,run}/spark",
"sudo ln -s /var/log/spark /srv/spark/logs",
"sudo ln -s `basename /srv/spark/python/lib/py4j-*-src.zip` /srv/spark/python/lib/py4j.zip",
"sudo cp /srv/spark/conf/log4j.properties.template /srv/spark/conf/log4j.properties",
"sudo cp /srv/spark/conf/fairscheduler.xml.template /srv/spark/conf/fairscheduler.xml",
"echo -e 'export SPARK_HOME=/srv/spark\\nexport PYSPARK_PYTHON=python3\\nexport PYTHONPATH=$SPARK_HOME/python/:$SPARK_HOME/python/lib/:$SPARK_HOME/python/lib/py4j.zip\\nexport PATH=$PATH:$SPARK_HOME/bin:$SPARK_HOME/sbin' | sudo tee /etc/profile.d/spark.sh > /dev/null",
"sudo sed -i -r -e 's/#DefaultEnvironment/DefaultEnvironment/;/DefaultEnvironment/s/([^=])$/\\1 /;/DefaultEnvironment/s/$/\"SPARK_HOME=\\/srv\\/spark\" \"PYSPARK_PYTHON=python3\" \"PYTHONPATH=\\/srv\\/spark\\/python\\/:\\/srv\\/spark\\/python\\/lib\\/:\\/srv\\/spark\\/python\\/lib\\/py4j.zip\"/' /etc/systemd/system.conf",
"sudo cp /tmp/spark-defaults.conf /srv/spark/conf/",
"sudo cp /tmp/spark-env.sh /srv/spark/conf/",
"sudo chown -R spark:spark /srv/spark /var/log/spark /var/run/spark",
"sudo cp /tmp/spark-master.service /lib/systemd/system/",
"sudo cp /tmp/spark-worker.service /lib/systemd/system/",
"sudo cp /tmp/spark-history.service /lib/systemd/system/",
"sudo systemctl daemon-reload",
"sudo systemctl disable spark-master.service",
"sudo systemctl disable spark-worker.service",
"sudo systemctl disable spark-history.service",
"sudo cp /tmp/spark_config /usr/local/bin/",
"sudo chown root:staff /usr/local/bin/spark_config",
"sudo chmod 0755 /usr/local/bin/spark_config",
"echo === System Cleanup ===",
"sudo rm -f /root/.bash_history",
"sudo rm -f /home/{{user `aws_ssh_username`}}/.bash_history",
"sudo rm -f /var/log/wtmp",
"sudo rm -f /var/log/btmp",
"sudo rm -rf /var/log/installer",
"sudo rm -rf /var/lib/cloud/instances",
"sudo rm -rf /tmp/* /var/tmp/* /tmp/.*-unix",
"sudo find /var/cache -type f -delete",
"sudo find /var/log -type f | while read f; do echo -n '' | sudo tee $f > /dev/null; done;",
"sudo find /var/lib/apt/lists -not -name lock -type f -delete",
"sudo sync",
"echo === All Done ==="
]
}
]
}