Skip to content

Commit 8dbe777

Browse files
meawopplDavies Liu
authored andcommitted
[SPARK-7806][EC2] Fixes that allow the spark_ec2.py tool to run with Python3
I have used this script to launch, destroy, start, and stop clusters successfully. Author: meawoppl <[email protected]> Closes #6336 from meawoppl/py3ec2spark and squashes the following commits: 2e87046 [meawoppl] Py3 compat fixes.
1 parent 8948ad3 commit 8dbe777

File tree

1 file changed

+9
-5
lines changed

1 file changed

+9
-5
lines changed

ec2/spark_ec2.py

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,9 @@
1919
# limitations under the License.
2020
#
2121

22-
from __future__ import with_statement, print_function
22+
from __future__ import division, print_function, with_statement
2323

24+
import codecs
2425
import hashlib
2526
import itertools
2627
import logging
@@ -47,6 +48,8 @@
4748
else:
4849
from urllib.request import urlopen, Request
4950
from urllib.error import HTTPError
51+
raw_input = input
52+
xrange = range
5053

5154
SPARK_EC2_VERSION = "1.3.1"
5255
SPARK_EC2_DIR = os.path.dirname(os.path.realpath(__file__))
@@ -423,13 +426,14 @@ def get_spark_ami(opts):
423426
b=opts.spark_ec2_git_branch)
424427

425428
ami_path = "%s/%s/%s" % (ami_prefix, opts.region, instance_type)
429+
reader = codecs.getreader("ascii")
426430
try:
427-
ami = urlopen(ami_path).read().strip()
428-
print("Spark AMI: " + ami)
431+
ami = reader(urlopen(ami_path)).read().strip()
429432
except:
430433
print("Could not resolve AMI at: " + ami_path, file=stderr)
431434
sys.exit(1)
432435

436+
print("Spark AMI: " + ami)
433437
return ami
434438

435439

@@ -750,7 +754,7 @@ def setup_cluster(conn, master_nodes, slave_nodes, opts, deploy_ssh_key):
750754
'mapreduce', 'spark-standalone', 'tachyon']
751755

752756
if opts.hadoop_major_version == "1":
753-
modules = filter(lambda x: x != "mapreduce", modules)
757+
modules = list(filter(lambda x: x != "mapreduce", modules))
754758

755759
if opts.ganglia:
756760
modules.append('ganglia')
@@ -1160,7 +1164,7 @@ def get_zones(conn, opts):
11601164

11611165
# Gets the number of items in a partition
11621166
def get_partition(total, num_partitions, current_partitions):
1163-
num_slaves_this_zone = total / num_partitions
1167+
num_slaves_this_zone = total // num_partitions
11641168
if (total % num_partitions) - current_partitions > 0:
11651169
num_slaves_this_zone += 1
11661170
return num_slaves_this_zone

0 commit comments

Comments
 (0)