diff --git a/dev/run-tests.py b/dev/run-tests.py index f24aac9a9386..04bfc98c9853 100755 --- a/dev/run-tests.py +++ b/dev/run-tests.py @@ -121,7 +121,7 @@ def determine_modules_to_test(changed_modules): if modules.root in modules_to_test: return [modules.root] return toposort_flatten( - {m: set(m.dependencies).intersection(modules_to_test) for m in modules_to_test}, sort=True) + dict((m, set(m.dependencies).intersection(modules_to_test)) for m in modules_to_test), sort=True) def determine_tags_to_exclude(changed_modules): diff --git a/dev/sparktestsupport/modules.py b/dev/sparktestsupport/modules.py index 9e293e7206d5..2bcc629189ea 100644 --- a/dev/sparktestsupport/modules.py +++ b/dev/sparktestsupport/modules.py @@ -15,10 +15,48 @@ # limitations under the License. # -from functools import total_ordering +import sys import itertools import re +if sys.version_info >= (2, 7): + from functools import total_ordering +else: + def total_ordering(cls): # backport python 2.6 + """Class decorator that fills in missing ordering methods""" + convert = { + '__lt__': [ + ('__gt__', lambda self, other: other < self), + ('__le__', lambda self, other: not other < self), + ('__ge__', lambda self, other: not self < other) + ], + '__le__': [ + ('__ge__', lambda self, other: other <= self), + ('__lt__', lambda self, other: not other <= self), + ('__gt__', lambda self, other: not self <= other) + ], + '__gt__': [ + ('__lt__', lambda self, other: other > self), + ('__ge__', lambda self, other: not other > self), + ('__le__', lambda self, other: not self > other) + ], + '__ge__': [ + ('__le__', lambda self, other: other >= self), + ('__gt__', lambda self, other: not other >= self), + ('__lt__', lambda self, other: not self >= other) + ] + } + roots = set(dir(cls)) & set(convert) + if not roots: + raise ValueError('must define at least one ordering operation: < > <= >=') + root = max(roots) # prefer __lt__ to __le__ to __gt__ to __ge__ + for opname, opfunc in convert[root]: + if opname not in roots: + opfunc.__name__ = opname + opfunc.__doc__ = getattr(int, opname).__doc__ + setattr(cls, opname, opfunc) + return cls + all_modules = [] diff --git a/dev/sparktestsupport/shellutils.py b/dev/sparktestsupport/shellutils.py index 05af87189b18..019c08586f2a 100644 --- a/dev/sparktestsupport/shellutils.py +++ b/dev/sparktestsupport/shellutils.py @@ -43,7 +43,7 @@ def subprocess_check_output(*popenargs, **kwargs): # backported from subprocess module in Python 2.7 def subprocess_check_call(*popenargs, **kwargs): - retcode = call(*popenargs, **kwargs) + retcode = subprocess.call(*popenargs, **kwargs) if retcode: cmd = kwargs.get("args") if cmd is None: diff --git a/dev/sparktestsupport/toposort.py b/dev/sparktestsupport/toposort.py index 6c67b4504bc3..180dca298eaf 100644 --- a/dev/sparktestsupport/toposort.py +++ b/dev/sparktestsupport/toposort.py @@ -60,15 +60,13 @@ def toposort(data): # Find all items that don't depend on anything. extra_items_in_deps = _reduce(set.union, data.values()) - set(data.keys()) # Add empty dependences where needed. - data.update({item: set() for item in extra_items_in_deps}) + data.update(dict((item, set()) for item in extra_items_in_deps)) while True: ordered = set(item for item, dep in data.items() if len(dep) == 0) if not ordered: break yield ordered - data = {item: (dep - ordered) - for item, dep in data.items() - if item not in ordered} + data = dict((item, (dep - ordered)) for item, dep in data.items() if item not in ordered) if len(data) != 0: raise ValueError('Cyclic dependencies exist among these items: {}'.format( ', '.join(repr(x) for x in data.items())))