[BRANCH-2.1][BACKPORT] Fixing python 2.6 tests for jenkings

dev/run-tests.py

@@ -121,7 +121,7 @@ def determine_modules_to_test(changed_modules):
     if modules.root in modules_to_test:
         return [modules.root]
     return toposort_flatten(
-        {m: set(m.dependencies).intersection(modules_to_test) for m in modules_to_test}, sort=True)
+        dict((m, set(m.dependencies).intersection(modules_to_test)) for m in modules_to_test), sort=True)
 
 
 def determine_tags_to_exclude(changed_modules):

dev/sparktestsupport/modules.py

@@ -15,10 +15,48 @@
 # limitations under the License.
 #
 
-from functools import total_ordering
+import sys
 import itertools
 import re
 
+if sys.version_info >= (2, 7):
+    from functools import total_ordering
+else:
+    def total_ordering(cls):  # backport python 2.6
+        """Class decorator that fills in missing ordering methods"""
+        convert = {
+            '__lt__': [
+                ('__gt__', lambda self, other: other < self),
+                ('__le__', lambda self, other: not other < self),
+                ('__ge__', lambda self, other: not self < other)
+            ],
+            '__le__': [
+                ('__ge__', lambda self, other: other <= self),
+                ('__lt__', lambda self, other: not other <= self),
+                ('__gt__', lambda self, other: not self <= other)
+            ],
+            '__gt__': [
+                ('__lt__', lambda self, other: other > self),
+                ('__ge__', lambda self, other: not other > self),
+                ('__le__', lambda self, other: not self > other)
+            ],
+            '__ge__': [
+                ('__le__', lambda self, other: other >= self),
+                ('__gt__', lambda self, other: not other >= self),
+                ('__lt__', lambda self, other: not self >= other)
+            ]
+        }
+        roots = set(dir(cls)) & set(convert)
+        if not roots:
+            raise ValueError('must define at least one ordering operation: < > <= >=')
+        root = max(roots)       # prefer __lt__ to __le__ to __gt__ to __ge__
+        for opname, opfunc in convert[root]:
+            if opname not in roots:
+                opfunc.__name__ = opname
+                opfunc.__doc__ = getattr(int, opname).__doc__
+                setattr(cls, opname, opfunc)
+        return cls
+
 all_modules = []
 
 

dev/sparktestsupport/shellutils.py

@@ -43,7 +43,7 @@ def subprocess_check_output(*popenargs, **kwargs):
 
     # backported from subprocess module in Python 2.7
     def subprocess_check_call(*popenargs, **kwargs):
-        retcode = call(*popenargs, **kwargs)
+        retcode = subprocess.call(*popenargs, **kwargs)
         if retcode:
             cmd = kwargs.get("args")
             if cmd is None:

dev/sparktestsupport/toposort.py

@@ -60,15 +60,13 @@ def toposort(data):
     # Find all items that don't depend on anything.
     extra_items_in_deps = _reduce(set.union, data.values()) - set(data.keys())
     # Add empty dependences where needed.
-    data.update({item: set() for item in extra_items_in_deps})
+    data.update(dict((item, set()) for item in extra_items_in_deps))
     while True:
         ordered = set(item for item, dep in data.items() if len(dep) == 0)
         if not ordered:
             break
         yield ordered
-        data = {item: (dep - ordered)
-                for item, dep in data.items()
-                if item not in ordered}
+        data = dict((item, (dep - ordered)) for item, dep in data.items() if item not in ordered)
     if len(data) != 0:
         raise ValueError('Cyclic dependencies exist among these items: {}'.format(
             ', '.join(repr(x) for x in data.items())))