diff --git a/dev/run-tests.py b/dev/run-tests.py index 5211aea03ea49..592fbee823eea 100755 --- a/dev/run-tests.py +++ b/dev/run-tests.py @@ -112,6 +112,7 @@ def determine_modules_to_test(changed_modules): ['graphx', 'examples'] >>> x = [x.name for x in determine_modules_to_test([modules.sql])] >>> x # doctest: +NORMALIZE_WHITESPACE + ... # doctest: +SKIP ['sql', 'avro', 'hive', 'mllib', 'sql-kafka-0-10', 'examples', 'hive-thriftserver', 'pyspark-sql', 'repl', 'sparkr', 'pyspark-mllib', 'pyspark-ml'] """ @@ -122,9 +123,15 @@ def determine_modules_to_test(changed_modules): # If we need to run all of the tests, then we should short-circuit and return 'root' if modules.root in modules_to_test: return [modules.root] - return toposort_flatten( + changed_modules = toposort_flatten( {m: set(m.dependencies).intersection(modules_to_test) for m in modules_to_test}, sort=True) + # TODO: Skip hive-thriftserver module for hadoop-3.2. remove this once hadoop-3.2 support it + if modules.hadoop_version == "hadoop3.2": + changed_modules = [m for m in changed_modules if m.name != "hive-thriftserver"] + + return changed_modules + def determine_tags_to_exclude(changed_modules): tags = [] diff --git a/dev/sparktestsupport/modules.py b/dev/sparktestsupport/modules.py index 812c2efae9932..7fe3b1be75d93 100644 --- a/dev/sparktestsupport/modules.py +++ b/dev/sparktestsupport/modules.py @@ -21,12 +21,6 @@ import re import os -if os.environ.get("AMPLAB_JENKINS"): - hadoop_version = os.environ.get("AMPLAB_JENKINS_BUILD_PROFILE", "hadoop2.7") -else: - hadoop_version = os.environ.get("HADOOP_PROFILE", "hadoop2.7") -print("[info] Choosing supported modules with Hadoop profile", hadoop_version) - all_modules = [] @@ -80,11 +74,7 @@ def __init__(self, name, dependencies, source_file_regexes, build_profile_flags= self.dependent_modules = set() for dep in dependencies: dep.dependent_modules.add(self) - # TODO: Skip hive-thriftserver module for hadoop-3.2. remove this once hadoop-3.2 support it - if name == "hive-thriftserver" and hadoop_version == "hadoop3.2": - print("[info] Skip unsupported module:", name) - else: - all_modules.append(self) + all_modules.append(self) def contains_file(self, filename): return any(re.match(p, filename) for p in self.source_file_prefixes) @@ -568,6 +558,15 @@ def __hash__(self): ] ) +# TODO: Skip hive-thriftserver module for hadoop-3.2. remove this once hadoop-3.2 support it +if os.environ.get("AMPLAB_JENKINS"): + hadoop_version = os.environ.get("AMPLAB_JENKINS_BUILD_PROFILE", "hadoop2.7") +else: + hadoop_version = os.environ.get("HADOOP_PROFILE", "hadoop2.7") +if hadoop_version == "hadoop3.2": + print("[info] Skip unsupported module:", "hive-thriftserver") + all_modules = [m for m in all_modules if m.name != "hive-thriftserver"] + # The root module is a dummy module which is used to run all of the tests. # No other modules should directly depend on this module. root = Module(