update Clang easyblock to add support for new directory structure in Clang versions >= 14 + support Flang + fix --module-only + fix sanity check for Clang 11.x

easybuild/easyblocks/c/clang.py

@@ -32,6 +32,7 @@
 @author: Dmitri Gribenko (National Technical University of Ukraine "KPI")
 @author: Ward Poelmans (Ghent University)
 @author: Alan O'Cais (Juelich Supercomputing Centre)
+@author: Maxime Boissonneault (Digital Research Alliance of Canada, Universite Laval)
 """
 
 import glob
@@ -86,6 +87,7 @@ def extra_options():
             'bootstrap': [True, "Bootstrap Clang using GCC", CUSTOM],
             'build_extra_clang_tools': [False, "Build extra Clang tools", CUSTOM],
             'build_lld': [False, "Build the LLVM lld linker", CUSTOM],
+            'build_lldb': [False, "Build the LLVM lldb debugger", CUSTOM],
             'build_targets': [None, "Build targets for LLVM (host architecture if None). Possible values: " +
                                     ', '.join(CLANG_TARGETS), CUSTOM],
             'default_cuda_capability': [None, "Default CUDA capability specified for clang, e.g. '7.5'", CUSTOM],
@@ -98,6 +100,7 @@ def extra_options():
             # The sanitizer tests often fail on HPC systems due to the 'weird' environment.
             'skip_sanitizer_tests': [True, "Do not run the sanitizer tests", CUSTOM],
             'usepolly': [False, "Build Clang with polly", CUSTOM],
+            'llvm_projects': [[], "LLVM projects to install", CUSTOM],
         })
         # disable regular out-of-source build, too simplistic for Clang to work
         extra_vars['separate_build_dir'][0] = False
@@ -113,6 +116,31 @@ def __init__(self, *args, **kwargs):
         self.llvm_obj_dir_stage3 = None
         self.make_parallel_opts = ""
 
+        if not self.cfg['llvm_projects']:
+            self.cfg['llvm_projects'] = []
+
+        # keep compatibility between using llvm_projects vs using flags
+        if LooseVersion(self.version) >= LooseVersion('14'):
+            self.cfg.update('llvm_projects', ['llvm', 'compiler-rt', 'clang', 'openmp'], allow_duplicate=False)
+            if self.cfg['usepolly']:
+                self.cfg.update('llvm_projects', 'polly', allow_duplicate=False)
+            if self.cfg['build_lld']:
+                self.cfg.update('llvm_projects', ['lld', 'libunwind'], allow_duplicate=False)
+            if self.cfg['build_lldb']:
+                self.cfg.update('llvm_projects', 'lldb', allow_duplicate=False)
+            if self.cfg['libcxx']:
+                self.cfg.update('llvm_projects', ['libcxx', 'libcxxabi'], allow_duplicate=False)
+            if self.cfg['build_extra_clang_tools']:
+                self.cfg.update('llvm_projects', 'clang-tools-extra', allow_duplicate=False)
+
+        # ensure libunwind is there if lld is there
+        if 'lld' in self.cfg['llvm_projects']:
+            self.cfg.update('llvm_projects', 'libunwind', allow_duplicate=False)
+
+        # ensure libcxxabi is there if libcxx is there
+        if 'libcxx' in self.cfg['llvm_projects']:
+            self.cfg.update('llvm_projects', 'libcxxabi', allow_duplicate=False)
+
     def check_readiness_step(self):
         """Fail early on RHEL 5.x and derivatives because of known bug in libc."""
         super(EB_Clang, self).check_readiness_step()
@@ -123,20 +151,7 @@ def check_readiness_step(self):
 
     def extract_step(self):
         """
-        Prepare a combined LLVM source tree.  The layout is:
-        llvm/             Unpack llvm-*.tar.gz here
-          projects/
-            compiler-rt/  Unpack compiler-rt-*.tar.gz here
-            openmp/       Unpack openmp-*.tar.xz here
-          tools/
-            clang/        Unpack clang-*.tar.gz here
-              tools/
-                extra/    Unpack clang-tools-extra-*.tar.gz here
-            polly/        Unpack polly-*.tar.gz here
-            libcxx/       Unpack libcxx-*.tar.gz here
-            libcxxabi/    Unpack libcxxabi-*.tar.gz here
-            lld/          Unpack lld-*.tar.gz here
-        libunwind/        Unpack libunwind-*.tar.gz here
+        Prepare a combined LLVM source tree.  The layout is different for versions earlier and later than 14.
         """
 
         # Extract everything into separate directories.
@@ -164,27 +179,51 @@ def find_source_dir(globpatterns, targetdir):
                                      glob_src_dirs)
             src_dirs[glob_src_dirs[0]] = targetdir
 
-        find_source_dir('compiler-rt-*', os.path.join(self.llvm_src_dir, 'projects', 'compiler-rt'))
-
-        if self.cfg["usepolly"]:
-            find_source_dir('polly-*', os.path.join(self.llvm_src_dir, 'tools', 'polly'))
-
-        if self.cfg["build_lld"]:
-            find_source_dir('lld-*', os.path.join(self.llvm_src_dir, 'tools', 'lld'))
-            if LooseVersion(self.version) >= LooseVersion('12.0.1'):
-                find_source_dir('libunwind-*', os.path.normpath(os.path.join(self.llvm_src_dir, '..', 'libunwind')))
-
-        if self.cfg["libcxx"]:
-            find_source_dir('libcxx-*', os.path.join(self.llvm_src_dir, 'projects', 'libcxx'))
-            find_source_dir('libcxxabi-*', os.path.join(self.llvm_src_dir, 'projects', 'libcxxabi'))
-
-        find_source_dir(['clang-[1-9]*', 'cfe-*'], os.path.join(self.llvm_src_dir, 'tools', 'clang'))
-
-        if self.cfg["build_extra_clang_tools"]:
-            find_source_dir('clang-tools-extra-*', os.path.join(self.llvm_src_dir, 'tools', 'clang', 'tools', 'extra'))
-
-        if LooseVersion(self.version) >= LooseVersion('3.8'):
-            find_source_dir('openmp-*', os.path.join(self.llvm_src_dir, 'projects', 'openmp'))
+        if any([x['name'].startswith('llvm-project') for x in self.src]):
+            # if sources contain 'llvm-project*', we use the full tarball
+            find_source_dir("../llvm-project-*", os.path.join(self.llvm_src_dir, "llvm-project-%s" % self.version))
+            self.cfg.update('configopts', '-DLLVM_ENABLE_PROJECTS="%s"' % ';'.join(self.cfg['llvm_projects']))
+        else:
+            # Layout for previous versions
+            #        llvm/             Unpack llvm-*.tar.gz here
+            #          projects/
+            #            compiler-rt/  Unpack compiler-rt-*.tar.gz here
+            #            openmp/       Unpack openmp-*.tar.xz here
+            #          tools/
+            #            clang/        Unpack clang-*.tar.gz here
+            #              tools/
+            #                extra/    Unpack clang-tools-extra-*.tar.gz here
+            #            polly/        Unpack polly-*.tar.gz here
+            #            libcxx/       Unpack libcxx-*.tar.gz here
+            #            libcxxabi/    Unpack libcxxabi-*.tar.gz here
+            #            lld/          Unpack lld-*.tar.gz here
+            #            lldb/         Unpack lldb-*.tar.gz here
+            #        libunwind/        Unpack libunwind-*.tar.gz here
+            find_source_dir('compiler-rt-*', os.path.join(self.llvm_src_dir, 'projects', 'compiler-rt'))
+
+            if 'polly' in self.cfg['llvm_projects']:
+                find_source_dir('polly-*', os.path.join(self.llvm_src_dir, 'tools', 'polly'))
+
+            if 'lld' in self.cfg['llvm_projects']:
+                find_source_dir('lld-*', os.path.join(self.llvm_src_dir, 'tools', 'lld'))
+                if LooseVersion(self.version) >= LooseVersion('12.0.1'):
+                    find_source_dir('libunwind-*', os.path.normpath(os.path.join(self.llvm_src_dir, '..', 'libunwind')))
+
+            if 'lldb' in self.cfg['llvm_projects']:
+                find_source_dir('lldb-*', os.path.join(self.llvm_src_dir, 'tools', 'lldb'))
+
+            if 'libcxx' in self.cfg['llvm_projects']:
+                find_source_dir('libcxx-*', os.path.join(self.llvm_src_dir, 'projects', 'libcxx'))
+                find_source_dir('libcxxabi-*', os.path.join(self.llvm_src_dir, 'projects', 'libcxxabi'))
+
+            find_source_dir(['clang-[1-9]*', 'cfe-*'], os.path.join(self.llvm_src_dir, 'tools', 'clang'))
+
+            if 'clang-tools-extra' in self.cfg['llvm_projects']:
+                find_source_dir('clang-tools-extra-*',
+                                os.path.join(self.llvm_src_dir, 'tools', 'clang', 'tools', 'extra'))
+
+            if LooseVersion(self.version) >= LooseVersion('3.8'):
+                find_source_dir('openmp-*', os.path.join(self.llvm_src_dir, 'projects', 'openmp'))
 
         for src in self.src:
             for (dirname, new_path) in src_dirs.items():
@@ -306,7 +345,7 @@ def configure_step(self):
         else:
             self.cfg.update('configopts', "-DLLVM_ENABLE_ASSERTIONS=OFF")
 
-        if self.cfg["usepolly"]:
+        if 'polly' in self.cfg['llvm_projects']:
             self.cfg.update('configopts', "-DLINK_POLLY_INTO_TOOLS=ON")
 
         # If Z3 is included as a dep, enable support in static analyzer (if enabled)
@@ -318,7 +357,7 @@ def configure_step(self):
 
         build_targets = self.cfg['build_targets']
 
-        if self.cfg["usepolly"] and "NVPTX" in build_targets:
+        if 'polly' in self.cfg['llvm_projects'] and "NVPTX" in build_targets:
             self.cfg.update('configopts', "-DPOLLY_ENABLE_GPGPU_CODEGEN=ON")
 
         self.cfg.update('configopts', '-DLLVM_TARGETS_TO_BUILD="%s"' % ';'.join(build_targets))
@@ -368,7 +407,12 @@ def configure_step(self):
             self.cfg.update('configopts', '-DLIBOMPTARGET_AMDGCN_GFXLIST=%s' % ' '.join(ec_amdgfx))
 
         self.log.info("Configuring")
-        super(EB_Clang, self).configure_step(srcdir=self.llvm_src_dir)
+
+        # directory structure has changed in version 14.x, cmake must start in llvm sub directory
+        if LooseVersion(self.version) >= LooseVersion('14'):
+            super(EB_Clang, self).configure_step(srcdir=os.path.join(self.llvm_src_dir, "llvm"))
+        else:
+            super(EB_Clang, self).configure_step(srcdir=self.llvm_src_dir)
 
     def disable_sanitizer_tests(self):
         """Disable the tests of all the sanitizers by removing the test directories from the build system"""
@@ -391,7 +435,10 @@ def disable_sanitizer_tests(self):
         else:
             # In Clang 3.6, the sanitizer tests are grouped together in one CMakeLists
             # We patch out adding the subdirectories with the sanitizer tests
-            cmakelists_tests = os.path.join(self.llvm_src_dir, 'projects', 'compiler-rt', 'test', 'CMakeLists.txt')
+            if LooseVersion(self.version) >= LooseVersion('14'):
+                cmakelists_tests = os.path.join(self.llvm_src_dir, 'compiler-rt', 'test', 'CMakeLists.txt')
+            else:
+                cmakelists_tests = os.path.join(self.llvm_src_dir, 'projects', 'compiler-rt', 'test', 'CMakeLists.txt')
             regex_subs = []
             if LooseVersion(self.version) >= LooseVersion('5.0'):
                 regex_subs.append((r'compiler_rt_test_runtime.*san.*', ''))
@@ -418,7 +465,10 @@ def build_with_prev_stage(self, prev_obj, next_obj):
         options += "-DCMAKE_BUILD_TYPE=%s" % self.build_type
 
         self.log.info("Configuring")
-        run_cmd("cmake %s %s" % (options, self.llvm_src_dir), log_all=True)
+        if LooseVersion(self.version) >= LooseVersion('14'):
+            run_cmd("cmake %s %s" % (options, os.path.join(self.llvm_src_dir, "llvm")), log_all=True)
+        else:
+            run_cmd("cmake %s %s" % (options, self.llvm_src_dir), log_all=True)
 
         self.log.info("Building")
         run_cmd("make %s" % self.make_parallel_opts, log_all=True)
@@ -493,7 +543,10 @@ def post_install_step(self):
 
         # copy Python bindings here in post-install step so that it is not done more than once in multi_deps context
         if self.cfg['python_bindings']:
-            python_bindings_source_dir = os.path.join(self.llvm_src_dir, "tools", "clang", "bindings", "python")
+            if LooseVersion(self.version) >= LooseVersion('14'):
+                python_bindings_source_dir = os.path.join(self.llvm_src_dir, "clang", "bindings", "python")
+            else:
+                python_bindings_source_dir = os.path.join(self.llvm_src_dir, "tools", "clang", "bindings", "python")
             python_bindins_target_dir = os.path.join(self.installdir, 'lib', 'python')
 
             shutil.copytree(python_bindings_source_dir, python_bindins_target_dir)
@@ -512,17 +565,20 @@ def sanity_check_step(self):
         if self.cfg['static_analyzer']:
             custom_paths['files'].extend(["bin/scan-build", "bin/scan-view"])
 
-        if self.cfg['build_extra_clang_tools'] and LooseVersion(self.version) >= LooseVersion('3.4'):
+        if 'clang-tools-extra' in self.cfg['llvm_projects'] and LooseVersion(self.version) >= LooseVersion('3.4'):
             custom_paths['files'].extend(["bin/clang-tidy"])
 
-        if self.cfg["usepolly"]:
+        if 'polly' in self.cfg['llvm_projects']:
             custom_paths['files'].extend(["lib/LLVMPolly.%s" % shlib_ext])
             custom_paths['dirs'].extend(["include/polly"])
 
-        if self.cfg["build_lld"]:
+        if 'lld' in self.cfg['llvm_projects']:
             custom_paths['files'].extend(["bin/lld"])
 
-        if self.cfg["libcxx"]:
+        if 'lldb' in self.cfg['llvm_projects']:
+            custom_paths['files'].extend(["bin/lldb"])
+
+        if 'libcxx' in self.cfg['llvm_projects']:
             custom_paths['files'].extend(["lib/libc++.%s" % shlib_ext])
             custom_paths['files'].extend(["lib/libc++abi.%s" % shlib_ext])