honor --module-search-path-headers in easyblocks that hardcode CPATH

easybuild/easyblocks/f/flexiblas.py

@@ -86,7 +86,7 @@ def __init__(self, *args, **kwargs):
         self.obj_builddir = os.path.join(self.builddir, 'easybuild_obj')
 
         # custom CPATH to FlexiBLAS headers
-        self.module_load_environment.CPATH = [os.path.join('include', 'flexiblas')]
+        self.module_load_environment.set_alias_vars('HEADERS', [os.path.join('include', 'flexiblas')])
 
     def configure_step(self):
         """Custom configuration for FlexiBLAS, based on which BLAS libraries are included as dependencies."""

easybuild/easyblocks/g/gcc.py

@@ -203,12 +203,13 @@ def __init__(self, *args, **kwargs):
             self.log.warning('Setting withnvptx to False, since we are building on a RISC-V system')
             self.cfg['withnvptx'] = False
 
-        # define list of subdirectories in search paths of module load environment
-        self.module_load_environment.PATH = ['bin']
-        self.module_load_environment.LD_LIBRARY_PATH = ['lib', 'lib64']
-        # GCC can find its own headers and libraries but the .so's need to be in LD_LIBRARY_PATH
-        self.module_load_environment.CPATH = []
-        self.module_load_environment.LIBRARY_PATH = ['lib', 'lib64'] if get_cpu_family() == RISCV else []
+        # GCC can find its own headers and libraries in most cases, but we had
+        # cases where paths top libraries needed to be set explicitly
+        # see: https://github.com/easybuilders/easybuild-easyblocks/pull/3256
+        # Therefore, remove paths from header search paths but keep paths in LIBRARY_PATH
+        for disallowed_var in self.module_load_environment.alias_vars('HEADERS'):
+            self.module_load_environment.remove(disallowed_var)
+            self.log.debug(f"Purposely not updating ${disallowed_var} in {self.name} module file")
 
     def create_dir(self, dirname):
         """

easybuild/easyblocks/generic/intelbase.py

@@ -137,13 +137,11 @@ def prepare_intel_tools_env(self):
 
         # make sure $CPATH, $LD_LIBRARY_PATH and $LIBRARY_PATH are not updated in generated module file,
         # because that leads to problem when the libraries included with VTune/Advisor/Inspector are being picked up
-        for disallowed_var in ['CPATH', 'LD_LIBRARY_PATH', 'LIBRARY_PATH']:
-            try:
-                delattr(self.module_load_environment, disallowed_var)
-            except AttributeError:
-                pass
-            else:
-                self.log.debug(f"Purposely not updating ${disallowed_var} in {self.name} module file")
+        mod_env_headers = self.module_load_environment.alias_vars('HEADERS')
+        mod_env_libs = ['LD_LIBRARY_PATH', 'LIBRARY_PATH']
+        for disallowed_var in mod_env_headers + mod_env_libs:
+            self.module_load_environment.remove(disallowed_var)
+            self.log.debug(f"Purposely not updating ${disallowed_var} in {self.name} module file")
 
     def get_custom_paths_tools(self, binaries):
         """Custom sanity check paths for certain Intel tools."""

easybuild/easyblocks/i/icc.py

@@ -100,42 +100,48 @@ def __init__(self, *args, **kwargs):
         # in deeper directories, and symlinked in top-level directories
         # however, not all binaries are symlinked (e.g. mcpcom is not)
         # we only need to include the deeper directories (same as compilervars.sh)
-
-        self.module_load_environment.PATH = [os.path.join(self.comp_libs_subdir, path) for path in (
+        def comp_libs_subdir_paths(*subdir_paths):
+            """Utility method to prepend self.comp_libs_subdir to list of paths"""
+            try:
+                return [os.path.join(self.comp_libs_subdir, path) for path in subdir_paths]
+            except TypeError:
+                raise EasyBuildError(f"Cannot prepend {self.comp_libs_subdir} to {subdir_paths}: wrong data type")
+
+        self.module_load_environment.PATH = comp_libs_subdir_paths(
             'bin/intel64',
             'ipp/bin/intel64',
             'mpi/intel64/bin',
             'tbb/bin/emt64',
             'tbb/bin/intel64',
-        )]
+        )
         # in the end we set 'LIBRARY_PATH' equal to 'LD_LIBRARY_PATH'
-        self.module_load_environment.LD_LIBRARY_PATH = [os.path.join(self.comp_libs_subdir, path) for path in (
+        self.module_load_environment.LD_LIBRARY_PATH = comp_libs_subdir_paths(
             'lib',
             'compiler/lib/intel64',
             'debugger/ipt/intel64/lib',
             'ipp/lib/intel64',
             'mkl/lib/intel64',
             'mpi/intel64',
             f"tbb/lib/intel64/{get_tbb_gccprefix(os.path.join(self.installdir, 'tbb/lib/intel64'))}",
-        )]
+        )
         # 'include' is deliberately omitted, including it causes problems, e.g. with complex.h and std::complex
         # cfr. https://software.intel.com/en-us/forums/intel-c-compiler/topic/338378
-        self.module_load_environment.CPATH = [os.path.join(self.comp_libs_subdir, path) for path in (
+        self.module_load_environment.set_alias_vars('HEADERS', comp_libs_subdir_paths(
             'daal/include',
             'ipp/include',
             'mkl/include',
             'tbb/include',
-        )]
-        self.module_load_environment.CLASSPATH = [os.path.join(self.comp_libs_subdir, 'daal/lib/daal.jar')]
-        self.module_load_environment.DAALROOT = [os.path.join(self.comp_libs_subdir, 'daal')]
-        self.module_load_environment.IPPROOT = [os.path.join(self.comp_libs_subdir, 'ipp')]
-        self.module_load_environment.MANPATH = [os.path.join(self.comp_libs_subdir, path) for path in (
+        ))
+        self.module_load_environment.CLASSPATH = comp_libs_subdir_paths('daal/lib/daal.jar')
+        self.module_load_environment.DAALROOT = comp_libs_subdir_paths('daal')
+        self.module_load_environment.IPPROOT = comp_libs_subdir_paths('ipp')
+        self.module_load_environment.MANPATH = comp_libs_subdir_paths(
             'debugger/gdb/intel64/share/man',
             'man/common',
             'man/en_US',
             'share/man',
-        )]
-        self.module_load_environment.TBBROOT = [os.path.join(self.comp_libs_subdir, 'tbb')]
+        )
+        self.module_load_environment.TBBROOT = comp_libs_subdir_paths('tbb')
 
         # Debugger requires INTEL_PYTHONHOME, which only allows for a single value
         self.debuggerpath = f"debugger_{self.version.split('.')[0]}"
@@ -148,7 +154,7 @@ def __init__(self, *args, **kwargs):
         )
 
         # 'lib/intel64' is deliberately listed last, so it gets precedence over subdirs
-        self.module_load_environment.LD_LIBRARY_PATH.append(os.path.join(self.comp_libs_subdir, 'lib/intel64'))
+        self.module_load_environment.LD_LIBRARY_PATH.extend(comp_libs_subdir_paths('lib/intel64'))
 
         self.module_load_environment.LIBRARY_PATH = self.module_load_environment.LD_LIBRARY_PATH
 
@@ -198,12 +204,13 @@ def make_module_extra(self, *args, **kwargs):
             txt += self.module_generator.set_environment('INTEL_PYTHONHOME', intel_pythonhome)
 
         # on Debian/Ubuntu, /usr/include/x86_64-linux-gnu needs to be included in $CPATH for icc
-        res = run_shell_cmd("gcc -print-multiarch", fail_on_error=False)
+        res = run_shell_cmd("gcc -print-multiarch", fail_on_error=False, hidden=True)
         multiarch_inc_subdir = res.output.strip()
         if res.exit_code == 0 and multiarch_inc_subdir:
             multiarch_inc_dir = os.path.join('/usr', 'include', multiarch_inc_subdir)
-            self.log.info("Adding multiarch include path %s to $CPATH in generated module file", multiarch_inc_dir)
-            # system location must be appended at the end, so use append_paths
-            txt += self.module_generator.append_paths('CPATH', [multiarch_inc_dir], allow_abs=True)
+            for envar in self.module_load_environment.alias_vars('HEADERS'):
+                self.log.info(f"Adding multiarch include path '{multiarch_inc_dir}' to ${envar} in generated module")
+                # system location must be appended at the end, so use append_paths
+                txt += self.module_generator.append_paths(envar, [multiarch_inc_dir], allow_abs=True)
 
         return txt

easybuild/easyblocks/i/iccifort.py

@@ -47,7 +47,8 @@ def __init__(self, *args, **kwargs):
 
         # Exclude 'compiler/include' for CPATH, including it causes problems, e.g. with complex.h and std::complex
         # cfr. https://software.intel.com/en-us/forums/intel-c-compiler/topic/338378
-        self.module_load_environment.CPATH.remove(os.path.join(self.comp_libs_subdir, 'compiler/include'))
+        for envar in self.module_load_environment.alias('HEADERS'):
+            envar.remove(os.path.join(self.comp_libs_subdir, 'compiler/include'))
 
         # remove entries from LIBRARY_PATH that icc and co already know about at compile time
         # only do this for iccifort merged installations so that icc can still find ifort

easybuild/easyblocks/i/ifort.py

@@ -60,7 +60,8 @@ def __init__(self, *args, **kwargs):
 
         # define list of subdirectories in search paths of module load environment
         # add additional paths to those of ICC only needed for separate ifort installations
-        self.module_load_environment.CPATH.append(os.path.join(self.comp_libs_subdir, 'compiler/include'))
+        for envar in self.module_load_environment.alias('HEADERS'):
+            envar.append(os.path.join(self.comp_libs_subdir, 'compiler/include'))
 
     def sanity_check_step(self):
         """Custom sanity check paths for ifort."""

easybuild/easyblocks/i/imkl.py

@@ -454,12 +454,15 @@ def make_module_step(self, *args, **kwargs):
             os.path.join(self.mkl_basedir, 'lib', 'intel64'),
         ]
         self.module_load_environment.LIBRARY_PATH = self.module_load_environment.LD_LIBRARY_PATH
-        self.module_load_environment.CPATH = [
+        self.module_load_environment.CMAKE_PREFIX_PATH = [self.mkl_basedir]
+        self.module_load_environment.PKG_CONFIG_PATH = pkg_config_path
+
+        # include paths to headers (e.g. CPATH)
+        include_dirs = [
             os.path.join(self.mkl_basedir, 'include'),
             os.path.join(self.mkl_basedir, 'include', 'fftw'),
         ]
-        self.module_load_environment.CMAKE_PREFIX_PATH = [self.mkl_basedir]
-        self.module_load_environment.PKG_CONFIG_PATH = pkg_config_path
+        self.module_load_environment.set_alias_vars('HEADERS', include_dirs)
 
         if LooseVersion(self.version) < LooseVersion('2021'):
             self.module_load_environment.MANPATH = ['man', os.path.join('man', 'en_US')]

easybuild/easyblocks/i/impi.py

@@ -284,13 +284,15 @@ def make_module_step(self, *args, **kwargs):
         self.module_load_environment.PATH = path_dirs
         self.module_load_environment.LD_LIBRARY_PATH = lib_dirs
         self.module_load_environment.LIBRARY_PATH = lib_dirs
-        self.module_load_environment.CPATH = include_dirs
         self.module_load_environment.MANPATH = [manpath]
         if fi_provider_path is not None:
             self.module_load_environment.FI_PROVIDER_PATH = fi_provider_path
         if mic_library_path is not None:
             self.module_load_environment.MIC_LD_LIBRARY_PATH = mic_library_path
 
+        # include paths to headers (e.g. CPATH)
+        self.module_load_environment.set_alias_vars('HEADERS', include_dirs)
+
         return super().make_module_step(*args, **kwargs)
 
     def make_module_extra(self, *args, **kwargs):

easybuild/easyblocks/i/intel_compilers.py

@@ -152,7 +152,6 @@ def make_module_step(self, *args, **kwargs):
             os.path.join(tbb_lib_prefix, tbb_lib_gccdir),
         ]
         self.module_load_environment.LIBRARY_PATH = self.module_load_environment.LD_LIBRARY_PATH
-        self.module_load_environment.CPATH = [os.path.join(self.tbb_subdir, 'include')]
         self.module_load_environment.MANPATH = [
             os.path.join(os.path.dirname(self.compilers_subdir), 'documentation', 'en', 'man', 'common'),
             os.path.join(self.compilers_subdir, 'share', 'man'),
@@ -163,6 +162,9 @@ def make_module_step(self, *args, **kwargs):
         )]
         self.module_load_environment.TBBROOT = [self.tbb_subdir]
 
+        # include paths to headers (e.g. CPATH)
+        self.module_load_environment.set_alias_vars('HEADERS', os.path.join(self.tbb_subdir, 'include'))
+
         return super().make_module_step(*args, **kwargs)
 
     def make_module_extra(self):
@@ -171,19 +173,23 @@ def make_module_extra(self):
 
         # On Debian/Ubuntu, /usr/include/x86_64-linux-gnu, or whatever dir gcc uses, needs to be included
         # in $CPATH for Intel C compiler
-        res = run_shell_cmd("gcc -print-multiarch")
+        res = run_shell_cmd("gcc -print-multiarch", hidden=True)
         multiarch_out = res.output.strip()
         if res.exit_code == 0 and multiarch_out:
             multi_arch_inc_dir_cmd = '|'.join([
                 "gcc -E -Wp,-v -xc /dev/null 2>&1",
-                "grep %s$" % multiarch_out,
+                f"grep {multiarch_out}$",
                 "grep -v /include-fixed/",
             ])
-            res = run_shell_cmd(multi_arch_inc_dir_cmd)
+            res = run_shell_cmd(multi_arch_inc_dir_cmd, hidden=True)
             multiarch_inc_dir = res.output.strip()
             if res.exit_code == 0 and multiarch_inc_dir:
-                self.log.info("Adding multiarch include path %s to $CPATH in generated module file", multiarch_inc_dir)
                 # system location must be appended at the end, so use append_paths
-                txt += self.module_generator.append_paths('CPATH', [multiarch_inc_dir], allow_abs=True)
+                for envar in self.module_load_environment.alias_vars('HEADERS'):
+                    self.log.info(
+                        f"Adding multiarch include path '{multiarch_inc_dir}' to ${envar} in generated module file"
+                    )
+                    # system location must be appended at the end, so use append_paths
+                    txt += self.module_generator.append_paths(envar, [multiarch_inc_dir], allow_abs=True)
 
         return txt