From 4ff134b937094b237836142caab7b351235de480 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Sun, 9 Feb 2025 02:04:28 +0100 Subject: [PATCH 01/13] honor --module-search-path-headers in flexiblas easyblock --- easybuild/easyblocks/f/flexiblas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/f/flexiblas.py b/easybuild/easyblocks/f/flexiblas.py index 5ead9049b64..68af9390885 100644 --- a/easybuild/easyblocks/f/flexiblas.py +++ b/easybuild/easyblocks/f/flexiblas.py @@ -86,7 +86,7 @@ def __init__(self, *args, **kwargs): self.obj_builddir = os.path.join(self.builddir, 'easybuild_obj') # custom CPATH to FlexiBLAS headers - self.module_load_environment.CPATH = [os.path.join('include', 'flexiblas')] + self.module_load_environment.set_alias_vars('HEADERS', [os.path.join('include', 'flexiblas')]) def configure_step(self): """Custom configuration for FlexiBLAS, based on which BLAS libraries are included as dependencies.""" From e53a5cf651605f911e9dfb47a7efd53291f6b697 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Mon, 10 Feb 2025 18:30:27 +0100 Subject: [PATCH 02/13] add comp_libs_subdir_paths method to icc easyblock --- easybuild/easyblocks/i/icc.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/easybuild/easyblocks/i/icc.py b/easybuild/easyblocks/i/icc.py index 3ffc36e3d5c..c67ea15b4ab 100644 --- a/easybuild/easyblocks/i/icc.py +++ b/easybuild/easyblocks/i/icc.py @@ -100,6 +100,12 @@ def __init__(self, *args, **kwargs): # in deeper directories, and symlinked in top-level directories # however, not all binaries are symlinked (e.g. mcpcom is not) # we only need to include the deeper directories (same as compilervars.sh) + def comp_libs_subdir_paths(*subdir_paths): + """Utility method to prepend self.comp_libs_subdir to list of paths""" + try: + return [os.path.join(self.comp_libs_subdir, path) for path in subdir_paths] + except TypeError: + raise EasyBuildError(f"Cannot prepend {self.comp_libs_subdir} to {subdir_paths}: wrong data type") self.module_load_environment.PATH = [os.path.join(self.comp_libs_subdir, path) for path in ( 'bin/intel64', From 9634a62b3b993c590db6e5d2bf9ec49f96b6ba62 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Mon, 10 Feb 2025 18:31:17 +0100 Subject: [PATCH 03/13] set paths in module load environment through comp_libs_subdir_paths and be aware of module-search-path-headers --- easybuild/easyblocks/i/icc.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/easybuild/easyblocks/i/icc.py b/easybuild/easyblocks/i/icc.py index c67ea15b4ab..774a582f3d2 100644 --- a/easybuild/easyblocks/i/icc.py +++ b/easybuild/easyblocks/i/icc.py @@ -107,15 +107,15 @@ def comp_libs_subdir_paths(*subdir_paths): except TypeError: raise EasyBuildError(f"Cannot prepend {self.comp_libs_subdir} to {subdir_paths}: wrong data type") - self.module_load_environment.PATH = [os.path.join(self.comp_libs_subdir, path) for path in ( + self.module_load_environment.PATH = comp_libs_subdir_paths( 'bin/intel64', 'ipp/bin/intel64', 'mpi/intel64/bin', 'tbb/bin/emt64', 'tbb/bin/intel64', - )] + ) # in the end we set 'LIBRARY_PATH' equal to 'LD_LIBRARY_PATH' - self.module_load_environment.LD_LIBRARY_PATH = [os.path.join(self.comp_libs_subdir, path) for path in ( + self.module_load_environment.LD_LIBRARY_PATH = comp_libs_subdir_paths( 'lib', 'compiler/lib/intel64', 'debugger/ipt/intel64/lib', @@ -123,25 +123,25 @@ def comp_libs_subdir_paths(*subdir_paths): 'mkl/lib/intel64', 'mpi/intel64', f"tbb/lib/intel64/{get_tbb_gccprefix(os.path.join(self.installdir, 'tbb/lib/intel64'))}", - )] + ) # 'include' is deliberately omitted, including it causes problems, e.g. with complex.h and std::complex # cfr. https://software.intel.com/en-us/forums/intel-c-compiler/topic/338378 - self.module_load_environment.CPATH = [os.path.join(self.comp_libs_subdir, path) for path in ( + self.module_load_environment.set_alias_vars('HEADERS', comp_libs_subdir_paths( 'daal/include', 'ipp/include', 'mkl/include', 'tbb/include', - )] - self.module_load_environment.CLASSPATH = [os.path.join(self.comp_libs_subdir, 'daal/lib/daal.jar')] - self.module_load_environment.DAALROOT = [os.path.join(self.comp_libs_subdir, 'daal')] - self.module_load_environment.IPPROOT = [os.path.join(self.comp_libs_subdir, 'ipp')] - self.module_load_environment.MANPATH = [os.path.join(self.comp_libs_subdir, path) for path in ( + )) + self.module_load_environment.CLASSPATH = comp_libs_subdir_paths('daal/lib/daal.jar') + self.module_load_environment.DAALROOT = comp_libs_subdir_paths('daal') + self.module_load_environment.IPPROOT = comp_libs_subdir_paths('ipp') + self.module_load_environment.MANPATH = comp_libs_subdir_paths( 'debugger/gdb/intel64/share/man', 'man/common', 'man/en_US', 'share/man', - )] - self.module_load_environment.TBBROOT = [os.path.join(self.comp_libs_subdir, 'tbb')] + ) + self.module_load_environment.TBBROOT = comp_libs_subdir_paths('tbb') # Debugger requires INTEL_PYTHONHOME, which only allows for a single value self.debuggerpath = f"debugger_{self.version.split('.')[0]}" @@ -154,7 +154,7 @@ def comp_libs_subdir_paths(*subdir_paths): ) # 'lib/intel64' is deliberately listed last, so it gets precedence over subdirs - self.module_load_environment.LD_LIBRARY_PATH.append(os.path.join(self.comp_libs_subdir, 'lib/intel64')) + self.module_load_environment.LD_LIBRARY_PATH.extend(comp_libs_subdir_paths('lib/intel64')) self.module_load_environment.LIBRARY_PATH = self.module_load_environment.LD_LIBRARY_PATH From cf5bbcb78588680f7e91c749a5c925191a03382f Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Mon, 10 Feb 2025 18:56:13 +0100 Subject: [PATCH 04/13] append multiarch include in icc easyblock to header environment variables set by module-search-paths-headers --- easybuild/easyblocks/i/icc.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/easybuild/easyblocks/i/icc.py b/easybuild/easyblocks/i/icc.py index 774a582f3d2..fe6d7ac683c 100644 --- a/easybuild/easyblocks/i/icc.py +++ b/easybuild/easyblocks/i/icc.py @@ -204,12 +204,13 @@ def make_module_extra(self, *args, **kwargs): txt += self.module_generator.set_environment('INTEL_PYTHONHOME', intel_pythonhome) # on Debian/Ubuntu, /usr/include/x86_64-linux-gnu needs to be included in $CPATH for icc - res = run_shell_cmd("gcc -print-multiarch", fail_on_error=False) + res = run_shell_cmd("gcc -print-multiarch", fail_on_error=False, hidden=True) multiarch_inc_subdir = res.output.strip() if res.exit_code == 0 and multiarch_inc_subdir: multiarch_inc_dir = os.path.join('/usr', 'include', multiarch_inc_subdir) - self.log.info("Adding multiarch include path %s to $CPATH in generated module file", multiarch_inc_dir) - # system location must be appended at the end, so use append_paths - txt += self.module_generator.append_paths('CPATH', [multiarch_inc_dir], allow_abs=True) + for envar in self.module_load_environment.alias_vars('HEADERS'): + self.log.info(f"Adding multiarch include path '{multiarch_inc_dir}' to ${envar} in generated module") + # system location must be appended at the end, so use append_paths + txt += self.module_generator.append_paths(envar, [multiarch_inc_dir], allow_abs=True) return txt From a3c4282964682cf89889a3a3097cd02f4fb6be6b Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Mon, 10 Feb 2025 18:57:58 +0100 Subject: [PATCH 05/13] make ifort easyblock aware of module-search-path-headers --- easybuild/easyblocks/i/ifort.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/easyblocks/i/ifort.py b/easybuild/easyblocks/i/ifort.py index 87c986d5dd6..6185f21ec16 100644 --- a/easybuild/easyblocks/i/ifort.py +++ b/easybuild/easyblocks/i/ifort.py @@ -60,7 +60,8 @@ def __init__(self, *args, **kwargs): # define list of subdirectories in search paths of module load environment # add additional paths to those of ICC only needed for separate ifort installations - self.module_load_environment.CPATH.append(os.path.join(self.comp_libs_subdir, 'compiler/include')) + for envar in self.module_load_environment.alias('HEADERS'): + envar.append(os.path.join(self.comp_libs_subdir, 'compiler/include')) def sanity_check_step(self): """Custom sanity check paths for ifort.""" From 96bbc19cdfdaf0e4de40b4eb4f529b8fe936fff7 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Mon, 10 Feb 2025 18:58:07 +0100 Subject: [PATCH 06/13] make iccifort easyblock aware of module-search-path-headers --- easybuild/easyblocks/i/iccifort.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/easyblocks/i/iccifort.py b/easybuild/easyblocks/i/iccifort.py index f0ebd432eff..0d49e24b4e9 100644 --- a/easybuild/easyblocks/i/iccifort.py +++ b/easybuild/easyblocks/i/iccifort.py @@ -47,7 +47,8 @@ def __init__(self, *args, **kwargs): # Exclude 'compiler/include' for CPATH, including it causes problems, e.g. with complex.h and std::complex # cfr. https://software.intel.com/en-us/forums/intel-c-compiler/topic/338378 - self.module_load_environment.CPATH.remove(os.path.join(self.comp_libs_subdir, 'compiler/include')) + for envar in self.module_load_environment.alias('HEADERS'): + envar.remove(os.path.join(self.comp_libs_subdir, 'compiler/include')) # remove entries from LIBRARY_PATH that icc and co already know about at compile time # only do this for iccifort merged installations so that icc can still find ifort From 81bfada5ca8e7963443694c5fd755b94be90a578 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Mon, 10 Feb 2025 19:18:59 +0100 Subject: [PATCH 07/13] make intel_compilers easyblock aware of module-search-path-headers --- easybuild/easyblocks/i/intel_compilers.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/easybuild/easyblocks/i/intel_compilers.py b/easybuild/easyblocks/i/intel_compilers.py index 267d5174093..3cad5b81a92 100644 --- a/easybuild/easyblocks/i/intel_compilers.py +++ b/easybuild/easyblocks/i/intel_compilers.py @@ -152,7 +152,6 @@ def make_module_step(self, *args, **kwargs): os.path.join(tbb_lib_prefix, tbb_lib_gccdir), ] self.module_load_environment.LIBRARY_PATH = self.module_load_environment.LD_LIBRARY_PATH - self.module_load_environment.CPATH = [os.path.join(self.tbb_subdir, 'include')] self.module_load_environment.MANPATH = [ os.path.join(os.path.dirname(self.compilers_subdir), 'documentation', 'en', 'man', 'common'), os.path.join(self.compilers_subdir, 'share', 'man'), @@ -163,6 +162,11 @@ def make_module_step(self, *args, **kwargs): )] self.module_load_environment.TBBROOT = [self.tbb_subdir] + # include paths to headers (e.g. CPATH) + self.module_load_environment.set_alias_vars('HEADERS', + os.path.join(self.tbb_subdir, 'include') + ) + return super().make_module_step(*args, **kwargs) def make_module_extra(self): @@ -171,19 +175,23 @@ def make_module_extra(self): # On Debian/Ubuntu, /usr/include/x86_64-linux-gnu, or whatever dir gcc uses, needs to be included # in $CPATH for Intel C compiler - res = run_shell_cmd("gcc -print-multiarch") + res = run_shell_cmd("gcc -print-multiarch", hidden=True) multiarch_out = res.output.strip() if res.exit_code == 0 and multiarch_out: multi_arch_inc_dir_cmd = '|'.join([ "gcc -E -Wp,-v -xc /dev/null 2>&1", - "grep %s$" % multiarch_out, + f"grep {multiarch_out}$" , "grep -v /include-fixed/", ]) - res = run_shell_cmd(multi_arch_inc_dir_cmd) + res = run_shell_cmd(multi_arch_inc_dir_cmd, hidden=True) multiarch_inc_dir = res.output.strip() if res.exit_code == 0 and multiarch_inc_dir: - self.log.info("Adding multiarch include path %s to $CPATH in generated module file", multiarch_inc_dir) # system location must be appended at the end, so use append_paths - txt += self.module_generator.append_paths('CPATH', [multiarch_inc_dir], allow_abs=True) + for envar in self.module_load_environment.alias_vars('HEADERS'): + self.log.info( + f"Adding multiarch include path '{multiarch_inc_dir}' to ${envar} in generated module file" + ) + # system location must be appended at the end, so use append_paths + txt += self.module_generator.append_paths(envar, [multiarch_inc_dir], allow_abs=True) return txt From 033962363fab7d1dfec717f21eb13df43dd311d7 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Mon, 10 Feb 2025 19:23:13 +0100 Subject: [PATCH 08/13] fix codestyle in intel_compilers easyblock --- easybuild/easyblocks/i/intel_compilers.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/easybuild/easyblocks/i/intel_compilers.py b/easybuild/easyblocks/i/intel_compilers.py index 3cad5b81a92..55567dc0c60 100644 --- a/easybuild/easyblocks/i/intel_compilers.py +++ b/easybuild/easyblocks/i/intel_compilers.py @@ -163,9 +163,7 @@ def make_module_step(self, *args, **kwargs): self.module_load_environment.TBBROOT = [self.tbb_subdir] # include paths to headers (e.g. CPATH) - self.module_load_environment.set_alias_vars('HEADERS', - os.path.join(self.tbb_subdir, 'include') - ) + self.module_load_environment.set_alias_vars('HEADERS', os.path.join(self.tbb_subdir, 'include')) return super().make_module_step(*args, **kwargs) @@ -180,7 +178,7 @@ def make_module_extra(self): if res.exit_code == 0 and multiarch_out: multi_arch_inc_dir_cmd = '|'.join([ "gcc -E -Wp,-v -xc /dev/null 2>&1", - f"grep {multiarch_out}$" , + f"grep {multiarch_out}$", "grep -v /include-fixed/", ]) res = run_shell_cmd(multi_arch_inc_dir_cmd, hidden=True) From 04dc2b383f7116d218e7fe984ae47cd7ab063eed Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Mon, 10 Feb 2025 22:27:38 +0100 Subject: [PATCH 09/13] make impi easyblock aware of module-search-path-headers --- easybuild/easyblocks/i/impi.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/easybuild/easyblocks/i/impi.py b/easybuild/easyblocks/i/impi.py index 2a343525355..f5aec659ac4 100644 --- a/easybuild/easyblocks/i/impi.py +++ b/easybuild/easyblocks/i/impi.py @@ -284,13 +284,15 @@ def make_module_step(self, *args, **kwargs): self.module_load_environment.PATH = path_dirs self.module_load_environment.LD_LIBRARY_PATH = lib_dirs self.module_load_environment.LIBRARY_PATH = lib_dirs - self.module_load_environment.CPATH = include_dirs self.module_load_environment.MANPATH = [manpath] if fi_provider_path is not None: self.module_load_environment.FI_PROVIDER_PATH = fi_provider_path if mic_library_path is not None: self.module_load_environment.MIC_LD_LIBRARY_PATH = mic_library_path + # include paths to headers (e.g. CPATH) + self.module_load_environment.set_alias_vars('HEADERS', include_dirs) + return super().make_module_step(*args, **kwargs) def make_module_extra(self, *args, **kwargs): From 9926576e2256d421d0a0cc768c2d6ca85d6b7119 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Mon, 10 Feb 2025 23:23:10 +0100 Subject: [PATCH 10/13] make intelbase easyblock aware of module-search-path-headers --- easybuild/easyblocks/generic/intelbase.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/easybuild/easyblocks/generic/intelbase.py b/easybuild/easyblocks/generic/intelbase.py index 8c749478c6a..bf8c9c6be26 100644 --- a/easybuild/easyblocks/generic/intelbase.py +++ b/easybuild/easyblocks/generic/intelbase.py @@ -137,7 +137,9 @@ def prepare_intel_tools_env(self): # make sure $CPATH, $LD_LIBRARY_PATH and $LIBRARY_PATH are not updated in generated module file, # because that leads to problem when the libraries included with VTune/Advisor/Inspector are being picked up - for disallowed_var in ['CPATH', 'LD_LIBRARY_PATH', 'LIBRARY_PATH']: + mod_env_headers = self.module_load_environment.alias_vars('HEADERS') + mod_env_libs = ['LD_LIBRARY_PATH', 'LIBRARY_PATH'] + for disallowed_var in mod_env_headers + mod_env_libs: try: delattr(self.module_load_environment, disallowed_var) except AttributeError: From adf3b83e28a9fe4d8eb9be9bbc3ce6dac96e4787 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Tue, 11 Feb 2025 00:27:16 +0100 Subject: [PATCH 11/13] make gcc easyblock aware of module-search-path-headers --- easybuild/easyblocks/g/gcc.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/easybuild/easyblocks/g/gcc.py b/easybuild/easyblocks/g/gcc.py index 072dbf3d49f..c80d4d6f68b 100644 --- a/easybuild/easyblocks/g/gcc.py +++ b/easybuild/easyblocks/g/gcc.py @@ -203,12 +203,17 @@ def __init__(self, *args, **kwargs): self.log.warning('Setting withnvptx to False, since we are building on a RISC-V system') self.cfg['withnvptx'] = False - # define list of subdirectories in search paths of module load environment - self.module_load_environment.PATH = ['bin'] - self.module_load_environment.LD_LIBRARY_PATH = ['lib', 'lib64'] - # GCC can find its own headers and libraries but the .so's need to be in LD_LIBRARY_PATH - self.module_load_environment.CPATH = [] - self.module_load_environment.LIBRARY_PATH = ['lib', 'lib64'] if get_cpu_family() == RISCV else [] + # GCC can find its own headers and libraries in most cases, but we had + # cases where paths top libraries needed to be set explicitly + # see: https://github.com/easybuilders/easybuild-easyblocks/pull/3256 + # Therefore, remove paths from header search paths but keep paths in LIBRARY_PATH + for header_search_path in self.module_load_environment.alias_vars('HEADERS'): + try: + delattr(self.module_load_environment, header_search_path) + except AttributeError: + pass + else: + self.log.debug(f"Removing ${header_search_path} from module file of {self.name}") def create_dir(self, dirname): """ From 9e5d3819ca9922c4854f56853b8fadf231df2462 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Tue, 11 Feb 2025 00:56:13 +0100 Subject: [PATCH 12/13] make imkl easyblock aware of module-search-path-headers --- easybuild/easyblocks/i/imkl.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/easybuild/easyblocks/i/imkl.py b/easybuild/easyblocks/i/imkl.py index 63677b25451..9db0b4fc1ab 100644 --- a/easybuild/easyblocks/i/imkl.py +++ b/easybuild/easyblocks/i/imkl.py @@ -454,12 +454,15 @@ def make_module_step(self, *args, **kwargs): os.path.join(self.mkl_basedir, 'lib', 'intel64'), ] self.module_load_environment.LIBRARY_PATH = self.module_load_environment.LD_LIBRARY_PATH - self.module_load_environment.CPATH = [ + self.module_load_environment.CMAKE_PREFIX_PATH = [self.mkl_basedir] + self.module_load_environment.PKG_CONFIG_PATH = pkg_config_path + + # include paths to headers (e.g. CPATH) + include_dirs = [ os.path.join(self.mkl_basedir, 'include'), os.path.join(self.mkl_basedir, 'include', 'fftw'), ] - self.module_load_environment.CMAKE_PREFIX_PATH = [self.mkl_basedir] - self.module_load_environment.PKG_CONFIG_PATH = pkg_config_path + self.module_load_environment.set_alias_vars('HEADERS', include_dirs) if LooseVersion(self.version) < LooseVersion('2021'): self.module_load_environment.MANPATH = ['man', os.path.join('man', 'en_US')] From 0e7ab1c50a31716cfe9531661d64163770a23e01 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Tue, 11 Feb 2025 01:50:15 +0100 Subject: [PATCH 13/13] replace delattr with ModuleLoadEnvironment.remove() in gcc and intelbase easyblocks --- easybuild/easyblocks/g/gcc.py | 10 +++------- easybuild/easyblocks/generic/intelbase.py | 8 ++------ 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/easybuild/easyblocks/g/gcc.py b/easybuild/easyblocks/g/gcc.py index c80d4d6f68b..c52c53f5edb 100644 --- a/easybuild/easyblocks/g/gcc.py +++ b/easybuild/easyblocks/g/gcc.py @@ -207,13 +207,9 @@ def __init__(self, *args, **kwargs): # cases where paths top libraries needed to be set explicitly # see: https://github.com/easybuilders/easybuild-easyblocks/pull/3256 # Therefore, remove paths from header search paths but keep paths in LIBRARY_PATH - for header_search_path in self.module_load_environment.alias_vars('HEADERS'): - try: - delattr(self.module_load_environment, header_search_path) - except AttributeError: - pass - else: - self.log.debug(f"Removing ${header_search_path} from module file of {self.name}") + for disallowed_var in self.module_load_environment.alias_vars('HEADERS'): + self.module_load_environment.remove(disallowed_var) + self.log.debug(f"Purposely not updating ${disallowed_var} in {self.name} module file") def create_dir(self, dirname): """ diff --git a/easybuild/easyblocks/generic/intelbase.py b/easybuild/easyblocks/generic/intelbase.py index bf8c9c6be26..0edbb1ee41f 100644 --- a/easybuild/easyblocks/generic/intelbase.py +++ b/easybuild/easyblocks/generic/intelbase.py @@ -140,12 +140,8 @@ def prepare_intel_tools_env(self): mod_env_headers = self.module_load_environment.alias_vars('HEADERS') mod_env_libs = ['LD_LIBRARY_PATH', 'LIBRARY_PATH'] for disallowed_var in mod_env_headers + mod_env_libs: - try: - delattr(self.module_load_environment, disallowed_var) - except AttributeError: - pass - else: - self.log.debug(f"Purposely not updating ${disallowed_var} in {self.name} module file") + self.module_load_environment.remove(disallowed_var) + self.log.debug(f"Purposely not updating ${disallowed_var} in {self.name} module file") def get_custom_paths_tools(self, binaries): """Custom sanity check paths for certain Intel tools."""