From bdb5964f8ac3618e00e952115396181b4135cdeb Mon Sep 17 00:00:00 2001 From: Damian Alvarez Date: Wed, 13 Jun 2018 12:17:19 +0200 Subject: [PATCH 1/7] Enabled filtering of paths in the CPATH variable generated for the wrapper --- easybuild/easyblocks/t/tensorflow.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/easybuild/easyblocks/t/tensorflow.py b/easybuild/easyblocks/t/tensorflow.py index ca96a739653..5ebe2d6aa9a 100644 --- a/easybuild/easyblocks/t/tensorflow.py +++ b/easybuild/easyblocks/t/tensorflow.py @@ -69,6 +69,7 @@ def extra_options(): extra_vars = { # see https://developer.nvidia.com/cuda-gpus 'cuda_compute_capabilities': [[], "List of CUDA compute capabilities to build with", CUSTOM], + 'wrapper_filter': [[], "List of paths to filter out in the wrapper creation", CUSTOM], 'with_jemalloc': [None, "Make TensorFlow use jemalloc (usually enabled by default)", CUSTOM], 'with_mkl_dnn': [None, "Make TensorFlow use Intel MKL-DNN (enabled unless cuDNN is used)", CUSTOM], } @@ -103,9 +104,16 @@ def configure_step(self): # cfr. https://github.com/bazelbuild/bazel/issues/663 if self.toolchain.comp_family() == toolchain.INTELCOMP: wrapper_dir = os.path.join(tmpdir, 'bin') + + # filter out paths from CPATH + wrapper_filter = self.cfg['wrapper_filter'] + cpath = os.getenv('CPATH').split(':') + filtered_cpath = [path for fil in wrapper_filter for path in cpath if not fil in path] + cpath = ':'.join(filtered_cpath) + icc_wrapper_txt = INTEL_COMPILER_WRAPPER % { 'compiler_path': which('icc'), - 'cpath': os.getenv('CPATH'), + 'cpath': cpath, 'intel_license_file': os.getenv('INTEL_LICENSE_FILE', os.getenv('LM_LICENSE_FILE')), 'wrapper_dir': wrapper_dir, } From 90a98fc1b234e4786f12ea7c13160dc0c454d4d5 Mon Sep 17 00:00:00 2001 From: Damian Alvarez Date: Thu, 14 Jun 2018 17:38:13 +0200 Subject: [PATCH 2/7] Removed paths also for any toolchain, and included LIBRARY_PATH in the filtering --- easybuild/easyblocks/t/tensorflow.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/easybuild/easyblocks/t/tensorflow.py b/easybuild/easyblocks/t/tensorflow.py index 5ebe2d6aa9a..421a9d5f962 100644 --- a/easybuild/easyblocks/t/tensorflow.py +++ b/easybuild/easyblocks/t/tensorflow.py @@ -69,7 +69,7 @@ def extra_options(): extra_vars = { # see https://developer.nvidia.com/cuda-gpus 'cuda_compute_capabilities': [[], "List of CUDA compute capabilities to build with", CUSTOM], - 'wrapper_filter': [[], "List of paths to filter out in the wrapper creation", CUSTOM], + 'package_filter': [[], "List of paths to filter out", CUSTOM], 'with_jemalloc': [None, "Make TensorFlow use jemalloc (usually enabled by default)", CUSTOM], 'with_mkl_dnn': [None, "Make TensorFlow use Intel MKL-DNN (enabled unless cuDNN is used)", CUSTOM], } @@ -100,17 +100,18 @@ def configure_step(self): tmpdir = tempfile.mkdtemp(suffix='-bazel-configure') + # filter out paths from CPATH and LIBRARY_PATH + package_filter = self.cfg['package_filter'] + for var in ['CPATH', 'LIBRARY_PATH']: + path = os.getenv(var).split(':') + filtered_path = [path for fil in package_filter for p in path if fil not in p] + os.environ[var] = ':'.join(filtered_path) + # put wrapper for Intel C compiler in place (required to make sure license server is found) # cfr. https://github.com/bazelbuild/bazel/issues/663 if self.toolchain.comp_family() == toolchain.INTELCOMP: wrapper_dir = os.path.join(tmpdir, 'bin') - # filter out paths from CPATH - wrapper_filter = self.cfg['wrapper_filter'] - cpath = os.getenv('CPATH').split(':') - filtered_cpath = [path for fil in wrapper_filter for path in cpath if not fil in path] - cpath = ':'.join(filtered_cpath) - icc_wrapper_txt = INTEL_COMPILER_WRAPPER % { 'compiler_path': which('icc'), 'cpath': cpath, From 043d756e2897c9948c8bff6413be3fae6bb6a0e6 Mon Sep 17 00:00:00 2001 From: Damian Alvarez Date: Thu, 14 Jun 2018 17:40:39 +0200 Subject: [PATCH 3/7] Fixed non defined cpath variable --- easybuild/easyblocks/t/tensorflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/t/tensorflow.py b/easybuild/easyblocks/t/tensorflow.py index 421a9d5f962..5a887af9bcd 100644 --- a/easybuild/easyblocks/t/tensorflow.py +++ b/easybuild/easyblocks/t/tensorflow.py @@ -114,7 +114,7 @@ def configure_step(self): icc_wrapper_txt = INTEL_COMPILER_WRAPPER % { 'compiler_path': which('icc'), - 'cpath': cpath, + 'cpath': os.getenv('CPATH'), 'intel_license_file': os.getenv('INTEL_LICENSE_FILE', os.getenv('LM_LICENSE_FILE')), 'wrapper_dir': wrapper_dir, } From 0a54a16ec28007241809868055015cc2a04bab24 Mon Sep 17 00:00:00 2001 From: Damian Alvarez Date: Fri, 15 Jun 2018 12:10:22 +0200 Subject: [PATCH 4/7] Fixed bug that created a list of lists, instead of a list of filtered paths --- easybuild/easyblocks/t/tensorflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/t/tensorflow.py b/easybuild/easyblocks/t/tensorflow.py index 5a887af9bcd..81ffe85b01d 100644 --- a/easybuild/easyblocks/t/tensorflow.py +++ b/easybuild/easyblocks/t/tensorflow.py @@ -104,7 +104,7 @@ def configure_step(self): package_filter = self.cfg['package_filter'] for var in ['CPATH', 'LIBRARY_PATH']: path = os.getenv(var).split(':') - filtered_path = [path for fil in package_filter for p in path if fil not in p] + filtered_path = [p for fil in package_filter for p in path if fil not in p] os.environ[var] = ':'.join(filtered_path) # put wrapper for Intel C compiler in place (required to make sure license server is found) From 1f4c7e51fa383b9438067ef98f756a82b8824512 Mon Sep 17 00:00:00 2001 From: Damian Alvarez Date: Fri, 22 Jun 2018 16:46:46 +0200 Subject: [PATCH 5/7] Added logging, and some extra explanation of what path_filter does and why --- easybuild/easyblocks/t/tensorflow.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/easybuild/easyblocks/t/tensorflow.py b/easybuild/easyblocks/t/tensorflow.py index 81ffe85b01d..ca6f47c404a 100644 --- a/easybuild/easyblocks/t/tensorflow.py +++ b/easybuild/easyblocks/t/tensorflow.py @@ -69,7 +69,7 @@ def extra_options(): extra_vars = { # see https://developer.nvidia.com/cuda-gpus 'cuda_compute_capabilities': [[], "List of CUDA compute capabilities to build with", CUSTOM], - 'package_filter': [[], "List of paths to filter out", CUSTOM], + 'path_filter': [[], "List of patterns to be filtered out in paths in $CPATH and $LIBRARY_PATH", CUSTOM], 'with_jemalloc': [None, "Make TensorFlow use jemalloc (usually enabled by default)", CUSTOM], 'with_mkl_dnn': [None, "Make TensorFlow use Intel MKL-DNN (enabled unless cuDNN is used)", CUSTOM], } @@ -100,12 +100,17 @@ def configure_step(self): tmpdir = tempfile.mkdtemp(suffix='-bazel-configure') - # filter out paths from CPATH and LIBRARY_PATH - package_filter = self.cfg['package_filter'] - for var in ['CPATH', 'LIBRARY_PATH']: - path = os.getenv(var).split(':') - filtered_path = [p for fil in package_filter for p in path if fil not in p] - os.environ[var] = ':'.join(filtered_path) + # filter out paths from CPATH and LIBRARY_PATH. This is needed since bazel will pull some dependencies that + # might conflict with dependencies on the system and/or installed with EB. For example: protobuf + path_filter = self.cfg['path_filter'] + if len(path_filter > 0): + self.log.info("Filtering $CPATH and $LIBRARY_PATH") + for var in ['CPATH', 'LIBRARY_PATH']: + path = os.getenv(var).split(':') + self.log.info("$%s old value was %s" % (var, path)) + filtered_path = os.pathsep.join([p for fil in path_filter for p in path if fil not in p]) + self.log.info("$%s new value is %s" % (var, filtered_path)) + setvar(var, filtered_path) # put wrapper for Intel C compiler in place (required to make sure license server is found) # cfr. https://github.com/bazelbuild/bazel/issues/663 From d9bdb3ff6effcfed43855508d566b8e7d24cbe8c Mon Sep 17 00:00:00 2001 From: Damian Alvarez Date: Fri, 22 Jun 2018 17:14:36 +0200 Subject: [PATCH 6/7] Fixed undefined var --- easybuild/easyblocks/t/tensorflow.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/t/tensorflow.py b/easybuild/easyblocks/t/tensorflow.py index ca6f47c404a..7c5d323f2d5 100644 --- a/easybuild/easyblocks/t/tensorflow.py +++ b/easybuild/easyblocks/t/tensorflow.py @@ -110,7 +110,7 @@ def configure_step(self): self.log.info("$%s old value was %s" % (var, path)) filtered_path = os.pathsep.join([p for fil in path_filter for p in path if fil not in p]) self.log.info("$%s new value is %s" % (var, filtered_path)) - setvar(var, filtered_path) + env.setvar(var, filtered_path) # put wrapper for Intel C compiler in place (required to make sure license server is found) # cfr. https://github.com/bazelbuild/bazel/issues/663 @@ -125,7 +125,7 @@ def configure_step(self): } icc_wrapper = os.path.join(wrapper_dir, 'icc') write_file(icc_wrapper, icc_wrapper_txt) - env.setvar('PATH', ':'.join([os.path.dirname(icc_wrapper), os.getenv('PATH')])) + env.setvar('PATH', os.pathsep.join([os.path.dirname(icc_wrapper), os.getenv('PATH')])) if self.dry_run: self.dry_run_msg("Wrapper for 'icc' was put in place: %s", icc_wrapper) else: From 6b466352f516a290f79d04f1d57a5206ed240d1e Mon Sep 17 00:00:00 2001 From: Damian Alvarez Date: Thu, 5 Jul 2018 18:29:35 +0200 Subject: [PATCH 7/7] Style fixes --- easybuild/easyblocks/t/tensorflow.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/easybuild/easyblocks/t/tensorflow.py b/easybuild/easyblocks/t/tensorflow.py index 7c5d323f2d5..7e77c6b950f 100644 --- a/easybuild/easyblocks/t/tensorflow.py +++ b/easybuild/easyblocks/t/tensorflow.py @@ -103,13 +103,12 @@ def configure_step(self): # filter out paths from CPATH and LIBRARY_PATH. This is needed since bazel will pull some dependencies that # might conflict with dependencies on the system and/or installed with EB. For example: protobuf path_filter = self.cfg['path_filter'] - if len(path_filter > 0): - self.log.info("Filtering $CPATH and $LIBRARY_PATH") + if path_filter: + self.log.info("Filtering $CPATH and $LIBRARY_PATH with path filter %s", path_filter) for var in ['CPATH', 'LIBRARY_PATH']: - path = os.getenv(var).split(':') + path = os.getenv(var).split(os.pathsep) self.log.info("$%s old value was %s" % (var, path)) filtered_path = os.pathsep.join([p for fil in path_filter for p in path if fil not in p]) - self.log.info("$%s new value is %s" % (var, filtered_path)) env.setvar(var, filtered_path) # put wrapper for Intel C compiler in place (required to make sure license server is found)