Skip to content

Set $HOME to temporary folder for TensorFlow build and tests #3751

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
boegel merged 3 commits into from
Jun 19, 2025
Merged

Set $HOME to temporary folder for TensorFlow build and tests #3751

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
f4af8b8
Set $HOME to temporary folder for TensorFlow build and tests
Flamefire Jun 2, 2025
f14505b
Use dict/set literals
Flamefire Jun 2, 2025
2810fbd
Mark unused variable
Flamefire Jun 2, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 15 additions & 11 deletions easybuild/easyblocks/t/tensorflow.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -197,12 +197,12 @@ def is_version_ok(version_range):
('wrapt', '2.0.0:'): 'wrapt',
}

dependency_mapping = dict((dep_name, tf_name)
for (dep_name, version_range), tf_name in available_system_libs.items()
if is_version_ok(version_range))
python_mapping = dict((pkg_name, tf_name)
for (pkg_name, version_range), tf_name in python_system_libs.items()
if is_version_ok(version_range))
dependency_mapping = {dep_name: tf_name
for (dep_name, version_range), tf_name in available_system_libs.items()
if is_version_ok(version_range)}
python_mapping = {pkg_name: tf_name
for (pkg_name, version_range), tf_name in python_system_libs.items()
if is_version_ok(version_range)}

if as_valid_libs:
tf_names = [tf_name for tf_name, version_range in unused_system_libs.items()
Expand Down Expand Up @@ -356,16 +356,16 @@ def get_system_libs(self):
"""
dependency_mapping, python_mapping = get_system_libs_for_version(self.version)
# Some TF dependencies require both a (usually C++) dependency and a Python package
deps_with_python_pkg = set(tf_name for tf_name in dependency_mapping.values()
if tf_name in python_mapping.values())
deps_with_python_pkg = {tf_name for tf_name in dependency_mapping.values()
if tf_name in python_mapping.values()}

system_libs = []
cpaths = []
libpaths = []
ignored_system_deps = []

# Check direct dependencies
dep_names = set(dep['name'] for dep in self.cfg.dependencies())
dep_names = {dep['name'] for dep in self.cfg.dependencies()}
for dep_name, tf_name in sorted(dependency_mapping.items(), key=lambda i: i[0].lower()):
if dep_name in dep_names:
if tf_name in deps_with_python_pkg:
Expand Down Expand Up @@ -446,6 +446,8 @@ def setup_build_dirs(self):
parent_dir = os.path.dirname(self.start_dir)
# Path where Bazel will store its output, build artefacts etc.
self.output_user_root_dir = os.path.join(parent_dir, 'bazel-root')
# Replace $HOME with a temporary folder to avoid using the user's home directory
self.home_dir = tempfile.mkdtemp(suffix='-tf-home')
# Folder where wrapper binaries can be placed, where required. TODO: Replace by --action_env cmds
self.wrapper_dir = os.path.join(parent_dir, 'wrapper_bin')
mkdir(self.wrapper_dir)
Expand Down Expand Up @@ -817,7 +819,7 @@ def patch_crosstool_files(self):
regex_subs.extend([('-fPIE', '-fPIC'), ('"-pie"', '"-fPIC"')])

# patch all CROSSTOOL* scripts to fix hardcoding of locations of binutils/GCC binaries
for path, dirnames, filenames in os.walk(os.getcwd()):
for path, _dirnames, filenames in os.walk(os.getcwd()):
for filename in filenames:
if filename.startswith('CROSSTOOL'):
full_path = os.path.join(path, filename)
Expand Down Expand Up @@ -849,6 +851,8 @@ def build_step(self):
# A value of None is interpreted as using the invoking environments value
INHERIT = None # For better readability

action_env['HOME'] = self.home_dir

jvm_max_memory = self.cfg['jvm_max_memory']
if jvm_max_memory:
jvm_startup_memory = min(512, int(jvm_max_memory))
Expand Down Expand Up @@ -1116,7 +1120,7 @@ def test_step(self):
self.log.warning(fail_msg)
# Try to enhance error message
failed_tests = []
failed_test_logs = dict()
failed_test_logs = {}
# Bazel outputs failed tests like "//tensorflow/c:kernels_test FAILED in[...]"
for match in re.finditer(r'^(//[a-zA-Z_/:]+)\s+FAILED', res.output, re.MULTILINE):
test_name = match.group(1)
Expand Down