diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 65ae805380..87e5660e2b 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1390,6 +1390,37 @@ def make_module_description(self): """ return self.module_generator.get_description() + def make_module_pythonpath(self): + """ + Add lines for module file to update $PYTHONPATH or $EBPYTHONPREFIXES, + if they aren't already present and the standard lib/python*/site-packages subdirectory exists + """ + lines = [] + if not os.path.isfile(os.path.join(self.installdir, 'bin', 'python')): # only needed when not a python install + python_subdir_pattern = os.path.join(self.installdir, 'lib', 'python*', 'site-packages') + candidate_paths = (os.path.relpath(path, self.installdir) for path in glob.glob(python_subdir_pattern)) + python_paths = [path for path in candidate_paths if re.match(r'lib/python\d+\.\d+/site-packages', path)] + + runtime_deps = [dep['name'] for dep in self.cfg.dependencies(runtime_only=True)] + use_ebpythonprefixes = all([ + 'Python' in runtime_deps, + build_option('prefer_python_search_path') == EBPYTHONPREFIXES, + not self.cfg['force_pythonpath'] + ]) + + if python_paths: + # add paths unless they were already added + if use_ebpythonprefixes: + path = '' # EBPYTHONPREFIXES are relative to the install dir + if path not in self.module_generator.added_paths_per_key[EBPYTHONPREFIXES]: + lines.append(self.module_generator.prepend_paths(EBPYTHONPREFIXES, path)) + else: + for python_path in python_paths: + if python_path not in self.module_generator.added_paths_per_key[PYTHONPATH]: + lines.append(self.module_generator.prepend_paths(PYTHONPATH, python_path)) + + return lines + def make_module_extra(self, altroot=None, altversion=None): """ Set extra stuff in module file, e.g. $EBROOT*, $EBVERSION*, etc. @@ -1438,26 +1469,8 @@ def make_module_extra(self, altroot=None, altversion=None): value, type(value)) lines.append(self.module_generator.append_paths(key, value, allow_abs=self.cfg['allow_append_abs_path'])) - # Add automatic PYTHONPATH or EBPYTHONPREFIXES if they aren't already present and python paths exist - if not os.path.isfile(os.path.join(self.installdir, 'bin/python')): # only needed when not a python install - candidate_paths = (os.path.relpath(path, self.installdir) - for path in glob.glob(f'{self.installdir}/lib/python*/site-packages')) - python_paths = [path for path in candidate_paths if re.match(r'lib/python\d+\.\d+/site-packages', path)] - - runtime_deps = [dep['name'] for dep in self.cfg.dependencies(runtime_only=True)] - use_ebpythonprefixes = 'Python' in runtime_deps and \ - build_option('prefer_python_search_path') == EBPYTHONPREFIXES and not self.cfg['force_pythonpath'] - - if python_paths: - # Add paths unless they were already added - if use_ebpythonprefixes: - path = '' # EBPYTHONPREFIXES are relative to the install dir - if path not in self.module_generator.added_paths_per_key[EBPYTHONPREFIXES]: - lines.append(self.module_generator.prepend_paths(EBPYTHONPREFIXES, path)) - else: - for python_path in python_paths: - if python_path not in self.module_generator.added_paths_per_key[PYTHONPATH]: - lines.append(self.module_generator.prepend_paths(PYTHONPATH, python_path)) + # add lines to update $PYTHONPATH or $EBPYTHONPREFIXES + lines.extend(self.make_module_pythonpath()) modloadmsg = self.cfg['modloadmsg'] if modloadmsg: diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index afe0b86bef..76ef640b29 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -4238,6 +4238,69 @@ def test_eb_error(self): stderr = stderr.getvalue() self.assertTrue(regex.search(stderr), f"Pattern '{regex.pattern}' should be found in {stderr}") + def test_toy_python(self): + """ + Test whether $PYTHONPATH or $EBPYTHONPREFIXES are set correctly. + """ + # generate fake Python module that we can use as runtime dependency for toy + # (required condition for use of $EBPYTHONPREFIXES) + fake_mods_path = os.path.join(self.test_prefix, 'modules') + fake_python_mod = os.path.join(fake_mods_path, 'Python', '3.6') + if get_module_syntax() == 'Lua': + fake_python_mod += '.lua' + write_file(fake_python_mod, '') + else: + write_file(fake_python_mod, '#%Module') + self.modtool.use(fake_mods_path) + + test_ecs = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs') + toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb') + + test_ec_txt = read_file(toy_ec) + test_ec_txt += "\npostinstallcmds.append('mkdir -p %(installdir)s/lib/python3.6/site-packages')" + test_ec_txt += "\npostinstallcmds.append('touch %(installdir)s/lib/python3.6/site-packages/foo.py')" + + test_ec = os.path.join(self.test_prefix, 'test.eb') + write_file(test_ec, test_ec_txt) + self.run_test_toy_build_with_output(ec_file=test_ec) + + toy_mod = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0') + if get_module_syntax() == 'Lua': + toy_mod += '.lua' + toy_mod_txt = read_file(toy_mod) + + pythonpath_regex = re.compile('^prepend.path.*PYTHONPATH.*lib/python3.6/site-packages', re.M) + + self.assertTrue(pythonpath_regex.search(toy_mod_txt), + f"Pattern '{pythonpath_regex.pattern}' found in: {toy_mod_txt}") + + # also check when opting in to use $EBPYTHONPREFIXES instead of $PYTHONPATH + args = ['--prefer-python-search-path=EBPYTHONPREFIXES'] + self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=args) + toy_mod_txt = read_file(toy_mod) + # if Python is not listed as a runtime dependency then $PYTHONPATH is still used, + # because the Python dependency used must be aware of $EBPYTHONPREFIXES + # (see sitecustomize.py installed by Python easyblock) + self.assertTrue(pythonpath_regex.search(toy_mod_txt), + f"Pattern '{pythonpath_regex.pattern}' found in: {toy_mod_txt}") + + test_ec_txt += "\ndependencies = [('Python', '3.6', '', SYSTEM)]" + write_file(test_ec, test_ec_txt) + self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=args) + toy_mod_txt = read_file(toy_mod) + + ebpythonprefixes_regex = re.compile('^prepend.path.*EBPYTHONPREFIXES.*root', re.M) + self.assertTrue(ebpythonprefixes_regex.search(toy_mod_txt), + f"Pattern '{ebpythonprefixes_regex.pattern}' found in: {toy_mod_txt}") + + test_ec_txt += "\nforce_pythonpath = True" + write_file(test_ec, test_ec_txt) + self.run_test_toy_build_with_output(ec_file=test_ec) + toy_mod_txt = read_file(toy_mod) + + self.assertTrue(pythonpath_regex.search(toy_mod_txt), + f"Pattern '{pythonpath_regex.pattern}' found in: {toy_mod_txt}") + def suite(): """ return all the tests in this file """