Skip to content

refactor search for python_cmd in PythonPackage and PythonBundle easyblocks #3475

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Micket merged 3 commits into from
Oct 10, 2024
Merged

refactor search for python_cmd in PythonPackage and PythonBundle easyblocks #3475

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
bb2b5b4
Improve string formatting in `pick_python_cmd`
Flamefire Oct 10, 2024
ad544cc
Factor out `find_python_cmd` from `PythonPackage` and `PythonBundle`
Flamefire Oct 9, 2024
aa0e320
Add wrapper `find_python_cmd_from_ec`
Flamefire Oct 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 15 additions & 48 deletions easybuild/easyblocks/generic/pythonbundle.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,14 +28,12 @@
@author: Kenneth Hoste (Ghent University)
"""
import os
import sys

from easybuild.easyblocks.generic.bundle import Bundle
from easybuild.easyblocks.generic.pythonpackage import EXTS_FILTER_PYTHON_PACKAGES
from easybuild.easyblocks.generic.pythonpackage import PythonPackage, get_pylibdirs, pick_python_cmd
from easybuild.easyblocks.generic.pythonpackage import PythonPackage, get_pylibdirs, find_python_cmd_from_ec
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.config import build_option, PYTHONPATH, EBPYTHONPREFIXES
from easybuild.tools.filetools import which
from easybuild.tools.modules import get_software_root
import easybuild.tools.environment as env

Expand Down Expand Up @@ -72,64 +70,33 @@ def __init__(self, *args, **kwargs):

self.log.info("exts_default_options: %s", self.cfg['exts_default_options'])

self.python_cmd = None
self.pylibdir = None
self.all_pylibdirs = []
self.all_pylibdirs = None

# figure out whether this bundle of Python packages is being installed for multiple Python versions
self.multi_python = 'Python' in self.cfg['multi_deps']

def prepare_step(self, *args, **kwargs):
"""Prepare for installing bundle of Python packages."""
super(Bundle, self).prepare_step(*args, **kwargs)
def prepare_python(self):
"""Python-specific preparations."""

python_root = get_software_root('Python')
if python_root is None:
if get_software_root('Python') is None:
raise EasyBuildError("Python not included as dependency!")
self.python_cmd = find_python_cmd_from_ec(self.log, self.cfg, required=True)

# when system Python is used, the first 'python' command in $PATH will not be $EBROOTPYTHON/bin/python,
# since $EBROOTPYTHON is set to just 'Python' in that case
# (see handling of allow_system_deps in EasyBlock.prepare_step)
if which('python') == os.path.join(python_root, 'bin', 'python'):
# if we're using a proper Python dependency, let det_pylibdir use 'python' like it does by default
python_cmd = None
else:
# since det_pylibdir will use 'python' by default as command to determine Python lib directory,
# we need to intervene when the system Python is used, by specifying version requirements
# to pick_python_cmd so the right 'python' command is used;
# if we're using the system Python and no Python version requirements are specified,
# use major/minor version of Python being used in this EasyBuild session (as we also do in PythonPackage)
req_py_majver = self.cfg['req_py_majver']
if req_py_majver is None:
req_py_majver = sys.version_info[0]
req_py_minver = self.cfg['req_py_minver']
if req_py_minver is None:
req_py_minver = sys.version_info[1]

# Get the max_py_majver and max_py_minver from the config
max_py_majver = self.cfg['max_py_majver']
max_py_minver = self.cfg['max_py_minver']

python_cmd = pick_python_cmd(req_maj_ver=req_py_majver, req_min_ver=req_py_minver,
max_py_majver=max_py_majver, max_py_minver=max_py_minver)

# If pick_python_cmd didn't find a (system) Python command, we should raise an error
if python_cmd:
self.log.info("Python command being used: %s", python_cmd)
else:
raise EasyBuildError(
"Failed to pick Python command that satisfies requirements in the easyconfig "
"(req_py_majver = %s, req_py_minver = %s, max_py_majver = %s, max_py_minver = %s)",
req_py_majver, req_py_minver, max_py_majver, max_py_minver
)

self.all_pylibdirs = get_pylibdirs(python_cmd=python_cmd)
self.all_pylibdirs = get_pylibdirs(python_cmd=self.python_cmd)
self.pylibdir = self.all_pylibdirs[0]

# if 'python' is not used, we need to take that into account in the extensions filter
# (which is also used during the sanity check)
if python_cmd:
if self.python_cmd != 'python':
orig_exts_filter = EXTS_FILTER_PYTHON_PACKAGES
self.cfg['exts_filter'] = (orig_exts_filter[0].replace('python', python_cmd), orig_exts_filter[1])
self.cfg['exts_filter'] = (orig_exts_filter[0].replace('python', self.python_cmd), orig_exts_filter[1])

def prepare_step(self, *args, **kwargs):
"""Prepare for installing bundle of Python packages."""
super(Bundle, self).prepare_step(*args, **kwargs)
self.prepare_python()

def extensions_step(self, *args, **kwargs):
"""Install extensions (usually PythonPackages)"""
Expand Down
138 changes: 71 additions & 67 deletions easybuild/easyblocks/generic/pythonpackage.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,31 +102,31 @@ def check_python_cmd(python_cmd):
# check whether specified Python command is available
if os.path.isabs(python_cmd):
if not os.path.isfile(python_cmd):
log.debug("Python command '%s' does not exist", python_cmd)
log.debug(f"Python command '{python_cmd}' does not exist")
return False
else:
python_cmd_path = which(python_cmd)
if python_cmd_path is None:
log.debug("Python command '%s' not available through $PATH", python_cmd)
log.debug(f"Python command '{python_cmd}' not available through $PATH")
return False

pyver = det_python_version(python_cmd)

if req_maj_ver is not None:
if req_min_ver is None:
req_majmin_ver = '%s.0' % req_maj_ver
else:
req_majmin_ver = '%s.%s' % (req_maj_ver, req_min_ver)

pyver = det_python_version(python_cmd)

# (strict) check for major version
maj_ver = pyver.split('.')[0]
if maj_ver != str(req_maj_ver):
log.debug("Major Python version does not match: %s vs %s", maj_ver, req_maj_ver)
log.debug(f"Major Python version does not match: {maj_ver} vs {req_maj_ver}")
return False

# check for minimal minor version
if LooseVersion(pyver) < LooseVersion(req_majmin_ver):
log.debug("Minimal requirement for minor Python version not satisfied: %s vs %s", pyver, req_majmin_ver)
log.debug(f"Minimal requirement for minor Python version not satisfied: {pyver} vs {req_majmin_ver}")
return False

if max_py_majver is not None:
Expand All @@ -135,45 +135,97 @@ def check_python_cmd(python_cmd):
else:
max_majmin_ver = '%s.%s' % (max_py_majver, max_py_minver)

pyver = det_python_version(python_cmd)

if LooseVersion(pyver) > LooseVersion(max_majmin_ver):
log.debug("Python version (%s) on the system is newer than the maximum supported "
"Python version specified in the easyconfig (%s)",
pyver, max_majmin_ver)
return False

# all check passed
log.debug("All check passed for Python command '%s'!", python_cmd)
log.debug(f"All check passed for Python command '{python_cmd}'!")
return True

# compose list of 'python' commands to consider
python_cmds = ['python']
if req_maj_ver:
python_cmds.append('python%s' % req_maj_ver)
python_cmds.append(f'python{req_maj_ver}')
if req_min_ver:
python_cmds.append('python%s.%s' % (req_maj_ver, req_min_ver))
python_cmds.append(f'python{req_maj_ver}.{req_min_ver}')
python_cmds.append(sys.executable)
log.debug("Considering Python commands: %s", ', '.join(python_cmds))
log.debug("Considering Python commands: " + ', '.join(python_cmds))

# try and find a 'python' command that satisfies the requirements
res = None
for python_cmd in python_cmds:
if check_python_cmd(python_cmd):
log.debug("Python command '%s' satisfies version requirements!", python_cmd)
log.debug(f"Python command '{python_cmd}' satisfies version requirements!")
if os.path.isabs(python_cmd):
res = python_cmd
else:
res = which(python_cmd)
log.debug("Absolute path to retained Python command: %s", res)
log.debug("Absolute path to retained Python command: " + res)
break
else:
log.debug("Python command '%s' does not satisfy version requirements (maj: %s, min: %s), moving on",
python_cmd, req_maj_ver, req_min_ver)
log.debug(f"Python command '{python_cmd}' does not satisfy version requirements "
f"(maj: {req_maj_ver}, min: {req_min_ver}), moving on")

return res


def find_python_cmd(log, req_py_majver, req_py_minver, max_py_majver, max_py_minver, required):
"""Return an appropriate python command to use.

When python is a dependency use the full path to that.
Else use req_py_maj/minver (defaulting to the Python being used in this EasyBuild session) to select one.
If no (matching) python command is found and raise an Error or log a warning depending on the required argument.
"""
python = None
python_root = get_software_root('Python')
# keep in mind that Python may be listed as an allowed system dependency,
# so just checking Python root is not sufficient
if python_root:
bin_python = os.path.join(python_root, 'bin', 'python')
if os.path.exists(bin_python) and os.path.samefile(which('python'), bin_python):
# if Python is listed as a (build) dependency, use 'python' command provided that way
python = bin_python
log.debug("Retaining 'python' command for Python dependency: " + python)

if python is None:
# if no Python version requirements are specified,
# use major/minor version of Python being used in this EasyBuild session
if req_py_majver is None:
req_py_majver = sys.version_info[0]
if req_py_minver is None:
req_py_minver = sys.version_info[1]
# if using system Python, go hunting for a 'python' command that satisfies the requirements
python = pick_python_cmd(req_maj_ver=req_py_majver, req_min_ver=req_py_minver,
max_py_majver=max_py_majver, max_py_minver=max_py_minver)

if python:
log.info("Python command being used: " + python)
elif required:
if all(v is None for v in (req_py_majver, req_py_minver, max_py_majver, max_py_minver)):
error_msg = "Failed to pick Python command to use"
else:
error_msg = (f"Failed to pick Python command that satisfies requirements in the easyconfig: "
f"req_py_majver = {req_py_majver}, req_py_minver = {req_py_minver}")
if max_py_majver is not None:
error_msg += f"max_py_majver = {max_py_majver}, max_py_minver = {max_py_minver}"
raise EasyBuildError(error_msg)
else:
log.warning("No Python command found!")
return python


def find_python_cmd_from_ec(log, cfg, required):
"""Find a python command using the constraints specified in the EasyConfig"""
return find_python_cmd(log,
cfg['req_py_majver'], cfg['req_py_minver'],
max_py_majver=cfg['max_py_majver'],
max_py_minver=cfg['max_py_minver'],
required=required)


def det_pylibdir(plat_specific=False, python_cmd=None):
"""Determine Python library directory."""
log = fancylogger.getLogger('det_pylibdir', fname=False)
Expand All @@ -188,8 +240,8 @@ def det_pylibdir(plat_specific=False, python_cmd=None):
if LooseVersion(det_python_version(python_cmd)) >= LooseVersion('3.12'):
# Python 3.12 removed distutils but has a core sysconfig module which is similar
pathname = 'platlib' if plat_specific else 'purelib'
vars = {'platbase': prefix, 'base': prefix}
pycode = 'import sysconfig; print(sysconfig.get_path("%s", vars=%s))' % (pathname, vars)
vars_param = {'platbase': prefix, 'base': prefix}
pycode = 'import sysconfig; print(sysconfig.get_path("%s", vars=%s))' % (pathname, vars_param)
else:
args = 'plat_specific=%s, prefix="%s"' % (plat_specific, prefix)
pycode = "import distutils.sysconfig; print(distutils.sysconfig.get_python_lib(%s))" % args
Expand Down Expand Up @@ -503,55 +555,7 @@ def set_pylibdirs(self):
def prepare_python(self):
"""Python-specific preparations."""

# pick 'python' command to use
python = None
python_root = get_software_root('Python')
# keep in mind that Python may be listed as an allowed system dependency,
# so just checking Python root is not sufficient
if python_root:
bin_python = os.path.join(python_root, 'bin', 'python')
if os.path.exists(bin_python) and os.path.samefile(which('python'), bin_python):
# if Python is listed as a (build) dependency, use 'python' command provided that way
python = os.path.join(python_root, 'bin', 'python')
self.log.debug("Retaining 'python' command for Python dependency: %s", python)

if python is None:
# if no Python version requirements are specified,
# use major/minor version of Python being used in this EasyBuild session
req_py_majver = self.cfg['req_py_majver']
if req_py_majver is None:
req_py_majver = sys.version_info[0]
req_py_minver = self.cfg['req_py_minver']
if req_py_minver is None:
req_py_minver = sys.version_info[1]

# Get the max_py_majver and max_py_minver from the config
max_py_majver = self.cfg['max_py_majver']
max_py_minver = self.cfg['max_py_minver']

# if using system Python, go hunting for a 'python' command that satisfies the requirements
python = pick_python_cmd(req_maj_ver=req_py_majver, req_min_ver=req_py_minver,
max_py_majver=max_py_majver, max_py_minver=max_py_minver)

# Check if we have Python by now. If not, and if self.require_python, raise a sensible error
if python:
self.python_cmd = python
self.log.info("Python command being used: %s", self.python_cmd)
elif self.require_python:
if (req_py_majver is not None or req_py_minver is not None
or max_py_majver is not None or max_py_minver is not None):
raise EasyBuildError(
"Failed to pick Python command that satisfies requirements in the easyconfig "
"(req_py_majver = %s, req_py_minver = %s, max_py_majver = %s, max_py_minver = %s)",
req_py_majver, req_py_minver, max_py_majver, max_py_minver
)
else:
raise EasyBuildError("Failed to pick Python command to use")
else:
self.log.warning("No Python command found!")
else:
self.python_cmd = python
self.log.info("Python command being used: %s", self.python_cmd)
self.python_cmd = find_python_cmd_from_ec(self.log, self.cfg, self.require_python)

if self.python_cmd:
# set Python lib directories
Expand Down