diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index 4bae0908c19..0b1bdf4db49 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -10,20 +10,19 @@ concurrency: jobs: python-linting: - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # 4.2.2 - name: set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 with: python-version: 3.8 - name: install Python packages run: | pip install --upgrade pip - # fix to this version for develop branch (to avoid needing to fix geant4.py) - pip install --upgrade "flake8" + pip install --upgrade flake8 - name: Run flake8 to verify PEP8-compliance of Python code run: flake8 diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 3cd95ff88de..d3a32651970 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -10,24 +10,22 @@ concurrency: jobs: build: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 strategy: matrix: - python: [3.6, 3.7, 3.8, 3.9, '3.10', '3.11'] - modules_tool: [Lmod-6.6.3, Lmod-7.8.22, Lmod-8.1.14, modules-tcl-1.147, modules-3.2.10, modules-4.1.4] + python: [3.7, 3.8, 3.9, '3.10', '3.11', '3.12'] + modules_tool: [Lmod-8.1.14, modules-tcl-1.147, modules-3.2.10, modules-4.5.3] module_syntax: [Lua, Tcl] # exclude some configuration for non-Lmod modules tool: # - don't test with Lua module syntax (only supported in Lmod) - # - don't test with Python 3.7+ (only with 3.6), to limit test configurations + # - don't test with Python 3.8+ (only with 3.7), to limit test configurations exclude: - modules_tool: modules-tcl-1.147 module_syntax: Lua - modules_tool: modules-3.2.10 module_syntax: Lua - - modules_tool: modules-4.1.4 + - modules_tool: modules-4.5.3 module_syntax: Lua - - modules_tool: modules-tcl-1.147 - python: 3.7 - modules_tool: modules-tcl-1.147 python: 3.8 - modules_tool: modules-tcl-1.147 @@ -36,8 +34,8 @@ jobs: python: '3.10' - modules_tool: modules-tcl-1.147 python: '3.11' - - modules_tool: modules-3.2.10 - python: 3.7 + - modules_tool: modules-tcl-1.147 + python: '3.12' - modules_tool: modules-3.2.10 python: 3.8 - modules_tool: modules-3.2.10 @@ -46,22 +44,24 @@ jobs: python: '3.10' - modules_tool: modules-3.2.10 python: '3.11' - - modules_tool: modules-4.1.4 - python: 3.7 - - modules_tool: modules-4.1.4 + - modules_tool: modules-3.2.10 + python: '3.12' + - modules_tool: modules-4.5.3 python: 3.8 - - modules_tool: modules-4.1.4 + - modules_tool: modules-4.5.3 python: 3.9 - - modules_tool: modules-4.1.4 + - modules_tool: modules-4.5.3 python: '3.10' - - modules_tool: modules-4.1.4 + - modules_tool: modules-4.5.3 python: '3.11' + - modules_tool: modules-4.5.3 + python: '3.12' fail-fast: false steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # 4.2.2 - name: set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 with: python-version: ${{matrix.python}} architecture: x64 @@ -73,20 +73,22 @@ jobs: # sudo apt-get update # for modules tool sudo apt-get install lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev - # fix for lua-posix packaging issue, see https://bugs.launchpad.net/ubuntu/+source/lua-posix/+bug/1752082 - # needed for Ubuntu 18.04, but not for Ubuntu 20.04, so skipping symlinking if posix.so already exists - if [ ! -e /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so ] ; then - sudo ln -s /usr/lib/x86_64-linux-gnu/lua/5.2/posix_c.so /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so - fi # for testing OpenMPI-system*eb we need to have Open MPI installed sudo apt-get install libopenmpi-dev openmpi-bin - + # Python packages + pip --version + pip install --upgrade pip + pip --version + if ! python -c "import distutils" 2> /dev/null; then + # we need setuptools for distutils in Python 3.12+, needed for python setup.py sdist + pip install --upgrade setuptools + fi - name: install EasyBuild framework run: | # first determine which branch of easybuild-framework repo to install - BRANCH=develop + BRANCH=5.0.x if [ "x$GITHUB_BASE_REF" = 'xmain' ]; then BRANCH=main; fi - if [ "x$GITHUB_BASE_REF" = 'x4.x' ]; then BRANCH=4.x; fi + if [ "x$GITHUB_BASE_REF" = 'x5.0.x' ]; then BRANCH=5.0.x; fi echo "Using easybuild-framework branch $BRANCH (\$GITHUB_BASE_REF $GITHUB_BASE_REF)" git clone -b $BRANCH --depth 10 --single-branch https://github.com/easybuilders/easybuild-framework.git cd easybuild-framework; git log -n 1; cd - @@ -113,7 +115,7 @@ jobs: python setup.py sdist ls dist export PREFIX=/tmp/$USER/$GITHUB_SHA - pip install --prefix $PREFIX dist/easybuild-easyblocks*tar.gz + pip install --prefix $PREFIX dist/easybuild[-_]easyblocks*tar.gz - name: run test suite env: diff --git a/RELEASE_NOTES b/RELEASE_NOTES index c847c65b3ac..a258188d141 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -1,12 +1,123 @@ This file contains a description of the major changes to the easybuild-easyblocks EasyBuild package. For more detailed information, please see the git log. -These release notes can also be consulted at http://easybuild.readthedocs.org/en/latest/Release_notes.html. +These release notes can also be consulted at https://docs.easybuild.io/release-notes . -The latest version of easybuild-easyblocks provides 259 software-specific easyblocks and 43 generic easyblocks. +The latest version of easybuild-easyblocks provides 196 software-specific easyblocks and 44 generic easyblocks. +v5.0.0 (18 March 2025) +---------------------- -v4.9.4 (22 september 2024) +- use `run_shell_cmd` rather than `run_cmd` and/or `run_cmd_qa` in all easyblocks (#3046, #3074, #3090, #3091, #3092, #3093, #3094, #3098, #3099, #3100, #3101, #3102, #3105, #3106, #3107, #3109, #3110, #3112, #3113, #3115, #3120, #3122, #3123, #3124, #3127, #3128, #3129, #3130, #3131, #3132, #3133, #3134, #3135, #3136, #3137, #3139, #3140, #3141, #3142, #3143, #3144, #3145, #3146, #3147, #3149, #3150, #3153, #3156, #3158, #3159, #3162, #3163, #3164, #3165, #3166, #3167, #3168, #3169, #3171, #3179, #3180, #3187, #3188, #3189, #3190, #3192, #3193, #3195, #3197, #3198, #3199, #3200, #3201, #3203, #3204, #3206, #3207, #3209, #3210, #3212, #3213, #3214, #3215, #3217, #3218, #3219, #3242, #3243, #3244, #3270, #3274, #3286, #3288, #3289, #3290, #3291, #3293, #3295, #3297, #3298, #3299, #3308, #3327, #3342, #3411, #3413, #3414, #3415, #3426, #3664) +- adopt `module_load_environment` in all easyblocks (#3513, #3529, #3530, #3550, #3551, #3552, #3553, #3555, #3556, #3559, #3560, #3561, #3562, #3577, #3578, #3579, #3585, #3586, #3587, #3592, #3593, #3594, #3596, #3597, #3598, #3600, #3601, #3602, #3603, #3604, #3606, #3607, #3608, #3609, #3610, #3611, #3612, #3613, #3615, #3616, #3618, #3619, #3620, #3622, #3625, #3626, #3627, #3628, #3629, #3630, #3631, #3634, #3647, #3635, #3641) +- enable `download_dep_fail`, `use_pip`, `sanity_pip_check` by default in `PythonPackage` easyblock (#3022, #3079, #3221) +- enhance `ConfigureMake` easyblock to error out on unknown configure options (#3025, #3563) +- adopt to changes in EasyBuild framework: + - stop testing with Python 3.5 and Lmod 6.x, stop using `toolchain.DUMMY` (#3014) + - stop importing from deprecated easybuild.tools.py2vs3 module + stop testing with Python 2.7 and 3.5 (#2916, #3015) + - use `LooseVersion` from `easybuild.tools` in all easyblocks (#3018) + - rename `run` method to `install_extension`, and likewise for `prerun` to `pre_install_extension`, `postrun` to `post_install_extension`, and `run_async` to `install_extension_async` (#3064) + - disable check for `.mod` files in Clang, AOCC, AOMP, and CPLEX easyblocks (#3067) + - stop running easyblocks test suite with Lmod 7.x (#3083) + - use `sysconfig.get_config_vars` from Python standard library in `PythonPackage` easyblock (instead of `distutils.sysconfig.get_config_vars`) (#3264) + - also run easyblocks test suite with Python 3.12 (#3266) + - use `ERROR` global from tools.config in Clang (#3318) and TensorFlow (#3320) easyblocks + - update easyblocks to let EasyBuild framework take care of prepend to `$PYTHONPATH` or `$EBPYTHONPREFIXES` (#3343) + - use `PYPI_SOURCE` constant in generic `PythonPackage` easyblock + custom easyblock for Python (#3410) + - use non-system toolchain for testing `--module-only` for ELPA and FFTW easyblocks, since they rely on toolchain options like 'pic' being set (#3425) + - take into account that '`pic`' toolchain option may not be defined in `CMakeMake` easyblock (#3470) + - remove use of deprecated `parse_log_for_error` in `RPackage` easyblock (#3515) + - rename `post_install_step` to `post_processing_step` (#3525) + - be more careful when getting value of particular easyconfig parameter in `PythonBundle` and `PythonPackage` easyblock, to avoid trouble with unresolved template values (#3537) + - remove use of deprecated `extract_errors_from_log` in `ConfigureMake` easyblock (#3540) + - remove Python 2 fallback in `openssl_wrapper` (#3544) + - update easyblocks to use `EasyConfig.parallel` property (#3557) + - honor `--module-search-path-headers` in easyblocks that hardcode CPATH (#3584) + - remove hardcoded CPLUS_INCLUDE_PATH in favor of module-search-path-headers in custom easyblock for Eigen (#3599) +- various updates: + - update version of `config.guess` used by `ConfigureMake` (#3013) + - enhance Score-P EasyBlock for future releases and better oneAPI support (#3548) + - enhance AOCC easyblock to work with LLVM 16 and newer (#3458) + - update VTune easyblock to work with v2024 and newer (#3465) + - enhance NAMD easyblock to support NAMD v3.0 (#3494) + - fix oneAPI sanity check for ifort removal in 2025.0 and newer (#3495) +- various other enhancements: + - set CMake installation `LIBDIR` to `lib` by default in `CMakeMake` easyblock (#3227) + - add warning if `.Renviron` detected during install of RPackage (#3263) + - make `PythonBundle` and `PythonPackage` aware of `--prefer-python-search-path` EasyBuild configuration option (#3343) + - enhance `LLVM` easyblock for compilation of clang/flang + other llvm-projects (#3373, #3657) + - enhance `CMakeMake` easyblock to check whether correct `Python` installation was picked up by `CMake` (#3399) + - enhance Qt5 easyblock to support building on RISC-V (#3462) + - let `CMakeMake` easyblock also set `Python_EXECUTABLE` option, as well as `Python3_EXECUTABLE` and `Python2_EXECUTABLE` derivatives (when appropriate) (#3463) + - update ROOT easyblock to support sysroot (#3467) + - enhance generic `Bundle` easyblock to transfer module requirements of components, but do not create logfile in components (#3472, #3504, #3509) + - enhance cuDNN easyblock to verify that EULA is accepted before installing it (#3473) + - enhance AOCC easyblock to correctly pass GCC toolchain and compiler driver (#3480) + - enhance Cargo easyblock for sources from git repositories (#3483, #3654) + - enhance Cargo easyblock to print message for generating `Cargo.lock` if it's missing (#3491) + - let `PythonPackage` easyblock fix python shebangs by default (#3499) + - make it possible to disable single precision as build target in GROMACS easyblock (#3501) + - pass netCDF-Fortran path via `$NETCDFF_DIR` in WPS easyblock (#3522) + - set `CMAKE_CUDA_HOST_COMPILER`, `CMAKE_CUDA_COMPILER`, and `CMAKE_CUDA_ARCHITECTURES` when using CUDA in `CMakeMake` easyblock (#3523) + - show path of output file produced by MATLAB installer (#3532) + - make path to temporary install directory used in test step of `PythonPackage` easyblock available through environment variable so that it may be used in easyconfigs (#3565) + - add support for Sapphire Rapids in LAMMPS (#3569) + - allow installing from commit for LAMMPS (#3582) + - add support to OpenFOAM easyblock for custom `sanity_check_motorbike` easyconfig parameter to opt out of running motorBike tutorial example during sanity check (#3595) + - enhance FlexiBLAS easyblock to add support for AOCL-BLAS backend (#3589, #3605) + - enhance handling of `PETSC_ARCH` in SLEPc easyblock (#3629) + - use unittest XML files to parse PyTorch test results (#3633) +- various bug fixes: + - don't change `installopts` easyconfig parameter value in-place in `PythonPackage` easyblock (#3080) + - fail on non-zero exit code in command to run Perl test suite (#3170) + - fix `--module-only` + cleanup for custom easyblock for Geant4 (#3302) + - consider both `easybuild-easyblocks*.tar.gz` and `easybuild_easyblocks*tar.gz` in CI workflows (#3309) + - make `CMakeMake` respect `debug` and `noopt` toolchain options when selecting build type (#3452) + - make `MesonNinja` respect the `debug`/`noopt`/`lowopt`/`opt` toolchain options to correctly set build type and use `--debug` and `--optimization` flags (#3454) + - strip leading `local/` from pylibdir where appropriate in `PythonPackage` easyblock (#3464) + - remove X11 flag for GROMACS 2023+ (#3466) + - fix setting of `CMAKE_OPTIONS` for SuiteSparse versions between 5.1.2 and 6.0.0 (#3471) + - add missing `return` for customised `install_extension_async` methods in Rserve and Rmpi easyblocks (#3474) + - disable CPU-specific optimizations for generic builds (by using a custom `processor_arch`) in LAMMPS easyblock (#3484) + - also add path to `libtorch.so` & co to `$LIBRARY_PATH` in generated module file for PyTorch (#3488) + - use `*DESCRIPTION` rather than `*/DESCRIPTION` in `RPackage.requires_deps` (#3490) + - use `DYNAMIC_ARCH=1` when building OpenBLAS with `--optarch=GENERIC` (#3492) + - fix check in Score-P's configure scripts that may fail if the path to certain dependencies include `yes` or `no` (#3496) + - let internal easyblock not create a log file in QuantumESPRESSO easyblock (#3505) + - update CUDA easyblock to add CUPTI and nvvm library directories to `$LIBRARY_PATH` (#3516) + - force use of `bash` for `Allwmake` scripts in OpenFOAM easyblock (#3519) + - fix detection of math library in numpy build (#3520) + - retain custom easyconfig parameters supported by `MesonNinja` easyblock in custom easyblock for `scipy` (#3526) + - inject custom '`%(python)s`' template value before getting value of '`buildcmd`' custom easyconfig parameter in `PythonPackage` easyblock (#3539) + - import from other easyblocks without the single folder letter (#3543) + - fix `$PYTHONPATH` for hermetic Python in TensorFlow builds (#3568) + - fix building PyTorch when using `setup.py` as the build command (#3574) + - set Cargo variables also for extensions (#3576) + - use bash for compiler wrapper in AOCC easyblock (#3648) + - fix compatibility of Molpro easyblock with `--module-only` (#3615) + - specify tmpdir for ANSYS installations (#3646) + - update + fixes in GitHub Actions workflows to use Ubuntu 22.04 (#3651, #3655) + - make AOCC compiler wrappers aware of keepsymlinks option (#3659) + - convert RUNPATH into RPATH for Rust binaries in recent Rust versions (> 1.79.0) (#3660) + - update configuration options for version 4+ of MPICH (#3663) +- various minor changes: + - don't auto-enable use of `-DSCOTCH_PTHREAD` when using MPI library other than Intel MPI in SCOTCH easyblock (#3019) + - replace `log.warn` with `log.warning` (#3245) + - refactor search for `python_cmd` in `PythonPackage` and `PythonBundle` easyblocks (#3475) + - update `Python` easyblock to move `sitecustomize.py` into `site-packages` (#3514) + - switch default to `install_pip=True` in custom Python easyblock (#3546) + - use context managers for templating changes in `Bundle` easyblock (#3547) + - use `PythonPackage` as the `default_easyblock` for `PythonBundle` (#3649) +- cleanup of dead code + - drop support for versions < 2.0 in custom easyblock for ScaLAPACK (#3106) + - drop support for versions < 9.5 in custom easyblock for Geant4 (#3296) + - remove code supporting obsolete versions from `IntelBase` family of easyblocks (#3533) + - clean up custom easyblock for Paraver, only support Paravar >= v4.7 (#3535) + - set minimum supported version of PETSc to v3.9 (#3627) +- remove unused software-specific easyblocks for ACML (#3172), ALADIN (#3173), Allinea (#3423), ARB (#3096), ATLAS (#3103), BamTools (#3424), Bioconductor (#3423), BiSearch (#3174), BLACS (#3104), Blender (#3423), BWISE (#3423), CFDEMcoupling (#3151), Chapel (#3423), CHARMM (#3108), cppcheck (#3111), DL_POLY_Classic (#3152), DOLFIN (#3154), Doris (#3175), Doxygen (#3114), EggLib (#3066), EPD (#3423), ESPResSo (#3117), fastStructure (#3155), FoldX (#3423), FreeFEM (#3157), HEALPix (#3285), IMOD (#3126), IPP (#3533), IronPython (#3125), libsmm (#3216), MetaVelvet (#3423), Modeller (#3292), Mono (#3211), Mothur (#3423), MSM (#3208), MTL4 (#3617), MyMediaLite (#3205), mutil (#3423), MVAPICH2 (#3423), ncurses (#3265), NEMO (#3202), OpenIFS (#3194), Pasha (#3423), pbdMPI (#3423), pbdSLAP (#3423), PGI (#3181), picard (#3423), pplacer (#3185), Primer3 (#3621), PyQuante (#3423), python_meep (#3182), Samcef (#3176), SAS (#3177), Scalasca v1.x (#3423), SHRiMP (#3423), SNPhylo (#3178), SOAPdenovo (#3423), TAU (#3535), TotalView (#3191), UFC (#3423), VSC-tools (#3138), WRF-Fire (#3271) + + +v4.9.4 (22 September 2024) -------------------------- update/bugfix release diff --git a/easybuild/__init__.py b/easybuild/__init__.py index 8d496189c34..d0005a450d9 100644 --- a/easybuild/__init__.py +++ b/easybuild/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/__init__.py b/easybuild/easyblocks/__init__.py index 7b5379a9516..ed02d77e509 100644 --- a/easybuild/easyblocks/__init__.py +++ b/easybuild/easyblocks/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -32,7 +32,6 @@ @author: Jens Timmerman (Ghent University) """ import os -from distutils.version import LooseVersion from pkgutil import extend_path # note: release candidates should be versioned as a pre-release, e.g. "1.1rc1" @@ -43,7 +42,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.9.4') +VERSION = '5.0.0' UNKNOWN = 'UNKNOWN' @@ -76,7 +75,7 @@ def get_git_revision(): if git_rev == UNKNOWN: VERBOSE_VERSION = VERSION else: - VERBOSE_VERSION = LooseVersion("%s-r%s" % (VERSION, git_rev)) + VERBOSE_VERSION = "%s-r%s" % (VERSION, git_rev) # extend path so python finds our easyblocks in the subdirectories where they are located subdirs = [chr(x) for x in range(ord('a'), ord('z') + 1)] + ['0'] diff --git a/easybuild/easyblocks/a/abaqus.py b/easybuild/easyblocks/a/abaqus.py index 8bb0ec72806..497220970fe 100644 --- a/easybuild/easyblocks/a/abaqus.py +++ b/easybuild/easyblocks/a/abaqus.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -34,18 +34,17 @@ """ import glob import os -from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.binary import Binary from easybuild.framework.easyblock import EasyBlock from easybuild.framework.easyconfig import CUSTOM +from easybuild.tools import LooseVersion from easybuild.tools.build_log import EasyBuildError from easybuild.tools.environment import setvar from easybuild.tools.filetools import change_dir, symlink, write_file from easybuild.tools.modules import get_software_root -from easybuild.tools.run import run_cmd_qa +from easybuild.tools.run import run_shell_cmd from easybuild.tools.systemtools import get_os_name -from easybuild.tools.py2vs3 import OrderedDict class EB_ABAQUS(Binary): @@ -72,6 +71,12 @@ def __init__(self, *args, **kwargs): self.log.info("Auto-enabling installation of fe-safe components for ABAQUS versions >= 2020") self.cfg['with_fe_safe'] = True + # add custom paths to $PATH + path_subdirs = ['Commands'] + if self.cfg['with_tosca']: + path_subdirs.append(os.path.join('cae', 'linux_a64', 'code', 'command')) + self.module_load_environment.PATH.extend(path_subdirs) + def extract_step(self): """Use default extraction procedure instead of the one for the Binary easyblock.""" EasyBlock.extract_step(self) @@ -107,14 +112,14 @@ def install_step(self): """Install ABAQUS using 'setup'.""" if LooseVersion(self.version) >= LooseVersion('2016'): change_dir(os.path.join(self.cfg['start_dir'], '1')) - qa = { - "Enter selection (default: Install):": '', - "Enter selection (default: Close):": '', - } + qa = [ + (r"Enter selection \(default: Install\):", ''), + (r"Enter selection \(default: Close\):", ''), + ] no_qa = [ - '___', + r"___", r"\(\d+\s*[KM]B\)", - r'\.\.\.$', + r"\.\.\.$", ] # Match string for continuing on with the selected items @@ -123,95 +128,93 @@ def install_step(self): # Allow for selection or deselection of components from lines of the form: # 5 [*] Tosca Fluid # This uses nextstr to make sure we only match the latest output in the Q&A process; - # negative lookahead (?!___) is used to exclude ___...___ lines, to avoid matching across questions - selectionstr = r"\s*(?P[-0-9]+) %%s %%s[ \w]*\n((?!%s)(?!___)[\S ]*\n)*%s$" % (nextstr, nextstr) + # negative lookahead (?!___) is used to exclude ___...___ lines, to avoid matching across questions; + # *? to use non-greedy matching in combination with negative lookahead + # (to avoid excessive backtracking by regex engine) + selectionstr = r"\s*(?P[-0-9]+) %%s %%s.*\n((?!%s)(?!___).*\n)*?%s" % (nextstr, nextstr) - # ensure question patterns are considered in order by using an OrderedDict instance, - # rather than a regular dictionary (where there's no guarantee on key order in general) - std_qa = OrderedDict() - - # Disable Extended Product Documentation because it has a troublesome Java dependency - std_qa[selectionstr % (r"\[*\]", "Extended Product Documentation")] = "%(nr)s" installed_docs = False # hard disabled, previous support was actually incomplete - # enable all ABAQUS components - std_qa[selectionstr % (r"\[ \]", "Abaqus.*")] = "%(nr)s" - std_qa[selectionstr % (r"\[ \]", "Cosimulation Services")] = "%(nr)s" - - # enable 3DSFlow Solver (used to be called "Abaqus/CFD Solver") - std_qa[selectionstr % (r"\[ \]", "3DSFlow Solver")] = "%(nr)s" + qa.extend([ + # disable Extended Product Documentation because it has a troublesome Java dependency + (selectionstr % (r"\[*\]", "Extended Product Documentation"), '%(nr)s'), + # enable all ABAQUS components + (selectionstr % (r"\[ \]", "Abaqus"), '%(nr)s'), + (selectionstr % (r"\[ \]", "Cosimulation Services"), '%(nr)s'), + # enable 3DSFlow Solver (used to be called "Abaqus/CFD Solver") + (selectionstr % (r"\[ \]", "3DSFlow Solver"), '%(nr)s'), + ]) # disable/enable fe-safe components if self.cfg['with_fe_safe']: - std_qa[selectionstr % (r"\[ \]", ".*fe-safe")] = "%(nr)s" + qa.append((selectionstr % (r"\[ \]", ".*fe-safe"), '%(nr)s')) else: - std_qa[selectionstr % (r"\[*\]", ".*fe-safe")] = "%(nr)s" + qa.append((selectionstr % (r"\[\*\]", ".*fe-safe"), '%(nr)s')) # Disable/enable Tosca if self.cfg['with_tosca']: - std_qa[selectionstr % (r"\[\ \]", "Tosca.*")] = "%(nr)s" + qa.append((selectionstr % (r"\[ \]", "Tosca.*"), '%(nr)s')) else: - std_qa[selectionstr % (r"\[\*\]", "Tosca.*")] = "%(nr)s" - - # disable CloudView - std_qa[r"(?P[0-9]+) \[X\] Search using CloudView\nEnter selection:"] = '%(cloudview)s\n\n' - # accept default port for documentation server - std_qa[r"Check that the port is free.\nDefault \[[0-9]+\]:"] = '\n' - - # disable feedback by users - std_qa[r"(?P[0-9]+) \[X\] Allow users to send feedback.\nEnter selection:"] = '%(feedback)s\n\n' - - # disable reverse proxy - std_qa[r"(?P[0-9]+) \[X\] Use a reverse proxy.\nEnter selection:"] = '%(proxy)s\n\n' - - # Disable Isight - std_qa[selectionstr % (r"\[\*\]", "Isight")] = "%(nr)s" - # Disable Search using EXALEAD - std_qa[r"\s*(?P[0-9]+) \[X\] Search using EXALEAD\nEnter selection:"] = '%(exalead)s\n\n' + qa.append((selectionstr % (r"\[\*\]", "Tosca.*"), '%(nr)s')) + + qa.extend([ + # disable CloudView + (r"(?P[0-9]+) \[X\] Search using CloudView\nEnter selection:", '%(cloudview)s\n\n'), + # accept default port for documentation server + (r"Check that the port is free.\nDefault \[[0-9]+\]:", '\n'), + # disable feedback by users + (r"(?P[0-9]+) \[X\] Allow users to send feedback.\nEnter selection:", '%(feedback)s\n\n'), + # disable reverse proxy + (r"(?P[0-9]+) \[X\] Use a reverse proxy.\nEnter selection:", '%(proxy)s\n\n'), + # Disable Isight + (selectionstr % (r"\[\*\]", "Isight"), '%(nr)s'), + # Disable Search using EXALEAD + (r"\s*(?P[0-9]+) \[X\] Search using EXALEAD\nEnter selection:", '%(exalead)s\n\n'), + ]) # Directories cae_subdir = os.path.join(self.installdir, 'cae') sim_subdir = os.path.join(self.installdir, 'sim') - std_qa[r"Default.*SIMULIA/EstProducts.*:"] = cae_subdir - std_qa[r"SIMULIA[0-9]*doc.*:"] = os.path.join(self.installdir, 'doc') # if docs are installed - std_qa[r"SimulationServices.*:"] = sim_subdir - std_qa[r"Choose the CODE installation directory.*:\n.*\n\n.*:"] = sim_subdir - std_qa[r"SIMULIA/CAE.*:"] = cae_subdir - std_qa[r"location of your Abaqus services \(solvers\).*(\n.*){8}:\s*"] = sim_subdir - std_qa[r"Default.*SIMULIA/Commands\]:\s*"] = os.path.join(self.installdir, 'Commands') - std_qa[r"Default.*SIMULIA/CAE/plugins.*:\s*"] = os.path.join(self.installdir, 'plugins') - std_qa[r"Default.*SIMULIA/Isight.*:\s*"] = os.path.join(self.installdir, 'Isight') - std_qa[r"Default.*SIMULIA/fe-safe/.*:"] = os.path.join(self.installdir, 'fe-safe') - std_qa[r"Default.*SIMULIA/Tosca.*:"] = os.path.join(self.installdir, 'tosca') - - # paths to STAR-CCM+, FLUENT are requested when Tosca is also installed; - # these do not strictly need to be specified at installation time, so we don't - std_qa[r"STAR-CCM.*\n((?!___)[\S ]*\n)*\nDefault \[\]:"] = '' - std_qa[r"FLUENT.*\n((?!___)[\S ]*\n)*\nDefault \[\]:"] = '' - - std_qa[r"location of your existing ANSA installation.*(\n.*){8}:"] = '' - std_qa[r"FLUENT Path.*(\n.*){7}:"] = '' - std_qa[r"working directory to be used by Tosca Fluid\s*(\n.*)*Default \[/usr/temp\]:\s*"] = '/tmp' - - # License server - std_qa[r"License Server [0-9]+\s*(\n.*){3}:"] = 'abaqusfea' # bypass value for license server - std_qa[r"License Server . \(redundant\)\s*(\n.*){3}:"] = '' - std_qa[r"License Server Configuration((?!___).*\n)*" + nextstr] = '' - - std_qa[r"Please choose an action:"] = '1' + qa.extend([ + (r"Default.*SIMULIA/EstProducts.*:", cae_subdir), + (r"SIMULIA[0-9]*doc.*:", os.path.join(self.installdir, 'doc')), # if docs are installed + (r"SimulationServices.*:", sim_subdir), + (r"Choose the CODE installation directory.*:\n.*\n\n.*:", sim_subdir), + (r"SIMULIA/CAE.*:", cae_subdir), + (r"location of your Abaqus services \(solvers\).*(\n.*){8}:\s*", sim_subdir), + (r"Default.*SIMULIA/Commands\]:\s*", os.path.join(self.installdir, 'Commands')), + (r"Default.*SIMULIA/CAE/plugins.*:\s*", os.path.join(self.installdir, 'plugins')), + (r"Default.*SIMULIA/Isight.*:\s*", os.path.join(self.installdir, 'Isight')), + (r"Default.*SIMULIA/fe-safe/.*:", os.path.join(self.installdir, 'fe-safe')), + (r"Default.*SIMULIA/Tosca.*:", os.path.join(self.installdir, 'tosca')), + # paths to STAR-CCM+, FLUENT are requested when Tosca is also installed; + # these do not strictly need to be specified at installation time, so we don't + (r"STAR-CCM.*\n((?!___).*\n)*?\nDefault \[\]:", ''), + (r"FLUENT.*\n((?!___).*\n)*?\nDefault \[\]:", ''), + (r"location of your existing ANSA installation.*(\n.*){8}:", ''), + (r"FLUENT Path.*(\n.*){7}:", ''), + (r"working directory to be used by Tosca Fluid\s*(\n.*)*Default \[/usr/temp\]:\s*", '/tmp'), + # License server + (r"License Server [0-9]+\s*(\n.*){3}:", 'abaqusfea'), # bypass value for license server + (r"License Server . \(redundant\)\s*(\n.*){3}:", ''), + (r"License Server Configuration((?!___).*\n)*?" + nextstr, ''), + (r"Please choose an action:", '1'), + ]) if LooseVersion(self.version) >= LooseVersion('2022') and installed_docs: java_root = get_software_root('Java') if java_root: - std_qa[r"Please enter .*Java Runtime Environment.* path.(\n.*)+Default \[\]:"] = java_root - std_qa[r"Please enter .*Java Runtime Environment.* path.(\n.*)+Default \[.+\]:"] = '' + qa.extend([ + (r"Please enter .*Java Runtime Environment.* path.(\n.*)+Default \[\]:", java_root), + (r"Please enter .*Java Runtime Environment.* path.(\n.*)+Default \[.+\]:", ''), + ]) else: raise EasyBuildError("Java is required for ABAQUS docs versions >= 2022, but it is missing") # Continue - std_qa[nextstr] = '' + qa.append((nextstr, '')) - run_cmd_qa('./StartTUI.sh', qa, no_qa=no_qa, std_qa=std_qa, log_all=True, simple=True, maxhits=1000) + run_shell_cmd('./StartTUI.sh', qa_patterns=qa, qa_wait_patterns=no_qa, qa_timeout=1000) else: change_dir(self.builddir) if self.cfg['install_cmd'] is None: @@ -240,11 +243,12 @@ def install_step(self): raise EasyBuildError("Failed to find expected subdir for hotfix: %s", subdirs) cwd = change_dir(os.path.join(subdir, '1')) - std_qa = OrderedDict() - std_qa[r"Enter selection \(default: Next\):"] = '' - std_qa["Choose the .*installation directory.*\n.*\n\n.*:"] = os.path.join(self.installdir, 'sim') - std_qa[r"Enter selection \(default: Install\):"] = '' - run_cmd_qa('./StartTUI.sh', {}, std_qa=std_qa, log_all=True, simple=True, maxhits=100) + qa = [ + (r"Enter selection \(default: Next\):", ''), + (r"Choose the .*installation directory.*\n.*\n\n.*:", os.path.join(self.installdir, 'sim')), + (r"Enter selection \(default: Install\):", ''), + ] + run_shell_cmd('./StartTUI.sh', qa_patterns=qa, qa_wait_patterns=no_qa, qa_timeout=1000) # F_CAASIMULIAComputeServicesBuildTime part change_dir(cwd) @@ -255,7 +259,7 @@ def install_step(self): raise EasyBuildError("Failed to find expected subdir for hotfix: %s", subdirs) cwd = change_dir(os.path.join(cwd, subdir, '1')) - run_cmd_qa('./StartTUI.sh', {}, std_qa=std_qa, log_all=True, simple=True, maxhits=100) + run_shell_cmd('./StartTUI.sh', qa_patterns=qa, qa_wait_patterns=no_qa, qa_timeout=1000) change_dir(cwd) # next install Part_SIMULIA_Abaqus_CAE hotfix (ABAQUS versions <= 2020) @@ -270,14 +274,15 @@ def install_step(self): raise EasyBuildError("Failed to find expected subdir for hotfix: %s", subdirs) cwd = change_dir(os.path.join(subdir, '1')) - std_qa = OrderedDict() - std_qa[r"Enter selection \(default: Next\):"] = '' - std_qa["Choose the .*installation directory.*\n.*\n\n.*:"] = os.path.join(self.installdir, 'cae') - std_qa[r"Enter selection \(default: Install\):"] = '' - std_qa[r"\[1\] Continue\n(?:.|\n)*Please choose an action:"] = '1' - std_qa[r"\[2\] Continue\n(?:.|\n)*Please choose an action:"] = '2' + qa = [ + (r"Enter selection \(default: Next\):", ''), + (r"Choose the .*installation directory.*\n.*\n\n.*:", os.path.join(self.installdir, 'cae')), + (r"Enter selection \(default: Install\):", ''), + (r"\[1\] Continue\n(?:.|\n)*Please choose an action:", '1'), + (r"\[2\] Continue\n(?:.|\n)*Please choose an action:", '2'), + ] no_qa = [r"Please be patient; it will take a few minutes to complete\.\n(\.)*"] - run_cmd_qa('./StartTUI.sh', {}, no_qa=no_qa, std_qa=std_qa, log_all=True, simple=True, maxhits=100) + run_shell_cmd('./StartTUI.sh', qa_patterns=qa, qa_wait_patterns=no_qa, qa_timeout=1000) change_dir(cwd) # install SIMULIA Established Products hotfix (ABAQUS versions > 2020) @@ -294,18 +299,19 @@ def install_step(self): cwd = change_dir(os.path.join(subdir, '1')) no_qa = [ - '___', - '...', + r'___', + r'\.\.\.', r'\(\d+[KM]B\)', ] - std_qa = OrderedDict() - std_qa[r"Enter selection \(default: Next\):"] = '' - std_qa["Choose the .*installation directory.*\n.*\n\n.*:"] = os.path.join(self.installdir, 'cae') - std_qa[r"Enter selection \(default: Install\):"] = '' - std_qa[r"The Abaqus commands directory.*:\n.*\n+Actions:\n.*\n_+\n\nPlease.*:"] = '1' - std_qa[r"Enter selection \(default: Close\):"] = '' - - run_cmd_qa('./StartTUI.sh', {}, no_qa=no_qa, std_qa=std_qa, log_all=True, simple=True, maxhits=100) + qa = [ + (r"Enter selection \(default: Next\):", ''), + (r"Choose the .*installation directory.*\n.*\n\n.*:", os.path.join(self.installdir, 'cae')), + (r"Enter selection \(default: Install\):", ''), + (r"The Abaqus commands directory.*:\n.*\n+Actions:\n.*\n_+\n\nPlease.*:", '1'), + (r"Enter selection \(default: Close\):", ''), + ] + + run_shell_cmd('./StartTUI.sh', qa_patterns=qa, qa_wait_patterns=no_qa, qa_timeout=1000) change_dir(cwd) # create 'abaqus' symlink for main command, which is not there anymore starting with ABAQUS 2022 @@ -353,20 +359,6 @@ def sanity_check_step(self): super(EB_ABAQUS, self).sanity_check_step(custom_paths=custom_paths, custom_commands=custom_commands) - def make_module_req_guess(self): - """Update $PATH guesses for ABAQUS.""" - guesses = super(EB_ABAQUS, self).make_module_req_guess() - - path_subdirs = ['Commands'] - if self.cfg['with_tosca']: - path_subdirs.append(os.path.join('cae', 'linux_a64', 'code', 'command')) - - guesses.update({ - 'PATH': path_subdirs, - }) - - return guesses - def make_module_extra(self): """Add LM_LICENSE_FILE path if specified""" txt = super(EB_ABAQUS, self).make_module_extra() diff --git a/easybuild/easyblocks/a/acml.py b/easybuild/easyblocks/a/acml.py deleted file mode 100644 index 296c49003ec..00000000000 --- a/easybuild/easyblocks/a/acml.py +++ /dev/null @@ -1,135 +0,0 @@ -## -# Copyright 2009-2024 Ghent University -# -# This file is part of EasyBuild, -# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), -# with support of Ghent University (http://ugent.be/hpc), -# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), -# Flemish Research Foundation (FWO) (http://www.fwo.be/en) -# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). -# -# https://github.com/easybuilders/easybuild -# -# EasyBuild is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation v2. -# -# EasyBuild is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with EasyBuild. If not, see . -## -""" -EasyBuild support for AMD Core Math Library (ACML), implemented as an easyblock - -@author: Stijn De Weirdt (Ghent University) -@author: Dries Verdegem (Ghent University) -@author: Kenneth Hoste (Ghent University) -@author: Pieter De Baets (Ghent University) -@author: Jens Timmerman (Ghent University) -""" - -import os -from easybuild.tools import LooseVersion - -from easybuild.framework.easyblock import EasyBlock -from easybuild.framework.easyconfig import CUSTOM -from easybuild.tools.run import run_cmd_qa -from easybuild.tools.systemtools import get_shared_lib_ext - - -class EB_ACML(EasyBlock): - - @staticmethod - def extra_options(): - extra_vars = { - 'use_fma4': [False, "Use library with FMA support.", CUSTOM], - } - return EasyBlock.extra_options(extra_vars) - - def __init__(self, *args, **kwargs): - """Constructor, adds extra class variables.""" - super(EB_ACML, self).__init__(*args, **kwargs) - - # determine base directory and suffix for ACML installation - vsuff_list = self.cfg['versionsuffix'].split('-') - self.basedir = '' - if len(vsuff_list) >= 3: - comp = vsuff_list[1] # gfortran, ifort, ... - bits = vsuff_list[2] # 32bit or 64bit - self.basedir += comp + bits[:2] - # specialized suffix, e.g., _fma4 for fused multiply-add - if LooseVersion(self.version) >= LooseVersion("5") and self.cfg['use_fma4']: - self.basedir += '_fma4' - - self.suffix = '' - if self.cfg['versionsuffix'].split('-')[-1] == "int64": - self.suffix = '_int64' - - def configure_step(self): - """No custom configure step for ACML.""" - pass - - def build_step(self): - """No custom build step for ACML.""" - pass - - def install_step(self): - """Install by running install script.""" - - altver = '-'.join(self.version.split('.')) - cmd = "./install-%s-%s%s.sh -accept" % (self.name.lower(), altver, self.cfg['versionsuffix']) - - qa = {'The directory will be created if it does not already exist. >': self.installdir} - - run_cmd_qa(cmd, qa, log_all=True, simple=True) - - def make_module_extra(self): - """Add extra entries in module file (various paths).""" - - txt = super(EB_ACML, self).make_module_extra() - - basepaths = ["%s%s" % (self.basedir, self.suffix), - "%s_mp%s" % (self.basedir, self.suffix)] - - txt += self.module_generator.set_environment('ACML_BASEDIR', basepaths[0]) - txt += self.module_generator.set_environment('ACML_BASEDIR_MT', basepaths[1]) - - for path in basepaths: - txt += self.module_generator.prepend_paths('CPATH', os.path.join(path, 'include')) - - for key in ['LD_LIBRARY_PATH', 'LIBRARY_PATH']: - for path in basepaths: - txt += self.module_generator.prepend_paths(key, os.path.join(path, 'lib')) - - return txt - - def sanity_check_step(self): - """Custom sanity check for ACML.""" - - inc_extra = [] - lib_extra = [] - if LooseVersion(self.version) < LooseVersion("5"): - inc_extra = ['_mv', '_mv_m128'] - lib_extra = ['_mv'] - - inc_files = [] - lib_files = [] - for suff in ['', '_mp']: - - fp = "%s%s%s" % (self.basedir, suff, self.suffix) - for inc in [''] + inc_extra: - inc_files.append(os.path.join(fp, 'include', 'acml%s.h' % inc)) - - for lib in [suff] + lib_extra: - for ext in ['a', get_shared_lib_ext()]: - lib_files.append(os.path.join(fp, 'lib', 'libacml%s.%s' % (lib, ext))) - - custom_paths = { - 'files': ['util/cpuid.exe'] + inc_files + lib_files, - 'dirs': [], - } - super(EB_ACML, self).sanity_check_step(custom_paths=custom_paths) diff --git a/easybuild/easyblocks/a/adf.py b/easybuild/easyblocks/a/adf.py index cc96cb86b44..7a35f73c889 100644 --- a/easybuild/easyblocks/a/adf.py +++ b/easybuild/easyblocks/a/adf.py @@ -1,5 +1,5 @@ ## -# Copyright 2016-2024 Ghent University +# Copyright 2016-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -33,7 +33,7 @@ import easybuild.tools.environment as env from easybuild.framework.easyblock import EasyBlock from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd class EB_ADF(EasyBlock): @@ -63,7 +63,7 @@ def configure_step(self): raise EasyBuildError("No or non-existing license file specified: %s", self.cfg['license_file']) cmd = './Install/configure' - run_cmd(cmd, log_all=True, simple=True, log_ok=True) + run_shell_cmd(cmd) def build_step(self): """No separate custom build procedure for ADF, see install step.""" @@ -80,8 +80,8 @@ def install_step(self): except OSError as err: raise EasyBuildError("Failed to copy %s to %s: %s", src_init_path, target_init_path, err) - cmd = "./bin/foray -j %d" % self.cfg['parallel'] - run_cmd(cmd, log_all=True, simple=True, log_ok=True) + cmd = f"./bin/foray -j {self.cfg.parallel}" + run_shell_cmd(cmd) def sanity_check_step(self): """Custom sanity check for ADF.""" diff --git a/easybuild/easyblocks/a/advisor.py b/easybuild/easyblocks/a/advisor.py index 2f24500b205..bc6c2791ea2 100644 --- a/easybuild/easyblocks/a/advisor.py +++ b/easybuild/easyblocks/a/advisor.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -34,33 +34,37 @@ from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.intelbase import IntelBase +from easybuild.tools.build_log import EasyBuildError class EB_Advisor(IntelBase): """ Support for installing Intel Advisor XE + - minimum version suported: 2020.x """ def __init__(self, *args, **kwargs): """Constructor, initialize class variables.""" super(EB_Advisor, self).__init__(*args, **kwargs) - if LooseVersion(self.version) < LooseVersion('2017'): - self.subdir = 'advisor_xe' - elif LooseVersion(self.version) < LooseVersion('2021'): + + if LooseVersion(self.version) < LooseVersion('2020'): + raise EasyBuildError( + f"Version {self.version} of {self.name} is unsupported. Mininum supported version is 2020.0." + ) + + if LooseVersion(self.version) < LooseVersion('2021'): self.subdir = 'advisor' else: self.subdir = os.path.join('advisor', 'latest') + # prepare module load environment + self.prepare_intel_tools_env() + def prepare_step(self, *args, **kwargs): """Since 2019u3 there is no license required.""" - if LooseVersion(self.version) >= LooseVersion('2019_update3'): - kwargs['requires_runtime_license'] = False + kwargs['requires_runtime_license'] = False super(EB_Advisor, self).prepare_step(*args, **kwargs) - def make_module_req_guess(self): - """Find reasonable paths for Advisor""" - return self.get_guesses_tools() - def sanity_check_step(self): """Custom sanity check paths for Advisor""" binaries = ['advixe-cl', 'advixe-feedback', 'advixe-gui', 'advixe-runss', 'advixe-runtrc', 'advixe-runtc'] diff --git a/easybuild/easyblocks/a/aedt.py b/easybuild/easyblocks/a/aedt.py index 53d04d506e8..4bdce936cc2 100644 --- a/easybuild/easyblocks/a/aedt.py +++ b/easybuild/easyblocks/a/aedt.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -33,7 +33,7 @@ from easybuild.easyblocks.generic.packedbinary import PackedBinary from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd class EB_AEDT(PackedBinary): @@ -69,10 +69,9 @@ def install_step(self): "-DSPECIFY_PORT=1", "-DLICENSE_PORT=%s" % licport, ]) - cmd = "./Linux/AnsysEM/Disk1/InstData/setup.exe %s" % options - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd("./Linux/AnsysEM/Disk1/InstData/setup.exe %s" % options) - def post_install_step(self): + def post_processing_step(self): """Disable OS check and set LC_ALL/LANG for runtime""" if not self.subdir: self._set_subdir() diff --git a/easybuild/easyblocks/a/aladin.py b/easybuild/easyblocks/a/aladin.py deleted file mode 100644 index 79454792fe6..00000000000 --- a/easybuild/easyblocks/a/aladin.py +++ /dev/null @@ -1,344 +0,0 @@ -## -# Copyright 2009-2024 Ghent University -# -# This file is part of EasyBuild, -# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), -# with support of Ghent University (http://ugent.be/hpc), -# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), -# Flemish Research Foundation (FWO) (http://www.fwo.be/en) -# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). -# -# https://github.com/easybuilders/easybuild -# -# EasyBuild is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation v2. -# -# EasyBuild is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with EasyBuild. If not, see . -## -""" -EasyBuild support for building and installing ALADIN, implemented as an easyblock - -@author: Kenneth Hoste (Ghent University) -""" -import fileinput -import os -import re -import shutil -import sys -import tempfile - -import easybuild.tools.environment as env -import easybuild.tools.toolchain as toolchain -from easybuild.framework.easyblock import EasyBlock -from easybuild.framework.easyconfig import CUSTOM -from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.filetools import apply_regex_substitutions, mkdir -from easybuild.tools.modules import get_software_root, get_software_libdir -from easybuild.tools.py2vs3 import OrderedDict -from easybuild.tools.run import run_cmd, run_cmd_qa - - -class EB_ALADIN(EasyBlock): - """Support for building/installing ALADIN.""" - - def __init__(self, *args, **kwargs): - """Initialisation of custom class variables for ALADIN.""" - super(EB_ALADIN, self).__init__(*args, **kwargs) - - self.conf_file = None - self.conf_filepath = None - self.rootpack_dir = 'UNKNOWN' - - self.orig_library_path = None - - @staticmethod - def extra_options(): - """Custom easyconfig parameters for ALADIN.""" - - extra_vars = { - 'optional_extra_param': ['default value', "short description", CUSTOM], - } - return EasyBlock.extra_options(extra_vars) - - def configure_step(self): - """Custom configuration procedure for ALADIN.""" - - # unset $LIBRARY_PATH set by modules of dependencies, because it may screw up linking - if 'LIBRARY_PATH' in os.environ: - self.log.debug("Unsetting $LIBRARY_PATH (was: %s)" % os.environ['LIBRARY_PATH']) - self.orig_library_path = os.environ.pop('LIBRARY_PATH') - - # build auxiliary libraries - auxlibs_dir = None - - my_gnu = None - if self.toolchain.comp_family() == toolchain.GCC: - my_gnu = 'y' # gfortran - for var in ['CFLAGS', 'CXXFLAGS', 'F90FLAGS', 'FFLAGS']: - flags = os.getenv(var) - env.setvar(var, "%s -fdefault-real-8 -fdefault-double-8" % flags) - self.log.info("Updated %s to '%s'" % (var, os.getenv(var))) - elif self.toolchain.comp_family() == toolchain.INTELCOMP: - my_gnu = 'i' # icc/ifort - else: - raise EasyBuildError("Don't know how to set 'my_gnu' variable in auxlibs build script.") - self.log.info("my_gnu set to '%s'" % my_gnu) - - tmp_installroot = tempfile.mkdtemp(prefix='aladin_auxlibs_') - - try: - cwd = os.getcwd() - - os.chdir(self.builddir) - builddirs = os.listdir(self.builddir) - - auxlibs_dir = [x for x in builddirs if x.startswith('auxlibs_installer')][0] - - os.chdir(auxlibs_dir) - - auto_driver = 'driver_automatic' - for line in fileinput.input(auto_driver, inplace=1, backup='.orig.eb'): - - line = re.sub(r"^(my_gnu\s*=\s*).*$", r"\1%s" % my_gnu, line) - line = re.sub(r"^(my_r32\s*=\s*).*$", r"\1n", line) # always 64-bit real precision - line = re.sub(r"^(my_readonly\s*=\s*).*$", r"\1y", line) # make libs read-only after build - line = re.sub(r"^(my_installroot\s*=\s*).*$", r"\1%s" % tmp_installroot, line) - - sys.stdout.write(line) - - run_cmd("./%s" % auto_driver) - - os.chdir(cwd) - - except OSError as err: - raise EasyBuildError("Failed to build ALADIN: %s", err) - - # build gmkpack, update PATH and set GMKROOT - # we build gmkpack here because a config file is generated in the gmkpack isntall path - try: - gmkpack_dir = [x for x in builddirs if x.startswith('gmkpack')][0] - os.chdir(os.path.join(self.builddir, gmkpack_dir)) - - qa = { - 'Do you want to run the configuration file maker assistant now (y) or later [n] ?': 'n', - } - - run_cmd_qa("./build_gmkpack", qa) - - os.chdir(cwd) - - paths = os.getenv('PATH').split(':') - paths.append(os.path.join(self.builddir, gmkpack_dir, 'util')) - env.setvar('PATH', ':'.join(paths)) - - env.setvar('GMKROOT', os.path.join(self.builddir, gmkpack_dir)) - - except OSError as err: - raise EasyBuildError("Failed to build gmkpack: %s", err) - - # generate gmkpack configuration file - self.conf_file = 'ALADIN_%s' % self.version - self.conf_filepath = os.path.join(self.builddir, 'gmkpack_support', 'arch', '%s.x' % self.conf_file) - - try: - if os.path.exists(self.conf_filepath): - os.remove(self.conf_filepath) - self.log.info("Removed existing gmpack config file %s" % self.conf_filepath) - - archdir = os.path.dirname(self.conf_filepath) - if not os.path.exists(archdir): - mkdir(archdir, parents=True) - - except OSError as err: - raise EasyBuildError("Failed to remove existing file %s: %s", self.conf_filepath, err) - - mpich = 'n' - known_mpi_libs = [toolchain.MPICH, toolchain.MPICH2, toolchain.INTELMPI] - if self.toolchain.options.get('usempi', None) and self.toolchain.mpi_family() in known_mpi_libs: - mpich = 'y' - - qpref = 'Please type the ABSOLUTE name of ' - qsuff = ', or ignore (environment variables allowed) :' - qsuff2 = ', or ignore : (environment variables allowed) :' - - comp_fam = self.toolchain.comp_family() - if comp_fam == toolchain.GCC: - gribdir = 'GNU' - elif comp_fam == toolchain.INTELCOMP: - gribdir = 'INTEL' - else: - raise EasyBuildError("Don't know which grib lib dir to use for compiler %s", comp_fam) - - aux_lib_gribex = os.path.join(tmp_installroot, gribdir, 'lib', 'libgribex.a') - aux_lib_ibm = os.path.join(tmp_installroot, gribdir, 'lib', 'libibmdummy.a') - grib_api_lib = os.path.join(get_software_root('grib_api'), 'lib', 'libgrib_api.a') - grib_api_f90_lib = os.path.join(get_software_root('grib_api'), 'lib', 'libgrib_api_f90.a') - grib_api_inc = os.path.join(get_software_root('grib_api'), 'include') - jasperlib = os.path.join(get_software_root('JasPer'), 'lib', 'libjasper.a') - mpilib = os.path.join(os.getenv('MPI_LIB_DIR'), os.getenv('MPI_LIB_SHARED')) - - # netCDF - netcdf = get_software_root('netCDF') - netcdf_fortran = get_software_root('netCDF-Fortran') - if netcdf: - netcdfinc = os.path.join(netcdf, 'include') - if netcdf_fortran: - netcdflib = os.path.join(netcdf_fortran, get_software_libdir('netCDF-Fortran'), 'libnetcdff.a') - else: - netcdflib = os.path.join(netcdf, get_software_libdir('netCDF'), 'libnetcdff.a') - if not os.path.exists(netcdflib): - raise EasyBuildError("%s does not exist", netcdflib) - else: - raise EasyBuildError("netCDF(-Fortran) not available") - - ldpaths = [ldflag[2:] for ldflag in os.getenv('LDFLAGS').split(' ')] # LDFLAGS have form '-L/path/to' - - lapacklibs = [] - for lib in os.getenv('LAPACK_STATIC_LIBS').split(','): - libpaths = [os.path.join(ldpath, lib) for ldpath in ldpaths] - lapacklibs.append([libpath for libpath in libpaths if os.path.exists(libpath)][0]) - lapacklib = ' '.join(lapacklibs) - blaslibs = [] - for lib in os.getenv('BLAS_STATIC_LIBS').split(','): - libpaths = [os.path.join(ldpath, lib) for ldpath in ldpaths] - blaslibs.append([libpath for libpath in libpaths if os.path.exists(libpath)][0]) - blaslib = ' '.join(blaslibs) - - qa = { - 'Do you want to run the configuration file maker assistant now (y) or later [n] ?': 'y', - 'Do you want to setup your configuration file for MPICH (y/n) [n] ?': mpich, - 'Please type the directory name where to find a dummy file mpif.h or ignore :': os.getenv('MPI_INC_DIR'), - '%sthe library gribex or emos%s' % (qpref, qsuff2): aux_lib_gribex, - '%sthe library ibm%s' % (qpref, qsuff): aux_lib_ibm, - '%sthe library grib_api%s' % (qpref, qsuff): grib_api_lib, - '%sthe library grib_api_f90%s' % (qpref, qsuff): grib_api_f90_lib, - '%sthe JPEG auxilary library if enabled by Grib_api%s' % (qpref, qsuff2): jasperlib, - '%sthe library netcdf%s' % (qpref, qsuff): netcdflib, - '%sthe library lapack%s' % (qpref, qsuff): lapacklib, - '%sthe library blas%s' % (qpref, qsuff): blaslib, - '%sthe library mpi%s' % (qpref, qsuff): mpilib, - '%sa MPI dummy library for serial executions, or ignore :' % qpref: '', - 'Please type the directory name where to find grib_api headers, or ignore :': grib_api_inc, - 'Please type the directory name where to find fortint.h or ignore :': '', - 'Please type the directory name where to find netcdf headers, or ignore :': netcdfinc, - 'Do you want to define CANARI (y/n) [y] ?': 'y', - 'Please type the name of the script file used to generate a preprocessed blacklist file, or ignore :': '', - 'Please type the name of the script file used to recover local libraries (gget), or ignore :': '', - 'Please type the options to tune the gnu compilers, or ignore :': os.getenv('F90FLAGS'), - } - - f90_seq = os.getenv('F90_SEQ') - if not f90_seq: - # F90_SEQ is only defined when usempi is enabled - f90_seq = os.getenv('F90') - - stdqa = OrderedDict([ - (r'Confirm library .* is .*', 'y'), # this one needs to be tried first! - (r'.*fortran 90 compiler name .*\s*:\n\(suggestions\s*: .*\)', f90_seq), - (r'.*fortran 90 compiler interfaced with .*\s*:\n\(suggestions\s*: .*\)', f90_seq), - (r'Please type the ABSOLUTE name of .*library.*, or ignore\s*[:]*\s*[\n]*.*', ''), - (r'Please .* to save this draft configuration file :\n.*', '%s.x' % self.conf_file), - ]) - - no_qa = [ - ".*ignored.", - ] - - env.setvar('GMKTMP', self.builddir) - env.setvar('GMKFILE', self.conf_file) - - run_cmd_qa("gmkfilemaker", qa, std_qa=stdqa, no_qa=no_qa) - - # set environment variables for installation dirs - env.setvar('ROOTPACK', os.path.join(self.installdir, 'rootpack')) - env.setvar('ROOTBIN', os.path.join(self.installdir, 'rootpack')) - env.setvar('HOMEPACK', os.path.join(self.installdir, 'pack')) - env.setvar('HOMEBIN', os.path.join(self.installdir, 'pack')) - - # patch config file to include right Fortran compiler flags - regex_subs = [(r"^(FRTFLAGS\s*=.*)$", r"\1 %s" % os.getenv('FFLAGS'))] - apply_regex_substitutions(self.conf_filepath, regex_subs) - - def build_step(self): - """No separate build procedure for ALADIN (see install_step).""" - - pass - - def test_step(self): - """Custom built-in test procedure for ALADIN.""" - - if self.cfg['runtest']: - cmd = "test-command" - run_cmd(cmd, simple=True, log_all=True, log_output=True) - - def install_step(self): - """Custom install procedure for ALADIN.""" - - try: - mkdir(os.getenv('ROOTPACK'), parents=True) - mkdir(os.getenv('HOMEPACK'), parents=True) - except OSError as err: - raise EasyBuildError("Failed to create rootpack dir in %s: %s", err) - - # create rootpack - [v1, v2] = self.version.split('_') - (out, _) = run_cmd("source $GMKROOT/util/berootpack && gmkpack -p master -a -r %s -b %s" % (v1, v2), - simple=False) - - packdir_regexp = re.compile(r"Creating main pack (.*) \.\.\.") - res = packdir_regexp.search(out) - if res: - self.rootpack_dir = os.path.join('rootpack', res.group(1)) - else: - raise EasyBuildError("Failed to determine rootpack dir.") - - # copy ALADIN sources to right directory - try: - src_dirs = [d for d in os.listdir(self.builddir) if not (d.startswith('auxlib') or d.startswith('gmk'))] - target = os.path.join(self.installdir, self.rootpack_dir, 'src', 'local') - self.log.info("Copying sources from %s to %s" % (self.builddir, target)) - for srcdir in src_dirs: - shutil.copytree(os.path.join(self.builddir, srcdir), os.path.join(target, srcdir)) - self.log.info("Copied %s" % srcdir) - except OSError as err: - raise EasyBuildError("Failed to copy ALADIN sources: %s", err) - - if self.cfg['parallel']: - env.setvar('GMK_THREADS', str(self.cfg['parallel'])) - - # build rootpack - run_cmd(os.path.join(self.installdir, self.rootpack_dir, 'ics_master')) - - # restore original $LIBRARY_PATH - if self.orig_library_path is not None: - os.environ['LIBRARY_PATH'] = self.orig_library_path - - def sanity_check_step(self): - """Custom sanity check for ALADIN.""" - bindir = os.path.join(self.rootpack_dir, 'bin') - libdir = os.path.join(self.rootpack_dir, 'lib') - custom_paths = { - 'files': [os.path.join(bindir, x) for x in ['MASTER']] + - [os.path.join(libdir, 'lib%s.local.a' % x) for x in ['aeo', 'ald', 'arp', 'bip', - 'bla', 'mpa', 'mse', 'obt', - 'odb', 'sat', 'scr', 'sct', - 'sur', 'surfex', 'tal', 'tfl', - 'uti', 'xla', 'xrd']], - 'dirs': [], - } - super(EB_ALADIN, self).sanity_check_step(custom_paths=custom_paths) - - def make_module_req_guess(self): - """Custom guesses for environment variables (PATH, ...) for ALADIN.""" - guesses = super(EB_ALADIN, self).make_module_req_guess() - guesses.update({ - 'PATH': [os.path.join(self.rootpack_dir, 'bin')], - }) - return guesses diff --git a/easybuild/easyblocks/a/allinea.py b/easybuild/easyblocks/a/allinea.py deleted file mode 100644 index 026b2f58d98..00000000000 --- a/easybuild/easyblocks/a/allinea.py +++ /dev/null @@ -1,117 +0,0 @@ -## -# Copyright 2013-2024 Ghent University -# -# This file is part of EasyBuild, -# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), -# with support of Ghent University (http://ugent.be/hpc), -# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), -# Flemish Research Foundation (FWO) (http://www.fwo.be/en) -# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). -# -# https://github.com/easybuilders/easybuild -# -# EasyBuild is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation v2. -# -# EasyBuild is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with EasyBuild. If not, see . -## -""" -EasyBuild support for building and installing Allinea tools, implemented as an easyblock - -@author: Kenneth Hoste (Ghent University) -""" -import os -import shutil -import stat - -from easybuild.easyblocks.generic.binary import Binary -from easybuild.framework.easyblock import EasyBlock -from easybuild.framework.easyconfig import CUSTOM -from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.filetools import adjust_permissions, copy_file - - -class EB_Allinea(Binary): - """Support for building/installing Allinea.""" - - @staticmethod - def extra_options(extra_vars=None): - """Define extra easyconfig parameters specific to Allinea.""" - extra = Binary.extra_options(extra_vars) - extra.update({ - 'templates': [[], "List of templates.", CUSTOM], - 'sysconfig': [None, "system.config file to install.", CUSTOM], - }) - return extra - - def extract_step(self): - """Extract Allinea installation files.""" - EasyBlock.extract_step(self) - - def configure_step(self): - """No configuration for Allinea.""" - # ensure a license file is specified - if self.cfg['license_file'] is None: - raise EasyBuildError("No license file specified.") - - def build_step(self): - """No build step for Allinea.""" - pass - - def install_step(self): - """Install Allinea using install script.""" - - if self.cfg['install_cmd'] is None: - self.cfg['install_cmd'] = "./textinstall.sh --accept-licence %s" % self.installdir - - super(EB_Allinea, self).install_step() - - # copy license file - lic_path = os.path.join(self.installdir, 'licences') - try: - shutil.copy2(self.cfg['license_file'], lic_path) - except OSError as err: - raise EasyBuildError("Failed to copy license file to %s: %s", lic_path, err) - - # copy templates - templ_path = os.path.join(self.installdir, 'templates') - for templ in self.cfg['templates']: - path = self.obtain_file(templ, extension='qtf') - if path: - self.log.debug('Template file %s found' % path) - else: - raise EasyBuildError('No template file named %s found', templ) - - try: - # use shutil.copy (not copy2) so that permissions of copied file match with rest of installation - shutil.copy(path, templ_path) - except OSError as err: - raise EasyBuildError("Failed to copy template %s to %s: %s", templ, templ_path, err) - - # copy system.config if requested - sysconf_path = os.path.join(self.installdir, 'system.config') - if self.cfg['sysconfig'] is not None: - path = self.obtain_file(self.cfg['sysconfig'], extension=False) - if path: - self.log.debug('system.config file %s found' % path) - else: - raise EasyBuildError('No system.config file named %s found', sysconf_path) - - copy_file(path, sysconf_path) - adjust_permissions(sysconf_path, stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH, - recursive=False, relative=False) - - def sanity_check_step(self): - """Custom sanity check for Allinea.""" - custom_paths = { - 'files': ['bin/ddt', 'bin/map'], - 'dirs': [], - } - super(EB_Allinea, self).sanity_check_step(custom_paths=custom_paths) diff --git a/easybuild/easyblocks/a/amber.py b/easybuild/easyblocks/a/amber.py index a306211086d..15972c0903a 100644 --- a/easybuild/easyblocks/a/amber.py +++ b/easybuild/easyblocks/a/amber.py @@ -1,6 +1,6 @@ ## -# Copyright 2009-2024 Ghent University -# Copyright 2015-2024 Stanford University +# Copyright 2009-2025 Ghent University +# Copyright 2015-2025 Stanford University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -41,7 +41,7 @@ from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.build_log import EasyBuildError from easybuild.tools.modules import get_software_root -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd from easybuild.tools.filetools import remove_dir, which @@ -107,7 +107,7 @@ def patch_step(self, *args, **kwargs): # he or she selects "latest". (Note: "latest" is not # recommended for this reason and others.) for _ in range(self.cfg['patchruns']): - run_cmd(cmd, log_all=True) + run_shell_cmd(cmd) else: for (tree, patch_level) in zip(['AmberTools', 'Amber'], self.cfg['patchlevels']): if patch_level == 0: @@ -116,7 +116,7 @@ def patch_step(self, *args, **kwargs): # Run as many times as specified. It is the responsibility # of the easyconfig author to get this right. for _ in range(self.cfg['patchruns']): - run_cmd(cmd, log_all=True) + run_shell_cmd(cmd) super(EB_Amber, self).patch_step(*args, **kwargs) @@ -304,7 +304,7 @@ def configuremake_install_step(self): for flag, testrule in build_targets: # configure cmd = "%s ./configure %s" % (self.cfg['preconfigopts'], ' '.join(common_configopts + [flag, comp_str])) - (out, _) = run_cmd(cmd, log_all=True, simple=False) + run_shell_cmd(cmd) # build in situ using 'make install' # note: not 'build' @@ -312,10 +312,10 @@ def configuremake_install_step(self): # test if self.cfg['runtest']: - run_cmd("make %s" % testrule, log_all=True, simple=False) + run_shell_cmd("make %s" % testrule) # clean, overruling the normal 'build' - run_cmd("make clean") + run_shell_cmd("make clean") def install_step(self): """Install procedure for Amber.""" @@ -332,24 +332,24 @@ def install_step(self): pretestcommands = 'source %s/amber.sh && cd %s' % (self.installdir, self.builddir) # serial tests - run_cmd("%s && make test.serial" % pretestcommands, log_all=True, simple=True) + run_shell_cmd("%s && make test.serial" % pretestcommands) if self.with_cuda: - (out, ec) = run_cmd("%s && make test.cuda_serial" % pretestcommands, log_all=True, simple=False) - if ec > 0: + res = run_shell_cmd("%s && make test.cuda_serial" % pretestcommands) + if res.exit_code > 0: self.log.warning("Check the output of the Amber cuda tests for possible failures") if self.with_mpi: # Hard-code parallel tests to use 4 threads env.setvar("DO_PARALLEL", self.toolchain.mpi_cmd_for('', 4)) - (out, ec) = run_cmd("%s && make test.parallel" % pretestcommands, log_all=True, simple=False) - if ec > 0: + res = run_shell_cmd("%s && make test.parallel" % pretestcommands) + if res.exit_code > 0: self.log.warning("Check the output of the Amber parallel tests for possible failures") if self.with_mpi and self.with_cuda: # Hard-code CUDA parallel tests to use 2 threads env.setvar("DO_PARALLEL", self.toolchain.mpi_cmd_for('', 2)) - (out, ec) = run_cmd("%s && make test.cuda_parallel" % pretestcommands, log_all=True, simple=False) - if ec > 0: + res = run_shell_cmd("%s && make test.cuda_parallel" % pretestcommands) + if res.exit_code > 0: self.log.warning("Check the output of the Amber cuda_parallel tests for possible failures") def sanity_check_step(self): @@ -381,7 +381,5 @@ def make_module_extra(self): txt = super(EB_Amber, self).make_module_extra() txt += self.module_generator.set_environment('AMBERHOME', self.installdir) - if self.pylibdir: - txt += self.module_generator.prepend_paths('PYTHONPATH', self.pylibdir) return txt diff --git a/easybuild/easyblocks/a/anaconda.py b/easybuild/easyblocks/a/anaconda.py index 0ac18f1b201..8efe6b5b2c9 100644 --- a/easybuild/easyblocks/a/anaconda.py +++ b/easybuild/easyblocks/a/anaconda.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -34,12 +34,28 @@ from easybuild.easyblocks.generic.binary import Binary from easybuild.tools.filetools import adjust_permissions, remove_dir -from easybuild.tools.run import run_cmd +from easybuild.tools.modules import MODULE_LOAD_ENV_HEADERS +from easybuild.tools.run import run_shell_cmd class EB_Anaconda(Binary): """Support for building/installing Anaconda and Miniconda.""" + def __init__(self, *args, **kwargs): + """Initialize class variables.""" + super().__init__(*args, **kwargs) + + # Do not add installation to search paths for headers or libraries to avoid + # that the Anaconda environment is used by other software at building or linking time. + # LD_LIBRARY_PATH issue discusses here: + # http://superuser.com/questions/980250/environment-module-cannot-initialize-tcl + mod_env_headers = self.module_load_environment.alias_vars(MODULE_LOAD_ENV_HEADERS) + mod_env_libs = ['LD_LIBRARY_PATH', 'LIBRARY_PATH'] + mod_env_cmake = ['CMAKE_LIBRARY_PATH', 'CMAKE_PREFIX_PATH'] + for disallowed_var in mod_env_headers + mod_env_libs + mod_env_cmake: + self.module_load_environment.remove(disallowed_var) + self.log.debug(f"Purposely not updating ${disallowed_var} in {self.name} module file") + def install_step(self): """Copy all files in build directory to the install directory""" @@ -51,17 +67,7 @@ def install_step(self): # Anacondas own install instructions specify "bash [script]" despite using different shebangs cmd = "%s bash ./%s -p %s -b -f" % (self.cfg['preinstallopts'], install_script, self.installdir) self.log.info("Installing %s using command '%s'..." % (self.name, cmd)) - run_cmd(cmd, log_all=True, simple=True) - - def make_module_req_guess(self): - """ - A dictionary of possible directories to look for. - """ - return { - 'MANPATH': ['man', os.path.join('share', 'man')], - 'PATH': ['bin', 'sbin'], - 'PKG_CONFIG_PATH': [os.path.join(x, 'pkgconfig') for x in ['lib', 'lib32', 'lib64', 'share']], - } + run_shell_cmd(cmd) def sanity_check_step(self): """ diff --git a/easybuild/easyblocks/a/ansys.py b/easybuild/easyblocks/a/ansys.py index b59d6f96b78..31057ef49e3 100644 --- a/easybuild/easyblocks/a/ansys.py +++ b/easybuild/easyblocks/a/ansys.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -35,8 +35,8 @@ from easybuild.easyblocks.generic.packedbinary import PackedBinary from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.filetools import adjust_permissions -from easybuild.tools.run import run_cmd +from easybuild.tools.filetools import adjust_permissions, mkdir +from easybuild.tools.run import run_shell_cmd class EB_ANSYS(PackedBinary): @@ -47,19 +47,44 @@ def __init__(self, *args, **kwargs): super(EB_ANSYS, self).__init__(*args, **kwargs) self.ansysver = None + # custom extra module environment entries for ANSYS + bin_dirs = [ + 'tgrid/bin', + 'Framework/bin/Linux64', + 'aisol/bin/linx64', + 'RSM/bin', + 'ansys/bin', + 'autodyn/bin', + 'CFD-Post/bin', + 'CFX/bin', + 'fluent/bin', + 'TurboGrid/bin', + 'polyflow/bin', + 'Icepak/bin', + 'icemcfd/linux64_amd/bin' + ] + if LooseVersion(self.version) >= LooseVersion('19.0'): + bin_dirs.append('CEI/bin') + # use glob pattern as we don't know exact version at this stage + # it will be expanded before injection into the module file + ansysver_glob = 'v[0-9]*' + self.module_load_environment.PATH = [os.path.join(ansysver_glob, d) for d in bin_dirs] + def install_step(self): """Custom install procedure for ANSYS.""" # Sources (e.g. iso files) may drop the execute permissions adjust_permissions('INSTALL', stat.S_IXUSR) - cmd = "./INSTALL -silent -install_dir %s" % self.installdir + + mkdir(f'{self.builddir}/tmp') + cmd = f"./INSTALL -silent -install_dir {self.installdir} -usetempdir {self.builddir}/tmp" # E.g. license.example.com or license1.example.com,license2.example.com - licserv = self.cfg.get('license_server', os.getenv('EB_ANSYS_LICENSE_SERVER')) + licserv = self.cfg.get('license_server') or os.getenv('EB_ANSYS_LICENSE_SERVER') # E.g. '2325:1055' or just ':' to use those defaults - licport = self.cfg.get('license_server_port', os.getenv('EB_ANSYS_LICENSE_SERVER_PORT')) + licport = self.cfg.get('license_server_port') or os.getenv('EB_ANSYS_LICENSE_SERVER_PORT') if licserv is not None and licport is not None: cmd += ' -licserverinfo %s:%s' % (licport, licserv) - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) adjust_permissions(self.installdir, stat.S_IWOTH, add=False) @@ -76,35 +101,6 @@ def set_ansysver(self): else: self.ansysver = 'v' + ''.join(self.version.split('.')[0:2]) - def make_module_req_guess(self): - """Custom extra module file entries for ANSYS.""" - - if self.ansysver is None: - self.set_ansysver() - - guesses = super(EB_ANSYS, self).make_module_req_guess() - dirs = [ - 'tgrid/bin', - 'Framework/bin/Linux64', - 'aisol/bin/linx64', - 'RSM/bin', - 'ansys/bin', - 'autodyn/bin', - 'CFD-Post/bin', - 'CFX/bin', - 'fluent/bin', - 'TurboGrid/bin', - 'polyflow/bin', - 'Icepak/bin', - 'icemcfd/linux64_amd/bin' - ] - if LooseVersion(self.version) >= LooseVersion('19.0'): - dirs.append('CEI/bin') - - guesses['PATH'] = [os.path.join(self.ansysver, d) for d in dirs] - - return guesses - def make_module_extra(self): """Define extra environment variables required by Ansys""" diff --git a/easybuild/easyblocks/a/ant.py b/easybuild/easyblocks/a/ant.py index cb6757e663f..b4020507149 100644 --- a/easybuild/easyblocks/a/ant.py +++ b/easybuild/easyblocks/a/ant.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -34,7 +34,7 @@ from easybuild.easyblocks.generic.packedbinary import PackedBinary from easybuild.tools.build_log import EasyBuildError from easybuild.tools.modules import get_software_root, get_software_version -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd class EB_ant(PackedBinary): @@ -58,4 +58,4 @@ def install_step(self): cmd = "sh build.sh -Ddist.dir=%s dist" % self.installdir - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) diff --git a/easybuild/easyblocks/a/aocc.py b/easybuild/easyblocks/a/aocc.py index ff33673afbc..5e942cb9d2b 100644 --- a/easybuild/easyblocks/a/aocc.py +++ b/easybuild/easyblocks/a/aocc.py @@ -1,5 +1,5 @@ ## -# Copyright 2020-2024 Forschungszentrum Juelich GmbH +# Copyright 2020-2025 Forschungszentrum Juelich GmbH # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. @@ -31,8 +31,11 @@ @author: Sebastian Achilles (Forschungszentrum Juelich GmbH) """ +import glob import os +import re import stat +import tempfile from easybuild.tools import LooseVersion @@ -40,12 +43,26 @@ from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import adjust_permissions, move_file, write_file -from easybuild.tools.systemtools import get_shared_lib_ext +from easybuild.tools.modules import get_software_root, get_software_version +from easybuild.tools.run import run_shell_cmd +from easybuild.tools.systemtools import get_shared_lib_ext, get_cpu_architecture # Wrapper script definition -WRAPPER_TEMPLATE = """#!/bin/sh +WRAPPER_TEMPLATE = """#!/bin/bash -%(compiler_name)s --gcc-toolchain=$EBROOTGCCCORE "$@" +# Patch argv[0] to the actual compiler so that the correct driver is used internally +(exec -a "$0" {compiler_name} --gcc-toolchain=$EBROOTGCCCORE "$@") +""" + +AOCC_MINIMAL_CPP_EXAMPLE = """ +#include + +int main(){ std::cout << "It works!" << std::endl; } +""" + +AOCC_MINIMAL_FORTRAN_EXAMPLE = """ +program main +end program main """ @@ -66,6 +83,15 @@ def __init__(self, *args, **kwargs): super(EB_AOCC, self).__init__(*args, **kwargs) self.clangversion = self.cfg['clangversion'] + # AOCC is based on Clang. Try to guess the clangversion from the AOCC version + # if clangversion is not specified in the easyconfig + if self.clangversion is None: + self.clangversion = self._aocc_guess_clang_version() + + self._gcc_prefix = None + + # Bypass the .mod file check for GCCcore installs + self.cfg['skip_mod_files_sanity_check'] = True def _aocc_guess_clang_version(self): map_aocc_to_clang_ver = { @@ -73,6 +99,10 @@ def _aocc_guess_clang_version(self): '3.0.0': '12.0.0', '3.1.0': '12.0.0', '3.2.0': '13.0.0', + '4.0.0': '14.0.6', + '4.1.0': '16.0.3', + '4.2.0': '16.0.3', + '5.0.0': '17.0.6', } if self.version in map_aocc_to_clang_ver: @@ -86,68 +116,155 @@ def _aocc_guess_clang_version(self): ] raise EasyBuildError('\n'.join(error_lines)) - def install_step(self): - # EULA for AOCC must be accepted via --accept-eula-for EasyBuild configuration option, - # or via 'accept_eula = True' in easyconfig file - self.check_accepted_eula(more_info='http://developer.amd.com/wordpress/media/files/AOCC_EULA.pdf') - - # AOCC is based on Clang. Try to guess the clangversion from the AOCC version - # if clangversion is not specified in the easyconfig - if self.clangversion is None: - self.clangversion = self._aocc_guess_clang_version() - - super(EB_AOCC, self).install_step() + def _create_compiler_wrappers(self, compilers_to_wrap): + if not compilers_to_wrap: + return - def post_install_step(self): - """Create wrappers for the compilers to make sure compilers picks up GCCcore as GCC toolchain""" - - orig_compiler_tmpl = '%s.orig' + orig_compiler_tmpl = f"{os.path.join(self.installdir, 'bin')}/{{}}.orig" def create_wrapper(wrapper_comp): """Create for a particular compiler, with a particular name""" wrapper_f = os.path.join(self.installdir, 'bin', wrapper_comp) - write_file(wrapper_f, WRAPPER_TEMPLATE % {'compiler_name': orig_compiler_tmpl % wrapper_comp}) + compiler_name = orig_compiler_tmpl.format(wrapper_comp) + write_file(wrapper_f, WRAPPER_TEMPLATE.format(compiler_name=compiler_name)) + perms = stat.S_IXUSR | stat.S_IRUSR | stat.S_IXGRP | stat.S_IRGRP | stat.S_IXOTH | stat.S_IROTH adjust_permissions(wrapper_f, perms) - compilers_to_wrap = [ - 'clang', - 'clang++', - 'clang-%s' % LooseVersion(self.clangversion).version[0], - 'clang-cpp', - 'flang', - ] - # Rename original compilers and prepare wrappers to pick up GCCcore as GCC toolchain for the compilers for comp in compilers_to_wrap: actual_compiler = os.path.join(self.installdir, 'bin', comp) if os.path.isfile(actual_compiler): - move_file(actual_compiler, orig_compiler_tmpl % actual_compiler) + move_file(actual_compiler, orig_compiler_tmpl.format(comp)) else: - err_str = "Tried to move '%s' to '%s', but it does not exist!" - raise EasyBuildError(err_str, actual_compiler, '%s.orig' % actual_compiler) + raise EasyBuildError(f"Cannot make wrapper for '{actual_compiler}', file does not exist") if not os.path.exists(actual_compiler): create_wrapper(comp) - self.log.info("Wrapper for %s successfully created", comp) + self.log.info(f"Wrapper for {comp} successfully created") else: - err_str = "Creating wrapper for '%s' not possible, since original compiler was not renamed!" - raise EasyBuildError(err_str, actual_compiler) + raise EasyBuildError(f"Cannot make wrapper for '{actual_compiler}', original compiler was not renamed!") + + def _create_compiler_config_files(self, compilers_to_add_config_file): + """For each of the compiler suites, add a .cfg file which points to the correct GCCcore as the GCC toolchain.""" + if not compilers_to_add_config_file: + return - super(EB_AOCC, self).post_install_step() + bin_dir = os.path.join(self.installdir, 'bin') + prefix_str = '--gcc-install-dir=%s' % self.gcc_prefix + for comp in compilers_to_add_config_file: + with open(os.path.join(bin_dir, '%s.cfg' % comp), 'w') as f: + f.write(prefix_str) + + def _sanity_check_gcc_prefix(self): + """Check if the GCC prefix is correct.""" + compilers_to_check = [ + 'clang', + 'clang++', + 'clang-%s' % LooseVersion(self.clangversion).version[0], + 'clang-cpp', + 'flang', + ] + + rgx = re.compile('Selected GCC installation: (.*)') + for comp in compilers_to_check: + cmd = "%s -v" % os.path.join(self.installdir, 'bin', comp) + res = run_shell_cmd(cmd, fail_on_error=False) + mch = rgx.search(res.output) + if mch is None: + self.log.debug(res.output) + raise EasyBuildError("Failed to extract GCC installation path from output of `%s`", cmd) + gcc_prefix = mch.group(1) + if gcc_prefix != self.gcc_prefix: + raise EasyBuildError( + "GCC installation path `%s` does not match expected path `%s`", gcc_prefix, self.gcc_prefix + ) + + @property + def gcc_prefix(self): + """Set the GCC prefix for the build.""" + if not self._gcc_prefix: + arch = get_cpu_architecture() + gcc_root = get_software_root('GCCcore') + gcc_version = get_software_version('GCCcore') + # If that doesn't work, try with GCC + if gcc_root is None: + gcc_root = get_software_root('GCC') + gcc_version = get_software_version('GCC') + # If that doesn't work either, print error and exit + if gcc_root is None: + raise EasyBuildError("Can't find GCC or GCCcore to use") + + pattern = os.path.join(gcc_root, 'lib', 'gcc', '%s-*' % arch, '%s' % gcc_version) + matches = glob.glob(pattern) + if not matches: + raise EasyBuildError("Can't find GCC version %s for architecture %s in %s", gcc_version, arch, pattern) + self._gcc_prefix = os.path.abspath(matches[0]) + self.log.debug("Using %s as the gcc install location", self._gcc_prefix) + + return self._gcc_prefix + + def install_step(self): + # EULA for AOCC must be accepted via --accept-eula-for EasyBuild configuration option, + # or via 'accept_eula = True' in easyconfig file + self.check_accepted_eula(more_info='http://developer.amd.com/wordpress/media/files/AOCC_EULA.pdf') + + super(EB_AOCC, self).install_step() + + def post_processing_step(self): + """ + For AOCC <5.0.0: + Create wrappers for the compilers to make sure compilers picks up GCCcore as GCC toolchain. + For AOCC >= 5.0.0: + Create [compiler-name].cfg files to point the compiler to the correct GCCcore as GCC toolchain. + For compilers not supporting this option, wrap the compilers using the old method. + """ + compilers_to_wrap = [] + compilers_to_add_config_files = [] + + if LooseVersion(self.version) < LooseVersion("5.0.0"): + compilers_to_wrap += [ + f'clang-{LooseVersion(self.clangversion).version[0]}', + ] + if not self.cfg['keepsymlinks']: + compilers_to_wrap += [ + 'clang', + 'clang++', + 'clang-cpp', + 'flang', + ] + else: + compilers_to_add_config_files += [ + 'clang', + 'clang++', + 'clang-cpp' + ] + compilers_to_wrap += [ + 'flang' + ] + + self._create_compiler_config_files(compilers_to_add_config_files) + self._create_compiler_wrappers(compilers_to_wrap) + super(EB_AOCC, self).post_processing_step() def sanity_check_step(self): """Custom sanity check for AOCC, based on sanity check for Clang.""" + + # Clang v16+ only use the major version number for the resource dir + resdir_version = self.clangversion + if LooseVersion(self.clangversion) >= LooseVersion('16.0.0'): + resdir_version = LooseVersion(self.clangversion).version[0] + shlib_ext = get_shared_lib_ext() custom_paths = { 'files': [ 'bin/clang', 'bin/clang++', 'bin/flang', 'bin/lld', 'bin/llvm-ar', 'bin/llvm-as', 'bin/llvm-config', 'bin/llvm-link', 'bin/llvm-nm', 'bin/llvm-symbolizer', 'bin/opt', 'bin/scan-build', 'bin/scan-view', - 'include/clang-c/Index.h', 'include/llvm-c/Core.h', 'lib/clang/%s/include/omp.h' % self.clangversion, - 'lib/clang/%s/include/stddef.h' % self.clangversion, 'lib/libclang.%s' % shlib_ext, + 'include/clang-c/Index.h', 'include/llvm-c/Core.h', 'lib/clang/%s/include/omp.h' % resdir_version, + 'lib/clang/%s/include/stddef.h' % resdir_version, 'lib/libclang.%s' % shlib_ext, 'lib/libomp.%s' % shlib_ext, ], - 'dirs': ['include/llvm', 'lib/clang/%s/lib' % self.clangversion, 'lib32'], + 'dirs': ['include/llvm', 'lib/clang/%s/lib' % resdir_version, 'lib32'], } custom_commands = [ @@ -158,6 +275,19 @@ def sanity_check_step(self): "flang --help", "llvm-config --cxxflags", ] + + self._sanity_check_gcc_prefix() + + # Check if clang++ can actually compile programs. This can fail if the wrong driver is picked up by LLVM. + tmpdir = tempfile.mkdtemp() + write_file(os.path.join(tmpdir, 'minimal.cpp'), AOCC_MINIMAL_CPP_EXAMPLE) + minimal_cpp_compiler_cmd = "cd %s && clang++ minimal.cpp -o minimal_cpp" % tmpdir + custom_commands.append(minimal_cpp_compiler_cmd) + # Check if flang can actually compile programs. This can fail if the wrong driver is picked up by LLVM. + write_file(os.path.join(tmpdir, 'minimal.f90'), AOCC_MINIMAL_FORTRAN_EXAMPLE) + minimal_f90_compiler_cmd = "cd %s && flang minimal.f90 -o minimal_f90" % tmpdir + custom_commands.append(minimal_f90_compiler_cmd) + super(EB_AOCC, self).sanity_check_step(custom_paths=custom_paths, custom_commands=custom_commands) def make_module_extra(self): @@ -169,13 +299,3 @@ def make_module_extra(self): # setting the AOCChome path txt += self.module_generator.set_environment('AOCChome', self.installdir) return txt - - def make_module_req_guess(self): - """ - A dictionary of possible directories to look for. - Include C_INCLUDE_PATH and CPLUS_INCLUDE_PATH as an addition to default ones - """ - guesses = super(EB_AOCC, self).make_module_req_guess() - guesses['C_INCLUDE_PATH'] = ['include'] - guesses['CPLUS_INCLUDE_PATH'] = ['include'] - return guesses diff --git a/easybuild/easyblocks/a/aomp.py b/easybuild/easyblocks/a/aomp.py index e44b6f06abb..99c4673651a 100644 --- a/easybuild/easyblocks/a/aomp.py +++ b/easybuild/easyblocks/a/aomp.py @@ -1,5 +1,5 @@ ## -# Copyright 2021-2024 Ghent University +# Copyright 2021-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -68,6 +68,8 @@ def __init__(self, *args, **kwargs): super(EB_AOMP, self).__init__(*args, **kwargs) self.cfg['extract_sources'] = True self.cfg['dontcreateinstalldir'] = True + # Bypass the .mod file check for GCCcore installs + self.cfg['skip_mod_files_sanity_check'] = True def configure_step(self): """Configure AOMP build and let 'Binary' install""" @@ -83,11 +85,7 @@ def configure_step(self): 'AOMP_APPLY_ROCM_PATCHES=0', 'AOMP_STANDALONE_BUILD=1', ] - if self.cfg['parallel']: - install_options.append( - 'NUM_THREADS={!s}'.format(self.cfg['parallel'])) - else: - install_options.append('NUM_THREADS=1') + install_options.append(f'NUM_THREADS={self.cfg.parallel}') # Check if CUDA is loaded and alternatively build CUDA backend if get_software_root('CUDA') or get_software_root('CUDAcore'): cuda_root = get_software_root('CUDA') or get_software_root('CUDAcore') @@ -131,8 +129,8 @@ def configure_step(self): # Only build selected components self.cfg['installopts'] = 'select ' + ' '.join(components) - def post_install_step(self): - super(EB_AOMP, self).post_install_step() + def post_processing_step(self): + super(EB_AOMP, self).post_processing_step() # The install script will create a symbolic link as the install # directory, this creates problems for EB as it won't remove the # symlink. To remedy this we remove the link here and rename the actual diff --git a/easybuild/easyblocks/a/arb.py b/easybuild/easyblocks/a/arb.py deleted file mode 100644 index 37cc7a6ce16..00000000000 --- a/easybuild/easyblocks/a/arb.py +++ /dev/null @@ -1,98 +0,0 @@ -## -# Copyright 2009-2024 Ghent University -# -# This file is part of EasyBuild, -# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), -# with support of Ghent University (http://ugent.be/hpc), -# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), -# Flemish Research Foundation (FWO) (http://www.fwo.be/en) -# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). -# -# https://github.com/easybuilders/easybuild -# -# EasyBuild is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation v2. -# -# EasyBuild is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with EasyBuild. If not, see . -## -""" -EasyBuild support for ARB, implemented as an easyblock - -@author: Jens Timmerman (Ghent University) -@author: Kenneth Hoste (Ghent University) -""" -import os - -from easybuild.easyblocks.generic.configuremake import ConfigureMake -from easybuild.tools.environment import setvar -from easybuild.tools.run import run_cmd - - -class EB_ARB(ConfigureMake): - """Support for building and installing ARB.""" - - def __init__(self, *args, **kwargs): - """Initialisation of custom class variables for ARB, specify building in install dir.""" - super(EB_ARB, self).__init__(*args, **kwargs) - - self.build_in_installdir = True - self.subdir = 'UNKNOWN' - - def configure_step(self): - """No separate configure step for ARB.""" - self.subdir = os.path.basename(self.cfg['start_dir'].strip(os.path.sep)) - - def build_step(self): - """Build ARB by running make.""" - # set/extend required environment variables - path = os.environ.get('PATH', '') - ld_library_path = os.environ.get('LD_LIBRARY_PATH', '') - setvar('ARBHOME', os.getcwd()) - setvar('PATH', os.pathsep.join([ - os.path.join(self.cfg['start_dir'], 'bin'), - path, - ])) - setvar('LD_LIBRARY_PATH', os.pathsep.join([ - os.path.join(self.cfg['start_dir'], 'lib'), - os.path.join(self.cfg['start_dir'], 'LIBLINK'), - ld_library_path, - ])) - - # update make options - # no OpenGL support, verbose, 64-bit - self.cfg.update('buildopts', 'all OPENGL=0 V=1 ARB_64=1') - - # run 'make' without arguments to configure build, ignore non-zero exit code - (out, ec) = run_cmd("make", simple=False, log_all=False, log_ok=False) - self.log.debug("Command 'make' used to configure exited with exitcode %s (ignored) and output:\n%s" % (ec, out)) - - super(EB_ARB, self).build_step() - - def install_step(self): - """No installation step, ARB was built in installdir""" - pass - - def make_module_req_guess(self): - """Specify correct LD_LIBRARY_PATH and CPATH for this installation.""" - guesses = super(EB_ARB, self).make_module_req_guess() - guesses.update({ - 'CPATH': [os.path.join(self.subdir, "include")], - 'PATH': [os.path.join(self.subdir, "bin")], - 'LD_LIBRARY_PATH': [os.path.join(self.subdir, "lib")], - }) - return guesses - - def sanity_check_step(self): - """Custom sanity check for ARB.""" - custom_paths = { - 'files': [os.path.join(self.subdir, "bin/arb")], - 'dirs': [os.path.join(self.subdir, x) for x in ["lib"]], - } - super(EB_ARB, self).sanity_check_step(custom_paths=custom_paths) diff --git a/easybuild/easyblocks/a/armadillo.py b/easybuild/easyblocks/a/armadillo.py index f5881372574..ee93ff63662 100644 --- a/easybuild/easyblocks/a/armadillo.py +++ b/easybuild/easyblocks/a/armadillo.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/a/atlas.py b/easybuild/easyblocks/a/atlas.py deleted file mode 100644 index 8a0ef0ad699..00000000000 --- a/easybuild/easyblocks/a/atlas.py +++ /dev/null @@ -1,244 +0,0 @@ -## -# Copyright 2009-2024 Ghent University -# -# This file is part of EasyBuild, -# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), -# with support of Ghent University (http://ugent.be/hpc), -# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), -# Flemish Research Foundation (FWO) (http://www.fwo.be/en) -# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). -# -# https://github.com/easybuilders/easybuild -# -# EasyBuild is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation v2. -# -# EasyBuild is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with EasyBuild. If not, see . -## -""" -EasyBuild support for building and installing ATLAS, implemented as an easyblock - -@author: Stijn De Weirdt (Ghent University) -@author: Dries Verdegem (Ghent University) -@author: Kenneth Hoste (Ghent University) -@author: Pieter De Baets (Ghent University) -@author: Jens Timmerman (Ghent University) -""" - -import fileinput -import re -import os -import sys -from easybuild.tools import LooseVersion - -from easybuild.easyblocks.generic.configuremake import ConfigureMake -from easybuild.framework.easyconfig import CUSTOM -from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.modules import get_software_root -from easybuild.tools.run import run_cmd -from easybuild.tools.systemtools import AMD, INTEL, get_cpu_speed, get_cpu_vendor, get_shared_lib_ext - - -class EB_ATLAS(ConfigureMake): - """ - Support for building ATLAS - - configure (and check if it failed due to CPU throttling being enabled) - - avoid parallel build (doesn't make sense for ATLAS and doesn't work) - - make (optionally with shared libs), and install - """ - - def __init__(self, *args, **kwargs): - super(EB_ATLAS, self).__init__(*args, **kwargs) - - @staticmethod - def extra_options(): - extra_vars = { - 'ignorethrottling': [False, "Ignore check done by ATLAS for CPU throttling (not recommended)", CUSTOM], - 'full_lapack': [False, "Build a full LAPACK library (requires netlib's LAPACK)", CUSTOM], - 'sharedlibs': [False, "Enable building of shared libs as well", CUSTOM], - } - return ConfigureMake.extra_options(extra_vars) - - def configure_step(self): - - # configure for 64-bit build - self.cfg.update('configopts', "-b 64") - - if self.cfg['ignorethrottling']: - # ignore CPU throttling check - # this is not recommended, it will disturb the measurements done by ATLAS - # used for the EasyBuild demo, to avoid requiring root privileges - if LooseVersion(self.version) < LooseVersion('3.10.0'): - self.cfg.update('configopts', '-Si cputhrchk 0') - else: - self.log.warning("Ignore CPU throttling check is not possible via command line.") - # apply patch to ignore CPU throttling: make ProbeCPUThrottle always return 0 - # see http://sourceforge.net/p/math-atlas/support-requests/857/ - cfg_file = os.path.join('CONFIG', 'src', 'config.c') - for line in fileinput.input(cfg_file, inplace=1, backup='.orig.eb'): - line = re.sub(r"^(\s*iret)\s*=\s*.*CPU THROTTLE.*$", r"\1 = 0;", line) - sys.stdout.write(line) - self.log.warning('CPU throttling check ignored: NOT recommended!') - - if get_cpu_vendor() in [AMD, INTEL]: - # use cycle accurate timer for timings - # see http://math-atlas.sourceforge.net/atlas_install/node23.html - # this should work on Linux with both GCC and Intel compilers - cpu_freq = int(get_cpu_speed()) - self.cfg.update('configopts', "-D c -DPentiumCPS=%s" % cpu_freq) - else: - # use -DWALL for non-x86, see http://math-atlas.sourceforge.net/atlas_install/node25.html - self.cfg.update('configopts', "-D c -DWALL") - - # if LAPACK is found, instruct ATLAS to provide a full LAPACK library - # ATLAS only provides a few LAPACK routines natively - if self.cfg['full_lapack']: - lapack_lib_version = LooseVersion('3.9') - if LooseVersion(self.version) < lapack_lib_version: - # pass built LAPACK library - lapack = get_software_root('LAPACK') - if lapack: - self.cfg.update('configopts', ' --with-netlib-lapack=%s/lib/liblapack.a' % lapack) - else: - raise EasyBuildError("netlib's LAPACK library not available, required to build ATLAS " - "with a full LAPACK library.") - else: - # pass LAPACK source tarball - lapack_src = None - for src in self.src: - if src['name'].startswith('lapack'): - lapack_src = src['path'] - if lapack_src is not None: - self.cfg.update('configopts', ' --with-netlib-lapack-tarfile=%s' % lapack_src) - else: - raise EasyBuildError("LAPACK source tarball not available, but required.") - - # enable building of shared libraries (requires -fPIC) - if self.cfg['sharedlibs'] or self.toolchain.options['pic']: - self.log.debug("Enabling -fPIC because we're building shared ATLAS libs, or just because.") - self.cfg.update('configopts', '-Fa alg -fPIC') - - # ATLAS only wants to be configured/built in a separate dir' - try: - objdir = "obj" - os.makedirs(objdir) - os.chdir(objdir) - except OSError as err: - raise EasyBuildError("Failed to create obj directory to build in: %s", err) - - # specify compilers - self.cfg.update('configopts', '-C ic %(cc)s -C if %(f77)s' % { - 'cc': os.getenv('CC'), - 'f77': os.getenv('F77') - }) - - # call configure in parent dir - cmd = "%s %s/configure --prefix=%s %s" % (self.cfg['preconfigopts'], self.cfg['start_dir'], - self.installdir, self.cfg['configopts']) - (out, exitcode) = run_cmd(cmd, log_all=False, log_ok=False, simple=False) - - if exitcode != 0: - throttling_regexp = re.compile("cpu throttling [a-zA-Z]* enabled", re.IGNORECASE) - if throttling_regexp.search(out): - errormsg = ( - "Configure failed, possible because CPU throttling is enabled; ATLAS doesn't like that. " - "You can either disable CPU throttling, or set 'ignorethrottling' to True in the ATLAS .eb file. " - "Also see http://math-atlas.sourceforge.net/errata.html#cputhrottle . " - "Configure output: %s" - ) % out - else: - errormsg = "configure output: %s\nConfigure failed, not sure why (see output above)." % out - raise EasyBuildError(errormsg) - - def build_step(self, verbose=False): - - if self.cfg['parallel'] != 1: - self.log.warning("Ignoring requested build parallelism, it breaks ATLAS, so setting to 1") - self.cfg['parallel'] = 1 - - # default make is fine - super(EB_ATLAS, self).build_step(verbose=verbose) - - # optionally also build shared libs - if self.cfg['sharedlibs']: - try: - os.chdir('lib') - except OSError as err: - raise EasyBuildError("Failed to change to 'lib' directory for building the shared libs.", err) - - self.log.debug("Building shared libraries") - cmd = "make shared cshared ptshared cptshared" - run_cmd(cmd, log_all=True, simple=True) - - try: - os.chdir('..') - except OSError as err: - raise EasyBuildError("Failed to get back to previous dir after building shared libs: %s ", err) - - def install_step(self): - """Install step - - Default make install and optionally remove incomplete lapack libs. - If the full_lapack option was set to false we don't - """ - super(EB_ATLAS, self).install_step() - if not self.cfg['full_lapack']: - for i in ['liblapack.a', 'liblapack.%s' % get_shared_lib_ext()]: - lib = os.path.join(self.installdir, "lib", i[0]) - if os.path.exists(lib): - os.rename(lib, os.path.join(self.installdir, "lib", - lib.replace("liblapack", "liblapack_atlas"))) - else: - self.log.warning("Tried to remove %s, but file didn't exist") - - def test_step(self): - - # always run tests - if self.cfg['runtest']: - self.log.warning("ATLAS testing is done using 'make check' and 'make ptcheck'," - " so no need to set 'runtest' in the .eb spec file.") - - # sanity tests - self.cfg['runtest'] = 'check' - super(EB_ATLAS, self).test_step() - - # checks of threaded code (only if required) - if os.path.exists(os.path.join(self.cfg['start_dir'], 'obj', 'include', 'atlas_pthreads.h')): - self.cfg['runtest'] = 'ptcheck' - super(EB_ATLAS, self).test_step() - - # performance summary - self.cfg['runtest'] = 'time' - super(EB_ATLAS, self).test_step() - - # default make install is fine - - def sanity_check_step(self): - """ - Custom sanity check for ATLAS - """ - - libs = ["atlas", "cblas", "f77blas", "lapack", "ptcblas", "ptf77blas"] - - static_libs = ["lib/lib%s.a" % x for x in libs] - - if self.cfg['sharedlibs']: - shlib_ext = get_shared_lib_ext() - shared_libs = ["lib/lib%s.%s" % (x, shlib_ext) for x in libs] - else: - shared_libs = [] - - custom_paths = { - 'files': ["include/%s" % x for x in ["cblas.h", "clapack.h"]] + - static_libs + shared_libs, - 'dirs': ["include/atlas"] - } - - super(EB_ATLAS, self).sanity_check_step(custom_paths=custom_paths) diff --git a/easybuild/easyblocks/b/bamtools.py b/easybuild/easyblocks/b/bamtools.py deleted file mode 100644 index e9c678b9548..00000000000 --- a/easybuild/easyblocks/b/bamtools.py +++ /dev/null @@ -1,88 +0,0 @@ -## -# Copyright 2009-2024 The Cyprus Institute -# -# This file is part of EasyBuild, -# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), -# with support of Ghent University (http://ugent.be/hpc), -# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), -# Flemish Research Foundation (FWO) (http://www.fwo.be/en) -# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). -# -# https://github.com/easybuilders/easybuild -# -# EasyBuild is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation v2. -# -# EasyBuild is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with EasyBuild. If not, see . -## -""" -EasyBuild support for BamTools, implemented as an easyblock - -@author: Andreas Panteli (The Cyprus Institute) -@author: Kenneth Hoste (Ghent University) -""" -from easybuild.tools import LooseVersion -from easybuild.easyblocks.generic.cmakemake import CMakeMake -from easybuild.easyblocks.generic.makecp import MakeCp -from easybuild.framework.easyconfig import CUSTOM -from easybuild.tools.systemtools import get_shared_lib_ext - - -class EB_BamTools(MakeCp, CMakeMake): - """Support for building and installing BamTools.""" - - @staticmethod - def extra_options(extra_vars=None): - """Extra easyconfig parameters for BamTools.""" - extra_vars = MakeCp.extra_options() - extra_vars.update(CMakeMake.extra_options()) - - # files_to_copy is not mandatory here, since we overwrite it in install_step - extra_vars['files_to_copy'][2] = CUSTOM - # BamTools requires an out of source build - extra_vars['separate_build_dir'][0] = True - - return extra_vars - - def configure_step(self): - """Configure BamTools build.""" - CMakeMake.configure_step(self) - - def install_step(self): - """Custom installation procedure for BamTools.""" - if LooseVersion(self.version) < LooseVersion('2.5.0'): - self.cfg['files_to_copy'] = ['bin', 'lib', 'include', 'docs', 'LICENSE', 'README'] - MakeCp.install_step(self) - else: - CMakeMake.install_step(self) - - def sanity_check_step(self): - """Custom sanity check for BamTools.""" - - shlib_ext = get_shared_lib_ext() - - custom_paths = { - 'files': ['bin/bamtools'], - 'dirs': [], - } - if LooseVersion(self.version) < LooseVersion('2.3.0'): - # bamtools-utils & jsoncpp libs now built as static libs by default since v2.3.0 - custom_paths['files'].extend(['lib/libbamtools-utils.%s' % shlib_ext, 'lib/libjsoncpp.%s' % shlib_ext]) - elif LooseVersion(self.version) < LooseVersion('2.5.0'): - custom_paths['files'].extend(['include/shared/bamtools_global.h', 'lib/libbamtools.a', - 'lib/libbamtools.%s' % shlib_ext, 'lib/libbamtools-utils.a', - 'lib/libjsoncpp.a']) - custom_paths['dirs'].extend(['include/api', 'docs']) - else: - custom_paths['files'].extend(['include/bamtools/shared/bamtools_global.h', - ('lib/libbamtools.a', 'lib64/libbamtools.a')]) - custom_paths['dirs'].extend(['include/bamtools/api', ('lib/pkgconfig', 'lib64/pkgconfig')]) - - super(EB_BamTools, self).sanity_check_step(custom_paths=custom_paths) diff --git a/easybuild/easyblocks/b/bazel.py b/easybuild/easyblocks/b/bazel.py index 249f1c864ea..e45c6ee605a 100644 --- a/easybuild/easyblocks/b/bazel.py +++ b/easybuild/easyblocks/b/bazel.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -35,7 +35,7 @@ from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import apply_regex_substitutions, copy_file, which from easybuild.tools.modules import get_software_root, get_software_version -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd from easybuild.framework.easyconfig import CUSTOM @@ -158,7 +158,7 @@ def configure_step(self): ]) # enable building in parallel - bazel_args = '--jobs=%d' % self.cfg['parallel'] + bazel_args = f'--jobs={self.cfg.parallel}' # Bazel provides a JDK by itself for some architectures # We want to enforce it using the JDK we provided via modules @@ -190,7 +190,7 @@ def build_step(self): self.cfg['prebuildopts'], "bash -c 'set -x && ./compile.sh'", # Show the commands the script is running to faster debug failures ]) - run_cmd(cmd, log_all=True, simple=True, log_ok=True) + run_shell_cmd(cmd) def test_step(self): """Test the compilation""" @@ -206,7 +206,7 @@ def test_step(self): # Avoid bazel using $HOME '--output_user_root=%s' % self.output_user_root, runtest, - '--jobs=%d' % self.cfg['parallel'], + f'--jobs={self.cfg.parallel}', '--host_javabase=@local_jdk//:jdk', # Be more verbose '--subcommands', '--verbose_failures', @@ -214,7 +214,7 @@ def test_step(self): '--build_tests_only', self.cfg['testopts'] ]) - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) def install_step(self): """Custom install procedure for Bazel.""" diff --git a/easybuild/easyblocks/b/berkeleygw.py b/easybuild/easyblocks/b/berkeleygw.py index 8bb5781890d..b7d28798042 100644 --- a/easybuild/easyblocks/b/berkeleygw.py +++ b/easybuild/easyblocks/b/berkeleygw.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -62,7 +62,7 @@ def configure_step(self): def build_step(self): """Custom build step for BerkeleyGW.""" - self.cfg['parallel'] = 1 + self.cfg.parallel = 1 self.cfg['buildopts'] = 'all-flavors' diff --git a/easybuild/easyblocks/b/binutils.py b/easybuild/easyblocks/b/binutils.py index 10e9761b09f..8720ad4f8bc 100644 --- a/easybuild/easyblocks/b/binutils.py +++ b/easybuild/easyblocks/b/binutils.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -38,7 +38,7 @@ from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import apply_regex_substitutions, copy_file from easybuild.tools.modules import get_software_libdir, get_software_root -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd from easybuild.tools.systemtools import RISCV, get_cpu_family, get_shared_lib_ext from easybuild.tools.utilities import nub @@ -70,11 +70,11 @@ def determine_used_library_paths(self): compiler_cmd = os.environ.get('CC', 'gcc') # determine library search paths for GCC - stdout, ec = run_cmd('LC_ALL=C "%s" -print-search-dirs' % compiler_cmd, simple=False, log_all=True) - if ec: + res = run_shell_cmd('LC_ALL=C "%s" -print-search-dirs' % compiler_cmd) + if res.exit_code: raise EasyBuildError("Failed to determine library search dirs from compiler %s", compiler_cmd) - m = re.search('^libraries: *=(.*)$', stdout, re.M) + m = re.search('^libraries: *=(.*)$', res.output, re.M) paths = nub(os.path.abspath(p) for p in m.group(1).split(os.pathsep)) self.log.debug('Unique library search paths from compiler %s: %s', compiler_cmd, paths) @@ -268,14 +268,14 @@ def sanity_check_step(self): if any(dep['name'] == 'zlib' for dep in build_deps): for binary in binaries: bin_path = os.path.join(self.installdir, 'bin', binary) - out, _ = run_cmd("file %s" % bin_path, simple=False) - if re.search(r'statically linked', out): + res = run_shell_cmd("file %s" % bin_path) + if re.search(r'statically linked', res.output): # binary is fully statically linked, so no chance for dynamically linked libz continue # check whether libz is linked dynamically, it shouldn't be - out, _ = run_cmd("ldd %s" % bin_path, simple=False) - if re.search(r'libz\.%s' % shlib_ext, out): - raise EasyBuildError("zlib is not statically linked in %s: %s", bin_path, out) + res = run_shell_cmd("ldd %s" % bin_path) + if re.search(r'libz\.%s' % shlib_ext, res.output): + raise EasyBuildError("zlib is not statically linked in %s: %s", bin_path, res.output) super(EB_binutils, self).sanity_check_step(custom_paths=custom_paths, custom_commands=custom_commands) diff --git a/easybuild/easyblocks/b/bioconductor.py b/easybuild/easyblocks/b/bioconductor.py deleted file mode 100644 index acacb2e8c9f..00000000000 --- a/easybuild/easyblocks/b/bioconductor.py +++ /dev/null @@ -1,59 +0,0 @@ -## -# Copyright 2009-2024 Ghent University -# -# This file is part of EasyBuild, -# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), -# with support of Ghent University (http://ugent.be/hpc), -# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), -# Flemish Research Foundation (FWO) (http://www.fwo.be/en) -# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). -# -# https://github.com/easybuilders/easybuild -# -# EasyBuild is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation v2. -# -# EasyBuild is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with EasyBuild. If not, see . -## -""" -EasyBuild support for building and installing the Bioconductor R library, implemented as an easyblock - -@author: Stijn De Weirdt (Ghent University) -@author: Dries Verdegem (Ghent University) -@author: Kenneth Hoste (Ghent University) -@author: Jens Timmerman (Ghent University) -@author: Toon Willems (Ghent University) -""" -from easybuild.easyblocks.generic.rpackage import RPackage -from easybuild.tools.build_log import EasyBuildError - - -class EB_Bioconductor(RPackage): - """ - The Bioconductor package extends RPackage to use a different source - And using the biocLite package to do the installation. - """ - - def make_cmdline_cmd(self): - """Create a command line to install an R library.""" - raise EasyBuildError("Don't know how to install a specific version of a Bioconductor package.") - - def make_r_cmd(self): - """Create a command to run in R to install an R library.""" - name = self.ext['name'] - self.log.debug("Installing Bioconductor package %s." % name) - - r_cmd = '\n'.join([ - 'source("http://bioconductor.org/biocLite.R")', - 'biocLite("%s")' % name, - ]) - cmd = "R -q --no-save" - - return cmd, r_cmd diff --git a/easybuild/easyblocks/b/bisearch.py b/easybuild/easyblocks/b/bisearch.py deleted file mode 100644 index 5c9ec28a9dd..00000000000 --- a/easybuild/easyblocks/b/bisearch.py +++ /dev/null @@ -1,64 +0,0 @@ -## -# Copyright 2009-2024 Ghent University -# -# This file is part of EasyBuild, -# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), -# with support of Ghent University (http://ugent.be/hpc), -# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), -# Flemish Research Foundation (FWO) (http://www.fwo.be/en) -# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). -# -# https://github.com/easybuilders/easybuild -# -# EasyBuild is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation v2. -# -# EasyBuild is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with EasyBuild. If not, see . -## -""" -EasyBuild support for BiSearch, implemented as an easyblock - -@author: Stijn De Weirdt (Ghent University) -@author: Dries Verdegem (Ghent University) -@author: Kenneth Hoste (Ghent University) -@author: Pieter De Baets (Ghent University) -@author: Jens Timmerman (Ghent University) -""" -import os - -from easybuild.easyblocks.generic.packedbinary import PackedBinary -from easybuild.tools.run import run_cmd_qa - - -class EB_BiSearch(PackedBinary): - """ - Support for building BiSearch. - Basically just run the interactive installation script install.sh. - """ - - def install_step(self): - cmd = "./install.sh" - - qanda = { - 'Please enter the BiSearch root directory: ': self.installdir, - 'Please enter the path of c++ compiler [/usr/bin/g++]: ': os.getenv('CXX') - } - - no_qa = [r'Compiling components\s*\.*'] - - run_cmd_qa(cmd, qanda, no_qa=no_qa, log_all=True, simple=True) - - def sanity_check_step(self): - """Custom sanity check for BiSearch.""" - custom_paths = { - 'files': ['bin/%s' % x for x in ['fpcr', 'indexing_cdna', 'indexing_genome', 'makecomp']], - 'dirs': [], - } - super(EB_BiSearch, self).sanity_check_step(custom_paths=custom_paths) diff --git a/easybuild/easyblocks/b/blacs.py b/easybuild/easyblocks/b/blacs.py deleted file mode 100644 index 482de074016..00000000000 --- a/easybuild/easyblocks/b/blacs.py +++ /dev/null @@ -1,223 +0,0 @@ -## -# Copyright 2009-2024 Ghent University -# -# This file is part of EasyBuild, -# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), -# with support of Ghent University (http://ugent.be/hpc), -# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), -# Flemish Research Foundation (FWO) (http://www.fwo.be/en) -# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). -# -# https://github.com/easybuilders/easybuild -# -# EasyBuild is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation v2. -# -# EasyBuild is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with EasyBuild. If not, see . -## -""" -EasyBuild support for building and installing BLACS, implemented as an easyblock - -@author: Stijn De Weirdt (Ghent University) -@author: Dries Verdegem (Ghent University) -@author: Kenneth Hoste (Ghent University) -@author: Pieter De Baets (Ghent University) -@author: Jens Timmerman (Ghent University) -""" - -import glob -import re -import os -import shutil - -from easybuild.easyblocks.generic.configuremake import ConfigureMake -from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.run import run_cmd - - -# also used by ScaLAPACK -def det_interface(log, path): - """Determine interface through 'xintface' heuristic tool""" - - (out, _) = run_cmd(os.path.join(path, "xintface"), log_all=True, simple=False) - - intregexp = re.compile(r".*INTFACE\s*=\s*-D(\S+)\s*") - res = intregexp.search(out) - if res: - return res.group(1) - else: - raise EasyBuildError("Failed to determine interface, output for xintface: %s", out) - - -class EB_BLACS(ConfigureMake): - """ - Support for building/installing BLACS - - configure: symlink BMAKES/Bmake.MPI-LINUX to Bmake.inc - - make install: copy files - """ - - def configure_step(self): - """Configure BLACS build by copying Bmake.inc file.""" - - src = os.path.join(self.cfg['start_dir'], 'BMAKES', 'Bmake.MPI-LINUX') - dest = os.path.join(self.cfg['start_dir'], 'Bmake.inc') - - if not os.path.isfile(src): - raise EasyBuildError("Can't find source file %s", src) - - if os.path.exists(dest): - raise EasyBuildError("Destination file %s exists", dest) - - try: - shutil.copy(src, dest) - except OSError as err: - raise EasyBuildError("Copying %s to %s failed: %s", src, dest, err) - - def build_step(self): - """Build BLACS using build_step, after figuring out the make options based on the heuristic tools available.""" - - opts = { - 'mpicc': "%s %s" % (os.getenv('MPICC'), os.getenv('CFLAGS')), - 'mpif77': "%s %s" % (os.getenv('MPIF77'), os.getenv('FFLAGS')), - 'f77': os.getenv('F77'), - 'cc': os.getenv('CC'), - 'builddir': os.getcwd(), - 'mpidir': os.path.dirname(os.getenv('MPI_LIB_DIR')), - } - - # determine interface and transcomm settings - comm = '' - interface = 'UNKNOWN' - try: - cwd = os.getcwd() - os.chdir('INSTALL') - - # need to build - cmd = "make" - cmd += " CC='%(mpicc)s' F77='%(mpif77)s' MPIdir=%(mpidir)s" \ - " MPILIB='' BTOPdir=%(builddir)s INTERFACE=NONE" % opts - - # determine interface using xintface - run_cmd("%s xintface" % cmd, log_all=True, simple=True) - - interface = det_interface(self.log, "./EXE") - - # try and determine transcomm using xtc_CsameF77 and xtc_UseMpich - if not comm: - - run_cmd("%s xtc_CsameF77" % cmd, log_all=True, simple=True) - - (out, _) = run_cmd(self.toolchain.mpi_cmd_for("./EXE/xtc_CsameF77", 2), log_all=True, simple=False) - - # get rid of first two lines, that inform about how to use this tool - out = '\n'.join(out.split('\n')[2:]) - - notregexp = re.compile("_NOT_") - - if not notregexp.search(out): - # if it doesn't say '_NOT_', set it - comm = "TRANSCOMM='-DCSameF77'" - - else: - (_, ec) = run_cmd("%s xtc_UseMpich" % cmd, log_all=False, log_ok=False, simple=False) - if ec == 0: - - (out, _) = run_cmd(self.toolchain.mpi_cmd_for("./EXE/xtc_UseMpich", 2), - log_all=True, simple=False) - - if not notregexp.search(out): - - commregexp = re.compile(r'Set TRANSCOMM\s*=\s*(.*)$') - - res = commregexp.search(out) - if res: - # found how to set TRANSCOMM, so set it - comm = "TRANSCOMM='%s'" % res.group(1) - else: - # no match, set empty TRANSCOMM - comm = "TRANSCOMM=''" - else: - # if it fails to compile, set empty TRANSCOMM - comm = "TRANSCOMM=''" - - os.chdir(cwd) - except OSError as err: - raise EasyBuildError("Failed to determine interface and transcomm settings: %s", err) - - opts.update({ - 'comm': comm, - 'int': interface, - }) - - add_makeopts = ' MPICC="%(mpicc)s" MPIF77="%(mpif77)s" %(comm)s ' % opts - add_makeopts += ' INTERFACE=%(int)s MPIdir=%(mpidir)s BTOPdir=%(builddir)s mpi ' % opts - - self.cfg.update('buildopts', add_makeopts) - - super(EB_BLACS, self).build_step() - - def install_step(self): - """Install by copying files to install dir.""" - - # include files and libraries - for (srcdir, destdir, ext) in [ - (os.path.join("SRC", "MPI"), "include", ".h"), # include files - ("LIB", "lib", ".a"), # libraries - ]: - - src = os.path.join(self.cfg['start_dir'], srcdir) - dest = os.path.join(self.installdir, destdir) - - try: - os.makedirs(dest) - os.chdir(src) - - for lib in glob.glob('*%s' % ext): - - # copy file - shutil.copy2(os.path.join(src, lib), dest) - - self.log.debug("Copied %s to %s" % (lib, dest)) - - if destdir == 'lib': - # create symlink with more standard name for libraries - symlink_name = "lib%s.a" % lib.split('_')[0] - os.symlink(os.path.join(dest, lib), os.path.join(dest, symlink_name)) - self.log.debug("Symlinked %s/%s to %s" % (dest, lib, symlink_name)) - - except OSError as err: - raise EasyBuildError("Copying %s/*.%s to installation dir %s failed: %s", src, ext, dest, err) - - # utilities - src = os.path.join(self.cfg['start_dir'], 'INSTALL', 'EXE', 'xintface') - dest = os.path.join(self.installdir, 'bin') - - try: - os.makedirs(dest) - - shutil.copy2(src, dest) - - self.log.debug("Copied %s to %s" % (src, dest)) - - except OSError as err: - raise EasyBuildError("Copying %s to installation dir %s failed: %s", src, dest, err) - - def sanity_check_step(self): - """Custom sanity check for BLACS.""" - - custom_paths = { - 'files': [fil for filptrn in ["blacs", "blacsCinit", "blacsF77init"] - for fil in ["lib/lib%s.a" % filptrn, - "lib/%s_MPI-LINUX-0.a" % filptrn]] + - ["bin/xintface"], - 'dirs': [] - } - - super(EB_BLACS, self).sanity_check_step(custom_paths=custom_paths) diff --git a/easybuild/easyblocks/b/blat.py b/easybuild/easyblocks/b/blat.py index bfdfc8fb0d5..d64a2a63c3a 100755 --- a/easybuild/easyblocks/b/blat.py +++ b/easybuild/easyblocks/b/blat.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 the Cyprus Institute +# Copyright 2009-2025 the Cyprus Institute # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/b/blender.py b/easybuild/easyblocks/b/blender.py deleted file mode 100644 index 2abf715f48d..00000000000 --- a/easybuild/easyblocks/b/blender.py +++ /dev/null @@ -1,112 +0,0 @@ -## -# Copyright 2009-2024 Ghent University -# -# This file is part of EasyBuild, -# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), -# with support of Ghent University (http://ugent.be/hpc), -# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), -# Flemish Research Foundation (FWO) (http://www.fwo.be/en) -# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). -# -# https://github.com/easybuilders/easybuild -# -# EasyBuild is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation v2. -# -# EasyBuild is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with EasyBuild. If not, see . -## -""" -EasyBuild support for Blender, implemented as an easyblock - -@author: Samuel Moors (Vrije Universiteit Brussel) -""" -import os - -from easybuild.easyblocks.generic.cmakemake import CMakeMake -from easybuild.tools.filetools import find_glob_pattern -from easybuild.tools.modules import get_software_root, get_software_version -from easybuild.tools.systemtools import get_shared_lib_ext - - -class EB_Blender(CMakeMake): - """Support for building Blender.""" - - @staticmethod - def extra_options(): - extra_vars = CMakeMake.extra_options() - extra_vars['separate_build_dir'][0] = True - return extra_vars - - def configure_step(self): - """Set CMake options for Blender""" - - default_config_opts = { - 'WITH_BUILDINFO': 'OFF', - # disable SSE detection to give EasyBuild full control over optimization compiler flags being used - 'WITH_CPU_SSE': 'OFF', - # needed unless extra dependencies are added for it to work - 'WITH_GAMEENGINE': 'OFF', - 'WITH_INSTALL_PORTABLE': 'OFF', - # needed unless extra dependencies are added for it to work - 'WITH_SYSTEM_GLEW': 'OFF', - } - - for key in default_config_opts: - opt = '-D%s=' % key - if opt not in self.cfg['configopts']: - self.cfg.update('configopts', opt + default_config_opts[key]) - - # Python paths - python_root = get_software_root('Python') - if python_root: - shlib_ext = get_shared_lib_ext() - pyshortver = '.'.join(get_software_version('Python').split('.')[:2]) - site_packages = os.path.join(python_root, 'lib', 'python%s' % pyshortver, 'site-packages') - - # We assume that numpy/requests are included with the Python installation (no longer true for 2019a) - numpy_root = find_glob_pattern( - os.path.join(site_packages, 'numpy-*-py%s-linux-x86_64.egg' % pyshortver)) - requests_root = find_glob_pattern(os.path.join(site_packages, 'requests-*-py%s.egg' % pyshortver)) - python_lib = find_glob_pattern( - os.path.join(python_root, 'lib', 'libpython%s*.%s' % (pyshortver, shlib_ext))) - python_include_dir = find_glob_pattern(os.path.join(python_root, 'include', 'python%s*' % pyshortver)) - - self.cfg.update('configopts', '-DPYTHON_VERSION=%s' % pyshortver) - self.cfg.update('configopts', '-DPYTHON_LIBRARY=%s' % python_lib) - self.cfg.update('configopts', '-DPYTHON_INCLUDE_DIR=%s' % python_include_dir) - self.cfg.update('configopts', '-DPYTHON_NUMPY_PATH=%s' % numpy_root) - self.cfg.update('configopts', '-DPYTHON_REQUESTS_PATH=%s' % requests_root) - - # OpenEXR - openexr_root = get_software_root('OpenEXR') - if openexr_root: - self.cfg.update('configopts', '-DOPENEXR_INCLUDE_DIR=%s' % os.path.join(openexr_root, 'include')) - - # OpenColorIO - if get_software_root('OpenColorIO'): - self.cfg.update('configopts', '-DWITH_OPENCOLORIO=ON') - - # CUDA - if get_software_root('CUDA'): - self.cfg.update('configopts', '-DWITH_CYCLES_CUDA_BINARIES=ON') - - super(EB_Blender, self).configure_step() - - def sanity_check_step(self): - """Custom sanity check for Blender.""" - custom_paths = { - 'files': ['bin/blender'], - 'dirs': [], - } - - # make sure Cycles render engine is available - custom_commands = ["blender -b -E help | grep CYCLES"] - - super(EB_Blender, self).sanity_check_step(custom_paths=custom_paths, custom_commands=custom_commands) diff --git a/easybuild/easyblocks/b/boost.py b/easybuild/easyblocks/b/boost.py index e0f51a2a440..e6d3981a829 100644 --- a/easybuild/easyblocks/b/boost.py +++ b/easybuild/easyblocks/b/boost.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -52,7 +52,7 @@ from easybuild.tools.config import ERROR from easybuild.tools.filetools import apply_regex_substitutions, read_file, symlink, which, write_file from easybuild.tools.modules import get_software_root, get_software_version -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd from easybuild.tools.systemtools import AARCH64, POWER, RISCV64, UNKNOWN from easybuild.tools.systemtools import get_cpu_architecture, get_glibc_version, get_shared_lib_ext @@ -144,7 +144,7 @@ def configure_step(self): cmd = "%s ./bootstrap.sh --with-toolset=%s --prefix=%s %s" tup = (self.cfg['preconfigopts'], toolset, self.installdir, self.cfg['configopts']) - run_cmd(cmd % tup, log_all=True, simple=True) + run_shell_cmd(cmd % tup) # Use build_toolset if specified or the bootstrap toolset without the OS suffix self.toolset = self.cfg['build_toolset'] or re.sub('-linux$', '', toolset) @@ -227,8 +227,8 @@ def build_step(self): self.bjamoptions += " -s%s_INCLUDE=%s/include" % (lib.upper(), libroot) self.bjamoptions += " -s%s_LIBPATH=%s/lib" % (lib.upper(), libroot) - if self.cfg['parallel']: - self.paracmd = "-j %s" % self.cfg['parallel'] + if self.cfg.parallel > 1: + self.paracmd = f"-j {self.cfg.parallel}" else: self.paracmd = '' @@ -282,7 +282,7 @@ def build_step(self): self.paracmd, self.cfg['buildopts'], ]) - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) def install_step(self): """Install Boost by copying files to install dir.""" @@ -298,7 +298,7 @@ def install_step(self): self.paracmd, self.cfg['installopts'], ]) - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) if self.cfg['tagged_layout']: if LooseVersion(self.version) >= LooseVersion("1.69.0") or not self.cfg['single_threaded']: diff --git a/easybuild/easyblocks/b/bowtie.py b/easybuild/easyblocks/b/bowtie.py index 119cb47ed0d..e6b4e813067 100644 --- a/easybuild/easyblocks/b/bowtie.py +++ b/easybuild/easyblocks/b/bowtie.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2024 Ghent University +# Copyright 2013-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/b/bowtie2.py b/easybuild/easyblocks/b/bowtie2.py index f768ac481d5..4f19a3c09be 100644 --- a/easybuild/easyblocks/b/bowtie2.py +++ b/easybuild/easyblocks/b/bowtie2.py @@ -1,7 +1,7 @@ ## # This file is an EasyBuild reciPY as per https://github.com/easybuilders/easybuild # -# Copyright:: Copyright 2012-2024 Uni.Lu/LCSB, NTUA +# Copyright:: Copyright 2012-2025 Uni.Lu/LCSB, NTUA # Authors:: Cedric Laczny , Fotis Georgatos , Kenneth Hoste # License:: MIT/GPL # $Id$ diff --git a/easybuild/easyblocks/b/bwa.py b/easybuild/easyblocks/b/bwa.py index 55bd60804f4..1948e87566f 100644 --- a/easybuild/easyblocks/b/bwa.py +++ b/easybuild/easyblocks/b/bwa.py @@ -1,7 +1,7 @@ ## # This file is an EasyBuild reciPY as per https://github.com/easybuilders/easybuild # -# Copyright:: Copyright 2012-2024 Uni.Lu/LCSB, NTUA +# Copyright:: Copyright 2012-2025 Uni.Lu/LCSB, NTUA # Authors:: Cedric Laczny , Kenneth Hoste # Authors:: George Tsouloupas , Fotis Georgatos # License:: MIT/GPL diff --git a/easybuild/easyblocks/b/bwise.py b/easybuild/easyblocks/b/bwise.py deleted file mode 100644 index 3e9b3d34e62..00000000000 --- a/easybuild/easyblocks/b/bwise.py +++ /dev/null @@ -1,114 +0,0 @@ -## -# Copyright 2018-2024 Free University of Brussels -# -# This file is part of EasyBuild, -# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), -# with support of Ghent University (http://ugent.be/hpc), -# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), -# Flemish Research Foundation (FWO) (http://www.fwo.be/en) -# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). -# -# https://github.com/easybuilders/easybuild -# -# EasyBuild is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation v2. -# -# EasyBuild is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with EasyBuild. If not, see . -## -""" -@author: Ward Poelmans (Free University of Brussels) -""" -import glob -import os -import stat - -from easybuild.easyblocks.generic.makecp import MakeCp -from easybuild.framework.easyconfig import CUSTOM -from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.filetools import adjust_permissions, apply_regex_substitutions, change_dir -from easybuild.tools.filetools import copy_file, write_file -from easybuild.tools.modules import get_software_root - - -class EB_BWISE(MakeCp): - """ - Custom easyblock to install BWISE - """ - @staticmethod - def extra_options(): - """Change default values of options""" - extra = MakeCp.extra_options() - # files_to_copy is not mandatory here - extra['files_to_copy'][2] = CUSTOM - return extra - - def build_step(self): - """Run multiple times for different sources""" - - # BCALM is a git submodule of BWISE but we use it as a dependency because - # it also has submodules and it's from a different developer - bcalm = get_software_root('BCALM') - if not bcalm: - raise EasyBuildError("BWISE needs BCALM to work") - - makefiles_fixes = [ - (r'^CC=.*$', 'CC=$(CXX)'), - (r'^CFLAGS=.*$', 'CFLAGS:=$(CFLAGS)'), - (r'^LDFLAGS=.*$', 'LDFLAGS:=$(LDFLAGS) -fopenmp') - ] - - def find_build_subdir(pattern): - """Changes to the sub directory that matches the given pattern""" - subdir = glob.glob(os.path.join(self.builddir, pattern)) - if subdir: - change_dir(subdir[0]) - apply_regex_substitutions('makefile', makefiles_fixes) - super(EB_BWISE, self).build_step() - return subdir[0] - else: - raise EasyBuildError("Could not find a subdirectory matching the pattern %s", pattern) - - # BWISE has 3 independant parts, we build them one by one - # first BWISE itself - subdir = find_build_subdir(os.path.join('BWISE-*', 'src')) - apply_regex_substitutions(os.path.join(subdir, '..', 'Bwise.py'), - [(r'^BWISE_MAIN = .*$', 'BWISE_MAIN = os.environ[\'EBROOTBWISE\']')]) - - # Onwards to BGREAT - subdir = find_build_subdir('BGREAT2-*') - copy_file(os.path.join(subdir, 'bgreat'), self.cfg['start_dir']) - - # Finally, BTRIM - subdir = find_build_subdir('BTRIM-*') - copy_file(os.path.join(subdir, 'btrim'), self.cfg['start_dir']) - - binaries = ['sequencesToNumbers', 'numbersFilter', 'path_counter', 'maximal_sr', 'simulator', - 'path_to_kmer', 'K2000/*.py', 'K2000/*.sh'] - self.cfg['files_to_copy'] = [(['bgreat', 'btrim', 'Bwise.py'] + ['src/%s' % x for x in binaries], 'bin'), - 'data'] - - def install_step(self): - super(EB_BWISE, self).install_step() - - # BWISE expects BCALM to be at exactly this location... - bcalmwrapper = """#!/bin/sh -$EBROOTBCALM/bin/bcalm "$@" - """ - write_file(os.path.join(self.installdir, "bin", "bcalm"), bcalmwrapper) - - adjust_permissions(os.path.join(self.installdir, "bin"), stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) - - def sanity_check_step(self): - """Custom sanity check for BWISE.""" - custom_paths = { - 'files': [os.path.join('bin', x) for x in ['bcalm', 'Bwise.py', 'bgreat', 'btrim']], - 'dirs': ['data'] - } - super(EB_BWISE, self).sanity_check_step(custom_paths=custom_paths) diff --git a/easybuild/easyblocks/b/bzip2.py b/easybuild/easyblocks/b/bzip2.py index aa5e598e935..68a22ee26b6 100644 --- a/easybuild/easyblocks/b/bzip2.py +++ b/easybuild/easyblocks/b/bzip2.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -36,7 +36,7 @@ from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd from easybuild.tools.systemtools import get_shared_lib_ext @@ -70,7 +70,7 @@ def install_step(self): if self.cfg['with_shared_libs']: cmd = "%s make -f Makefile-libbz2_so %s" % (self.cfg['prebuildopts'], self.cfg['buildopts']) - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) # copy shared libraries to /lib shlib_ext = get_shared_lib_ext() diff --git a/easybuild/easyblocks/c/cblas.py b/easybuild/easyblocks/c/cblas.py index 4a1ca2f349d..84bb88c96fb 100644 --- a/easybuild/easyblocks/c/cblas.py +++ b/easybuild/easyblocks/c/cblas.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2024 Ghent University +# Copyright 2013-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/c/cfdemcoupling.py b/easybuild/easyblocks/c/cfdemcoupling.py deleted file mode 100644 index 4bb0fd85a5e..00000000000 --- a/easybuild/easyblocks/c/cfdemcoupling.py +++ /dev/null @@ -1,169 +0,0 @@ -## -# Copyright 2018-2024 Ghent University -# -# This file is part of EasyBuild, -# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), -# with support of Ghent University (http://ugent.be/hpc), -# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), -# Flemish Research Foundation (FWO) (http://www.fwo.be/en) -# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). -# -# https://github.com/easybuilders/easybuild -# -# EasyBuild is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation v2. -# -# EasyBuild is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with EasyBuild. If not, see . -## -""" -EasyBuild support for building and installing CFDEMcoupling, implemented as an easyblock - -@author: Kenneth Hoste (Ghent University) -""" -import glob -import os - -import easybuild.tools.environment as env -import easybuild.tools.toolchain as toolchain -from easybuild.framework.easyblock import EasyBlock -from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.filetools import copy_file, mkdir, move_file -from easybuild.tools.modules import get_software_root, get_software_version -from easybuild.tools.run import run_cmd - - -class EB_CFDEMcoupling(EasyBlock): - """Support for building/installing CFDEMcoupling.""" - - def __init__(self, *args, **kwargs): - """Initialisation of custom class variables for CFDEMcoupling.""" - super(EB_CFDEMcoupling, self).__init__(*args, **kwargs) - - self.build_in_installdir = True - - self.cfdem_project_dir = os.path.join(self.installdir, '%s-%s' % (self.name, self.version)) - self.liggghts_dir = os.path.join(self.installdir, 'LIGGGHTS-%s' % self.version) - - def configure_step(self): - """Set up environment for building/installing CFDEMcoupling.""" - - # rename top-level directory to CFDEMcoupling- - top_dirs = os.listdir(self.builddir) - - for (pkgname, target_dir) in [('CFDEMcoupling', self.cfdem_project_dir), ('LIGGGHTS', self.liggghts_dir)]: - pkg_topdirs = [d for d in top_dirs if d.startswith(pkgname)] - if len(pkg_topdirs) == 1: - orig_dir = os.path.join(self.builddir, pkg_topdirs[0]) - move_file(orig_dir, target_dir) - else: - error_msg = "Failed to find subdirectory for %s in %s %s (missing sources for %s?)", - raise EasyBuildError(error_msg, pkgname, self.builddir, top_dirs, pkgname) - - env.setvar('CFDEM_VERSION', self.version) - env.setvar('CFDEM_PROJECT_DIR', self.cfdem_project_dir) - - # define $CFDEM_PROJECT_USER_DIR to an empty existing directory - project_user_dir = os.path.join(self.builddir, 'project_user_dir') - env.setvar('CFDEM_PROJECT_USER_DIR', project_user_dir) - mkdir(project_user_dir, parents=True) - - cfdem_bashrc = os.path.join(self.cfdem_project_dir, 'src', 'lagrangian', 'cfdemParticle', 'etc', 'bashrc') - env.setvar('CFDEM_bashrc', cfdem_bashrc) - - env.setvar('CFDEM_LIGGGHTS_SRC_DIR', os.path.join(self.liggghts_dir, 'src')) - env.setvar('CFDEM_LIGGGHTS_MAKEFILE_NAME', 'auto') - - lpp_dirs = glob.glob(os.path.join(self.builddir, 'LPP-*')) - if len(lpp_dirs) == 1: - env.setvar('CFDEM_LPP_DIR', lpp_dirs[0]) - else: - raise EasyBuildError("Failed to isolate LPP-* directory in %s", self.builddir) - - # build in parallel - env.setvar("WM_NCOMPPROCS", str(self.cfg['parallel'])) - - vtk_root = get_software_root('VTK') - if vtk_root: - vtk_ver_maj_min = '.'.join(get_software_version('VTK').split('.')[:2]) - vtk_inc = os.path.join(vtk_root, 'include', 'vtk-%s' % vtk_ver_maj_min) - if os.path.exists(vtk_inc): - env.setvar('VTK_INC_USR', '-I%s' % vtk_inc) - else: - raise EasyBuildError("Expected directory %s does not exist!", vtk_inc) - - vtk_lib = os.path.join(vtk_root, 'lib') - if os.path.exists(vtk_lib): - env.setvar('VTK_LIB_USR', '-L%s' % vtk_lib) - else: - raise EasyBuildError("Expected directory %s does not exist!", vtk_lib) - else: - raise EasyBuildError("VTK not included as dependency") - - # can't seem to use defined 'cfdemSysTest' alias, so call cfdemSystemTest.sh script directly... - cmd = "source $CFDEM_bashrc && $CFDEM_SRC_DIR/lagrangian/cfdemParticle/etc/cfdemSystemTest.sh" - run_cmd(cmd, log_all=True, simple=True, log_ok=True) - - def build_step(self): - """Custom build procedure for CFDEMcoupling.""" - - if get_software_root('OpenFOAM'): - openfoam_ver = get_software_version('OpenFOAM') - openfoam_maj_ver = openfoam_ver.split('.')[0] - else: - raise EasyBuildError("OpenFOAM not included as dependency") - - # make sure expected additionalLibs_* file is available - addlibs_subdir = os.path.join('src', 'lagrangian', 'cfdemParticle', 'etc', 'addLibs_universal') - src_addlibs = os.path.join(self.cfdem_project_dir, addlibs_subdir, 'additionalLibs_%s.x' % openfoam_maj_ver) - target_addlibs = os.path.join(self.cfdem_project_dir, addlibs_subdir, 'additionalLibs_%s' % openfoam_ver) - copy_file(src_addlibs, target_addlibs) - - # can't seem to use defined 'cfdemCompCFDEMall' alias... - cmd = "$CFDEM_SRC_DIR/lagrangian/cfdemParticle/etc/compileCFDEMcoupling_all.sh" - run_cmd("source $FOAM_BASH && source $CFDEM_bashrc && %s" % cmd, log_all=True, simple=True, log_ok=True) - - def install_step(self): - """No custom install procedure for CFDEMcoupling.""" - pass - - def sanity_check_step(self): - """Custom sanity check for CFDEMcoupling.""" - comp_fam = self.toolchain.comp_family() - if comp_fam == toolchain.GCC: - wm_compiler = 'Gcc' - elif comp_fam == toolchain.INTELCOMP: - wm_compiler = 'Icc' - else: - raise EasyBuildError("Unknown compiler family, don't know how to set WM_COMPILER") - - psubdir = "linux64%sDPInt32Opt" % wm_compiler - - cfdem_base_dir = os.path.basename(self.cfdem_project_dir) - bins = ['cfdemPostproc', 'cfdemSolverIB', 'cfdemSolverPiso', 'cfdemSolverPisoScalar', 'cfdemSolverPisoSTM'] - custom_paths = { - 'files': [os.path.join(cfdem_base_dir, 'platforms', psubdir, 'bin', b) for b in bins], - 'dirs': [os.path.join(cfdem_base_dir, 'platforms', psubdir, 'lib')], - } - super(EB_CFDEMcoupling, self).sanity_check_step(custom_paths=custom_paths) - - def make_module_extra(self): - """Custom extra module file entries for CFDEMcoupling.""" - - txt = super(EB_CFDEMcoupling, self).make_module_extra() - - txt += self.module_generator.set_environment('CFDEM_VERSION', self.version) - txt += self.module_generator.set_environment('CFDEM_PROJECT_DIR', self.cfdem_project_dir) - txt += self.module_generator.set_environment('CFDEM_LIGGGHTS_SRC_DIR', os.path.join(self.liggghts_dir, 'src')) - txt += self.module_generator.set_environment('CFDEM_LIGGGHTS_MAKEFILE_NAME', 'auto') - - cfdem_bashrc = os.path.join(self.cfdem_project_dir, 'src', 'lagrangian', 'cfdemParticle', 'etc', 'bashrc') - txt += self.module_generator.set_environment('CFDEM_bashrc', cfdem_bashrc) - - return txt diff --git a/easybuild/easyblocks/c/cgal.py b/easybuild/easyblocks/c/cgal.py index 431b843b50a..47deae247dd 100644 --- a/easybuild/easyblocks/c/cgal.py +++ b/easybuild/easyblocks/c/cgal.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/c/chapel.py b/easybuild/easyblocks/c/chapel.py deleted file mode 100644 index 2386f5b7dbd..00000000000 --- a/easybuild/easyblocks/c/chapel.py +++ /dev/null @@ -1,65 +0,0 @@ -## -# This file is an EasyBuild reciPY as per https://github.com/easybuilders/easybuild -# -# Copyright:: Copyright 2012-2024 Uni.Lu/LCSB, NTUA -# Authors:: Fotis Georgatos , Kenneth Hoste -# License:: MIT/GPL -# $Id$ -# -# This work implements a part of the HPCBIOS project and is a component of the policy: -# http://hpcbios.readthedocs.org/en/latest/ -## -""" -EasyBuild support for Chapel, implemented as an easyblock - -@author: Fotis Georgatos (Uni.Lu) -@author: Kenneth Hoste (Ghent University) -""" -import os - -from easybuild.easyblocks.generic.configuremake import ConfigureMake - - -class EB_Chapel(ConfigureMake): - """Support for building Chapel.""" - - def __init__(self, *args, **kwargs): - """Initialize Chapel-specific variables.""" - super(EB_Chapel, self).__init__(*args, **kwargs) - self.build_in_installdir = True - - def configure_step(self): - """No configure step for Chapel.""" - pass - - # building is done via make, so taken care of by ConfigureMake easyblock - - def install_step(self): - """Installation of Chapel has already been done as part of the build procedure""" - pass - - def sanity_check_step(self): - """Custom sanity check for Chapel.""" - - libpath = os.path.join('lib', 'linux64', 'gnu', 'comm-none', 'substrate-none', 'seg-none', - 'mem-default', 'tasks-fifo', 'threads-pthreads', 'atomics-intrinsics') - custom_paths = { - 'files': ['bin/linux64/chpl', 'bin/linux64/chpldoc', - os.path.join(libpath, 'libchpl.a'), os.path.join(libpath, 'main.o')], - 'dirs': [] - } - - super(EB_Chapel, self).sanity_check_step(custom_paths=custom_paths) - - def make_module_req_guess(self): - """ - A dictionary of possible directories to look for; this is needed since bin/linux64 of chapel is non standard - """ - guesses = super(EB_Chapel, self).make_module_req_guess() - lib_paths = ['lib', 'lib/linux64', 'lib64'] - guesses.update({ - 'PATH': ['bin', 'bin/linux64', 'bin64'], - 'LD_LIBRARY_PATH': lib_paths, - 'LIBRARY_PATH': lib_paths, - }) - return guesses diff --git a/easybuild/easyblocks/c/charmm.py b/easybuild/easyblocks/c/charmm.py deleted file mode 100644 index 6b4791e30b0..00000000000 --- a/easybuild/easyblocks/c/charmm.py +++ /dev/null @@ -1,159 +0,0 @@ -## -# Copyright 2013-2024 Ghent University -# -# This file is part of EasyBuild, -# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), -# with support of Ghent University (http://ugent.be/hpc), -# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), -# Flemish Research Foundation (FWO) (http://www.fwo.be/en) -# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). -# -# https://github.com/easybuilders/easybuild -# -# EasyBuild is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation v2. -# -# EasyBuild is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with EasyBuild. If not, see . -## -""" -EasyBuild support for building and installing CHARMM, implemented as an easyblock - -@author: Ward Poelmans (Ghent University) -""" -# TODO: add support for more QC software (q-chem, gamess, ...) - -import shutil - -from easybuild.framework.easyconfig import CUSTOM -from easybuild.framework.easyblock import EasyBlock -from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.modules import get_software_root, get_software_version -from easybuild.tools.run import run_cmd -import easybuild.tools.toolchain as toolchain - -# Possible systemsizes for CHARMM -KNOWN_SYSTEM_SIZES = ['huge', 'xxlarge', 'xlarge', 'large', 'medium', 'small', 'xsmall', 'reduce'] - - -class EB_CHARMM(EasyBlock): - """ - Support for building/installing CHARMM - """ - - @staticmethod - def extra_options(): - """Add extra easyconfig parameters custom to CHARMM.""" - extra_vars = { - 'build_options': ["FULL", "Specify the options to the build script", CUSTOM], - 'system_size': ["medium", "Specify the supported systemsize: %s" % ', '.join(KNOWN_SYSTEM_SIZES), CUSTOM], - 'runtest': [True, "Run tests after each build", CUSTOM], - } - return EasyBlock.extra_options(extra_vars) - - def __init__(self, *args, **kwargs): - """Initialisation of custom class variables for CHARMM.""" - super(EB_CHARMM, self).__init__(*args, **kwargs) - self.arch = 'UNKNOWN' - - def configure_step(self): - # Clean out old dir but don't create new one - self.cfg['dontcreateinstalldir'] = True - - if self.toolchain.comp_family() == toolchain.INTELCOMP: - self.arch = "em64t" - else: - self.arch = "gnu" - - super(EB_CHARMM, self).make_dir(self.installdir, True, dontcreateinstalldir=True) - - def build_step(self, verbose=False): - """Start the actual build""" - if self.cfg['system_size'] not in KNOWN_SYSTEM_SIZES: - raise EasyBuildError("Unknown system size '%s' specified, known: %s", - self.cfg['system_size'], KNOWN_SYSTEM_SIZES) - - self.log.info("Building for size: %s" % self.cfg['system_size']) - self.log.info("Build options from the easyconfig: %s" % self.cfg['build_options']) - build_options = self.cfg['build_options'] - - # FFTW and MKL are mutally exclusive - if get_software_root("FFTW"): - self.log.info("Using FFTW") - build_options += " FFTW" - else: - self.log.info("Not using FFTW") - if get_software_root("imkl"): - self.log.info("Using the MKL") - build_options += " MKL" - else: - self.log.info("Not using MKL") - - # Currently, only support for g09 added - if get_software_root("Gaussian") and 'g09' in get_software_version('Gaussian'): - self.log.info("Using g09") - build_options += " G09" - else: - self.log.info("Not using g09") - - if self.toolchain.options.get('usempi', None): - self.log.info("Using MPI") - # M means use MPI and MPIF90 means let mpif90 handle all MPI stuff - build_options += " M MPIF90" - - # By default, CHARMM uses gfortran. We need to specify if we want ifort - if self.toolchain.comp_family() == toolchain.INTELCOMP: - build_options += " IFORT" - - cmd = ' '.join([self.cfg['prebuildopts'], './install.com', self.arch, self.cfg['system_size'], build_options]) - (out, _) = run_cmd(cmd, log_all=True, simple=False, log_output=verbose) - return out - - def test_step(self): - """Run the testsuite""" - # Allow to skip the tests by setting runtest to False - if self.cfg['runtest']: - if self.toolchain.options.get('usempi', None): - cmd = "cd test && ./test.com M %s %s" % (self.cfg['parallel'], self.arch) - else: - cmd = "cd test && ./test.com %s" % self.arch - (out, _) = run_cmd(cmd, log_all=True, simple=False) - return out - - def sanity_check_step(self): - """Custom sanity check for CHARMM.""" - custom_paths = { - 'files': [], - 'dirs': [], - } - if self.toolchain.options.get('usempi', None): - custom_paths['files'].append('exec/%s_M/charmm' % self.arch) - else: - custom_paths['files'].append('exec/%s/charmm' % self.arch) - super(EB_CHARMM, self).sanity_check_step(custom_paths=custom_paths) - - def install_step(self): - """Copy the build directory to the install path""" - self.log.info("Copying CHARMM dir %s to %s" % (self.cfg['start_dir'], self.installdir)) - try: - shutil.copytree(self.cfg['start_dir'], self.installdir) - except OSError as err: - raise EasyBuildError("Failed to copy CHARMM dir to install dir: %s", err) - - def make_module_req_guess(self): - """Custom guesses for environment variable PATH for CHARMM.""" - guesses = super(EB_CHARMM, self).make_module_req_guess() - if self.toolchain.options.get('usempi', None): - suffix = "_M" - else: - suffix = "" - guesses.update({ - 'PATH': ['exec/%s%s' % (self.arch, suffix)], - }) - return guesses diff --git a/easybuild/easyblocks/c/chimera.py b/easybuild/easyblocks/c/chimera.py index 6aab8f1f5af..2945a53e0fc 100644 --- a/easybuild/easyblocks/c/chimera.py +++ b/easybuild/easyblocks/c/chimera.py @@ -11,7 +11,7 @@ from easybuild.easyblocks.generic.packedbinary import PackedBinary from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import mkdir, symlink -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd class EB_Chimera(PackedBinary): @@ -22,7 +22,7 @@ def extract_step(self, verbose=False): to obtain chimera.bin installer.""" cmd = "unzip -d %s %s" % (self.builddir, self.src[0]['path']) - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) def install_step(self): """Install using chimera.bin.""" @@ -41,7 +41,7 @@ def install_step(self): # root directory. cmd = "./chimera.bin -q -d %s" % os.path.join(self.installdir, 'chimera') - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) # Create a symlink to the Chimera startup script; this symlink # will end up in PATH. The startup script sets up the diff --git a/easybuild/easyblocks/c/clang.py b/easybuild/easyblocks/c/clang.py index af9c1059104..6c700b08741 100644 --- a/easybuild/easyblocks/c/clang.py +++ b/easybuild/easyblocks/c/clang.py @@ -1,6 +1,6 @@ ## -# Copyright 2013-2024 Dmitri Gribenko -# Copyright 2013-2024 Ghent University +# Copyright 2013-2025 Dmitri Gribenko +# Copyright 2013-2025 Ghent University # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. @@ -43,12 +43,12 @@ from easybuild.easyblocks.generic.cmakemake import CMakeMake from easybuild.framework.easyconfig import CUSTOM from easybuild.toolchains.compiler.clang import Clang -from easybuild.tools import run +from easybuild.tools.config import ERROR from easybuild.tools.build_log import EasyBuildError, print_warning from easybuild.tools.config import build_option from easybuild.tools.filetools import apply_regex_substitutions, change_dir, mkdir, symlink, which -from easybuild.tools.modules import get_software_root -from easybuild.tools.run import run_cmd +from easybuild.tools.modules import MODULE_LOAD_ENV_HEADERS, get_software_root +from easybuild.tools.run import run_shell_cmd from easybuild.tools.systemtools import AARCH32, AARCH64, POWER, RISCV64, X86_64 from easybuild.tools.systemtools import get_cpu_architecture, get_os_name, get_os_version, get_shared_lib_ext from easybuild.tools.environment import setvar @@ -124,9 +124,11 @@ def __init__(self, *args, **kwargs): self.llvm_obj_dir_stage1 = None self.llvm_obj_dir_stage2 = None self.llvm_obj_dir_stage3 = None - self.make_parallel_opts = "" self.runtime_lib_path = "lib" + # Bypass the .mod file check for GCCcore installs + self.cfg['skip_mod_files_sanity_check'] = True + if not self.cfg['llvm_projects']: self.cfg['llvm_projects'] = [] if not self.cfg['llvm_runtimes']: @@ -325,21 +327,21 @@ def configure_step(self): disable_san_tests = False # all sanitizer tests will fail when there's a limit on the vmem # this is ugly but I haven't found a cleaner way so far - (vmemlim, ec) = run_cmd("ulimit -v", regexp=False) - if not vmemlim.startswith("unlimited"): + res = run_shell_cmd("ulimit -v", fail_on_error=False) + if not res.output.startswith("unlimited"): disable_san_tests = True - self.log.warn("There is a virtual memory limit set of %s KB. The tests of the " - "sanitizers will be disabled as they need unlimited virtual " - "memory unless --strict=error is used." % vmemlim.strip()) + self.log.warning("There is a virtual memory limit set of %s KB. The tests of the " + "sanitizers will be disabled as they need unlimited virtual " + "memory unless --strict=error is used." % res.output.strip()) # the same goes for unlimited stacksize - (stacklim, ec) = run_cmd("ulimit -s", regexp=False) - if stacklim.startswith("unlimited"): + res = run_shell_cmd("ulimit -s", fail_on_error=False) + if res.output.startswith("unlimited"): disable_san_tests = True - self.log.warn("The stacksize limit is set to unlimited. This causes the ThreadSanitizer " - "to fail. The sanitizers tests will be disabled unless --strict=error is used.") + self.log.warning("The stacksize limit is set to unlimited. This causes the ThreadSanitizer " + "to fail. The sanitizers tests will be disabled unless --strict=error is used.") - if (disable_san_tests or self.cfg['skip_sanitizer_tests']) and build_option('strict') != run.ERROR: + if (disable_san_tests or self.cfg['skip_sanitizer_tests']) and build_option('strict') != ERROR: self.log.debug("Disabling the sanitizer tests") self.disable_sanitizer_tests() @@ -401,9 +403,6 @@ def configure_step(self): self.cfg.update('configopts', '-DLLVM_TARGETS_TO_BUILD="%s"' % ';'.join(build_targets)) - if self.cfg['parallel']: - self.make_parallel_opts = "-j %s" % self.cfg['parallel'] - # If hwloc is included as a dep, use it in OpenMP runtime for affinity hwloc_root = get_software_root('hwloc') if hwloc_root: @@ -546,12 +545,12 @@ def build_with_prev_stage(self, prev_obj, next_obj): self.log.info("Configuring") if LooseVersion(self.version) >= LooseVersion('14'): - run_cmd("cmake %s %s" % (' '.join(options), os.path.join(self.llvm_src_dir, "llvm")), log_all=True) + run_shell_cmd("cmake %s %s" % (' '.join(options), os.path.join(self.llvm_src_dir, "llvm"))) else: - run_cmd("cmake %s %s" % (' '.join(options), self.llvm_src_dir), log_all=True) + run_shell_cmd("cmake %s %s" % (' '.join(options), self.llvm_src_dir)) self.log.info("Building") - run_cmd("make %s VERBOSE=1" % self.make_parallel_opts, log_all=True) + run_shell_cmd(f"make {self.parallel_flag} VERBOSE=1") # restore $PATH setvar('PATH', orig_path) @@ -578,7 +577,7 @@ def test_step(self): change_dir(self.llvm_obj_dir_stage3) else: change_dir(self.llvm_obj_dir_stage1) - run_cmd("make %s check-all" % self.make_parallel_opts, log_all=True) + run_shell_cmd(f"make {self.parallel_flag} check-all") def install_step(self): """Install stage 3 binaries.""" @@ -609,9 +608,9 @@ def install_step(self): except OSError as err: raise EasyBuildError("Failed to copy static analyzer dirs to install dir: %s", err) - def post_install_step(self): + def post_processing_step(self): """Install python bindings.""" - super(EB_Clang, self).post_install_step() + super(EB_Clang, self).post_processing_step() # copy Python bindings here in post-install step so that it is not done more than once in multi_deps context if self.cfg['python_bindings']: @@ -754,6 +753,21 @@ def sanity_check_step(self): super(EB_Clang, self).sanity_check_step(custom_paths=custom_paths, custom_commands=custom_commands) + def make_module_step(self, *args, **kwargs): + """ + Set paths for module load environment based on the actual installation files + """ + # Ensure that installation files are not added to search paths to headers and libs + mod_env_headers = self.module_load_environment.alias_vars(MODULE_LOAD_ENV_HEADERS) + mod_env_libs = ['LIBRARY_PATH'] + for disallowed_var in mod_env_headers + mod_env_libs: + self.module_load_environment.remove(disallowed_var) + self.log.debug(f"Purposely not updating ${disallowed_var} in {self.name} module file") + # Clang can find its own headers and libraries but the .so's need to be in LD_LIBRARY_PATH + self.module_load_environment.LD_LIBRARY_PATH = ['lib', 'lib64', self.runtime_lib_path] + + return super().make_module_step(*args, **kwargs) + def make_module_extra(self): """Custom variables for Clang module.""" txt = super(EB_Clang, self).make_module_extra() @@ -763,15 +777,3 @@ def make_module_extra(self): if self.cfg['python_bindings']: txt += self.module_generator.prepend_paths('PYTHONPATH', os.path.join("lib", "python")) return txt - - def make_module_req_guess(self): - """ - Clang can find its own headers and libraries but the .so's need to be in LD_LIBRARY_PATH - """ - guesses = super(EB_Clang, self).make_module_req_guess() - guesses.update({ - 'CPATH': [], - 'LIBRARY_PATH': [], - 'LD_LIBRARY_PATH': ['lib', 'lib64', self.runtime_lib_path], - }) - return guesses diff --git a/easybuild/easyblocks/c/clang_aomp.py b/easybuild/easyblocks/c/clang_aomp.py index 6f046cdb28c..e3d86502f2c 100644 --- a/easybuild/easyblocks/c/clang_aomp.py +++ b/easybuild/easyblocks/c/clang_aomp.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -145,7 +145,7 @@ def configure_step(self): raise EasyBuildError("Could not find 'ROCm-Device-Libs' source directory in %s", self.builddir) num_comps = len(self.cfg['components']) - for idx, comp in enumerate(self.comp_cfgs): + for idx, (comp, _) in enumerate(self.comp_instances): name = comp['name'] msg = "configuring bundle component %s %s (%d/%d)..." % (name, comp['version'], idx + 1, num_comps) print_msg(msg) @@ -153,7 +153,7 @@ def configure_step(self): self.cfg_method[name](comp) self.log.info(msg) else: - self.log.warn("Component %s has no configure method!" % name) + self.log.warning("Component %s has no configure method!" % name) def sanity_check_step(self): """ diff --git a/easybuild/easyblocks/c/cmake.py b/easybuild/easyblocks/c/cmake.py index c17750ae459..91356bf73fc 100644 --- a/easybuild/easyblocks/c/cmake.py +++ b/easybuild/easyblocks/c/cmake.py @@ -1,5 +1,5 @@ ## -# Copyright 2020-2024 Alexander Grund +# Copyright 2020-2025 Alexander Grund # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/c/code_server.py b/easybuild/easyblocks/c/code_server.py index d8ae7276cc2..fc9d615d502 100644 --- a/easybuild/easyblocks/c/code_server.py +++ b/easybuild/easyblocks/c/code_server.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2024 Ghent University +# Copyright 2012-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/c/comsol.py b/easybuild/easyblocks/c/comsol.py index d38dd75d646..03e768e5a54 100644 --- a/easybuild/easyblocks/c/comsol.py +++ b/easybuild/easyblocks/c/comsol.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -37,7 +37,7 @@ from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import adjust_permissions, copy_file, find_flexlm_license, read_file, write_file from easybuild.tools.modules import get_software_root -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd from easybuild.tools.systemtools import get_shared_lib_ext @@ -124,7 +124,7 @@ def install_step(self): env.unset_env_vars(['DISPLAY']) cmd = ' '.join([self.cfg['preinstallopts'], setup_script, '-s', self.configfile, self.cfg['installopts']]) - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) def sanity_check_step(self): """Custom sanity check for COMSOL.""" diff --git a/easybuild/easyblocks/c/cp2k.py b/easybuild/easyblocks/c/cp2k.py index 1a0f109878c..8251194b100 100644 --- a/easybuild/easyblocks/c/cp2k.py +++ b/easybuild/easyblocks/c/cp2k.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -52,7 +52,7 @@ from easybuild.tools.filetools import change_dir, copy_dir, copy_file, mkdir, write_file from easybuild.tools.config import build_option from easybuild.tools.modules import get_software_root, get_software_version -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd from easybuild.tools.systemtools import get_avail_core_count @@ -290,7 +290,7 @@ def prepmodinc(self): else: raise EasyBuildError("prepmodinc: Unknown value specified for F77 (%s)", f77) - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) return modincpath else: @@ -392,7 +392,8 @@ def configure_common(self): # build libint wrapper cmd = "%s -c libint_cpp_wrapper.cpp -I%s/include" % (libintcompiler, libint) - if not run_cmd(cmd, log_all=True, simple=True): + res = run_shell_cmd(cmd, fail_on_error=False) + if res.exit_code: raise EasyBuildError("Building the libint wrapper failed") libint_wrapper = '%s/libint_cpp_wrapper.o' % libinttools_path @@ -668,18 +669,17 @@ def build_step(self): change_dir(makefiles) # modify makefile for parallel build - parallel = self.cfg['parallel'] - if parallel: - + parallel = self.cfg.parallel + if parallel > 1: try: for line in fileinput.input('Makefile', inplace=1, backup='.orig.patchictce'): - line = re.sub(r"^PMAKE\s*=.*$", "PMAKE\t= $(SMAKE) -j %s" % parallel, line) + line = re.sub(r"^PMAKE\s*=.*$", f"PMAKE\t= $(SMAKE) -j {parallel}", line) sys.stdout.write(line) except IOError as err: raise EasyBuildError("Can't modify/write Makefile in %s: %s", makefiles, err) # update make options with MAKE - self.cfg.update('buildopts', 'MAKE="make -j %s"' % self.cfg['parallel']) + self.cfg.update('buildopts', f'MAKE="make -j {self.cfg.parallel}"') # update make options with ARCH and VERSION self.cfg.update('buildopts', 'ARCH=%s VERSION=%s' % (self.typearch, self.cfg['type'])) @@ -687,16 +687,16 @@ def build_step(self): cmd = "make %s" % self.cfg['buildopts'] # clean first - run_cmd(cmd + " clean", log_all=True, simple=True, log_output=True) + run_shell_cmd(cmd + " clean") # build and install # compile regularly first with the default make target # and only then build the library - run_cmd(cmd + ' all', log_all=True, simple=True, log_output=True) + run_shell_cmd(cmd + ' all') # build as a library if self.cfg['library']: - run_cmd(cmd + 'libcp2k', log_all=True, simple=True, log_output=True) + run_shell_cmd(cmd + 'libcp2k') def test_step(self): """Run regression test.""" @@ -756,7 +756,7 @@ def test_step(self): self.log.info("No reference output found for regression test, just continuing without it...") # prefer using 4 cores, since some tests require/prefer square (n^2) numbers or powers of 2 (2^n) - test_core_cnt = min(self.cfg['parallel'], 4) + test_core_cnt = min(self.cfg.parallel, 4) if get_avail_core_count() < test_core_cnt: raise EasyBuildError("Cannot run MPI tests as not enough cores (< %s) are available", test_core_cnt) else: @@ -787,19 +787,19 @@ def test_step(self): self.log.debug("Contents of %s: %s" % (cfg_fn, cfg_txt)) # run regression test - (regtest_output, ec) = run_cmd(regtest_cmd, log_all=True, simple=False, log_output=True) + regtest = run_shell_cmd(regtest_cmd, fail_on_error=False) - if ec == 0: - self.log.info("Regression test output:\n%s" % regtest_output) + if regtest.exit_code == 0: + self.log.info("Regression test output:\n%s" % regtest.output) else: - raise EasyBuildError("Regression test failed (non-zero exit code): %s", regtest_output) + raise EasyBuildError("Regression test failed (non-zero exit code): %s", regtest.output) # pattern to search for regression test summary re_pattern = r"number\s+of\s+%s\s+tests\s+(?P[0-9]+)" # find total number of tests regexp = re.compile(re_pattern % "", re.M | re.I) - res = regexp.search(regtest_output) + res = regexp.search(regtest.output) tot_cnt = None if res: tot_cnt = int(res.group('cnt')) @@ -816,7 +816,7 @@ def test_report(test_result): regexp = re.compile(re_pattern % test_result, re.M | re.I) cnt = None - res = regexp.search(regtest_output) + res = regexp.search(regtest.output) if not res: raise EasyBuildError("Finding number of %s tests in regression test summary failed", test_result.lower()) diff --git a/easybuild/easyblocks/c/cplex.py b/easybuild/easyblocks/c/cplex.py index 286683bad9d..d5c8ab53ce5 100644 --- a/easybuild/easyblocks/c/cplex.py +++ b/easybuild/easyblocks/c/cplex.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -38,12 +38,10 @@ import easybuild.tools.environment as env from easybuild.easyblocks.generic.binary import Binary -from easybuild.easyblocks.generic.pythonpackage import det_pylibdir -from easybuild.easyblocks.python import EBPYTHONPREFIXES from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import adjust_permissions, change_dir, mkdir from easybuild.tools.modules import get_software_root -from easybuild.tools.run import run_cmd, run_cmd_qa +from easybuild.tools.run import run_shell_cmd class EB_CPLEX(Binary): @@ -60,6 +58,9 @@ def __init__(self, *args, **kwargs): self.with_python = False self.multi_python = 'Python' in self.cfg['multi_deps'] + # Bypass the .mod file check for GCCcore installs + self.cfg['skip_mod_files_sanity_check'] = True + def prepare_step(self, *args, **kwargs): """Prepare build environment.""" super(EB_CPLEX, self).prepare_step(*args, **kwargs) @@ -81,23 +82,23 @@ def install_step(self): cmd = "%s -i console" % dst - qanda = { - "PRESS TO CONTINUE:": '', - 'Press Enter to continue viewing the license agreement, or enter' - ' "1" to accept the agreement, "2" to decline it, "3" to print it,' - ' or "99" to go back to the previous screen.:': '1', - 'ENTER AN ABSOLUTE PATH, OR PRESS TO ACCEPT THE DEFAULT :': self.installdir, - 'IS THIS CORRECT? (Y/N):': 'y', - 'PRESS TO INSTALL:': '', - "PRESS TO EXIT THE INSTALLER:": '', - "CHOOSE LOCALE BY NUMBER:": '', - "Choose Instance Management Option:": '', - "No model content or proprietary data will be sent.\n1- Yes\n2- No\n" - "ENTER THE NUMBER OF THE DESIRED CHOICE:": '2', - } - noqanda = [r'Installing\.\.\..*\n.*------.*\n\n.*============.*\n.*$'] - - run_cmd_qa(cmd, qanda, no_qa=noqanda, log_all=True, simple=True) + qa = [ + (r"PRESS TO CONTINUE:", ''), + (r'Press Enter to continue viewing the license agreement, or enter' + r' "1" to accept the agreement, "2" to decline it, "3" to print it,' + r' or "99" to go back to the previous screen\.:', '1'), + (r'ENTER AN ABSOLUTE PATH, OR PRESS TO ACCEPT THE DEFAULT :', self.installdir), + (r'IS THIS CORRECT\? \(Y/N\):', 'y'), + (r'PRESS TO INSTALL:', ''), + (r"PRESS TO EXIT THE INSTALLER:", ''), + (r"CHOOSE LOCALE BY NUMBER:", ''), + (r"Choose Instance Management Option:", ''), + (r"No model content or proprietary data will be sent.\n1- Yes\n2- No\n" + r"ENTER THE NUMBER OF THE DESIRED CHOICE:", '2'), + ] + no_qa = [r'Installing\.\.\..*\n.*------.*\n\n.*============.*\n.*$'] + + run_shell_cmd(cmd, qa_patterns=qa, qa_wait_patterns=no_qa) # fix permissions on install dir perms = stat.S_IRWXU | stat.S_IXOTH | stat.S_IXGRP | stat.S_IROTH | stat.S_IRGRP @@ -106,7 +107,7 @@ def install_step(self): # also install Python bindings if Python is included as a dependency if self.with_python: cwd = change_dir(os.path.join(self.installdir, 'python')) - run_cmd("python setup.py install --prefix=%s" % self.installdir) + run_shell_cmd("python setup.py install --prefix=%s" % self.installdir) change_dir(cwd) def det_bindir(self): @@ -147,12 +148,6 @@ def make_module_extra(self): txt += self.module_generator.set_environment('CPLEX_HOME', os.path.join(self.installdir, 'cplex')) txt += self.module_generator.set_environment('CPLEXDIR', os.path.join(self.installdir, 'cplex')) - if self.with_python: - if self.multi_python: - txt += self.module_generator.prepend_paths(EBPYTHONPREFIXES, '') - else: - txt += self.module_generator.prepend_paths('PYTHONPATH', [det_pylibdir()]) - self.log.debug("make_module_extra added %s" % txt) return txt diff --git a/easybuild/easyblocks/c/cppcheck.py b/easybuild/easyblocks/c/cppcheck.py deleted file mode 100644 index 3fec7d84ad3..00000000000 --- a/easybuild/easyblocks/c/cppcheck.py +++ /dev/null @@ -1,139 +0,0 @@ -## -# Copyright 2016-2024 Forschungszentrum Juelich GmbH -# -# This file is part of EasyBuild, -# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), -# with support of Ghent University (http://ugent.be/hpc), -# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), -# Flemish Research Foundation (FWO) (http://www.fwo.be/en) -# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). -# -# https://github.com/easybuilders/easybuild -# -# EasyBuild is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation v2. -# -# EasyBuild is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with EasyBuild. If not, see . -## -""" -@author: Damian Alvarez (Forschungszentrum Juelich GmbH) -""" -import os -import shutil - -from easybuild.easyblocks.generic.configuremake import ConfigureMake -from easybuild.framework.easyconfig import CUSTOM -from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.run import run_cmd - - -class EB_cppcheck(ConfigureMake): - """ - EasyBlock to install cppcheck - """ - - @staticmethod - def extra_options(extra_vars=None): - """ - Define if we are using rules or not, and if we are building the GUI - """ - extra = { - 'have_rules': [False, "Use rules", CUSTOM], - 'build_gui': [False, "Build GUI", CUSTOM], - } - if extra_vars is None: - extra_vars = {} - extra.update(extra_vars) - return ConfigureMake.extra_options(extra_vars=extra) - - def configure_step(self): - """ - Run qmake on the GUI, if necessary - """ - if self.cfg['build_gui']: - cmd = 'qmake QMAKE_CXX="$CXX" QMAKE_LINK="$CXX"' - - if self.cfg['have_rules']: - cmd = ' '.join([cmd, 'HAVE_RULES=yes']) - - gui_dir = os.path.join(self.cfg['start_dir'], 'gui') - - try: - os.chdir(gui_dir) - run_cmd(cmd, log_all=True, simple=True) - os.chdir(self.cfg['start_dir']) - except OSError as err: - raise EasyBuildError("Moving to %s and configure the GUI build failed: %s", gui_dir, err) - - else: - self.log.debug("Configuration of the GUI skipped") - - def build_step(self, verbose=False): - """ - Compile cppcheck with make and cppcheck-gui with qmake and make - """ - self.cfg.update('buildopts', 'CFGDIR=%(installdir)s/cfg') - - if self.cfg['have_rules']: - self.cfg.update('buildopts', 'HAVE_RULES=yes') - - super(EB_cppcheck, self).build_step(verbose=verbose) - - if self.cfg['build_gui']: - super(EB_cppcheck, self).build_step(verbose=verbose, path='gui') - - def install_step(self): - """ - Install cppcheck with make install and cppcheck-gui copying the file - """ - self.cfg.update('installopts', 'DESTDIR=%(installdir)s/ PREFIX="" CFGDIR=cfg') - - super(EB_cppcheck, self).install_step() - - # The GUI doesn't have make install, so we copy it manually - if self.cfg['build_gui']: - filepath = os.path.join(self.cfg['start_dir'], 'gui/', 'cppcheck-gui') - - try: - # make sure we're (still) in the start dir - os.chdir(self.cfg['start_dir']) - - # Perform the copy - target_dest = os.path.join(self.installdir, 'bin') - if os.path.isfile(filepath): - self.log.debug("Copying file %s to %s" % (filepath, target_dest)) - shutil.copy2(filepath, target_dest) - else: - raise EasyBuildError("Can't copy non-existing file %s to %s", filepath, target_dest) - - # Create cfg symlink. It shouldn't be necessary, but cppcheck-gui complains otherwise - # unless called with --data-dir - src = os.path.join(self.installdir, 'cfg') - dst = os.path.join(self.installdir, 'bin/cfg') - self.log.debug("Creating symlink %s to %s" % (src, dst)) - os.symlink(src, dst) - - except OSError as err: - raise EasyBuildError("Copying %s to installation dir failed: %s", filepath, err) - - def sanity_check_step(self): - """ - Custom sanity check for cppcheck. - """ - custom_paths = { - 'files': ['bin/cppcheck'], - 'dirs': [] - } - - if self.cfg['build_gui']: - custom_paths['files'].append('bin/cppcheck-gui') - custom_paths['dirs'].append('bin/cfg') - - super(EB_cppcheck, self).sanity_check_step(custom_paths=custom_paths) diff --git a/easybuild/easyblocks/c/crispr_dav.py b/easybuild/easyblocks/c/crispr_dav.py index e85da91da26..4d11b3f1eee 100644 --- a/easybuild/easyblocks/c/crispr_dav.py +++ b/easybuild/easyblocks/c/crispr_dav.py @@ -1,5 +1,5 @@ ## -# Copyright 2020-2024 Ghent University +# Copyright 2020-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -46,7 +46,7 @@ def __init__(self, *args, **kwargs): super(EB_CRISPR_minus_DAV, self).__init__(*args, **kwargs) self.cfg['extract_sources'] = True - def post_install_step(self): + def post_processing_step(self): """Update configuration files with correct paths to dependencies and files in installation.""" # getting paths of deps + files we will work with diff --git a/easybuild/easyblocks/c/cryptography.py b/easybuild/easyblocks/c/cryptography.py index 12cee791117..251499a70d6 100644 --- a/easybuild/easyblocks/c/cryptography.py +++ b/easybuild/easyblocks/c/cryptography.py @@ -1,5 +1,5 @@ ## -# Copyright 2017-2024 Ghent University +# Copyright 2017-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -30,7 +30,7 @@ from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.pythonpackage import PythonPackage -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd class EB_cryptography(PythonPackage): @@ -57,5 +57,5 @@ def sanity_check_step(self, *args, **kwargs): if success: # Check module added in v0.7 leading to issue #9446 (see above) if LooseVersion(self.version) >= LooseVersion("0.7"): - run_cmd("python -c 'from cryptography.hazmat.bindings.openssl import binding'") + run_shell_cmd("python -c 'from cryptography.hazmat.bindings.openssl import binding'") return success, fail_msg diff --git a/easybuild/easyblocks/c/cuda.py b/easybuild/easyblocks/c/cuda.py index 10d6bc30837..98a4bd0eeef 100644 --- a/easybuild/easyblocks/c/cuda.py +++ b/easybuild/easyblocks/c/cuda.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2024 Ghent University +# Copyright 2012-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -46,7 +46,7 @@ from easybuild.tools.config import IGNORE from easybuild.tools.filetools import adjust_permissions, change_dir, copy_dir, expand_glob_paths from easybuild.tools.filetools import patch_perl_script_autoflush, remove_file, symlink, which, write_file -from easybuild.tools.run import run_cmd, run_cmd_qa +from easybuild.tools.run import run_shell_cmd from easybuild.tools.systemtools import AARCH64, POWER, X86_64, get_cpu_architecture, get_shared_lib_ext import easybuild.tools.environment as env @@ -92,6 +92,28 @@ def __init__(self, *args, **kwargs): self.cfg.template_values['cudaarch'] = cudaarch self.cfg.generate_template_values() + # Specify CUDA custom values for module load environment + # The dirs should be in the order ['open64/bin', 'bin'] + bin_path = [] + if LooseVersion(self.version) < LooseVersion('7'): + bin_path.append(os.path.join('open64', 'bin')) + bin_path.append('bin') + + lib_path = ['lib64'] + inc_path = ['include'] + if LooseVersion(self.version) >= LooseVersion('7'): + lib_path.append(os.path.join('extras', 'CUPTI', 'lib64')) + inc_path.append(os.path.join('extras', 'CUPTI', 'include')) + bin_path.append(os.path.join('nvvm', 'bin')) + lib_path.append(os.path.join('nvvm', 'lib64')) + inc_path.append(os.path.join('nvvm', 'include')) + + self.module_load_environment.CPATH = inc_path + self.module_load_environment.LD_LIBRARY_PATH = lib_path + self.module_load_environment.LIBRARY_PATH = lib_path + [os.path.join('stubs', 'lib64')] + self.module_load_environment.PATH = bin_path + self.module_load_environment.PKG_CONFIG_PATH = ['pkgconfig'] + def fetch_step(self, *args, **kwargs): """Check for EULA acceptance prior to getting sources.""" # EULA for CUDA must be accepted via --accept-eula-for EasyBuild configuration option, @@ -105,7 +127,7 @@ def fetch_step(self, *args, **kwargs): def extract_step(self): """Extract installer to have more control, e.g. options, patching Perl scripts, etc.""" execpath = self.src[0]['path'] - run_cmd("/bin/sh " + execpath + " --noexec --nox11 --target " + self.builddir) + run_shell_cmd("/bin/sh " + execpath + " --noexec --nox11 --target " + self.builddir) self.src[0]['finalpath'] = self.builddir def install_step(self): @@ -172,12 +194,11 @@ def install_step(self): } # prepare for running install script autonomously - qanda = {} - stdqa = { + qa = [ # this question is only asked if CUDA tools are already available system-wide - r"Would you like to remove all CUDA files under .*? (yes/no/abort): ": "no", - } - noqanda = [ + (r"Would you like to remove all CUDA files under .*\? \(yes/no/abort\): ", "no"), + ] + no_qa = [ r"^Configuring", r"Installation Complete", r"Verifying archive integrity.*", @@ -206,9 +227,9 @@ def install_step(self): # instead of segfaulting in the cuda-installer. remove_file('/tmp/cuda-installer.log') - # overriding maxhits default value to 1000 (seconds to wait for nothing to change in the output + # overriding qa_timeout default value to 1000 (seconds to wait for nothing to change in the output # without seeing a known question) - run_cmd_qa(cmd, qanda, std_qa=stdqa, no_qa=noqanda, log_all=True, simple=True, maxhits=1000) + run_shell_cmd(cmd, qa_patterns=qa, qa_wait_patterns=no_qa, qa_timeout=1000) # Remove the cuda-installer log file remove_file('/tmp/cuda-installer.log') @@ -217,9 +238,9 @@ def install_step(self): if len(self.src) > 1: for patch in self.src[1:]: self.log.debug("Running patch %s", patch['name']) - run_cmd("/bin/sh " + patch['path'] + " --accept-eula --silent --installdir=" + self.installdir) + run_shell_cmd("/bin/sh " + patch['path'] + " --accept-eula --silent --installdir=" + self.installdir) - def post_install_step(self): + def post_processing_step(self): """ Create wrappers for the specified host compilers, generate the appropriate stub symlinks, and create version independent pkgconfig files @@ -265,7 +286,7 @@ def create_wrapper(wrapper_name, wrapper_comp): # Run ldconfig to create missing symlinks in the stubs directory (libcuda.so.1, etc) cmd = ' '.join([ldconfig, '-N', stubs_dir]) - run_cmd(cmd) + run_shell_cmd(cmd) # GCC searches paths in LIBRARY_PATH and the system paths suffixed with ../lib64 or ../lib first # This means stubs/../lib64 is searched before the system /lib64 folder containing a potentially older libcuda. @@ -289,7 +310,7 @@ def create_wrapper(wrapper_name, wrapper_comp): symlink(pc_file, link, use_abspath_source=False) change_dir(cwd) - super(EB_CUDA, self).post_install_step() + super(EB_CUDA, self).post_processing_step() def sanity_check_step(self): """Custom sanity check for CUDA.""" @@ -335,33 +356,3 @@ def make_module_extra(self): txt += self.module_generator.set_environment('CUDA_PATH', self.installdir) self.log.debug("make_module_extra added this: %s", txt) return txt - - def make_module_req_guess(self): - """Specify CUDA custom values for PATH etc.""" - - guesses = super(EB_CUDA, self).make_module_req_guess() - - # The dirs should be in the order ['open64/bin', 'bin'] - bin_path = [] - if LooseVersion(self.version) < LooseVersion('7'): - bin_path.append(os.path.join('open64', 'bin')) - bin_path.append('bin') - - lib_path = ['lib64'] - inc_path = ['include'] - if LooseVersion(self.version) >= LooseVersion('7'): - lib_path.append(os.path.join('extras', 'CUPTI', 'lib64')) - inc_path.append(os.path.join('extras', 'CUPTI', 'include')) - bin_path.append(os.path.join('nvvm', 'bin')) - lib_path.append(os.path.join('nvvm', 'lib64')) - inc_path.append(os.path.join('nvvm', 'include')) - - guesses.update({ - 'CPATH': inc_path, - 'LD_LIBRARY_PATH': lib_path, - 'LIBRARY_PATH': ['lib64', os.path.join('stubs', 'lib64')], - 'PATH': bin_path, - 'PKG_CONFIG_PATH': ['pkgconfig'], - }) - - return guesses diff --git a/easybuild/easyblocks/c/cudacompat.py b/easybuild/easyblocks/c/cudacompat.py index 1013b57137c..e9f7ca9f368 100644 --- a/easybuild/easyblocks/c/cudacompat.py +++ b/easybuild/easyblocks/c/cudacompat.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2024 Ghent University +# Copyright 2012-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -38,7 +38,7 @@ from easybuild.tools.build_log import EasyBuildError, print_warning from easybuild.tools.config import build_option, IGNORE from easybuild.tools.filetools import copy_file, find_glob_pattern, mkdir, symlink, which -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd class EB_CUDAcompat(Binary): @@ -65,6 +65,8 @@ def __init__(self, *args, **kwargs): """Initialize custom class variables for CUDACompat.""" super(EB_CUDAcompat, self).__init__(*args, **kwargs) self._has_nvidia_smi = None + # avoid building software with this compat libraries + self.module_load_environment.remove('LIBRARY_PATH') @property def has_nvidia_smi(self): @@ -85,12 +87,12 @@ def _run_nvidia_smi(self, args): if not self.has_nvidia_smi: raise RuntimeError('Could not find nvidia-smi.') cmd = 'nvidia-smi ' + args - out, ec = run_cmd(cmd, log_ok=False, log_all=False, regexp=False) - if ec != 0: - raise RuntimeError("`%s` returned exit code %s with output:\n%s" % (cmd, ec, out)) + res = run_shell_cmd(cmd, fail_on_error=False) + if res.exit_code != 0: + raise RuntimeError("`%s` returned exit code %s with output:\n%s" % (cmd, res.exit_code, res.output)) else: - self.log.info('`%s` succeeded with output:\n%s' % (cmd, out)) - return out.strip().split('\n') + self.log.info('`%s` succeeded with output:\n%s' % (cmd, res.output)) + return res.output.strip().split('\n') def prepare_step(self, *args, **kwargs): """Parse and check the compatible_driver_versions value of the EasyConfig""" @@ -127,7 +129,7 @@ def extract_step(self): execpath = self.src[0]['path'] tmpdir = os.path.join(self.builddir, 'tmp') targetdir = os.path.join(self.builddir, 'extracted') - run_cmd("/bin/sh " + execpath + " --extract-only --tmpdir='%s' --target '%s'" % (tmpdir, targetdir)) + run_shell_cmd("/bin/sh " + execpath + " --extract-only --tmpdir='%s' --target '%s'" % (tmpdir, targetdir)) self.src[0]['finalpath'] = targetdir def test_step(self): @@ -214,17 +216,6 @@ def install_step(self): unversioned_symlink = versioned_symlink.rsplit('.', 1)[0] symlink(versioned_symlink, os.path.join(libdir, unversioned_symlink), use_abspath_source=False) - def make_module_req_guess(self): - """Don't try to guess anything.""" - return dict() - - def make_module_extra(self): - """Skip the changes from the Binary EasyBlock and (only) set LD_LIBRARY_PATH.""" - - txt = super(Binary, self).make_module_extra() - txt += self.module_generator.prepend_paths('LD_LIBRARY_PATH', 'lib') - return txt - def sanity_check_step(self): """Check for core files (unversioned libs, symlinks)""" libraries = [ diff --git a/easybuild/easyblocks/c/cudnn.py b/easybuild/easyblocks/c/cudnn.py index 9c1b3557b76..37af2df5537 100644 --- a/easybuild/easyblocks/c/cudnn.py +++ b/easybuild/easyblocks/c/cudnn.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2024 Ghent University +# Copyright 2012-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -68,3 +68,13 @@ def __init__(self, *args, **kwargs): self.cfg['keepsymlinks'] = True self.cfg.template_values['cudnnarch'] = cudnnarch self.cfg.generate_template_values() + + def fetch_step(self, *args, **kwargs): + """Check for EULA acceptance prior to getting sources.""" + # EULA for cuDNN must be accepted via --accept-eula-for EasyBuild configuration option, + # or via 'accept_eula = True' in easyconfig file + self.check_accepted_eula( + name='cuDNN', + more_info='https://docs.nvidia.com/deeplearning/cudnn/latest/reference/eula.html' + ) + return super(EB_cuDNN, self).fetch_step(*args, **kwargs) diff --git a/easybuild/easyblocks/c/cufflinks.py b/easybuild/easyblocks/c/cufflinks.py index c6753a07d2f..b666da4e71d 100644 --- a/easybuild/easyblocks/c/cufflinks.py +++ b/easybuild/easyblocks/c/cufflinks.py @@ -1,7 +1,7 @@ ## # This file is an EasyBuild reciPY as per https://github.com/easybuilders/easybuild # -# Copyright:: Copyright 2012-2024 Uni.Lu/LCSB, NTUA +# Copyright:: Copyright 2012-2025 Uni.Lu/LCSB, NTUA # Authors:: Cedric Laczny , Fotis Georgatos , Kenneth Hoste # License:: MIT/GPL # $Id$ diff --git a/easybuild/easyblocks/d/db.py b/easybuild/easyblocks/d/db.py index b4d1e015903..67628ce97ad 100644 --- a/easybuild/easyblocks/d/db.py +++ b/easybuild/easyblocks/d/db.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2024 Ghent University +# Copyright 2013-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/d/dl_poly_classic.py b/easybuild/easyblocks/d/dl_poly_classic.py deleted file mode 100644 index 991b32e042a..00000000000 --- a/easybuild/easyblocks/d/dl_poly_classic.py +++ /dev/null @@ -1,111 +0,0 @@ -## -# Copyright 2013-2024 Ghent University -# -# This file is part of EasyBuild, -# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), -# with support of Ghent University (http://ugent.be/hpc), -# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), -# Flemish Research Foundation (FWO) (http://www.fwo.be/en) -# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). -# -# https://github.com/easybuilders/easybuild -# -# EasyBuild is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation v2. -# -# EasyBuild is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with EasyBuild. If not, see . -## -""" -EasyBuild support for DL_POLY Classic, implemented as an easyblock - -@author: Jens Timmerman (Ghent University) -@author: Kenneth Hoste (Ghent University) -""" -import glob -import os - -from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.filetools import copy_file, copy_dir -from easybuild.tools.run import run_cmd -from easybuild.easyblocks.generic.configuremake import ConfigureMake - - -class EB_DL_underscore_POLY_underscore_Classic(ConfigureMake): - """Support for building and installing DL_POLY Classic.""" - - def __init__(self, *args, **kwargs): - """Easyblock constructor; initialize class variables.""" - super(EB_DL_underscore_POLY_underscore_Classic, self).__init__(*args, **kwargs) - - # check whether PLUMED is listed as a dependency - self.with_plumed = 'PLUMED' in [dep['name'] for dep in self.cfg['dependencies']] - - # create PLUMED patch in prepare_step rather than patch_step, - # so we can rely on being in the unpacked source directory - def prepare_step(self, *args, **kwargs): - """Generate PLUMED patch if PLUMED is listed as a dependency.""" - super(EB_DL_underscore_POLY_underscore_Classic, self).prepare_step(*args, **kwargs) - - if self.with_plumed: - # see https://groups.google.com/d/msg/plumed-users/cWaIDU5F6Bw/bZUW3J9cCAAJ - diff_pat = 'dlpoly-*.diff' - try: - diff_hits = glob.glob(os.path.join(self.builddir, diff_pat)) - except OSError as err: - raise EasyBuildError("Failed to find list of files/dirs that match '%s': %s", diff_pat, err) - - if len(diff_hits) == 1: - plumed_patch = diff_hits[0] - elif not self.dry_run: - raise EasyBuildError("Expected to find exactly one match for '%s' in %s, found: %s", - diff_pat, self.builddir, diff_hits) - - if not self.dry_run: - try: - os.rename('source', 'srcmod') - except OSError as err: - raise EasyBuildError("Failed to move 'source' directory to 'srcmod': %s", err) - - engine = os.path.splitext(os.path.basename(plumed_patch))[0] - cmd = "plumed-patch -p --runtime -e %s -d %s" % (engine, plumed_patch) - run_cmd(cmd, log_all=True, simple=True) - - def configure_step(self): - """Copy the makefile to the source directory and use MPIF90 to do a parrallel build""" - - self.cfg.update('buildopts', 'LD="$MPIF90 -o" FC="$MPIF90 -c" par') - - if self.with_plumed: - source_dir = 'srcmod' - self.cfg.update('buildopts', 'LDFLAGS="${LDFLAGS} -lplumed -ldl"') - else: - source_dir = 'source' - - copy_file(os.path.join('build', 'MakePAR'), os.path.join(source_dir, 'Makefile')) - try: - os.chdir(source_dir) - except OSError as err: - raise EasyBuildError("Failed to change to %s: %s", source_dir, err) - - def install_step(self): - """Copy the executables to the installation directory""" - self.log.debug("copying %s/execute to %s, (from %s)", self.cfg['start_dir'], self.installdir, os.getcwd()) - # create a 'bin' subdir, this way we also get $PATH to be set correctly automatically - install_path = os.path.join(self.cfg['start_dir'], 'execute') - bin_path = os.path.join(self.installdir, 'bin') - copy_dir(install_path, bin_path) - - def sanity_check_step(self): - """Custom sanity check step for DL_POLY Classic""" - custom_paths = { - 'files': ['bin/DLPOLY.X'], - 'dirs': [], - } - super(EB_DL_underscore_POLY_underscore_Classic, self).sanity_check_step(custom_paths=custom_paths) diff --git a/easybuild/easyblocks/d/dm_reverb.py b/easybuild/easyblocks/d/dm_reverb.py index 104d1c8841d..b94ba015f78 100644 --- a/easybuild/easyblocks/d/dm_reverb.py +++ b/easybuild/easyblocks/d/dm_reverb.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -33,7 +33,7 @@ from easybuild.tools.environment import setvar from easybuild.tools.filetools import mkdir, remove_dir from easybuild.tools.modules import get_software_root, get_software_version -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd class EB_dm_minus_reverb(PythonPackage): @@ -60,7 +60,8 @@ def configure_step(self, *args, **kwargs): # execute custom configuration script conf_cmd = "python configure.py" - return run_cmd(conf_cmd, log_all=True, simple=True, log_output=True) + res = run_shell_cmd(conf_cmd) + return res.exit_code def build_step(self, *args, **kwargs): """Build with Bazel""" @@ -90,14 +91,15 @@ def build_step(self, *args, **kwargs): # use JDK from EB bazel_build_opts += " --host_javabase=@local_jdk//:jdk" # explicitly set the number of processes - bazel_build_opts += " --jobs=%d" % self.cfg['parallel'] + bazel_build_opts += f" --jobs={self.cfg.parallel}" # print full compilation commands bazel_build_opts += " --subcommands" bazel_cmd = "%s bazel %s build %s %s" % (self.cfg['prebuildopts'], bazel_opts, bazel_build_opts, bazel_build_pkg) - return run_cmd(bazel_cmd, log_all=True, simple=True, log_output=True) + res = run_shell_cmd(bazel_cmd) + return res.exit_code def install_step(self, *args, **kwargs): """Package deepmind/reverb in a wheel and install it with pip""" @@ -111,15 +113,17 @@ def install_step(self, *args, **kwargs): whl_cmd = "./bazel-bin/reverb/pip_package/build_pip_package %s" % whl_build_opts - run_cmd(whl_cmd, log_all=True, simple=True, log_output=True) + run_shell_cmd(whl_cmd) # install wheel with pip pymajmin = ''.join(get_software_version('Python').split('.')[:2]) whl_file = '%s-%s-cp%s*.whl' % (self.name.replace('-', '_'), self.version, pymajmin) whl_path = os.path.join(self.builddir, whl_file) + installopts = ' '.join([self.cfg['installopts']] + self.py_installopts) + self.install_cmd = PIP_INSTALL_CMD % { - 'installopts': self.cfg['installopts'], + 'installopts': installopts, 'loc': whl_path, 'prefix': self.installdir, 'python': self.python_cmd, diff --git a/easybuild/easyblocks/d/dolfin.py b/easybuild/easyblocks/d/dolfin.py deleted file mode 100644 index 955fcb52ee6..00000000000 --- a/easybuild/easyblocks/d/dolfin.py +++ /dev/null @@ -1,359 +0,0 @@ -## -# Copyright 2009-2024 Ghent University -# -# This file is part of EasyBuild, -# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), -# with support of Ghent University (http://ugent.be/hpc), -# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), -# Flemish Research Foundation (FWO) (http://www.fwo.be/en) -# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). -# -# https://github.com/easybuilders/easybuild -# -# EasyBuild is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation v2. -# -# EasyBuild is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with EasyBuild. If not, see . -## -""" -EasyBuild support for DOLFIN, implemented as an easyblock - -@author: Kenneth Hoste (Ghent University) -@author: Jens Timmerman (Ghent University) -""" -import glob -import os -import re -import tempfile -from easybuild.tools import LooseVersion - -import easybuild.tools.environment as env -import easybuild.tools.toolchain as toolchain -from easybuild.easyblocks.generic.cmakepythonpackage import CMakePythonPackage -from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.filetools import change_dir, remove -from easybuild.tools.modules import get_software_root, get_software_version -from easybuild.tools.run import run_cmd -from easybuild.tools.systemtools import get_shared_lib_ext - - -class EB_DOLFIN(CMakePythonPackage): - """Support for building and installing DOLFIN.""" - - @staticmethod - def extra_options(): - extra_vars = CMakePythonPackage.extra_options() - extra_vars['separate_build_dir'][0] = True - return extra_vars - - def __init__(self, *args, **kwargs): - """Initialize class variables.""" - super(EB_DOLFIN, self).__init__(*args, **kwargs) - - self.boost_dir = None - self.saved_configopts = None - - def configure_step(self): - """Set DOLFIN-specific configure options and configure with CMake.""" - - shlib_ext = get_shared_lib_ext() - - # compiler flags - cflags = os.getenv('CFLAGS') - cxxflags = os.getenv('CXXFLAGS') - fflags = os.getenv('FFLAGS') - - # fix for "SEEK_SET is #defined but must not be for the C++ binding of MPI. Include mpi.h before stdio.h" - if self.toolchain.mpi_family() in [toolchain.INTELMPI, toolchain.MPICH, toolchain.MPICH2, toolchain.MVAPICH2]: - cflags += " -DMPICH_IGNORE_CXX_SEEK" - cxxflags += " -DMPICH_IGNORE_CXX_SEEK" - fflags += " -DMPICH_IGNORE_CXX_SEEK" - - self.cfg.update('configopts', '-DCMAKE_C_FLAGS="%s"' % cflags) - self.cfg.update('configopts', '-DCMAKE_CXX_FLAGS="%s"' % cxxflags) - self.cfg.update('configopts', '-DCMAKE_Fortran_FLAGS="%s"' % fflags) - - # set correct compilers to be used at runtime - self.cfg.update('configopts', '-DMPI_C_COMPILER="$MPICC"') - self.cfg.update('configopts', '-DMPI_CXX_COMPILER="$MPICXX"') - - # specify MPI library - self.cfg.update('configopts', '-DMPI_COMPILER="%s"' % os.getenv('MPICC')) - - if os.getenv('MPI_LIB_SHARED') and os.getenv('MPI_INC_DIR'): - self.cfg.update('configopts', '-DMPI_LIBRARY="%s"' % os.getenv('MPI_LIB_SHARED')) - self.cfg.update('configopts', '-DMPI_INCLUDE_PATH="%s"' % os.getenv('MPI_INC_DIR')) - else: - raise EasyBuildError("MPI_LIB_SHARED or MPI_INC_DIR not set, could not determine MPI-related paths.") - - # save config options to reuse them later (e.g. for sanity check commands) - self.saved_configopts = self.cfg['configopts'] - - # make sure that required dependencies are loaded - deps = ['Boost', 'CGAL', 'ParMETIS', 'PETSc', 'Python', - 'SCOTCH', 'SLEPc', 'SuiteSparse', 'Trilinos', 'zlib'] - # Armadillo was replaced by Eigen in v1.3 - if LooseVersion(self.version) < LooseVersion('1.3'): - deps.append('Armadillo') - else: - deps.append('Eigen') - - # UFC has been integrated into FFC in v1.4, cfr. https://bitbucket.org/fenics-project/ufc-deprecated - if LooseVersion(self.version) < LooseVersion('1.4'): - deps.append('UFC') - - # PLY, petsc4py, slepc4py are required since v1.5 - if LooseVersion(self.version) >= LooseVersion('1.5'): - deps.extend(['petsc4py', 'PLY', 'slepc4py']) - - # pybind11 is required to build Python bindings since v2018.1 - if LooseVersion(self.version) >= LooseVersion('2018.1'): - deps.append('pybind11') - - depsdict = {} - for dep in deps: - deproot = get_software_root(dep) - if not deproot: - raise EasyBuildError("Dependency %s not available.", dep) - else: - depsdict.update({dep: deproot}) - - # zlib - self.cfg.update('configopts', '-DZLIB_INCLUDE_DIR=%s' % os.path.join(depsdict['zlib'], "include")) - self.cfg.update('configopts', '-DZLIB_LIBRARY=%s' % os.path.join(depsdict['zlib'], "lib", "libz.a")) - - # set correct openmp options - openmp = self.toolchain.get_flag('openmp') - self.cfg.update('configopts', '-DOpenMP_CXX_FLAGS="%s"' % openmp) - self.cfg.update('configopts', '-DOpenMP_C_FLAGS="%s"' % openmp) - - # Boost config parameters - self.cfg.update('configopts', "-DBOOST_INCLUDEDIR=%s/include" % depsdict['Boost']) - self.cfg.update('configopts', "-DBoost_DEBUG=ON -DBOOST_ROOT=%s" % depsdict['Boost']) - self.boost_dir = depsdict['Boost'] - - # UFC and Armadillo config params - if 'UFC' in depsdict: - self.cfg.update('configopts', "-DUFC_DIR=%s" % depsdict['UFC']) - if 'Armadillo' in depsdict: - self.cfg.update('configopts', "-DARMADILLO_DIR:PATH=%s " % depsdict['Armadillo']) - - # Eigen config params - if 'Eigen' in depsdict: - self.cfg.update('configopts', "-DEIGEN3_INCLUDE_DIR=%s " % os.path.join(depsdict['Eigen'], 'include')) - - # specify Python paths - if LooseVersion(self.version) < LooseVersion('2018.1'): - python = depsdict['Python'] - pyver = '.'.join(get_software_version('Python').split('.')[:2]) - self.cfg.update('configopts', "-DPYTHON_INCLUDE_PATH=%s/include/python%s" % (python, pyver)) - self.cfg.update('configopts', "-DPYTHON_LIBRARY=%s/lib/libpython%s.%s" % (python, pyver, shlib_ext)) - - # SuiteSparse config params - suitesparse = depsdict['SuiteSparse'] - umfpack_params = [ - '-DUMFPACK_DIR="%(sp)s/UMFPACK"', - '-DUMFPACK_INCLUDE_DIRS="%(sp)s/UMFPACK/include;%(sp)s/UFconfig"', - '-DAMD_DIR="%(sp)s/UMFPACK"', - '-DCHOLMOD_DIR="%(sp)s/CHOLMOD"', - '-DCHOLMOD_INCLUDE_DIRS="%(sp)s/CHOLMOD/include;%(sp)s/UFconfig"', - '-DUFCONFIG_DIR="%(sp)s/UFconfig"', - '-DCAMD_LIBRARY:PATH="%(sp)s/CAMD/lib/libcamd.a"', - '-DCCOLAMD_LIBRARY:PATH="%(sp)s/CCOLAMD/lib/libccolamd.a"', - '-DCOLAMD_LIBRARY:PATH="%(sp)s/COLAMD/lib/libcolamd.a"' - ] - - self.cfg.update('configopts', ' '.join(umfpack_params) % {'sp': suitesparse}) - - # ParMETIS and SCOTCH - self.cfg.update('configopts', '-DPARMETIS_DIR="%s"' % depsdict['ParMETIS']) - self.cfg.update('configopts', '-DSCOTCH_DIR="%s" -DSCOTCH_DEBUG:BOOL=ON' % depsdict['SCOTCH']) - - # BLACS and LAPACK - self.cfg.update('configopts', '-DBLAS_LIBRARIES:PATH="%s"' % os.getenv('LIBBLAS')) - self.cfg.update('configopts', '-DLAPACK_LIBRARIES:PATH="%s"' % os.getenv('LIBLAPACK')) - - # CGAL - self.cfg.update('configopts', '-DCGAL_DIR:PATH="%s"' % depsdict['CGAL']) - - # PETSc - # need to specify PETSC_ARCH explicitely (env var alone is not sufficient) - for env_var in ["PETSC_DIR", "PETSC_ARCH"]: - val = os.getenv(env_var) - if val: - self.cfg.update('configopts', '-D%s=%s' % (env_var, val)) - - # MTL4 - if 'MTL4' in depsdict: - self.cfg.update('configopts', '-DMTL4_DIR:PATH="%s"' % depsdict['MTL4']) - - # SUNDIALS - if 'SUNDIALS' in depsdict: - self.cfg.update('configopts', '-DSUNDIALS_DIR:PATH="%s"' % depsdict['SUNDIALS']) - - # configure - out = super(EB_DOLFIN, self).configure_step() - - # make sure that all optional packages are found - not_found_re = re.compile("The following optional packages could not be found") - if not_found_re.search(out): - raise EasyBuildError("Optional packages could not be found, this should not happen...") - - # enable verbose build, so we have enough information if something goes wrong - self.cfg.update('buildopts', "VERBOSE=1") - - def test_step(self): - """Run DOLFIN demos by means of test.""" - - if self.cfg['runtest']: - - # set cache/error dirs for Instant - tmpdir = tempfile.mkdtemp() - instant_cache_dir = os.path.join(tmpdir, '.instant', 'cache') - instant_error_dir = os.path.join(tmpdir, '.instant', 'error') - try: - os.makedirs(instant_cache_dir) - os.makedirs(instant_error_dir) - except OSError as err: - raise EasyBuildError("Failed to create Instant cache/error dirs: %s", err) - - env_vars = [ - ('INSTANT_CACHE_DIR', instant_cache_dir), - ('INSTANT_ERROR_DIR', instant_error_dir), - ] - env_var_cmds = ' && '.join(['export %s="%s"' % (var, val) for (var, val) in env_vars]) - - cpp_cmds = [ - env_var_cmds, - "cd %(dir)s", - ] - if LooseVersion(self.version) < LooseVersion('1.1'): - cpp_cmds.append("cmake . %s" % self.saved_configopts) - - cpp_cmds.extend([ - "make VERBOSE=1", - "./demo_%(name)s", - "cd -", - ]) - cmd_template_cpp = " && ".join(cpp_cmds) - - # list based on demos available for DOLFIN v1.0.0 - pde_demos = ['biharmonic', 'cahn-hilliard', 'hyperelasticity', 'mixed-poisson', - 'navier-stokes', 'poisson', 'stokes-iterative'] - - if LooseVersion(self.version) < LooseVersion('1.1'): - demos = [os.path.join('demo', 'la', 'eigenvalue')] + [os.path.join('demo', 'pde', x) for x in pde_demos] - else: - # verified with v1.6.0 - demos = [os.path.join('demo', 'documented', x) for x in pde_demos] - - # construct commands - cmds = [tmpl % {'dir': os.path.join(d, subdir), 'name': os.path.basename(d)} - for d in demos for (tmpl, subdir) in [(cmd_template_cpp, 'cpp')]] - - # exclude Python tests for now, because they 'hang' sometimes (unclear why) - # they can be reinstated once run_cmd (or its equivalent) has support for timeouts - # see https://github.com/easybuilders/easybuild-framework/issues/581 - # test command templates - # cmd_template_python = " && ".join([ - # env_var_cmds, - # "cd %(dir)s", - # "python demo_%(name)s.py", - # "cd -", - # ]) - - # for (tmpl, subdir) in [(cmd_template_python, 'python'), (cmd_template_cpp, 'cpp')] - - # subdomains-poisson has no C++ get_version, only Python - # Python tests excluded, see above - # name = 'subdomains-poisson' - # path = os.path.join('demo', 'pde', name, 'python') - # cmds += [cmd_template_python % {'dir': path, 'name': name}] - - # supply empty argument to each command - for cmd in cmds: - run_cmd(cmd, log_all=True) - - # clean up temporary dir - remove(tmpdir) - - def install_step(self): - """Custom install procedure for DOLFIN: also install Python bindings.""" - super(EB_DOLFIN, self).install_step() - - # avoid that pip (ab)uses $HOME/.cache/pip - # cfr. https://pip.pypa.io/en/stable/reference/pip_install/#caching - env.setvar('XDG_CACHE_HOME', tempfile.gettempdir()) - self.log.info("Using %s as pip cache directory", os.environ['XDG_CACHE_HOME']) - - if LooseVersion(self.version) >= LooseVersion('2018.1'): - # see https://bitbucket.org/fenics-project/dolfin/issues/897/switch-from-swig-to-pybind11-for-python - # and https://github.com/FEniCS/dolfin/blob/master/python/README.rst - cwd = change_dir(os.path.join(self.start_dir, 'python')) - - env.setvar('CMAKE_PREFIX_PATH', self.installdir) - env.setvar('PYBIND11_DIR', get_software_root('pybind11')) - - run_cmd("pip install --prefix %s ." % self.installdir) - - change_dir(cwd) - - def post_install_step(self): - """Post install actions: extend RPATH paths in .so libraries part of the DOLFIN Python package.""" - if LooseVersion(self.version) >= LooseVersion('1.1'): - # cfr. https://github.com/hashdist/hashstack/blob/master/pkgs/dolfin/dolfin.yaml (look for patchelf) - - # determine location of libdolfin.so - dolfin_lib = 'libdolfin.so' - dolfin_libdir = None - for libdir in ['lib', 'lib64']: - if os.path.exists(os.path.join(self.installdir, libdir, dolfin_lib)): - dolfin_libdir = os.path.join(self.installdir, libdir) - break - if dolfin_libdir is None: - raise EasyBuildError("Failed to locate %s", dolfin_lib) - - for pylibdir in self.all_pylibdirs: - libs = glob.glob(os.path.join(self.installdir, pylibdir, 'dolfin', 'cpp', '_*.so')) - for lib in libs: - out, _ = run_cmd("patchelf --print-rpath %s" % lib, simple=False, log_all=True) - curr_rpath = out.strip() - cmd = "patchelf --set-rpath '%s:%s' %s" % (curr_rpath, dolfin_libdir, lib) - run_cmd(cmd, log_all=True) - - def make_module_extra(self): - """Set extra environment variables for DOLFIN.""" - - txt = super(EB_DOLFIN, self).make_module_extra() - - # Dolfin needs to find Boost - # check whether boost_dir is defined for compatibility with --module-only - if self.boost_dir: - txt += self.module_generator.set_environment('BOOST_DIR', self.boost_dir) - - envvars = ['I_MPI_CXX', 'I_MPI_CC'] - for envvar in envvars: - envar_val = os.getenv(envvar) - # if environment variable is set, also set it in module - if envar_val: - txt += self.module_generator.set_environment(envvar, envar_val) - - return txt - - def sanity_check_step(self): - """Custom sanity check for DOLFIN.""" - - # custom sanity check paths - custom_paths = { - 'files': ['bin/dolfin-%s' % x for x in ['version', 'convert', 'order', 'plot']] + ['include/dolfin.h'], - 'dirs': ['%s/dolfin' % self.pylibdir], - } - - super(EB_DOLFIN, self).sanity_check_step(custom_paths=custom_paths) diff --git a/easybuild/easyblocks/d/doris.py b/easybuild/easyblocks/d/doris.py deleted file mode 100644 index c0de79b461c..00000000000 --- a/easybuild/easyblocks/d/doris.py +++ /dev/null @@ -1,136 +0,0 @@ -## -# Copyright 2009-2024 Ghent University -# -# This file is part of EasyBuild, -# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), -# with support of Ghent University (http://ugent.be/hpc), -# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), -# Flemish Research Foundation (FWO) (http://www.fwo.be/en) -# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). -# -# https://github.com/easybuilders/easybuild -# -# EasyBuild is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation v2. -# -# EasyBuild is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with EasyBuild. If not, see . -## -""" -EasyBuild support for building and installing Doris, implemented as an easyblock - -author: Kenneth Hoste (HPC-UGent) -""" -import os - -from easybuild.easyblocks.generic.configuremake import ConfigureMake -from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.filetools import change_dir, mkdir -from easybuild.tools.modules import get_software_root -from easybuild.tools.run import run_cmd_qa - - -class EB_Doris(ConfigureMake): - """Support for building/installing Doris.""" - - def configure_step(self): - """Custom configuration procedure for Doris.""" - fftw = get_software_root('FFTW') - if fftw is None: - raise EasyBuildError("Required dependency FFTW is missing") - - # create installation directory (and /bin subdirectory) early, make sure it doesn't get removed later - self.make_installdir() - mkdir(os.path.join(self.installdir, 'bin')) - self.cfg['keeppreviousinstall'] = True - - # configure/build/install should be done from 'src' subdirectory - change_dir(os.path.join(self.cfg['start_dir'], 'src')) - - qa = { - "===> Press enter to continue.": '', - "===> Enter installation directory (use absolute path):": os.path.join(self.installdir, 'bin'), - "===> Press enter to continue (CTRL-C to exit).": '', - } - std_qa = { - r"===> Do you want to compile a more verbose DEBUG version \(y/n\)\? \[n\](.|\n)*expected results\)": 'n', - r"===> What is your C\+\+ compiler\? \[.*\]": os.getenv('CXX'), - r"===> Do you have the FFTW library \(y/n\)\? \[.*\]": 'y', - r"===> What is the path to the FFTW library \(libfftw3f.*\)\? \[.*\]": os.path.join(fftw, 'lib'), - r"===> What is the path to the FFTW include file \(fftw3.h\)\? \[.*\]": os.path.join(fftw, 'include'), - r"===> Do you have the VECLIB library \(y/n\)\? \[.*\]": 'n', - r"===> Do you have the LAPACK library \(y/n\)\? \[.*\]": 'y', - r"===> What is the path to the LAPACK library liblapack.a\? \[.*\]": os.getenv('LAPACK_LIB_DIR'), - r"===> Are you working on a Little Endian \(X86 PC, Intel\) machine \(y/n\)\? \[.*\]": 'y', - r"===> Installation of Doris in directory: /usr/local/bin \(y/n\)\? \[.*\]": 'n', - } - - run_cmd_qa('./configure', qa, std_qa=std_qa, log_all=True, simple=True) - - def build_step(self): - """Custom build procedure for Doris.""" - common_buildopts = self.cfg['buildopts'] - - # build Doris - change_dir(os.path.join(self.cfg['start_dir'], 'src')) - - # override some of the settings via options to 'make' - lflags = "-L%s -lfftw3f " % os.path.join(get_software_root('FFTW'), 'lib') - lflags += "-L%s %s" % (os.getenv('LAPACK_LIB_DIR'), os.getenv('LIBLAPACK_MT')) - self.cfg.update('buildopts', 'LFLAGS="%s"' % lflags) - self.cfg.update('buildopts', r'CFLAGSOPT="%s \$(DEFS)"' % os.getenv('CXXFLAGS')) - - super(EB_Doris, self).build_step() - - # build SARtools - change_dir(os.path.join(self.cfg['start_dir'], 'SARtools')) - - self.cfg['buildopts'] = common_buildopts - self.cfg.update('buildopts', 'CC="%s"' % os.getenv('CXX')) - cflags = os.getenv('CXXFLAGS') + " -D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE -D_LARGEFILE64_SOURCE" - self.cfg.update('buildopts', 'CFLAGS="%s"' % cflags) - - super(EB_Doris, self).build_step() - - # build ENVISAT_TOOLS - change_dir(os.path.join(self.cfg['start_dir'], 'ENVISAT_TOOLS')) - - self.cfg['buildopts'] = common_buildopts - self.cfg.update('buildopts', 'CC="%s"' % os.getenv('CC')) - self.cfg.update('buildopts', 'CFLAGS="%s"' % os.getenv('CFLAGS')) - - super(EB_Doris, self).build_step() - - def install_step(self): - """Custom build procedure for Doris.""" - # install Doris - change_dir(os.path.join(self.cfg['start_dir'], 'src')) - super(EB_Doris, self).install_step() - - # install SARtools - self.cfg.update('installopts', 'INSTALL_DIR=%s' % os.path.join(self.installdir, 'bin')) - change_dir(os.path.join(self.cfg['start_dir'], 'SARtools')) - super(EB_Doris, self).install_step() - - # install ENVISAT_TOOLS - change_dir(os.path.join(self.cfg['start_dir'], 'ENVISAT_TOOLS')) - self.cfg.update('installopts', 'CC="%s"' % os.getenv('CC')) - self.cfg.update('installopts', 'CFLAGS="%s"' % os.getenv('CFLAGS')) - super(EB_Doris, self).install_step() - - def sanity_check_step(self): - """Custom sanity check for Doris.""" - doris_bins = ['cpx2ps', 'doris', 'plotcpm', 'run'] - sartools_bins = ['bkconvert', 'cpxfiddle', 'flapjack', 'floatmult', 'wrap'] - envisat_tools_bins = ['envisat_dump_header', 'envisat_dump_data'] - custom_paths = { - 'files': [os.path.join('bin', x) for x in doris_bins + sartools_bins + envisat_tools_bins], - 'dirs': [], - } - super(EB_Doris, self).sanity_check_step(custom_paths=custom_paths) diff --git a/easybuild/easyblocks/d/doxygen.py b/easybuild/easyblocks/d/doxygen.py deleted file mode 100644 index 9fa4fdde321..00000000000 --- a/easybuild/easyblocks/d/doxygen.py +++ /dev/null @@ -1,65 +0,0 @@ -## -# Copyright 2009-2024 Ghent University -# -# This file is part of EasyBuild, -# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), -# with support of Ghent University (http://ugent.be/hpc), -# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), -# Flemish Research Foundation (FWO) (http://www.fwo.be/en) -# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). -# -# https://github.com/easybuilders/easybuild -# -# EasyBuild is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation v2. -# -# EasyBuild is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with EasyBuild. If not, see . -## -""" -EasyBuild support for building and installing Doxygen, implemented as an easyblock - -@author: Stijn De Weirdt (Ghent University) -@author: Dries Verdegem (Ghent University) -@author: Kenneth Hoste (Ghent University) -@author: Pieter De Baets (Ghent University) -@author: Jens Timmerman (Ghent University) -@author: Balazs Hajgato (Free University Brussels (VUB)) -""" - -from easybuild.tools import LooseVersion -from easybuild.tools.run import run_cmd -from easybuild.easyblocks.generic.cmakemake import CMakeMake - - -class EB_Doxygen(CMakeMake): - """Support for building/installing Doxygen""" - - def configure_step(self): - """Configure build using non-standard configure prefix option (without `=`) - for versions before 1.8.10. Newer versions use cmake instead of configure""" - if LooseVersion(self.version) < LooseVersion("1.8.10"): - - cmd = "%s ./configure --prefix %s %s" % (self.cfg['preconfigopts'], self.installdir, - self.cfg['configopts']) - run_cmd(cmd, log_all=True, simple=True) - else: - super(EB_Doxygen, self).configure_step() - - def sanity_check_step(self): - """Custom sanity check for Doxygen""" - - custom_paths = { - 'files': ["bin/doxygen"], - 'dirs': [] - } - - custom_commands = ["doxygen --help"] - - super(EB_Doxygen, self).sanity_check_step(custom_paths=custom_paths, custom_commands=custom_commands) diff --git a/easybuild/easyblocks/d/dualsphysics.py b/easybuild/easyblocks/d/dualsphysics.py index fd1b923c29a..0f50bece895 100644 --- a/easybuild/easyblocks/d/dualsphysics.py +++ b/easybuild/easyblocks/d/dualsphysics.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2024 Ghent University +# Copyright 2013-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -37,7 +37,7 @@ from easybuild.tools.config import build_option from easybuild.tools.filetools import adjust_permissions from easybuild.tools.modules import get_software_root -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd class EB_DualSPHysics(CMakeMakeCp): @@ -90,9 +90,9 @@ def install_step(self): ] super(EB_DualSPHysics, self).install_step() - def post_install_step(self): + def post_processing_step(self): """Custom post-installation step: ensure rpath is patched into binaries/libraries if configured.""" - super(EB_DualSPHysics, self).post_install_step() + super(EB_DualSPHysics, self).post_processing_step() if build_option('rpath'): # only the compiled binary (e.g. DualSPHysics5.0CPU_linux64) is rpath'd, the precompiled libraries @@ -102,8 +102,8 @@ def post_install_step(self): self.installdir, 'bin', 'DualSPHysics%s%s_linux64' % (self.shortver, self.dsph_target) ) - out, _ = run_cmd("patchelf --print-rpath %s" % rpathed_bin, simple=False, trace=False) - comp_rpath = out.strip() + res = run_shell_cmd("patchelf --print-rpath %s" % rpathed_bin, hidden=True) + comp_rpath = res.output.strip() files_to_patch = [] for x in [('bin', '*_linux64'), ('bin', '*.so'), ('lib', '*.so')]: @@ -111,14 +111,14 @@ def post_install_step(self): try: for x in files_to_patch: - out, _ = run_cmd("patchelf --print-rpath %s" % x, trace=False) - self.log.debug("Original RPATH for %s: %s" % (out, x)) + res = run_shell_cmd("patchelf --print-rpath %s" % x, hidden=True) + self.log.debug("Original RPATH for %s: %s" % (res.output, x)) - run_cmd("patchelf --set-rpath '%s' --force-rpath %s" % (comp_rpath, x), trace=False) - run_cmd("patchelf --shrink-rpath --force-rpath %s" % x, trace=False) + run_shell_cmd("patchelf --set-rpath '%s' --force-rpath %s" % (comp_rpath, x), hidden=True) + run_shell_cmd("patchelf --shrink-rpath --force-rpath %s" % x, hidden=True) - out, _ = run_cmd("patchelf --print-rpath %s" % x, trace=False) - self.log.debug("RPATH for %s (after patching and shrinking): %s" % (out, x)) + res = run_shell_cmd("patchelf --print-rpath %s" % x, hidden=True) + self.log.debug("RPATH for %s (after patching and shrinking): %s" % (res.output, x)) except OSError as err: raise EasyBuildError("Failed to patch RPATH section in binaries/libraries: %s", err) diff --git a/easybuild/easyblocks/e/easybuildmeta.py b/easybuild/easyblocks/e/easybuildmeta.py index ac05730a7f4..cba49132dfd 100644 --- a/easybuild/easyblocks/e/easybuildmeta.py +++ b/easybuild/easyblocks/e/easybuildmeta.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2024 Ghent University +# Copyright 2013-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -31,13 +31,13 @@ import os import re import sys -from easybuild.tools import LooseVersion +from collections import OrderedDict -from easybuild.easyblocks.generic.pythonpackage import PythonPackage, det_pip_version +from easybuild.easyblocks.generic.pythonpackage import PythonPackage +from easybuild.tools import LooseVersion from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import apply_regex_substitutions, change_dir, read_file from easybuild.tools.modules import get_software_root_env_var_name -from easybuild.tools.py2vs3 import OrderedDict from easybuild.tools.utilities import flatten @@ -66,27 +66,6 @@ def __init__(self, *args, **kwargs): # consider setuptools first, in case it is listed as a sources self.easybuild_pkgs.insert(0, 'setuptools') - # opt-in to using pip for recent version of EasyBuild, if: - # - EasyBuild is being installed for Python >= 3.6; - # - pip is available, and recent enough (>= 21.0); - # - use_pip is not specified; - pyver = sys.version.split(' ')[0] - self.log.info("Python version: %s", pyver) - if sys.version_info >= (3, 6) and self.cfg['use_pip'] is None: - # try to determine pip version, ignore any failures that occur while doing so; - # problems may occur due changes in environment ($PYTHONPATH, etc.) - pip_version = None - try: - pip_version = det_pip_version(python_cmd=sys.executable) - self.log.info("Found Python v%s + pip: %s", pyver, pip_version) - except Exception as err: - self.log.warning("Failed to determine pip version: %s", err) - - if pip_version and LooseVersion(pip_version) >= LooseVersion('21.0'): - self.log.info("Auto-enabling use of pip to install EasyBuild!") - self.cfg['use_pip'] = True - self.determine_install_command() - # Override this function since we want to respect the user choice for the python installation to use # (which can be influenced by EB_PYTHON and EB_INSTALLPYTHON) def prepare_python(self): @@ -157,10 +136,10 @@ def install_step(self): except OSError as err: raise EasyBuildError("Failed to install EasyBuild packages: %s", err) - def post_install_step(self): + def post_processing_step(self): """Remove setuptools.pth file that hard includes a system-wide (site-packages) path, if it is there.""" - super(EB_EasyBuildMeta, self).post_install_step() + super(EB_EasyBuildMeta, self).post_processing_step() setuptools_pth = os.path.join(self.installdir, self.pylibdir, 'setuptools.pth') if os.path.exists(setuptools_pth): diff --git a/easybuild/easyblocks/e/egglib.py b/easybuild/easyblocks/e/egglib.py deleted file mode 100644 index 17916ec4107..00000000000 --- a/easybuild/easyblocks/e/egglib.py +++ /dev/null @@ -1,88 +0,0 @@ -## -# Copyright 2009-2024 Ghent University -# -# This file is part of EasyBuild, -# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), -# with support of Ghent University (http://ugent.be/hpc), -# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), -# Flemish Research Foundation (FWO) (http://www.fwo.be/en) -# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). -# -# https://github.com/easybuilders/easybuild -# -# EasyBuild is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation v2. -# -# EasyBuild is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with EasyBuild. If not, see . -## -""" -EasyBuild support for building and installing EggLib, implemented as an easyblock - -@author: Kenneth Hoste (HPC-UGent) -""" -import os - -import easybuild.tools.environment as env -from easybuild.easyblocks.generic.configuremake import ConfigureMake -from easybuild.easyblocks.generic.pythonpackage import PythonPackage -from easybuild.tools.build_log import EasyBuildError - - -class EB_EggLib(PythonPackage, ConfigureMake): - """Support for building/installing EggLib.""" - - def configure_step(self): - """Configure EggLib build/install procedure.""" - # only need to configure Python library here, configuration of C++ library is done in install step - PythonPackage.configure_step(self) - - def build_step(self): - """No custom build procedure for EggLib; build/install is done in install_step.""" - pass - - def install_step(self): - """Custom install procedure for EggLib: first build/install C++ library, then build Python library.""" - - # build/install C++ library - cpp_subdir = os.path.join(self.builddir, 'egglib-cpp-%s' % self.version) - try: - os.chdir(cpp_subdir) - except OSError as err: - raise EasyBuildError("Failed to move to: %s", err) - - ConfigureMake.configure_step(self) - ConfigureMake.build_step(self) - ConfigureMake.install_step(self) - - # header files and libraries must be found when building Python library - for varname, subdir in [('CPATH', 'include'), ('LIBRARY_PATH', 'lib')]: - env.setvar(varname, '%s:%s' % (os.path.join(self.installdir, subdir), os.environ.get(varname, ''))) - - # build/install Python package - py_subdir = os.path.join(self.builddir, 'egglib-py-%s' % self.version) - try: - os.chdir(py_subdir) - except OSError as err: - raise EasyBuildError("Failed to move to: %s", err) - - PythonPackage.build_step(self) - - self.cfg.update('installopts', "--install-lib %s" % os.path.join(self.installdir, self.pylibdir)) - self.cfg.update('installopts', "--install-scripts %s" % os.path.join(self.installdir, 'bin')) - - PythonPackage.install_step(self) - - def sanity_check_step(self): - """Custom sanity check for EggLib.""" - custom_paths = { - 'files': ['bin/egglib', 'lib/libegglib-cpp.a'], - 'dirs': ['include/egglib-cpp', self.pylibdir], - } - super(EB_EggLib, self).sanity_check_step(custom_paths=custom_paths) diff --git a/easybuild/easyblocks/e/eigen.py b/easybuild/easyblocks/e/eigen.py index ed92ee2c144..d34b12f8113 100644 --- a/easybuild/easyblocks/e/eigen.py +++ b/easybuild/easyblocks/e/eigen.py @@ -1,7 +1,7 @@ ## # This file is an EasyBuild reciPY as per https://github.com/easybuilders/easybuild # -# Copyright:: Copyright 2012-2024 Uni.Lu/LCSB, NTUA +# Copyright:: Copyright 2012-2025 Uni.Lu/LCSB, NTUA # Authors:: Cedric Laczny , Fotis Georgatos , Kenneth Hoste # License:: MIT/GPL # $Id$ @@ -98,12 +98,3 @@ def sanity_check_step(self): % os.path.join(self.installdir, cmake_config_dir)) super(EB_Eigen, self).sanity_check_step(custom_paths=custom_paths, custom_commands=custom_commands) - - def make_module_req_guess(self): - """ - A dictionary of possible directories to look for. - Include CPLUS_INCLUDE_PATH as an addition to default ones - """ - guesses = super(EB_Eigen, self).make_module_req_guess() - guesses.update({'CPLUS_INCLUDE_PATH': ['include']}) - return guesses diff --git a/easybuild/easyblocks/e/elpa.py b/easybuild/easyblocks/e/elpa.py index f2fd2be7008..5acb8e6f740 100644 --- a/easybuild/easyblocks/e/elpa.py +++ b/easybuild/easyblocks/e/elpa.py @@ -1,6 +1,6 @@ ## -# Copyright 2009-2024 Ghent University -# Copyright 2019-2024 Micael Oliveira +# Copyright 2009-2025 Ghent University +# Copyright 2019-2025 Micael Oliveira # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/e/elsi.py b/easybuild/easyblocks/e/elsi.py index 053ec3698d8..220b2187cb9 100644 --- a/easybuild/easyblocks/e/elsi.py +++ b/easybuild/easyblocks/e/elsi.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/e/epd.py b/easybuild/easyblocks/e/epd.py deleted file mode 100644 index d3dbfc8e65b..00000000000 --- a/easybuild/easyblocks/e/epd.py +++ /dev/null @@ -1,45 +0,0 @@ -## -# Copyright 2009-2024 Ghent University -# -# This file is part of EasyBuild, -# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), -# with support of Ghent University (http://ugent.be/hpc), -# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), -# Flemish Research Foundation (FWO) (http://www.fwo.be/en) -# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). -# -# https://github.com/easybuilders/easybuild -# -# EasyBuild is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation v2. -# -# EasyBuild is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with EasyBuild. If not, see . -## -""" -General EasyBuild support for installing the Enthought Python Distribution - -@author: Jens Timmerman -""" -import os - -from easybuild.easyblocks.generic.binary import Binary - - -class EB_EPD(Binary): - """Easyblock implementing the build step for EPD, - this is just running the installer script, with an argument to the installdir - """ - - def install_step(self): - """Overwrite install_step from Binary""" - os.chdir(self.builddir) - if self.cfg['install_cmd'] is None: - self.cfg['install_cmd'] = "./epd_free-%s-x86_64.sh -b -p %s" % (self.version, self.installdir) - super(EB_EPD, self).install_step() diff --git a/easybuild/easyblocks/e/esmf.py b/easybuild/easyblocks/e/esmf.py index e6fd4d16620..1429ec88acc 100644 --- a/easybuild/easyblocks/e/esmf.py +++ b/easybuild/easyblocks/e/esmf.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2024 Ghent University +# Copyright 2013-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -37,7 +37,7 @@ from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.tools.build_log import EasyBuildError from easybuild.tools.modules import get_software_root, get_software_version -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd from easybuild.tools.systemtools import get_shared_lib_ext from easybuild.framework.easyconfig import CUSTOM @@ -127,8 +127,7 @@ def configure_step(self): env.setvar('ESMF_NETCDF_LIBS', ' '.join(netcdf_libs)) # 'make info' provides useful debug info - cmd = "make info" - run_cmd(cmd, log_all=True, simple=True, log_ok=True) + run_shell_cmd("make info") def install_step(self): # first, install the software @@ -146,10 +145,10 @@ def install_step(self): cmd = "python setup.py build --ESMFMKFILE=%s/lib/esmf.mk " % self.installdir cmd += " && python setup.py install --prefix=%s" % self.installdir - run_cmd(cmd, log_all=True, simple=True, log_ok=True) + run_shell_cmd(cmd) def make_module_extra(self): - """Add install path to PYTHONPATH or EBPYTHONPREFIXES""" + """Set $ESMFMKFILE environment variable""" txt = super(EB_ESMF, self).make_module_extra() # set environment variable ESMFMKFILE @@ -157,15 +156,6 @@ def make_module_extra(self): esmf_mkfile_path = os.path.join(self.installdir, "lib", "esmf.mk") txt += self.module_generator.set_environment('ESMFMKFILE', esmf_mkfile_path) - if self.cfg['multi_deps'] and 'Python' in self.cfg['multi_deps']: - txt += self.module_generator.prepend_paths('EBPYTHONPREFIXES', '') - else: - python = get_software_version('Python') - if python: - pyshortver = '.'.join(get_software_version('Python').split('.')[:2]) - pythonpath = os.path.join('lib', 'python%s' % pyshortver, 'site-packages') - txt += self.module_generator.prepend_paths('PYTHONPATH', [pythonpath]) - return txt def sanity_check_step(self): diff --git a/easybuild/easyblocks/e/espresso.py b/easybuild/easyblocks/e/espresso.py deleted file mode 100644 index 7665663a6cd..00000000000 --- a/easybuild/easyblocks/e/espresso.py +++ /dev/null @@ -1,78 +0,0 @@ -## -# This file is an EasyBuild reciPY as per https://github.com/easybuilders/easybuild -# -# Copyright:: Copyright 2012-2024 Uni.Lu/LCSB, NTUA -# Authors:: Josh Berryman , Fotis Georgatos , Kenneth Hoste -# License:: MIT/GPL -# $Id$ -# -# This work implements a part of the HPCBIOS project and is a component of the policy: -# http://hpcbios.readthedocs.org/en/latest/HPCBIOS_2012-80.html -## -""" -EasyBuild support for building and installing ESPResSo, implemented as an easyblock - -@author: Josh Berryman -@author: Fotis Georgatos (Uni.Lu) -@author: Kenneth Hoste (Ghent University) -""" -import os - -from easybuild.easyblocks.generic.configuremake import ConfigureMake -from easybuild.framework.easyconfig import CUSTOM -from easybuild.tools.run import run_cmd - - -class EB_ESPResSo(ConfigureMake): - """Support for building/installing ESPResSo, parallel version.""" - - def __init__(self, *args, **kwargs): - """Specify to build in install dir.""" - super(EB_ESPResSo, self).__init__(*args, **kwargs) - - self.build_in_installdir = True - self.install_subdir = '%s-%s' % (self.name.lower(), self.version) - - @staticmethod - def extra_options(): - extra_vars = { - 'runtest': [True, "Run ESPResSo tests.", CUSTOM], - } - return ConfigureMake.extra_options(extra_vars) - - def test_step(self): - """Custom built-in test procedure for ESPResSo, parallel version.""" - - if self.cfg['runtest']: - cmd = './runtest.sh -p 2 *.tcl' - (out, ec) = run_cmd(cmd, simple=False, log_all=False, log_ok=False, path="testsuite") - - if ec: - # ESPResSo fails many of its tests in version 3.1.1, and the test script itself is buggy - # so, just provide output in log file, but ignore things if it fails - self.log.warning("ESPResSo test failed (exit code: %s): %s" % (ec, out)) - else: - self.log.info("Successful ESPResSo test completed: %s" % out) - - def install_step(self): - """Build is done in install dir, so no separate install step.""" - pass - - def sanity_check_step(self): - """Custom sanity check for ESPResSo.""" - - custom_paths = { - 'files': [os.path.join(self.install_subdir, 'Espresso')], - 'dirs': [os.path.join(self.install_subdir, x) for x in ['samples', 'scripts', 'tools']], - } - - super(EB_ESPResSo, self).sanity_check_step(custom_paths=custom_paths) - - def make_module_req_guess(self): - """Customize PATH for ESPResSo.""" - - guesses = super(EB_ESPResSo, self).make_module_req_guess() - - guesses.update({'PATH': [self.install_subdir]}) - - return guesses diff --git a/easybuild/easyblocks/e/extrae.py b/easybuild/easyblocks/e/extrae.py index f6f47ffcea1..0ce705f8ab8 100644 --- a/easybuild/easyblocks/e/extrae.py +++ b/easybuild/easyblocks/e/extrae.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/f/faststructure.py b/easybuild/easyblocks/f/faststructure.py deleted file mode 100644 index 922d12e0df2..00000000000 --- a/easybuild/easyblocks/f/faststructure.py +++ /dev/null @@ -1,87 +0,0 @@ -# -*- coding: utf-8 -*- -## -# Copyright 2009-2024 Ghent University -# -# This file is part of EasyBuild, -# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), -# with support of Ghent University (http://ugent.be/hpc), -# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), -# Flemish Research Foundation (FWO) (http://www.fwo.be/en) -# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). -# -# https://github.com/easybuilders/easybuild -# -# EasyBuild is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation v2. -# -# EasyBuild is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with EasyBuild. If not, see . -## -""" -EasyBuild support for building and installing fastStructure, implemented as an easyblock - -@author: Bob Dröge (University of Groningen) -""" -import os -import stat - -from easybuild.easyblocks.generic.cmdcp import CmdCp -from easybuild.framework.easyconfig import CUSTOM -from easybuild.tools.filetools import adjust_permissions, change_dir, read_file, write_file -from easybuild.tools.run import run_cmd - - -class EB_fastStructure(CmdCp): - """Support for building and installing fastStructure.""" - - @staticmethod - def extra_options(extra_vars=None): - """Change default values of options""" - extra = CmdCp.extra_options() - # files_to_copy is not mandatory here - extra['files_to_copy'][2] = CUSTOM - return extra - - def __init__(self, *args, **kwargs): - """Initialisation of custom class variables for fastStructure.""" - super(EB_fastStructure, self).__init__(*args, **kwargs) - - self.cfg['files_to_copy'] = ['*'] - self.pyfiles = ['distruct.py', 'chooseK.py', 'structure.py'] - - def build_step(self): - """Build fastStructure using setup.py.""" - cwd = change_dir('vars') - run_cmd("python setup.py build_ext --inplace") - change_dir(cwd) - run_cmd("python setup.py build_ext --inplace") - - def post_install_step(self): - """Add a shebang to the .py files and make them executable.""" - for pyfile in self.pyfiles: - pf_path = os.path.join(self.installdir, pyfile) - pf_contents = read_file(pf_path) - write_file(pf_path, "#!/usr/bin/env python\n" + pf_contents) - adjust_permissions(pf_path, stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) - - super(EB_fastStructure, self).post_install_step() - - def sanity_check_step(self): - """Custom sanity check for fastStructure.""" - custom_paths = { - 'files': self.pyfiles, - 'dirs': ['vars'], - } - super(EB_fastStructure, self).sanity_check_step(custom_paths=custom_paths) - - def make_module_req_guess(self): - """Make sure PATH is set correctly.""" - guesses = super(EB_fastStructure, self).make_module_req_guess() - guesses['PATH'] = [''] - return guesses diff --git a/easybuild/easyblocks/f/fdtd_solutions.py b/easybuild/easyblocks/f/fdtd_solutions.py index bacaf32144a..17b6ee24130 100644 --- a/easybuild/easyblocks/f/fdtd_solutions.py +++ b/easybuild/easyblocks/f/fdtd_solutions.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2024 Ghent University +# Copyright 2013-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -32,7 +32,7 @@ from easybuild.easyblocks.generic.packedbinary import PackedBinary from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import copy_dir -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd class EB_FDTD_underscore_Solutions(PackedBinary): @@ -49,7 +49,7 @@ def extract_step(self): if len(rpms) != 1: raise EasyBuildError("Incorrect number of RPMs found, was expecting exactly one: %s", rpms) cmd = "rpm2cpio %s | cpio -idm " % rpms[0] - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) def make_installdir(self): """Override installdir creation""" diff --git a/easybuild/easyblocks/f/ferret.py b/easybuild/easyblocks/f/ferret.py index 9df9958e7f6..184445a2732 100644 --- a/easybuild/easyblocks/f/ferret.py +++ b/easybuild/easyblocks/f/ferret.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/f/fftw.py b/easybuild/easyblocks/f/fftw.py index baa287e2d50..0b745acf229 100644 --- a/easybuild/easyblocks/f/fftw.py +++ b/easybuild/easyblocks/f/fftw.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/f/fftwmpi.py b/easybuild/easyblocks/f/fftwmpi.py index cb285a576b3..5a083629f2d 100644 --- a/easybuild/easyblocks/f/fftwmpi.py +++ b/easybuild/easyblocks/f/fftwmpi.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -59,7 +59,7 @@ def prepare_step(self, *args, **kwargs): if not fftw_root: raise EasyBuildError("Required FFTW dependency is missing!") - def post_install_step(self): + def post_processing_step(self): """Custom post install step for FFTW.MPI""" # remove everything except include files that are already in non-MPI FFTW dependency. @@ -68,7 +68,7 @@ def post_install_step(self): glob.glob(os.path.join(self.installdir, 'lib*/pkgconfig')) + glob.glob(os.path.join(self.installdir, 'lib*/cmake')) + [os.path.join(self.installdir, p) for p in ['bin', 'share']]) - super(EB_FFTW_period_MPI, self).post_install_step() + super(EB_FFTW_period_MPI, self).post_processing_step() def sanity_check_step(self): """Custom sanity check for FFTW.MPI: check if all libraries/headers for MPI interfaces are there.""" diff --git a/easybuild/easyblocks/f/flex.py b/easybuild/easyblocks/f/flex.py index 0c9fb9b54cf..04c43a68954 100644 --- a/easybuild/easyblocks/f/flex.py +++ b/easybuild/easyblocks/f/flex.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/f/flexiblas.py b/easybuild/easyblocks/f/flexiblas.py index c1a7be76799..fa110dacc51 100644 --- a/easybuild/easyblocks/f/flexiblas.py +++ b/easybuild/easyblocks/f/flexiblas.py @@ -1,5 +1,5 @@ ## -# Copyright 2021-2024 Ghent University +# Copyright 2021-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -36,7 +36,8 @@ from easybuild.tools.config import build_option from easybuild.tools.environment import setvar from easybuild.tools.filetools import write_file -from easybuild.tools.run import run_cmd +from easybuild.tools.modules import MODULE_LOAD_ENV_HEADERS +from easybuild.tools.run import run_shell_cmd from easybuild.tools.systemtools import get_shared_lib_ext IMKL_CONF_TEMPLATE = """[IMKL] @@ -85,6 +86,9 @@ def __init__(self, *args, **kwargs): self.obj_builddir = os.path.join(self.builddir, 'easybuild_obj') + # custom CPATH to FlexiBLAS headers + self.module_load_environment.set_alias_vars(MODULE_LOAD_ENV_HEADERS, [os.path.join('include', 'flexiblas')]) + def configure_step(self): """Custom configuration for FlexiBLAS, based on which BLAS libraries are included as dependencies.""" @@ -94,7 +98,7 @@ def configure_step(self): 'FLEXIBLAS_DEFAULT': self.cfg['flexiblas_default'] or self.blas_libs[0], } - supported_blas_libs = ['BLIS', 'NETLIB', 'OpenBLAS', 'imkl'] + supported_blas_libs = ['AOCL-BLAS', 'BLIS', 'NETLIB', 'OpenBLAS', 'imkl'] # make sure that default backend is a supported library flexiblas_default = configopts['FLEXIBLAS_DEFAULT'] @@ -105,7 +109,11 @@ def configure_step(self): unsupported_libs = [x for x in self.blas_libs if x not in supported_blas_libs] if unsupported_libs: raise EasyBuildError("One or more unsupported libraries used: %s", ', '.join(unsupported_libs)) - + # need to use CMake naming convention + if 'AOCL-BLAS' in self.blas_libs: + self.blas_libs[self.blas_libs.index('AOCL-BLAS')] = 'AOCL_mt' + if configopts['FLEXIBLAS_DEFAULT'] == 'AOCL-BLAS': + configopts['FLEXIBLAS_DEFAULT'] = 'AOCL_mt' # list of BLAS libraries to use is specified via -DEXTRA=... configopts['EXTRA'] = ';'.join(self.blas_libs) @@ -131,6 +139,8 @@ def configure_step(self): configopts[key] = mkl_compiler_mapping[comp_family] except KeyError: raise EasyBuildError("Compiler family not supported yet: %s", comp_family) + elif blas_lib == 'AOCL_mt': + configopts[key] = 'blis-mt' else: configopts[key] = blas_lib.lower() @@ -182,7 +192,7 @@ def test_step(self): "make test", self.cfg['testopts'], ]) - run_cmd(test_cmd) + run_shell_cmd(test_cmd) def sanity_check_step(self): """Custom sanity check for FlexiBLAS.""" @@ -223,12 +233,3 @@ def sanity_check_step(self): custom_commands.append("flexiblas list | grep %s" % blas_lib.upper()) super(EB_FlexiBLAS, self).sanity_check_step(custom_paths=custom_paths, custom_commands=custom_commands) - - def make_module_req_guess(self): - """Customize CPATH for FlexiBLAS.""" - - guesses = super(EB_FlexiBLAS, self).make_module_req_guess() - - guesses.update({'CPATH': [os.path.join('include', 'flexiblas')]}) - - return guesses diff --git a/easybuild/easyblocks/f/flook.py b/easybuild/easyblocks/f/flook.py index 83d6173e3be..93c9b047f92 100644 --- a/easybuild/easyblocks/f/flook.py +++ b/easybuild/easyblocks/f/flook.py @@ -1,5 +1,5 @@ ## -# Copyright 2023-2024 Utrecht University +# Copyright 2023-2025 Utrecht University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -55,7 +55,7 @@ def __init__(self, *args, **kwargs): else: local_comp_flags = 'FFLAGS="$FFLAGS" CFLAGS="$CFLAGS"' self.cfg.update('buildopts', 'liball %s' % local_comp_flags) - self.cfg['parallel'] = 1 + self.cfg.parallel = 1 def configure_step(self): # flook has no configure step diff --git a/easybuild/easyblocks/f/fluent.py b/easybuild/easyblocks/f/fluent.py index f8e764ae803..c95ae9086ee 100644 --- a/easybuild/easyblocks/f/fluent.py +++ b/easybuild/easyblocks/f/fluent.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -34,7 +34,7 @@ from easybuild.easyblocks.generic.packedbinary import PackedBinary from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.filetools import adjust_permissions -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd class EB_FLUENT(PackedBinary): @@ -56,6 +56,12 @@ def __init__(self, *args, **kwargs): self.fluent_verdir = 'v%s' % subdir_version + self.module_load_environment.PATH = [ + os.path.join(self.fluent_verdir, 'fluent', 'bin'), + os.path.join(self.fluent_verdir, 'Framework', 'bin', 'Linux64'), + ] + self.module_load_environment.LD_LIBRARY_PATH = [os.path.join(self.fluent_verdir, 'fluent', 'lib')] + def install_step(self): """Custom install procedure for FLUENT.""" extra_args = '' @@ -64,7 +70,7 @@ def install_step(self): extra_args += '-noroot' cmd = "./INSTALL %s -debug -silent -install_dir %s %s" % (extra_args, self.installdir, self.cfg['installopts']) - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) adjust_permissions(self.installdir, stat.S_IWOTH, add=False) @@ -76,17 +82,3 @@ def sanity_check_step(self): 'dirs': [os.path.join(self.fluent_verdir, x) for x in ['aisol', 'CFD-Post']] } super(EB_FLUENT, self).sanity_check_step(custom_paths=custom_paths) - - def make_module_req_guess(self): - """Custom extra module file entries for FLUENT.""" - guesses = super(EB_FLUENT, self).make_module_req_guess() - - guesses.update({ - 'PATH': [ - os.path.join(self.fluent_verdir, 'fluent', 'bin'), - os.path.join(self.fluent_verdir, 'Framework', 'bin', 'Linux64'), - ], - 'LD_LIBRARY_PATH': [os.path.join(self.fluent_verdir, 'fluent', 'lib')], - }) - - return guesses diff --git a/easybuild/easyblocks/f/foldx.py b/easybuild/easyblocks/f/foldx.py deleted file mode 100644 index e07441d64f7..00000000000 --- a/easybuild/easyblocks/f/foldx.py +++ /dev/null @@ -1,71 +0,0 @@ -## -# Copyright 2013-2024 Ghent University -# -# This file is part of EasyBuild, -# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), -# with support of Ghent University (http://ugent.be/hpc), -# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), -# Flemish Research Foundation (FWO) (http://www.fwo.be/en) -# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). -# -# https://github.com/easybuilders/easybuild -# -# EasyBuild is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation v2. -# -# EasyBuild is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with EasyBuild. If not, see . -## -""" -EasyBuild support for installing FoldX, implemented as an easyblock - -@author: Kenneth Hoste (Ghent University) -""" -import os -import shutil -import stat - -from easybuild.easyblocks.generic.tarball import Tarball -from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.filetools import adjust_permissions - - -class EB_FoldX(Tarball): - """ - Support for installing FoldX - """ - - def install_step(self): - """Install by copying files to install dir and binary to 'bin' install subdir, and fixing permissions.""" - - bindir = os.path.join(self.installdir, 'bin') - binaries = [ - 'foldx_%s.linux' % self.version, # FoldX v2.x - 'FoldX.linux64', # FoldX 3.x - 'foldx64Linux', # FoldX 3.0-beta6 - 'foldx3b6', # FoldX 3.0 beta 6.1 >_< - ] - try: - os.makedirs(bindir) - for item in os.listdir(self.cfg['start_dir']): - if os.path.isfile(item): - if item in binaries: - shutil.copy2(os.path.join(self.cfg['start_dir'], item), bindir) - # make sure binary has executable permissions - perms = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH - adjust_permissions(os.path.join(bindir, item), perms, add=True) - self.log.debug("Copied %s to %s and fixed permissions" % (item, bindir)) - else: - # copy everything else straight into install directory - shutil.copy2(os.path.join(self.cfg['start_dir'], item), self.installdir) - self.log.debug("Copied %s to install dir %s" % (item, self.installdir)) - else: - self.log.warning("Skipping non-file %s in %s, not copying it." % (item, self.cfg['start_dir'])) - except OSError as err: - raise EasyBuildError("Copying binaries in %s to install dir 'bin' failed: %s", self.cfg['start_dir'], err) diff --git a/easybuild/easyblocks/f/freefem.py b/easybuild/easyblocks/f/freefem.py deleted file mode 100644 index 125df068c57..00000000000 --- a/easybuild/easyblocks/f/freefem.py +++ /dev/null @@ -1,169 +0,0 @@ -## -# Copyright 2009-2024 Ghent University -# -# This file is part of EasyBuild, -# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), -# with support of Ghent University (http://ugent.be/hpc), -# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), -# Flemish Research Foundation (FWO) (http://www.fwo.be/en) -# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). -# -# https://github.com/easybuilders/easybuild -# -# EasyBuild is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation v2. -# -# EasyBuild is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with EasyBuild. If not, see . -## -""" -EasyBuild support for FreeFem++, implemented as an easyblock - -@author: Balazs Hajgato (Free University Brussels - VUB) -@author: Kenneth Hoste (HPC-UGent) -""" -import os -import re - -import easybuild.tools.environment as env -import easybuild.tools.toolchain as toolchain -from easybuild.easyblocks.generic.configuremake import ConfigureMake -from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.config import ERROR -from easybuild.tools.modules import get_software_root -from easybuild.tools.filetools import change_dir -from easybuild.tools.run import check_log_for_errors, run_cmd - - -class EB_FreeFEM(ConfigureMake): - """Support for building and installing FreeFem++.""" - - def configure_step(self): - """ - FreeFem++ configure should run twice. - First to configure PETSc (and then build it), - then to configure FreeFem++ with the built PETSc. - """ - - # first Autoreconf has to be run - if not get_software_root('Autoconf'): - raise EasyBuildError("Autoconf is required to build FreeFem++. Please add it as build dependency") - - run_cmd("autoreconf -i", log_all=True, simple=False) - - configopts = [ - '--disable-optim', # disable custom optimizations (not needed, $CFLAGS set by EasyBuild is picked up) - '--enable-download', # enable downloading of dependencies - '--disable-openblas', # do not download OpenBLAS - ] - - blas_family = self.toolchain.blas_family() - if blas_family == toolchain.OPENBLAS: - openblas_root = get_software_root('OpenBLAS') - configopts.append('--with-blas=%s' % os.path.join(openblas_root, 'lib')) - elif blas_family == toolchain.INTELMKL: - mkl_root = os.getenv('MKLROOT') - configopts.append("--with-mkl=%s" % os.path.join(mkl_root, 'mkl', 'lib', 'intel64')) - - # specify which MPI to build with based on MPI component of toolchain - if self.toolchain.mpi_family() in toolchain.OPENMPI: - self.cfg.update('configopts', '--with-mpi=openmpi') - elif self.toolchain.mpi_family() in toolchain.INTELMPI: - self.cfg.update('configopts', '--with-mpi=mpic++') - - # if usempi is enabled, configure with --with-mpi=yes - elif self.toolchain.options.get('usempi', None): - self.cfg.update('configopts', '--with-mpi=yes') - else: - self.cfg.update('configopts', '--with-mpi=no') - - # check dependencies and add the corresponding configure options for FreeFEM - hdf5_root = get_software_root('HDF5') - if hdf5_root: - configopts.append('--with-hdf5=%s ' % os.path.join(hdf5_root, 'bin', 'h5pcc')) - - gsl_root = get_software_root('GSL') - if gsl_root: - configopts.append('--with-gsl-prefix=%s' % gsl_root) - - petsc_root = get_software_root('PETSc') - if petsc_root: - configopts.append('--with-petsc=%s' % os.path.join(petsc_root, 'lib', 'petsc', 'conf', 'petscvariables')) - - bemtool_root = get_software_root('BemTool') - if bemtool_root: - configopts.append('--with-bem-include=%s' % bemtool_root) - - for configopt in configopts: - self.cfg.update('configopts', configopt) - - # initial configuration - out = super(EB_FreeFEM, self).configure_step() - - regex = re.compile("WARNING: unrecognized options: (.*)", re.M) - res = regex.search(out) - if res: - raise EasyBuildError("One or more configure options not recognized: %s" % res.group(1)) - - if not petsc_root: - # re-create installation dir (deletes old installation), - # then set keeppreviousinstall to True (to avoid deleting PETSc installation) - self.make_installdir() - self.cfg['keeppreviousinstall'] = True - - # configure and make petsc-slepc - # download & build PETSc as recommended by FreeFEM if no PETSc dependency was provided - cwd = change_dir(os.path.join('3rdparty', 'ff-petsc')) - - cmd = ['make'] - if self.cfg['parallel']: - cmd.append('-j %s' % self.cfg['parallel']) - cmd.append('petsc-slepc') - - run_cmd(' '.join(cmd), log_all=True, simple=False) - - change_dir(cwd) - - # reconfigure for FreeFEM build - super(EB_FreeFEM, self).configure_step() - - def test_step(self): - """Run tests.""" - - # run tests unless they're disabled explicitly - if self.cfg['runtest'] is None or self.cfg['runtest']: - - # avoid oversubscribing, by using both OpenMP threads and having OpenBLAS use threads as well - env.setvar('OMP_NUM_THREADS', '1') - - (out, _) = run_cmd("make check", log_all=True, simple=False, regexp=False) - - # Raise an error if any test failed - check_log_for_errors(out, [('# (ERROR|FAIL): +[1-9]', ERROR)]) - - def sanity_check_step(self): - """Custom sanity check step for FreeFem++.""" - - binaries = [os.path.join('bin', x) for x in ['bamg', 'cvmsh2', 'ffglut', 'ffmaster', 'ffmedit']] - binaries.extend(os.path.join('bin', 'ff-%s' % x) for x in ['c++', 'get-dep', 'mpirun', 'pkg-download']) - binaries.extend(os.path.join('bin', 'FreeFem++%s' % x) for x in ['', '-mpi', '-nw']) - - custom_paths = { - 'files': binaries, - 'dirs': [os.path.join('share', 'FreeFEM', self.version)] + - [os.path.join('lib', 'ff++', self.version, x) for x in ['bin', 'etc', 'idp', 'include', 'lib']], - } - super(EB_FreeFEM, self).sanity_check_step(custom_paths=custom_paths) - - def make_module_extra(self): - txt = super(EB_FreeFEM, self).make_module_extra() - if self.toolchain.blas_family() == toolchain.OPENBLAS: - # avoid oversubscribing the available cores, by only running one OpenMP thread - txt += self.module_generator.set_environment('OMP_NUM_THREADS', "1") - return txt diff --git a/easybuild/easyblocks/f/freesurfer.py b/easybuild/easyblocks/f/freesurfer.py index 1c63b53e01f..74885b0cc0c 100644 --- a/easybuild/easyblocks/f/freesurfer.py +++ b/easybuild/easyblocks/f/freesurfer.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2024 Ghent University +# Copyright 2013-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -52,13 +52,14 @@ def install_step(self): super(EB_FreeSurfer, self).install_step() write_file(os.path.join(self.installdir, '.license'), self.cfg['license_text']) - def make_module_req_guess(self): - """Include correct subdirectories to $PATH for FreeSurfer.""" - guesses = super(EB_FreeSurfer, self).make_module_req_guess() + def __init__(self, *args, **kwargs): + """Custom constructor for FLUENT easyblock, initialize/define class parameters.""" + super(EB_FreeSurfer, self).__init__(*args, **kwargs) - guesses['PATH'].extend([os.path.join('fsfast', 'bin'), os.path.join('mni', 'bin'), 'tktools']) - - return guesses + self.module_load_environment.PATH.extend([ + os.path.join('fsfast', 'bin'), + os.path.join('mni', 'bin'), 'tktools', + ]) def make_module_extra(self): """Define FreeSurfer-specific environment variable in generated module file.""" diff --git a/easybuild/easyblocks/f/freetype.py b/easybuild/easyblocks/f/freetype.py index 2621a1fd6fe..0a47c52b96c 100644 --- a/easybuild/easyblocks/f/freetype.py +++ b/easybuild/easyblocks/f/freetype.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -29,6 +29,7 @@ """ from easybuild.easyblocks.generic.configuremake import ConfigureMake +from easybuild.tools.modules import MODULE_LOAD_ENV_HEADERS from easybuild.tools.systemtools import get_shared_lib_ext @@ -41,6 +42,8 @@ def __init__(self, *args, **kwargs): self.maj_ver = self.version.split('.')[0] + self.module_load_environment.set_alias_vars(MODULE_LOAD_ENV_HEADERS, [f'include/freetype{self.maj_ver}']) + def sanity_check_step(self): """Custom sanity check for freetype.""" custom_paths = { @@ -49,14 +52,3 @@ def sanity_check_step(self): 'dirs': ['include/freetype%s' % self.maj_ver], } super(EB_freetype, self).sanity_check_step(custom_paths=custom_paths) - - def make_module_req_guess(self): - """Custom guess for CPATH for freetype.""" - - guesses = super(EB_freetype, self).make_module_req_guess() - - guesses.update({ - 'CPATH': ['include/freetype%s' % self.maj_ver], - }) - - return guesses diff --git a/easybuild/easyblocks/f/fsl.py b/easybuild/easyblocks/f/fsl.py index 5064fb30dda..435e3faa958 100644 --- a/easybuild/easyblocks/f/fsl.py +++ b/easybuild/easyblocks/f/fsl.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -36,7 +36,7 @@ import easybuild.tools.environment as env from easybuild.framework.easyblock import EasyBlock from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd from easybuild.tools.filetools import read_file, copy_dir from easybuild.tools.utilities import nub @@ -53,6 +53,9 @@ def __init__(self, *args, **kwargs): self.fsldir = None + self.module_load_environment.PATH = [os.path.join('fsl', 'bin')] + self.module_load_environment.LD_LIBRARY_PATH = [os.path.join('fsl', 'lib')] + def configure_step(self): """Configure FSL build: set FSLDIR env var.""" @@ -61,8 +64,8 @@ def configure_step(self): # determine FSL machine type cmd = ". %s/etc/fslconf/fslmachtype.sh" % self.fsldir - (out, _) = run_cmd(cmd, log_all=True, simple=False) - fslmachtype = out.strip() + res = run_shell_cmd(cmd) + fslmachtype = res.output.strip() self.log.debug("FSL machine type: %s" % fslmachtype) best_cfg = None @@ -117,7 +120,7 @@ def build_step(self): """Build FSL using supplied script.""" cmd = ". %s/etc/fslconf/fsl.sh && ./build" % self.fsldir - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) # check build.log file for success buildlog = os.path.join(self.installdir, "fsl", "build.log") @@ -133,18 +136,6 @@ def install_step(self): """Building was performed in install dir, no explicit install step required.""" pass - def make_module_req_guess(self): - """Set correct PATH and LD_LIBRARY_PATH variables.""" - - guesses = super(EB_FSL, self).make_module_req_guess() - - guesses.update({ - 'PATH': ["fsl/bin"], - 'LD_LIBRARY_PATH': ["fsl/lib"], - }) - - return guesses - def make_module_extra(self): """Add setting of FSLDIR in module.""" diff --git a/easybuild/easyblocks/g/g2clib.py b/easybuild/easyblocks/g/g2clib.py index e15d945e5e1..0c4e5a7088c 100644 --- a/easybuild/easyblocks/g/g2clib.py +++ b/easybuild/easyblocks/g/g2clib.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/g/g2lib.py b/easybuild/easyblocks/g/g2lib.py index 4c316c3b1dd..b107c284254 100644 --- a/easybuild/easyblocks/g/g2lib.py +++ b/easybuild/easyblocks/g/g2lib.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/g/gamess_us.py b/easybuild/easyblocks/g/gamess_us.py index ca8985bfcc5..a1717d448de 100644 --- a/easybuild/easyblocks/g/gamess_us.py +++ b/easybuild/easyblocks/g/gamess_us.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -48,7 +48,7 @@ from easybuild.tools.config import build_option from easybuild.tools.filetools import change_dir, copy_file, mkdir, read_file, write_file, remove_dir from easybuild.tools.modules import get_software_root, get_software_version -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd from easybuild.tools.systemtools import POWER, X86_64 from easybuild.tools.systemtools import get_cpu_architecture from easybuild.tools import LooseVersion, toolchain @@ -142,10 +142,10 @@ def configure_step(self): fortran_comp, fortran_version = None, None if comp_fam == toolchain.INTELCOMP: fortran_comp = 'ifort' - (out, _) = run_cmd("ifort -v", simple=False) - res = re.search(r"^ifort version ([0-9]+)\.[0-9.]+$", out) + res = run_shell_cmd("ifort -v") + regex = re.compile(r"^ifort version ([0-9]+)\.[0-9.]+$") try: - version_num = res.group(1) + version_num = re.search(regex, res.output).group(1) except (AttributeError, IndexError): raise EasyBuildError("Failed to determine ifort major version number") fortran_version = {"GMS_IFORT_VERNO": version_num} @@ -203,7 +203,7 @@ def configure_step(self): installinfo_opts["GMS_MATHLIB"] = mathlib installinfo_opts["GMS_MATHLIB_PATH"] = mathlib_path - installinfo_opts["GMS_LAPACK_LINK_LINE"] = '"%s"' % mathlib_flags + installinfo_opts["GMS_LAPACK_LINK_LINE"] = f'"{mathlib_flags}"' # verify selected DDI communication layer known_ddi_comms = ['mpi', 'mixed', 'shmem', 'sockets'] @@ -286,7 +286,7 @@ def configure_step(self): try: lked = os.path.join(self.builddir, 'lked') for line in fileinput.input(lked, inplace=1, backup='.orig'): - line = re.sub(r"^(\s*set\sLIBXC_FLAGS)=.*GMS_PATH.*", r'\1="%s"' % libxc_linker_flags, line) + line = re.sub(r"^(\s*set\sLIBXC_FLAGS)=.*GMS_PATH.*", rf'\1="{libxc_linker_flags}"', line) sys.stdout.write(line) except IOError as err: raise EasyBuildError("Failed to patch %s: %s", lked, err) @@ -314,7 +314,7 @@ def configure_step(self): installinfo_opts['XMVB'] = False # add include paths from dependencies - installinfo_opts["GMS_FPE_FLAGS"] = '"%s"' % os.environ['CPPFLAGS'] + installinfo_opts["GMS_FPE_FLAGS"] = f"\"{os.environ['CPPFLAGS']}\"" # might be useful for debugging # installinfo_opts["GMS_FPE_FLAGS"] = '"%s"' % os.environ['CPPFLAGS'] + "-ffpe-trap=invalid,zero,overflow" @@ -324,27 +324,27 @@ def configure_step(self): boolean_opts = {opt: str(val).lower() for opt, val in installinfo_opts.items() if val in [True, False]} installinfo_opts.update(boolean_opts) # format: setenv KEY VALUE - installinfo_txt = '\n'.join(["setenv %s %s" % (k, installinfo_opts[k]) for k in installinfo_opts]) + installinfo_txt = '\n'.join([f"setenv {k} {installinfo_opts[k]}" for k in installinfo_opts]) write_file(installinfo_file, installinfo_txt) - self.log.debug("Contents of %s:\n%s" % (installinfo_file, read_file(installinfo_file))) + self.log.debug(f"Contents of {installinfo_file}:\n{read_file(installinfo_file)}") # patch hardcoded settings in rungms to use values specified in easyconfig file rungms = os.path.join(self.builddir, 'rungms') extra_gmspath_lines = "set ERICFMT=$GMSPATH/auxdata/ericfmt.dat\nset MCPPATH=$GMSPATH/auxdata/MCP\n" try: for line in fileinput.input(rungms, inplace=1, backup='.orig'): - line = re.sub(r"^(\s*set\s*TARGET)=.*", r"\1=%s" % self.cfg['ddi_comm'], line) - line = re.sub(r"^(\s*set\s*GMSPATH)=.*", r"\1=%s\n%s" % (self.installdir, extra_gmspath_lines), line) - line = re.sub(r"(null\) set VERNO)=.*", r"\1=%s" % self.version, line) - line = re.sub(r"^(\s*set DDI_MPI_CHOICE)=.*", r"\1=%s" % mpilib, line) - line = re.sub(r"^(\s*set DDI_MPI_ROOT)=.*%s.*" % mpilib.lower(), r"\1=%s" % mpilib_path, line) - line = re.sub(r"^(\s*set GA_MPI_ROOT)=.*%s.*" % mpilib.lower(), r"\1=%s" % mpilib_path, line) + line = re.sub(r"^(\s*set\s*TARGET)=.*", rf"\1={self.cfg['ddi_comm']}", line) + line = re.sub(r"^(\s*set\s*GMSPATH)=.*", rf"\1={self.installdir}\n{extra_gmspath_lines}", line) + line = re.sub(r"(null\) set VERNO)=.*", rf"\1={self.version}", line) + line = re.sub(r"^(\s*set DDI_MPI_CHOICE)=.*", rf"\1={mpilib}", line) + line = re.sub(rf"^(\s*set DDI_MPI_ROOT)=.*{mpilib.lower()}.*", rf"\1={mpilib_path}", line) + line = re.sub(rf"^(\s*set GA_MPI_ROOT)=.*{mpilib.lower()}.*", rf"\1={mpilib_path}", line) # comment out all adjustments to $LD_LIBRARY_PATH that involves hardcoded paths line = re.sub(r"^(\s*)(setenv\s*LD_LIBRARY_PATH\s*/.*)", r"\1#\2", line) # scratch directory paths - line = re.sub(r"^(\s*set\s*SCR)=.*", r"if ( ! $?SCR ) \1=%s" % self.cfg['scratch_dir'], line) + line = re.sub(r"^(\s*set\s*SCR)=.*", rf"if ( ! $?SCR ) \1={self.cfg['scratch_dir']}", line) line = re.sub( - r"^(\s*set\s*USERSCR)=.*", r"if ( ! $?USERSCR ) \1=%s" % self.cfg['user_scratch_dir'], line + r"^(\s*set\s*USERSCR)=.*", rf"if ( ! $?USERSCR ) \1={self.cfg['user_scratch_dir']}", line ) line = re.sub(r"^(df -k \$SCR)$", r"mkdir -p $SCR && mkdir -p $USERSCR && \1", line) if self.cfg['hyperthreading'] is False: @@ -359,8 +359,8 @@ def configure_step(self): compddi = os.path.join(self.builddir, 'ddi/compddi') try: for line in fileinput.input(compddi, inplace=1, backup='.orig'): - line = re.sub(r"^(\s*set MAXCPUS)=.*", r"\1=%s" % self.cfg['maxcpus'], line, 1) - line = re.sub(r"^(\s*set MAXNODES)=.*", r"\1=%s" % self.cfg['maxnodes'], line, 1) + line = re.sub(r"^(\s*set MAXCPUS)=.*", rf"\1={self.cfg['maxcpus']}", line, 1) + line = re.sub(r"^(\s*set MAXNODES)=.*", rf"\1={self.cfg['maxnodes']}", line, 1) sys.stdout.write(line) except IOError as err: raise EasyBuildError("Failed to patch compddi", compddi, err) @@ -375,39 +375,37 @@ def configure_step(self): except IOError as err: raise EasyBuildError("Failed to patch actvte.code", actvte, err) # compiling - run_cmd("mv %s/tools/actvte.code" % self.builddir + " %s/tools/actvte.f" % self.builddir) - run_cmd( - "%s -o " % fortran_comp + " %s/tools/actvte.x" % self.builddir + " %s/tools/actvte.f" % self.builddir - ) + run_shell_cmd(f"mv {self.builddir}/tools/actvte.code {self.builddir}/tools/actvte.f") + run_shell_cmd(f"{fortran_comp} -o {self.builddir}/tools/actvte.x {self.builddir}/tools/actvte.f") def build_step(self): """Custom build procedure for GAMESS-US: using compddi, compall and lked scripts.""" compddi = os.path.join(self.cfg['start_dir'], 'ddi', 'compddi') - run_cmd(compddi, log_all=True, simple=True) + run_shell_cmd(compddi) # make sure the libddi.a library is present libddi = os.path.join(self.cfg['start_dir'], 'ddi', 'libddi.a') if not os.path.isfile(libddi): raise EasyBuildError("The libddi.a library (%s) was never built", libddi) else: - self.log.info("The libddi.a library (%s) was successfully built." % libddi) + self.log.info("The libddi.a library (%s) was successfully built.", libddi) ddikick = os.path.join(self.cfg['start_dir'], 'ddi', 'ddikick.x') if os.path.isfile(ddikick): - self.log.info("The ddikick.x executable (%s) was successfully built." % ddikick) + self.log.info("The ddikick.x executable (%s) was successfully built.", ddikick) if self.cfg['ddi_comm'] == 'sockets': src = ddikick dst = os.path.join(self.cfg['start_dir'], 'ddikick.x') - self.log.info("Moving ddikick.x executable from %s to %s." % (src, dst)) + self.log.info("Moving ddikick.x executable from %s to %s.", src, dst) os.rename(src, dst) compall_cmd = os.path.join(self.cfg['start_dir'], 'compall') - compall = "%s %s %s" % (self.cfg['prebuildopts'], compall_cmd, self.cfg['buildopts']) - run_cmd(compall, log_all=True, simple=True) + compall = " ".join([self.cfg['prebuildopts'], compall_cmd, self.cfg['buildopts']]) + run_shell_cmd(compall) - cmd = "%s gamess %s" % (os.path.join(self.cfg['start_dir'], 'lked'), self.version) - run_cmd(cmd, log_all=True, simple=True) + gamess_cmd = f"{os.path.join(self.cfg['start_dir'], 'lked')} gamess {self.version}" + run_shell_cmd(gamess_cmd) def test_step(self): """Run GAMESS-US tests (if 'runtest' easyconfig parameter is set to True).""" @@ -429,11 +427,11 @@ def test_step(self): return # MPI builds can only run tests that support parallel execution - if int(self.cfg['parallel']) < 2: + if self.cfg.parallel < 2: self.log.info("Skipping testing of GAMESS-US as MPI tests need at least 2 CPU cores to run") return - test_procs = str(self.cfg['parallel']) + test_procs = str(self.cfg.parallel) target_tests = [exam for exam in target_tests if exam[0] not in GAMESS_SERIAL_TESTS] if self.toolchain.mpi_family() == toolchain.INTELMPI: @@ -457,27 +455,27 @@ def test_step(self): except OSError as err: raise EasyBuildError("Failed to copy test '%s' to %s: %s", exam, self.testdir, err) - test_env_vars.append('SCR=%s' % self.testdir) + test_env_vars.append(f"SCR={self.testdir}") # run target exam tests, dump output to exam.log rungms = os.path.join(self.installdir, 'rungms') for exam, exam_file in target_tests: rungms_prefix = ' && '.join(test_env_vars) test_cmd = [rungms_prefix, rungms, exam_file, self.version, test_procs, test_procs] - (out, _) = run_cmd(' '.join(test_cmd), log_all=True, simple=False) - write_file('%s.log' % exam, out) + res = run_shell_cmd(' '.join(test_cmd)) + write_file(f"{exam}.log", res.output) check_cmd = os.path.join(self.installdir, 'tests', 'standard', 'checktst') - (out, _) = run_cmd(check_cmd, log_all=True, simple=False) + res = run_shell_cmd(check_cmd) # verify output of tests failed_regex = re.compile(r"^.*!!FAILED\.$", re.M) - failed_tests = set([exam[0:6] for exam in failed_regex.findall(out)]) + failed_tests = set([exam[0:6] for exam in failed_regex.findall(res.output)]) done_tests = set([exam[0] for exam in target_tests]) if done_tests - failed_tests == done_tests: info_msg = "All target tests ran successfully!" if self.cfg['ddi_comm'] == 'mpi': - info_msg += " (serial tests ignored: %s)" % ", ".join(GAMESS_SERIAL_TESTS) + info_msg += f" (serial tests ignored: {', '.join(GAMESS_SERIAL_TESTS)})" self.log.info(info_msg) else: raise EasyBuildError("ERROR: Not all target tests ran successfully") @@ -496,7 +494,7 @@ def install_step(self): def sanity_check_step(self): """Custom sanity check for GAMESS-US.""" custom_paths = { - 'files': ['gamess.%s.x' % self.version, 'rungms'], + 'files': [f'gamess.{self.version}.x', 'rungms'], 'dirs': [], } super(EB_GAMESS_minus_US, self).sanity_check_step(custom_paths=custom_paths) diff --git a/easybuild/easyblocks/g/gate.py b/easybuild/easyblocks/g/gate.py index 6347694df73..e0258a9bd27 100644 --- a/easybuild/easyblocks/g/gate.py +++ b/easybuild/easyblocks/g/gate.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -40,7 +40,7 @@ from easybuild.easyblocks.generic.cmakemake import CMakeMake from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd from easybuild.tools.systemtools import get_shared_lib_ext @@ -144,8 +144,8 @@ def install_step(self): raise EasyBuildError("Failed to copy %s to %s: %s", self.cfg['start_dir'], self.installdir, err) cmd = "source %s &> /dev/null && echo $G4SYSTEM" % os.path.join(self.cfg['start_dir'], 'env_gate.sh') - out, _ = run_cmd(cmd, simple=False) - self.g4system = out.strip() + res = run_shell_cmd(cmd) + self.g4system = res.output.strip() if self.g4system: self.log.debug("Value obtained for $G4SYSTEM: %s" % self.g4system) else: diff --git a/easybuild/easyblocks/g/gcc.py b/easybuild/easyblocks/g/gcc.py index 76562c40013..4e120fbaae1 100644 --- a/easybuild/easyblocks/g/gcc.py +++ b/easybuild/easyblocks/g/gcc.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -50,8 +50,8 @@ from easybuild.tools.config import build_option from easybuild.tools.filetools import apply_regex_substitutions, adjust_permissions, change_dir, copy_file from easybuild.tools.filetools import mkdir, move_file, read_file, symlink, which, write_file -from easybuild.tools.modules import get_software_root -from easybuild.tools.run import run_cmd +from easybuild.tools.modules import MODULE_LOAD_ENV_HEADERS, get_software_root +from easybuild.tools.run import run_shell_cmd from easybuild.tools.systemtools import RISCV, check_os_dependency, get_cpu_architecture, get_cpu_family from easybuild.tools.systemtools import get_gcc_version, get_shared_lib_ext, get_os_name, get_os_type from easybuild.tools.toolchain.compiler import OPTARCH_GENERIC @@ -203,6 +203,14 @@ def __init__(self, *args, **kwargs): self.log.warning('Setting withnvptx to False, since we are building on a RISC-V system') self.cfg['withnvptx'] = False + # GCC can find its own headers and libraries in most cases, but we had + # cases where paths top libraries needed to be set explicitly + # see: https://github.com/easybuilders/easybuild-easyblocks/pull/3256 + # Therefore, remove paths from header search paths but keep paths in LIBRARY_PATH + for disallowed_var in self.module_load_environment.alias_vars(MODULE_LOAD_ENV_HEADERS): + self.module_load_environment.remove(disallowed_var) + self.log.debug(f"Purposely not updating ${disallowed_var} in {self.name} module file") + def create_dir(self, dirname): """ Create a dir to build in. @@ -258,10 +266,11 @@ def disable_lto_mpfr_old_gcc(self, objdir): # check whether GCC actually supports LTO (it may be configured with --disable-lto), # by compiling a simple C program using -flto - out, ec = run_cmd("echo 'void main() {}' | gcc -x c -flto - -o /dev/null", simple=False, log_ok=False) + res = run_shell_cmd("echo 'void main() {}' | gcc -x c -flto - -o /dev/null", fail_on_error=False) gcc_path = which('gcc') - if ec: - self.log.info("GCC command %s doesn't seem to support LTO, test compile failed: %s", gcc_path, out) + if res.exit_code: + self.log.info("GCC command %s doesn't seem to support LTO, test compile failed: %s", gcc_path, + res.output) disable_mpfr_lto = True else: self.log.info("GCC command %s provides LTO support, so using it when building MPFR", gcc_path) @@ -442,16 +451,16 @@ def run_configure_cmd(self, cmd): if host_type: cmd += ' --host=' + host_type - (out, ec) = run_cmd("%s %s" % (self.cfg['preconfigopts'], cmd), log_all=True, simple=False) + res = run_shell_cmd("%s %s" % (self.cfg['preconfigopts'], cmd)) - if ec != 0: - raise EasyBuildError("Command '%s' exited with exit code != 0 (%s)", cmd, ec) + if res.exit_code != 0: + raise EasyBuildError("Command '%s' exited with exit code != 0 (%s)", cmd, res.exit_code) # configure scripts tend to simply ignore unrecognized options # we should be more strict here, because GCC is very much a moving target unknown_re = re.compile("WARNING: unrecognized options") - unknown_options = unknown_re.findall(out) + unknown_options = unknown_re.findall(res.output) if unknown_options: raise EasyBuildError("Unrecognized options found during configure: %s", unknown_options) @@ -591,7 +600,7 @@ def configure_step(self): "-DCMAKE_BUILD_TYPE=Release", self.llvm_dir, ]) - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) # Need to terminate the current configuration step, but we can't run 'configure' since LLVM uses # CMake, we therefore run 'CMake' manually and then return nothing. # The normal make stage will build LLVM for us as expected, note that we override the install step @@ -732,14 +741,14 @@ def build_step(self): # make and install stage 1 build of GCC paracmd = '' - if self.cfg['parallel']: - paracmd = "-j %s" % self.cfg['parallel'] + if self.cfg.parallel > 1: + paracmd = f"-j {self.cfg.parallel}" cmd = "%s make %s %s" % (self.cfg['prebuildopts'], paracmd, self.cfg['buildopts']) - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) cmd = "make install %s" % (self.cfg['installopts']) - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) # register built GCC as compiler to use for stage 2/3 path = "%s/bin:%s" % (self.stage1installdir, os.getenv('PATH')) @@ -858,10 +867,10 @@ def build_step(self): # build and 'install' cmd = "make %s" % paracmd - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) cmd = "make install" - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) if lib == "gmp": # make sure correct GMP is found @@ -965,11 +974,11 @@ def install_step(self, *args, **kwargs): else: super(EB_GCC, self).install_step(*args, **kwargs) - def post_install_step(self, *args, **kwargs): + def post_processing_step(self, *args, **kwargs): """ Post-processing after installation: add symlinks for cc, c++, f77, f95 """ - super(EB_GCC, self).post_install_step(*args, **kwargs) + super(EB_GCC, self).post_processing_step(*args, **kwargs) # Add symlinks for cc/c++/f77/f95. bindir = os.path.join(self.installdir, 'bin') @@ -1193,17 +1202,3 @@ def sanity_check_step(self): super(EB_GCC, self).sanity_check_step(custom_paths=custom_paths, custom_commands=custom_commands, extra_modules=extra_modules) - - def make_module_req_guess(self): - """ - GCC can find its own headers and libraries but the .so's need to be in LD_LIBRARY_PATH - """ - guesses = super(EB_GCC, self).make_module_req_guess() - guesses.update({ - 'PATH': ['bin'], - 'CPATH': [], - 'LIBRARY_PATH': ['lib', 'lib64'] if get_cpu_family() == RISCV else [], - 'LD_LIBRARY_PATH': ['lib', 'lib64'], - 'MANPATH': ['man', 'share/man'] - }) - return guesses diff --git a/easybuild/easyblocks/g/gctf.py b/easybuild/easyblocks/g/gctf.py index cd2b9315fa0..a82cacb01b4 100644 --- a/easybuild/easyblocks/g/gctf.py +++ b/easybuild/easyblocks/g/gctf.py @@ -1,5 +1,5 @@ ## -# Copyright 2019-2024 Ghent University +# Copyright 2019-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (https://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/g/geant4.py b/easybuild/easyblocks/g/geant4.py index ed3f5ff3b93..b7c9216f888 100644 --- a/easybuild/easyblocks/g/geant4.py +++ b/easybuild/easyblocks/g/geant4.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -33,23 +33,15 @@ """ import os -import shutil -import re -from easybuild.tools import LooseVersion -import easybuild.tools.environment as env -from easybuild.framework.easyconfig import CUSTOM from easybuild.easyblocks.generic.cmakemake import CMakeMake -from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.modules import get_software_root -from easybuild.tools.run import run_cmd, run_cmd_qa -from easybuild.tools.filetools import copy_file, read_file, write_file +from easybuild.framework.easyconfig import CUSTOM +from easybuild.tools import LooseVersion class EB_Geant4(CMakeMake): """ Support for building Geant4. - Note: Geant4 moved to a CMake-based build system as of version 9.4. """ @staticmethod @@ -69,339 +61,37 @@ def extra_options(): extra_vars['separate_build_dir'][0] = True return extra_vars - def __init__(self, *args, **kwargs): - """Initialisation of custom class variables for Geant4.""" - super(EB_Geant4, self).__init__(*args, **kwargs) - self.g4system = 'UNKNOWN' - self.datadst = 'UNKNOWN' - - def configure_step(self): - """ - Configure Geant4 build, either via CMake for versions more recent than 9.4, - or using an interactive configuration procedure otherwise. - """ - - # Geant4 switched to a CMake build system in version 9.4 - if LooseVersion(self.version) >= LooseVersion("9.4"): - super(EB_Geant4, self).configure_step() - - else: - pwd = self.cfg['start_dir'] - dst = self.installdir - clhepdir = get_software_root('CLHEP') - cmd = "%s/Configure -E -build" % pwd - - self.qanda = { - # questions and answers for version 9.1.p03 - "There exists a config.sh file. Shall I use it to set the defaults? [y]": "n", - "Would you like to see the instructions? [n]": "", - "[Type carriage return to continue]": "", - "Definition of G4SYSTEM variable is Linux-g++. That stands for: 1) OS : Linux" - "2) Compiler : g++ To modify default settings, select number above (e.g. 2) " - "[Press [Enter] for default settings]": "2", - "Which C++ compiler? [g++]": "$(GPP)", - "Confirm your selection or set recommended 'g++'! [*]": "", - "Definition of G4SYSTEM variable is Linux-icc. That stands for: 1) OS : Linux 2)" - "Compiler : icc To modify default settings, select number above (e.g. 2) " - "[Press [Enter] for default settings]": "", - "Do you expect to run these scripts and binaries on multiple machines? [n]": "y", - "Where is Geant4 source installed? [%s]" % pwd: "", - "Specify the path where Geant4 libraries and source files should be installed." - " [%s]" % pwd: dst, - "Do you want to copy all Geant4 headers in one directory? [n]": "y", - "Please, specify default directory where ALL the Geant4 data is installed:" - "G4LEVELGAMMADATA: %(pwd)s/data/PhotonEvaporation2.0 G4RADIOACTIVEDATA: " - "%(pwd)s/data/RadioactiveDecay3.2 G4LEDATA: %(pwd)s/data/G4EMLOW5.1 G4NEUTRONHPDATA: " - "%(pwd)s/data/G4NDL3.12 G4ABLADATA: %(pwd)s/data/G4ABLA3.0 You will be asked about " - "customizing these next. [%(pwd)s/data]" % {'pwd': pwd}: "%s/data" % dst, - "Directory %s/data doesn't exist. Use that name anyway? [n]" % dst: "y", - "Please, specify default directory where the Geant4 data is installed: " - "1) G4LEVELGAMMADATA: %(dst)s/data/PhotonEvaporation2.0 2) G4RADIOACTIVEDATA: " - "%(dst)s/data/RadioactiveDecay3.2 3) G4LEDATA: %(dst)s/data/G4EMLOW5.1 4) G4NEUTRONHPDATA: " - "%(dst)s/data/G4NDL3.12 5) G4ABLADATA: %(dst)s/data/G4ABLA3.0 To modify default settings, " - "select number above (e.g. 2) [Press [Enter] for default settings]" % {'dst': dst}: "", - "Please, specify where CLHEP is installed: CLHEP_BASE_DIR: ": clhepdir, - "Please, specify where CLHEP is installed: CLHEP_BASE_DIR: [%s]" % clhepdir: "", - "You can customize paths and library name of you CLHEP installation: 1) CLHEP_INCLUDE_DIR: " - "%(clhepdir)s/include 2) CLHEP_LIB_DIR: %(clhepdir)s/lib 3) CLHEP_LIB: CLHEP To modify " - "default settings, select number above (e.g. 2) [Press [Enter] for default settings]" % - {'clhepdir': clhepdir}: "", - "By default 'static' (.a) libraries are built. Do you want to build 'shared' (.so) " - "libraries? [n]": "y", - "You selected to build 'shared' (.so) libraries. Do you want to build 'static' (.a) " - "libraries too? [n]": "y", - "Do you want to build 'global' compound libraries? [n]": "", - "Do you want to compile libraries in DEBUG mode (-g)? [n]": "", - "G4UI_NONE If this variable is set, no UI sessions nor any UI libraries are built. " - "This can be useful when running a pure batch job or in a user framework having its own " - "UI system. Do you want to set this variable ? [n]": "", - "G4UI_BUILD_XAW_SESSION G4UI_USE_XAW Specifies to include and use the XAW interfaces in " - "the application to be built. The XAW (X11 Athena Widget set) extensions are required to " - "activate and build this driver. [n]": "", - "G4UI_BUILD_XM_SESSION G4UI_USE_XM Specifies to include and use the XM Motif based user " - "interfaces. The XM Motif extensions are required to activate and build this driver. [n]": "", - "G4VIS_NONE If this variable is set, no visualization drivers will be built or used. Do " - "you want to set this variable ? [n]": "n", - "G4VIS_BUILD_OPENGLX_DRIVER G4VIS_USE_OPENGLX It is an interface to the de facto standard " - "3D graphics library, OpenGL. It is well suited for real-time fast visualization and " - "prototyping. The X11 version of the OpenGL libraries is required. [n]": "", - "G4VIS_BUILD_OPENGLXM_DRIVER G4VIS_USE_OPENGLXM It is an interface to the de facto " - "standard 3D graphics library, OpenGL. It is well suited for real-time fast visualization " - "and prototyping. The X11 version of the OpenGL libraries and the Motif Xm extension is " - "required. [n]": "", - "G4VIS_BUILD_DAWN_DRIVER G4VIS_USE_DAWN DAWN drivers are interfaces to the Fukui Renderer " - "DAWN. DAWN is a vectorized 3D PostScript processor suited to prepare technical high " - "quality outputs for presentation and/or documentation. [n]": "", - "G4VIS_BUILD_OIX_DRIVER G4VIS_USE_OIX The OpenInventor driver is based on OpenInventor tech" - "nology for scientific visualization. The X11 version of OpenInventor is required. [n]": "", - "G4VIS_BUILD_RAYTRACERX_DRIVER G4VIS_USE_RAYTRACERX Allows for interactive ray-tracing " - "graphics through X11. The X11 package is required. [n]": "", - "G4VIS_BUILD_VRML_DRIVER G4VIS_USE_VRML These driver generate VRML files, which describe " - "3D scenes to be visualized with a proper VRML viewer. [n]": "", - "G4LIB_BUILD_GDML Setting this variable will enable building of the GDML plugin module " - "embedded in Geant4 for detector description persistency. It requires your system to have " - "the XercesC library and headers installed. Do you want to set this variable? [n]": "", - "G4LIB_BUILD_G3TOG4 The utility module 'g3tog4' will be built by setting this variable. " - "NOTE: it requires a valid FORTRAN compiler to be installed on your system and the " - "'cernlib' command in the path, in order to build the ancillary tools! Do you want to " - "build 'g3tog4' ? [n]": "", - "G4LIB_BUILD_ZLIB Do you want to activate compression for output files generated by the " - "HepRep visualization driver? [n]": "y", - "G4ANALYSIS_USE Activates the configuration setup for allowing plugins to analysis tools " - "based on AIDA (Astract Interfaces for Data Analysis). In order to use AIDA features and " - "compliant analysis tools, the proper environment for these tools will have to be set " - "(see documentation for the specific analysis tools). [n]": "", - "Press [Enter] to start installation or use a shell escape to edit config.sh: ": "", - # extra questions and answers for version 9.2.p03 - "Directory %s doesn't exist. Use that name anyway? [n]" % dst: "y", - "Specify the path where the Geant4 data libraries PhotonEvaporation%s " - "RadioactiveDecay%s G4EMLOW%s G4NDL%s G4ABLA%s are " - "installed. For now, a flat directory structure is assumed, and this can be customized " - "at the next step if needed. [%s/data]" % (self.cfg['PhotonEvaporationVersion'], - self.cfg['G4RadioactiveDecayVersion'], - self.cfg['G4EMLOWVersion'], - self.cfg['G4NDLVersion'], - self.cfg['G4ABLAVersion'], - pwd - ): "%s/data" % dst, - "Please enter 1) Another path to search in 2) 'f' to force the use of the path " - "you entered previously (the data libraries are not needed to build Geant4, but " - "are needed to run applications later). 3) 'c' to customize the data paths, e.g. " - "if you have the data libraries installed in different locations. [f]": "", - "G4UI_BUILD_QT_SESSION G4UI_USE_QT Setting these variables will enable the building " - "of the G4 Qt based user interface module and the use of this module in your " - "applications respectively. The Qt3 or Qt4 headers, libraries and moc application are " - "required to enable the building of this module. Do you want to enable build and use of " - "this module? [n]": "", - # extra questions and answers for version 9.4.po1 - "What is the path to the Geant4 source tree? [%s]" % pwd: "", - "Where should Geant4 be installed? [%s]" % pwd: dst, - "Do you want to install all Geant4 headers in one directory? [n]": "y", - "Do you want to build shared libraries? [y]": "", - "Do you want to build static libraries too? [n]": "", - "Do you want to build global libraries? [y]": "", - "Do you want to build granular libraries as well? [n]": "", - "Do you want to build libraries with debugging information? [n]": "", - "Specify the path where the Geant4 data libraries are installed: [%s/data]" % pwd: "%s/data" % dst, - "How many parallel jobs should make launch? [1]": "%s" % self.cfg['parallel'], - "Please enter 1) Another path to search in 2) 'f' to force the use of the path you entered " - "previously (the data libraries are NOT needed to build Geant4, but are needed to run " - "applications later). 3) 'c' to customize the data paths, e.g. if you have the data " - "libraries installed in different locations. [f]": "", - "Enable building of User Interface (UI) modules? [y]": "", - "Enable building of the XAW (X11 Athena Widget set) UI module? [n]": "", - "Enable building of the X11-Motif (Xm) UI module? [n]": "", - "Enable building of the Qt UI module? [n]": "", - "Enable building of visualization drivers? [y]": "n", - "Enable the Geometry Description Markup Language (GDML) module? [n]": "", - "Enable build of the g3tog4 utility module? [n]": "", - "Enable internal zlib compression for HepRep visualization? [n] ": "", - } - - self.noqanda = [ - r"Compiling\s+.*?\s+\.\.\.", - r"Making\s+dependency\s+for\s+file\s+.*?\s+\.\.\.", - r"Making\s+libname\.map\s+starter\s+file\s+\.\.\.", - r"Making\s+libname\.map\s+\.\.\.", - r"Reading\s+library\s+get_name\s+map\s+file\s*\.\.\.", - r"Reading\s+dependency\s+files\s*\.\.\.", - r"Creating\s+shared\s+library\s+.*?\s+\.\.\.", - ] - - run_cmd_qa(cmd, self.qanda, self.noqanda, log_all=True, simple=True) - - # determining self.g4system - try: - scriptdirbase = os.path.join(pwd, '.config', 'bin') - filelist = os.listdir(scriptdirbase) - except OSError as err: - raise EasyBuildError("Failed to determine self.g4system: %s", err) - - if len(filelist) != 1: - raise EasyBuildError("Exactly one directory is expected in %s; found back: %s", scriptdirbase, filelist) - else: - self.g4system = filelist[0] - - self.scriptdir = os.path.join(scriptdirbase, self.g4system) - if not os.path.isdir(self.scriptdir): - raise EasyBuildError("Something went wrong. Dir: %s doesn't exist.", self.scriptdir) - self.log.info("The directory containing several important scripts to be copied was found: %s", - self.scriptdir) - - # copying config.sh to pwd - self.log.info("copying config.sh to %s", pwd) - copy_file(os.path.join(self.scriptdir, 'config.sh'), pwd) - - # creating several scripts containing environment variables - cmd = "%s/Configure -S -f config.sh -D g4conf=%s -D abssrc=%s" % (pwd, self.scriptdir, pwd) - run_cmd(cmd, log_all=True, simple=True) - - def build_step(self): - """Build Geant4.""" - if LooseVersion(self.version) >= LooseVersion("9.4"): - super(EB_Geant4, self).build_step() - else: - pwd = self.cfg['start_dir'] - cmd = "%s/Configure -build" % pwd - run_cmd_qa(cmd, self.qanda, no_qa=self.noqanda, log_all=True, simple=True) - - def install_step(self): - """Install Geant4.""" - - if LooseVersion(self.version) >= LooseVersion("9.4"): - super(EB_Geant4, self).install_step() - # '10.01.p03' -> '10.1.3' - shortver = self.version.replace('.0', '.').replace('.p0', '.') - self.datadst = os.path.join(self.installdir, 'share', '%s-%s' % (self.name, shortver), 'data') - - if LooseVersion(self.version) < LooseVersion("9.5"): - version_parts = self.version.split('.') - name = 'geant4-%s' % '.'.join(version_parts[:2] + [version_parts[-1][-1]]) - for (source, target) in [("%s.sh" % name, "env.sh"), ("%s.csh" % name, "env.csh")]: - source = os.path.join(self.installdir, 'share', name, 'config', source) - target = os.path.join(self.installdir, target) - try: - os.symlink(source, target) - except OSError as err: - raise EasyBuildError("Failed to symlink %s to %s: %s", source, target, err) - else: - pwd = self.cfg['start_dir'] - - try: - datasrc = os.path.join(pwd, '..') - self.datadst = os.path.join(self.installdir, 'data') - os.mkdir(self.datadst) - except OSError as err: - raise EasyBuildError("Failed to create data destination dir %s: %s", self.datadst, err) - - datalist = [ - 'G4ABLA%s' % self.cfg['G4ABLAVersion'], - 'G4EMLOW%s' % self.cfg['G4EMLOWVersion'], - 'G4NDL%s' % self.cfg['G4NDLVersion'], - 'PhotonEvaporation%s' % self.cfg['PhotonEvaporationVersion'], - 'RadioactiveDecay%s' % self.cfg['G4RadioactiveDecayVersion'], - ] - try: - for dat in datalist: - self.log.info("Copying %s to %s" % (dat, self.datadst)) - shutil.copytree(os.path.join(datasrc, dat), os.path.join(self.datadst, dat)) - except IOError as err: - raise EasyBuildError("Something went wrong during data copying (%s) to %s: %s", dat, self.datadst, err) - - try: - for fil in ['config', 'environments', 'examples']: - self.log.info("Copying %s to %s" % (fil, self.installdir)) - if not os.path.exists(os.path.join(pwd, fil)): - raise EasyBuildError("No such file or directory: %s", fil) - if os.path.isdir(os.path.join(pwd, fil)): - shutil.copytree(os.path.join(pwd, fil), os.path.join(self.installdir, fil)) - elif os.path.isfile(os.path.join(pwd, fil)): - shutil.copy2(os.path.join(pwd, fil), os.path.join(self.installdir, fil)) - except IOError as err: - raise EasyBuildError("Something went wrong during copying of %s to %s: %s", fil, self.installdir, err) - - try: - for fil in ['config.sh', 'env.sh', 'env.csh']: - self.log.info("Copying %s to %s" % (fil, self.installdir)) - if not os.path.exists(os.path.join(self.scriptdir, fil)): - raise EasyBuildError("No such file or directory: %s", fil) - if os.path.isdir(os.path.join(self.scriptdir, fil)): - shutil.copytree(os.path.join(self.scriptdir, fil), os.path.join(self.installdir, fil)) - elif os.path.isfile(os.path.join(self.scriptdir, fil)): - shutil.copy2(os.path.join(self.scriptdir, fil), os.path.join(self.installdir, fil)) - except IOError as err: - raise EasyBuildError("Something went wrong during copying of (%s) to %s: %s", fil, self.installdir, err) - - cmd = "%(pwd)s/Configure -f %(pwd)s/config.sh -d -install" % {'pwd': pwd} - run_cmd(cmd, log_all=True, simple=True) - - mpiuidir = os.path.join(self.installdir, "examples/extended/parallel/MPI/mpi_interface") - os.chdir(mpiuidir) - - # tweak config file as needed - G4MPItxt = read_file('G4MPI.gmk') - - root_re = re.compile(r"(.*G4MPIROOT\s+=\s+).*", re.MULTILINE) - cxx_re = re.compile(r"(.*CXX\s+:=\s+).*", re.MULTILINE) - cppflags_re = re.compile(r"(.*CPPFLAGS\s+\+=\s+.*)", re.MULTILINE) - - G4MPItxt = root_re.sub(r"\1%s/intel64" % get_software_root('IMPI'), G4MPItxt) - G4MPItxt = cxx_re.sub(r"\1mpicxx -cxx=icpc", G4MPItxt) - G4MPItxt = cppflags_re.sub(r"\1 -I$(G4INCLUDE) -I%s)/include" % get_software_root('CLHEP'), G4MPItxt) - - self.log.debug("contents of G4MPI.gmk: %s" % G4MPItxt) - - shutil.copyfile("G4MPI.gmk", "G4MPI.gmk.ORIG") - write_file('G4MPI.gmk', G4MPItxt) - - # make sure the required environment variables are there - env.setvar("G4INSTALL", self.installdir) - env.setvar("G4SYSTEM", self.g4system) - env.setvar("G4LIB", "%s/lib/geant4/" % self.installdir) - env.setvar("G4INCLUDE", "%s/include/geant4/" % self.installdir) - - run_cmd("make", log_all=True, simple=True) - run_cmd("make includes", log_all=True, simple=True) - def make_module_extra(self): """Define Geant4-specific environment variables in module file.""" g4version = '.'.join(self.version.split('.')[:2]) + # '10.01.p03' -> '10.1.3' + shortver = self.version.replace('.0', '.').replace('.p0', '.') + datadst = os.path.join(self.installdir, 'share', '%s-%s' % (self.name, shortver), 'data') + txt = super(EB_Geant4, self).make_module_extra() txt += self.module_generator.set_environment('G4INSTALL', self.installdir) # no longer needed in > 9.5, but leave it there for now. txt += self.module_generator.set_environment('G4VERSION', g4version) incdir = os.path.join(self.installdir, 'include') - libdir = os.path.join(self.installdir, 'lib') - if LooseVersion(self.version) >= LooseVersion("9.5"): - txt += self.module_generator.set_environment('G4INCLUDE', os.path.join(incdir, 'Geant4')) - txt += self.module_generator.set_environment('G4LIB', os.path.join(self.installdir, 'lib64', 'Geant4')) - elif LooseVersion(self.version) >= LooseVersion("9.4"): - txt += self.module_generator.set_environment('G4INCLUDE', os.path.join(incdir, 'geant4')) - txt += self.module_generator.set_environment('G4LIB', libdir) - else: - txt += self.module_generator.set_environment('G4INCLUDE', os.path.join(incdir, 'geant4')) - txt += self.module_generator.set_environment('G4LIB', os.path.join(libdir, 'geant4')) - txt += self.module_generator.set_environment('G4SYSTEM', self.g4system) - if self.cfg['G4ABLAVersion']: - g4abladata = os.path.join(self.datadst, 'G4ABLA%s' % self.cfg['G4ABLAVersion']) - txt += self.module_generator.set_environment('G4ABLADATA', g4abladata) + txt += self.module_generator.set_environment('G4INCLUDE', os.path.join(incdir, 'Geant4')) + txt += self.module_generator.set_environment('G4LIB', os.path.join(self.installdir, 'lib64', 'Geant4')) if self.cfg['PhotonEvaporationVersion']: - g4levelgammadata = os.path.join(self.datadst, 'PhotonEvaporation%s' % self.cfg['PhotonEvaporationVersion']) + g4levelgammadata = os.path.join(datadst, 'PhotonEvaporation%s' % self.cfg['PhotonEvaporationVersion']) txt += self.module_generator.set_environment('G4LEVELGAMMADATA', g4levelgammadata) if self.cfg['G4RadioactiveDecayVersion']: - g4radioactivedata = os.path.join(self.datadst, 'RadioactiveDecay%s' % self.cfg['G4RadioactiveDecayVersion']) + g4radioactivedata = os.path.join(datadst, 'RadioactiveDecay%s' % self.cfg['G4RadioactiveDecayVersion']) txt += self.module_generator.set_environment('G4RADIOACTIVEDATA', g4radioactivedata) if self.cfg['G4EMLOWVersion']: - g4ledata = os.path.join(self.datadst, 'G4EMLOW%s' % self.cfg['G4EMLOWVersion']) + g4ledata = os.path.join(datadst, 'G4EMLOW%s' % self.cfg['G4EMLOWVersion']) txt += self.module_generator.set_environment('G4LEDATA', g4ledata) if self.cfg['G4NDLVersion']: - g4neutronhpdata = os.path.join(self.datadst, 'G4NDL%s' % self.cfg['G4NDLVersion']) + g4neutronhpdata = os.path.join(datadst, 'G4NDL%s' % self.cfg['G4NDLVersion']) txt += self.module_generator.set_environment('G4NEUTRONHPDATA', g4neutronhpdata) return txt @@ -410,25 +100,18 @@ def sanity_check_step(self): """ Custom sanity check for Geant4 """ - bin_files = ["bin/geant4-config"] - if LooseVersion(self.version) >= LooseVersion("9.5"): - bin_files.extend(["bin/geant4.sh", "bin/geant4.csh"]) - libs = ['analysis', 'event', 'GMocren', 'materials', 'readout', 'Tree', 'VRML'] + bin_files = ["bin/geant4-config", "bin/geant4.sh", "bin/geant4.csh"] + libs = ['analysis', 'event', 'GMocren', 'materials', 'readout', 'Tree', 'VRML'] - # G4Persistency library was split up in Geant v11.2, - # see https://geant4.web.cern.ch/download/release-notes/notes-v11.2.0.html - if LooseVersion(self.version) >= LooseVersion('11.2'): - libs.extend(['gdml', 'geomtext', 'mctruth', 'geomtext']) - else: - libs.append('persistency') - - lib_files = ["lib64/libG4%s.so" % x for x in libs] - include_dir = 'include/Geant4' + # G4Persistency library was split up in Geant v11.2, + # see https://geant4.web.cern.ch/download/release-notes/notes-v11.2.0.html + if LooseVersion(self.version) >= LooseVersion('11.2'): + libs.extend(['gdml', 'geomtext', 'mctruth', 'geomtext']) else: - # paths for v9.4, untested for prior versions - lib_files = ["lib/libG4%s.so" % x for x in ['event', 'GMocren', 'materials', 'persistency', - 'readout', 'Tree', 'VRML']] - include_dir = 'include/geant4' + libs.append('persistency') + + lib_files = ["lib64/libG4%s.so" % x for x in libs] + include_dir = 'include/Geant4' custom_paths = { 'files': bin_files + lib_files, diff --git a/easybuild/easyblocks/g/ghc.py b/easybuild/easyblocks/g/ghc.py index cfae0650626..ffd3a3a3c8b 100644 --- a/easybuild/easyblocks/g/ghc.py +++ b/easybuild/easyblocks/g/ghc.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/g/go.py b/easybuild/easyblocks/g/go.py index 25d06272e3a..d3967539a1e 100644 --- a/easybuild/easyblocks/g/go.py +++ b/easybuild/easyblocks/g/go.py @@ -1,5 +1,5 @@ ## -# Copyright 2014-2024 Ghent University +# Copyright 2014-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -36,7 +36,7 @@ from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import remove_dir -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd from easybuild.tools.modules import get_software_root @@ -59,10 +59,6 @@ def install_step(self): specifying the final installation prefix by setting $GOROOT_FINAL. """ srcdir = os.path.join(self.cfg['start_dir'], 'src') - try: - os.chdir(srcdir) - except OSError as err: - raise EasyBuildError("Failed to move to %s: %s", srcdir, err) # $GOROOT_FINAL only specifies the location of the final installation, which gets baked into the binaries # the installation itself is *not* done by the all.bash script, that needs to be done manually @@ -77,7 +73,7 @@ def install_step(self): else: cmd = "GOROOT_FINAL=%s ./all.bash" % self.installdir - run_cmd(cmd, log_all=True, simple=False) + run_shell_cmd(cmd, work_dir=srcdir) try: remove_dir(self.installdir) diff --git a/easybuild/easyblocks/g/gromacs.py b/easybuild/easyblocks/g/gromacs.py index 6aa96f92ac8..cba739f3817 100644 --- a/easybuild/easyblocks/g/gromacs.py +++ b/easybuild/easyblocks/g/gromacs.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2024 Ghent University +# Copyright 2013-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -32,25 +32,27 @@ @author: Guilherme Peretti-Pezzi (CSCS) @author: Oliver Stueker (Compute Canada/ACENET) @author: Davide Vanzo (Vanderbilt University) +@author: Alex Domingo (Vrije Universiteit Brussel) """ import glob import os import re import shutil -from easybuild.tools import LooseVersion import easybuild.tools.environment as env import easybuild.tools.toolchain as toolchain -from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.easyblocks.generic.cmakemake import CMakeMake +from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.framework.easyconfig import CUSTOM +from easybuild.tools import LooseVersion from easybuild.tools.build_log import EasyBuildError, print_warning from easybuild.tools.config import build_option from easybuild.tools.filetools import copy_dir, find_backup_name_candidate, remove_dir, which from easybuild.tools.modules import get_software_libdir, get_software_root, get_software_version -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd +from easybuild.tools.systemtools import X86_64, get_cpu_architecture, get_cpu_features, get_shared_lib_ext from easybuild.tools.toolchain.compiler import OPTARCH_GENERIC -from easybuild.tools.systemtools import X86_64, get_cpu_architecture, get_shared_lib_ext, get_cpu_features +from easybuild.tools.utilities import nub from easybuild.tools.version import VERBOSE_VERSION as EASYBUILD_VERSION @@ -63,6 +65,8 @@ def extra_options(): extra_vars.update({ 'double_precision': [None, "Build with double precision enabled (-DGMX_DOUBLE=ON), " + "default is to build double precision unless CUDA is enabled", CUSTOM], + 'single_precision': [True, "Build with single precision enabled (-DGMX_DOUBLE=OFF), " + + "default is to build single precision", CUSTOM], 'mpisuffix': ['_mpi', "Suffix to append to MPI-enabled executables (only for GROMACS < 4.6)", CUSTOM], 'mpiexec': ['mpirun', "MPI executable to use when running tests", CUSTOM], 'mpiexec_numproc_flag': ['-np', "Flag to introduce the number of MPI tasks when running tests", CUSTOM], @@ -77,12 +81,16 @@ def extra_options(): def __init__(self, *args, **kwargs): """Initialize GROMACS-specific variables.""" super(EB_GROMACS, self).__init__(*args, **kwargs) - self.lib_subdir = '' + + self._lib_subdirs = [] # list of directories with libraries + self.pre_env = '' self.cfg['build_shared_libs'] = self.cfg.get('build_shared_libs', False) + if LooseVersion(self.version) >= LooseVersion('2019'): # Building the gmxapi interface requires shared libraries self.cfg['build_shared_libs'] = True + if self.cfg['build_shared_libs']: self.libext = get_shared_lib_ext() else: @@ -140,6 +148,7 @@ def get_gromacs_arch(self): return res + @property def is_double_precision_cuda_build(self): """Check if the current build step involves double precision and CUDA""" cuda = get_software_root('CUDA') @@ -223,10 +232,11 @@ def configure_step(self): # Need to check if PLUMED has an engine for this version engine = 'gromacs-%s' % self.version - (out, _) = run_cmd("plumed-patch -l", log_all=True, simple=False) - if not re.search(engine, out): + res = run_shell_cmd("plumed-patch -l") + if not re.search(engine, res.output): plumed_ver = get_software_version('PLUMED') - msg = "There is no support in PLUMED version %s for GROMACS %s: %s" % (plumed_ver, self.version, out) + msg = "There is no support in PLUMED version %s for GROMACS %s: %s" % (plumed_ver, self.version, + res.output) if self.cfg['ignore_plumed_version_check']: self.log.warning(msg) else: @@ -283,19 +293,19 @@ def configure_step(self): # Now patch GROMACS for PLUMED between configure and build if plumed_root: - run_cmd(plumed_cmd, log_all=True, simple=True) + run_shell_cmd(plumed_cmd) else: if '-DGMX_MPI=ON' in self.cfg['configopts']: mpi_numprocs = self.cfg.get('mpi_numprocs', 0) if mpi_numprocs == 0: self.log.info("No number of test MPI tasks specified -- using default: %s", - self.cfg['parallel']) - mpi_numprocs = self.cfg['parallel'] + self.cfg.parallel) + mpi_numprocs = self.cfg.parallel - elif mpi_numprocs > self.cfg['parallel']: + elif mpi_numprocs > self.cfg.parallel: self.log.warning("Number of test MPI tasks (%s) is greater than value for 'parallel': %s", - mpi_numprocs, self.cfg['parallel']) + mpi_numprocs, self.cfg.parallel) mpiexec = self.cfg.get('mpiexec') if mpiexec: @@ -343,7 +353,7 @@ def configure_step(self): mode = 'static' plumed_cmd = plumed_cmd + ' -m %s' % mode - run_cmd(plumed_cmd, log_all=True, simple=True) + run_shell_cmd(plumed_cmd) # prefer static libraries, if available if self.cfg['build_shared_libs']: @@ -354,8 +364,9 @@ def configure_step(self): # always specify to use external BLAS/LAPACK self.cfg.update('configopts', "-DGMX_EXTERNAL_BLAS=ON -DGMX_EXTERNAL_LAPACK=ON") - # disable GUI tools - self.cfg.update('configopts', "-DGMX_X11=OFF") + if gromacs_version < '2023': + # disable GUI tools, removed in v2023 + self.cfg.update('configopts', "-DGMX_X11=OFF") # convince to build for an older architecture than present on the build node by setting GMX_SIMD CMake flag # it does not make sense for Cray, because OPTARCH is defined by the Cray Toolchain @@ -457,7 +468,7 @@ def build_step(self): iteration is for double precision """ - if self.is_double_precision_cuda_build(): + if self.is_double_precision_cuda_build: self.log.info("skipping build step") else: super(EB_GROMACS, self).build_step() @@ -465,7 +476,7 @@ def build_step(self): def test_step(self): """Run the basic tests (but not necessarily the full regression tests) using make check""" - if self.is_double_precision_cuda_build(): + if self.is_double_precision_cuda_build: self.log.info("skipping test step") else: # allow to escape testing by setting runtest to False @@ -501,7 +512,7 @@ def test_step(self): # run 'make check' or whatever the easyconfig specifies # in parallel since it involves more compilation - self.cfg.update('runtest', "-j %s" % self.cfg['parallel']) + self.cfg.update('runtest', f"-j {self.cfg.parallel}") super(EB_GROMACS, self).test_step() if build_option('rpath'): @@ -521,11 +532,11 @@ def install_step(self): Custom install step for GROMACS; figure out where libraries were installed to. """ # Skipping if CUDA is enabled and the current iteration is double precision - if self.is_double_precision_cuda_build(): + if self.is_double_precision_cuda_build: self.log.info("skipping install step") else: # run 'make install' in parallel since it involves more compilation - self.cfg.update('installopts', "-j %s" % self.cfg['parallel']) + self.cfg.update('installopts', f"-j {self.cfg.parallel}") super(EB_GROMACS, self).install_step() def extensions_step(self, fetch=False): @@ -544,48 +555,60 @@ def extensions_step(self, fetch=False): super(EB_GROMACS, self).extensions_step(fetch) self.cfg['runtest'] = orig_runtest - def get_lib_subdir(self): - # the GROMACS libraries get installed in different locations (deeper subdirectory), - # depending on the platform; - # this is determined by the GNUInstallDirs CMake module; - # rather than trying to replicate the logic, we just figure out where the library was placed - - if LooseVersion(self.version) < LooseVersion('5.0'): - libname = 'libgmx*.%s' % self.libext - else: - libname = 'libgromacs*.%s' % self.libext - lib_subdir = None - for libdir in ['lib', 'lib64']: - if os.path.exists(os.path.join(self.installdir, libdir)): - for subdir in [libdir, os.path.join(libdir, '*')]: - libpaths = glob.glob(os.path.join(self.installdir, subdir, libname)) - if libpaths: - lib_subdir = os.path.dirname(libpaths[0])[len(self.installdir) + 1:] - self.log.info("Found lib subdirectory that contains %s: %s", libname, lib_subdir) - break - if not lib_subdir: - raise EasyBuildError("Failed to determine lib subdirectory in %s", self.installdir) - - return lib_subdir - - def make_module_req_guess(self): - """Custom library subdirectories for GROMACS.""" - guesses = super(EB_GROMACS, self).make_module_req_guess() - if not self.lib_subdir: + @property + def lib_subdirs(self): + """Return list of relative paths to subdirs holding library files""" + if len(self._lib_subdirs) == 0: try: - self.lib_subdir = self.get_lib_subdir() + self._lib_subdirs = self.get_lib_subdirs() except EasyBuildError as error: if build_option('force') and build_option('module_only'): - self.log.info("No lib subdirectory directory found in installation: %s", error) + self.log.info(f"No sub-directory with GROMACS libraries found in installation: {error}") self.log.info("You are forcing module creation for a non-existent installation!") else: raise error - guesses.update({ - 'LD_LIBRARY_PATH': [self.lib_subdir], - 'LIBRARY_PATH': [self.lib_subdir], - 'PKG_CONFIG_PATH': [os.path.join(self.lib_subdir, 'pkgconfig')], - }) - return guesses + + return self._lib_subdirs + + def get_lib_subdirs(self): + """ + Return list of relative paths to sub-directories that contain GROMACS libraries + + The GROMACS libraries get installed in different locations (deeper subdirectory), + depending on the platform; + this is determined by the GNUInstallDirs CMake module; + rather than trying to replicate the logic, we just figure out where the library was placed + """ + + if LooseVersion(self.version) < LooseVersion('5.0'): + libname = f'libgmx*.{self.libext}' + else: + libname = f'libgromacs*.{self.libext}' + + lib_subdirs = [] + real_installdir = os.path.realpath(self.installdir) + for lib_path in glob.glob(os.path.join(real_installdir, '**', libname), recursive=True): + lib_relpath = os.path.realpath(lib_path) # avoid symlinks + lib_relpath = lib_relpath[len(real_installdir) + 1:] # relative path from installdir + subdir = lib_relpath.split(os.sep)[0:-1] + lib_subdirs.append(os.path.join(*subdir)) + + if len(lib_subdirs) == 0: + raise EasyBuildError(f"Failed to determine sub-directory with {libname} in {self.installdir}") + + # remove duplicates, 'libname' pattern can match symlinks to actual library file + lib_subdirs = nub(lib_subdirs) + self.log.info(f"Found sub-directories that contain {libname}: {', '.join(lib_subdirs)}") + + return lib_subdirs + + def make_module_step(self, *args, **kwargs): + """Custom library subdirectories for GROMACS.""" + self.module_load_environment.LD_LIBRARY_PATH = self.lib_subdirs + self.module_load_environment.LIBRARY_PATH = self.lib_subdirs + self.module_load_environment.PKG_CONFIG_PATH = [os.path.join(ld, 'pkgconfig') for ld in self.lib_subdirs] + + return super().make_module_step(*args, **kwargs) def sanity_check_step(self): """Custom sanity check for GROMACS.""" @@ -656,21 +679,16 @@ def sanity_check_step(self): if dsuff: suffixes.extend([dsuff]) - lib_files.extend([ - 'lib%s%s.%s' % (x, suff, self.libext) for x in libnames + mpi_libnames for suff in suffixes - ]) + lib_files.extend([f'lib{x}{suff}.{self.libext}' for x in libnames + mpi_libnames for suff in suffixes]) bin_files.extend([b + suff for b in bins + mpi_bins for suff in suffixes]) - if not self.lib_subdir: - self.lib_subdir = self.get_lib_subdir() - # pkgconfig dir not available for earlier versions, exact version to use here is unclear if LooseVersion(self.version) >= LooseVersion('4.6'): - dirs.append(os.path.join(self.lib_subdir, 'pkgconfig')) + dirs.extend([os.path.join(ld, 'pkgconfig') for ld in self.lib_subdirs]) custom_paths = { 'files': [os.path.join('bin', b) for b in bin_files] + - [os.path.join(self.lib_subdir, lib) for lib in lib_files], + [os.path.join(libdir, lib) for libdir in self.lib_subdirs for lib in lib_files], 'dirs': dirs, } super(EB_GROMACS, self).sanity_check_step(custom_paths=custom_paths) @@ -743,10 +761,15 @@ def run_all_steps(self, *args, **kwargs): 'mpi': 'install' } - precisions = ['single'] + precisions = [] + if self.cfg.get('single_precision'): + precisions.append('single') if self.cfg.get('double_precision') is None or self.cfg.get('double_precision'): precisions.append('double') + if precisions == []: + raise EasyBuildError("No precision selected. At least one of single/double_precision must be unset or True") + mpitypes = ['nompi'] if self.toolchain.options.get('usempi', None): mpitypes.append('mpi') diff --git a/easybuild/easyblocks/g/gurobi.py b/easybuild/easyblocks/g/gurobi.py index c8b90e9dfe9..c36cf450e2e 100644 --- a/easybuild/easyblocks/g/gurobi.py +++ b/easybuild/easyblocks/g/gurobi.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -31,14 +31,13 @@ """ import os -from easybuild.easyblocks.generic.pythonpackage import det_pylibdir from easybuild.easyblocks.generic.tarball import Tarball from easybuild.framework.easyconfig import CUSTOM from easybuild.tools import LooseVersion from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import copy_file from easybuild.tools.modules import get_software_root -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd class EB_Gurobi(Tarball): @@ -77,7 +76,7 @@ def install_step(self): copy_file(self.orig_license_file, self.license_file) if get_software_root('Python') and LooseVersion(self.version) < LooseVersion('11'): - run_cmd("python setup.py install --prefix=%s" % self.installdir) + run_shell_cmd("python setup.py install --prefix=%s" % self.installdir) def sanity_check_step(self): """Custom sanity check for Gurobi.""" @@ -101,10 +100,6 @@ def make_module_extra(self): txt = super(EB_Gurobi, self).make_module_extra() txt += self.module_generator.set_environment('GUROBI_HOME', self.installdir) txt += self.module_generator.set_environment('GRB_LICENSE_FILE', self.license_file) - - if get_software_root('Python'): - txt += self.module_generator.prepend_paths('PYTHONPATH', det_pylibdir()) - txt += self.module_generator.prepend_paths('MATLABPATH', 'matlab') return txt diff --git a/easybuild/easyblocks/generic/__init__.py b/easybuild/easyblocks/generic/__init__.py index 8d496189c34..d0005a450d9 100644 --- a/easybuild/easyblocks/generic/__init__.py +++ b/easybuild/easyblocks/generic/__init__.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/generic/binariestarball.py b/easybuild/easyblocks/generic/binariestarball.py index 53b8de19f7c..dc8fa7173f0 100644 --- a/easybuild/easyblocks/generic/binariestarball.py +++ b/easybuild/easyblocks/generic/binariestarball.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2024 Ghent University +# Copyright 2013-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/generic/binary.py b/easybuild/easyblocks/generic/binary.py index 288859ecbb1..fc85ad20ad3 100644 --- a/easybuild/easyblocks/generic/binary.py +++ b/easybuild/easyblocks/generic/binary.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -40,7 +40,7 @@ from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import adjust_permissions, copy_file, mkdir, remove_dir -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd PREPEND_TO_PATH_DEFAULT = [''] @@ -126,12 +126,12 @@ def install_step(self): for install_cmd in install_cmds: cmd = ' '.join([self.cfg['preinstallopts'], install_cmd, self.cfg['installopts']]) self.log.info("Running install command for %s: '%s'..." % (self.name, cmd)) - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) else: raise EasyBuildError("Incorrect value type for install_cmds, should be list or tuple: ", install_cmds) - def post_install_step(self): + def post_processing_step(self): """Copy installation to actual installation directory in case of a staged installation.""" if self.cfg.get('staged_install', False): staged_installdir = self.installdir @@ -145,7 +145,7 @@ def post_install_step(self): raise EasyBuildError("Failed to move staged install from %s to %s: %s", staged_installdir, self.installdir, err) - super(Binary, self).post_install_step() + super(Binary, self).post_processing_step() def sanity_check_rpath(self): """Skip the rpath sanity check, this is binary software""" diff --git a/easybuild/easyblocks/generic/buildenv.py b/easybuild/easyblocks/generic/buildenv.py index 35be80739ce..85a7d204b4c 100644 --- a/easybuild/easyblocks/generic/buildenv.py +++ b/easybuild/easyblocks/generic/buildenv.py @@ -1,5 +1,5 @@ ## -# Copyright 2015-2024 Ghent University +# Copyright 2015-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/generic/bundle.py b/easybuild/easyblocks/generic/bundle.py index b8abecf8fc0..2a5b9751718 100644 --- a/easybuild/easyblocks/generic/bundle.py +++ b/easybuild/easyblocks/generic/bundle.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -31,6 +31,7 @@ @author: Pieter De Baets (Ghent University) @author: Jens Timmerman (Ghent University) @author: Jasper Grimm (University of York) +@author: Jan Andre Reuter (Juelich Supercomputing Centre) """ import copy import os @@ -41,7 +42,7 @@ from easybuild.framework.easyconfig.easyconfig import get_easyblock_class from easybuild.tools.build_log import EasyBuildError, print_msg from easybuild.tools.modules import get_software_root, get_software_version -from easybuild.tools.py2vs3 import string_type +from easybuild.tools.utilities import nub class Bundle(EasyBlock): @@ -71,8 +72,8 @@ def __init__(self, *args, **kwargs): self.altroot = None self.altversion = None - # list of EasyConfig instances for components - self.comp_cfgs = [] + # list of EasyConfig instances and their EasyBlocks for components + self.comp_instances = [] # list of EasyConfig instances of components for which to run sanity checks self.comp_cfgs_sanity_check = [] @@ -80,9 +81,9 @@ def __init__(self, *args, **kwargs): check_for_sources = getattr(self, 'check_for_sources', True) # list of sources for bundle itself *must* be empty (unless overridden by subclass) if check_for_sources: - if self.cfg['sources']: + if self.cfg.get_ref('sources'): raise EasyBuildError("List of sources for bundle itself must be empty, found %s", self.cfg['sources']) - if self.cfg['patches']: + if self.cfg.get_ref('patches'): raise EasyBuildError("List of patches for bundle itself must be empty, found %s", self.cfg['patches']) # copy EasyConfig instance before we make changes to it @@ -90,119 +91,123 @@ def __init__(self, *args, **kwargs): self.cfg = self.cfg.copy() # disable templating to avoid premature resolving of template values - self.cfg.enable_templating = False - - # list of checksums for patches (must be included after checksums for sources) - checksums_patches = [] - - if self.cfg['sanity_check_components'] and self.cfg['sanity_check_all_components']: - raise EasyBuildError("sanity_check_components and sanity_check_all_components cannot be enabled together") - - # backup and reset general sanity checks from main body of ec, if component-specific sanity checks are enabled - # necessary to avoid: - # - duplicating the general sanity check across all components running sanity checks - # - general sanity checks taking precedence over those defined in a component's easyblock - self.backup_sanity_paths = self.cfg['sanity_check_paths'] - self.backup_sanity_cmds = self.cfg['sanity_check_commands'] - if self.cfg['sanity_check_components'] or self.cfg['sanity_check_all_components']: - # reset general sanity checks, to be restored later - self.cfg['sanity_check_paths'] = {} - self.cfg['sanity_check_commands'] = {} - - for comp in self.cfg['components']: - comp_name, comp_version, comp_specs = comp[0], comp[1], {} - if len(comp) == 3: - comp_specs = comp[2] - - comp_cfg = self.cfg.copy() - - comp_cfg['name'] = comp_name - comp_cfg['version'] = comp_version - - # determine easyblock to use for this component - # - if an easyblock is specified explicitely, that will be used - # - if not, a software-specific easyblock will be considered by get_easyblock_class - # - if no easyblock was found, default_easyblock is considered - comp_easyblock = comp_specs.get('easyblock') - easyblock_class = get_easyblock_class(comp_easyblock, name=comp_name, error_on_missing_easyblock=False) - if easyblock_class is None: - if self.cfg['default_easyblock']: - easyblock = self.cfg['default_easyblock'] - easyblock_class = get_easyblock_class(easyblock) - + # Note that self.cfg.update also resolves templates! + with self.cfg.disable_templating(): + # list of checksums for patches (must be included after checksums for sources) + checksums_patches = [] + + if self.cfg['sanity_check_components'] and self.cfg['sanity_check_all_components']: + raise EasyBuildError("sanity_check_components and sanity_check_all_components" + "cannot be enabled together") + + # backup and reset general sanity checks from main body of ec, + # if component-specific sanity checks are enabled necessary to avoid: + # - duplicating the general sanity check across all components running sanity checks + # - general sanity checks taking precedence over those defined in a component's easyblock + self.backup_sanity_paths = self.cfg['sanity_check_paths'] + self.backup_sanity_cmds = self.cfg['sanity_check_commands'] + if self.cfg['sanity_check_components'] or self.cfg['sanity_check_all_components']: + # reset general sanity checks, to be restored later + self.cfg['sanity_check_paths'] = {} + self.cfg['sanity_check_commands'] = {} + components = self.cfg['components'] + + for comp in components: + comp_name, comp_version, comp_specs = comp[0], comp[1], {} + if len(comp) == 3: + comp_specs = comp[2] + + comp_cfg = self.cfg.copy() + + comp_cfg['name'] = comp_name + comp_cfg['version'] = comp_version + + # determine easyblock to use for this component + # - if an easyblock is specified explicitely, that will be used + # - if not, a software-specific easyblock will be considered by get_easyblock_class + # - if no easyblock was found, default_easyblock is considered + comp_easyblock = comp_specs.get('easyblock') + easyblock_class = get_easyblock_class(comp_easyblock, name=comp_name, error_on_missing_easyblock=False) if easyblock_class is None: - raise EasyBuildError("No easyblock found for component %s v%s", comp_name, comp_version) - else: - self.log.info("Using default easyblock %s for component %s", easyblock, comp_name) - else: - easyblock = easyblock_class.__name__ - self.log.info("Using easyblock %s for component %s", easyblock, comp_name) + if self.cfg['default_easyblock']: + easyblock = self.cfg['default_easyblock'] + easyblock_class = get_easyblock_class(easyblock) - if easyblock == 'Bundle': - raise EasyBuildError("The Bundle easyblock can not be used to install components in a bundle") + if easyblock_class is None: + raise EasyBuildError("No easyblock found for component %s v%s", comp_name, comp_version) + self.log.info("Using default easyblock %s for component %s", easyblock, comp_name) + else: + easyblock = easyblock_class.__name__ + self.log.info("Using easyblock %s for component %s", easyblock, comp_name) - comp_cfg.easyblock = easyblock_class + if easyblock == 'Bundle': + raise EasyBuildError("The Bundle easyblock can not be used to install components in a bundle") - # make sure that extra easyconfig parameters are known, so they can be set - extra_opts = comp_cfg.easyblock.extra_options() - comp_cfg.extend_params(copy.deepcopy(extra_opts)) + comp_cfg.easyblock = easyblock_class - comp_cfg.generate_template_values() + # make sure that extra easyconfig parameters are known, so they can be set + extra_opts = comp_cfg.easyblock.extra_options() + comp_cfg.extend_params(copy.deepcopy(extra_opts)) - # do not inherit easyblock to use from parent (since that would result in an infinite loop in install_step) - comp_cfg['easyblock'] = None + comp_cfg.generate_template_values() - # reset list of sources/source_urls/checksums - comp_cfg['sources'] = comp_cfg['source_urls'] = comp_cfg['checksums'] = comp_cfg['patches'] = [] + # do not inherit easyblock to use from parent + # (since that would result in an infinite loop in install_step) + comp_cfg['easyblock'] = None - for key in self.cfg['default_component_specs']: - comp_cfg[key] = self.cfg['default_component_specs'][key] + # reset list of sources/source_urls/checksums + comp_cfg['sources'] = comp_cfg['source_urls'] = comp_cfg['checksums'] = comp_cfg['patches'] = [] - for key in comp_specs: - comp_cfg[key] = comp_specs[key] + for key in self.cfg['default_component_specs']: + comp_cfg[key] = self.cfg['default_component_specs'][key] - # enable resolving of templates for component-specific EasyConfig instance - comp_cfg.enable_templating = True + for key in comp_specs: + comp_cfg[key] = comp_specs[key] - # 'sources' is strictly required - if comp_cfg['sources']: + # Don't require that all template values can be resolved at this point but still resolve them. + # This is important to ensure that template values like %(name)s and %(version)s + # are correctly resolved with the component name/version before values are copied over to self.cfg + with comp_cfg.allow_unresolved_templates(): + comp_sources = comp_cfg['sources'] + comp_source_urls = comp_cfg['source_urls'] + if not comp_sources: + raise EasyBuildError("No sources specification for component %s v%s", comp_name, comp_version) # If per-component source URLs are provided, attach them directly to the relevant sources - if comp_cfg['source_urls']: - for source in comp_cfg['sources']: - if isinstance(source, string_type): - self.cfg.update('sources', [{'filename': source, 'source_urls': comp_cfg['source_urls']}]) + if comp_source_urls: + for source in comp_sources: + if isinstance(source, str): + self.cfg.update('sources', [{'filename': source, 'source_urls': comp_source_urls[:]}]) elif isinstance(source, dict): # Update source_urls in the 'source' dict to use the one for the components # (if it doesn't already exist) if 'source_urls' not in source: - source['source_urls'] = comp_cfg['source_urls'] + source['source_urls'] = comp_source_urls[:] self.cfg.update('sources', [source]) else: raise EasyBuildError("Source %s for component %s is neither a string nor a dict, cannot " "process it.", source, comp_cfg['name']) else: # add component sources to list of sources - self.cfg.update('sources', comp_cfg['sources']) - else: - raise EasyBuildError("No sources specification for component %s v%s", comp_name, comp_version) + self.cfg.update('sources', comp_sources) - if comp_cfg['checksums']: - src_cnt = len(comp_cfg['sources']) + comp_checksums = comp_cfg['checksums'] + if comp_checksums: + src_cnt = len(comp_sources) - # add per-component checksums for sources to list of checksums - self.cfg.update('checksums', comp_cfg['checksums'][:src_cnt]) + # add per-component checksums for sources to list of checksums + self.cfg.update('checksums', comp_checksums[:src_cnt]) - # add per-component checksums for patches to list of checksums for patches - checksums_patches.extend(comp_cfg['checksums'][src_cnt:]) + # add per-component checksums for patches to list of checksums for patches + checksums_patches.extend(comp_checksums[src_cnt:]) - if comp_cfg['patches']: - self.cfg.update('patches', comp_cfg['patches']) + with comp_cfg.allow_unresolved_templates(): + comp_patches = comp_cfg['patches'] + if comp_patches: + self.cfg.update('patches', comp_patches) - self.comp_cfgs.append(comp_cfg) + self.comp_instances.append((comp_cfg, comp_cfg.easyblock(comp_cfg, logfile=self.logfile))) - self.cfg.update('checksums', checksums_patches) - - self.cfg.enable_templating = True + self.cfg.update('checksums', checksums_patches) # restore general sanity checks if using component-specific sanity checks if self.cfg['sanity_check_components'] or self.cfg['sanity_check_all_components']: @@ -217,7 +222,7 @@ def check_checksums(self): """ checksum_issues = super(Bundle, self).check_checksums() - for comp in self.comp_cfgs: + for comp, _ in self.comp_instances: checksum_issues.extend(self.check_checksums_for(comp, sub="of component %s" % comp['name'])) return checksum_issues @@ -247,14 +252,12 @@ def build_step(self): def install_step(self): """Install components, if specified.""" comp_cnt = len(self.cfg['components']) - for idx, cfg in enumerate(self.comp_cfgs): + for idx, (cfg, comp) in enumerate(self.comp_instances): print_msg("installing bundle component %s v%s (%d/%d)..." % (cfg['name'], cfg['version'], idx + 1, comp_cnt)) self.log.info("Installing component %s v%s using easyblock %s", cfg['name'], cfg['version'], cfg.easyblock) - comp = cfg.easyblock(cfg) - # correct build/install dirs comp.builddir = self.builddir comp.install_subdir, comp.installdir = self.install_subdir, self.installdir @@ -270,9 +273,11 @@ def install_step(self): comp.src = [] - # find match entries in self.src for this component - for source in comp.cfg['sources']: - if isinstance(source, string_type): + # find matching entries in self.src for this component + with comp.cfg.allow_unresolved_templates(): + comp_sources = comp.cfg['sources'] + for source in comp_sources: + if isinstance(source, str): comp_src_fn = source elif isinstance(source, dict): if 'filename' in source: @@ -306,23 +311,84 @@ def install_step(self): else: comp.run_step(step_name, [lambda x: getattr(x, '%s_step' % step_name)]) - # update environment to ensure stuff provided by former components can be picked up by latter components - # once the installation is finalised, this is handled by the generated module - reqs = comp.make_module_req_guess() - for envvar in reqs: - curr_val = os.getenv(envvar, '') - curr_paths = curr_val.split(os.pathsep) - for subdir in reqs[envvar]: - path = os.path.join(self.installdir, subdir) - if path not in curr_paths: - if curr_val: - new_val = '%s:%s' % (path, curr_val) + if comp.make_module_req_guess.__qualname__ != 'EasyBlock.make_module_req_guess': + depr_msg = f"Easyblock used to install component {comp.name} still uses make_module_req_guess" + self.log.deprecated(depr_msg, '6.0') + # update environment to ensure stuff provided by former components can be picked up by latter components + # once the installation is finalised, this is handled by the generated module + reqs = comp.make_module_req_guess() + for envvar in reqs: + curr_val = os.getenv(envvar, '') + curr_paths = curr_val.split(os.pathsep) + for subdir in reqs[envvar]: + path = os.path.join(self.installdir, subdir) + if path not in curr_paths: + if curr_val: + new_val = '%s:%s' % (path, curr_val) + else: + new_val = path + env.setvar(envvar, new_val) + else: + # Update current environment with component environment to ensure stuff provided + # by this component can be picked up by installation of subsequent components + # - this is a stripped down version of EasyBlock.make_module_req for fake modules + # - once bundle installation is complete, this is handled by the generated module as usual + for mod_envar, mod_paths in comp.module_load_environment.items(): + # expand glob patterns in module load environment to existing absolute paths + mod_expand = [x for p in mod_paths for x in comp.expand_module_search_path(p, False)] + mod_expand = nub(mod_expand) + mod_expand = [os.path.join(self.installdir, path) for path in mod_expand] + # prepend to current environment variable if new stuff added to installation + curr_env = os.getenv(mod_envar, '') + curr_paths = [path for path in curr_env.split(os.pathsep) if path] + new_paths = nub(mod_expand + curr_paths) + new_env = os.pathsep.join(new_paths) + if new_env and new_env != curr_env: + env.setvar(mod_envar, new_env) + + def make_module_step(self, *args, **kwargs): + """ + Set module requirements from all components, e.g. $PATH, etc. + During the install step, we only set these requirements temporarily. + Later on when building the module, those paths are not considered. + Therefore, iterate through all the components again and gather + the requirements. + + Do not remove duplicates or check for existence of folders, + as this is done in the generic EasyBlock while creating + the module file already. + """ + for cfg, comp in self.comp_instances: + self.log.info("Gathering module paths for component %s v%s", cfg['name'], cfg['version']) + + # take into account that easyblock used for component may not be migrated yet to module_load_environment + if comp.make_module_req_guess.__qualname__ != 'EasyBlock.make_module_req_guess': + + depr_msg = f"Easyblock used to install component {cfg['name']} still uses make_module_req_guess" + self.log.deprecated(depr_msg, '6.0') + + reqs = comp.make_module_req_guess() + + # Try-except block to fail with an easily understandable error message. + # This should only trigger when an EasyBlock returns non-dict module requirements + # for make_module_req_guess() which should then be fixed in the components EasyBlock. + try: + for key, value in sorted(reqs.items()): + if key in self.module_load_environment: + getattr(self.module_load_environment, key).extend(value) else: - new_val = path - env.setvar(envvar, new_val) + setattr(self.module_load_environment, key, value) + except AttributeError: + raise EasyBuildError("Cannot process module requirements of bundle component %s v%s", + cfg['name'], cfg['version']) + else: + for env_var_name, env_var_val in comp.module_load_environment.items(): + if env_var_name in self.module_load_environment: + getattr(self.module_load_environment, env_var_name).extend(env_var_val) + else: + setattr(self.module_load_environment, env_var_name, env_var_val) - # close log for this component - comp.close_log() + return super().make_module_step(*args, **kwargs) def make_module_extra(self, *args, **kwargs): """Set extra stuff in module file, e.g. $EBROOT*, $EBVERSION*, etc.""" diff --git a/easybuild/easyblocks/generic/cargo.py b/easybuild/easyblocks/generic/cargo.py old mode 100644 new mode 100755 index d03407862db..10c6ac86859 --- a/easybuild/easyblocks/generic/cargo.py +++ b/easybuild/easyblocks/generic/cargo.py @@ -1,5 +1,6 @@ +#!/usr/bin/env python ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -27,20 +28,22 @@ @author: Mikael Oehman (Chalmers University of Technology) @author: Alex Domingo (Vrije Universiteit Brussel) +@author: Alexander Grund (TU Dresden) """ import os import re +from glob import glob import easybuild.tools.environment as env import easybuild.tools.systemtools as systemtools -from easybuild.tools.build_log import EasyBuildError, print_warning from easybuild.framework.easyconfig import CUSTOM from easybuild.framework.extensioneasyblock import ExtensionEasyBlock -from easybuild.tools.filetools import extract_file, change_dir -from easybuild.tools.run import run_cmd +from easybuild.tools.build_log import EasyBuildError, print_warning from easybuild.tools.config import build_option -from easybuild.tools.filetools import compute_checksum, mkdir, write_file +from easybuild.tools.filetools import CHECKSUM_TYPE_SHA256, compute_checksum, extract_file, mkdir, move_file +from easybuild.tools.filetools import read_file, write_file +from easybuild.tools.run import run_shell_cmd from easybuild.tools.toolchain.compiler import OPTARCH_GENERIC CRATESIO_SOURCE = "https://crates.io/api/v1/crates" @@ -54,14 +57,93 @@ """ -CONFIG_TOML_PATCH_GIT = """ -[patch."{repo}"] -{crates} +CONFIG_TOML_SOURCE_GIT = """ +[source."{url}?rev={rev}"] +git = "{url}" +rev = "{rev}" +replace-with = "vendored-sources" """ -CONFIG_TOML_PATCH_GIT_CRATES = """{0} = {{ path = "{1}" }} + +CONFIG_TOML_SOURCE_GIT_BRANCH = """ +[source."{url}?rev={rev}"] +git = "{url}" +rev = "{rev}" +branch = "{branch}" +replace-with = "vendored-sources" +""" + +CONFIG_TOML_SOURCE_GIT_WORKSPACE = """ +[source."real-{url}?rev={rev}"] +directory = "{workspace_dir}" + +[source."{url}?rev={rev}"] +git = "{url}" +rev = "{rev}" +replace-with = "real-{url}?rev={rev}" + """ -CARGO_CHECKSUM_JSON = '{{"files": {{}}, "package": "{chksum}"}}' +CARGO_CHECKSUM_JSON = '{{"files": {{}}, "package": "{checksum}"}}' + + +def get_workspace_members(crate_dir): + """Find all members of a cargo workspace in crate_dir. + + (Minimally) parse the Cargo.toml file. + If it is a workspace return all members (subfolder names). + Otherwise return None. + """ + cargo_toml = os.path.join(crate_dir, 'Cargo.toml') + + # We are looking for this: + # [workspace] + # members = [ + # "reqwest-middleware", + # "reqwest-tracing", + # "reqwest-retry", + # ] + + lines = [line.strip() for line in read_file(cargo_toml).splitlines()] + try: + start_idx = lines.index('[workspace]') + except ValueError: + return None + # Find "members = [" and concatenate the value, stop at end of section or file + member_str = None + for line in lines[start_idx + 1:]: + if line.startswith('#'): + continue # Skip comments + if re.match(r'\[\w+\]', line): + break + if member_str is None: + m = re.match(r'members\s+=\s+\[', line) + if m: + member_str = line[m.end():] + elif line.endswith(']'): + member_str += line[:-1].strip() + break + else: + member_str += line + # Split at commas after removing possibly trailing ones and remove the quotes + members = [member.strip().strip('"') for member in member_str.rstrip(',').split(',')] + # Sanity check that we didn't pick up anything unexpected + invalid_members = [member for member in members if not re.match(r'(\w|-)+', member)] + if invalid_members: + raise EasyBuildError('Failed to parse %s: Found seemingly invalid members: %s', + cargo_toml, ', '.join(invalid_members)) + return [os.path.join(crate_dir, m) for m in members] + + +def get_checksum(src, log): + """Get the checksum from an extracted source""" + checksum = src['checksum'] + if isinstance(checksum, dict): + try: + checksum = checksum[src['name']] + except KeyError: + log.warning('No checksum for %s in %s', checksum, src['name']) + checksum = None + return checksum class Cargo(ExtensionEasyBlock): @@ -81,24 +163,31 @@ def extra_options(extra_vars=None): return extra_vars @staticmethod - def crate_src_filename(pkg_name, pkg_version, *args): - """Crate tarball filename based on package name and version""" - return "{0}-{1}.tar.gz".format(pkg_name, pkg_version) + def crate_src_filename(pkg_name, pkg_version, _url=None, rev=None): + """Crate tarball filename based on package name, version and optionally git revision""" + filename = [pkg_name, pkg_version] + filename_ext = '.tar.gz' + + if rev is not None: + # sources from a git repo + filename.append(rev[:8]) # append short commit hash + filename_ext = '.tar.xz' # use a reproducible archive format + + return '-'.join(filename) + filename_ext @staticmethod - def crate_download_filename(pkg_name, pkg_version, *args): + def crate_download_filename(pkg_name, pkg_version): """Crate download filename based on package name and version""" - return "{0}/{1}/download".format(pkg_name, pkg_version) + return f"{pkg_name}/{pkg_version}/download" def rustc_optarch(self): """Determines what architecture to target. Translates GENERIC optarch, and respects rustc specific optarch. General optarchs are ignored as there is no direct translation. """ - if systemtools.X86_64 == systemtools.get_cpu_architecture(): + generic = '-C target-cpu=generic' + if systemtools.get_cpu_architecture() == systemtools.X86_64: generic = '-C target-cpu=x86-64' - else: - generic = '-C target-cpu=generic' optimal = '-C target-cpu=native' @@ -111,25 +200,19 @@ def rustc_optarch(self): return generic else: return '-' + rust_optarch - self.log.info("no rustc information in the optarch dict, so using %s" % optimal) + self.log.info(f"Given 'optarch' has no specific information on rustc, so using {optimal}") elif optarch == OPTARCH_GENERIC: return generic else: - self.log.warning("optarch is ignored as there is no translation for rustc, so using %s" % optimal) + self.log.warning(f"Ignoring 'optarch' because there is no translation for rustc, so using {optimal}") + return optimal def __init__(self, *args, **kwargs): """Constructor for Cargo easyblock.""" super(Cargo, self).__init__(*args, **kwargs) self.cargo_home = os.path.join(self.builddir, '.cargo') - self.vendor_dir = os.path.join(self.builddir, 'easybuild_vendor') - env.setvar('CARGO_HOME', self.cargo_home) - env.setvar('RUSTC', 'rustc') - env.setvar('RUSTDOC', 'rustdoc') - env.setvar('RUSTFMT', 'rustfmt') - env.setvar('RUSTFLAGS', self.rustc_optarch()) - env.setvar('RUST_LOG', 'DEBUG') - env.setvar('RUST_BACKTRACE', '1') + self.set_cargo_vars() # Populate sources from "crates" list of tuples sources = [] @@ -148,7 +231,7 @@ def __init__(self, *args, **kwargs): repo_name = repo_name[:-4] sources.append({ 'git_config': {'url': url, 'repo_name': repo_name, 'commit': rev}, - 'filename': self.crate_src_filename(crate, version), + 'filename': self.crate_src_filename(crate, version, rev=rev), }) # copy EasyConfig instance before we make changes to it @@ -156,79 +239,207 @@ def __init__(self, *args, **kwargs): self.cfg.update('sources', sources) + def set_cargo_vars(self): + """Set environment variables for Rust compilation and Cargo""" + env.setvar('CARGO_HOME', self.cargo_home) + env.setvar('RUSTC', 'rustc') + env.setvar('RUSTDOC', 'rustdoc') + env.setvar('RUSTFMT', 'rustfmt') + env.setvar('RUSTFLAGS', self.rustc_optarch()) + env.setvar('RUST_LOG', 'DEBUG') + env.setvar('RUST_BACKTRACE', '1') + @property def crates(self): """Return the crates as defined in the EasyConfig""" return self.cfg['crates'] + def load_module(self, *args, **kwargs): + """(Re)set environment variables after loading module file. + + Required here to ensure the variables are defined for stand-alone installations and extensions, + because the environment is reset to the initial environment right before loading the module. + """ + super(Cargo, self).load_module(*args, **kwargs) + self.set_cargo_vars() + def extract_step(self): """ Unpack the source files and populate them with required .cargo-checksum.json if offline """ + self.vendor_dir = os.path.join(self.builddir, 'easybuild_vendor') mkdir(self.vendor_dir) + # Sources from git repositories might contain multiple crates/folders in a so-called "workspace". + # If we put such a workspace into the vendor folder, cargo fails with + # "found a virtual manifest at [...]Cargo.toml instead of a package manifest". + # Hence we put those in a separate folder and only move "regular" crates into the vendor folder. + self.git_vendor_dir = os.path.join(self.builddir, 'easybuild_vendor_git') + mkdir(self.git_vendor_dir) vendor_crates = {self.crate_src_filename(*crate): crate for crate in self.crates} - git_sources = {crate[2]: [] for crate in self.crates if len(crate) == 4} + # Track git sources for building the cargo config and avoiding duplicated folders + git_sources = {} for src in self.src: - extraction_dir = self.builddir + # Check if the source is a vendored crate + is_vendor_crate = src['name'] in vendor_crates + if is_vendor_crate: + # Store crate for later + src['crate'] = vendor_crates[src['name']] + crate_name = src['crate'][0] + + # Check for git crates, `git_key` will be set to a true-ish value for those + if not is_vendor_crate or len(src['crate']) == 2: + git_key = None + else: + git_key = src['crate'][2:] + git_repo, rev = git_key + self.log.debug("Sources of %s(%s) belong to git repo: %s rev %s", + crate_name, src['name'], git_repo, rev) + # Do a sanity check that sources for the same repo and revision are the same + try: + previous_source = git_sources[git_key] + except KeyError: + git_sources[git_key] = src + else: + previous_checksum = get_checksum(previous_source, self.log) + current_checksum = get_checksum(src, self.log) + if previous_checksum and current_checksum and previous_checksum != current_checksum: + raise EasyBuildError("Sources for the same git repository need to be identical. " + "Mismatch found for %s rev %s in %s (checksum: %s) vs %s (checksum: %s)", + git_repo, rev, previous_source['name'], previous_checksum, + src['name'], current_checksum) + self.log.info("Source %s already extracted to %s by %s. Skipping extraction.", + src['name'], previous_source['finalpath'], previous_source['name']) + src['finalpath'] = previous_source['finalpath'] + continue + # Extract dependency crates into vendor subdirectory, separate from sources of main package - if src['name'] in vendor_crates: - extraction_dir = self.vendor_dir + extraction_dir = self.builddir + if is_vendor_crate: + extraction_dir = self.git_vendor_dir if git_key else self.vendor_dir self.log.info("Unpacking source of %s", src['name']) existing_dirs = set(os.listdir(extraction_dir)) - crate_dir = None src_dir = extract_file(src['path'], extraction_dir, cmd=src['cmd'], extra_options=self.cfg['unpack_options'], change_into_dir=False) new_extracted_dirs = set(os.listdir(extraction_dir)) - existing_dirs - if len(new_extracted_dirs) == 1: - # Expected crate tarball with 1 folder - crate_dir = new_extracted_dirs.pop() - src_dir = os.path.join(extraction_dir, crate_dir) - self.log.debug("Unpacked sources of %s into: %s", src['name'], src_dir) - elif len(new_extracted_dirs) == 0: + if len(new_extracted_dirs) == 0: # Extraction went wrong raise EasyBuildError("Unpacking sources of '%s' failed", src['name']) + # Expected crate tarball with 1 folder # TODO: properly handle case with multiple extracted folders # this is currently in a grey area, might still be used by cargo + if len(new_extracted_dirs) == 1: + src_dir = os.path.join(extraction_dir, new_extracted_dirs.pop()) + self.log.debug("Unpacked sources of %s into: %s", src['name'], src_dir) - change_dir(src_dir) - self.src[self.src.index(src)]['finalpath'] = src_dir - - if self.cfg['offline'] and crate_dir: - # Create checksum file for extracted sources required by vendored crates.io sources - self.log.info('creating .cargo-checksums.json file for : %s', crate_dir) - chksum = compute_checksum(src['path'], checksum_type='sha256') - chkfile = os.path.join(extraction_dir, crate_dir, '.cargo-checksum.json') - write_file(chkfile, CARGO_CHECKSUM_JSON.format(chksum=chksum)) - # Add path to extracted sources for any crate from a git repo - try: - crate_name, _, crate_repo, _ = vendor_crates[src['name']] - except (ValueError, KeyError): - pass - else: - self.log.debug("Sources of %s belong to git repo: %s", src['name'], crate_repo) - git_src_dir = (crate_name, src_dir) - git_sources[crate_repo].append(git_src_dir) - - if self.cfg['offline']: - self.log.info("Setting vendored crates dir for offline operation") - config_toml = os.path.join(self.cargo_home, 'config.toml') - # Replace crates-io with vendored sources using build dir wide toml file in CARGO_HOME - # because the rust source subdirectories might differ with python packages - self.log.debug("Writting config.toml entry for vendored crates from crate.io") - write_file(config_toml, CONFIG_TOML_SOURCE_VENDOR.format(vendor_dir=self.vendor_dir), append=True) - - # also vendor sources from other git sources (could be many crates for one git source) - for git_repo, repo_crates in git_sources.items(): - self.log.debug("Writting config.toml entry for git repo: %s", git_repo) - config_crates = ''.join([CONFIG_TOML_PATCH_GIT_CRATES.format(*crate) for crate in repo_crates]) - write_file(config_toml, CONFIG_TOML_PATCH_GIT.format(repo=git_repo, crates=config_crates), append=True) - - # Use environment variable since it would also be passed along to builds triggered via python packages - env.setvar('CARGO_NET_OFFLINE', 'true') + if is_vendor_crate and self.cfg['offline']: + # Create checksum file for extracted sources required by vendored crates + + # By default there is only a single crate + crate_dirs = [src_dir] + # For git sources determine the folders that contain crates by taking workspaces into account + if git_key: + member_dirs = get_workspace_members(src_dir) + if member_dirs: + crate_dirs = member_dirs + + try: + checksum = src[CHECKSUM_TYPE_SHA256] + except KeyError: + checksum = compute_checksum(src['path'], checksum_type=CHECKSUM_TYPE_SHA256) + for crate_dir in crate_dirs: + self.log.info('creating .cargo-checksums.json file for %s', os.path.basename(crate_dir)) + chkfile = os.path.join(src_dir, crate_dir, '.cargo-checksum.json') + write_file(chkfile, CARGO_CHECKSUM_JSON.format(checksum=checksum)) + # Move non-workspace git crates to the vendor folder + if git_key and member_dirs is None: + src_dir = os.path.join(self.vendor_dir, os.path.basename(crate_dirs[0])) + self.log.debug('Moving crate %s without workspaces to vendor folder', crate_name) + move_file(crate_dirs[0], src_dir) + + src['finalpath'] = src_dir + + self._setup_offline_config(git_sources) + + def _setup_offline_config(self, git_sources): + """ + Setup the configuration required for offline builds + :param git_sources: dict mapping (git_repo, rev) to extracted source + """ + if not self.cfg['offline']: + return + self.log.info("Setting up vendored crates for offline operation") + self.log.debug("Writting config.toml entry for vendored crates from crate.io") + config_toml = os.path.join(self.cargo_home, 'config.toml') + # Replace crates-io with vendored sources using build dir wide toml file in CARGO_HOME + write_file(config_toml, CONFIG_TOML_SOURCE_VENDOR.format(vendor_dir=self.vendor_dir)) + + # Tell cargo about the vendored git sources to avoid it failing with: + # Unable to update https://github.com/[...] + # can't checkout from 'https://github.com/[...]]': you are in the offline mode (--offline) + + for (git_repo, rev), src in git_sources.items(): + crate_name = src['crate'][0] + src_dir = src['finalpath'] + if os.path.dirname(src_dir) == self.vendor_dir: + # Non-workspace sources are in vendor_dir + git_branch = self._get_crate_git_repo_branch(crate_name) + template = CONFIG_TOML_SOURCE_GIT_BRANCH if git_branch else CONFIG_TOML_SOURCE_GIT + self.log.debug(f"Writing config.toml entry for git repo: {git_repo} branch {git_branch}, rev {rev}") + write_file( + config_toml, + template.format(url=git_repo, rev=rev, branch=git_branch), + append=True + ) + else: + self.log.debug("Writing config.toml entry for git repo: %s rev %s", git_repo, rev) + # Workspace sources stay in their own separate folder. + # We cannot have a `directory = ""` entry where a folder containing a workspace is inside + write_file( + config_toml, + CONFIG_TOML_SOURCE_GIT_WORKSPACE.format(url=git_repo, rev=rev, workspace_dir=src_dir), + append=True + ) + + # Use environment variable since it would also be passed along to builds triggered via python packages + env.setvar('CARGO_NET_OFFLINE', 'true') + + def _get_crate_git_repo_branch(self, crate_name): + """ + Find the dependency definition for given crate in all Cargo.toml files of sources + Return branch target for given crate_name if any + """ + # Search all Cargo.toml files in main source and vendored crates + cargo_toml_files = [] + for cargo_source_dir in (self.src[0]['finalpath'], self.vendor_dir, self.git_vendor_dir): + cargo_toml_files.extend(glob(os.path.join(cargo_source_dir, '**', 'Cargo.toml'), recursive=True)) + + if not cargo_toml_files: + raise EasyBuildError("Cargo.toml file not found in sources") + + self.log.debug( + f"Searching definition of crate '{crate_name}' in the following files: {', '.join(cargo_toml_files)}" + ) + + git_repo_spec = re.compile(re.escape(crate_name) + r"\s*=\s*{([^}]*)}", re.M) + git_branch_spec = re.compile(r'branch\s*=\s*"([^"]*)"', re.M) + + for cargo_toml in cargo_toml_files: + git_repo_crate = git_repo_spec.search(read_file(cargo_toml)) + if git_repo_crate: + self.log.debug(f"Found specification in {cargo_toml} for crate '{crate_name}': " + + git_repo_crate.group()) + git_repo_crate_contents = git_repo_crate.group(1) + git_branch_crate = git_branch_spec.search(git_repo_crate_contents) + if git_branch_crate: + self.log.debug(f"Found git branch requirement for crate '{crate_name}': " + + git_branch_crate.group()) + return git_branch_crate.group(1) + + return None def configure_step(self): """Empty configuration step.""" @@ -241,8 +452,8 @@ def profile(self): def build_step(self): """Build with cargo""" parallel = '' - if self.cfg['parallel']: - parallel = "-j %s" % self.cfg['parallel'] + if self.cfg.parallel > 1: + parallel = f"-j {self.cfg.parallel}" tests = '' if self.cfg['enable_tests']: @@ -250,9 +461,9 @@ def build_step(self): lto = '' if self.cfg['lto'] is not None: - lto = '--config profile.%s.lto=\\"%s\\"' % (self.profile, self.cfg['lto']) + lto = f'--config profile.{self.profile}.lto="{self.cfg["lto"]}"' - run_cmd('rustc --print cfg', log_all=True, simple=True) # for tracking in log file + run_shell_cmd('rustc --print cfg') # for tracking in log file cmd = ' '.join([ self.cfg['prebuildopts'], 'cargo build', @@ -262,7 +473,7 @@ def build_step(self): parallel, self.cfg['buildopts'], ]) - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) def test_step(self): """Test with cargo""" @@ -273,7 +484,7 @@ def test_step(self): '--profile=' + self.profile, self.cfg['testopts'], ]) - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) def install_step(self): """Install with cargo""" @@ -285,58 +496,76 @@ def install_step(self): '--path=.', self.cfg['installopts'], ]) - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) def generate_crate_list(sourcedir): """Helper for generating crate list""" - import toml + from urllib.parse import parse_qs, urlsplit # pylint: disable=import-outside-toplevel + + import toml # pylint: disable=import-outside-toplevel cargo_toml = toml.load(os.path.join(sourcedir, 'Cargo.toml')) - cargo_lock = toml.load(os.path.join(sourcedir, 'Cargo.lock')) + + try: + cargo_lock = toml.load(os.path.join(sourcedir, 'Cargo.lock')) + except FileNotFoundError as err: + print("\nNo Cargo.lock file found. Generate one with 'cargo generate-lockfile'\n") + raise err try: app_name = cargo_toml['package']['name'] except KeyError: app_name = os.path.basename(os.path.abspath(sourcedir)) print_warning('Did not find a [package] name= entry. Assuming it is the folder name: ' + app_name) - deps = cargo_lock['package'] app_in_cratesio = False crates = [] other_crates = [] - for dep in deps: + for dep in cargo_lock['package']: name = dep['name'] version = dep['version'] - if 'source' in dep: - if name == app_name: - app_in_cratesio = True # exclude app itself, needs to be first in crates list or taken from pypi - else: - if dep['source'] == 'registry+https://github.com/rust-lang/crates.io-index': - crates.append((name, version)) - else: - # Lock file has #revision in the url - url, rev = dep['source'].rsplit('#', maxsplit=1) - for prefix in ('registry+', 'git+'): - if url.startswith(prefix): - url = url[len(prefix):] - # Remove branch name and revision URL parameters if present - url = re.sub(r'\?branch=\w+$', '', url) - url = re.sub(r'\?rev=%s+$' % rev, '', url) - crates.append((name, version, url, rev)) - else: + try: + source_url = dep['source'] + except KeyError: other_crates.append((name, version)) + continue + if name == app_name: + app_in_cratesio = True # exclude app itself, needs to be first in crates list or taken from pypi + else: + if source_url == 'registry+https://github.com/rust-lang/crates.io-index': + crates.append((name, version)) + else: + # Lock file has revision and branch in the url + url = re.sub(r'^(registry|git)\+', '', source_url) # Strip prefix if present + parsed_url = urlsplit(url) + url = re.split('[#?]', url, maxsplit=1)[0] # Remove query and fragment + rev = parsed_url.fragment + if not rev: + raise ValueError(f"Revision not found in URL {url}") + qs = parse_qs(parsed_url.query) + rev_qs = qs.get('rev', [None])[0] + if rev_qs is not None and rev_qs != rev: + raise ValueError(f"Found different revision in query of URL {url}: {rev_qs} (expected: {rev})") + crates.append((name, version, url, rev)) return app_in_cratesio, crates, other_crates -if __name__ == '__main__': - import sys +def main(): + import sys # pylint: disable=import-outside-toplevel + if len(sys.argv) != 2: + print('Expected path to folder containing Cargo.[toml,lock]') + sys.exit(1) app_in_cratesio, crates, other = generate_crate_list(sys.argv[1]) - print(other) + print('Other crates (no source in Cargo.lock):', other) if app_in_cratesio or crates: print('crates = [') if app_in_cratesio: print(' (name, version),') - for crate_info in crates: + for crate_info in sorted(crates): print(" %s," % str(crate_info)) print(']') + + +if __name__ == '__main__': + main() diff --git a/easybuild/easyblocks/generic/cargopythonbundle.py b/easybuild/easyblocks/generic/cargopythonbundle.py index d76ccde6129..893c03a8959 100644 --- a/easybuild/easyblocks/generic/cargopythonbundle.py +++ b/easybuild/easyblocks/generic/cargopythonbundle.py @@ -1,5 +1,5 @@ ## -# Copyright 2018-2024 Ghent University +# Copyright 2018-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/generic/cargopythonpackage.py b/easybuild/easyblocks/generic/cargopythonpackage.py index fa85cad411c..5a073333384 100644 --- a/easybuild/easyblocks/generic/cargopythonpackage.py +++ b/easybuild/easyblocks/generic/cargopythonpackage.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/generic/cmakemake.py b/easybuild/easyblocks/generic/cmakemake.py index eb0d5820186..a83296b6ab5 100644 --- a/easybuild/easyblocks/generic/cmakemake.py +++ b/easybuild/easyblocks/generic/cmakemake.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -42,12 +42,13 @@ from easybuild.framework.easyconfig import BUILD, CUSTOM from easybuild.tools.build_log import EasyBuildError, print_warning from easybuild.tools.config import build_option -from easybuild.tools.filetools import change_dir, create_unused_dir, mkdir, which +from easybuild.tools.filetools import change_dir, create_unused_dir, mkdir, read_file, which from easybuild.tools.environment import setvar from easybuild.tools.modules import get_software_root, get_software_version -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd from easybuild.tools.systemtools import get_shared_lib_ext from easybuild.tools.utilities import nub +import easybuild.tools.toolchain as toolchain DEFAULT_CONFIGURE_CMD = 'cmake' @@ -63,7 +64,8 @@ def det_cmake_version(): regex = re.compile(r"^[cC][mM]ake version (?P[0-9]\.[0-9a-zA-Z.-]+)$", re.M) cmd = "cmake --version" - (out, _) = run_cmd(cmd, simple=False, log_ok=False, log_all=False, trace=False) + cmd_res = run_shell_cmd(cmd, hidden=True, fail_on_error=False) + out = cmd_res.output res = regex.search(out) if res: cmake_version = res.group('version') @@ -87,6 +89,52 @@ def setup_cmake_env(tc): setvar("CMAKE_LIBRARY_PATH", library_paths) +def setup_cmake_env_python_hints(cmake_version=None): + """Set environment variables as hints for CMake to prefer the Python module, if loaded. + Useful when there is no way to specify arguments for CMake directly, + e.g. when CMake is called from within another build system. + Otherwise get_cmake_python_config_[str/dict] should be used instead. + """ + if cmake_version is None: + cmake_version = det_cmake_version() + if LooseVersion(cmake_version) < '3.12': + raise EasyBuildError("Setting Python hints for CMake requires CMake version 3.12 or newer") + python_root = get_software_root('Python') + if python_root: + python_version = LooseVersion(get_software_version('Python')) + setvar('Python_ROOT_DIR', python_root) + if python_version >= "3": + setvar('Python3_ROOT_DIR', python_root) + else: + setvar('Python2_ROOT_DIR', python_root) + + +def get_cmake_python_config_dict(): + """Get a dictionary with CMake configuration options for finding Python if loaded as a module.""" + options = {} + python_root = get_software_root('Python') + if python_root: + python_version = LooseVersion(get_software_version('Python')) + python_exe = os.path.join(python_root, 'bin', 'python') + # This is required for (deprecated) `find_package(PythonInterp ...)` + options['PYTHON_EXECUTABLE'] = python_exe + # Ensure that both `find_package(Python) and find_package(Python2/3)` work as intended + options['Python_EXECUTABLE'] = python_exe + if python_version >= "3": + options['Python3_EXECUTABLE'] = python_exe + else: + options['Python2_EXECUTABLE'] = python_exe + return options + + +def get_cmake_python_config_str(): + """Get CMake configuration arguments for finding Python if loaded as a module. + This string is intended to be passed to the invocation of `cmake`. + """ + options = get_cmake_python_config_dict() + return ' '.join('-D%s=%s' % (key, value) for key, value in options.items()) + + class CMakeMake(ConfigureMake): """Support for configuring build with CMake instead of traditional configure script""" @@ -101,10 +149,12 @@ def extra_options(extra_vars=None): 'build_shared_libs': [None, "Build shared library (instead of static library)" "None can be used to add no flag (usually results in static library)", CUSTOM], 'build_type': [None, "Build type for CMake, e.g. Release." - "Defaults to 'Release' or 'Debug' depending on toolchainopts[debug]", CUSTOM], + "Defaults to 'Release', 'RelWithDebInfo' or 'Debug' depending on " + "toolchainopts[debug,noopt]", CUSTOM], 'configure_cmd': [DEFAULT_CONFIGURE_CMD, "Configure command to use", CUSTOM], 'generator': [None, "Build file generator to use. None to use CMakes default", CUSTOM], 'install_target_subdir': [None, "Subdirectory to use as installation target", CUSTOM], + 'install_libdir': ['lib', "Subdirectory to use for library installation files", CUSTOM], 'runtest': [None, "Make target to test build or True to use CTest", BUILD], 'srcdir': [None, "Source directory location to provide to cmake command", CUSTOM], 'separate_build_dir': [True, "Perform build in a separate directory", CUSTOM], @@ -146,7 +196,12 @@ def build_type(self): """Build type set in the EasyConfig with default determined by toolchainopts""" build_type = self.cfg.get('build_type') if build_type is None: - build_type = 'Debug' if self.toolchain.options.get('debug', None) else 'Release' + if self.toolchain.options.get('noopt', None): # also implies debug but is the closest match + build_type = 'Debug' + elif self.toolchain.options.get('debug', None): + build_type = 'RelWithDebInfo' + else: + build_type = 'Release' return build_type def prepend_config_opts(self, config_opts): @@ -200,12 +255,24 @@ def configure_step(self, srcdir=None, builddir=None): if '-DCMAKE_BUILD_TYPE=' in self.cfg['configopts']: if self.cfg.get('build_type') is not None: - self.log.warning('CMAKE_BUILD_TYPE is set in configopts. Ignoring build_type') + self.log.info("CMAKE_BUILD_TYPE is set in configopts. Ignoring 'build_type' easyconfig parameter.") else: options['CMAKE_BUILD_TYPE'] = self.build_type + # Set installation directory for libraries + # any CMAKE_INSTALL_DIR[:PATH] setting defined in 'configopts' has precedence over 'install_libdir' + if self.cfg['install_libdir'] is not None: + cmake_install_dir_pattern = re.compile(r"-DCMAKE_INSTALL_LIBDIR(:PATH)?=[^\s]") + if cmake_install_dir_pattern.search(self.cfg['configopts']): + self.log.info( + "CMAKE_INSTALL_LIBDIR is set in configopts. Ignoring 'install_libdir' easyconfig parameter." + ) + else: + # set CMAKE_INSTALL_LIBDIR including its type to PATH, otherwise CMake can silently ignore it + options['CMAKE_INSTALL_LIBDIR:PATH'] = self.cfg['install_libdir'] + # Add -fPIC flag if necessary - if self.toolchain.options['pic']: + if self.toolchain.options.get('pic', False): options['CMAKE_POSITION_INDEPENDENT_CODE'] = 'ON' if self.cfg['generator']: @@ -231,7 +298,7 @@ def configure_step(self, srcdir=None, builddir=None): # Usually you want to remove -DBUILD_SHARED_LIBS from configopts and set build_shared_libs to True or False # If you need it in configopts don't set build_shared_libs (or explicitely set it to `None` (Default)) if '-DBUILD_SHARED_LIBS=' in self.cfg['configopts']: - print_warning('Ignoring BUILD_SHARED_LIBS is set in configopts because build_shared_libs is set') + print_warning('Ignoring BUILD_SHARED_LIBS setting in configopts because build_shared_libs is set') self.cfg.update('configopts', '-DBUILD_SHARED_LIBS=%s' % ('ON' if build_shared_libs else 'OFF')) # If the cache does not exist CMake reads the environment variables @@ -273,17 +340,28 @@ def configure_step(self, srcdir=None, builddir=None): # see https://github.com/Kitware/CMake/commit/3ec9226779776811240bde88a3f173c29aa935b5 options['CMAKE_SKIP_RPATH'] = 'ON' - # make sure that newer CMAKE picks python based on location, not just the newest python - # Avoids issues like e.g. https://github.com/EESSI/software-layer/pull/370#issuecomment-1785594932 - if LooseVersion(self.cmake_version) >= '3.15': - options['CMAKE_POLICY_DEFAULT_CMP0094'] = 'NEW' - # show what CMake is doing by default options['CMAKE_VERBOSE_MAKEFILE'] = 'ON' # disable CMake user package repository options['CMAKE_FIND_USE_PACKAGE_REGISTRY'] = 'OFF' + # ensure CMake uses EB python, not system or virtualenv python + options.update(get_cmake_python_config_dict()) + + # pass the preferred host compiler, CUDA compiler, and CUDA architectures to the CUDA compiler + cuda_root = get_software_root('CUDA') + if cuda_root: + options['CMAKE_CUDA_HOST_COMPILER'] = which(os.getenv('CXX', 'g++')) + options['CMAKE_CUDA_COMPILER'] = which('nvcc') + cuda_cc = build_option('cuda_compute_capabilities') or self.cfg['cuda_compute_capabilities'] + if cuda_cc: + options['CMAKE_CUDA_ARCHITECTURES'] = '"%s"' % ';'.join([cc.replace('.', '') for cc in cuda_cc]) + else: + raise EasyBuildError('List of CUDA compute capabilities must be specified, either via ' + 'cuda_compute_capabilities easyconfig parameter or via ' + '--cuda-compute-capabilities') + if not self.cfg.get('allow_system_boost', False): boost_root = get_software_root('Boost') if boost_root: @@ -302,6 +380,8 @@ def configure_step(self, srcdir=None, builddir=None): options['BOOST_ROOT'] = boost_root options['Boost_NO_SYSTEM_PATHS'] = 'ON' + self.cmake_options = options + if self.cfg.get('configure_cmd') == DEFAULT_CONFIGURE_CMD: self.prepend_config_opts(options) command = ' '.join([ @@ -316,9 +396,63 @@ def configure_step(self, srcdir=None, builddir=None): self.cfg.get('configure_cmd'), self.cfg['configopts']]) - (out, _) = run_cmd(command, log_all=True, simple=False) - - return out + res = run_shell_cmd(command) + self.check_python_paths() + return res.output + + def check_python_paths(self): + """Check that there are no detected Python paths outside the Python dependency provided by EasyBuild""" + if not os.path.exists('CMakeCache.txt'): + self.log.warning("CMakeCache.txt not found. Python paths checks skipped.") + return + cmake_cache = read_file('CMakeCache.txt') + if not cmake_cache: + self.log.warning("CMake Cache could not be read. Python paths checks skipped.") + return + + self.log.info("Checking Python paths") + + python_paths = { + "executable": [], + "include_dir": [], + "library": [], + } + python_regex = re.compile(r"_?(Python|PYTHON)\d?_" + r"(?PEXECUTABLE|INCLUDE_DIR|LIBRARY)\w*(:\w+)?\s*=(?P.*)") + cmake_false_expressions = {'', '0', 'OFF', 'NO', 'FALSE', 'N', 'IGNORE', 'NOTFOUND'} + for line in cmake_cache.splitlines(): + match = python_regex.match(line) + if match: + self.log.debug("Python related CMake cache line found: " + line) + path_type = match['type'].lower() + path = match['value'].strip() + if path.endswith('-NOTFOUND') or path.upper() in cmake_false_expressions: + continue + self.log.info("Python %s path: %s", path_type, path) + python_paths[path_type].append(path) + + ebrootpython_path = get_software_root("Python") + if not ebrootpython_path: + if any(python_paths.values()) and not self.toolchain.comp_family() == toolchain.SYSTEM: + self.log.warning("Found Python paths in CMake cache but Python is not a dependency") + # Can't do the check + return + ebrootpython_path = os.path.realpath(ebrootpython_path) + + errors = [] + for path_type, paths in python_paths.items(): + for path in paths: + if not os.path.exists(path): + errors.append("Python %s path does not exist: %s" % (path_type, path)) + elif not os.path.realpath(path).startswith(ebrootpython_path): + errors.append("Python %s path '%s' is outside EBROOTPYTHON (%s)" % + (path_type, path, ebrootpython_path)) + + if errors: + # Combine all errors into a single message + error_message = "\n".join(errors) + raise EasyBuildError("Python path errors:\n" + error_message) + self.log.info("Python check successful") def test_step(self): """CMake specific test setup""" diff --git a/easybuild/easyblocks/generic/cmakemakecp.py b/easybuild/easyblocks/generic/cmakemakecp.py index 51a9cf3d29f..bfaf2d4b76a 100644 --- a/easybuild/easyblocks/generic/cmakemakecp.py +++ b/easybuild/easyblocks/generic/cmakemakecp.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/generic/cmakeninja.py b/easybuild/easyblocks/generic/cmakeninja.py index f9c4d7556ba..c0decbb684a 100644 --- a/easybuild/easyblocks/generic/cmakeninja.py +++ b/easybuild/easyblocks/generic/cmakeninja.py @@ -1,5 +1,5 @@ ## -# Copyright 2019-2024 Ghent University +# Copyright 2019-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/generic/cmakepythonpackage.py b/easybuild/easyblocks/generic/cmakepythonpackage.py index 5a4bc30f537..67a112a9521 100644 --- a/easybuild/easyblocks/generic/cmakepythonpackage.py +++ b/easybuild/easyblocks/generic/cmakepythonpackage.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/generic/cmdcp.py b/easybuild/easyblocks/generic/cmdcp.py index ee240551696..22ff6dbf8d9 100644 --- a/easybuild/easyblocks/generic/cmdcp.py +++ b/easybuild/easyblocks/generic/cmdcp.py @@ -1,5 +1,5 @@ ## -# Copyright 2014-2024 Ghent University +# Copyright 2014-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -32,7 +32,7 @@ from easybuild.easyblocks.generic.makecp import MakeCp from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd class CmdCp(MakeCp): @@ -80,4 +80,4 @@ def build_step(self): raise EasyBuildError("No match for %s in %s, don't know which command to use.", src, self.cfg['cmds_map']) - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) diff --git a/easybuild/easyblocks/generic/conda.py b/easybuild/easyblocks/generic/conda.py index 5fce4607e6f..d4d0362f3f1 100644 --- a/easybuild/easyblocks/generic/conda.py +++ b/easybuild/easyblocks/generic/conda.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -28,13 +28,10 @@ @author: Jillian Rowe (New York University Abu Dhabi) @author: Kenneth Hoste (HPC-UGent) """ - -import os - from easybuild.easyblocks.generic.binary import Binary from easybuild.framework.easyconfig import CUSTOM -from easybuild.tools.run import run_cmd -from easybuild.tools.modules import get_software_root +from easybuild.tools.run import run_shell_cmd +from easybuild.tools.modules import MODULE_LOAD_ENV_HEADERS, get_software_root from easybuild.tools.build_log import EasyBuildError @@ -53,6 +50,21 @@ def extra_options(extra_vars=None): }) return extra_vars + def __init__(self, *args, **kwargs): + """Initialize class variables.""" + super().__init__(*args, **kwargs) + + # Do not add installation to search paths for headers or libraries to avoid + # that the conda environment is used by other software at building or linking time. + # LD_LIBRARY_PATH issue discusses here: + # http://superuser.com/questions/980250/environment-module-cannot-initialize-tcl + mod_env_headers = self.module_load_environment.alias_vars(MODULE_LOAD_ENV_HEADERS) + mod_env_libs = ['LD_LIBRARY_PATH', 'LIBRARY_PATH'] + mod_env_cmake = ['CMAKE_LIBRARY_PATH', 'CMAKE_PREFIX_PATH'] + for disallowed_var in mod_env_headers + mod_env_libs + mod_env_cmake: + self.module_load_environment.remove(disallowed_var) + self.log.debug(f"Purposely not updating ${disallowed_var} in {self.name} module file") + def extract_step(self): """Copy sources via extract_step of parent, if any are specified.""" if self.src: @@ -74,8 +86,8 @@ def install_step(self): # initialize conda environment # setuptools is just a choice, but *something* needs to be there - cmd = "%s config --add create_default_packages setuptools" % conda_cmd - run_cmd(cmd, log_all=True, simple=True) + cmd = f"{conda_cmd} config --add create_default_packages setuptools" + run_shell_cmd(cmd) if self.cfg['environment_file'] or self.cfg['remote_environment']: @@ -85,27 +97,27 @@ def install_step(self): env_spec = self.cfg['remote_environment'] # use --force to ignore existing installation directory - cmd = "%s %s env create --force %s -p %s" % (self.cfg['preinstallopts'], conda_cmd, - env_spec, self.installdir) - run_cmd(cmd, log_all=True, simple=True) + cmd = f"{self.cfg['preinstallopts']} {conda_cmd} env create " + cmd += f"--force {env_spec} -p {self.installdir}" + run_shell_cmd(cmd) else: if self.cfg['requirements']: - install_args = "-y %s " % self.cfg['requirements'] + install_args = f"-y {self.cfg['requirements']} " if self.cfg['channels']: install_args += ' '.join('-c ' + chan for chan in self.cfg['channels']) self.log.info("Installed conda requirements") - cmd = "%s %s create --force -y -p %s %s" % (self.cfg['preinstallopts'], conda_cmd, - self.installdir, install_args) - run_cmd(cmd, log_all=True, simple=True) + cmd = f"{self.cfg['preinstallopts']} {conda_cmd} create " + cmd += f"--force -y -p {self.installdir} {install_args}" + run_shell_cmd(cmd) # clean up - cmd = "%s clean -ya" % conda_cmd - run_cmd(cmd, log_all=True, simple=True) + cmd = f"{conda_cmd} clean -ya" + run_shell_cmd(cmd) def make_module_extra(self): """Add the install directory to the PATH.""" @@ -115,15 +127,3 @@ def make_module_extra(self): txt += self.module_generator.set_environment('CONDA_DEFAULT_ENV', self.installdir) self.log.debug("make_module_extra added this: %s", txt) return txt - - def make_module_req_guess(self): - """ - A dictionary of possible directories to look for. - """ - # LD_LIBRARY_PATH issue discusses here - # http://superuser.com/questions/980250/environment-module-cannot-initialize-tcl - return { - 'PATH': ['bin', 'sbin'], - 'MANPATH': ['man', os.path.join('share', 'man')], - 'PKG_CONFIG_PATH': [os.path.join(x, 'pkgconfig') for x in ['lib', 'lib32', 'lib64', 'share']], - } diff --git a/easybuild/easyblocks/generic/configuremake.py b/easybuild/easyblocks/generic/configuremake.py index 8ad952db5ec..408595ac467 100644 --- a/easybuild/easyblocks/generic/configuremake.py +++ b/easybuild/easyblocks/generic/configuremake.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -34,6 +34,7 @@ @author: Toon Willems (Ghent University) @author: Maxime Boissonneault (Compute Canada - Universite Laval) @author: Alan O'Cais (Juelich Supercomputing Centre) +@author: Sebastian Achilles (Juelich Supercomputing Centre) """ import os import re @@ -44,12 +45,12 @@ from easybuild.easyblocks import VERSION as EASYBLOCKS_VERSION from easybuild.framework.easyblock import EasyBlock from easybuild.framework.easyconfig import CUSTOM -from easybuild.tools.build_log import print_warning -from easybuild.tools.config import source_paths, build_option +from easybuild.tools.build_log import print_warning, EasyBuildError +from easybuild.tools.config import source_paths, build_option, ERROR, IGNORE, WARN from easybuild.tools.filetools import CHECKSUM_TYPE_SHA256, adjust_permissions, compute_checksum, download_file from easybuild.tools.filetools import read_file, remove_file -from easybuild.tools.py2vs3 import string_type -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd +from easybuild.tools.utilities import nub # string that indicates that a configure script was generated by Autoconf # note: bytes string since this constant is used to check the contents of 'configure' which is read as bytes @@ -57,10 +58,10 @@ AUTOCONF_GENERATED_MSG = b"Generated by GNU Autoconf" # download location & SHA256 for config.guess script -CONFIG_GUESS_VERSION = '2018-08-29' -CONFIG_GUESS_URL_STUB = "https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=" -CONFIG_GUESS_COMMIT_ID = "59e2ce0e6b46bb47ef81b68b600ed087e14fdaad" -CONFIG_GUESS_SHA256 = "c02eb9cc55c86cfd1e9a794e548d25db5c9539e7b2154beb649bc6e2cbffc74c" +CONFIG_GUESS_VERSION = '2023-08-22' +CONFIG_GUESS_URL_STUB = "https://git.savannah.gnu.org/cgit/config.git/plain/config.guess?id=" +CONFIG_GUESS_COMMIT_ID = "28ea239c53a2d5d8800c472bc2452eaa16e37af2" +CONFIG_GUESS_SHA256 = "ec2577614252326f889df1de97b9a457c03a9a94811048563c211a44496d8ba3" DEFAULT_BUILD_CMD = 'make' @@ -195,6 +196,11 @@ def extra_options(extra_vars=None): 'tar_config_opts': [False, "Override tar settings as determined by configure.", CUSTOM], 'test_cmd': [None, "Test command to use ('runtest' value is appended, default: '%s')" % DEFAULT_TEST_CMD, CUSTOM], + 'unrecognized_configure_options': [ERROR, + "Action to do when unrecognized options passed to ./configure are" + " detected, defaults to aborting the build. Can be set to '" + WARN + + "' or '" + IGNORE + "' (NOT RECOMMENDED! It might hide actual errors" + " e.g. misspelling of intended or changed options)", CUSTOM], }) return extra_vars @@ -204,6 +210,11 @@ def __init__(self, *args, **kwargs): self.config_guess = None + @property + def parallel_flag(self): + """Return the flag to enable parallelism or empty for serial""" + return f'-j {self.cfg.parallel}' if self.cfg.parallel > 1 else '' + def obtain_config_guess(self, download_source_path=None, search_source_paths=None): """ Locate or download an up-to-date config.guess for use with ConfigureMake @@ -247,8 +258,8 @@ def determine_build_and_host_type(self): "EasyBuild attempts to download a recent config.guess but seems to have failed!") else: self.check_config_guess() - system_type, _ = run_cmd(self.config_guess, log_all=True) - system_type = system_type.strip() + res = run_shell_cmd(self.config_guess) + system_type = res.output.strip() self.log.info("%s returned a system type '%s'", self.config_guess, system_type) if build_type is None: @@ -323,35 +334,53 @@ def configure_step(self, cmd_prefix=''): ] + build_and_host_options + [self.cfg['configopts']] ) - (out, _) = run_cmd(cmd, log_all=True, simple=False) - - return out - - def build_step(self, verbose=False, path=None): + res = run_shell_cmd(cmd) + + action = self.cfg['unrecognized_configure_options'] + valid_actions = (ERROR, WARN, IGNORE) + # Always verify the EC param + if action not in valid_actions: + raise EasyBuildError("Invalid value for 'unrecognized_configure_options': %s. Must be one of: %s", + action, ', '.join(valid_actions)) + if action != IGNORE: + unrecognized_options_str = 'configure: WARNING: unrecognized options:' + unrecognized_options = re.findall(rf"^{unrecognized_options_str}.*", res.output, flags=re.I | re.M) + # Keep only unique options (remove the warning string and strip whitespace) + unrecognized_options = nub(x.split(unrecognized_options_str)[-1].strip() for x in unrecognized_options) + if unrecognized_options: + msg = 'Found unrecognized configure options: ' + '; '.join(unrecognized_options) + if action == WARN: + print_warning(msg) + else: + raise EasyBuildError(msg) + + return res.output + + def build_step(self, verbose=None, path=None): """ Start the actual build - typical: make -j X """ - paracmd = '' - if self.cfg['parallel']: - paracmd = "-j %s" % self.cfg['parallel'] + if verbose is not None: + self.log.deprecated("The 'verbose' parameter to build_step is deprecated and unneeded.", '6.0') targets = self.cfg.get('build_cmd_targets') or DEFAULT_BUILD_TARGET # ensure strings are converted to list - targets = [targets] if isinstance(targets, string_type) else targets + targets = [targets] if isinstance(targets, str) else targets for target in targets: cmd = ' '.join([ self.cfg['prebuildopts'], self.cfg.get('build_cmd') or DEFAULT_BUILD_CMD, target, - paracmd, + self.parallel_flag, self.cfg['buildopts'], ]) self.log.info("Building target '%s'", target) - (out, _) = run_cmd(cmd, path=path, log_all=True, simple=False, log_output=verbose) + res = run_shell_cmd(cmd, work_dir=path) + out = res.output return out @@ -365,13 +394,13 @@ def test_step(self): runtest = self.cfg['runtest'] if runtest or test_cmd != DEFAULT_TEST_CMD: # Make run_test a string (empty if it is e.g. a boolean) - if not isinstance(runtest, string_type): + if not isinstance(runtest, str): runtest = '' # Compose command filtering out empty values cmd = ' '.join([x for x in (self.cfg['pretestopts'], test_cmd, runtest, self.cfg['testopts']) if x]) - (out, _) = run_cmd(cmd, log_all=True, simple=False) + res = run_shell_cmd(cmd) - return out + return res.output def install_step(self): """ @@ -385,6 +414,6 @@ def install_step(self): self.cfg['installopts'], ]) - (out, _) = run_cmd(cmd, log_all=True, simple=False) + res = run_shell_cmd(cmd) - return out + return res.output diff --git a/easybuild/easyblocks/generic/configuremakepythonpackage.py b/easybuild/easyblocks/generic/configuremakepythonpackage.py index 8da4ac0331b..9be68aaf7a1 100644 --- a/easybuild/easyblocks/generic/configuremakepythonpackage.py +++ b/easybuild/easyblocks/generic/configuremakepythonpackage.py @@ -1,5 +1,5 @@ ## -# Copyright 2015-2024 Ghent University +# Copyright 2015-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -31,7 +31,7 @@ """ from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.easyblocks.generic.pythonpackage import PythonPackage -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd class ConfigureMakePythonPackage(ConfigureMake, PythonPackage): @@ -58,7 +58,7 @@ def configure_step(self, *args, **kwargs): """Configure build using ``python configure``.""" PythonPackage.configure_step(self, *args, **kwargs) cmd = ' '.join([self.cfg['preconfigopts'], self.python_cmd, self.cfg['configopts']]) - run_cmd(cmd, log_all=True) + run_shell_cmd(cmd) def build_step(self, *args, **kwargs): """Build Python package with ``make``.""" diff --git a/easybuild/easyblocks/generic/craytoolchain.py b/easybuild/easyblocks/generic/craytoolchain.py index 210b7e9f478..1bd6fd99ecc 100644 --- a/easybuild/easyblocks/generic/craytoolchain.py +++ b/easybuild/easyblocks/generic/craytoolchain.py @@ -1,5 +1,5 @@ ## -# Copyright 2015-2024 Ghent University +# Copyright 2015-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/generic/fortranpythonpackage.py b/easybuild/easyblocks/generic/fortranpythonpackage.py index 1a8a512f1da..3d72497b832 100644 --- a/easybuild/easyblocks/generic/fortranpythonpackage.py +++ b/easybuild/easyblocks/generic/fortranpythonpackage.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -36,7 +36,7 @@ import easybuild.tools.toolchain as toolchain from easybuild.easyblocks.generic.pythonpackage import PythonPackage from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd class FortranPythonPackage(PythonPackage): @@ -70,4 +70,4 @@ def build_step(self): raise EasyBuildError("Unknown family of compilers being used: %s", comp_fam) cmd = "%s %s setup.py build %s" % (self.cfg['prebuildopts'], self.python_cmd, self.cfg['buildopts']) - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) diff --git a/easybuild/easyblocks/generic/gopackage.py b/easybuild/easyblocks/generic/gopackage.py index 268d0eea617..ca86e247786 100644 --- a/easybuild/easyblocks/generic/gopackage.py +++ b/easybuild/easyblocks/generic/gopackage.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -35,7 +35,7 @@ from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.build_log import EasyBuildError from easybuild.tools.modules import get_software_root, get_software_version -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd class GoPackage(EasyBlock): @@ -73,8 +73,8 @@ def configure_step(self): env.setvar('GOBIN', os.path.join(self.installdir, 'bin'), verbose=False) # creates log entries for go being used, for debugging - run_cmd("go version", verbose=False, trace=False) - run_cmd("go env", verbose=False, trace=False) + run_shell_cmd("go version", hidden=True) + run_shell_cmd("go env", hidden=True) def build_step(self): """If Go package is not native go module, lets try to make the module.""" @@ -83,7 +83,7 @@ def build_step(self): go_sum_file = 'go.sum' if not os.path.exists(go_mod_file) or not os.path.isfile(go_mod_file): - self.log.warn("go.mod not found! This is not natively supported go module. Trying to init module.") + self.log.warning("go.mod not found! This is not natively supported go module. Trying to init module.") if self.cfg['modulename'] is None: raise EasyBuildError("Installing non-native go module. You need to specify 'modulename' in easyconfig") @@ -93,13 +93,13 @@ def build_step(self): # go mod init cmd = ' '.join(['go', 'mod', 'init', self.cfg['modulename']]) - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) if self.cfg['forced_deps']: for dep in self.cfg['forced_deps']: # go get specific dependencies which locks them in go.mod cmd = ' '.join(['go', 'get', '%s@%s' % dep]) - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) # note: ... (tripledot) used below is not a typo, but go wildcard pattern # which means: anything you can find in this directory, including all subdirectories @@ -107,20 +107,20 @@ def build_step(self): # see: https://stackoverflow.com/a/28031651/2047157 # building and testing will add packages to go.mod - run_cmd('go build ./...', log_all=True, simple=True) - run_cmd('go test ./...', log_all=True, simple=True) + run_shell_cmd('go build ./...') + run_shell_cmd('go test ./...') # tidy up go.mod - run_cmd('go mod tidy', log_all=True, simple=True) + run_shell_cmd('go mod tidy') # build and test again, to ensure go mod tidy didn't removed anything needed - run_cmd('go build ./...', log_all=True, simple=True) - run_cmd('go test ./...', log_all=True, simple=True) + run_shell_cmd('go build ./...') + run_shell_cmd('go test ./...') - self.log.warn('Include generated go.mod and go.sum via patch to ensure locked dependencies ' - 'and run this easyconfig again.') - run_cmd('cat go.mod', log_all=True, simple=True) - run_cmd('cat go.sum', log_all=True, simple=True) + self.log.warning('Include generated go.mod and go.sum via patch to ensure locked dependencies ' + 'and run this easyconfig again.') + run_shell_cmd('cat go.mod') + run_shell_cmd('cat go.sum') if not os.path.exists(go_sum_file) or not os.path.isfile(go_sum_file): raise EasyBuildError("go.sum not found! This module has no locked dependency versions.") @@ -138,7 +138,7 @@ def install_step(self): '-x', self.cfg['installopts'], ]) - run_cmd(cmd, log_all=True, log_ok=True, simple=True) + run_shell_cmd(cmd) def sanity_check_step(self): """Custom sanity check for Go package.""" diff --git a/easybuild/easyblocks/generic/intelbase.py b/easybuild/easyblocks/generic/intelbase.py index 5c03926642f..194e6a06a6f 100644 --- a/easybuild/easyblocks/generic/intelbase.py +++ b/easybuild/easyblocks/generic/intelbase.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -48,8 +48,9 @@ from easybuild.framework.easyconfig.types import ensure_iterable_license_specs from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import adjust_permissions, find_flexlm_license -from easybuild.tools.filetools import mkdir, read_file, remove_file, write_file -from easybuild.tools.run import run_cmd +from easybuild.tools.filetools import read_file, remove_file, write_file +from easybuild.tools.modules import MODULE_LOAD_ENV_HEADERS +from easybuild.tools.run import run_shell_cmd # different supported activation types (cfr. Intel documentation) @@ -68,20 +69,14 @@ # silent.cfg parameter name for type of license activation (cfr. options listed above) ACTIVATION_NAME = 'ACTIVATION_TYPE' # since icc/ifort v2013_sp1, impi v4.1.1, imkl v11.1 -ACTIVATION_NAME_2012 = 'ACTIVATION' # previous activation type parameter used in older versions # silent.cfg parameter name for install prefix INSTALL_DIR_NAME = 'PSET_INSTALL_DIR' # silent.cfg parameter name for install mode INSTALL_MODE_NAME = 'PSET_MODE' -# Older (2015 and previous) silent.cfg parameter name for install mode -INSTALL_MODE_NAME_2015 = 'INSTALL_MODE' -# Install mode for 2016 version +# Install mode since 2016 version INSTALL_MODE = 'install' -# Install mode for 2015 and older versions -INSTALL_MODE_2015 = 'NONRPM' # silent.cfg parameter name for license file/server specification LICENSE_FILE_NAME = 'ACTIVATION_LICENSE_FILE' # since icc/ifort v2013_sp1, impi v4.1.1, imkl v11.1 -LICENSE_FILE_NAME_2012 = 'PSET_LICENSE_FILE' # previous license file parameter used in older versions LICENSE_SERIAL_NUMBER = 'ACTIVATION_SERIAL_NUMBER' COMP_ALL = 'ALL' @@ -134,34 +129,25 @@ def set_versioned_subdir(self, subdir, path): def get_guesses_tools(self): """Find reasonable paths for a subset of Intel tools, ignoring CPATH, LD_LIBRARY_PATH and LIBRARY_PATH""" + self.log.deprecated("IntelBase.get_guesses_tools() is replaced by IntelBase.prepare_intel_tools_env()", '6.0') - guesses = super(IntelBase, self).make_module_req_guess() - - if self.cfg['m32']: - guesses['PATH'] = [os.path.join(self.subdir, 'bin32')] - else: - guesses['PATH'] = [os.path.join(self.subdir, 'bin64')] - - guesses['MANPATH'] = [os.path.join(self.subdir, 'man')] + def prepare_intel_tools_env(self): + """Find reasonable paths for a subset of Intel tools, ignoring CPATH, LD_LIBRARY_PATH and LIBRARY_PATH""" + self.module_load_environment.PATH = [os.path.join(self.subdir, 'bin64')] + self.module_load_environment.MANPATH = [os.path.join(self.subdir, 'man')] # make sure $CPATH, $LD_LIBRARY_PATH and $LIBRARY_PATH are not updated in generated module file, # because that leads to problem when the libraries included with VTune/Advisor/Inspector are being picked up - for key in ['CPATH', 'LD_LIBRARY_PATH', 'LIBRARY_PATH']: - if key in guesses: - self.log.debug("Purposely not updating $%s in %s module file", key, self.name) - del guesses[key] - - return guesses + mod_env_headers = self.module_load_environment.alias_vars(MODULE_LOAD_ENV_HEADERS) + mod_env_libs = ['LD_LIBRARY_PATH', 'LIBRARY_PATH'] + for disallowed_var in mod_env_headers + mod_env_libs: + self.module_load_environment.remove(disallowed_var) + self.log.debug(f"Purposely not updating ${disallowed_var} in {self.name} module file") def get_custom_paths_tools(self, binaries): """Custom sanity check paths for certain Intel tools.""" - if self.cfg['m32']: - files = [os.path.join('bin32', b) for b in binaries] - dirs = ['lib32', 'include'] - else: - files = [os.path.join('bin64', b) for b in binaries] - dirs = ['lib64', 'include'] - + files = [os.path.join('bin64', b) for b in binaries] + dirs = ['lib64', 'include'] custom_paths = { 'files': [os.path.join(self.subdir, f) for f in files], 'dirs': [os.path.join(self.subdir, d) for d in dirs], @@ -176,12 +162,6 @@ def extra_options(extra_vars=None): 'serial_number': [None, "Serial number for the product", CUSTOM], 'requires_runtime_license': [True, "Boolean indicating whether or not a runtime license is required", CUSTOM], - # 'usetmppath': - # workaround for older SL5 version (5.5 and earlier) - # used to be True, but False since SL5.6/SL6 - # disables TMP_PATH env and command line option - 'usetmppath': [False, "Use temporary path for installation", CUSTOM], - 'm32': [False, "Enable 32-bit toolchain", CUSTOM], 'components': [None, "List of components to install", CUSTOM], }) @@ -374,8 +354,8 @@ def install_step_classic(self, silent_cfg_names_map=None, silent_cfg_extras=None ]) % { 'install_dir_name': silent_cfg_names_map.get('install_dir_name', INSTALL_DIR_NAME), 'install_dir': silent_cfg_names_map.get('install_dir', self.installdir), - 'install_mode': silent_cfg_names_map.get('install_mode', INSTALL_MODE_2015), - 'install_mode_name': silent_cfg_names_map.get('install_mode_name', INSTALL_MODE_NAME_2015), + 'install_mode': silent_cfg_names_map.get('install_mode', INSTALL_MODE), + 'install_mode_name': silent_cfg_names_map.get('install_mode_name', INSTALL_MODE_NAME), } if self.install_components is not None: @@ -404,15 +384,6 @@ def install_step_classic(self, silent_cfg_names_map=None, silent_cfg_extras=None write_file(silentcfg, silent) self.log.debug("Contents of %s:\n%s", silentcfg, silent) - # workaround for mktmp: create tmp dir and use it - tmpdir = os.path.join(self.cfg['start_dir'], 'mytmpdir') - mkdir(tmpdir, parents=True) - - tmppathopt = '' - if self.cfg['usetmppath']: - env.setvar('TMP_PATH', tmpdir) - tmppathopt = "-t %s" % tmpdir - # set some extra env variables env.setvar('LOCAL_INSTALL_VERBOSE', '1') env.setvar('VERBOSE_MODE', '1') @@ -423,12 +394,11 @@ def install_step_classic(self, silent_cfg_names_map=None, silent_cfg_extras=None cmd = ' '.join([ self.cfg['preinstallopts'], './install.sh', - tmppathopt, '-s ' + silentcfg, self.cfg['installopts'], ]) - return run_cmd(cmd, log_all=True, simple=True, log_output=True) + run_shell_cmd(cmd) def install_step_oneapi(self, *args, **kwargs): """ @@ -468,16 +438,16 @@ def install_step_oneapi(self, *args, **kwargs): cmd.append(self.cfg['installopts']) - return run_cmd(' '.join(cmd), log_all=True, simple=True, log_output=True) + run_shell_cmd(' '.join(cmd)) def install_step(self, *args, **kwargs): """ Install Intel software """ if LooseVersion(self.version) >= LooseVersion('2021'): - return self.install_step_oneapi(*args, **kwargs) + self.install_step_oneapi(*args, **kwargs) else: - return self.install_step_classic(*args, **kwargs) + self.install_step_classic(*args, **kwargs) def move_after_install(self): """Move installed files to correct location after installation.""" diff --git a/easybuild/easyblocks/generic/jar.py b/easybuild/easyblocks/generic/jar.py index 2ac88971af9..d502f9440c4 100644 --- a/easybuild/easyblocks/generic/jar.py +++ b/easybuild/easyblocks/generic/jar.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/generic/juliabundle.py b/easybuild/easyblocks/generic/juliabundle.py index 7f2c7080ce0..cce74517282 100644 --- a/easybuild/easyblocks/generic/juliabundle.py +++ b/easybuild/easyblocks/generic/juliabundle.py @@ -1,5 +1,5 @@ ## -# Copyright 2022-2024 Vrije Universiteit Brussel +# Copyright 2022-2025 Vrije Universiteit Brussel # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/generic/juliapackage.py b/easybuild/easyblocks/generic/juliapackage.py index 5a3e4acbb51..800b5a85e8e 100644 --- a/easybuild/easyblocks/generic/juliapackage.py +++ b/easybuild/easyblocks/generic/juliapackage.py @@ -1,5 +1,5 @@ ## -# Copyright 2022-2024 Vrije Universiteit Brussel +# Copyright 2022-2025 Vrije Universiteit Brussel # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -40,7 +40,7 @@ from easybuild.tools.build_log import EasyBuildError from easybuild.tools.modules import get_software_root, get_software_version from easybuild.tools.filetools import copy_dir, mkdir -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd from easybuild.tools.utilities import trace_msg EXTS_FILTER_JULIA_PACKAGES = ("julia -e 'using %(ext_name)s'", "") @@ -107,14 +107,14 @@ def get_julia_env(env_var): } try: - out, _ = run_cmd(julia_read_cmd[env_var], log_all=True, simple=False, trace=False) + res = run_shell_cmd(julia_read_cmd[env_var], hidden=True) except KeyError: raise EasyBuildError("Unknown Julia environment variable requested: %s", env_var) try: - parsed_var = ast.literal_eval(out) + parsed_var = ast.literal_eval(res.output) except SyntaxError: - raise EasyBuildError("Failed to parse %s from julia shell: %s", env_var, out) + raise EasyBuildError("Failed to parse %s from julia shell: %s", env_var, res.output) return parsed_var @@ -227,9 +227,9 @@ def install_pkg_source(self, pkg_source, environment, trace=True): "julia -e '%s'" % julia_pkg_cmd, self.cfg['installopts'], ]) - (out, _) = run_cmd(cmd, log_all=True, simple=False, trace=trace) + res = run_shell_cmd(cmd) - return out + return res.output def include_pkg_dependencies(self): """Add to installation environment all Julia packages already present in its dependencies""" @@ -283,13 +283,13 @@ def install_step(self): return self.install_pkg() - def run(self): + def install_extension(self): """Install Julia package as an extension.""" if not self.src: errmsg = "No source found for Julia package %s, required for installation. (src: %s)" raise EasyBuildError(errmsg, self.name, self.src) - ExtensionEasyBlock.run(self, unpack_src=True) + ExtensionEasyBlock.install_extension(self, unpack_src=True) self.prepare_julia_env() self.install_pkg() diff --git a/easybuild/easyblocks/generic/makecp.py b/easybuild/easyblocks/generic/makecp.py index 230608d3aa5..f4f1870c5ce 100644 --- a/easybuild/easyblocks/generic/makecp.py +++ b/easybuild/easyblocks/generic/makecp.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2024 the Cyprus Institute +# Copyright 2013-2025 the Cyprus Institute # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -35,7 +35,6 @@ from easybuild.framework.easyconfig import BUILD, MANDATORY from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import change_dir, copy_dir, copy_file, mkdir -from easybuild.tools.py2vs3 import string_type class MakeCp(ConfigureMake): @@ -78,13 +77,13 @@ def install_step(self): for fil in files_to_copy: if isinstance(fil, tuple): # ([src1, src2], targetdir) - if len(fil) == 2 and isinstance(fil[0], list) and isinstance(fil[1], string_type): + if len(fil) == 2 and isinstance(fil[0], list) and isinstance(fil[1], str): files_specs = fil[0] target = os.path.join(self.installdir, fil[1]) else: raise EasyBuildError("Only tuples of format '([], )' supported.") # 'src_file' or 'src_dir' - elif isinstance(fil, string_type): + elif isinstance(fil, str): files_specs = [fil] target = self.installdir else: diff --git a/easybuild/easyblocks/generic/mesonninja.py b/easybuild/easyblocks/generic/mesonninja.py index c432bfd0211..22bf6ae68eb 100644 --- a/easybuild/easyblocks/generic/mesonninja.py +++ b/easybuild/easyblocks/generic/mesonninja.py @@ -1,5 +1,5 @@ ## -# Copyright 2018-2024 Ghent University +# Copyright 2018-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -34,7 +34,7 @@ from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import change_dir, create_unused_dir, which from easybuild.tools.modules import get_software_version -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd DEFAULT_CONFIGURE_CMD = 'meson' DEFAULT_BUILD_CMD = 'ninja' @@ -53,12 +53,28 @@ def extra_options(extra_vars=None): extra_vars.update({ 'build_dir': [None, "build_dir to pass to meson", CUSTOM], 'build_cmd': [DEFAULT_BUILD_CMD, "Build command to use", CUSTOM], + 'build_type': [None, "Build type for meson, e.g. release." + "Replaces use of toolchain options debug, noopt, lowopt, opt", CUSTOM], + 'ndebug': [True, "Sets -Db_ndebug which in turn defines NDEBUG for C/C++ builds." + "This disabled costly asserts in code, typical for production.", CUSTOM], 'configure_cmd': [DEFAULT_CONFIGURE_CMD, "Configure command to use", CUSTOM], 'install_cmd': [DEFAULT_INSTALL_CMD, "Install command to use", CUSTOM], 'separate_build_dir': [True, "Perform build in a separate directory", CUSTOM], }) return extra_vars + @property + def optimization(self): + """Optimization level""" + if self.toolchain.options.get('noopt', False): + return 0 + elif self.toolchain.options.get('lowopt', False): + return 1 + elif self.toolchain.options.get('opt', False): + return 3 + else: + return 2 + def configure_step(self, cmd_prefix=''): """ Configure with Meson. @@ -92,15 +108,27 @@ def configure_step(self, cmd_prefix=''): build_dir = self.cfg.get('build_dir') or self.start_dir - cmd = "%(preconfigopts)s %(configure_cmd)s --prefix %(installdir)s %(configopts)s %(source_dir)s" % { + # Build type is either specified directly or via --optimization and --debug flags. + if self.cfg['build_type'] is not None: + build_type = '--buildtype=' + self.cfg['build_type'] + else: + build_type = '--optimization=%(optimization)s %(debug)s' % { + 'optimization': self.optimization, + 'debug': '--debug' if self.toolchain.options.get('debug', False) else '', + } + + cmd = ("%(preconfigopts)s %(configure_cmd)s --prefix %(installdir)s %(build_type)s %(configopts)s " + "-Db_ndebug=%(ndebug)s %(source_dir)s") % { + 'build_type': build_type, 'configopts': self.cfg['configopts'], 'configure_cmd': configure_cmd, 'installdir': self.installdir, 'preconfigopts': self.cfg['preconfigopts'], + 'ndebug': str(self.cfg.get('ndebug')).lower(), 'source_dir': build_dir, } - (out, _) = run_cmd(cmd, log_all=True, simple=False) - return out + res = run_shell_cmd(cmd) + return res.output def build_step(self, verbose=False, path=None): """ @@ -108,18 +136,16 @@ def build_step(self, verbose=False, path=None): """ build_cmd = self.cfg.get('build_cmd', DEFAULT_BUILD_CMD) - parallel = '' - if self.cfg['parallel']: - parallel = "-j %s" % self.cfg['parallel'] + parallel = f'-j {self.cfg.parallel}' if self.cfg.parallel > 1 else '' - cmd = "%(prebuildopts)s %(build_cmd)s %(parallel)s %(buildopts)s" % { + cmd = "%(prebuildopts)s %(build_cmd)s -v %(parallel)s %(buildopts)s" % { 'buildopts': self.cfg['buildopts'], 'build_cmd': build_cmd, 'parallel': parallel, 'prebuildopts': self.cfg['prebuildopts'], } - (out, _) = run_cmd(cmd, log_all=True, simple=False) - return out + res = run_shell_cmd(cmd) + return res.output def test_step(self): """ @@ -127,8 +153,8 @@ def test_step(self): """ if self.cfg['runtest']: cmd = "%s %s %s" % (self.cfg['pretestopts'], self.cfg['runtest'], self.cfg['testopts']) - (out, _) = run_cmd(cmd, log_all=True, simple=False) - return out + res = run_shell_cmd(cmd) + return res.output def install_step(self): """ @@ -136,9 +162,7 @@ def install_step(self): """ install_cmd = self.cfg.get('install_cmd', DEFAULT_INSTALL_CMD) - parallel = '' - if self.cfg['parallel']: - parallel = "-j %s" % self.cfg['parallel'] + parallel = f'-j {self.cfg.parallel}' if self.cfg.parallel > 1 else '' cmd = "%(preinstallopts)s %(install_cmd)s %(parallel)s %(installopts)s install" % { 'installopts': self.cfg['installopts'], @@ -146,5 +170,5 @@ def install_step(self): 'install_cmd': install_cmd, 'preinstallopts': self.cfg['preinstallopts'], } - (out, _) = run_cmd(cmd, log_all=True, simple=False) - return out + res = run_shell_cmd(cmd) + return res.output diff --git a/easybuild/easyblocks/generic/modulerc.py b/easybuild/easyblocks/generic/modulerc.py index e9547aada1a..886775e04c6 100644 --- a/easybuild/easyblocks/generic/modulerc.py +++ b/easybuild/easyblocks/generic/modulerc.py @@ -1,5 +1,5 @@ ## -# Copyright 2018-2024 Ghent University +# Copyright 2018-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/generic/ocamlpackage.py b/easybuild/easyblocks/generic/ocamlpackage.py index 538ecb8c3ff..ed788c256f5 100644 --- a/easybuild/easyblocks/generic/ocamlpackage.py +++ b/easybuild/easyblocks/generic/ocamlpackage.py @@ -1,5 +1,5 @@ ## -# Copyright 2015-2024 Ghent University +# Copyright 2015-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -29,7 +29,7 @@ """ from easybuild.framework.extensioneasyblock import ExtensionEasyBlock from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd class OCamlPackage(ExtensionEasyBlock): @@ -39,11 +39,11 @@ def configure_step(self): """Raise error when configure step is run: installing OCaml packages stand-alone is not supported (yet)""" raise EasyBuildError("Installing OCaml packages stand-alone is not supported (yet)") - def run(self): + def install_extension(self): """Perform OCaml package installation (as extension).""" # install using 'opam install' - run_cmd("eval `opam config env` && opam install -yv %s.%s" % (self.name, self.version)) + run_shell_cmd("eval `opam config env` && opam install -yv %s.%s" % (self.name, self.version)) # 'opam pin add' fixes the version of the package # see https://opam.ocaml.org/doc/Usage.html#opampin - run_cmd("eval `opam config env` && opam pin -yv add %s %s" % (self.name, self.version)) + run_shell_cmd("eval `opam config env` && opam pin -yv add %s %s" % (self.name, self.version)) diff --git a/easybuild/easyblocks/generic/octavepackage.py b/easybuild/easyblocks/generic/octavepackage.py index c7891460cc1..83a24e5bcb9 100644 --- a/easybuild/easyblocks/generic/octavepackage.py +++ b/easybuild/easyblocks/generic/octavepackage.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -34,7 +34,7 @@ from easybuild.framework.extensioneasyblock import ExtensionEasyBlock from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import change_dir -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd class OctavePackage(ExtensionEasyBlock): @@ -44,19 +44,19 @@ def configure_step(self): """Raise error when configure step is run: installing Octave toolboxes stand-alone is not supported (yet)""" raise EasyBuildError("Installing Octave toolboxes stand-alone is not supported (yet)") - def run(self): + def install_extension(self): """Perform Octave package installation (as extension).""" # if patches are specified, we need to unpack the source tarball, apply the patch, # and create a temporary tarball to use for installation if self.patches: # call out to ExtensionEasyBlock to unpack & apply patches - super(OctavePackage, self).run(unpack_src=True) + super(OctavePackage, self).install_extension(unpack_src=True) # create temporary tarball from unpacked & patched source src = os.path.join(tempfile.gettempdir(), '%s-%s-patched.tar.gz' % (self.name, self.version)) cwd = change_dir(os.path.dirname(self.ext_dir)) - run_cmd("tar cfvz %s %s" % (src, os.path.basename(self.ext_dir))) + run_shell_cmd("tar cfvz %s %s" % (src, os.path.basename(self.ext_dir))) change_dir(cwd) else: src = self.src @@ -69,4 +69,4 @@ def run(self): octave_cmd += "pkg install -global %s" % src - run_cmd("octave --eval '%s'" % octave_cmd) + run_shell_cmd("octave --eval '%s'" % octave_cmd) diff --git a/easybuild/easyblocks/generic/packedbinary.py b/easybuild/easyblocks/generic/packedbinary.py index 18ebbbf28d4..5e6ee8d77a4 100644 --- a/easybuild/easyblocks/generic/packedbinary.py +++ b/easybuild/easyblocks/generic/packedbinary.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/generic/perlbundle.py b/easybuild/easyblocks/generic/perlbundle.py index 928dfe6886e..e86bdb3de01 100644 --- a/easybuild/easyblocks/generic/perlbundle.py +++ b/easybuild/easyblocks/generic/perlbundle.py @@ -1,5 +1,5 @@ ## -# Copyright 2018-2024 Ghent University +# Copyright 2018-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/generic/perlmodule.py b/easybuild/easyblocks/generic/perlmodule.py index 648b5d6dab4..16c784c640a 100644 --- a/easybuild/easyblocks/generic/perlmodule.py +++ b/easybuild/easyblocks/generic/perlmodule.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -35,7 +35,7 @@ from easybuild.framework.extensioneasyblock import ExtensionEasyBlock from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd from easybuild.tools.environment import unset_env_vars @@ -79,7 +79,7 @@ def install_perl_module(self): '%s=%s' % (prefix_opt, self.installdir), self.cfg['configopts'], ]) - run_cmd(install_cmd) + run_shell_cmd(install_cmd) ConfigureMake.build_step(self) ConfigureMake.test_step(self) @@ -98,22 +98,22 @@ def install_perl_module(self): self.installdir, self.cfg['configopts'], ]) - run_cmd(install_cmd) + run_shell_cmd(install_cmd) - run_cmd("%s perl Build build %s" % (self.cfg['prebuildopts'], self.cfg['buildopts'])) + run_shell_cmd("%s perl Build build %s" % (self.cfg['prebuildopts'], self.cfg['buildopts'])) runtest = self.cfg['runtest'] if runtest: - run_cmd('%s perl Build %s %s' % (self.cfg['pretestopts'], runtest, self.cfg['testopts'])) - run_cmd('%s perl Build install %s' % (self.cfg['preinstallopts'], self.cfg['installopts'])) + run_shell_cmd('%s perl Build %s %s' % (self.cfg['pretestopts'], runtest, self.cfg['testopts'])) + run_shell_cmd('%s perl Build install %s' % (self.cfg['preinstallopts'], self.cfg['installopts'])) - def run(self): + def install_extension(self): """Perform the actual Perl module build/installation procedure""" if not self.src: raise EasyBuildError("No source found for Perl module %s, required for installation. (src: %s)", self.name, self.src) - ExtensionEasyBlock.run(self, unpack_src=True) + ExtensionEasyBlock.install_extension(self, unpack_src=True) self.install_perl_module() @@ -139,14 +139,13 @@ def sanity_check_step(self, *args, **kwargs): """ return ExtensionEasyBlock.sanity_check_step(self, EXTS_FILTER_PERL_MODULES, *args, **kwargs) - def make_module_req_guess(self): - """Customized dictionary of paths to look for with PERL*LIB.""" - majver = get_major_perl_version() + def make_module_step(self, *args, **kwargs): + """ + Custom paths to look for with PERL*LIB + """ + perl_lib_var = f"PERL{get_major_perl_version()}LIB" sitearchsuffix = get_site_suffix('sitearch') sitelibsuffix = get_site_suffix('sitelib') + setattr(self.module_load_environment, perl_lib_var, ['', sitearchsuffix, sitelibsuffix]) - guesses = super(PerlModule, self).make_module_req_guess() - guesses.update({ - "PERL%sLIB" % majver: ['', sitearchsuffix, sitelibsuffix], - }) - return guesses + return super().make_module_step(*args, **kwargs) diff --git a/easybuild/easyblocks/generic/pythonbundle.py b/easybuild/easyblocks/generic/pythonbundle.py index 5e7d402f120..cb43d3ee332 100644 --- a/easybuild/easyblocks/generic/pythonbundle.py +++ b/easybuild/easyblocks/generic/pythonbundle.py @@ -1,5 +1,5 @@ ## -# Copyright 2018-2024 Ghent University +# Copyright 2018-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -28,13 +28,12 @@ @author: Kenneth Hoste (Ghent University) """ import os -import sys from easybuild.easyblocks.generic.bundle import Bundle -from easybuild.easyblocks.generic.pythonpackage import EBPYTHONPREFIXES, EXTS_FILTER_PYTHON_PACKAGES -from easybuild.easyblocks.generic.pythonpackage import PythonPackage, get_pylibdirs, pick_python_cmd +from easybuild.easyblocks.generic.pythonpackage import EXTS_FILTER_PYTHON_PACKAGES +from easybuild.easyblocks.generic.pythonpackage import PythonPackage, get_pylibdirs, find_python_cmd_from_ec from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.filetools import which +from easybuild.tools.config import build_option, PYTHONPATH, EBPYTHONPREFIXES from easybuild.tools.modules import get_software_root import easybuild.tools.environment as env @@ -52,6 +51,7 @@ def extra_options(extra_vars=None): extra_vars = {} # combine custom easyconfig parameters of Bundle & PythonPackage extra_vars = Bundle.extra_options(extra_vars) + extra_vars['default_easyblock'][0] = 'PythonPackage' return PythonPackage.extra_options(extra_vars) def __init__(self, *args, **kwargs): @@ -69,69 +69,35 @@ def __init__(self, *args, **kwargs): if key not in self.cfg['exts_default_options']: self.cfg['exts_default_options'][key] = self.cfg[key] - self.cfg['exts_default_options']['download_dep_fail'] = True - self.log.info("Detection of downloaded extension dependencies is enabled") - self.log.info("exts_default_options: %s", self.cfg['exts_default_options']) + self.python_cmd = None self.pylibdir = None - self.all_pylibdirs = [] + self.all_pylibdirs = None # figure out whether this bundle of Python packages is being installed for multiple Python versions self.multi_python = 'Python' in self.cfg['multi_deps'] - def prepare_step(self, *args, **kwargs): - """Prepare for installing bundle of Python packages.""" - super(Bundle, self).prepare_step(*args, **kwargs) + def prepare_python(self): + """Python-specific preparations.""" - python_root = get_software_root('Python') - if python_root is None: + if get_software_root('Python') is None: raise EasyBuildError("Python not included as dependency!") + self.python_cmd = find_python_cmd_from_ec(self.log, self.cfg, required=True) - # when system Python is used, the first 'python' command in $PATH will not be $EBROOTPYTHON/bin/python, - # since $EBROOTPYTHON is set to just 'Python' in that case - # (see handling of allow_system_deps in EasyBlock.prepare_step) - if which('python') == os.path.join(python_root, 'bin', 'python'): - # if we're using a proper Python dependency, let det_pylibdir use 'python' like it does by default - python_cmd = None - else: - # since det_pylibdir will use 'python' by default as command to determine Python lib directory, - # we need to intervene when the system Python is used, by specifying version requirements - # to pick_python_cmd so the right 'python' command is used; - # if we're using the system Python and no Python version requirements are specified, - # use major/minor version of Python being used in this EasyBuild session (as we also do in PythonPackage) - req_py_majver = self.cfg['req_py_majver'] - if req_py_majver is None: - req_py_majver = sys.version_info[0] - req_py_minver = self.cfg['req_py_minver'] - if req_py_minver is None: - req_py_minver = sys.version_info[1] - - # Get the max_py_majver and max_py_minver from the config - max_py_majver = self.cfg['max_py_majver'] - max_py_minver = self.cfg['max_py_minver'] - - python_cmd = pick_python_cmd(req_maj_ver=req_py_majver, req_min_ver=req_py_minver, - max_py_majver=max_py_majver, max_py_minver=max_py_minver) - - # If pick_python_cmd didn't find a (system) Python command, we should raise an error - if python_cmd: - self.log.info("Python command being used: %s", python_cmd) - else: - raise EasyBuildError( - "Failed to pick Python command that satisfies requirements in the easyconfig " - "(req_py_majver = %s, req_py_minver = %s, max_py_majver = %s, max_py_minver = %s)", - req_py_majver, req_py_minver, max_py_majver, max_py_minver - ) - - self.all_pylibdirs = get_pylibdirs(python_cmd=python_cmd) + self.all_pylibdirs = get_pylibdirs(python_cmd=self.python_cmd) self.pylibdir = self.all_pylibdirs[0] # if 'python' is not used, we need to take that into account in the extensions filter # (which is also used during the sanity check) - if python_cmd: + if self.python_cmd != 'python': orig_exts_filter = EXTS_FILTER_PYTHON_PACKAGES - self.cfg['exts_filter'] = (orig_exts_filter[0].replace('python', python_cmd), orig_exts_filter[1]) + self.cfg['exts_filter'] = (orig_exts_filter[0].replace('python', self.python_cmd), orig_exts_filter[1]) + + def prepare_step(self, *args, **kwargs): + """Prepare for installing bundle of Python packages.""" + super(Bundle, self).prepare_step(*args, **kwargs) + self.prepare_python() def extensions_step(self, *args, **kwargs): """Install extensions (usually PythonPackages)""" @@ -149,9 +115,20 @@ def make_module_extra(self, *args, **kwargs): txt = super(Bundle, self).make_module_extra(*args, **kwargs) # update $EBPYTHONPREFIXES rather than $PYTHONPATH - # if this Python package was installed for multiple Python versions - if self.multi_python: - txt += self.module_generator.prepend_paths(EBPYTHONPREFIXES, '') + # if this Python package was installed for multiple Python versions, or if we prefer it + use_ebpythonprefixes = False + runtime_deps = [dep['name'] for dep in self.cfg.dependencies(runtime_only=True)] + + if 'Python' in runtime_deps: + self.log.info("Found Python runtime dependency, so considering $EBPYTHONPREFIXES...") + if build_option('prefer_python_search_path') == EBPYTHONPREFIXES: + self.log.info("Preferred Python search path is $EBPYTHONPREFIXES, so using that") + use_ebpythonprefixes = True + + if self.multi_python or use_ebpythonprefixes: + path = '' # EBPYTHONPREFIXES are relative to the install dir + if path not in self.module_generator.added_paths_per_key[EBPYTHONPREFIXES]: + txt += self.module_generator.prepend_paths(EBPYTHONPREFIXES, path) else: # the temporary module file that is generated before installing extensions @@ -166,7 +143,8 @@ def make_module_extra(self, *args, **kwargs): ] for pylibdir in new_pylibdirs: - txt += self.module_generator.prepend_paths('PYTHONPATH', pylibdir) + if pylibdir not in self.module_generator.added_paths_per_key[PYTHONPATH]: + txt += self.module_generator.prepend_paths(PYTHONPATH, pylibdir) return txt diff --git a/easybuild/easyblocks/generic/pythonpackage.py b/easybuild/easyblocks/generic/pythonpackage.py index eae50cae7cb..254482717d2 100644 --- a/easybuild/easyblocks/generic/pythonpackage.py +++ b/easybuild/easyblocks/generic/pythonpackage.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -38,21 +38,20 @@ import sys import tempfile from easybuild.tools import LooseVersion -from distutils.sysconfig import get_config_vars +from sysconfig import get_config_vars import easybuild.tools.environment as env from easybuild.base import fancylogger -from easybuild.easyblocks.python import EBPYTHONPREFIXES, EXTS_FILTER_PYTHON_PACKAGES +from easybuild.easyblocks.python import EXTS_FILTER_PYTHON_PACKAGES from easybuild.framework.easyconfig import CUSTOM from easybuild.framework.easyconfig.default import DEFAULT_CONFIG -from easybuild.framework.easyconfig.templates import TEMPLATE_CONSTANTS +from easybuild.framework.easyconfig.templates import PYPI_SOURCE from easybuild.framework.extensioneasyblock import ExtensionEasyBlock from easybuild.tools.build_log import EasyBuildError, print_msg -from easybuild.tools.config import build_option +from easybuild.tools.config import build_option, PYTHONPATH, EBPYTHONPREFIXES from easybuild.tools.filetools import change_dir, mkdir, remove_dir, symlink, which -from easybuild.tools.modules import get_software_root -from easybuild.tools.py2vs3 import string_type, subprocess_popen_text -from easybuild.tools.run import run_cmd +from easybuild.tools.modules import ModEnvVarType, get_software_root +from easybuild.tools.run import run_shell_cmd, subprocess_popen_text from easybuild.tools.utilities import nub from easybuild.tools.hooks import CONFIGURE_STEP, BUILD_STEP, TEST_STEP, INSTALL_STEP @@ -79,8 +78,8 @@ def det_python_version(python_cmd): """Determine version of specified 'python' command.""" pycode = 'import sys; print("%s.%s.%s" % sys.version_info[:3])' - out, _ = run_cmd("%s -c '%s'" % (python_cmd, pycode), simple=False, trace=False) - return out.strip() + res = run_shell_cmd("%s -c '%s'" % (python_cmd, pycode), hidden=True) + return res.output.strip() def pick_python_cmd(req_maj_ver=None, req_min_ver=None, max_py_majver=None, max_py_minver=None): @@ -103,31 +102,31 @@ def check_python_cmd(python_cmd): # check whether specified Python command is available if os.path.isabs(python_cmd): if not os.path.isfile(python_cmd): - log.debug("Python command '%s' does not exist", python_cmd) + log.debug(f"Python command '{python_cmd}' does not exist") return False else: python_cmd_path = which(python_cmd) if python_cmd_path is None: - log.debug("Python command '%s' not available through $PATH", python_cmd) + log.debug(f"Python command '{python_cmd}' not available through $PATH") return False + pyver = det_python_version(python_cmd) + if req_maj_ver is not None: if req_min_ver is None: req_majmin_ver = '%s.0' % req_maj_ver else: req_majmin_ver = '%s.%s' % (req_maj_ver, req_min_ver) - pyver = det_python_version(python_cmd) - # (strict) check for major version maj_ver = pyver.split('.')[0] if maj_ver != str(req_maj_ver): - log.debug("Major Python version does not match: %s vs %s", maj_ver, req_maj_ver) + log.debug(f"Major Python version does not match: {maj_ver} vs {req_maj_ver}") return False # check for minimal minor version if LooseVersion(pyver) < LooseVersion(req_majmin_ver): - log.debug("Minimal requirement for minor Python version not satisfied: %s vs %s", pyver, req_majmin_ver) + log.debug(f"Minimal requirement for minor Python version not satisfied: {pyver} vs {req_majmin_ver}") return False if max_py_majver is not None: @@ -136,8 +135,6 @@ def check_python_cmd(python_cmd): else: max_majmin_ver = '%s.%s' % (max_py_majver, max_py_minver) - pyver = det_python_version(python_cmd) - if LooseVersion(pyver) > LooseVersion(max_majmin_ver): log.debug("Python version (%s) on the system is newer than the maximum supported " "Python version specified in the easyconfig (%s)", @@ -145,36 +142,90 @@ def check_python_cmd(python_cmd): return False # all check passed - log.debug("All check passed for Python command '%s'!", python_cmd) + log.debug(f"All check passed for Python command '{python_cmd}'!") return True # compose list of 'python' commands to consider python_cmds = ['python'] if req_maj_ver: - python_cmds.append('python%s' % req_maj_ver) + python_cmds.append(f'python{req_maj_ver}') if req_min_ver: - python_cmds.append('python%s.%s' % (req_maj_ver, req_min_ver)) + python_cmds.append(f'python{req_maj_ver}.{req_min_ver}') python_cmds.append(sys.executable) - log.debug("Considering Python commands: %s", ', '.join(python_cmds)) + log.debug("Considering Python commands: " + ', '.join(python_cmds)) # try and find a 'python' command that satisfies the requirements res = None for python_cmd in python_cmds: if check_python_cmd(python_cmd): - log.debug("Python command '%s' satisfies version requirements!", python_cmd) + log.debug(f"Python command '{python_cmd}' satisfies version requirements!") if os.path.isabs(python_cmd): res = python_cmd else: res = which(python_cmd) - log.debug("Absolute path to retained Python command: %s", res) + log.debug("Absolute path to retained Python command: " + res) break else: - log.debug("Python command '%s' does not satisfy version requirements (maj: %s, min: %s), moving on", - python_cmd, req_maj_ver, req_min_ver) + log.debug(f"Python command '{python_cmd}' does not satisfy version requirements " + f"(maj: {req_maj_ver}, min: {req_min_ver}), moving on") return res +def find_python_cmd(log, req_py_majver, req_py_minver, max_py_majver, max_py_minver, required): + """Return an appropriate python command to use. + + When python is a dependency use the full path to that. + Else use req_py_maj/minver (defaulting to the Python being used in this EasyBuild session) to select one. + If no (matching) python command is found and raise an Error or log a warning depending on the required argument. + """ + python = None + python_root = get_software_root('Python') + # keep in mind that Python may be listed as an allowed system dependency, + # so just checking Python root is not sufficient + if python_root: + bin_python = os.path.join(python_root, 'bin', 'python') + if os.path.exists(bin_python) and os.path.samefile(which('python'), bin_python): + # if Python is listed as a (build) dependency, use 'python' command provided that way + python = bin_python + log.debug("Retaining 'python' command for Python dependency: " + python) + + if python is None: + # if no Python version requirements are specified, + # use major/minor version of Python being used in this EasyBuild session + if req_py_majver is None: + req_py_majver = sys.version_info[0] + if req_py_minver is None: + req_py_minver = sys.version_info[1] + # if using system Python, go hunting for a 'python' command that satisfies the requirements + python = pick_python_cmd(req_maj_ver=req_py_majver, req_min_ver=req_py_minver, + max_py_majver=max_py_majver, max_py_minver=max_py_minver) + + if python: + log.info("Python command being used: " + python) + elif required: + if all(v is None for v in (req_py_majver, req_py_minver, max_py_majver, max_py_minver)): + error_msg = "Failed to pick Python command to use" + else: + error_msg = (f"Failed to pick Python command that satisfies requirements in the easyconfig: " + f"req_py_majver = {req_py_majver}, req_py_minver = {req_py_minver}") + if max_py_majver is not None: + error_msg += f"max_py_majver = {max_py_majver}, max_py_minver = {max_py_minver}" + raise EasyBuildError(error_msg) + else: + log.warning("No Python command found!") + return python + + +def find_python_cmd_from_ec(log, cfg, required): + """Find a python command using the constraints specified in the EasyConfig""" + return find_python_cmd(log, + cfg['req_py_majver'], cfg['req_py_minver'], + max_py_majver=cfg['max_py_majver'], + max_py_minver=cfg['max_py_minver'], + required=required) + + def det_pylibdir(plat_specific=False, python_cmd=None): """Determine Python library directory.""" log = fancylogger.getLogger('det_pylibdir', fname=False) @@ -184,13 +235,13 @@ def det_pylibdir(plat_specific=False, python_cmd=None): python_cmd = 'python' # determine Python lib dir via distutils - # use run_cmd, we can to talk to the active Python, not the system Python running EasyBuild + # use run_shell_cmd, we can to talk to the active Python, not the system Python running EasyBuild prefix = '/tmp/' if LooseVersion(det_python_version(python_cmd)) >= LooseVersion('3.12'): # Python 3.12 removed distutils but has a core sysconfig module which is similar pathname = 'platlib' if plat_specific else 'purelib' - vars = {'platbase': prefix, 'base': prefix} - pycode = 'import sysconfig; print(sysconfig.get_path("%s", vars=%s))' % (pathname, vars) + vars_param = {'platbase': prefix, 'base': prefix} + pycode = 'import sysconfig; print(sysconfig.get_path("%s", vars=%s))' % (pathname, vars_param) else: args = 'plat_specific=%s, prefix="%s"' % (plat_specific, prefix) pycode = "import distutils.sysconfig; print(distutils.sysconfig.get_python_lib(%s))" % args @@ -198,15 +249,24 @@ def det_pylibdir(plat_specific=False, python_cmd=None): log.debug("Determining Python library directory using command '%s'", cmd) - out, ec = run_cmd(cmd, simple=False, force_in_dry_run=True, trace=False) - txt = out.strip().split('\n')[-1] + res = run_shell_cmd(cmd, in_dry_run=True, hidden=True) + txt = res.output.strip().split('\n')[-1] # value obtained should start with specified prefix, otherwise something is very wrong if not txt.startswith(prefix): raise EasyBuildError("Last line of output of %s does not start with specified prefix %s: %s (exit code %s)", - cmd, prefix, out, ec) + cmd, prefix, res.output, res.exit_code) pylibdir = txt[len(prefix):] + + # Ubuntu 24.04: the pylibdir has a leading local/, which causes issues later + # e.g. when symlinking /local/* to /* + # we can safely strip this to get a working installation + local = 'local/' + if pylibdir.startswith(local): + log.info("Removing leading /local from determined pylibdir: %s" % pylibdir) + pylibdir = pylibdir[len(local):] + log.debug("Determined pylibdir using '%s': %s", cmd, pylibdir) return pylibdir @@ -242,7 +302,8 @@ def det_pip_version(python_cmd='python'): log = fancylogger.getLogger('det_pip_version', fname=False) log.info("Determining pip version...") - out, _ = run_cmd("%s -m pip --version" % python_cmd, verbose=False, simple=False, trace=False) + res = run_shell_cmd("%s -m pip --version" % python_cmd, hidden=True) + out = res.output pip_version_regex = re.compile('^pip ([0-9.]+)') res = pip_version_regex.search(out) @@ -276,8 +337,8 @@ def det_py_install_scheme(python_cmd='python'): cmd = "%s -c 'import sysconfig; print(sysconfig.%s())'" % (python_cmd, get_default_scheme) log.debug("Determining active Python installation scheme with: %s", cmd) - out, _ = run_cmd(cmd, verbose=False, simple=False, trace=False) - py_install_scheme = out.strip() + res = run_shell_cmd(cmd, hidden=True) + py_install_scheme = res.output.strip() if py_install_scheme in PY_INSTALL_SCHEMES: log.info("Active Python installation scheme: %s", py_install_scheme) @@ -351,7 +412,10 @@ def extra_options(extra_vars=None): "Otherwise it will be used as-is. A value of None then skips the build step. " "The template %(python)s will be replace by the currently used Python binary.", CUSTOM], 'check_ldshared': [None, 'Check Python value of $LDSHARED, correct if needed to "$CC -shared"', CUSTOM], - 'download_dep_fail': [None, "Fail if downloaded dependencies are detected", CUSTOM], + 'download_dep_fail': [True, "Fail if downloaded dependencies are detected", CUSTOM], + 'fix_python_shebang_for': [['bin/*'], "List of files for which Python shebang should be fixed " + "to '#!/usr/bin/env python' (glob patterns supported) " + "(default: ['bin/*'])", CUSTOM], 'install_src': [None, "Source path to pass to the install command (e.g. a whl file)." "Defaults to '.' for unpacked sources or the first source file specified", CUSTOM], 'install_target': ['install', "Option to pass to setup.py", CUSTOM], @@ -364,8 +428,8 @@ def extra_options(extra_vars=None): 'req_py_minver': [None, "Required minor Python version (only relevant when using system Python)", CUSTOM], 'max_py_majver': [None, "Maximum major Python version (only relevant when using system Python)", CUSTOM], 'max_py_minver': [None, "Maximum minor Python version (only relevant when using system Python)", CUSTOM], - 'sanity_pip_check': [False, "Run 'python -m pip check' to ensure all required Python packages are " - "installed and check for any package with an invalid (0.0.0) version.", CUSTOM], + 'sanity_pip_check': [True, "Run 'python -m pip check' to ensure all required Python packages are " + "installed and check for any package with an invalid (0.0.0) version.", CUSTOM], 'runtest': [True, "Run unit tests.", CUSTOM], # overrides default 'testinstall': [False, "Install into temporary directory prior to running the tests.", CUSTOM], 'unpack_sources': [None, "Unpack sources prior to build/install. Defaults to 'True' except for whl files", @@ -374,7 +438,7 @@ def extra_options(extra_vars=None): # version. Those would fail the (extended) sanity_pip_check. So as a last resort they can be added here # and will be excluded from that check. Note that the display name is required, i.e. from `pip list`. 'unversioned_packages': [[], "List of packages that don't have a version at all, i.e. show 0.0.0", CUSTOM], - 'use_pip': [None, "Install using '%s'" % PIP_INSTALL_CMD, CUSTOM], + 'use_pip': [True, "Install using '%s'" % PIP_INSTALL_CMD, CUSTOM], 'use_pip_editable': [False, "Install using 'pip install --editable'", CUSTOM], # see https://packaging.python.org/tutorials/installing-packages/#installing-setuptools-extras 'use_pip_extras': [None, "String with comma-separated list of 'extras' to install via pip", CUSTOM], @@ -388,7 +452,7 @@ def extra_options(extra_vars=None): if 'source_urls' not in extra_vars: # Create a copy so the defaults are not modified by the following line src_urls = DEFAULT_CONFIG['source_urls'][:] - src_urls[0] = [url for name, url, _ in TEMPLATE_CONSTANTS if name == 'PYPI_SOURCE'] + src_urls[0] = [PYPI_SOURCE] extra_vars['source_urls'] = src_urls return ExtensionEasyBlock.extra_options(extra_vars=extra_vars) @@ -434,8 +498,6 @@ def __init__(self, *args, **kwargs): # figure out whether this Python package is being installed for multiple Python versions self.multi_python = 'Python' in self.cfg['multi_deps'] - # determine install command - self.use_setup_py = False self.determine_install_command() # avoid that pip (ab)uses $HOME/.cache/pip @@ -448,16 +510,23 @@ def __init__(self, *args, **kwargs): # Don't let pip connect to PYPI to check for a new version env.setvar('PIP_DISABLE_PIP_VERSION_CHECK', 'true') + # avoid that lib subdirs are appended to $*LIBRARY_PATH if they don't provide libraries + # typically, only lib/pythonX.Y/site-packages should be added to $PYTHONPATH (see make_module_extra) + self.module_load_environment.LD_LIBRARY_PATH.type = ModEnvVarType.PATH_WITH_TOP_FILES + self.module_load_environment.LIBRARY_PATH.type = ModEnvVarType.PATH_WITH_TOP_FILES + def determine_install_command(self): """ Determine install command to use. """ - if self.cfg.get('use_pip', False) or self.cfg.get('use_pip_editable', False): + self.py_installopts = [] + if self.cfg.get('use_pip', True) or self.cfg.get('use_pip_editable', False): + self.use_setup_py = False self.install_cmd = PIP_INSTALL_CMD pip_verbose = self.cfg.get('pip_verbose', None) if pip_verbose or (pip_verbose is None and build_option('debug')): - self.cfg.update('installopts', '--verbose') + self.py_installopts.append('--verbose') # don't auto-install dependencies with pip unless use_pip_for_deps=True # the default is use_pip_for_deps=False @@ -465,29 +534,30 @@ def determine_install_command(self): self.log.info("Using pip to also install the dependencies") else: self.log.info("Using pip with --no-deps option") - self.cfg.update('installopts', '--no-deps') + self.py_installopts.append('--no-deps') if self.cfg.get('pip_ignore_installed', True): # don't (try to) uninstall already availale versions of the package being installed - self.cfg.update('installopts', '--ignore-installed') + self.py_installopts.append('--ignore-installed') if self.cfg.get('zipped_egg', False): - self.cfg.update('installopts', '--egg') + self.py_installopts.append('--egg') pip_no_index = self.cfg.get('pip_no_index', None) - if pip_no_index or (pip_no_index is None and self.cfg.get('download_dep_fail')): - self.cfg.update('installopts', '--no-index') + if pip_no_index or (pip_no_index is None and self.cfg.get('download_dep_fail', True)): + self.py_installopts.append('--no-index') else: self.use_setup_py = True self.install_cmd = SETUP_PY_INSTALL_CMD - if self.cfg['install_target'] == EASY_INSTALL_TARGET: + install_target = self.cfg.get_ref('install_target') + if install_target == EASY_INSTALL_TARGET: self.install_cmd += " %(loc)s" - self.cfg.update('installopts', '--no-deps') + self.py_installopts.append('--no-deps') if self.cfg.get('zipped_egg', False): - if self.cfg['install_target'] == EASY_INSTALL_TARGET: - self.cfg.update('installopts', '--zip-ok') + if install_target == EASY_INSTALL_TARGET: + self.py_installopts.append('--zip-ok') else: raise EasyBuildError("Installing zipped eggs requires using easy_install or pip") @@ -502,55 +572,7 @@ def set_pylibdirs(self): def prepare_python(self): """Python-specific preparations.""" - # pick 'python' command to use - python = None - python_root = get_software_root('Python') - # keep in mind that Python may be listed as an allowed system dependency, - # so just checking Python root is not sufficient - if python_root: - bin_python = os.path.join(python_root, 'bin', 'python') - if os.path.exists(bin_python) and os.path.samefile(which('python'), bin_python): - # if Python is listed as a (build) dependency, use 'python' command provided that way - python = os.path.join(python_root, 'bin', 'python') - self.log.debug("Retaining 'python' command for Python dependency: %s", python) - - if python is None: - # if no Python version requirements are specified, - # use major/minor version of Python being used in this EasyBuild session - req_py_majver = self.cfg['req_py_majver'] - if req_py_majver is None: - req_py_majver = sys.version_info[0] - req_py_minver = self.cfg['req_py_minver'] - if req_py_minver is None: - req_py_minver = sys.version_info[1] - - # Get the max_py_majver and max_py_minver from the config - max_py_majver = self.cfg['max_py_majver'] - max_py_minver = self.cfg['max_py_minver'] - - # if using system Python, go hunting for a 'python' command that satisfies the requirements - python = pick_python_cmd(req_maj_ver=req_py_majver, req_min_ver=req_py_minver, - max_py_majver=max_py_majver, max_py_minver=max_py_minver) - - # Check if we have Python by now. If not, and if self.require_python, raise a sensible error - if python: - self.python_cmd = python - self.log.info("Python command being used: %s", self.python_cmd) - elif self.require_python: - if (req_py_majver is not None or req_py_minver is not None - or max_py_majver is not None or max_py_minver is not None): - raise EasyBuildError( - "Failed to pick Python command that satisfies requirements in the easyconfig " - "(req_py_majver = %s, req_py_minver = %s, max_py_majver = %s, max_py_minver = %s)", - req_py_majver, req_py_minver, max_py_majver, max_py_minver - ) - else: - raise EasyBuildError("Failed to pick Python command to use") - else: - self.log.warning("No Python command found!") - else: - self.python_cmd = python - self.log.info("Python command being used: %s", self.python_cmd) + self.python_cmd = find_python_cmd_from_ec(self.log, self.cfg, self.require_python) if self.python_cmd: # set Python lib directories @@ -650,7 +672,7 @@ def compose_install_command(self, prefix, extrapath=None, installopts=None): # since we provide all required dependencies already, we disable this via --no-build-isolation if LooseVersion(pip_version) >= LooseVersion('10.0'): if '--no-build-isolation' not in self.cfg['installopts']: - self.cfg.update('installopts', '--no-build-isolation') + self.py_installopts.append('--no-build-isolation') elif not self.dry_run: raise EasyBuildError("Failed to determine pip version!") @@ -664,7 +686,7 @@ def compose_install_command(self, prefix, extrapath=None, installopts=None): if self._should_unpack_source() or not self.src: # specify current directory loc = '.' - elif isinstance(self.src, string_type): + elif isinstance(self.src, str): # for extensions, self.src specifies the location of the source file loc = self.src else: @@ -677,7 +699,7 @@ def compose_install_command(self, prefix, extrapath=None, installopts=None): loc += '[%s]' % extras if installopts is None: - installopts = self.cfg['installopts'] + installopts = ' '.join([self.cfg['installopts']] + self.py_installopts) if self.cfg.get('use_pip_editable', False): # add --editable option when requested, in the right place (i.e. right before the location specification) @@ -721,9 +743,9 @@ def extract_step(self): if self._should_unpack_source(): super(PythonPackage, self).extract_step() - def prerun(self): + def pre_install_extension(self): """Prepare for installing Python package.""" - super(PythonPackage, self).prerun() + super(PythonPackage, self).pre_install_extension() self.prepare_python() def prepare_step(self, *args, **kwargs): @@ -796,11 +818,15 @@ def configure_step(self): # creates log entries for python being used, for debugging cmd = "%(python)s -V; %(python)s -c 'import sys; print(sys.executable, sys.path)'" - run_cmd(cmd % {'python': self.python_cmd}, verbose=False, trace=False) + run_shell_cmd(cmd % {'python': self.python_cmd}, hidden=True) def build_step(self): """Build Python package using setup.py""" + + # inject extra '%(python)s' template value before getting value of 'buildcmd' custom easyconfig parameter + self.cfg.template_values['python'] = self.python_cmd build_cmd = self.cfg['buildcmd'] + if self.use_setup_py: if get_software_root('CMake'): @@ -811,17 +837,16 @@ def build_step(self): if not build_cmd: build_cmd = 'build' # Default value for setup.py - build_cmd = '%(python)s setup.py ' + build_cmd + build_cmd = f"{self.python_cmd} setup.py {build_cmd}" if build_cmd: - build_cmd = build_cmd % {'python': self.python_cmd} cmd = ' '.join([self.cfg['prebuildopts'], build_cmd, self.cfg['buildopts']]) - (out, _) = run_cmd(cmd, log_all=True, simple=False) + res = run_shell_cmd(cmd) # keep track of all output, so we can check for auto-downloaded dependencies; # take into account that build/install steps may be run multiple times # We consider the build and install output together as downloads likely happen here if this is run - self.install_cmd_output += out + self.install_cmd_output += res.output def test_step(self, return_output_ec=False): """ @@ -830,7 +855,7 @@ def test_step(self, return_output_ec=False): :param return_output: return output and exit code of test command """ - if isinstance(self.cfg['runtest'], string_type): + if isinstance(self.cfg['runtest'], str): self.testcmd = self.cfg['runtest'] if self.cfg['runtest'] and self.testcmd is not None: @@ -851,6 +876,9 @@ def test_step(self, return_output_ec=False): actual_installdir = os.path.join(test_installdir, 'local') else: actual_installdir = test_installdir + # Export the temporary installdir as an environment variable + # Some tests (e.g. for astropy) require to be run in the installdir + env.setvar('EB_PYTHONPACKAGE_TEST_INSTALLDIR', actual_installdir) self.log.debug("Pre-creating subdirectories in %s: %s", actual_installdir, self.all_pylibdirs) for pylibdir in self.all_pylibdirs: @@ -859,13 +887,13 @@ def test_step(self, return_output_ec=False): raise EasyBuildError("Failed to create test install dir: %s", err) # print Python search path (just debugging purposes) - run_cmd("%s -c 'import sys; print(sys.path)'" % self.python_cmd, verbose=False, trace=False) + run_shell_cmd("%s -c 'import sys; print(sys.path)'" % self.python_cmd, hidden=True) abs_pylibdirs = [os.path.join(actual_installdir, pylibdir) for pylibdir in self.all_pylibdirs] extrapath = "export PYTHONPATH=%s &&" % os.pathsep.join(abs_pylibdirs + ['$PYTHONPATH']) cmd = self.compose_install_command(test_installdir, extrapath=extrapath) - run_cmd(cmd, log_all=True, simple=True, verbose=False) + run_shell_cmd(cmd) self.py_post_install_shenanigans(test_installdir) @@ -879,11 +907,12 @@ def test_step(self, return_output_ec=False): ]) if return_output_ec: - (out, ec) = run_cmd(cmd, log_all=False, log_ok=False, simple=False, regexp=False) - # need to log seperately, since log_all and log_ok need to be false to retrieve out and ec + res = run_shell_cmd(cmd, fail_on_error=False) + # need to retrieve ec by not failing on error + (out, ec) = (res.output, res.exit_code) self.log.info("cmd '%s' exited with exit code %s and output:\n%s", cmd, ec, out) else: - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) if test_installdir: remove_dir(test_installdir) @@ -921,12 +950,12 @@ def install_step(self): # actually install Python package cmd = self.compose_install_command(self.installdir) - (out, _) = run_cmd(cmd, log_all=True, log_ok=True, simple=False) + res = run_shell_cmd(cmd) # keep track of all output from install command, so we can check for auto-downloaded dependencies; # take into account that install step may be run multiple times # (for iterated installations over multiply Python versions) - self.install_cmd_output += out + self.install_cmd_output += res.output self.py_post_install_shenanigans(self.installdir) @@ -939,12 +968,12 @@ def install_step(self): if value is not None: env.setvar(name, value, verbose=False) - def run(self, *args, **kwargs): + def install_extension(self, *args, **kwargs): """Perform the actual Python package build/installation procedure""" # we unpack unless explicitly told otherwise kwargs.setdefault('unpack_src', self._should_unpack_source()) - super(PythonPackage, self).run(*args, **kwargs) + super(PythonPackage, self).install_extension(*args, **kwargs) # configure, build, test, install # See EasyBlock.get_steps @@ -1003,8 +1032,8 @@ def sanity_check_step(self, *args, **kwargs): # see also https://github.com/easybuilders/easybuild-easyblocks/issues/1877 env.setvar('PYTHONNOUSERSITE', '1', verbose=False) - if self.cfg.get('download_dep_fail', False): - self.log.info("Detection of downloaded dependencies enabled, checking output of installation command...") + if self.cfg.get('download_dep_fail', True): + self.log.info("Detection of downloaded depdenencies enabled, checking output of installation command...") patterns = [ 'Downloading .*/packages/.*', # setuptools r'Collecting .*', # pip @@ -1047,7 +1076,7 @@ def sanity_check_step(self, *args, **kwargs): exts_filter = (orig_exts_filter[0].replace('python', self.python_cmd), orig_exts_filter[1]) kwargs.update({'exts_filter': exts_filter}) - if self.cfg.get('sanity_pip_check', False): + if self.cfg.get('sanity_pip_check', True): pip_version = det_pip_version(python_cmd=python_cmd) if pip_version: @@ -1068,8 +1097,9 @@ def sanity_check_step(self, *args, **kwargs): pip_check_errors = [] - pip_check_msg, ec = run_cmd(pip_check_command, log_ok=False) - if ec: + res = run_shell_cmd(pip_check_command, fail_on_error=False) + pip_check_msg = res.output + if res.exit_code: pip_check_errors.append('`%s` failed:\n%s' % (pip_check_command, pip_check_msg)) else: self.log.info('`%s` completed successfully' % pip_check_command) @@ -1132,42 +1162,32 @@ def sanity_check_step(self, *args, **kwargs): return (parent_success and success, parent_fail_msg + fail_msg) - def make_module_req_guess(self): - """ - Define list of subdirectories to consider for updating path-like environment variables ($PATH, etc.). - """ - guesses = super(PythonPackage, self).make_module_req_guess() - - # avoid that lib subdirs are appended to $*LIBRARY_PATH if they don't provide libraries - # typically, only lib/pythonX.Y/site-packages should be added to $PYTHONPATH (see make_module_extra) - for envvar in ['LD_LIBRARY_PATH', 'LIBRARY_PATH']: - newlist = [] - for subdir in guesses[envvar]: - # only subdirectories that contain one or more files/libraries should be retained - fullpath = os.path.join(self.installdir, subdir) - if os.path.exists(fullpath): - if any([os.path.isfile(os.path.join(fullpath, x)) for x in os.listdir(fullpath)]): - newlist.append(subdir) - self.log.debug("Only retaining %s subdirs from %s for $%s (others don't provide any libraries)", - newlist, guesses[envvar], envvar) - guesses[envvar] = newlist - - return guesses - def make_module_extra(self, *args, **kwargs): """Add install path to PYTHONPATH""" txt = '' # update $EBPYTHONPREFIXES rather than $PYTHONPATH - # if this Python package was installed for multiple Python versions - if self.multi_python: - txt += self.module_generator.prepend_paths(EBPYTHONPREFIXES, '') + # if this Python package was installed for multiple Python versions, or if we prefer it; + # note: although EasyBuild framework also has logic for this in EasyBlock.make_module_extra, + # we retain full control here, since the logic is slightly different + use_ebpythonprefixes = False + runtime_deps = [dep['name'] for dep in self.cfg.dependencies(runtime_only=True)] + + if 'Python' in runtime_deps: + self.log.info("Found Python runtime dependency, so considering $EBPYTHONPREFIXES...") + if build_option('prefer_python_search_path') == EBPYTHONPREFIXES: + self.log.info("Preferred Python search path is $EBPYTHONPREFIXES, so using that") + use_ebpythonprefixes = True + + if self.multi_python or use_ebpythonprefixes: + path = '' # EBPYTHONPREFIXES are relative to the install dir + txt += self.module_generator.prepend_paths(EBPYTHONPREFIXES, path) elif self.require_python: self.set_pylibdirs() for path in self.all_pylibdirs: fullpath = os.path.join(self.installdir, path) # only extend $PYTHONPATH with existing, non-empty directories if os.path.exists(fullpath) and os.listdir(fullpath): - txt += self.module_generator.prepend_paths('PYTHONPATH', path) + txt += self.module_generator.prepend_paths(PYTHONPATH, path) return super(PythonPackage, self).make_module_extra(txt, *args, **kwargs) diff --git a/easybuild/easyblocks/generic/rpackage.py b/easybuild/easyblocks/generic/rpackage.py index 3c9fd700183..ee77d0555ec 100644 --- a/easybuild/easyblocks/generic/rpackage.py +++ b/easybuild/easyblocks/generic/rpackage.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -33,16 +33,17 @@ @author: Balazs Hajgato (Vrije Universiteit Brussel) """ import os +import pathlib import re from easybuild.easyblocks.r import EXTS_FILTER_R_PACKAGES, EB_R from easybuild.easyblocks.generic.configuremake import check_config_guess, obtain_config_guess from easybuild.framework.easyconfig import CUSTOM from easybuild.framework.extensioneasyblock import ExtensionEasyBlock -from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.build_log import EasyBuildError, print_warning from easybuild.tools.environment import setvar from easybuild.tools.filetools import mkdir, copy_file -from easybuild.tools.run import run_cmd, parse_log_for_error +from easybuild.tools.run import run_shell_cmd def make_R_install_option(opt, values, cmdline=False): @@ -89,6 +90,12 @@ def __init__(self, *args, **kwargs): self.ext_src = None self._required_deps = None + Renviron = pathlib.Path.home() / '.Renviron' + if Renviron.exists(): + msg = f".Renviron file detected ({Renviron}). This file may impact the building of R packages. " + msg += "If you did not expect this file to exist then you should remove it." + print_warning(msg) + def make_r_cmd(self, prefix=None): """Create a command to run in R to install an R package.""" confvars = "confvars" @@ -169,15 +176,17 @@ def build_step(self): def install_R_package(self, cmd, inp=None): """Install R package as specified, and check for errors.""" - output, _ = run_cmd(cmd, log_all=True, simple=False, inp=inp, regexp=False) - self.check_install_output(output) + res = run_shell_cmd(cmd, stdin=inp) + self.check_install_output(res.output) def check_install_output(self, output): """ Check output of installation command, and clean up installation if needed. """ - errors = parse_log_for_error(output, regExp="^ERROR:") + errors = re.findall(r"^ERROR:.*", output, flags=re.I | re.M) + if errors: + self.log.info("R package %s failed with error:\n%s", self.name, '\n'.join(errors)) self.handle_installation_errors() cmd = "R -q --no-save" stdin = """ @@ -185,7 +194,7 @@ def check_install_output(self, output): """ % self.name # remove package if errors were detected # it's possible that some of the dependencies failed, but the package itself was installed - run_cmd(cmd, log_all=False, log_ok=False, simple=False, inp=stdin, regexp=False) + run_shell_cmd(cmd, fail_on_error=False, stdin=stdin) raise EasyBuildError("Errors detected during installation of R package %s!", self.name) else: self.log.debug("R package %s installed succesfully", self.name) @@ -216,12 +225,12 @@ def required_deps(self): if self._required_deps is None: if self.src: - cmd = "tar --wildcards --extract --file %s --to-stdout '*/DESCRIPTION'" % self.src - out, _ = run_cmd(cmd, simple=False, trace=False) + cmd = "tar --wildcards --extract --file %s --to-stdout '*DESCRIPTION'" % self.src + res = run_shell_cmd(cmd, hidden=True) # lines that start with whitespace are merged with line above lines = [] - for line in out.splitlines(): + for line in res.output.splitlines(): if line and line[0] in (' ', '\t'): lines[-1] = lines[-1] + line else: @@ -277,8 +286,8 @@ def prepare_r_ext_install(self): # determine location if isinstance(self.master, EB_R): # extension is being installed as part of an R installation/module - (out, _) = run_cmd("R RHOME", log_all=True, simple=False, trace=False) - rhome = out.strip() + res = run_shell_cmd("R RHOME", hidden=True) + rhome = res.output.strip() lib_install_prefix = os.path.join(rhome, 'library') else: # extension is being installed in a separate installation prefix @@ -286,7 +295,7 @@ def prepare_r_ext_install(self): mkdir(lib_install_prefix, parents=True) if self.src: - super(RPackage, self).run(unpack_src=True) + super(RPackage, self).install_extension(unpack_src=True) self.ext_src = self.src self.update_config_guess(self.ext_dir) self.log.debug("Installing R package %s version %s." % (self.name, self.version)) @@ -299,19 +308,21 @@ def prepare_r_ext_install(self): return cmd, stdin - def run(self): + def install_extension(self): """ Install R package as an extension. """ cmd, stdin = self.prepare_r_ext_install() self.install_R_package(cmd, inp=stdin) - def run_async(self): + def install_extension_async(self, thread_pool): """ Start installation of R package as an extension asynchronously. """ cmd, stdin = self.prepare_r_ext_install() - self.async_cmd_start(cmd, inp=stdin) + task_id = f'ext_{self.name}_{self.version}' + return thread_pool.submit(run_shell_cmd, cmd, stdin=stdin, asynchronous=True, env=os.environ.copy(), + fail_on_error=False, task_id=task_id, work_dir=os.getcwd()) def async_cmd_check(self): """ diff --git a/easybuild/easyblocks/generic/rpm.py b/easybuild/easyblocks/generic/rpm.py index d23d4427e0b..baf6a8c2020 100644 --- a/easybuild/easyblocks/generic/rpm.py +++ b/easybuild/easyblocks/generic/rpm.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -46,7 +46,7 @@ from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import change_dir, mkdir, symlink, which -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd _log = fancylogger.getLogger('easyblocks.generic.rpm') @@ -88,7 +88,7 @@ def rebuild_rpm(rpm_path, targetdir): targetdir, rpm_path, ]) - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) class Rpm(Binary): @@ -105,6 +105,11 @@ def __init__(self, *args, **kwargs): self.rebuild_rpm = False + # Add common PATH/LD_LIBRARY_PATH paths found in RPMs to module load environment + self.module_load_environment.PATH = [os.path.join('usr', 'bin'), 'sbin', os.path.join('usr', 'sbin')] + self.module_load_environment.LD_LIBRARY_PATH = [os.path.join('usr', 'lib'), os.path.join('usr', 'lib64')] + self.module_load_environment.MANPATH = [os.path.join('usr', 'share', 'man')] + @staticmethod def extra_options(extra_vars=None): """Extra easyconfig parameters specific to RPMs.""" @@ -142,10 +147,10 @@ def configure_step(self): # determine whether RPMs need to be rebuilt to make relocation work cmd = "rpm --version" - (out, _) = run_cmd(cmd, log_all=True, simple=False) + res = run_shell_cmd(cmd) rpmver_re = re.compile(r"^RPM\s+version\s+(?P[0-9.]+).*") - res = rpmver_re.match(out) + res = rpmver_re.match(res.output) self.log.debug("RPM version found: %s" % res.group()) if res: @@ -185,7 +190,7 @@ def install_step(self): cmd = "rpm --initdb --dbpath /rpm --root %s" % self.installdir - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) force = '' if self.cfg['force']: @@ -219,7 +224,7 @@ def install_step(self): 'post': postinstall, 'installopts': self.cfg['installopts'], } - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) for path in self.cfg['makesymlinks']: # allow globs, always use first hit. @@ -232,16 +237,3 @@ def install_step(self): symlink(realdirs[0], os.path.join(self.installdir, os.path.basename(path))) else: self.log.debug("No match found for symlink glob %s." % path) - - def make_module_req_guess(self): - """Add common PATH/LD_LIBRARY_PATH paths found in RPMs to list of guesses.""" - - guesses = super(Rpm, self).make_module_req_guess() - - guesses.update({ - 'PATH': guesses.get('PATH', []) + ['usr/bin', 'sbin', 'usr/sbin'], - 'LD_LIBRARY_PATH': guesses.get('LD_LIBRARY_PATH', []) + ['usr/lib', 'usr/lib64'], - 'MANPATH': guesses.get('MANPATH', []) + ['usr/share/man'], - }) - - return guesses diff --git a/easybuild/easyblocks/generic/rubygem.py b/easybuild/easyblocks/generic/rubygem.py index ddfc9010cad..fc83ea20b4f 100644 --- a/easybuild/easyblocks/generic/rubygem.py +++ b/easybuild/easyblocks/generic/rubygem.py @@ -1,5 +1,5 @@ ## -# Copyright 2015-2024 Ghent University +# Copyright 2015-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -36,7 +36,7 @@ from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import copy_file from easybuild.tools.modules import get_software_root -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd class RubyGem(ExtensionEasyBlock): @@ -56,12 +56,12 @@ def __init__(self, *args, **kwargs): super(RubyGem, self).__init__(*args, **kwargs) self.ext_src = None - def run(self): + def install_extension(self): """Perform the actual Ruby gem build/install""" if not self.src: raise EasyBuildError("No source found for Ruby Gem %s, required for installation.", self.name) - super(RubyGem, self).run() + super(RubyGem, self).install_extension() self.ext_src = self.src self.log.debug("Installing Ruby gem %s version %s." % (self.name, self.version)) @@ -103,10 +103,10 @@ def build_step(self): gemspec = "%s.gemspec" % self.name gemspec_lower = "%s.gemspec" % self.name.lower() if os.path.exists(gemspec): - run_cmd("gem build %s -o %s.gem" % (gemspec, self.name)) + run_shell_cmd("gem build %s -o %s.gem" % (gemspec, self.name)) self.ext_src = "%s.gem" % self.name elif os.path.exists(gemspec_lower): - run_cmd("gem build %s -o %s.gem" % (gemspec_lower, self.name.lower())) + run_shell_cmd("gem build %s -o %s.gem" % (gemspec_lower, self.name.lower())) self.ext_src = "%s.gem" % self.name.lower() else: raise EasyBuildError("No gem_file specified and no" @@ -126,9 +126,13 @@ def install_step(self): if not self.is_extension or self.master.name != 'Ruby': env.setvar('GEM_HOME', self.installdir) - bindir = os.path.join(self.installdir, 'bin') - cmd = "%s gem install --bindir %s --local %s" % (self.cfg['preinstallopts'], bindir, self.ext_src) - run_cmd(cmd) + cmd = ' '.join([ + self.cfg['preinstallopts'], + 'gem install', + '--bindir ' + os.path.join(self.installdir, 'bin'), + '--local ' + self.ext_src, + ]) + run_shell_cmd(cmd) def make_module_extra(self): """Extend $GEM_PATH in module file.""" diff --git a/easybuild/easyblocks/generic/scons.py b/easybuild/easyblocks/generic/scons.py index 9e8b615cb03..88c55222dee 100644 --- a/easybuild/easyblocks/generic/scons.py +++ b/easybuild/easyblocks/generic/scons.py @@ -1,5 +1,5 @@ ## -# Copyright 2015-2024 Ghent University +# Copyright 2015-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -29,7 +29,7 @@ """ from easybuild.framework.easyblock import EasyBlock from easybuild.framework.easyconfig import CUSTOM -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd class SCons(EasyBlock): @@ -56,9 +56,7 @@ def build_step(self, verbose=False): Build with SCons """ - par = '' - if self.cfg['parallel']: - par = "-j %s" % self.cfg['parallel'] + par = f'-j {self.cfg.parallel}' if self.cfg.parallel > 1 else '' cmd = "%(prebuildopts)s scons %(par)s %(buildopts)s %(prefix)s" % { 'buildopts': self.cfg['buildopts'], @@ -66,9 +64,9 @@ def build_step(self, verbose=False): 'prefix': self.prefix, 'par': par, } - (out, _) = run_cmd(cmd, log_all=True, log_output=verbose) + res = run_shell_cmd(cmd) - return out + return res.output def test_step(self): """ @@ -76,7 +74,7 @@ def test_step(self): """ if self.cfg['runtest']: cmd = "%s scons %s %s" % (self.cfg['pretestopts'], self.cfg['runtest'], self.cfg['testopts']) - run_cmd(cmd, log_all=True) + run_shell_cmd(cmd) def install_step(self): """ @@ -87,6 +85,6 @@ def install_step(self): 'preinstallopts': self.cfg['preinstallopts'], 'prefix': self.prefix, } - (out, _) = run_cmd(cmd, log_all=True) + res = run_shell_cmd(cmd) - return out + return res.output diff --git a/easybuild/easyblocks/generic/systemcompiler.py b/easybuild/easyblocks/generic/systemcompiler.py index ff9db971e08..8151ce44f21 100644 --- a/easybuild/easyblocks/generic/systemcompiler.py +++ b/easybuild/easyblocks/generic/systemcompiler.py @@ -1,5 +1,5 @@ ## -# Copyright 2015-2024 Ghent University +# Copyright 2015-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -23,28 +23,34 @@ # along with EasyBuild. If not, see . ## """ -EasyBuild support for using (already installed/existing) system compiler instead of a full install via EasyBuild. +EasyBuild support for using (already installed/existing) system compiler +instead of a full install via EasyBuild. @author Bernd Mohr (Juelich Supercomputing Centre) @author Kenneth Hoste (Ghent University) @author Alan O'Cais (Juelich Supercomputing Centre) +@author Alex Domingo (Vrije Universiteit Brussel) """ import os import re -from easybuild.tools import LooseVersion from easybuild.base import fancylogger from easybuild.easyblocks.generic.bundle import Bundle -from easybuild.easyblocks.icc import EB_icc -from easybuild.easyblocks.ifort import EB_ifort -from easybuild.easyblocks.gcc import EB_GCC from easybuild.framework.easyconfig import CUSTOM +from easybuild.tools import LooseVersion from easybuild.tools.build_log import EasyBuildError, print_warning from easybuild.tools.filetools import read_file, resolve_path, which -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd _log = fancylogger.getLogger('easyblocks.generic.systemcompiler') +KNOWN_SYSTEM_COMPILERS = { + 'GCC': 'gcc', + 'GCCcore': 'gcc', + 'icc': 'icc', + 'ifort': 'ifort', +} + def extract_compiler_version(compiler_name): """Extract compiler version for provided compiler_name.""" @@ -54,7 +60,8 @@ def extract_compiler_version(compiler_name): # Intel(R) C Intel(R) 64 Compiler XE for applications running on Intel(R) 64, Version 15.0.1.133 Build 20141023 version_regex = re.compile(r'\s([0-9]+(?:\.[0-9]+){1,3})\s', re.M) if compiler_name == 'gcc': - out, _ = run_cmd("gcc --version", simple=False) + res = run_shell_cmd("gcc --version") + out = res.output res = version_regex.search(out) if res is None: raise EasyBuildError("Could not extract GCC version from %s", out) @@ -70,8 +77,8 @@ def extract_compiler_version(compiler_name): raise EasyBuildError("Compiler command '%s' not found", compiler_name) # Check what we have looks like a version number (the regex we use requires spaces around the version number) if version_regex.search(' ' + compiler_version + ' ') is None: - error_msg = "Derived Intel compiler version '%s' doesn't look correct, " % compiler_version - error_msg += "is compiler installed in a path like '.../composer_xe_2015.3.187/bin/intel64/icc'?" + error_msg = f"Derived Intel compiler version '{compiler_version}' doesn't look correct, " + error_msg += "is compiler installed in a path like '.../composer_xe_0000.0.000/bin/intel64/icc'?" raise EasyBuildError(error_msg) else: raise EasyBuildError("Unknown compiler %s", compiler_name) @@ -85,8 +92,7 @@ def extract_compiler_version(compiler_name): return compiler_version -# No need to inherit from EB_icc since EB_ifort already inherits from that -class SystemCompiler(Bundle, EB_GCC, EB_ifort): +class SystemCompiler(Bundle): """ Support for generating a module file for the system compiler with specified name. @@ -100,11 +106,8 @@ class SystemCompiler(Bundle, EB_GCC, EB_ifort): def extra_options(): """Add custom easyconfig parameters for SystemCompiler easyblock.""" # Gather extra_vars from inherited classes, order matters here to make this work without problems in __init__ - extra_vars = EB_GCC.extra_options() - extra_vars.update(EB_icc.extra_options()) - extra_vars.update(EB_ifort.extra_options()) - extra_vars.update(Bundle.extra_options()) - # Add an option to add all module path extensions to the resultant easyconfig + extra_vars = Bundle.extra_options() + # Add an option to add all module path extensions to the resulting easyconfig # This is useful if you are importing a compiler from a non-default path extra_vars.update({ 'generate_standalone_module': [ @@ -117,7 +120,11 @@ def extra_options(): def __init__(self, *args, **kwargs): """Extra initialization: keep track of values that may change due to modifications to the version.""" - super(SystemCompiler, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) + + # by default rely on Bundle easyblock, child SytemCompiler subclasses + # might define other easyblocks to handle standalone modules + self.compiler_class = Bundle # Keep track of original values of vars that are subject to change, for restoring later. # The version is determined/matched from the installation and the installdir is determined from the system @@ -127,28 +134,23 @@ def __init__(self, *args, **kwargs): def prepare_step(self, *args, **kwargs): """Do compiler appropriate prepare step, determine system compiler version and prefix.""" - if self.cfg['generate_standalone_module']: - if self.cfg['name'] in ['GCC', 'GCCcore']: - EB_GCC.prepare_step(self, *args, **kwargs) - elif self.cfg['name'] in ['icc']: - EB_icc.prepare_step(self, *args, **kwargs) - elif self.cfg['name'] in ['ifort']: - EB_ifort.prepare_step(self, *args, **kwargs) - else: - raise EasyBuildError("I don't know how to do the prepare_step for %s", self.cfg['name']) - else: - Bundle.prepare_step(self, *args, **kwargs) + # disable RPATH as SystemCompiler just generates wrapper modules + self.toolchain.options['rpath'] = False + + self.compiler_class.prepare_step(self, *args, **kwargs) # Determine compiler path (real path, with resolved symlinks) - compiler_name = self.cfg['name'].lower() - if compiler_name == 'gcccore': - compiler_name = 'gcc' + try: + compiler_name = KNOWN_SYSTEM_COMPILERS[self.cfg['name']] + except KeyError as err: + raise EasyBuildError(f"EasyConfig '{self.cfg['name']}' has no known system compilers") from err + path_to_compiler = which(compiler_name) if path_to_compiler: path_to_compiler = resolve_path(path_to_compiler) - self.log.info("Found path to compiler '%s' (with symlinks resolved): %s", compiler_name, path_to_compiler) + self.log.info(f"Found path to compiler '{compiler_name}' (with symlinks resolved): {path_to_compiler}") else: - raise EasyBuildError("%s not found in $PATH", compiler_name) + raise EasyBuildError(f"{compiler_name} not found in $PATH") # Determine compiler version self.compiler_version = extract_compiler_version(compiler_name) @@ -180,48 +182,32 @@ def prepare_step(self, *args, **kwargs): self.compiler_prefix = os.path.dirname(os.path.dirname(self.compiler_prefix)) else: - raise EasyBuildError("Unknown system compiler %s" % self.cfg['name']) + raise EasyBuildError(f"Unknown system compiler {self.cfg['name']}") if not os.path.exists(self.compiler_prefix): - raise EasyBuildError("Path derived for system compiler (%s) does not exist: %s!", - compiler_name, self.compiler_prefix) - self.log.debug("Derived version/install prefix for system compiler %s: %s, %s", - compiler_name, self.compiler_version, self.compiler_prefix) + raise EasyBuildError( + f"Prefix path derived for system compiler ({compiler_name}) does not exist: {self.compiler_prefix}!" + ) + self.log.debug( + f"Derived version/install prefix for system compiler {compiler_name}: " + f"{self.compiler_version}, {self.compiler_prefix}" + ) # If EasyConfig specified "real" version (not 'system' which means 'derive automatically'), check it if self.cfg['version'] == 'system': - self.log.info("Found specified version '%s', going with derived compiler version '%s'", - self.cfg['version'], self.compiler_version) + self.log.info( + f"Found requested version '{self.cfg['version']}', derived from compiler as '{self.compiler_version}'" + ) elif self.cfg['version'] != self.compiler_version: - raise EasyBuildError("Specified version (%s) does not match version reported by compiler (%s)" % - (self.cfg['version'], self.compiler_version)) + raise EasyBuildError( + f"Requested version ({self.cfg['version']}) does not match version " + f"reported by compiler ({self.compiler_version})" + ) def make_installdir(self, dontcreate=None): """Custom implementation of make installdir: do nothing, do not touch system compiler directories and files.""" pass - def make_module_req_guess(self): - """ - A dictionary of possible directories to look for. Return known dict for the system compiler, or empty dict if - generate_standalone_module parameter is False - """ - guesses = {} - if self.cfg['generate_standalone_module']: - if self.compiler_prefix in ['/usr', '/usr/local']: - # Force off adding paths to module since unloading such a module would be a potential shell killer - print_warning("Ignoring option 'generate_standalone_module' since installation prefix is %s", - self.compiler_prefix) - else: - if self.cfg['name'] in ['GCC', 'GCCcore']: - guesses = EB_GCC.make_module_req_guess(self) - elif self.cfg['name'] in ['icc']: - guesses = EB_icc.make_module_req_guess(self) - elif self.cfg['name'] in ['ifort']: - guesses = EB_ifort.make_module_req_guess(self) - else: - raise EasyBuildError("I don't know how to generate module var guesses for %s", self.cfg['name']) - return guesses - def make_module_step(self, fake=False): """ Custom module step for SystemCompiler: make 'EBROOT' and 'EBVERSION' reflect actual system compiler version @@ -232,12 +218,29 @@ def make_module_step(self, fake=False): self.installdir = self.compiler_prefix # Generate module - res = super(SystemCompiler, self).make_module_step(fake=fake) + module_generator_class = Bundle + if self.compiler_class is not None: + if self.compiler_prefix in ['/usr', '/usr/local']: + # reset module load environment for system compilers in default $PATHS + # as such a module would be a potential shell killer + print_warning( + "Ignoring option 'generate_standalone_module' since installation prefix is " + f"already in $PATH: {self.compiler_prefix}" + ) + module_vars = [str(env_var) for env_var in self.module_load_environment] + for env_var in module_vars: + self.module_load_environment.remove(env_var) + else: + # rely on compiler class module step to generate standalone module + module_generator_class = self.compiler_class + + module_path = module_generator_class.make_module_step(self, fake=fake) # Reset version and installdir to EasyBuild values self.installdir = self.orig_installdir self.cfg['version'] = self.orig_version - return res + + return module_path def make_module_extend_modpath(self): """ @@ -248,28 +251,18 @@ def make_module_extend_modpath(self): self.cfg['version'] = self.orig_version # Retrieve module path extensions - res = super(SystemCompiler, self).make_module_extend_modpath() + extended_modpath = self.compiler_class.make_module_extend_modpath(self) # Reset to actual compiler version (e.g., "4.8.2") self.cfg['version'] = self.compiler_version - return res + + return extended_modpath def make_module_extra(self, *args, **kwargs): """Add any additional module text.""" - if self.cfg['generate_standalone_module']: - if self.cfg['name'] in ['GCC', 'GCCcore']: - extras = EB_GCC.make_module_extra(self, *args, **kwargs) - elif self.cfg['name'] in ['icc']: - extras = EB_icc.make_module_extra(self, *args, **kwargs) - elif self.cfg['name'] in ['ifort']: - extras = EB_ifort.make_module_extra(self, *args, **kwargs) - else: - raise EasyBuildError("I don't know how to generate extra module text for %s", self.cfg['name']) - else: - extras = super(SystemCompiler, self).make_module_extra(*args, **kwargs) - return extras + return self.compiler_class.make_module_extra(self, *args, **kwargs) - def post_install_step(self, *args, **kwargs): + def post_processing_step(self, *args, **kwargs): """Do nothing.""" pass @@ -281,11 +274,11 @@ def permissions_step(self): """Do nothing.""" pass - def sanity_check_step(self, *args, **kwargs): + def sanity_check_step(self): """ Nothing is being installed, so just being able to load the (fake) module is sufficient """ - self.log.info("Testing loading of module '%s' by means of sanity check" % self.full_mod_name) + self.log.info(f"Testing loading of module '{self.full_mod_name}' by means of sanity check") fake_mod_data = self.load_fake_module(purge=True) self.log.debug("Cleaning up after testing loading of module") self.clean_up_fake_module(fake_mod_data) diff --git a/easybuild/easyblocks/generic/systemcompilergcc.py b/easybuild/easyblocks/generic/systemcompilergcc.py new file mode 100644 index 00000000000..ab3b5a33646 --- /dev/null +++ b/easybuild/easyblocks/generic/systemcompilergcc.py @@ -0,0 +1,61 @@ +## +# Copyright 2015-2024 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for using (already installed/existing) system compiler based +on GCC instead of a full install via EasyBuild. + +@author Bernd Mohr (Juelich Supercomputing Centre) +@author Kenneth Hoste (Ghent University) +@author Alan O'Cais (Juelich Supercomputing Centre) +@author Alex Domingo (Vrije Universiteit Brussel) +""" +from easybuild.easyblocks.gcc import EB_GCC +from easybuild.easyblocks.generic.systemcompiler import SystemCompiler + + +# order matters, SystemCompiler goes first to avoid recursion whenever EB_GCC calls super() +class SystemCompilerGCC(SystemCompiler, EB_GCC): + """ + Support for generating a module file for a system compiler based on GCC with specified name. + + The compiler is expected to be available in $PATH, required libraries are assumed to be readily available. + + Specifying 'system' as a version leads to using the derived compiler version in the generated module; + if an actual version is specified, it is checked against the derived version of the system compiler that was found. + """ + @staticmethod + def extra_options(): + """Add custom easyconfig parameters for SystemCompilerGCC easyblock.""" + extra_vars = EB_GCC.extra_options() + extra_vars.update(SystemCompiler.extra_options()) + return extra_vars + + def __init__(self, *args, **kwargs): + """Extra initialization: keep track of values that may change due to modifications to the version.""" + super().__init__(*args, **kwargs) + + # use GCC compiler class to generate standalone module + if self.cfg['generate_standalone_module']: + self.compiler_class = EB_GCC diff --git a/easybuild/easyblocks/generic/systemmpi.py b/easybuild/easyblocks/generic/systemmpi.py index 84cb89d7ebe..0a5348a7188 100644 --- a/easybuild/easyblocks/generic/systemmpi.py +++ b/easybuild/easyblocks/generic/systemmpi.py @@ -1,5 +1,5 @@ ## -# Copyright 2015-2024 Ghent University +# Copyright 2015-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -38,7 +38,7 @@ from easybuild.tools.build_log import EasyBuildError, print_warning from easybuild.tools.filetools import read_file, resolve_path, which from easybuild.tools.modules import get_software_version -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd class SystemMPI(Bundle, ConfigureMake, EB_impi): @@ -124,7 +124,8 @@ def prepare_step(self, *args, **kwargs): # Determine MPI version, installation prefix and underlying compiler if mpi_name in ('openmpi', 'spectrummpi'): # Spectrum MPI is based on Open MPI so is also covered by this logic - output_of_ompi_info, _ = run_cmd("ompi_info", simple=False) + res = run_shell_cmd("ompi_info") + output_of_ompi_info = res.output # Extract the version of the MPI implementation if mpi_name == 'spectrummpi': @@ -183,8 +184,9 @@ def prepare_step(self, *args, **kwargs): self.mpi_env_vars[key] = value # Extract the C compiler used underneath Intel MPI - compile_info, exit_code = run_cmd("%s -compile-info" % mpi_c_wrapper, simple=False) - if exit_code == 0: + res == run_shell_cmd("%s -compile-info" % mpi_c_wrapper) + compile_info = res.output + if res.exit_code == 0: self.mpi_c_compiler = compile_info.split(' ', 1)[0] else: raise EasyBuildError("Could not determine C compiler underneath Intel MPI, '%s -compiler-info' " @@ -219,29 +221,10 @@ def make_installdir(self, dontcreate=None): """Custom implementation of make installdir: do nothing, do not touch system MPI directories and files.""" pass - def post_install_step(self): + def post_processing_step(self): """Do nothing.""" pass - def make_module_req_guess(self): - """ - A dictionary of possible directories to look for. Return known dict for the system MPI. - """ - guesses = {} - if self.cfg['generate_standalone_module']: - if self.mpi_prefix in ['/usr', '/usr/local']: - # Force off adding paths to module since unloading such a module would be a potential shell killer - print_warning("Ignoring option 'generate_standalone_module' since installation prefix is %s", - self.mpi_prefix) - else: - if self.cfg['name'] in ['OpenMPI', 'SpectrumMPI']: - guesses = ConfigureMake.make_module_req_guess(self) - elif self.cfg['name'] in ['impi']: - guesses = EB_impi.make_module_req_guess(self) - else: - raise EasyBuildError("I don't know how to generate module var guesses for %s", self.cfg['name']) - return guesses - def make_module_step(self, fake=False): """ Custom module step for SystemMPI: make 'EBROOT' and 'EBVERSION' reflect actual system MPI version @@ -267,12 +250,34 @@ def make_module_step(self, fake=False): self.installdir = self.mpi_prefix # Generate module - res = super(SystemMPI, self).make_module_step(fake=fake) + module_generator_class = Bundle + if self.cfg['generate_standalone_module']: + if self.mpi_prefix in ['/usr', '/usr/local']: + # reset module load environment for system MPI in default $PATHS + # as such a module would be a potential shell killer + print_warning( + "Ignoring option 'generate_standalone_module' since installation prefix is " + f"already in $PATH: {self.mpi_prefix}" + ) + module_vars = [str(env_var) for env_var in self.module_load_environment] + for env_var in module_vars: + self.module_load_environment.remove(env_var) + else: + # determine system MPI easyblock module to generate standalone module + if self.cfg['name'] in ['OpenMPI', 'SpectrumMPI']: + module_generator_class = ConfigureMake + elif self.cfg['name'] in ['impi']: + module_generator_class = EB_impi + else: + raise EasyBuildError(f"Cannot generate standalone module for SystemMPI of {self.cfg['name']}") + + module_path = module_generator_class.make_module_step(self, fake=fake) # Reset version and installdir to EasyBuild values self.installdir = self.orig_installdir self.cfg['version'] = self.orig_version - return res + + return module_path def make_module_extend_modpath(self): """ diff --git a/easybuild/easyblocks/generic/tarball.py b/easybuild/easyblocks/generic/tarball.py index 1eef4528ae9..9930aadc28d 100644 --- a/easybuild/easyblocks/generic/tarball.py +++ b/easybuild/easyblocks/generic/tarball.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -41,7 +41,7 @@ from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import copy_dir, extract_file, remove_dir -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd class Tarball(ExtensionEasyBlock): @@ -72,7 +72,7 @@ def build_step(self): """ pass - def run(self, *args, **kwargs): + def install_extension(self, *args, **kwargs): """Install as extension: unpack sources and copy (via install step).""" if self.cfg['install_type'] is None: self.log.info("Auto-enabled install_type=merge because Tarball is being used to install an extension") @@ -94,7 +94,7 @@ def install_step(self, src=None): preinstall_cmd = '&& '.join([cmd for cmd in [preinstall_cmd, self.cfg['preinstall_cmd']] if cmd]) if preinstall_cmd: self.log.info("Preparing installation of %s using command '%s'..." % (self.name, preinstall_cmd)) - run_cmd(preinstall_cmd, log_all=True, simple=True) + run_shell_cmd(preinstall_cmd) # Copy source directory source_path = src or self.cfg['start_dir'] diff --git a/easybuild/easyblocks/generic/toolchain.py b/easybuild/easyblocks/generic/toolchain.py index 957971d03a9..ba4dfbefa1e 100644 --- a/easybuild/easyblocks/generic/toolchain.py +++ b/easybuild/easyblocks/generic/toolchain.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/generic/versionindependentpythonpackage.py b/easybuild/easyblocks/generic/versionindependentpythonpackage.py index 9c7dc3fc8be..d03c5d8b62b 100644 --- a/easybuild/easyblocks/generic/versionindependentpythonpackage.py +++ b/easybuild/easyblocks/generic/versionindependentpythonpackage.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2024 Ghent University +# Copyright 2013-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -37,7 +37,7 @@ import easybuild.tools.environment as env from easybuild.easyblocks.generic.pythonpackage import PythonPackage from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd class VersionIndependentPythonPackage(PythonPackage): @@ -72,14 +72,14 @@ def install_step(self): '--record %s' % os.path.join(self.builddir, 'record'), '--no-compile', ] - self.cfg.update('installopts', ' '.join(extra_installopts)) + self.py_installopts.extend(extra_installopts) else: # using easy_install or pip always results in installation that is specific to Python version eb_name = self.__class__.__name__ raise EasyBuildError("%s easyblock is not compatible with using easy_install or pip", eb_name) cmd = self.compose_install_command(self.installdir) - run_cmd(cmd, log_all=True, simple=True, log_output=True) + run_shell_cmd(cmd) # setuptools stubbornly replaces the shebang line in scripts with # the full path to the Python interpreter used to install; diff --git a/easybuild/easyblocks/generic/vscpythonpackage.py b/easybuild/easyblocks/generic/vscpythonpackage.py index ea2012878a1..fe9fbd6fe3c 100644 --- a/easybuild/easyblocks/generic/vscpythonpackage.py +++ b/easybuild/easyblocks/generic/vscpythonpackage.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2024 Ghent University +# Copyright 2013-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/generic/waf.py b/easybuild/easyblocks/generic/waf.py index 8cec720fd9a..bbfd7d3ea71 100644 --- a/easybuild/easyblocks/generic/waf.py +++ b/easybuild/easyblocks/generic/waf.py @@ -1,5 +1,5 @@ ## -# Copyright 2015-2024 Ghent University +# Copyright 2015-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -29,7 +29,7 @@ """ from easybuild.framework.easyblock import EasyBlock -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd class Waf(EasyBlock): @@ -48,9 +48,9 @@ def configure_step(self, cmd_prefix=''): '--prefix=%s' % self.installdir, self.cfg['configopts'], ]) - (out, _) = run_cmd(cmd, log_all=True, simple=False) + res = run_shell_cmd(cmd) - return out + return res.output def build_step(self, verbose=False, path=None): """ @@ -62,9 +62,9 @@ def build_step(self, verbose=False, path=None): 'build', self.cfg['buildopts'], ]) - (out, _) = run_cmd(cmd, log_all=True, simple=False) + res = run_shell_cmd(cmd) - return out + return res.output def install_step(self, verbose=False, path=None): """ @@ -76,6 +76,6 @@ def install_step(self, verbose=False, path=None): 'install', self.cfg['installopts'], ]) - (out, _) = run_cmd(cmd, log_all=True, simple=False) + res = run_shell_cmd(cmd) - return out + return res.output diff --git a/easybuild/easyblocks/h/hadoop.py b/easybuild/easyblocks/h/hadoop.py index 856def79fd8..b1162f828dd 100644 --- a/easybuild/easyblocks/h/hadoop.py +++ b/easybuild/easyblocks/h/hadoop.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -36,7 +36,7 @@ from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import copy_file from easybuild.tools.modules import get_software_root -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd from easybuild.tools.systemtools import get_shared_lib_ext @@ -64,9 +64,9 @@ def build_step(self): raise EasyBuildError("%s not found. Failing install" % native_lib) cmd += ' -Drequire.%s=true -D%s.prefix=%s' % (native_lib, native_lib, lib_root) - if self.cfg['parallel'] > 1: - cmd += " -T%d" % self.cfg['parallel'] - run_cmd(cmd, log_all=True, simple=True, log_ok=True) + if self.cfg.parallel > 1: + cmd += f" -T{self.cfg.parallel}" + run_shell_cmd(cmd) def install_step(self): """Custom install procedure for Hadoop: install-by-copy.""" @@ -76,7 +76,7 @@ def install_step(self): else: super(EB_Hadoop, self).install_step() - def post_install_step(self): + def post_processing_step(self): """After the install, copy the extra native libraries into place.""" for native_library, lib_path in self.cfg['extra_native_libs']: lib_root = get_software_root(native_library) @@ -111,14 +111,14 @@ def sanity_check_step(self): fake_mod_data = self.load_fake_module(purge=True) # exit code is ignored, since this cmd exits with 1 if not all native libraries were found cmd = "hadoop checknative -a" - out, _ = run_cmd(cmd, simple=False, log_all=False, log_ok=False) + res = run_shell_cmd(cmd, fail_on_error=False) self.clean_up_fake_module(fake_mod_data) not_found = [] installdir = os.path.realpath(self.installdir) lib_src = os.path.join(installdir, 'lib', 'native') for native_lib, _ in self.cfg['extra_native_libs']: - if not re.search(r'%s: *true *%s' % (native_lib, lib_src), out): + if not re.search(r'%s: *true *%s' % (native_lib, lib_src), res.output): not_found.append(native_lib) if not_found: raise EasyBuildError("%s not found by 'hadoop checknative -a'.", ', '.join(not_found)) diff --git a/easybuild/easyblocks/h/hdf5.py b/easybuild/easyblocks/h/hdf5.py index 5501a9db54c..313b3b4e5a7 100644 --- a/easybuild/easyblocks/h/hdf5.py +++ b/easybuild/easyblocks/h/hdf5.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -143,13 +143,6 @@ def sanity_check_step(self): } super(EB_HDF5, self).sanity_check_step(custom_paths=custom_paths) - def make_module_req_guess(self): - """Specify pkgconfig path for HDF5.""" - guesses = super(EB_HDF5, self).make_module_req_guess() - guesses.update({'PKG_CONFIG_PATH': [os.path.join('lib', 'pkgconfig')]}) - - return guesses - def make_module_extra(self): """Also define $HDF5_DIR to installation directory.""" txt = super(EB_HDF5, self).make_module_extra() diff --git a/easybuild/easyblocks/h/healpix.py b/easybuild/easyblocks/h/healpix.py deleted file mode 100644 index 89698368987..00000000000 --- a/easybuild/easyblocks/h/healpix.py +++ /dev/null @@ -1,176 +0,0 @@ -## -# Copyright 2009-2024 Ghent University -# -# This file is part of EasyBuild, -# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), -# with support of Ghent University (http://ugent.be/hpc), -# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), -# Flemish Research Foundation (FWO) (http://www.fwo.be/en) -# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). -# -# https://github.com/easybuilders/easybuild -# -# EasyBuild is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation v2. -# -# EasyBuild is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with EasyBuild. If not, see . -## -""" -EasyBuild support for building and installing HEALPix, implemented as an easyblock - -@author: Kenneth Hoste (HPC-UGent) -@author: Josef Dvoracek (Institute of Physics, Czech Academy of Sciences) -""" -import os -from easybuild.tools import LooseVersion - -import easybuild.tools.toolchain as toolchain -from easybuild.easyblocks.generic.configuremake import ConfigureMake -from easybuild.framework.easyconfig import CUSTOM -from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.modules import get_software_root -from easybuild.tools.run import run_cmd_qa -from easybuild.tools.systemtools import get_shared_lib_ext - - -class EB_HEALPix(ConfigureMake): - """Support for building/installing HEALPix.""" - - @staticmethod - def extra_options(): - """There 3 variants of GCC build""" - extra_vars = { - 'gcc_target': ['generic_gcc', "Target to use when using a GCC-based compiler toolchain", CUSTOM], - } - - return ConfigureMake.extra_options(extra_vars) - - def __init__(self, *args, **kwargs): - """Initialisation of custom class variables for HEALPix.""" - super(EB_HEALPix, self).__init__(*args, **kwargs) - - self.build_in_installdir = True - - # target: - # 1: basic_gcc - # 2: generic_gcc - # 3: linux_icc - # 4: optimized_gcc - # 5: osx - # 6: osx_icc - self.target_string = None - - comp_fam = self.toolchain.comp_family() - if comp_fam == toolchain.INTELCOMP: # @UndefinedVariable - self.target_string = 'linux_icc' - elif comp_fam in [toolchain.DUMMY, toolchain.SYSTEM, toolchain.GCC]: # @UndefinedVariable - - self.target_string = self.cfg['gcc_target'] - - if self.target_string not in ['basic_gcc', 'generic_gcc', 'optimized_gcc']: - raise EasyBuildError("Unknown GCC target specified: %s", self.target_string) - else: - raise EasyBuildError("Don't know how which C++ configuration to use for toolchain family '%s'", comp_fam) - - def extract_step(self): - """Extract sources.""" - # strip off 'Healpix_' part to avoid having everything in a subdirectory - self.cfg['unpack_options'] = "--strip-components=1" - super(EB_HEALPix, self).extract_step() - - def configure_step(self): - """Custom configuration procedure for HEALPix.""" - - cfitsio = get_software_root('CFITSIO') - if not cfitsio: - raise EasyBuildError("Failed to determine root for CFITSIO, module not loaded?") - - cmd = "./configure -L" - qa = { - r"Should I attempt to create these directories (Y\|n)?": 'Y', - "full name of cfitsio library (libcfitsio.a):": '', - r"Do you want this modification to be done (y\|N)?": 'y', - "enter suffix for directories ():": '', - # configure for C (2), Fortran (3), C++ (4), then exit (0) - "Enter your choice (configuration of packages can be done in any order):": ['2', '3', '4', '0'], - } - std_qa = { - r"C compiler you want to use \(\S*\):": os.environ['CC'], - r"enter name of your F90 compiler \(\S*\):": os.environ['F90'], - r"enter name of your C compiler \(\S*\):": os.environ['CC'], - r"options for C compiler \([^)]*\):": os.environ['CFLAGS'], - r"enter compilation/optimisation flags for C compiler \([^)]*\):": os.environ['CFLAGS'], - r"compilation flags for %s compiler \([^:]*\):" % os.environ['F90']: '', - r"enter optimisation flags for %s compiler \([^)]*\):" % os.environ['F90']: os.environ['F90FLAGS'], - r"location of cfitsio library \(\S*\):": os.path.join(cfitsio, 'lib'), - r"cfitsio header fitsio.h \(\S*\):": os.path.join(cfitsio, 'include'), - r"enter command for library archiving \([^)]*\):": '', - r"archive creation \(and indexing\) command \([^)]*\):": '', - r"A static library is produced by default. Do you also want a shared library.*": 'y', - r"Available configurations for C\+\+ compilation are:[\s\n\S]*" + - r"(?P[0-9]+): %s[\s\n\S]*Choose one number:" % self.target_string: '%(nr)s', - r"PGPLOT.[\s\n]*Do you want to enable this option \?[\s\n]*\([^)]*\) \(y\|N\)": 'N', - r"the parallel implementation[\s\n]*Enter choice.*": '1', - r"do you want the HEALPix/C library to include CFITSIO-related functions \? \(Y\|n\):": 'Y', - r"\(recommended if the Healpix-F90 library is to be linked to external codes\) \(Y\|n\):": 'Y', # PIC -> Y - r"Do you rather want a shared/dynamic library.*": 'n', # shared instead static? -> N - } - run_cmd_qa(cmd, qa, std_qa=std_qa, log_all=True, simple=True, log_ok=True) - - def build_step(self): - """Custom build procedure for HEALPix.""" - # disable parallel build - self.cfg['parallel'] = '1' - self.log.debug("Disabled parallel build") - super(EB_HEALPix, self).build_step() - - def install_step(self): - """No dedicated install procedure for HEALPix.""" - pass - - def make_module_extra(self): - """additional paths""" - txt = super(EB_HEALPix, self).make_module_extra() - paths = { - 'CPATH': 'include', - 'LIBRARY_PATH': 'lib', - 'PATH': 'bin', - } - for key in sorted(paths): - txt += self.module_generator.prepend_paths(key, os.path.join('src', 'cxx', self.target_string, paths[key])) - - return txt - - def sanity_check_step(self): - """Custom sanity check for HEALPix.""" - binaries = [os.path.join('bin', x) for x in ['alteralm', 'anafast', 'hotspot', 'map2gif', 'median_filter', - 'plmgen', 'sky_ng_sim', 'sky_ng_sim_bin', 'smoothing', - 'synfast', 'ud_grade']] - libraries = [os.path.join('lib', 'lib%s.a' % x) for x in ['chealpix', 'gif', 'healpix', 'hpxgif']] - - target_subdirs = ['bin', 'lib'] - if LooseVersion(self.version) >= LooseVersion('3'): - # 'include' subdir is only there for recent HEALPix versions - target_subdirs.append('include') - - custom_paths = { - 'files': binaries + libraries + [os.path.join('lib', 'libchealpix.%s' % get_shared_lib_ext())], - 'dirs': ['include'] + [os.path.join('src', 'cxx', self.target_string, x) for x in target_subdirs], - } - - custom_commands = [] - if LooseVersion(self.version) >= LooseVersion('3'): - custom_commands.extend([ - "cd %s && make test &> build/testresults.txt" % self.installdir, - 'test $(grep -c "test completed" %s/build/testresults.txt) -eq 4' % self.installdir, - 'test $(grep -c "normal completion" %s/build/testresults.txt) -eq 10' % self.installdir, - ]) - - super(EB_HEALPix, self).sanity_check_step(custom_paths=custom_paths, custom_commands=custom_commands) diff --git a/easybuild/easyblocks/h/hpcc.py b/easybuild/easyblocks/h/hpcc.py index c8137cc1435..9f2f75f08cb 100644 --- a/easybuild/easyblocks/h/hpcc.py +++ b/easybuild/easyblocks/h/hpcc.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/h/hpcg.py b/easybuild/easyblocks/h/hpcg.py index e634111bced..9d4ef99245a 100644 --- a/easybuild/easyblocks/h/hpcg.py +++ b/easybuild/easyblocks/h/hpcg.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -36,7 +36,7 @@ from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_option from easybuild.tools.filetools import mkdir -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd from easybuild.tools import LooseVersion @@ -53,14 +53,14 @@ def configure_step(self): arg = "MPI_GCC_OMP" else: arg = "../setup/Make.MPI_GCC_OMP" - run_cmd("../configure %s" % arg, log_all=True, simple=True, log_ok=True, path='obj') + run_shell_cmd("../configure %s" % arg, work_dir='obj') def build_step(self): """Run build in build subdirectory.""" cxx = os.environ['CXX'] cxxflags = os.environ['CXXFLAGS'] cmd = "make CXX='%s' CXXFLAGS='$(HPCG_DEFS) %s -DMPICH_IGNORE_CXX_SEEK'" % (cxx, cxxflags) - run_cmd(cmd, log_all=True, simple=True, log_ok=True, path='obj') + run_shell_cmd(cmd, work_dir='obj') def test_step(self): """Custom built-in test procedure for HPCG.""" @@ -75,7 +75,7 @@ def test_step(self): hpcg_mpi_cmd = self.toolchain.mpi_cmd_for("xhpcg", 2) # 2 threads per MPI process (4 threads in total) cmd = "PATH=%s:$PATH OMP_NUM_THREADS=2 %s" % (objbindir, hpcg_mpi_cmd) - run_cmd(cmd, simple=True, log_all=True, log_ok=True) + run_shell_cmd(cmd) # find log file, check for success success_regex = re.compile(r"Scaled Residual \[[0-9.e-]+\]") diff --git a/easybuild/easyblocks/h/hpl.py b/easybuild/easyblocks/h/hpl.py index 1f1b23e0dff..472656a322b 100644 --- a/easybuild/easyblocks/h/hpl.py +++ b/easybuild/easyblocks/h/hpl.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -37,7 +37,7 @@ from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import change_dir, copy_file, mkdir, remove_file, symlink -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd class EB_HPL(ConfigureMake): @@ -65,7 +65,7 @@ def configure_step(self, subdir=None): cmd = "/bin/bash make_generic" - run_cmd(cmd, log_all=True, simple=True, log_output=True) + run_shell_cmd(cmd) remove_file(makeincfile) symlink(os.path.join(setupdir, 'Make.UNKNOWN'), makeincfile) diff --git a/easybuild/easyblocks/h/hypre.py b/easybuild/easyblocks/h/hypre.py index 1d9615bd742..9e142d2de38 100644 --- a/easybuild/easyblocks/h/hypre.py +++ b/easybuild/easyblocks/h/hypre.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/i/icc.py b/easybuild/easyblocks/i/icc.py index 527921d0fa1..6482931c54b 100644 --- a/easybuild/easyblocks/i/icc.py +++ b/easybuild/easyblocks/i/icc.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -38,102 +38,146 @@ import re from easybuild.tools import LooseVersion -from easybuild.easyblocks.generic.intelbase import IntelBase, ACTIVATION_NAME_2012, COMP_ALL -from easybuild.easyblocks.generic.intelbase import LICENSE_FILE_NAME_2012 -from easybuild.easyblocks.t.tbb import get_tbb_gccprefix -from easybuild.tools.run import run_cmd +from easybuild.easyblocks.generic.intelbase import IntelBase, COMP_ALL +from easybuild.easyblocks.tbb import get_tbb_gccprefix +from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.modules import MODULE_LOAD_ENV_HEADERS +from easybuild.tools.run import run_shell_cmd from easybuild.tools.systemtools import get_shared_lib_ext def get_icc_version(): """Obtain icc version string via 'icc --version'.""" cmd = "icc --version" - (out, _) = run_cmd(cmd, log_all=True, simple=False) + res = run_shell_cmd(cmd) ver_re = re.compile(r"^icc \(ICC\) (?P[0-9.]+) [0-9]+$", re.M) - version = ver_re.search(out).group('version') + version = ver_re.search(res.output).group('version') return version class EB_icc(IntelBase): - """Support for installing icc - - - tested with 11.1.046 - - will fail for all older versions (due to newer silent installer) + """ + Support for installing icc + - minimum version suported: 2020.0 """ def __init__(self, *args, **kwargs): """Constructor, initialize class variables.""" super(EB_icc, self).__init__(*args, **kwargs) - self.debuggerpath = None - - self.comp_libs_subdir = None + self.comp_libs_subdir = "" # need to make sure version is an actual version # required because of support in SystemCompiler generic easyblock to specify 'system' as version, # which results in deriving the actual compiler version # comparing a non-version like 'system' with an actual version like '2016' fails with TypeError in Python 3.x - if re.match(r'^[0-9]+.*', self.version) and LooseVersion(self.version) >= LooseVersion('2016'): + if re.match(r'^[0-9]+.*', self.version): + + if LooseVersion(self.version) < LooseVersion('2020'): + raise EasyBuildError( + f"Version {self.version} of {self.name} is unsupported. Mininum supported version is 2020.0." + ) - self.comp_libs_subdir = os.path.join('compilers_and_libraries_%s' % self.version, 'linux') + self.comp_libs_subdir = os.path.join(f'compilers_and_libraries_{self.version}', 'linux') if self.cfg['components'] is None: # we need to use 'ALL' by default, # using 'DEFAULTS' results in key things not being installed (e.g. bin/icc) self.cfg['components'] = [COMP_ALL] - self.log.debug("Nothing specified for components, but required for version 2016, using %s instead", - self.cfg['components']) - - def install_step(self): - """ - Actual installation - - create silent cfg file - - execute command - """ - silent_cfg_names_map = None - - if LooseVersion(self.version) < LooseVersion('2013_sp1'): - # since icc v2013_sp1, silent.cfg has been slightly changed to be 'more standard' + self.log.debug( + f"Missing components specification, required for version {self.version}. " + f"Using {self.cfg['components']} instead." + ) - silent_cfg_names_map = { - 'activation_name': ACTIVATION_NAME_2012, - 'license_file_name': LICENSE_FILE_NAME_2012, - } + # define list of subdirectories in search paths of module load environment + # some of these paths only apply to certain versions, but that doesn't really matter + # existence of paths is checked by module generator before 'prepend-paths' statements are included - super(EB_icc, self).install_step(silent_cfg_names_map=silent_cfg_names_map) + # new directory layout since Intel Parallel Studio XE 2016 + # https://software.intel.com/en-us/articles/new-directory-layout-for-intel-parallel-studio-xe-2016 + # in recent Intel compiler distributions, the actual binaries are + # in deeper directories, and symlinked in top-level directories + # however, not all binaries are symlinked (e.g. mcpcom is not) + # we only need to include the deeper directories (same as compilervars.sh) + def comp_libs_subdir_paths(*subdir_paths): + """Utility method to prepend self.comp_libs_subdir to list of paths""" + try: + return [os.path.join(self.comp_libs_subdir, path) for path in subdir_paths] + except TypeError: + raise EasyBuildError(f"Cannot prepend {self.comp_libs_subdir} to {subdir_paths}: wrong data type") + + self.module_load_environment.PATH = comp_libs_subdir_paths( + 'bin/intel64', + 'ipp/bin/intel64', + 'mpi/intel64/bin', + 'tbb/bin/emt64', + 'tbb/bin/intel64', + ) + # in the end we set 'LIBRARY_PATH' equal to 'LD_LIBRARY_PATH' + self.module_load_environment.LD_LIBRARY_PATH = comp_libs_subdir_paths( + 'lib', + 'compiler/lib/intel64', + 'debugger/ipt/intel64/lib', + 'ipp/lib/intel64', + 'mkl/lib/intel64', + 'mpi/intel64', + f"tbb/lib/intel64/{get_tbb_gccprefix(os.path.join(self.installdir, 'tbb/lib/intel64'))}", + ) + # 'include' is deliberately omitted, including it causes problems, e.g. with complex.h and std::complex + # cfr. https://software.intel.com/en-us/forums/intel-c-compiler/topic/338378 + self.module_load_environment.set_alias_vars(MODULE_LOAD_ENV_HEADERS, comp_libs_subdir_paths( + 'daal/include', + 'ipp/include', + 'mkl/include', + 'tbb/include', + )) + self.module_load_environment.CLASSPATH = comp_libs_subdir_paths('daal/lib/daal.jar') + self.module_load_environment.DAALROOT = comp_libs_subdir_paths('daal') + self.module_load_environment.IPPROOT = comp_libs_subdir_paths('ipp') + self.module_load_environment.MANPATH = comp_libs_subdir_paths( + 'debugger/gdb/intel64/share/man', + 'man/common', + 'man/en_US', + 'share/man', + ) + self.module_load_environment.TBBROOT = comp_libs_subdir_paths('tbb') + + # Debugger requires INTEL_PYTHONHOME, which only allows for a single value + self.debuggerpath = f"debugger_{self.version.split('.')[0]}" + self.module_load_environment.LD_LIBRARY_PATH.extend([ + os.path.join(self.debuggerpath, 'libipt/intel64/lib'), + 'daal/lib/intel64_lin', # out of self.comp_libs_subdir to not break libipt library loading for gdb + ]) + self.module_load_environment.PATH.append( + os.path.join(self.debuggerpath, 'gdb', 'intel64', 'bin') + ) + + # 'lib/intel64' is deliberately listed last, so it gets precedence over subdirs + self.module_load_environment.LD_LIBRARY_PATH.extend(comp_libs_subdir_paths('lib/intel64')) + + self.module_load_environment.LIBRARY_PATH = self.module_load_environment.LD_LIBRARY_PATH def sanity_check_step(self): """Custom sanity check paths for icc.""" - binprefix = 'bin/intel64' - libprefix = 'lib/intel64' - if LooseVersion(self.version) >= LooseVersion('2011'): - if LooseVersion(self.version) <= LooseVersion('2011.3.174'): - binprefix = 'bin' - elif LooseVersion(self.version) >= LooseVersion('2013_sp1'): - binprefix = 'bin' - else: - libprefix = 'compiler/lib/intel64' - + binprefix = 'bin' binfiles = ['icc', 'icpc'] - if LooseVersion(self.version) < LooseVersion('2014'): - binfiles += ['idb'] - binaries = [os.path.join(binprefix, f) for f in binfiles] - libraries = [os.path.join(libprefix, 'lib%s' % lib) for lib in ['iomp5.a', 'iomp5.%s' % get_shared_lib_ext()]] - sanity_check_files = binaries + libraries - if LooseVersion(self.version) > LooseVersion('2015'): - sanity_check_files.append('include/omp.h') + + libprefix = 'lib/intel64' + libraries = [os.path.join(libprefix, f'lib{lib}') for lib in ['iomp5.a', f'iomp5.{get_shared_lib_ext()}']] + + headers = ['include/omp.h'] custom_paths = { - 'files': sanity_check_files, + 'files': binaries + libraries + headers, 'dirs': [], } # make very sure that expected 'compilers_and_libraries_/linux' subdir is there for recent versions, - # since we rely on it being there in make_module_req_guess + # since we rely on it being there in module_load_environment if self.comp_libs_subdir: custom_paths['dirs'].append(self.comp_libs_subdir) @@ -141,126 +185,33 @@ def sanity_check_step(self): super(EB_icc, self).sanity_check_step(custom_paths=custom_paths, custom_commands=custom_commands) - def make_module_req_guess(self): + def make_module_step(self, *args, **kwargs): """ - Additional paths to consider for prepend-paths statements in module file + Additional paths for module load environment that are conditional """ - prefix = None - - guesses = super(EB_icc, self).make_module_req_guess() - - # guesses per environment variables - # some of these paths only apply to certain versions, but that doesn't really matter - # existence of paths is checked by module generator before 'prepend-paths' statements are included - guesses.update({ - 'CLASSPATH': ['daal/lib/daal.jar'], - # 'include' is deliberately omitted, including it causes problems, e.g. with complex.h and std::complex - # cfr. https://software.intel.com/en-us/forums/intel-c-compiler/topic/338378 - 'CPATH': ['daal/include', 'ipp/include', 'mkl/include', 'tbb/include'], - 'DAALROOT': ['daal'], - 'IPPROOT': ['ipp'], - 'LD_LIBRARY_PATH': ['lib'], - 'MANPATH': ['debugger/gdb/intel64/share/man', 'man/common', 'man/en_US', 'share/man'], - 'PATH': [], - 'TBBROOT': ['tbb'], - }) - - if self.cfg['m32']: - # 32-bit toolchain - guesses['PATH'].extend(['bin/ia32', 'tbb/bin/ia32']) - # in the end we set 'LIBRARY_PATH' equal to 'LD_LIBRARY_PATH' - guesses['LD_LIBRARY_PATH'].append('lib/ia32') - - else: - # 64-bit toolkit - guesses['PATH'].extend([ - 'bin/intel64', - 'debugger/gdb/intel64/bin', - 'ipp/bin/intel64', - 'mpi/intel64/bin', - 'tbb/bin/emt64', - 'tbb/bin/intel64', - ]) - - # in the end we set 'LIBRARY_PATH' equal to 'LD_LIBRARY_PATH' - guesses['LD_LIBRARY_PATH'].extend([ - 'compiler/lib/intel64', - 'debugger/ipt/intel64/lib', - 'ipp/lib/intel64', - 'mkl/lib/intel64', - 'mpi/intel64', - 'tbb/lib/intel64/%s' % get_tbb_gccprefix(os.path.join(self.installdir, 'tbb/lib/intel64')), - ]) - - if LooseVersion(self.version) < LooseVersion('2016'): - prefix = 'composer_xe_%s' % self.version - # for some older versions, name of subdirectory is slightly different - if not os.path.isdir(os.path.join(self.installdir, prefix)): - cand_prefix = 'composerxe-%s' % self.version - if os.path.isdir(os.path.join(self.installdir, cand_prefix)): - prefix = cand_prefix - - # debugger is dependent on $INTEL_PYTHONHOME since version 2015 and newer - if LooseVersion(self.version) >= LooseVersion('2015'): - self.debuggerpath = os.path.join(prefix, 'debugger') - - else: - # new directory layout for Intel Parallel Studio XE 2016 - # https://software.intel.com/en-us/articles/new-directory-layout-for-intel-parallel-studio-xe-2016 - prefix = self.comp_libs_subdir - # Debugger requires INTEL_PYTHONHOME, which only allows for a single value - self.debuggerpath = 'debugger_%s' % self.version.split('.')[0] - - guesses['LD_LIBRARY_PATH'].extend([ - os.path.join(self.debuggerpath, 'libipt/intel64/lib'), - 'daal/lib/intel64_lin', - ]) - - # 'lib/intel64' is deliberately listed last, so it gets precedence over subdirs - guesses['LD_LIBRARY_PATH'].append('lib/intel64') - - guesses['LIBRARY_PATH'] = guesses['LD_LIBRARY_PATH'] - - # set debugger path - if self.debuggerpath: - guesses['PATH'].append(os.path.join(self.debuggerpath, 'gdb', 'intel64', 'bin')) - - # in recent Intel compiler distributions, the actual binaries are - # in deeper directories, and symlinked in top-level directories - # however, not all binaries are symlinked (e.g. mcpcom is not) - # we only need to include the deeper directories (same as compilervars.sh) - if prefix and os.path.isdir(os.path.join(self.installdir, prefix)): - for key, subdirs in guesses.items(): - guesses[key] = [os.path.join(prefix, subdir) for subdir in subdirs] - - # The for loop above breaks libipt library loading for gdb - this fixes that - guesses['LD_LIBRARY_PATH'].append('daal/lib/intel64_lin') - if self.debuggerpath: - guesses['LD_LIBRARY_PATH'].append(os.path.join(self.debuggerpath, 'libipt/intel64/lib')) - # only set $IDB_HOME if idb exists idb_home_subdir = 'bin/intel64' if os.path.isfile(os.path.join(self.installdir, idb_home_subdir, 'idb')): - guesses['IDB_HOME'] = [idb_home_subdir] + self.module_load_environment.IDB_HOME = [idb_home_subdir] - return guesses + return super().make_module_step(*args, **kwargs) def make_module_extra(self, *args, **kwargs): """Additional custom variables for icc: $INTEL_PYTHONHOME.""" txt = super(EB_icc, self).make_module_extra(*args, **kwargs) - if self.debuggerpath: - intel_pythonhome = os.path.join(self.installdir, self.debuggerpath, 'python', 'intel64') - if os.path.isdir(intel_pythonhome): - txt += self.module_generator.set_environment('INTEL_PYTHONHOME', intel_pythonhome) + intel_pythonhome = os.path.join(self.installdir, self.debuggerpath, 'python', 'intel64') + if os.path.isdir(intel_pythonhome): + txt += self.module_generator.set_environment('INTEL_PYTHONHOME', intel_pythonhome) # on Debian/Ubuntu, /usr/include/x86_64-linux-gnu needs to be included in $CPATH for icc - out, ec = run_cmd("gcc -print-multiarch", simple=False, log_all=False, log_ok=False) - multiarch_inc_subdir = out.strip() - if ec == 0 and multiarch_inc_subdir: + res = run_shell_cmd("gcc -print-multiarch", fail_on_error=False, hidden=True) + multiarch_inc_subdir = res.output.strip() + if res.exit_code == 0 and multiarch_inc_subdir: multiarch_inc_dir = os.path.join('/usr', 'include', multiarch_inc_subdir) - self.log.info("Adding multiarch include path %s to $CPATH in generated module file", multiarch_inc_dir) - # system location must be appended at the end, so use append_paths - txt += self.module_generator.append_paths('CPATH', [multiarch_inc_dir], allow_abs=True) + for envar in self.module_load_environment.alias_vars(MODULE_LOAD_ENV_HEADERS): + self.log.info(f"Adding multiarch include path '{multiarch_inc_dir}' to ${envar} in generated module") + # system location must be appended at the end, so use append_paths + txt += self.module_generator.append_paths(envar, [multiarch_inc_dir], allow_abs=True) return txt diff --git a/easybuild/easyblocks/i/iccifort.py b/easybuild/easyblocks/i/iccifort.py index 1072de00d7b..b9064ca459b 100644 --- a/easybuild/easyblocks/i/iccifort.py +++ b/easybuild/easyblocks/i/iccifort.py @@ -1,5 +1,5 @@ ## -# Copyright 2019-2024 Bart Oldeman, McGill University, Compute Canada +# Copyright 2019-2025 Bart Oldeman, McGill University, Compute Canada # # This file is triple-licensed under GPLv2 (see below), MIT, and # BSD three-clause licenses. @@ -34,6 +34,7 @@ import os from easybuild.easyblocks.icc import EB_icc from easybuild.easyblocks.ifort import EB_ifort +from easybuild.tools.modules import MODULE_LOAD_ENV_HEADERS class EB_iccifort(EB_ifort, EB_icc): @@ -41,6 +42,27 @@ class EB_iccifort(EB_ifort, EB_icc): Class that can be used to install iccifort """ + def __init__(self, *args, **kwargs): + """Constructor, initialize class variables.""" + super().__init__(*args, **kwargs) + + # Exclude 'compiler/include' for CPATH, including it causes problems, e.g. with complex.h and std::complex + # cfr. https://software.intel.com/en-us/forums/intel-c-compiler/topic/338378 + for envar in self.module_load_environment.alias(MODULE_LOAD_ENV_HEADERS): + envar.remove(os.path.join(self.comp_libs_subdir, 'compiler/include')) + + # remove entries from LIBRARY_PATH that icc and co already know about at compile time + # only do this for iccifort merged installations so that icc can still find ifort + # libraries and vice versa for split installations + if self.comp_libs_subdir: + excluded_library_paths = [os.path.join(self.comp_libs_subdir, path) for path in ( + 'compiler/lib/intel64', + 'lib', + 'lib/intel64', + )] + for excluded_path in excluded_library_paths: + self.module_load_environment.LIBRARY_PATH.remove(excluded_path) + def sanity_check_step(self): """Custom sanity check paths for iccifort.""" EB_icc.sanity_check_step(self) @@ -56,19 +78,3 @@ def make_module_extra(self): txt += self.module_generator.set_environment('EBVERSIONIFORT', self.version) return txt - - def make_module_req_guess(self): - # Use EB_icc because its make_module_req_guess deliberately omits 'include' for CPATH: - # including it causes problems, e.g. with complex.h and std::complex - # cfr. https://software.intel.com/en-us/forums/intel-c-compiler/topic/338378 - # whereas EB_ifort adds 'include' but that's only needed if icc and ifort are separate - guesses = EB_icc.make_module_req_guess(self) - - # remove entries from LIBRARY_PATH that icc and co already know about at compile time - # only do this for iccifort merged installations so that icc can still find ifort - # libraries and vice versa for split installations - if self.comp_libs_subdir is not None: - compiler_library_paths = [os.path.join(self.comp_libs_subdir, p) - for p in ('lib', 'compiler/lib/intel64', 'lib/ia32', 'lib/intel64')] - guesses['LIBRARY_PATH'] = [p for p in guesses['LIBRARY_PATH'] if p not in compiler_library_paths] - return guesses diff --git a/easybuild/easyblocks/i/ifort.py b/easybuild/easyblocks/i/ifort.py index 6497c1124fa..7479ff85849 100644 --- a/easybuild/easyblocks/i/ifort.py +++ b/easybuild/easyblocks/i/ifort.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -34,61 +34,58 @@ """ import os -from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.intelbase import IntelBase from easybuild.easyblocks.icc import EB_icc # @UnresolvedImport +from easybuild.tools import LooseVersion +from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.modules import MODULE_LOAD_ENV_HEADERS from easybuild.tools.systemtools import get_shared_lib_ext class EB_ifort(EB_icc, IntelBase): """ Class that can be used to install ifort - - tested with 11.1.046 - -- will fail for all older versions (due to newer silent installer) + - minimum version suported: 2020.0 + - will fail for all older versions (due to newer silent installer) """ + def __init__(self, *args, **kwargs): + """Constructor, initialize class variables.""" + super().__init__(*args, **kwargs) + + if LooseVersion(self.version) < LooseVersion('2020'): + raise EasyBuildError( + f"Version {self.version} of {self.name} is unsupported. Mininum supported version is 2020.0." + ) + + # define list of subdirectories in search paths of module load environment + # add additional paths to those of ICC only needed for separate ifort installations + for envar in self.module_load_environment.alias(MODULE_LOAD_ENV_HEADERS): + envar.append(os.path.join(self.comp_libs_subdir, 'compiler/include')) + def sanity_check_step(self): """Custom sanity check paths for ifort.""" shlib_ext = get_shared_lib_ext() - binprefix = 'bin/intel64' - libprefix = 'lib/intel64' - if LooseVersion(self.version) >= LooseVersion('2011'): - if LooseVersion(self.version) <= LooseVersion('2011.3.174'): - binprefix = 'bin' - elif LooseVersion(self.version) >= LooseVersion('2013_sp1'): - binprefix = 'bin' - else: - libprefix = 'compiler/lib/intel64' + binprefix = 'bin' + binfiles = ['ifort'] + binaries = [os.path.join(binprefix, f) for f in binfiles] - bins = ['ifort'] - if LooseVersion(self.version) < LooseVersion('2013'): - # idb is not shipped with ifort anymore in 2013.x versions (it is with icc though) - bins.append('idb') + libprefix = 'lib/intel64' + libfiles = [f'lib{lib}' for lib in ['ifcore.a', f'ifcore.{shlib_ext}', 'iomp5.a', f'iomp5.{shlib_ext}']] + libraries = [os.path.join(libprefix, f) for f in libfiles] - libs = ['lib%s' % lib for lib in ['ifcore.a', 'ifcore.%s' % shlib_ext, 'iomp5.a', 'iomp5.%s' % shlib_ext]] custom_paths = { - 'files': [os.path.join(binprefix, x) for x in bins] + [os.path.join(libprefix, lib) for lib in libs], + 'files': binaries + libraries, 'dirs': [], } # make very sure that expected 'compilers_and_libraries_/linux' subdir is there for recent versions, - # since we rely on it being there in make_module_req_guess + # since we rely on it being there in module_load_environment if self.comp_libs_subdir: custom_paths['dirs'].append(self.comp_libs_subdir) custom_commands = ["which ifort"] IntelBase.sanity_check_step(self, custom_paths=custom_paths, custom_commands=custom_commands) - - def make_module_req_guess(self): - """ - Additional paths to consider for prepend-paths statements in module file - """ - guesses = super(EB_ifort, self).make_module_req_guess() - if LooseVersion(self.version) >= LooseVersion('2016'): - # This enables the creation of fortran 2008 bindings in MPI - guesses['CPATH'].append('include') - - return guesses diff --git a/easybuild/easyblocks/i/imkl.py b/easybuild/easyblocks/i/imkl.py index 59ce1159a65..47cbcc96405 100644 --- a/easybuild/easyblocks/i/imkl.py +++ b/easybuild/easyblocks/i/imkl.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -43,20 +43,20 @@ import easybuild.tools.environment as env import easybuild.tools.toolchain as toolchain -from easybuild.easyblocks.generic.intelbase import IntelBase, ACTIVATION_NAME_2012, LICENSE_FILE_NAME_2012 +from easybuild.easyblocks.generic.intelbase import IntelBase from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_option from easybuild.tools.filetools import apply_regex_substitutions, change_dir, mkdir, move_file, remove_dir, write_file -from easybuild.tools.modules import get_software_root -from easybuild.tools.run import run_cmd +from easybuild.tools.modules import MODULE_LOAD_ENV_HEADERS, get_software_root +from easybuild.tools.run import run_shell_cmd from easybuild.tools.systemtools import get_shared_lib_ext class EB_imkl(IntelBase): """ Class that can be used to install mkl - - tested with 10.2.1.017 + - minimum version suported: 2020.x -- will fail for all older versions (due to newer silent installer) """ @@ -73,6 +73,10 @@ def extra_options(): def __init__(self, *args, **kwargs): """Constructor for imkl easyblock.""" super(EB_imkl, self).__init__(*args, **kwargs) + if LooseVersion(self.version) < LooseVersion('2020'): + raise EasyBuildError( + f"Version {self.version} of {self.name} is unsupported. Mininum supported version is 2020.0." + ) # make sure $MKLROOT isn't set, it's known to cause problems with the installation self.cfg.update('unwanted_env_vars', ['MKLROOT']) @@ -103,17 +107,12 @@ def mkl_basedir(self, path): def prepare_step(self, *args, **kwargs): """Prepare build environment.""" - if LooseVersion(self.version) >= LooseVersion('2017.2.174'): - kwargs['requires_runtime_license'] = False - super(EB_imkl, self).prepare_step(*args, **kwargs) - else: - super(EB_imkl, self).prepare_step(*args, **kwargs) + kwargs['requires_runtime_license'] = False + super(EB_imkl, self).prepare_step(*args, **kwargs) # build the mkl interfaces, if desired if self.cfg['interfaces']: - self.cdftlibs = ['fftw2x_cdft'] - if LooseVersion(self.version) >= LooseVersion('10.3'): - self.cdftlibs.append('fftw3x_cdft') + self.cdftlibs = ['fftw2x_cdft', 'fftw3x_cdft'] # check whether MPI_FAMILY constant is defined, so mpi_family() can be used if hasattr(self.toolchain, 'MPI_FAMILY') and self.toolchain.MPI_FAMILY is not None: mpi_spec_by_fam = { @@ -146,45 +145,22 @@ def install_step(self): - create silent cfg file - execute command """ - silent_cfg_names_map = None silent_cfg_extras = None - - if LooseVersion(self.version) < LooseVersion('11.1'): - # since imkl v11.1, silent.cfg has been slightly changed to be 'more standard' - - silent_cfg_names_map = { - 'activation_name': ACTIVATION_NAME_2012, - 'license_file_name': LICENSE_FILE_NAME_2012, - } - - if LooseVersion(self.version) >= LooseVersion('11.1') and self.install_components is None: + if self.install_components is None: silent_cfg_extras = { 'COMPONENTS': 'ALL', } - - super(EB_imkl, self).install_step( - silent_cfg_names_map=silent_cfg_names_map, - silent_cfg_extras=silent_cfg_extras) + super(EB_imkl, self).install_step(silent_cfg_extras=silent_cfg_extras) def build_mkl_fftw_interfaces(self, libdir): """Build the Intel MKL FFTW interfaces.""" - mkdir(libdir) - loosever = LooseVersion(self.version) - - if loosever >= LooseVersion('10.3'): - intsubdir = self.mkl_basedir - if loosever >= LooseVersion('2024'): - intsubdir = os.path.join(intsubdir, 'share', 'mkl') - intsubdir = os.path.join(intsubdir, 'interfaces') - inttarget = 'libintel64' - else: - intsubdir = 'interfaces' - if self.cfg['m32']: - inttarget = 'lib32' - else: - inttarget = 'libem64t' + intsubdir = self.mkl_basedir + if LooseVersion(self.version) >= LooseVersion('2024'): + intsubdir = os.path.join(intsubdir, 'share', 'mkl') + intsubdir = os.path.join(intsubdir, 'interfaces') + inttarget = 'libintel64' cmd = "make -f makefile %s" % inttarget @@ -259,12 +235,12 @@ def build_mkl_fftw_interfaces(self, libdir): buildopts.append('mpi=%s' % self.mpi_spec) precflags = [''] - if lib.startswith('fftw2x') and not self.cfg['m32']: + if lib.startswith('fftw2x'): # build both single and double precision variants precflags = ['PRECISION=MKL_DOUBLE', 'PRECISION=MKL_SINGLE'] intflags = [''] - if lib in self.cdftlibs and not self.cfg['m32']: + if lib in self.cdftlibs: # build both 32-bit and 64-bit interfaces intflags = ['interface=lp64', 'interface=ilp64'] @@ -298,8 +274,8 @@ def build_mkl_fftw_interfaces(self, libdir): self.log.info("Changed to interface %s directory %s", lib, intdir) fullcmd = "%s %s" % (cmd, ' '.join(buildopts + extraopts)) - res = run_cmd(fullcmd, log_all=True, simple=True) - if not res: + res = run_shell_cmd(fullcmd) + if res.exit_code: raise EasyBuildError("Building %s (flags: %s, fullcmd: %s) failed", lib, flags, fullcmd) for fn in os.listdir(tmpbuild): @@ -344,22 +320,22 @@ def build_mkl_flexiblas(self, flexiblasdir): 'intel_thread parallel=intel SYSTEM_LIBS="-lm -ldl -L%s"' % compilerdir]] for cmd in cmds: - res = run_cmd(cmd, log_all=True, simple=True) - if not res: + res = run_shell_cmd(cmd) + if res.exit_code: raise EasyBuildError("Building FlexiBLAS-compatible library (cmd: %s) failed", cmd) - def post_install_step(self): + def post_processing_step(self): """ Install group libraries and interfaces (if desired). """ - super(EB_imkl, self).post_install_step() + super(EB_imkl, self).post_processing_step() # extract examples examples_subdir = os.path.join(self.installdir, self.mkl_basedir, self.examples_subdir) if os.path.exists(examples_subdir): cwd = change_dir(examples_subdir) for examples_tarball in glob.glob('examples_*.tgz'): - run_cmd("tar xvzf %s -C ." % examples_tarball) + run_shell_cmd("tar xvzf %s -C ." % examples_tarball) change_dir(cwd) # reload the dependencies @@ -367,35 +343,16 @@ def post_install_step(self): shlib_ext = get_shared_lib_ext() - if self.cfg['m32']: - extra = { - 'libmkl.%s' % shlib_ext: 'GROUP (-lmkl_intel -lmkl_intel_thread -lmkl_core)', - 'libmkl_em64t.a': 'GROUP (libmkl_intel.a libmkl_intel_thread.a libmkl_core.a)', - 'libmkl_solver.a': 'GROUP (libmkl_solver.a)', - 'libmkl_scalapack.a': 'GROUP (libmkl_scalapack_core.a)', - 'libmkl_lapack.a': 'GROUP (libmkl_intel.a libmkl_intel_thread.a libmkl_core.a)', - 'libmkl_cdft.a': 'GROUP (libmkl_cdft_core.a)' - } - else: - extra = { - 'libmkl.%s' % shlib_ext: 'GROUP (-lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core)', - 'libmkl_em64t.a': 'GROUP (libmkl_intel_lp64.a libmkl_intel_thread.a libmkl_core.a)', - 'libmkl_solver.a': 'GROUP (libmkl_solver_lp64.a)', - 'libmkl_scalapack.a': 'GROUP (libmkl_scalapack_lp64.a)', - 'libmkl_lapack.a': 'GROUP (libmkl_intel_lp64.a libmkl_intel_thread.a libmkl_core.a)', - 'libmkl_cdft.a': 'GROUP (libmkl_cdft_core.a)' - } - - loosever = LooseVersion(self.version) - - if loosever >= LooseVersion('10.3'): - libsubdir = os.path.join(self.mkl_basedir, 'lib', 'intel64') - else: - if self.cfg['m32']: - libsubdir = os.path.join('lib', '32') - else: - libsubdir = os.path.join('lib', 'em64t') + extra = { + 'libmkl.%s' % shlib_ext: 'GROUP (-lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core)', + 'libmkl_em64t.a': 'GROUP (libmkl_intel_lp64.a libmkl_intel_thread.a libmkl_core.a)', + 'libmkl_solver.a': 'GROUP (libmkl_solver_lp64.a)', + 'libmkl_scalapack.a': 'GROUP (libmkl_scalapack_lp64.a)', + 'libmkl_lapack.a': 'GROUP (libmkl_intel_lp64.a libmkl_intel_thread.a libmkl_core.a)', + 'libmkl_cdft.a': 'GROUP (libmkl_cdft_core.a)' + } + libsubdir = os.path.join(self.mkl_basedir, 'lib', 'intel64') libdir = os.path.join(self.installdir, libsubdir) for fil, txt in extra.items(): dest = os.path.join(libdir, fil) @@ -425,28 +382,15 @@ def get_mkl_fftw_interface_libs(self): "don't know compiler suffix for FFTW libraries.") precs = ['_double', '_single'] - ver = LooseVersion(self.version) - if ver < LooseVersion('11'): - # no precision suffix in libfftw2 libs before imkl v11 - precs = [''] - fftw_vers = ['2x%s%s' % (x, prec) for x in ['c', 'f'] for prec in precs] + ['3xc', '3xf'] + fftw_vers = [f'2x{x}{prec}' for x in ['c', 'f'] for prec in precs] + ['3xc', '3xf'] + pics = ['', '_pic'] - libs = ['libfftw%s%s%s.a' % (fftwver, compsuff, pic) for fftwver in fftw_vers for pic in pics] + libs = [f'libfftw{fftwver}{compsuff}{pic}.a' for fftwver in fftw_vers for pic in pics] if self.cdftlibs: - fftw_cdft_vers = ['2x_cdft_DOUBLE'] - if not self.cfg['m32']: - fftw_cdft_vers.append('2x_cdft_SINGLE') - if ver >= LooseVersion('10.3'): - fftw_cdft_vers.append('3x_cdft') - if ver >= LooseVersion('11.0.2'): - bits = ['_lp64'] - if not self.cfg['m32']: - bits.append('_ilp64') - else: - # no bits suffix in cdft libs before imkl v11.0.2 - bits = [''] - libs += ['libfftw%s%s%s.a' % x for x in itertools.product(fftw_cdft_vers, bits, pics)] + fftw_cdft_vers = ['2x_cdft_DOUBLE', '2x_cdft_SINGLE', '3x_cdft'] + bits = ['_lp64', '_ilp64'] + libs += [f'libfftw{x[0]}{x[1]}{x[2]}.a' for x in itertools.product(fftw_cdft_vers, bits, pics)] return libs @@ -456,7 +400,6 @@ def sanity_check_step(self): mklfiles = None mkldirs = None - ver = LooseVersion(self.version) libs = ['libmkl_core.%s' % shlib_ext, 'libmkl_gnu_thread.%s' % shlib_ext, 'libmkl_intel_thread.%s' % shlib_ext, 'libmkl_sequential.%s' % shlib_ext] extralibs = ['libmkl_blacs_intelmpi_%(suff)s.' + shlib_ext, 'libmkl_scalapack_%(suff)s.' + shlib_ext] @@ -468,50 +411,21 @@ def sanity_check_step(self): libs += [os.path.join('flexiblas', 'libflexiblas_imkl_%s.so' % thread) for thread in ['gnu_thread', 'intel_thread', 'sequential']] - if ver >= LooseVersion('10.3') and self.cfg['m32']: - raise EasyBuildError("Sanity check for 32-bit not implemented yet for IMKL v%s (>= 10.3)", self.version) - - if ver >= LooseVersion('10.3'): - mkldirs = [ - os.path.join(self.mkl_basedir, 'bin'), - os.path.join(self.mkl_basedir, 'lib', 'intel64'), - os.path.join(self.mkl_basedir, 'include'), - ] - libs += [lib % {'suff': suff} for lib in extralibs for suff in ['lp64', 'ilp64']] - - mklfiles = [os.path.join(self.mkl_basedir, 'include', 'mkl.h')] - mklfiles.extend([os.path.join(self.mkl_basedir, 'lib', 'intel64', lib) for lib in libs]) + mkldirs = [ + os.path.join(self.mkl_basedir, 'bin'), + os.path.join(self.mkl_basedir, 'lib', 'intel64'), + os.path.join(self.mkl_basedir, 'include'), + ] + libs += [lib % {'suff': suff} for lib in extralibs for suff in ['lp64', 'ilp64']] - if ver >= LooseVersion('2021'): + mklfiles = [os.path.join(self.mkl_basedir, 'include', 'mkl.h')] + mklfiles.extend([os.path.join(self.mkl_basedir, 'lib', 'intel64', lib) for lib in libs]) + if LooseVersion(self.version) >= LooseVersion('2021'): mklfiles.append(os.path.join(self.mkl_basedir, 'lib', 'intel64', 'libmkl_core.%s' % shlib_ext)) - - elif ver >= LooseVersion('10.3'): - if ver < LooseVersion('11.3'): - mkldirs.append(os.path.join(self.mkl_basedir, 'bin', 'intel64')) - - mklfiles.append(os.path.join(self.mkl_basedir, 'lib', 'intel64', 'libmkl.%s' % shlib_ext)) - - if ver >= LooseVersion('10.3.4') and ver < LooseVersion('11.1'): - mkldirs += [os.path.join('compiler', 'lib', 'intel64')] - elif ver >= LooseVersion('2017.0.0'): - mkldirs += [os.path.join('lib', 'intel64_lin')] - else: - mkldirs += [os.path.join('lib', 'intel64')] - else: - if self.cfg['m32']: - lib_subdir = '32' - else: - lib_subdir = 'em64t' - libs += [lib % {'suff': suff} for lib in extralibs for suff in ['lp64', 'ilp64']] - - mklfiles = [ - os.path.join('lib', lib_subdir, 'libmkl.%s' % shlib_ext), - os.path.join('include', 'mkl.h'), - ] - mklfiles.extend([os.path.join('lib', lib_subdir, lib) for lib in libs]) - mkldirs = [os.path.join('lib', lib_subdir), os.path.join('include', lib_subdir), 'interfaces'] + mklfiles.append(os.path.join(self.mkl_basedir, 'lib', 'intel64', 'libmkl.%s' % shlib_ext)) + mkldirs += [os.path.join('lib', 'intel64_lin')] custom_paths = { 'files': mklfiles, @@ -520,76 +434,49 @@ def sanity_check_step(self): super(EB_imkl, self).sanity_check_step(custom_paths=custom_paths) - def make_module_req_guess(self): + def make_module_step(self, *args, **kwargs): """ - A dictionary of possible directories to look for + Set paths for module load environment based on the actual installation files """ - guesses = super(EB_imkl, self).make_module_req_guess() - - if LooseVersion(self.version) >= LooseVersion('10.3'): - if self.cfg['m32']: - raise EasyBuildError("32-bit not supported yet for IMKL v%s (>= 10.3)", self.version) - else: - if LooseVersion(self.version) >= LooseVersion('2021'): - compiler_subdir = os.path.join(self.get_versioned_subdir('compiler'), self.compiler_libdir) - pkg_config_path = [os.path.join(self.mkl_basedir, 'tools', 'pkgconfig'), - os.path.join(self.mkl_basedir, 'lib', 'pkgconfig')] - else: - compiler_subdir = os.path.join('lib', 'intel64') - pkg_config_path = [os.path.join(self.mkl_basedir, 'bin', 'pkgconfig')] - guesses['MANPATH'] = ['man', os.path.join('man', 'en_US')] - if LooseVersion(self.version) >= LooseVersion('11.0'): - if LooseVersion(self.version) >= LooseVersion('11.3'): - guesses['MIC_LD_LIBRARY_PATH'] = [ - os.path.join('lib', 'intel64_lin_mic'), - os.path.join(self.mkl_basedir, 'lib', 'mic'), - ] - elif LooseVersion(self.version) >= LooseVersion('11.1'): - guesses['MIC_LD_LIBRARY_PATH'] = [ - os.path.join('lib', 'mic'), - os.path.join(self.mkl_basedir, 'lib', 'mic'), - ] - else: - guesses['MIC_LD_LIBRARY_PATH'] = [ - os.path.join('compiler', 'lib', 'mic'), - os.path.join(self.mkl_basedir, 'lib', 'mic'), - ] - library_path = [ - compiler_subdir, - os.path.join(self.mkl_basedir, 'lib', 'intel64'), - ] - cpath = [ - os.path.join(self.mkl_basedir, 'include'), - os.path.join(self.mkl_basedir, 'include', 'fftw'), - ] - cmake_prefix_path = [self.mkl_basedir] - guesses.update({ - 'PATH': [], - 'LD_LIBRARY_PATH': library_path, - 'LIBRARY_PATH': library_path, - 'CPATH': cpath, - 'CMAKE_PREFIX_PATH': cmake_prefix_path, - 'PKG_CONFIG_PATH': pkg_config_path, - }) - if self.cfg['flexiblas']: - guesses['FLEXIBLAS_LIBRARY_PATH'] = os.path.join(library_path[1], 'flexiblas') + if LooseVersion(self.version) >= LooseVersion('2021'): + compiler_subdir = os.path.join(self.get_versioned_subdir('compiler'), self.compiler_libdir) + pkg_config_path = [ + os.path.join(self.mkl_basedir, 'tools', 'pkgconfig'), + os.path.join(self.mkl_basedir, 'lib', 'pkgconfig'), + ] else: - if self.cfg['m32']: - guesses.update({ - 'PATH': ['bin', 'bin/ia32', 'tbb/bin/ia32'], - 'LD_LIBRARY_PATH': ['lib', 'lib/32'], - 'LIBRARY_PATH': ['lib', 'lib/32'], - 'MANPATH': ['man', 'share/man', 'man/en_US'], - }) + compiler_subdir = os.path.join('lib', 'intel64') + pkg_config_path = [os.path.join(self.mkl_basedir, 'bin', 'pkgconfig')] + + self.module_load_environment.PATH = [] + self.module_load_environment.LD_LIBRARY_PATH = [ + compiler_subdir, + os.path.join(self.mkl_basedir, 'lib', 'intel64'), + ] + self.module_load_environment.LIBRARY_PATH = self.module_load_environment.LD_LIBRARY_PATH + self.module_load_environment.CMAKE_PREFIX_PATH = [self.mkl_basedir] + self.module_load_environment.PKG_CONFIG_PATH = pkg_config_path + + # include paths to headers (e.g. CPATH) + include_dirs = [ + os.path.join(self.mkl_basedir, 'include'), + os.path.join(self.mkl_basedir, 'include', 'fftw'), + ] + self.module_load_environment.set_alias_vars(MODULE_LOAD_ENV_HEADERS, include_dirs) + + if LooseVersion(self.version) < LooseVersion('2021'): + self.module_load_environment.MANPATH = ['man', os.path.join('man', 'en_US')] + self.module_load_environment.MIC_LD_LIBRARY_PATH = [ + os.path.join('lib', 'intel64_lin_mic'), + os.path.join(self.mkl_basedir, 'lib', 'mic'), + ] - else: - guesses.update({ - 'PATH': ['bin', 'bin/intel64', 'tbb/bin/em64t'], - 'LD_LIBRARY_PATH': ['lib', 'lib/em64t'], - 'LIBRARY_PATH': ['lib', 'lib/em64t'], - 'MANPATH': ['man', 'share/man', 'man/en_US'], - }) - return guesses + if self.cfg['flexiblas']: + self.module_load_environment.FLEXIBLAS_LIBRARY_PATH = os.path.join( + self.mkl_basedir, 'lib', 'intel64', 'flexiblas' + ) + + return super().make_module_step(*args, **kwargs) def make_module_extra(self): """Overwritten from Application to add extra txt""" diff --git a/easybuild/easyblocks/i/imkl_fftw.py b/easybuild/easyblocks/i/imkl_fftw.py index 18cd02246be..fdc8b5fe5b1 100644 --- a/easybuild/easyblocks/i/imkl_fftw.py +++ b/easybuild/easyblocks/i/imkl_fftw.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -54,19 +54,21 @@ def install_step(self): self.mkl_basedir = os.getenv('MKLROOT') self.build_mkl_fftw_interfaces(os.path.join(self.installdir, 'lib')) - def make_module_req_guess(self): - """Custom guesses for imkl-FFTW module file""" - # bypass custom paths for imkl, only use standard library location - return super(EB_imkl, self).make_module_req_guess() + def make_module_step(self, *args, **kwargs): + """ + Custom paths of imkl are unnecessary as imkl-FFTW only ships libraries under the 'lib' subdir + Use generic make_module_step skipping imkl + """ + return super(EB_imkl, self).make_module_step(*args, **kwargs) def make_module_extra(self): """Custom extra variables to set in module file""" # bypass extra module variables for imkl return super(EB_imkl, self).make_module_extra() - def post_install_step(self): + def post_processing_step(self): """Custom post install step for imkl-FFTW""" - # bypass post_install_step of imkl easyblock + # bypass post_processing_step of imkl easyblock pass def sanity_check_step(self): diff --git a/easybuild/easyblocks/i/imod.py b/easybuild/easyblocks/i/imod.py deleted file mode 100644 index cda5c3a70fd..00000000000 --- a/easybuild/easyblocks/i/imod.py +++ /dev/null @@ -1,95 +0,0 @@ -## -# Copyright 2013-2024 Ghent University -# -# This file is part of EasyBuild, -# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), -# with support of Ghent University (http://ugent.be/hpc), -# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), -# Flemish Research Foundation (FWO) (http://www.fwo.be/en) -# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). -# -# https://github.com/easybuilders/easybuild -# -# EasyBuild is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation v2. -# -# EasyBuild is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with EasyBuild. If not, see . -## -""" -EasyBuild support for building and installing IMOD, implemented as an easyblock - -@author: Benjamin Roberts (Landcare Research NZ Ltd) -""" -import os -import shutil - -from easybuild.easyblocks.generic.binary import Binary -from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.filetools import remove_dir -from easybuild.tools.run import run_cmd - - -class EB_IMOD(Binary): - """Support for building/installing IMOD.""" - - def install_step(self): - """Install IMOD using install script.""" - - # -dir: Choose location of installation directory - # -skip: do not attempt to deploy resource files in /etc - # -yes: do not prompt for confirmation - script = '{0}_{1}{2}.csh'.format(self.name.lower(), self.version, self.cfg['versionsuffix']) - cmd = "tcsh {0} -dir {1} -script {1} -skip -yes".format(script, self.installdir) - run_cmd(cmd, log_all=True, simple=True) - - # The assumption by the install script is that installdir will be something - # like /usr/local. So it creates, within the specified install location, a - # number of additional directories within which to install IMOD. We will, - # therefore, move the contents of these directories up and throw away the - # directories themselves. Doing so apparently is not a problem so long as - # IMOD_DIR is correctly set in the module. - link_to_remove = os.path.join(self.installdir, self.name) - dir_to_remove = os.path.join(self.installdir, "{0}_{1}".format(self.name.lower(), self.version)) - try: - for entry in os.listdir(dir_to_remove): - shutil.move(os.path.join(dir_to_remove, entry), self.installdir) - if os.path.realpath(link_to_remove) != os.path.realpath(dir_to_remove): - raise EasyBuildError("Something went wrong: %s doesn't point to %s", link_to_remove, dir_to_remove) - remove_dir(dir_to_remove) - os.remove(link_to_remove) - except OSError as err: - raise EasyBuildError("Failed to clean up install dir: %s", err) - - def sanity_check_step(self): - """Custom sanity check for IMOD.""" - custom_paths = { - 'files': ['bin/imod', 'IMOD-linux.sh', 'IMOD-linux.csh', 'installIMOD'], - 'dirs': ['lib'], - } - super(EB_IMOD, self).sanity_check_step(custom_paths=custom_paths) - - def make_module_extra(self): - """Define IMOD specific variables in generated module file.""" - txt = super(EB_IMOD, self).make_module_extra() - txt += self.module_generator.set_environment('IMOD_DIR', self.installdir) - txt += self.module_generator.set_environment('IMOD_PLUGIN_DIR', - os.path.join(self.installdir, 'lib', 'imodplug')) - txt += self.module_generator.set_environment('IMOD_QTLIBDIR', os.path.join(self.installdir, 'qtlib')) - if os.getenv('JAVA_HOME') is None: - raise EasyBuildError("$JAVA_HOME is not defined for some reason -- check environment") - else: - txt += self.module_generator.set_environment('IMOD_JAVADIR', os.getenv('JAVA_HOME')) - txt += self.module_generator.set_environment('FOR_DISABLE_STACK_TRACE', '1') - txt += self.module_generator.set_alias('subm', "submfg $* &") - txt += self.module_generator.msg_on_load("Please set the environment variable $IMOD_CALIB_DIR if appropriate.") - - txt += self.module_generator.msg_on_load("bash users run: 'source $EBROOTIMOD/IMOD-linux.sh") - txt += self.module_generator.msg_on_load("csh users run: 'source $EBROOTIMOD/IMOD-linux.csh'") - return txt diff --git a/easybuild/easyblocks/i/impi.py b/easybuild/easyblocks/i/impi.py index 42919b4a459..d210b48e758 100644 --- a/easybuild/easyblocks/i/impi.py +++ b/easybuild/easyblocks/i/impi.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -38,13 +38,13 @@ from easybuild.tools import LooseVersion import easybuild.tools.toolchain as toolchain -from easybuild.easyblocks.generic.intelbase import IntelBase, ACTIVATION_NAME_2012, LICENSE_FILE_NAME_2012 +from easybuild.easyblocks.generic.intelbase import IntelBase from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_option -from easybuild.tools.filetools import apply_regex_substitutions, change_dir, extract_file, mkdir, write_file -from easybuild.tools.modules import get_software_root, get_software_version -from easybuild.tools.run import run_cmd +from easybuild.tools.filetools import apply_regex_substitutions, change_dir, extract_file +from easybuild.tools.modules import MODULE_LOAD_ENV_HEADERS, get_software_root, get_software_version +from easybuild.tools.run import run_shell_cmd from easybuild.tools.systemtools import get_shared_lib_ext from easybuild.tools.toolchain.mpi import get_mpi_cmd_template @@ -52,6 +52,7 @@ class EB_impi(IntelBase): """ Support for installing Intel MPI library + - minimum version suported: 2018.x """ @staticmethod def extra_options(): @@ -67,11 +68,8 @@ def extra_options(): return IntelBase.extra_options(extra_vars) def prepare_step(self, *args, **kwargs): - if LooseVersion(self.version) >= LooseVersion('2017.2.174'): - kwargs['requires_runtime_license'] = False - super(EB_impi, self).prepare_step(*args, **kwargs) - else: - super(EB_impi, self).prepare_step(*args, **kwargs) + kwargs['requires_runtime_license'] = False + super(EB_impi, self).prepare_step(*args, **kwargs) def install_step(self): """ @@ -81,59 +79,19 @@ def install_step(self): """ impiver = LooseVersion(self.version) + if impiver < LooseVersion('2018'): + raise EasyBuildError( + f"Version {self.version} of {self.name} is unsupported. Mininum supported version is 2018.0." + ) + if impiver >= LooseVersion('2021'): super(EB_impi, self).install_step() - - elif impiver >= LooseVersion('4.0.1'): + else: # impi starting from version 4.0.1.x uses standard installation procedure. - silent_cfg_names_map = {} - - if impiver < LooseVersion('4.1.1'): - # since impi v4.1.1, silent.cfg has been slightly changed to be 'more standard' - silent_cfg_names_map.update({ - 'activation_name': ACTIVATION_NAME_2012, - 'license_file_name': LICENSE_FILE_NAME_2012, - }) - super(EB_impi, self).install_step(silent_cfg_names_map=silent_cfg_names_map) - - # impi v4.1.1 and v5.0.1 installers create impi/ subdir, so stuff needs to be moved afterwards - if impiver == LooseVersion('4.1.1.036') or impiver >= LooseVersion('5.0.1.035'): - super(EB_impi, self).move_after_install() - else: - # impi up until version 4.0.0.x uses custom installation procedure. - silent = """[mpi] -INSTALLDIR=%(ins)s -LICENSEPATH=%(lic)s -INSTALLMODE=NONRPM -INSTALLUSER=NONROOT -UPDATE_LD_SO_CONF=NO -PROCEED_WITHOUT_PYTHON=yes -AUTOMOUNTED_CLUSTER=yes -EULA=accept -[mpi-rt] -INSTALLDIR=%(ins)s -LICENSEPATH=%(lic)s -INSTALLMODE=NONRPM -INSTALLUSER=NONROOT -UPDATE_LD_SO_CONF=NO -PROCEED_WITHOUT_PYTHON=yes -AUTOMOUNTED_CLUSTER=yes -EULA=accept - -""" % {'lic': self.license_file, 'ins': self.installdir} - - # already in correct directory - silentcfg = os.path.join(os.getcwd(), "silent.cfg") - write_file(silentcfg, silent) - self.log.debug("Contents of %s: %s", silentcfg, silent) - - tmpdir = os.path.join(os.getcwd(), self.version, 'mytmpdir') - mkdir(tmpdir, parents=True) - - cmd = "./install.sh --tmp-dir=%s --silent=%s" % (tmpdir, silentcfg) - run_cmd(cmd, log_all=True, simple=True) + # since v5.0.1 installers create impi/ subdir, so stuff needs to be moved afterwards + super(EB_impi, self).move_after_install() # recompile libfabric (if requested) # some Intel MPI versions (like 2019 update 6) no longer ship libfabric sources @@ -148,36 +106,32 @@ def install_step(self): libfabric_installpath = os.path.join(self.installdir, 'intel64', 'libfabric') make = 'make' - if self.cfg['parallel']: - make += ' -j %d' % self.cfg['parallel'] + if self.cfg.parallel > 1: + make += f' -j {self.cfg.parallel}' cmds = [ - './configure --prefix=%s %s' % (libfabric_installpath, self.cfg['libfabric_configopts']), + f"./configure --prefix={libfabric_installpath} {self.cfg['libfabric_configopts']}", make, - 'make install' + "make install", ] for cmd in cmds: - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) else: self.log.info("Rebuild of libfabric is requested, but %s does not exist, so skipping...", libfabric_src_tgz_fn) else: raise EasyBuildError("Rebuild of libfabric is requested, but ofi_internal is set to False.") - def post_install_step(self): + def post_processing_step(self): """Custom post install step for IMPI, fix broken env scripts after moving installed files.""" - super(EB_impi, self).post_install_step() + super(EB_impi, self).post_processing_step() impiver = LooseVersion(self.version) if impiver >= LooseVersion('2021'): self.log.info("No post-install action for impi v%s", self.version) - - elif impiver == LooseVersion('4.1.1.036') or impiver >= LooseVersion('5.0.1.035'): - if impiver >= LooseVersion('2018.0.128'): - script_paths = [os.path.join('intel64', 'bin')] - else: - script_paths = [os.path.join('intel64', 'bin'), os.path.join('mic', 'bin')] + else: + script_paths = [os.path.join('intel64', 'bin')] # fix broken env scripts after the move regex_subs = [(r"^setenv I_MPI_ROOT.*", r"setenv I_MPI_ROOT %s" % self.installdir)] for script in [os.path.join(script_path, 'mpivars.csh') for script_path in script_paths]: @@ -201,12 +155,8 @@ def sanity_check_step(self): impi_ver = LooseVersion(self.version) suff = '64' - if self.cfg['m32']: - suff = '' - mpi_mods = ['mpi.mod'] - if impi_ver > LooseVersion('4.0'): - mpi_mods.extend(['mpi_base.mod', 'mpi_constants.mod', 'mpi_sizeofs.mod']) + mpi_mods = ['mpi.mod', 'mpi_base.mod', 'mpi_constants.mod', 'mpi_sizeofs.mod'] if impi_ver >= LooseVersion('2021'): mpi_subdir = self.get_versioned_subdir('mpi') @@ -215,7 +165,6 @@ def sanity_check_step(self): lib_dir = os.path.join(mpi_subdir, 'lib') if impi_ver < LooseVersion('2021.11'): lib_dir = os.path.join(lib_dir, 'release') - elif impi_ver >= LooseVersion('2019'): bin_dir = os.path.join('intel64', 'bin') include_dir = os.path.join('intel64', 'include') @@ -243,108 +192,108 @@ def sanity_check_step(self): custom_commands = [] if build_option('mpi_tests'): - if impi_ver >= LooseVersion('2017'): - # Add minimal test program to sanity checks - if build_option('sanity_check_only'): - # When only running the sanity check we need to manually make sure that - # variables for compilers and parallelism have been set - self.set_parallel() - self.prepare_step(start_dir=False) - - impi_testexe = os.path.join(tempfile.mkdtemp(), 'mpi_test') - else: - impi_testexe = os.path.join(self.builddir, 'mpi_test') + # Add minimal test program to sanity checks + if build_option('sanity_check_only'): + # When only running the sanity check we need to manually make sure that + # variables for compilers and parallelism have been set + self.set_parallel() + self.prepare_step(start_dir=False) + + impi_testexe = os.path.join(tempfile.mkdtemp(), 'mpi_test') + else: + impi_testexe = os.path.join(self.builddir, 'mpi_test') - if impi_ver >= LooseVersion('2021'): - impi_testsrc = os.path.join(self.installdir, self.get_versioned_subdir('mpi')) - if impi_ver >= LooseVersion('2021.11'): - impi_testsrc = os.path.join(impi_testsrc, 'opt', 'mpi') - impi_testsrc = os.path.join(impi_testsrc, 'test', 'test.c') - else: - impi_testsrc = os.path.join(self.installdir, 'test', 'test.c') + if impi_ver >= LooseVersion('2021'): + impi_testsrc = os.path.join(self.installdir, self.get_versioned_subdir('mpi')) + if impi_ver >= LooseVersion('2021.11'): + impi_testsrc = os.path.join(impi_testsrc, 'opt', 'mpi') + impi_testsrc = os.path.join(impi_testsrc, 'test', 'test.c') + else: + impi_testsrc = os.path.join(self.installdir, 'test', 'test.c') - self.log.info("Adding minimal MPI test program to sanity checks: %s", impi_testsrc) + self.log.info("Adding minimal MPI test program to sanity checks: %s", impi_testsrc) - # Build test program with appropriate compiler from current toolchain - build_cmd = "mpicc -cc=%s %s -o %s" % (os.getenv('CC'), impi_testsrc, impi_testexe) + # Build test program with appropriate compiler from current toolchain + build_cmd = "mpicc -cc=%s %s -o %s" % (os.getenv('CC'), impi_testsrc, impi_testexe) - # Execute test program with appropriate MPI executable for target toolchain - params = {'nr_ranks': self.cfg['parallel'], 'cmd': impi_testexe} - mpi_cmd_tmpl, params = get_mpi_cmd_template(toolchain.INTELMPI, params, mpi_version=self.version) + # Execute test program with appropriate MPI executable for target toolchain + params = {'nr_ranks': self.cfg.parallel, 'cmd': impi_testexe} + mpi_cmd_tmpl, params = get_mpi_cmd_template(toolchain.INTELMPI, params, mpi_version=self.version) - custom_commands.extend([ - build_cmd, # build test program - mpi_cmd_tmpl % params, # run test program - ]) + custom_commands.extend([ + build_cmd, # build test program + mpi_cmd_tmpl % params, # run test program + ]) super(EB_impi, self).sanity_check_step(custom_paths=custom_paths, custom_commands=custom_commands) - def make_module_req_guess(self): + def make_module_step(self, *args, **kwargs): """ - A dictionary of possible directories to look for + Set paths for module load environment based on the actual installation files """ - guesses = super(EB_impi, self).make_module_req_guess() - if self.cfg['m32']: - lib_dirs = ['lib', 'lib/ia32', 'ia32/lib'] - guesses.update({ - 'PATH': ['bin', 'bin/ia32', 'ia32/bin'], - 'LD_LIBRARY_PATH': lib_dirs, - 'LIBRARY_PATH': lib_dirs, - 'MIC_LD_LIBRARY_PATH': ['mic/lib'], - }) - else: - manpath = 'man' + manpath = 'man' + fi_provider_path = None + mic_library_path = None - impi_ver = LooseVersion(self.version) - if impi_ver >= LooseVersion('2021'): - mpi_subdir = self.get_versioned_subdir('mpi') - lib_dirs = [ - os.path.join(mpi_subdir, 'lib'), - os.path.join(mpi_subdir, 'libfabric', 'lib'), - ] - if impi_ver < LooseVersion('2021.11'): - lib_dirs.insert(1, os.path.join(mpi_subdir, 'lib', 'release')) - include_dirs = [os.path.join(mpi_subdir, 'include')] - path_dirs = [ - os.path.join(mpi_subdir, 'bin'), - os.path.join(mpi_subdir, 'libfabric', 'bin'), - ] - if impi_ver >= LooseVersion('2021.11'): - manpath = os.path.join(mpi_subdir, 'share', 'man') - else: - manpath = os.path.join(mpi_subdir, 'man') - - if self.cfg['ofi_internal']: - libfabric_dir = os.path.join(mpi_subdir, 'libfabric') - lib_dirs.append(os.path.join(libfabric_dir, 'lib')) - path_dirs.append(os.path.join(libfabric_dir, 'bin')) - guesses['FI_PROVIDER_PATH'] = [os.path.join(libfabric_dir, 'lib', 'prov')] - - elif impi_ver >= LooseVersion('2019'): - # The "release" library is default in v2019. Give it precedence over intel64/lib. - # (remember paths are *prepended*, so the last path in the list has highest priority) - lib_dirs = [os.path.join('intel64', x) for x in ['lib', os.path.join('lib', 'release')]] - include_dirs = [os.path.join('intel64', 'include')] - path_dirs = [os.path.join('intel64', 'bin')] - if self.cfg['ofi_internal']: - lib_dirs.append(os.path.join('intel64', 'libfabric', 'lib')) - path_dirs.append(os.path.join('intel64', 'libfabric', 'bin')) - guesses['FI_PROVIDER_PATH'] = [os.path.join('intel64', 'libfabric', 'lib', 'prov')] + impi_ver = LooseVersion(self.version) + if impi_ver >= LooseVersion('2021'): + mpi_subdir = self.get_versioned_subdir('mpi') + path_dirs = [ + os.path.join(mpi_subdir, 'bin'), + os.path.join(mpi_subdir, 'libfabric', 'bin'), + ] + lib_dirs = [ + os.path.join(mpi_subdir, 'lib'), + os.path.join(mpi_subdir, 'libfabric', 'lib'), + ] + if impi_ver < LooseVersion('2021.11'): + lib_dirs.insert(1, os.path.join(mpi_subdir, 'lib', 'release')) + include_dirs = [os.path.join(mpi_subdir, 'include')] + + if impi_ver >= LooseVersion('2021.11'): + manpath = os.path.join(mpi_subdir, 'share', 'man') else: - lib_dirs = [os.path.join('lib', 'em64t'), 'lib64'] - include_dirs = ['include64'] - path_dirs = [os.path.join('bin', 'intel64'), 'bin64'] - guesses['MIC_LD_LIBRARY_PATH'] = [os.path.join('mic', 'lib')] - - guesses.update({ - 'PATH': path_dirs, - 'LD_LIBRARY_PATH': lib_dirs, - 'LIBRARY_PATH': lib_dirs, - 'MANPATH': [manpath], - 'CPATH': include_dirs, - }) - - return guesses + manpath = os.path.join(mpi_subdir, 'man') + + if self.cfg['ofi_internal']: + lib_dirs.append(os.path.join(mpi_subdir, 'libfabric', 'lib')) + path_dirs.append(os.path.join(mpi_subdir, 'libfabric', 'bin')) + fi_provider_path = [os.path.join(mpi_subdir, 'libfabric', 'lib', 'prov')] + + elif impi_ver >= LooseVersion('2019'): + path_dirs = [os.path.join('intel64', 'bin')] + # The "release" library is default in v2019. Give it precedence over intel64/lib. + # (remember paths are *prepended*, so the last path in the list has highest priority) + lib_dirs = [ + os.path.join('intel64', 'lib'), + os.path.join('intel64', 'lib', 'release'), + ] + include_dirs = [os.path.join('intel64', 'include')] + + if self.cfg['ofi_internal']: + lib_dirs.append(os.path.join('intel64', 'libfabric', 'lib')) + path_dirs.append(os.path.join('intel64', 'libfabric', 'bin')) + fi_provider_path = [os.path.join('intel64', 'libfabric', 'lib', 'prov')] + + else: + path_dirs = [os.path.join('bin', 'intel64'), 'bin64'] + lib_dirs = [os.path.join('lib', 'em64t'), 'lib64'] + include_dirs = ['include64'] + mic_library_path = [os.path.join('mic', 'lib')] + + self.module_load_environment.PATH = path_dirs + self.module_load_environment.LD_LIBRARY_PATH = lib_dirs + self.module_load_environment.LIBRARY_PATH = lib_dirs + self.module_load_environment.MANPATH = [manpath] + if fi_provider_path is not None: + self.module_load_environment.FI_PROVIDER_PATH = fi_provider_path + if mic_library_path is not None: + self.module_load_environment.MIC_LD_LIBRARY_PATH = mic_library_path + + # include paths to headers (e.g. CPATH) + self.module_load_environment.set_alias_vars(MODULE_LOAD_ENV_HEADERS, include_dirs) + + return super().make_module_step(*args, **kwargs) def make_module_extra(self, *args, **kwargs): """Overwritten from Application to add extra txt""" diff --git a/easybuild/easyblocks/i/inspector.py b/easybuild/easyblocks/i/inspector.py index 7d7ddbf062b..65403ce0aef 100644 --- a/easybuild/easyblocks/i/inspector.py +++ b/easybuild/easyblocks/i/inspector.py @@ -1,5 +1,5 @@ # # -# Copyright 2013-2024 Ghent University +# Copyright 2013-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -31,52 +31,40 @@ import os from easybuild.tools import LooseVersion -from easybuild.easyblocks.generic.intelbase import IntelBase, ACTIVATION_NAME_2012, LICENSE_FILE_NAME_2012 +from easybuild.easyblocks.generic.intelbase import IntelBase +from easybuild.tools.build_log import EasyBuildError class EB_Inspector(IntelBase): """ Support for installing Intel Inspector + - minimum version suported: 2020.x """ def __init__(self, *args, **kwargs): """Easyblock constructor; define class variables.""" super(EB_Inspector, self).__init__(*args, **kwargs) + loosever = LooseVersion(self.version) + if loosever < LooseVersion('2020'): + raise EasyBuildError( + f"Version {self.version} of {self.name} is unsupported. Mininum supported version is 2020.0." + ) + # recent versions of Inspector are installed to a subdirectory self.subdir = '' - loosever = LooseVersion(self.version) - if loosever >= LooseVersion('2013_update7') and loosever < LooseVersion('2017'): - self.subdir = 'inspector_xe' - elif loosever >= LooseVersion('2017') and loosever < LooseVersion('2021'): + if loosever < LooseVersion('2021'): self.subdir = 'inspector' elif loosever >= LooseVersion('2021'): self.subdir = os.path.join('inspector', 'latest') + # prepare module load environment + self.prepare_intel_tools_env() + def make_installdir(self): """Do not create installation directory, install script handles that already.""" super(EB_Inspector, self).make_installdir(dontcreate=True) - def install_step(self): - """ - Actual installation - - create silent cfg file - - execute command - """ - silent_cfg_names_map = None - - if LooseVersion(self.version) <= LooseVersion('2013_update6'): - silent_cfg_names_map = { - 'activation_name': ACTIVATION_NAME_2012, - 'license_file_name': LICENSE_FILE_NAME_2012, - } - - super(EB_Inspector, self).install_step(silent_cfg_names_map=silent_cfg_names_map) - - def make_module_req_guess(self): - """Find reasonable paths for Inspector""" - return self.get_guesses_tools() - def sanity_check_step(self): """Custom sanity check paths for Intel Inspector.""" binaries = ['inspxe-cl', 'inspxe-feedback', 'inspxe-gui', 'inspxe-runmc', 'inspxe-runtc'] diff --git a/easybuild/easyblocks/i/intel_compilers.py b/easybuild/easyblocks/i/intel_compilers.py index a9c49d04e1f..eddfadd7927 100644 --- a/easybuild/easyblocks/i/intel_compilers.py +++ b/easybuild/easyblocks/i/intel_compilers.py @@ -1,5 +1,5 @@ # # -# Copyright 2021-2024 Ghent University +# Copyright 2021-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -31,9 +31,10 @@ from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.intelbase import IntelBase -from easybuild.easyblocks.t.tbb import get_tbb_gccprefix +from easybuild.easyblocks.tbb import get_tbb_gccprefix from easybuild.tools.build_log import EasyBuildError, print_msg -from easybuild.tools.run import run_cmd +from easybuild.tools.modules import MODULE_LOAD_ENV_HEADERS +from easybuild.tools.run import run_shell_cmd class EB_intel_minus_compilers(IntelBase): @@ -106,7 +107,9 @@ def sanity_check_step(self): ] bindir = os.path.join(self.compilers_subdir, 'bin') oneapi_compiler_paths = [os.path.join(bindir, x) for x in oneapi_compiler_cmds] - if LooseVersion(self.version) >= LooseVersion('2024'): + if LooseVersion(self.version) >= LooseVersion('2025'): + classic_compiler_cmds = [] + elif LooseVersion(self.version) >= LooseVersion('2024'): classic_compiler_cmds = ['ifort'] classic_bindir = bindir else: @@ -132,41 +135,38 @@ def sanity_check_step(self): super(EB_intel_minus_compilers, self).sanity_check_step(custom_paths=custom_paths, custom_commands=custom_commands) - def make_module_req_guess(self): + def make_module_step(self, *args, **kwargs): """ - Paths to consider for prepend-paths statements in module file + Set paths for module load environment based on the actual installation files """ - libdirs = [ - 'lib', - os.path.join('lib', 'x64'), - os.path.join('compiler', 'lib', 'intel64_lin'), + tbb_lib_prefix = os.path.join(self.tbb_subdir, 'lib', 'intel64') + tbb_lib_gccdir = get_tbb_gccprefix(os.path.join(self.installdir, tbb_lib_prefix)) + + self.module_load_environment.PATH = [os.path.join(self.compilers_subdir, path) for path in ( + 'bin', + os.path.join('bin', 'intel64'), + )] + self.module_load_environment.LD_LIBRARY_PATH = [ + os.path.join(self.compilers_subdir, 'lib'), + os.path.join(self.compilers_subdir, 'lib', 'x64'), + os.path.join(self.compilers_subdir, 'compiler', 'lib', 'intel64_lin'), + os.path.join(tbb_lib_prefix, tbb_lib_gccdir), ] - libdirs = [os.path.join(self.compilers_subdir, x) for x in libdirs] - tbb_subdir = self.tbb_subdir - tbb_libsubdir = os.path.join(tbb_subdir, 'lib', 'intel64') - libdirs.append(os.path.join(tbb_libsubdir, - get_tbb_gccprefix(os.path.join(self.installdir, tbb_libsubdir)))) - guesses = { - 'PATH': [ - os.path.join(self.compilers_subdir, 'bin'), - os.path.join(self.compilers_subdir, 'bin', 'intel64'), - ], - 'LD_LIBRARY_PATH': libdirs, - 'LIBRARY_PATH': libdirs, - 'MANPATH': [ - os.path.join(os.path.dirname(self.compilers_subdir), 'documentation', 'en', 'man', 'common'), - os.path.join(self.compilers_subdir, 'share', 'man'), - ], - 'OCL_ICD_FILENAMES': [ - os.path.join(self.compilers_subdir, 'lib', 'x64', 'libintelocl.so'), - os.path.join(self.compilers_subdir, 'lib', 'libintelocl.so'), - ], - 'CPATH': [ - os.path.join(tbb_subdir, 'include'), - ], - 'TBBROOT': [tbb_subdir], - } - return guesses + self.module_load_environment.LIBRARY_PATH = self.module_load_environment.LD_LIBRARY_PATH + self.module_load_environment.MANPATH = [ + os.path.join(os.path.dirname(self.compilers_subdir), 'documentation', 'en', 'man', 'common'), + os.path.join(self.compilers_subdir, 'share', 'man'), + ] + self.module_load_environment.OCL_ICD_FILENAMES = [os.path.join(self.compilers_subdir, path) for path in ( + os.path.join('lib', 'x64', 'libintelocl.so'), + os.path.join('lib', 'libintelocl.so'), + )] + self.module_load_environment.TBBROOT = [self.tbb_subdir] + + # include paths to headers (e.g. CPATH) + self.module_load_environment.set_alias_vars(MODULE_LOAD_ENV_HEADERS, os.path.join(self.tbb_subdir, 'include')) + + return super().make_module_step(*args, **kwargs) def make_module_extra(self): """Additional custom variables for intel-compiler""" @@ -174,19 +174,23 @@ def make_module_extra(self): # On Debian/Ubuntu, /usr/include/x86_64-linux-gnu, or whatever dir gcc uses, needs to be included # in $CPATH for Intel C compiler - multiarch_out, ec = run_cmd("gcc -print-multiarch", simple=False) - multiarch_out = multiarch_out.strip() - if ec == 0 and multiarch_out: + res = run_shell_cmd("gcc -print-multiarch", hidden=True) + multiarch_out = res.output.strip() + if res.exit_code == 0 and multiarch_out: multi_arch_inc_dir_cmd = '|'.join([ "gcc -E -Wp,-v -xc /dev/null 2>&1", - "grep %s$" % multiarch_out, + f"grep {multiarch_out}$", "grep -v /include-fixed/", ]) - multiarch_inc_dir, ec = run_cmd(multi_arch_inc_dir_cmd) - if ec == 0 and multiarch_inc_dir: - multiarch_inc_dir = multiarch_inc_dir.strip() - self.log.info("Adding multiarch include path %s to $CPATH in generated module file", multiarch_inc_dir) + res = run_shell_cmd(multi_arch_inc_dir_cmd, hidden=True) + multiarch_inc_dir = res.output.strip() + if res.exit_code == 0 and multiarch_inc_dir: # system location must be appended at the end, so use append_paths - txt += self.module_generator.append_paths('CPATH', [multiarch_inc_dir], allow_abs=True) + for envar in self.module_load_environment.alias_vars(MODULE_LOAD_ENV_HEADERS): + self.log.info( + f"Adding multiarch include path '{multiarch_inc_dir}' to ${envar} in generated module file" + ) + # system location must be appended at the end, so use append_paths + txt += self.module_generator.append_paths(envar, [multiarch_inc_dir], allow_abs=True) return txt diff --git a/easybuild/easyblocks/i/ipp.py b/easybuild/easyblocks/i/ipp.py deleted file mode 100644 index d39c55b07b8..00000000000 --- a/easybuild/easyblocks/i/ipp.py +++ /dev/null @@ -1,129 +0,0 @@ -## -# Copyright 2009-2024 Ghent University -# -# This file is part of EasyBuild, -# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), -# with support of Ghent University (http://ugent.be/hpc), -# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), -# Flemish Research Foundation (FWO) (http://www.fwo.be/en) -# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). -# -# https://github.com/easybuilders/easybuild -# -# EasyBuild is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation v2. -# -# EasyBuild is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with EasyBuild. If not, see . -## -""" -EasyBuild support for installing the Intel Performance Primitives (IPP) library, implemented as an easyblock - -@author: Stijn De Weirdt (Ghent University) -@author: Dries Verdegem (Ghent University) -@author: Kenneth Hoste (Ghent University) -@author: Pieter De Baets (Ghent University) -@author: Jens Timmerman (Ghent University) -@author: Lumir Jasiok (IT4Innovations) -@author: Damian Alvarez (Forschungszentrum Juelich GmbH) -""" - -from easybuild.tools import LooseVersion -import os - -from easybuild.easyblocks.generic.intelbase import IntelBase, ACTIVATION_NAME_2012, LICENSE_FILE_NAME_2012 -from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.systemtools import get_platform_name -from easybuild.tools.systemtools import get_shared_lib_ext - - -class EB_ipp(IntelBase): - """ - Support for installing Intel Integrated Performance Primitives library - """ - - def install_step(self): - """ - Actual installation - - create silent cfg file - - execute command - """ - - platform_name = get_platform_name() - if platform_name.startswith('x86_64'): - self.arch = "intel64" - elif platform_name.startswith('i386') or platform_name.startswith('i686'): - self.arch = 'ia32' - else: - raise EasyBuildError("Failed to determine system architecture based on %s", platform_name) - - silent_cfg_names_map = None - silent_cfg_extras = None - - if LooseVersion(self.version) < LooseVersion('8.0'): - silent_cfg_names_map = { - 'activation_name': ACTIVATION_NAME_2012, - 'license_file_name': LICENSE_FILE_NAME_2012, - } - - # in case of IPP 9.x, we have to specify ARCH_SELECTED in silent.cfg - if LooseVersion(self.version) >= LooseVersion('9.0'): - silent_cfg_extras = { - 'ARCH_SELECTED': self.arch.upper() - } - - super(EB_ipp, self).install_step(silent_cfg_names_map=silent_cfg_names_map, silent_cfg_extras=silent_cfg_extras) - - def sanity_check_step(self): - """Custom sanity check paths for IPP.""" - shlib_ext = get_shared_lib_ext() - - dirs = [os.path.join('ipp', x) for x in ['bin', 'include', os.path.join('tools', 'intel64')]] - if LooseVersion(self.version) < LooseVersion('8.0'): - dirs.extend([ - os.path.join('compiler', 'lib', 'intel64'), - os.path.join('ipp', 'interfaces', 'data-compression'), - ]) - elif LooseVersion(self.version) < LooseVersion('9.0'): - dirs.extend([ - os.path.join('composerxe', 'lib', 'intel64'), - ]) - - ipp_libs = ['cc', 'ch', 'core', 'cv', 'dc', 'i', 's', 'vm'] - if LooseVersion(self.version) < LooseVersion('9.0'): - ipp_libs.extend(['ac', 'di', 'j', 'm', 'r', 'sc', 'vc']) - - custom_paths = { - 'files': [ - os.path.join('ipp', 'lib', 'intel64', 'libipp%s') % y for x in ipp_libs - for y in ['%s.a' % x, '%s.%s' % (x, shlib_ext)] - ], - 'dirs': dirs, - } - - super(EB_ipp, self).sanity_check_step(custom_paths=custom_paths) - - def make_module_req_guess(self): - """ - A dictionary of possible directories to look for - """ - guesses = super(EB_ipp, self).make_module_req_guess() - - if LooseVersion(self.version) >= LooseVersion('9.0'): - lib_path = [os.path.join('ipp', 'lib', self.arch), os.path.join('lib', self.arch)] - include_path = os.path.join('ipp', 'include') - - guesses.update({ - 'LD_LIBRARY_PATH': lib_path, - 'LIBRARY_PATH': lib_path, - 'CPATH': [include_path], - 'INCLUDE': [include_path], - }) - - return guesses diff --git a/easybuild/easyblocks/i/ironpython.py b/easybuild/easyblocks/i/ironpython.py deleted file mode 100644 index dedbbe8feca..00000000000 --- a/easybuild/easyblocks/i/ironpython.py +++ /dev/null @@ -1,77 +0,0 @@ -## -# Copyright 2009-2024 Ghent University -# -# This file is part of EasyBuild, -# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), -# with support of Ghent University (http://ugent.be/hpc), -# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), -# the Hercules foundation (http://www.herculesstichting.be/in_English) -# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). -# -# https://github.com/easybuilders/easybuild -# -# EasyBuild is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation v2. -# -# EasyBuild is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with EasyBuild. If not, see . -## -""" -EasyBuild support for IronPython, implemented as an easyblock - -@author: Stijn De Weirdt (Ghent University) -@author: Dries Verdegem (Ghent University) -@author: Kenneth Hoste (Ghent University) -@author: Pieter De Baets (Ghent University) -@author: Jens Timmerman (Ghent University) -""" - -import os - -from easybuild.easyblocks.generic.packedbinary import PackedBinary -from easybuild.tools.run import run_cmd - - -class EB_IronPython(PackedBinary): - """Support for building/installing IronPython.""" - - def __init__(self, *args, **kwargs): - """Custom constructor for IronPython easyblock, indicate building in installdir.""" - super(EB_IronPython, self).__init__(*args, **kwargs) - - self.build_in_installdir = True - - def extract_step(self): - """Extract sources; strip off parent directory during unpack""" - self.cfg.update('unpack_options', "--strip-components=1") - super(EB_IronPython, self).extract_step() - - def install_step(self): - """Custom install step for IronPython, using xbuild command.""" - - cmd = "xbuild /p:Configuration=Release Solutions/%s.sln" % self.name - run_cmd(cmd, log_all=True, simple=True) - - def sanity_check_step(self): - """Custom sanity check for IronPython.""" - - binpath = os.path.join('bin', 'Release') - custom_paths = { - 'files': [os.path.join(binpath, x) for x in ['ipy.exe', 'ipy64.exe', 'ipyw.exe', 'ipyw64.exe']], - 'dirs': ['Config', 'Runtime', 'Tools', 'Util'], - } - super(EB_IronPython, self).sanity_check_step(custom_paths=custom_paths) - - def make_module_req_guess(self): - """Add IronPython binaries path to $PATH.""" - guesses = super(EB_IronPython, self).make_module_req_guess() - guesses.update({ - 'PATH': [os.path.join('bin', 'Release')], - }) - return guesses diff --git a/easybuild/easyblocks/i/itac.py b/easybuild/easyblocks/i/itac.py index d0811cbcc57..aecf7894c46 100644 --- a/easybuild/easyblocks/i/itac.py +++ b/easybuild/easyblocks/i/itac.py @@ -1,5 +1,5 @@ # # -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -32,20 +32,19 @@ @author: Jens Timmerman (Ghent University) """ -import os - from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.intelbase import IntelBase from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.run import run_cmd +from easybuild.tools.modules import MODULE_LOAD_ENV_HEADERS +from easybuild.tools.run import run_shell_cmd class EB_itac(IntelBase): """ Class that can be used to install itac - - tested with Intel Trace Analyzer and Collector 7.2.1.008 + - minimum version suported: 2019.x """ @staticmethod @@ -55,6 +54,23 @@ def extra_options(): } return IntelBase.extra_options(extra_vars) + def __init__(self, *args, **kwargs): + """Constructor, initialize class variables.""" + super().__init__(*args, **kwargs) + + if LooseVersion(self.version) < LooseVersion('2019'): + raise EasyBuildError( + f"Version {self.version} of {self.name} is unsupported. Mininum supported version is 2019.0." + ) + + # add cutom paths to the module load environment + self.module_load_environment.PATH = ['bin', 'bin/intel64', 'bin64'] + self.module_load_environment.LD_LIBRARY_PATH = ['lib', 'lib/intel64', 'lib64', 'slib'] + # avoid software building against itac + self.module_load_environment.remove('LIBRARY_PATH') + for disallowed_var in self.module_load_environment.alias_vars(MODULE_LOAD_ENV_HEADERS): + self.module_load_environment.remove(disallowed_var) + def prepare_step(self, *args, **kwargs): """ Custom prepare step for itac: don't require runtime license for oneAPI versions (>= 2021) @@ -71,42 +87,9 @@ def install_step_classic(self): - create silent cfg file - execute command """ - - if LooseVersion(self.version) >= LooseVersion('8.1'): - super(EB_itac, self).install_step_classic(silent_cfg_names_map=None) - - # itac v9.0.1 installer create itac/ subdir, so stuff needs to be moved afterwards - if LooseVersion(self.version) >= LooseVersion('9.0'): - super(EB_itac, self).move_after_install() - else: - silent = """ -[itac] -INSTALLDIR=%(ins)s -LICENSEPATH=%(lic)s -INSTALLMODE=NONRPM -INSTALLUSER=NONROOT -INSTALL_ITA=YES -INSTALL_ITC=YES -DEFAULT_MPI=%(mpi)s -EULA=accept -""" % {'lic': self.license_file, 'ins': self.installdir, 'mpi': self.cfg['preferredmpi']} - - # already in correct directory - silentcfg = os.path.join(os.getcwd(), "silent.cfg") - f = open(silentcfg, 'w') - f.write(silent) - f.close() - self.log.debug("Contents of %s: %s" % (silentcfg, silent)) - - tmpdir = os.path.join(os.getcwd(), self.version, 'mytmpdir') - try: - os.makedirs(tmpdir) - except OSError as err: - raise EasyBuildError("Directory %s can't be created: %s", tmpdir, err) - - cmd = "./install.sh --tmp-dir=%s --silent=%s" % (tmpdir, silentcfg) - - run_cmd(cmd, log_all=True, simple=True) + super(EB_itac, self).install_step_classic(silent_cfg_names_map=None) + # since itac v9.0.1 installer create itac/ subdir, so stuff needs to be moved afterwards + super(EB_itac, self).move_after_install() def install_step_oneapi(self, *args, **kwargs): """ @@ -131,7 +114,7 @@ def install_step_oneapi(self, *args, **kwargs): "--install-dir=%s" % self.installdir, ]) - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) # itac installer create itac/ subdir, so stuff needs to be moved afterwards super(EB_itac, self).move_after_install() @@ -146,32 +129,6 @@ def sanity_check_step(self): super(EB_itac, self).sanity_check_step(custom_paths=custom_paths) - def make_module_req_guess(self): - """ - A dictionary of possible directories to look for - """ - guesses = {} - if LooseVersion(self.version) < LooseVersion('9.0'): - preferredmpi = self.cfg["preferredmpi"] - guesses.update({ - 'MANPATH': ['man'], - 'CLASSPATH': ['itac/lib_%s' % preferredmpi], - 'VT_LIB_DIR': ['itac/lib_%s' % preferredmpi], - 'VT_SLIB_DIR': ['itac/lib_s%s' % preferredmpi] - }) - - if self.cfg['m32']: - guesses.update({ - 'PATH': ['bin', 'bin/ia32', 'ia32/bin'], - 'LD_LIBRARY_PATH': ['lib', 'lib/ia32', 'ia32/lib'], - }) - else: - guesses.update({ - 'PATH': ['bin', 'bin/intel64', 'bin64'], - 'LD_LIBRARY_PATH': ['lib', 'lib/intel64', 'lib64', 'slib'], - }) - return guesses - def make_module_extra(self): """Overwritten from IntelBase to add extra txt""" txt = super(EB_itac, self).make_module_extra() diff --git a/easybuild/easyblocks/j/java.py b/easybuild/easyblocks/j/java.py index 5731a193bbf..039e251388e 100644 --- a/easybuild/easyblocks/j/java.py +++ b/easybuild/easyblocks/j/java.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2024 Ghent University +# Copyright 2012-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -37,7 +37,7 @@ from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_option from easybuild.tools.filetools import adjust_permissions, change_dir, copy_dir, copy_file, remove_dir, which -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd from easybuild.tools.systemtools import AARCH64, POWER, RISCV64, X86_64, get_cpu_architecture, get_shared_lib_ext from easybuild.tools.utilities import nub @@ -74,7 +74,7 @@ def extract_step(self): adjust_permissions(os.path.join(self.builddir, self.src[0]['name']), stat.S_IXUSR, add=True) change_dir(self.builddir) - run_cmd(os.path.join(self.builddir, self.src[0]['name']), log_all=True, simple=True, inp='') + run_shell_cmd(os.path.join(self.builddir, self.src[0]['name']), stdin='') else: PackedBinary.extract_step(self) adjust_permissions(self.builddir, stat.S_IWUSR, add=True, recursive=True) @@ -87,12 +87,12 @@ def install_step(self): else: PackedBinary.install_step(self) - def post_install_step(self): + def post_processing_step(self): """ Custom post-installation step: - ensure correct glibc is used when installing into custom sysroot and using RPATH """ - super(EB_Java, self).post_install_step() + super(EB_Java, self).post_processing_step() # patch binaries and libraries when using alternate sysroot in combination with RPATH sysroot = build_option('sysroot') @@ -129,68 +129,66 @@ def post_install_step(self): if elf_interp is None: raise EasyBuildError("Failed to isolate ELF interpreter!") - module_guesses = self.make_module_req_guess() - - bindirs = [os.path.join(self.installdir, bindir) for bindir in module_guesses['PATH'] if - os.path.exists(os.path.join(self.installdir, bindir))] - # Make sure these are unique real paths - bindirs = list(set([os.path.realpath(path) for path in bindirs])) + # Expand paths in PATH and make sure these are unique real paths + bindirs = nub([x for bindir in self.module_load_environment.PATH + for x in self.expand_module_search_path(bindir)]) + bindirs = [os.path.realpath(os.path.join(self.installdir, bindir)) for bindir in bindirs] for bindir in bindirs: for path in os.listdir(bindir): path = os.path.join(bindir, path) - out, _ = run_cmd("file %s" % path, trace=False) - if "dynamically linked" in out: + res = run_shell_cmd("file %s" % path, hidden=True) + if "dynamically linked" in res.output: - out, _ = run_cmd("patchelf --print-interpreter %s" % path, trace=False) - self.log.debug("ELF interpreter for %s: %s" % (path, out)) + res = run_shell_cmd("patchelf --print-interpreter %s" % path) + self.log.debug("ELF interpreter for %s: %s" % (path, res.output)) - run_cmd("patchelf --set-interpreter %s %s" % (elf_interp, path), trace=False) + run_shell_cmd("patchelf --set-interpreter %s %s" % (elf_interp, path), hidden=True) - out, _ = run_cmd("patchelf --print-interpreter %s" % path, trace=False) - self.log.debug("ELF interpreter for %s: %s" % (path, out)) + res = run_shell_cmd("patchelf --print-interpreter %s" % path, hidden=True) + self.log.debug("ELF interpreter for %s: %s" % (path, res.output)) - out, _ = run_cmd("patchelf --print-rpath %s" % path, simple=False, trace=False) - curr_rpath = out.strip() + res = run_shell_cmd("patchelf --print-rpath %s" % path, hidden=True) + curr_rpath = res.output.strip() self.log.debug("RPATH for %s: %s" % (path, curr_rpath)) new_rpath = ':'.join([curr_rpath] + sysroot_lib_paths) # note: it's important to wrap the new RPATH value in single quotes, # to avoid magic values like $ORIGIN being resolved by the shell - run_cmd("patchelf --set-rpath '%s' %s" % (new_rpath, path), trace=False) + run_shell_cmd("patchelf --set-rpath '%s' %s" % (new_rpath, path), hidden=True) - curr_rpath, _ = run_cmd("patchelf --print-rpath %s" % path, simple=False, trace=False) - self.log.debug("RPATH for %s (prior to shrinking): %s" % (path, curr_rpath)) + res = run_shell_cmd("patchelf --print-rpath %s" % path, hidden=True) + self.log.debug("RPATH for %s (prior to shrinking): %s" % (path, res.output)) - run_cmd("patchelf --shrink-rpath %s" % path, trace=False) + run_shell_cmd("patchelf --shrink-rpath %s" % path, hidden=True) - curr_rpath, _ = run_cmd("patchelf --print-rpath %s" % path, simple=False, trace=False) - self.log.debug("RPATH for %s (after shrinking): %s" % (path, curr_rpath)) + res = run_shell_cmd("patchelf --print-rpath %s" % path, hidden=True) + self.log.debug("RPATH for %s (after shrinking): %s" % (path, res.output)) - libdirs = [os.path.join(self.installdir, libdir) for libdir in module_guesses['LIBRARY_PATH'] if - os.path.exists(os.path.join(self.installdir, libdir))] - # Make sure these are unique real paths - libdirs = list(set([os.path.realpath(path) for path in libdirs])) + # Expand paths in LIBRARY_PATH and make sure these are unique real paths + libdirs = nub([x for ld in self.module_load_environment.LIBRARY_PATH + for x in self.expand_module_search_path(ld)]) + libdirs = [os.path.realpath(os.path.join(self.installdir, ld)) for ld in libdirs] shlib_ext = '.' + get_shared_lib_ext() for libdir in libdirs: for path, _, filenames in os.walk(libdir): shlibs = [os.path.join(path, x) for x in filenames if x.endswith(shlib_ext)] for shlib in shlibs: - out, _ = run_cmd("patchelf --print-rpath %s" % shlib, simple=False, trace=False) - curr_rpath = out.strip() + res = run_shell_cmd("patchelf --print-rpath %s" % shlib, hidden=True) + curr_rpath = res.output.strip() self.log.debug("RPATH for %s: %s" % (shlib, curr_rpath)) new_rpath = ':'.join([curr_rpath] + sysroot_lib_paths) # note: it's important to wrap the new RPATH value in single quotes, # to avoid magic values like $ORIGIN being resolved by the shell - run_cmd("patchelf --set-rpath '%s' %s" % (new_rpath, shlib), trace=False) + run_shell_cmd("patchelf --set-rpath '%s' %s" % (new_rpath, shlib), hidden=True) - curr_rpath, _ = run_cmd("patchelf --print-rpath %s" % shlib, simple=False, trace=False) - self.log.debug("RPATH for %s (prior to shrinking): %s" % (path, curr_rpath)) + res = run_shell_cmd("patchelf --print-rpath %s" % shlib, hidden=True) + self.log.debug("RPATH for %s (prior to shrinking): %s" % (path, res.output)) - run_cmd("patchelf --shrink-rpath %s" % shlib, trace=False) + run_shell_cmd("patchelf --shrink-rpath %s" % shlib, hidden=True) - curr_rpath, _ = run_cmd("patchelf --print-rpath %s" % shlib, simple=False, trace=False) - self.log.debug("RPATH for %s (after shrinking): %s" % (path, curr_rpath)) + res = run_shell_cmd("patchelf --print-rpath %s" % shlib, hidden=True) + self.log.debug("RPATH for %s (after shrinking): %s" % (path, res.output)) except OSError as err: raise EasyBuildError("Failed to patch RPATH section in binaries/libraries: %s", err) diff --git a/easybuild/easyblocks/j/jaxlib.py b/easybuild/easyblocks/j/jaxlib.py index d4dca3245cb..e876258b089 100644 --- a/easybuild/easyblocks/j/jaxlib.py +++ b/easybuild/easyblocks/j/jaxlib.py @@ -1,5 +1,5 @@ ## -# Copyright 2012-2024 Ghent University +# Copyright 2012-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -51,7 +51,6 @@ def extra_options(): """Custom easyconfig parameters specific to jaxlib.""" extra_vars = PythonPackage.extra_options() - extra_vars['use_pip'][0] = True # Run custom build script and install the generated whl file extra_vars['buildcmd'][0] = '%(python)s build/build.py' extra_vars['install_src'][0] = 'dist/*.whl' @@ -89,7 +88,7 @@ def configure_step(self): # Passed to the build command of bazel bazel_options = [ - '--jobs=%s' % self.cfg['parallel'], + f'--jobs={self.cfg.parallel}', '--subcommands', '--action_env=PYTHONPATH', '--action_env=EBPYTHONPREFIXES', diff --git a/easybuild/easyblocks/l/lammps.py b/easybuild/easyblocks/l/lammps.py index 9e8d2861944..53f00452b5a 100644 --- a/easybuild/easyblocks/l/lammps.py +++ b/easybuild/easyblocks/l/lammps.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -39,13 +39,15 @@ import easybuild.tools.environment as env import easybuild.tools.toolchain as toolchain +from easybuild.base import fancylogger from easybuild.framework.easyconfig import CUSTOM, MANDATORY from easybuild.tools.build_log import EasyBuildError, print_warning, print_msg from easybuild.tools.config import build_option from easybuild.tools.filetools import copy_dir, mkdir from easybuild.tools.modules import get_software_root, get_software_version -from easybuild.tools.run import run_cmd -from easybuild.tools.systemtools import get_shared_lib_ext +from easybuild.tools.run import run_shell_cmd +from easybuild.tools.systemtools import AARCH64, get_cpu_architecture, get_shared_lib_ext +from easybuild.tools.toolchain.compiler import OPTARCH_GENERIC from easybuild.easyblocks.generic.cmakemake import CMakeMake @@ -124,6 +126,7 @@ 'skylake_avx512': 'SKX', 'cascadelake': 'SKX', 'icelake': 'SKX', + 'sapphirerapids': 'SKX', 'knights-landing': 'KNL', 'zen': 'ZEN', 'zen2': 'ZEN2', @@ -153,6 +156,8 @@ # lammps version, which caused the most changes. This may not be precise, but it does work with existing easyconfigs ref_version = '29Sep2021' +_log = fancylogger.getLogger('easyblocks.lammps') + def translate_lammps_version(version): """Translate the LAMMPS version into something that can be used in a comparison""" @@ -236,6 +241,10 @@ def update_kokkos_cpu_mapping(self): self.kokkos_cpu_mapping['a64fx'] = 'A64FX' self.kokkos_cpu_mapping['zen4'] = 'ZEN3' + if LooseVersion(self.cur_version) >= LooseVersion(translate_lammps_version('2Aug2023')): + self.kokkos_cpu_mapping['icelake'] = 'ICX' + self.kokkos_cpu_mapping['sapphirerapids'] = 'SPR' + def prepare_step(self, *args, **kwargs): """Custom prepare step for LAMMPS.""" super(EB_LAMMPS, self).prepare_step(*args, **kwargs) @@ -426,18 +435,18 @@ def configure_step(self, **kwargs): if python_dir: # Find the Python .so lib cmd = 'python -c "import sysconfig; print(sysconfig.get_config_var(\'LDLIBRARY\'))"' - (python_lib, _) = run_cmd(cmd, log_all=True, simple=False, trace=False) - if not python_lib: - raise EasyBuildError("Failed to determine Python .so library: %s", python_lib) - python_lib_path = glob.glob(os.path.join(python_dir, 'lib*', python_lib.strip()))[0] + res = run_shell_cmd(cmd, hidden=True) + if not res.output: + raise EasyBuildError("Failed to determine Python .so library: %s", res.output) + python_lib_path = glob.glob(os.path.join(python_dir, 'lib*', res.output.strip()))[0] if not python_lib_path: - raise EasyBuildError("Could not find path to Python .so library: %s", python_lib) + raise EasyBuildError("Could not find path to Python .so library: %s", res.output) # and the path to the Python include folder cmd = 'python -c "import sysconfig; print(sysconfig.get_config_var(\'INCLUDEPY\'))"' - (python_include_dir, _) = run_cmd(cmd, log_all=True, simple=False, trace=False) - if not python_include_dir: - raise EasyBuildError("Failed to determine Python include dir: %s", python_include_dir) - python_include_dir = python_include_dir.strip() + res = run_shell_cmd(cmd, hidden=True) + if not res.output: + raise EasyBuildError("Failed to determine Python include dir: %s", res.output) + python_include_dir = res.output.strip() # Whether you need one or the other of the options below depends on the version of CMake and LAMMPS # Rather than figure this out, use both (and one will be ignored) @@ -471,7 +480,7 @@ def install_step(self): mkdir(site_packages, parents=True) - self.lammpsdir = os.path.join(self.builddir, '%s-*_%s' % (self.name.lower(), self.version)) + self.lammpsdir = os.path.join(self.builddir, '%s-*' % self.name.lower()) self.python_dir = os.path.join(self.lammpsdir, 'python') # The -i flag is added through a patch to the lammps source file python/install.py @@ -486,7 +495,7 @@ def install_step(self): 'site_packages': site_packages, } - run_cmd(cmd, log_all=True, simple=False) + run_shell_cmd(cmd) def sanity_check_step(self, *args, **kwargs): """Run custom sanity checks for LAMMPS files, dirs and commands.""" @@ -548,19 +557,6 @@ def sanity_check_step(self, *args, **kwargs): return super(EB_LAMMPS, self).sanity_check_step(custom_commands=custom_commands, custom_paths=custom_paths) - def make_module_extra(self): - """Add install path to PYTHONPATH""" - - txt = super(EB_LAMMPS, self).make_module_extra() - - python = get_software_version('Python') - if python: - pyshortver = '.'.join(get_software_version('Python').split('.')[:2]) - pythonpath = os.path.join('lib', 'python%s' % pyshortver, 'site-packages') - txt += self.module_generator.prepend_paths('PYTHONPATH', [pythonpath]) - - return txt - def get_cuda_gpu_arch(cuda_cc): """Return CUDA gpu ARCH in LAMMPS required format. Example: 'sm_32' """ @@ -579,7 +575,22 @@ def get_kokkos_arch(kokkos_cpu_mapping, cuda_cc, kokkos_arch, cuda=None): processor_arch = None - if kokkos_arch: + if build_option('optarch') == OPTARCH_GENERIC: + # For generic Arm builds we use an existing target; + # this ensures that KOKKOS_ARCH_ARM_NEON is enabled (Neon is required for armv8-a). + # For other architectures we set a custom/non-existent type, which will disable all optimizations, + # and it should use the compiler (optimization) flags set by EasyBuild for this architecture. + if get_cpu_architecture() == AARCH64: + processor_arch = 'ARMV80' + else: + processor_arch = 'EASYBUILD_GENERIC' + + _log.info("Generic build requested, setting CPU ARCH to %s." % processor_arch) + if kokkos_arch: + msg = "The specified kokkos_arch (%s) will be ignored " % kokkos_arch + msg += "because a generic build was requested (via --optarch=GENERIC)" + print_warning(msg) + elif kokkos_arch: if kokkos_arch not in KOKKOS_CPU_ARCH_LIST: warning_msg = "Specified CPU ARCH (%s) " % kokkos_arch warning_msg += "was not found in listed options [%s]." % KOKKOS_CPU_ARCH_LIST @@ -666,7 +677,7 @@ def get_cpu_arch(): :return: returns detected cpu architecture """ - out, ec = run_cmd("python -c 'from archspec.cpu import host; print(host())'", simple=False) - if ec: - raise EasyBuildError("Failed to determine CPU architecture: %s", out) - return out.strip() + res = run_shell_cmd("python -c 'from archspec.cpu import host; print(host())'") + if res.exit_code: + raise EasyBuildError("Failed to determine CPU architecture: %s", res.output) + return res.output.strip() diff --git a/easybuild/easyblocks/l/lapack.py b/easybuild/easyblocks/l/lapack.py index f2f3ce6705c..5e052519139 100644 --- a/easybuild/easyblocks/l/lapack.py +++ b/easybuild/easyblocks/l/lapack.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -44,7 +44,7 @@ from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import copy_file from easybuild.tools.modules import get_software_root -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd class EB_LAPACK(ConfigureMake): @@ -215,7 +215,7 @@ def test_step(self): for lib in ["blas", "lapack"]: self.log.info("Running %s tests..." % lib.upper()) cmd = "make BLASLIB='%s' %s_testing" % (blaslib, lib) - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) else: super(EB_LAPACK, self).test_step() diff --git a/easybuild/easyblocks/l/libdrm.py b/easybuild/easyblocks/l/libdrm.py index dd3d2c2b872..75902e4cef0 100644 --- a/easybuild/easyblocks/l/libdrm.py +++ b/easybuild/easyblocks/l/libdrm.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2024 Ghent University +# Copyright 2013-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/l/libint.py b/easybuild/easyblocks/l/libint.py index ea4f86cb78e..67f8c87eba5 100644 --- a/easybuild/easyblocks/l/libint.py +++ b/easybuild/easyblocks/l/libint.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2024 Ghent University +# Copyright 2013-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -37,7 +37,8 @@ from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.build_log import EasyBuildError, print_msg from easybuild.tools.filetools import change_dir, extract_file -from easybuild.tools.run import run_cmd +from easybuild.tools.modules import MODULE_LOAD_ENV_HEADERS +from easybuild.tools.run import run_shell_cmd from easybuild.tools.systemtools import get_shared_lib_ext @@ -52,6 +53,19 @@ def extra_options(): } return CMakeMake.extra_options(extra_vars) + def __init__(self, *args, **kwargs): + """Easyblock constructor.""" + super(EB_Libint, self).__init__(*args, **kwargs) + + # add custom paths to headers to module load environment + libint_includes = ['include'] + if LooseVersion(self.version) >= LooseVersion('2.0'): + libint_includes.append(os.path.join('include', 'libint2')) + else: + libint_includes.append(os.path.join('include', 'libint')) + + self.module_load_environment.set_alias_vars(MODULE_LOAD_ENV_HEADERS, libint_includes) + def configure_step(self): """Add some extra configure options.""" @@ -67,7 +81,7 @@ def configure_step(self): print_msg("configuring Libint compiler...") # first run autogen.sh script to generate initial configure script - run_cmd("./autogen.sh") + run_shell_cmd("./autogen.sh") cmd = ' '.join([ self.cfg['preconfigopts'], @@ -75,10 +89,10 @@ def configure_step(self): self.cfg['configopts'], self.cfg['libint_compiler_configopts'], ]) - run_cmd(cmd) + run_shell_cmd(cmd) print_msg("generating Libint library...") - run_cmd("make export") + run_shell_cmd("make export") source_fn = 'libint-%s.tgz' % self.version if os.path.exists(source_fn): @@ -167,15 +181,3 @@ def sanity_check_step(self): 'dirs': [], } super(EB_Libint, self).sanity_check_step(custom_paths=custom_paths) - - def make_module_req_guess(self): - """Specify correct CPATH for this installation.""" - guesses = super(EB_Libint, self).make_module_req_guess() - if LooseVersion(self.version) >= LooseVersion('2.0'): - libint_include = os.path.join('include', 'libint2') - else: - libint_include = os.path.join('include', 'libint') - guesses.update({ - 'CPATH': ['include', libint_include], - }) - return guesses diff --git a/easybuild/easyblocks/l/libqglviewer.py b/easybuild/easyblocks/l/libqglviewer.py index 6c38a1103b0..10bc1e5dfaa 100644 --- a/easybuild/easyblocks/l/libqglviewer.py +++ b/easybuild/easyblocks/l/libqglviewer.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -29,7 +29,7 @@ """ from easybuild.easyblocks.generic.configuremake import ConfigureMake -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd from easybuild.tools.systemtools import get_shared_lib_ext from easybuild.tools.modules import get_software_root from easybuild.tools.build_log import EasyBuildError @@ -47,7 +47,7 @@ def configure_step(self): 'installdir': self.installdir, 'configopts': self.cfg['configopts'], } - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) def sanity_check_step(self): """Custom sanity check for libQGLViewer.""" diff --git a/easybuild/easyblocks/l/libsmm.py b/easybuild/easyblocks/l/libsmm.py deleted file mode 100644 index 6010a7bb6c3..00000000000 --- a/easybuild/easyblocks/l/libsmm.py +++ /dev/null @@ -1,212 +0,0 @@ -## -# Copyright 2009-2024 Ghent University -# -# This file is part of EasyBuild, -# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), -# with support of Ghent University (http://ugent.be/hpc), -# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), -# Flemish Research Foundation (FWO) (http://www.fwo.be/en) -# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). -# -# https://github.com/easybuilders/easybuild -# -# EasyBuild is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation v2. -# -# EasyBuild is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with EasyBuild. If not, see . -## -""" -EasyBuild support for building and installing the libsmm library, implemented as an easyblock - -@author: Stijn De Weirdt (Ghent University) -@author: Dries Verdegem (Ghent University) -@author: Kenneth Hoste (Ghent University) -@author: Pieter De Baets (Ghent University) -@author: Jens Timmerman (Ghent University) -""" - -import os -from easybuild.tools import LooseVersion - -import easybuild.tools.toolchain as toolchain -from easybuild.framework.easyblock import EasyBlock -from easybuild.framework.easyconfig import CUSTOM -from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.filetools import copy_dir -from easybuild.tools.modules import get_software_version -from easybuild.tools.run import run_cmd - - -class EB_libsmm(EasyBlock): - """ - Support for the CP2K small matrix library - Notes: - build can take really really long, and no real rebuilding needed for each get_version - - CP2K can be built without this - """ - - @staticmethod - def extra_options(): - # default dimensions - dd = [1, 4, 5, 6, 9, 13, 16, 17, 22] - extra_vars = { - 'transpose_flavour': [1, "Transpose flavour of routines", CUSTOM], - 'max_tiny_dim': [12, "Maximum tiny dimension", CUSTOM], - 'dims': [dd, "Generate routines for these matrix dims", CUSTOM], - } - return EasyBlock.extra_options(extra_vars) - - def configure_step(self): - """Configure build: change to tools/build_libsmm dir""" - try: - dst = 'tools/build_libsmm' - os.chdir(dst) - self.log.debug('Change to directory %s' % dst) - except OSError as err: - raise EasyBuildError("Failed to change to directory %s: %s", dst, err) - - def build_step(self): - """Build libsmm - Possible iterations over precision (single/double) and type (real/complex) - - also type of transpose matrix - - all set in the config file - - Make the config.in file (is source afterwards in the build) - """ - - fn = 'config.in' - cfg_tpl = """# This config file was generated by EasyBuild - -# the build script can generate optimized routines packed in a library for -# 1) 'nn' => C=C+MATMUL(A,B) -# 2) 'tn' => C=C+MATMUL(TRANSPOSE(A),B) -# 3) 'nt' => C=C+MATMUL(A,TRANSPOSE(B)) -# 4) 'tt' => C=C+MATMUL(TRANPOSE(A),TRANPOSE(B)) -# -# select a tranpose_flavor from the list 1 2 3 4 -# -transpose_flavor=%(transposeflavour)s - -# 1) d => double precision real -# 2) s => single precision real -# 3) z => double precision complex -# 4) c => single precision complex -# -# select a data_type from the list 1 2 3 4 -# -data_type=%(datatype)s - -# target compiler... this are the options used for building the library. -# They should be aggessive enough to e.g. perform vectorization for the specific CPU -# (e.g. -ftree-vectorize -march=native), -# and allow some flexibility in reordering floating point expressions (-ffast-math). -# Higher level optimisation (in particular loop nest optimization) should not be used. -# -target_compile="%(targetcompile)s" - -# target dgemm link options... these are the options needed to link blas (e.g. -lblas) -# blas is used as a fall back option for sizes not included in the library or in those cases where it is faster -# the same blas library should thus also be used when libsmm is linked. -# -OMP_NUM_THREADS=1 -blas_linking="%(LIBBLAS)s" - -# matrix dimensions for which optimized routines will be generated. -# since all combinations of M,N,K are being generated the size of the library becomes very large -# if too many sizes are being optimized for. Numbers have to be ascending. -# -dims_small="%(dims)s" - -# tiny dimensions are used as primitves and generated in an 'exhaustive' search. -# They should be a sequence from 1 to N, -# where N is a number that is large enough to have good cache performance -# (e.g. for modern SSE cpus 8 to 12) -# Too large (>12?) is not beneficial, but increases the time needed to build the library -# Too small (<8) will lead to a slow library, but the build might proceed quickly -# The minimum number for a successful build is 4 -# -dims_tiny="%(tiny_dims)s" - -# host compiler... this is used only to compile a few tools needed to build the library. -# The library itself is not compiled this way. -# This compiler needs to be able to deal with some Fortran2003 constructs. -# -host_compile="%(hostcompile)s " - -# number of processes to use in parallel for compiling / building and benchmarking the library. -# Should *not* be more than the physical (available) number of cores of the machine -# -tasks=%(tasks)s - - """ - - # only GCC is supported for now - if self.toolchain.comp_family() == toolchain.GCC: # @UndefinedVariable - hostcompile = os.getenv('F90') - - # optimizations - opts = "-O2 -funroll-loops -ffast-math -ftree-vectorize -march=native -fno-inline-functions" - - # Depending on the get_version, we need extra options - extra = '' - gccVersion = LooseVersion(get_software_version('GCC')) - if gccVersion >= LooseVersion('4.6'): - extra = "-flto" - - targetcompile = "%s %s %s" % (hostcompile, opts, extra) - else: - raise EasyBuildError("No supported compiler found (tried GCC)") - - if not os.getenv('LIBBLAS'): - raise EasyBuildError("No BLAS library specifications found (LIBBLAS not set)!") - - cfgdict = { - 'datatype': None, - 'transposeflavour': self.cfg['transpose_flavour'], - 'targetcompile': targetcompile, - 'hostcompile': hostcompile, - 'dims': ' '.join([str(d) for d in self.cfg['dims']]), - 'tiny_dims': ' '.join([str(d) for d in range(1, self.cfg['max_tiny_dim'] + 1)]), - 'tasks': self.cfg['parallel'], - 'LIBBLAS': "%s %s" % (os.getenv('LDFLAGS'), os.getenv('LIBBLAS')) - } - - # configure for various iterations - datatypes = [(1, 'double precision real'), (3, 'double precision complex')] - - for (dt, descr) in datatypes: - cfgdict['datatype'] = dt - try: - txt = cfg_tpl % cfgdict - f = open(fn, 'w') - f.write(txt) - f.close() - self.log.debug("config file %s for datatype %s ('%s'): %s" % (fn, dt, descr, txt)) - except IOError as err: - raise EasyBuildError("Failed to write %s: %s", fn, err) - - self.log.info("Building for datatype %s ('%s')..." % (dt, descr)) - run_cmd("./do_clean") - run_cmd("./do_all") - - def install_step(self): - """Install CP2K: clean, and copy lib directory to install dir""" - - run_cmd("./do_clean") - copy_dir('lib', os.path.join(self.installdir, 'lib')) - - def sanity_check_step(self): - """Custom sanity check for libsmm""" - - custom_paths = { - 'files': ["lib/libsmm_%s.a" % x for x in ["dnn", "znn"]], - 'dirs': [], - } - - super(EB_libsmm, self).sanity_check_step(custom_paths=custom_paths) diff --git a/easybuild/easyblocks/l/libxml2.py b/easybuild/easyblocks/l/libxml2.py index dd430895792..931bddc7762 100644 --- a/easybuild/easyblocks/l/libxml2.py +++ b/easybuild/easyblocks/l/libxml2.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/l/llvm.py b/easybuild/easyblocks/l/llvm.py index 1193828e6ff..cb783db6003 100644 --- a/easybuild/easyblocks/l/llvm.py +++ b/easybuild/easyblocks/l/llvm.py @@ -1,5 +1,5 @@ ## -# Copyright 2020-2024 Ghent University +# Copyright 2020-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -25,19 +25,134 @@ """ EasyBuild support for building and installing LLVM, implemented as an easyblock +@author: Dmitri Gribenko (National Technical University of Ukraine "KPI") +@author: Ward Poelmans (Ghent University) +@author: Alan O'Cais (Juelich Supercomputing Centre) +@author: Maxime Boissonneault (Digital Research Alliance of Canada, Universite Laval) @author: Simon Branford (University of Birmingham) @author: Kenneth Hoste (Ghent University) +@author: Davide Grassano (CECAM HQ - Lausanne) """ +import contextlib +import glob import os +import re -from easybuild.easyblocks.clang import CLANG_TARGETS, DEFAULT_TARGETS_MAP -from easybuild.easyblocks.generic.cmakemake import CMakeMake from easybuild.framework.easyconfig import CUSTOM -from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.filetools import move_file -from easybuild.tools.modules import get_software_root -from easybuild.tools.systemtools import get_cpu_architecture +from easybuild.toolchains.compiler.clang import Clang from easybuild.tools import LooseVersion +from easybuild.tools.build_log import EasyBuildError, print_msg, print_warning +from easybuild.tools.config import ERROR, SEARCH_PATH_LIB_DIRS, build_option +from easybuild.tools.environment import setvar +from easybuild.tools.filetools import apply_regex_substitutions, change_dir, copy_dir, copy_file +from easybuild.tools.filetools import mkdir, remove_file, symlink, which, write_file +from easybuild.tools.modules import MODULE_LOAD_ENV_HEADERS, get_software_root, get_software_version +from easybuild.tools.run import run_shell_cmd +from easybuild.tools.systemtools import AARCH32, AARCH64, POWER, RISCV64, X86_64 +from easybuild.tools.systemtools import get_cpu_architecture, get_shared_lib_ext + +from easybuild.easyblocks.generic.cmakemake import CMakeMake, get_cmake_python_config_dict + +BUILD_TARGET_AMDGPU = 'AMDGPU' +BUILD_TARGET_NVPTX = 'NVPTX' + +LLVM_TARGETS = [ + 'AArch64', BUILD_TARGET_AMDGPU, 'ARM', 'AVR', 'BPF', 'Hexagon', 'Lanai', 'LoongArch', 'Mips', 'MSP430', + BUILD_TARGET_NVPTX, 'PowerPC', 'RISCV', 'Sparc', 'SystemZ', 'VE', 'WebAssembly', 'X86', 'XCore', + 'all' +] +LLVM_EXPERIMENTAL_TARGETS = [ + 'ARC', 'CSKY', 'DirectX', 'M68k', 'SPIRV', 'Xtensa', +] +ALL_TARGETS = LLVM_TARGETS + LLVM_EXPERIMENTAL_TARGETS + +DEFAULT_TARGETS_MAP = { + AARCH32: ['ARM'], + AARCH64: ['AArch64'], + POWER: ['PowerPC'], + RISCV64: ['RISCV'], + X86_64: ['X86'], +} + +AMDGPU_GFX_SUPPORT = [ + 'gfx700', 'gfx701', 'gfx801', 'gfx803', 'gfx900', 'gfx902', 'gfx906', 'gfx908', 'gfx90a', 'gfx90c', + 'gfx940', 'gfx941', 'gfx942', 'gfx1010', 'gfx1030', 'gfx1031', 'gfx1032', 'gfx1033', 'gfx1034', + 'gfx1035', 'gfx1036', 'gfx1100', 'gfx1101', 'gfx1102', 'gfx1103', 'gfx1150', 'gfx1151' +] + +remove_gcc_dependency_opts = { + 'CLANG_DEFAULT_CXX_STDLIB': 'libc++', + 'CLANG_DEFAULT_RTLIB': 'compiler-rt', + # Moved to general_opts for ease of building with openmp offload (or other multi-stage builds) + # 'CLANG_DEFAULT_LINKER': 'lld', + 'CLANG_DEFAULT_UNWINDLIB': 'libunwind', + + 'COMPILER_RT_BUILD_GWP_ASAN': 'Off', + 'COMPILER_RT_ENABLE_INTERNAL_SYMBOLIZER': 'On', + 'COMPILER_RT_ENABLE_STATIC_UNWINDER': 'On', # https://lists.llvm.org/pipermail/llvm-bugs/2016-July/048424.html + 'COMPILER_RT_USE_BUILTINS_LIBRARY': 'On', + 'COMPILER_RT_USE_LIBCXX': 'On', + 'COMPILER_RT_USE_LLVM_UNWINDER': 'On', + + 'LIBCXX_CXX_ABI': 'libcxxabi', + 'LIBCXX_DEFAULT_ABI_LIBRARY': 'libcxxabi', + # Needed as libatomic could not be present on the system (compilation and tests will succeed because of the + # GCCcore builddep, but usage/sanity check will fail due to missing libatomic) + 'LIBCXX_HAS_ATOMIC_LIB': 'NO', + 'LIBCXX_HAS_GCC_S_LIB': 'Off', + 'LIBCXX_USE_COMPILER_RT': 'On', + + 'LIBCXXABI_HAS_GCC_S_LIB': 'Off', + 'LIBCXXABI_USE_LLVM_UNWINDER': 'On', + 'LIBCXXABI_USE_COMPILER_RT': 'On', + + 'LIBUNWIND_HAS_GCC_S_LIB': 'Off', + 'LIBUNWIND_USE_COMPILER_RT': 'On', + + # Libxml2 from system gets automatically detected and linked in bringing dependencies from stdc++, gcc_s, icuuc, etc + # Moved to a check at the configure step. See https://github.com/easybuilders/easybuild-easyconfigs/issues/22491 + # 'LLVM_ENABLE_LIBXML2': 'Off', + + 'SANITIZER_USE_STATIC_LLVM_UNWINDER': 'On', +} + +disable_werror = { + 'BENCHMARK_ENABLE_WERROR': 'Off', + 'COMPILER_RT_ENABLE_WERROR': 'Off', + 'FLANG_ENABLE_WERROR': 'Off', + 'LIBC_WNO_ERROR': 'On', + 'LIBCXX_ENABLE_WERROR': 'Off', + 'LIBUNWIND_ENABLE_WERROR': 'Off', + 'LLVM_ENABLE_WERROR': 'Off', + 'OPENMP_ENABLE_WERROR': 'Off', +} + +general_opts = { + 'CMAKE_VERBOSE_MAKEFILE': 'ON', + 'LLVM_INCLUDE_BENCHMARKS': 'OFF', + 'LLVM_INSTALL_UTILS': 'ON', + # If EB is launched from a venv, avoid giving priority to the venv's python + 'Python3_FIND_VIRTUALENV': 'STANDARD', +} + + +@contextlib.contextmanager +def _wrap_env(path="", ld_path=""): + """Wrap the environment with $PATH and $LD_LIBRARY_PATH.""" + orig_path = os.getenv('PATH', '') + orig_ld_library_path = os.getenv('LD_LIBRARY_PATH', '') + + path = ':'.join(filter(None, [path, orig_path])) + ld_path = ':'.join(filter(None, [ld_path, orig_ld_library_path])) + + setvar('PATH', path) + setvar('LD_LIBRARY_PATH', ld_path) + + try: + yield + finally: + setvar('PATH', orig_path) + setvar('LD_LIBRARY_PATH', orig_ld_library_path) class EB_LLVM(CMakeMake): @@ -45,13 +160,65 @@ class EB_LLVM(CMakeMake): Support for building and installing LLVM """ + minimal_conflicts = [ + 'build_bolt', + 'build_clang_extras', + 'build_lld', + 'build_lldb', + 'build_openmp', + 'build_openmp_tools', + 'build_runtimes', + 'bootstrap', + 'full_llvm', + 'python_bindings', + 'usepolly', + ] + + # Create symlink between equivalent host triples, useful so that other build processes that relies on older + # triple names can still work when passing the old name to --target + symlink_lst = [ + ('x86_64-unknown-linux-gnu', 'x86_64-pc-linux'), + ('x86_64-unknown-linux-gnu', 'x86_64-pc-linux-gnu'), + ] + + # From LLVM 19, GCC_INSTALL_PREFIX is not supported anymore to hardcode the GCC installation path into the binaries; + # Now every compilers needs a .cfg file with the --gcc-install-dir option + # This list tells which compilers need to have a .cfg file created + # NOTE: flang is the expected name also for the 'flang-new' compiler + cfg_compilers = ['clang', 'clang++', 'flang'] + @staticmethod def extra_options(): extra_vars = CMakeMake.extra_options() extra_vars.update({ + 'amd_gfx_list': [None, "List of AMDGPU targets to build for. Possible values: " + + ', '.join(AMDGPU_GFX_SUPPORT), CUSTOM], + 'assertions': [False, "Enable assertions. Helps to catch bugs in Clang.", CUSTOM], + 'bootstrap': [True, "Build LLVM-Clang using itself", CUSTOM], + 'build_bolt': [False, "Build the LLVM bolt binary optimizer", CUSTOM], + 'build_clang_extras': [False, "Build the LLVM Clang extra tools", CUSTOM], + 'build_lld': [False, "Build the LLVM lld linker", CUSTOM], + 'build_lldb': [False, "Build the LLVM lldb debugger", CUSTOM], + 'build_openmp': [True, "Build the LLVM OpenMP runtime", CUSTOM], + 'build_openmp_offload': [True, "Build the LLVM OpenMP offload runtime", CUSTOM], + 'build_openmp_tools': [True, "Build the LLVM OpenMP tools interface", CUSTOM], + 'build_runtimes': [False, "Build the LLVM runtimes (compiler-rt, libunwind, libcxx, libcxxabi)", CUSTOM], 'build_targets': [None, "Build targets for LLVM (host architecture if None). Possible values: " + - ', '.join(CLANG_TARGETS), CUSTOM], + ', '.join(ALL_TARGETS), CUSTOM], + 'debug_tests': [True, "Enable verbose output for tests", CUSTOM], + 'disable_werror': [False, "Disable -Werror for all projects", CUSTOM], 'enable_rtti': [True, "Enable RTTI", CUSTOM], + 'full_llvm': [False, "Build LLVM without any dependency", CUSTOM], + 'minimal': [False, "Build LLVM only", CUSTOM], + 'python_bindings': [False, "Install python bindings", CUSTOM], + 'skip_all_tests': [False, "Skip running of tests", CUSTOM], + 'skip_sanitizer_tests': [True, "Do not run the sanitizer tests", CUSTOM], + 'test_suite_ignore_patterns': [None, "List of test to ignore (if the string matches)", CUSTOM], + 'test_suite_max_failed': [0, "Maximum number of failing tests (does not count allowed failures)", CUSTOM], + 'test_suite_timeout_single': [None, "Timeout for each individual test in the test suite", CUSTOM], + 'test_suite_timeout_total': [None, "Timeout for total running time of the testsuite", CUSTOM], + 'use_pic': [True, "Build with Position Independent Code (PIC)", CUSTOM], + 'usepolly': [False, "Build Clang with polly", CUSTOM], }) return extra_vars @@ -60,65 +227,1019 @@ def __init__(self, *args, **kwargs): """Initialize LLVM-specific variables.""" super(EB_LLVM, self).__init__(*args, **kwargs) + self.llvm_src_dir = None + self.llvm_obj_dir_stage1 = None + self.llvm_obj_dir_stage2 = None + self.llvm_obj_dir_stage3 = None + self.intermediate_projects = ['llvm', 'clang'] + self.intermediate_runtimes = ['compiler-rt', 'libunwind', 'libcxx', 'libcxxabi'] + if not self.cfg['minimal']: + self.final_projects = ['llvm', 'mlir', 'clang', 'flang'] + else: + self.final_projects = ['llvm'] + self.final_runtimes = [] + self.gcc_prefix = None + self.runtimes_cmake_args = { + 'CMAKE_C_COMPILER': [], + 'CMAKE_C_FLAGS': [], + 'CMAKE_CXX_COMPILER': [], + 'CMAKE_CXX_FLAGS': [], + 'CMAKE_EXE_LINKER_FLAGS': [], + } + self.offload_targets = ['host'] + # self._added_librt = None + + # Shared + off_opts, on_opts = [], [] self.build_shared = self.cfg.get('build_shared_libs', False) - if LooseVersion(self.version) >= LooseVersion('14'): - self.cfg['start_dir'] = '%s-%s.src' % (self.name.lower(), self.version) - # avoid using -DBUILD_SHARED_LIBS directly, use -DLLVM_{BUILD,LINK}_LLVM_DYLIB flags instead - if self.build_shared: - self.cfg['build_shared_libs'] = None + if self.build_shared: + self.cfg['build_shared_libs'] = None + on_opts.extend(['LLVM_BUILD_LLVM_DYLIB', 'LLVM_LINK_LLVM_DYLIB', 'LIBCXX_ENABLE_SHARED', + 'LIBCXXABI_ENABLE_SHARED', 'LIBUNWIND_ENABLE_SHARED']) + else: + off_opts.extend(['LIBCXX_ENABLE_ABI_LINKER_SCRIPT', 'LIBCXX_ENABLE_SHARED', 'LIBCXXABI_ENABLE_SHARED', + 'LIBUNWIND_ENABLE_SHARED', 'LLVM_BUILD_LLVM_DYLIB', 'LLVM_LINK_LLVM_DYLIB']) + on_opts.extend(['LIBCXX_ENABLE_STATIC', 'LIBCXX_ENABLE_STATIC_ABI_LIBRARY', 'LIBCXXABI_ENABLE_STATIC', + 'LIBUNWIND_ENABLE_STATIC']) - def configure_step(self): - """ - Install extra tools in bin/; enable zlib if it is a dep; optionally enable rtti; and set the build target - """ - if LooseVersion(self.version) >= LooseVersion('14'): - self.cfg.update('configopts', '-DLLVM_INCLUDE_BENCHMARKS=OFF') - if self.build_shared: - self.cfg.update('configopts', '-DLLVM_BUILD_LLVM_DYLIB=ON -DLLVM_LINK_LLVM_DYLIB=ON') + # RTTI + if self.cfg['enable_rtti']: + on_opts.extend(['LLVM_ENABLE_RTTI', 'LLVM_REQUIRES_RTTI']) + # Does not work yet with Flang + # on_opts.append('LLVM_ENABLE_EH') + + if self.cfg['use_pic']: + on_opts.append('CMAKE_POSITION_INDEPENDENT_CODE') + + for opt in on_opts: + general_opts[opt] = 'ON' + + for opt in off_opts: + general_opts[opt] = 'OFF' + + self.full_llvm = self.cfg['full_llvm'] + + if self.cfg['minimal']: + conflicts = [_ for _ in self.minimal_conflicts if self.cfg[_]] + if conflicts: + raise EasyBuildError("Minimal build conflicts with '%s'", ', '.join(conflicts)) + + # Other custom options + if self.full_llvm: + if not self.cfg['bootstrap']: + raise EasyBuildError("Full LLVM build requires bootstrap build") + if not self.cfg['build_lld']: + raise EasyBuildError("Full LLVM build requires building lld") + if not self.cfg['build_runtimes']: + raise EasyBuildError("Full LLVM build requires building runtimes") + self.log.info("Building LLVM without any GCC dependency") + + if self.cfg['disable_werror']: + general_opts.update(disable_werror) + + if self.cfg['build_runtimes']: + self.final_runtimes += ['compiler-rt', 'libunwind', 'libcxx', 'libcxxabi'] + + if self.cfg['build_openmp']: + self.final_projects.append('openmp') + + if self.cfg['build_openmp_offload']: + if not self.cfg['build_openmp']: + raise EasyBuildError("Building OpenMP offload requires building OpenMP runtime") + # LLVM 19 added a new runtime target for explicit offloading + # https://discourse.llvm.org/t/llvm-19-1-0-no-library-libomptarget-nvptx-sm-80-bc-found/81343 + if LooseVersion(self.version) >= LooseVersion('19'): + self.log.debug("Explicitly enabling OpenMP offloading for LLVM >= 19") + self.final_runtimes.append('offload') + else: + self.log.warning("OpenMP offloading is included with the OpenMP runtime for LLVM < 19") + + if self.cfg['build_openmp_tools']: + if not self.cfg['build_openmp']: + raise EasyBuildError("Building OpenMP tools requires building OpenMP runtime") + + if self.cfg['usepolly']: + self.final_projects.append('polly') + + if self.cfg['build_clang_extras']: + self.final_projects.append('clang-tools-extra') + + if self.cfg['build_lld']: + self.intermediate_projects.append('lld') + self.final_projects.append('lld') + # This should be the default to make offload multi-stage compilations easier + general_opts['CLANG_DEFAULT_LINKER'] = 'lld' + general_opts['FLANG_DEFAULT_LINKER'] = 'lld' + + if self.cfg['build_lldb']: + self.final_projects.append('lldb') + if self.full_llvm: + remove_gcc_dependency_opts['LLDB_ENABLE_LIBXML2'] = 'Off' + remove_gcc_dependency_opts['LLDB_ENABLE_LZMA'] = 'Off' + remove_gcc_dependency_opts['LLDB_ENABLE_PYTHON'] = 'Off' - self.cfg.update('configopts', '-DLLVM_INSTALL_UTILS=ON') + if self.cfg['build_bolt']: + self.final_projects.append('bolt') - if get_software_root('zlib'): - self.cfg.update('configopts', '-DLLVM_ENABLE_ZLIB=ON') + # Sysroot + sysroot = build_option('sysroot') + if sysroot: + general_opts['DEFAULT_SYSROOT'] = sysroot + general_opts['CMAKE_SYSROOT'] = sysroot - if self.cfg["enable_rtti"]: - self.cfg.update('configopts', '-DLLVM_ENABLE_RTTI=ON') + # list of CUDA compute capabilities to use can be specifed in two ways (where (2) overrules (1)): + # (1) in the easyconfig file, via the custom cuda_compute_capabilities; + # (2) in the EasyBuild configuration, via --cuda-compute-capabilities configuration option; + cuda_cc_list = build_option('cuda_compute_capabilities') or self.cfg['cuda_compute_capabilities'] or [] + amd_gfx_list = self.cfg['amd_gfx_list'] or [] - build_targets = self.cfg['build_targets'] - if build_targets is None: + # Build targets + build_targets = self.cfg['build_targets'] or [] + if not build_targets: + self.log.debug("No build targets specified, using default detection") + deps = [dep['name'].lower() for dep in self.cfg.dependencies()] arch = get_cpu_architecture() - try: - default_targets = DEFAULT_TARGETS_MAP[arch][:] - self.cfg['build_targets'] = build_targets = default_targets - self.log.debug("Using %s as default build targets for CPU architecture %s.", default_targets, arch) - except KeyError: + if arch not in DEFAULT_TARGETS_MAP: raise EasyBuildError("No default build targets defined for CPU architecture %s.", arch) + build_targets += DEFAULT_TARGETS_MAP[arch] - unknown_targets = [target for target in build_targets if target not in CLANG_TARGETS] + # If CUDA is included as a dep, add NVPTX as a target + # There are (old) toolchains with CUDA as part of the toolchain + cuda_toolchain = hasattr(self.toolchain, 'COMPILER_CUDA_FAMILY') + if 'cuda' in deps or cuda_toolchain or cuda_cc_list: + if LooseVersion(self.version) < LooseVersion('18'): + self.log.info(f"Not auto-enabling {BUILD_TARGET_NVPTX} offload target, only done for LLVM >= 18") + else: + build_targets.append(BUILD_TARGET_NVPTX) + self.offload_targets += ['cuda'] # Used for LLVM >= 19 + self.log.debug(f"{BUILD_TARGET_NVPTX} enabled by CUDA dependency/cuda_compute_capabilities") + + # For AMDGPU support we need ROCR-Runtime and + # ROCT-Thunk-Interface, however, since ROCT is a dependency of + # ROCR we only check for the ROCR-Runtime here + # https://openmp.llvm.org/SupportAndFAQ.html#q-how-to-build-an-openmp-amdgpu-offload-capable-compiler + if 'rocr-runtime' in deps or amd_gfx_list: + if LooseVersion(self.version) < LooseVersion('18'): + self.log.info(f"Not auto-enabling {BUILD_TARGET_AMDGPU} offload target, only done for LLVM >= 18") + else: + build_targets.append(BUILD_TARGET_AMDGPU) + self.offload_targets += ['amdgpu'] # Used for LLVM >= 19 + self.log.debug(f"{BUILD_TARGET_AMDGPU} enabled by rocr-runtime dependency/amd_gfx_list") + + self.cfg['build_targets'] = build_targets + self.log.debug("Using %s as default build targets for CPU architecture %s.", build_targets, arch) + + unknown_targets = set(build_targets) - set(ALL_TARGETS) if unknown_targets: raise EasyBuildError("Some of the chosen build targets (%s) are not in %s.", - ', '.join(unknown_targets), ', '.join(CLANG_TARGETS)) + ', '.join(unknown_targets), ', '.join(ALL_TARGETS)) + exp_targets = set(build_targets) & set(LLVM_EXPERIMENTAL_TARGETS) + if exp_targets: + self.log.warning("Experimental targets %s are being used.", ', '.join(exp_targets)) + + self.build_targets = build_targets or [] + + self.cuda_cc = [cc.replace('.', '') for cc in cuda_cc_list] + if BUILD_TARGET_NVPTX in self.build_targets and not self.cuda_cc: + raise EasyBuildError("Can't build Clang with CUDA support without specifying 'cuda-compute-capabilities'") + if self.cuda_cc and BUILD_TARGET_NVPTX not in self.build_targets: + print_warning("CUDA compute capabilities specified, but NVPTX not in manually specified build targets.") + + self.amd_gfx = amd_gfx_list + if BUILD_TARGET_AMDGPU in self.build_targets and not self.amd_gfx: + raise EasyBuildError("Can't build Clang with AMDGPU support without specifying 'amd_gfx_list'") + if self.amd_gfx and BUILD_TARGET_AMDGPU not in self.build_targets: + print_warning("'amd_gfx' specified, but AMDGPU not in manually specified build targets.") + + general_opts['CMAKE_BUILD_TYPE'] = self.build_type + general_opts['CMAKE_INSTALL_PREFIX'] = self.installdir + + general_opts['LLVM_TARGETS_TO_BUILD'] = '"%s"' % ';'.join(build_targets) + + self._cmakeopts = {} + self._cfgopts = list(filter(None, self.cfg.get('configopts', '').split())) + self.llvm_src_dir = os.path.join(self.builddir, 'llvm-project-%s.src' % self.version) + + def _add_cmake_runtime_args(self): + """Generate the value for 'RUNTIMES_CMAKE_ARGS' and add it to the cmake options.""" + if self.runtimes_cmake_args: + args = [] + for key, val in self.runtimes_cmake_args.items(): + if isinstance(val, list): + val = ' '.join(val) + if val: + args.append('-D%s=%s' % (key, val)) + self._cmakeopts['RUNTIMES_CMAKE_ARGS'] = '"%s"' % ';'.join(args) + + def _configure_general_build(self): + """General configuration step for LLVM.""" + self._cmakeopts.update(general_opts) + self._add_cmake_runtime_args() + + def _configure_intermediate_build(self): + """Configure the intermediate stages of the build.""" + self._cmakeopts['LLVM_ENABLE_PROJECTS'] = '"%s"' % ';'.join(self.intermediate_projects) + self._cmakeopts['LLVM_ENABLE_RUNTIMES'] = '"%s"' % ';'.join(self.intermediate_runtimes) + + def _configure_final_build(self): + """Configure the final stage of the build.""" + self._cmakeopts['LLVM_ENABLE_PROJECTS'] = '"%s"' % ';'.join(self.final_projects) + self._cmakeopts['LLVM_ENABLE_RUNTIMES'] = '"%s"' % ';'.join(self.final_runtimes) + + hwloc_root = get_software_root('hwloc') + if hwloc_root: + self.log.info("Using %s as hwloc root", hwloc_root) + self._cmakeopts['LIBOMP_USE_HWLOC'] = 'ON' + self._cmakeopts['LIBOMP_HWLOC_INSTALL_DIR'] = hwloc_root + + if 'openmp' in self.final_projects: + if LooseVersion(self.version) >= LooseVersion('19') and self.cfg['build_openmp_offload']: + self._cmakeopts['LIBOMPTARGET_PLUGINS_TO_BUILD'] = ';'.join(self.offload_targets) + self._cmakeopts['OPENMP_ENABLE_LIBOMPTARGET'] = 'ON' + self._cmakeopts['LIBOMP_INSTALL_ALIASES'] = 'OFF' + if not self.cfg['build_openmp_tools']: + self._cmakeopts['OPENMP_ENABLE_OMPT_TOOLS'] = 'OFF' + + # Make sure tests are not running with more than 'parallel' tasks + parallel = self.cfg.parallel + if not build_option('mpi_tests'): + parallel = 1 + lit_args = [f'-j {parallel}'] + if self.cfg['debug_tests']: + lit_args += ['-v'] + timeout_single = self.cfg['test_suite_timeout_single'] + if timeout_single: + lit_args += ['--timeout', str(timeout_single)] + timeout_total = self.cfg['test_suite_timeout_total'] + if timeout_total: + lit_args += ['--max-time', str(timeout_total)] + self._cmakeopts['LLVM_LIT_ARGS'] = '"%s"' % ' '.join(lit_args) + + if self.cfg['usepolly']: + self._cmakeopts['LLVM_POLLY_LINK_INTO_TOOLS'] = 'ON' + if not self.cfg['skip_all_tests']: + self._cmakeopts['LLVM_INCLUDE_TESTS'] = 'ON' + self._cmakeopts['LLVM_BUILD_TESTS'] = 'ON' + + @staticmethod + def _get_gcc_prefix(): + """Get the GCC prefix for the build.""" + arch = get_cpu_architecture() + gcc_root = get_software_root('GCCcore') + gcc_version = get_software_version('GCCcore') + # If that doesn't work, try with GCC + if gcc_root is None: + gcc_root = get_software_root('GCC') + gcc_version = get_software_version('GCC') + # If that doesn't work either, print error and exit + if gcc_root is None: + raise EasyBuildError("Can't find GCC or GCCcore to use") + + pattern = os.path.join(gcc_root, 'lib', 'gcc', f'{arch}-*', gcc_version) + matches = glob.glob(pattern) + if not matches: + raise EasyBuildError("Can't find GCC version %s for architecture %s in %s", gcc_version, arch, pattern) + gcc_prefix = os.path.abspath(matches[0]) + + return gcc_root, gcc_prefix + + def _set_gcc_prefix(self): + """Set the GCC prefix for the build.""" + if self.gcc_prefix is None: + gcc_root, gcc_prefix = self._get_gcc_prefix() + + # --gcc-toolchain and --gcc-install-dir for flang are not supported before LLVM 19 + # https://github.com/llvm/llvm-project/pull/87360 + if LooseVersion(self.version) < LooseVersion('19'): + self.log.debug("Using GCC_INSTALL_PREFIX") + general_opts['GCC_INSTALL_PREFIX'] = gcc_root + else: + # See https://github.com/llvm/llvm-project/pull/85891#issuecomment-2021370667 + self.log.debug("Using '--gcc-install-dir' in CMAKE_C_FLAGS and CMAKE_CXX_FLAGS") + self.runtimes_cmake_args['CMAKE_C_FLAGS'] += ['--gcc-install-dir=%s' % gcc_prefix] + self.runtimes_cmake_args['CMAKE_CXX_FLAGS'] += ['--gcc-install-dir=%s' % gcc_prefix] + + self.gcc_prefix = gcc_prefix + self.log.debug("Using %s as the gcc install location", self.gcc_prefix) + + def configure_step(self): + """ + Install extra tools in bin/; enable zlib if it is a dep; optionally enable rtti; and set the build target + """ + # Allow running with older versions of LLVM for minimal builds in order to replace EB_LLVM easyblock + if not self.cfg['minimal'] and LooseVersion(self.version) < LooseVersion('18.1.6'): + raise EasyBuildError("LLVM version %s is not supported, please use version 18.1.6 or newer", self.version) + + # Allow running with older versions of LLVM for minimal builds in order to replace EB_LLVM easyblock + gcc_version = get_software_version('GCCcore') + if not self.cfg['minimal'] and LooseVersion(gcc_version) < LooseVersion('13'): + raise EasyBuildError("LLVM %s requires GCC 13 or newer, found %s", self.version, gcc_version) + + # Lit is needed for running tests-suite + lit_root = get_software_root('lit') + if not lit_root: + if not self.cfg['skip_all_tests']: + raise EasyBuildError("Can't find 'lit', needed for running tests-suite") + + timeouts = self.cfg['test_suite_timeout_single'] or self.cfg['test_suite_timeout_total'] + if not self.cfg['skip_all_tests'] and timeouts: + psutil_root = get_software_root('psutil') + if not psutil_root: + raise EasyBuildError("Can't find 'psutil', needed for running tests-suite with timeout") + + # Parallel build + self.make_parallel_opts = "" + if self.cfg.parallel: + self.make_parallel_opts = f"-j {self.cfg.parallel}" + + # Bootstrap + self.llvm_obj_dir_stage1 = os.path.join(self.builddir, 'llvm.obj.1') + if self.cfg['bootstrap']: + self.log.info("Initialising for bootstrap build.") + self.llvm_obj_dir_stage2 = os.path.join(self.builddir, 'llvm.obj.2') + self.llvm_obj_dir_stage3 = os.path.join(self.builddir, 'llvm.obj.3') + self.final_dir = self.llvm_obj_dir_stage3 + mkdir(self.llvm_obj_dir_stage2) + mkdir(self.llvm_obj_dir_stage3) + else: + self.log.info("Initialising for single stage build.") + self.final_dir = self.llvm_obj_dir_stage1 + + general_opts['LLVM_ENABLE_ASSERTIONS'] = 'ON' if self.cfg['assertions'] else 'OFF' + + # Dependencies based persistent options (should be reused across stages) + # Libxml2 + xml2_root = get_software_root('libxml2') + # Explicitly disable libxml2 if not found to avoid linking against system libxml2 + if xml2_root: + if self.full_llvm: + self.log.warning("LLVM is being built in 'full_llvm' mode, libxml2 will not be used") + general_opts['LLVM_ENABLE_LIBXML2'] = 'OFF' + else: + general_opts['LLVM_ENABLE_LIBXML2'] = 'ON' + else: + general_opts['LLVM_ENABLE_LIBXML2'] = 'OFF' + + # If 'ON', risk finding a system zlib or zstd leading to including /usr/include as -isystem that can lead + # to errors during compilation of 'offload.tools.kernelreplay' due to the inclusion of LLVMSupport (19.x) + general_opts['LLVM_ENABLE_ZLIB'] = 'ON' if get_software_root('zlib') else 'OFF' + general_opts['LLVM_ENABLE_ZSTD'] = 'ON' if get_software_root('zstd') else 'OFF' + # Should not use system SWIG if present + general_opts['LLDB_ENABLE_SWIG'] = 'ON' if get_software_root('SWIG') else 'OFF' + + z3_root = get_software_root("Z3") + if z3_root: + self.log.info("Using %s as Z3 root", z3_root) + general_opts['LLVM_ENABLE_Z3_SOLVER'] = 'ON' + general_opts['LLVM_Z3_INSTALL_DIR'] = z3_root + else: + general_opts['LLVM_ENABLE_Z3_SOLVER'] = 'OFF' + + python_opts = get_cmake_python_config_dict() + general_opts.update(python_opts) + self.runtimes_cmake_args.update(python_opts) + + if self.cfg['bootstrap']: + self._configure_intermediate_build() + else: + self._configure_final_build() + + if self.cfg['skip_sanitizer_tests'] and build_option('strict') != ERROR: + self.log.info("Disabling the sanitizer tests") + self.disable_sanitizer_tests() + + # Remove python bindings tests causing uncaught exception in the build + cmakelists_tests = os.path.join(self.llvm_src_dir, 'clang', 'CMakeLists.txt') + regex_subs = [] + regex_subs.append((r'add_subdirectory\(bindings/python/tests\)', '')) + apply_regex_substitutions(cmakelists_tests, regex_subs) - self.cfg.update('configopts', '-DLLVM_TARGETS_TO_BUILD="%s"' % ';'.join(build_targets)) + self._set_gcc_prefix() - if LooseVersion(self.version) >= LooseVersion('15.0'): - # make sure that CMake modules are available in build directory, - # by moving the extracted folder to the expected location - cmake_modules_path = os.path.join(self.builddir, 'cmake-%s.src' % self.version) - if os.path.exists(cmake_modules_path): - move_file(cmake_modules_path, os.path.join(self.builddir, 'cmake')) + # If we don't want to build with CUDA (not in dependencies) trick CMakes FindCUDA module into not finding it by + # using the environment variable which is used as-is and later checked for a falsy value when determining + # whether CUDA was found + if not get_software_root('CUDA'): + setvar('CUDA_NVCC_EXECUTABLE', 'IGNORE') + + if 'openmp' in self.final_projects: + gpu_archs = [] + gpu_archs += ['sm_%s' % cc for cc in self.cuda_cc] + gpu_archs += self.amd_gfx + if gpu_archs: + general_opts['LIBOMPTARGET_DEVICE_ARCHITECTURES'] = '"%s"' % ';'.join(gpu_archs) + + self._configure_general_build() + self.add_cmake_opts() + + src_dir = os.path.join(self.llvm_src_dir, 'llvm') + super(EB_LLVM, self).configure_step(builddir=self.llvm_obj_dir_stage1, srcdir=src_dir) + + def disable_sanitizer_tests(self): + """Disable the tests of all the sanitizers by removing the test directories from the build system""" + cmakelists_tests = os.path.join(self.llvm_src_dir, 'compiler-rt', 'test', 'CMakeLists.txt') + regex_subs = [(r'compiler_rt_test_runtime.*san.*', '')] + apply_regex_substitutions(cmakelists_tests, regex_subs) + + def add_cmake_opts(self): + """Add LLVM-specific CMake options.""" + base_opts = self._cfgopts.copy() + for k, v in self._cmakeopts.items(): + base_opts.append('-D%s=%s' % (k, v)) + self.cfg['configopts'] = ' '.join(base_opts) + + def configure_step2(self): + """Configure the second stage of the bootstrap.""" + self._cmakeopts = {} + self._configure_general_build() + self._configure_intermediate_build() + if self.full_llvm: + self._cmakeopts.update(remove_gcc_dependency_opts) + + def configure_step3(self): + """Configure the third stage of the bootstrap.""" + self._cmakeopts = {} + self._configure_general_build() + self._configure_final_build() + if self.full_llvm: + self._cmakeopts.update(remove_gcc_dependency_opts) + + @staticmethod + def _create_compiler_config_file(compilers, gcc_prefix, installdir): + """Create a config file for the compiler to point to the correct GCC installation.""" + bin_dir = os.path.join(installdir, 'bin') + prefix_str = '--gcc-install-dir=%s' % gcc_prefix + for comp in compilers: + write_file(os.path.join(bin_dir, f'{comp}.cfg'), prefix_str) + + def build_with_prev_stage(self, prev_dir, stage_dir): + """Build LLVM using the previous stage.""" + curdir = os.getcwd() + + bin_dir = os.path.join(prev_dir, 'bin') + lib_dir_runtime = self.get_runtime_lib_path(prev_dir, fail_ok=False) + + # Give priority to the libraries in the current stage if compiled to avoid failures due to undefined symbols + # e.g. when calling the compiled clang-ast-dump for stage 3 + lib_path = ':'.join(filter(None, [ + os.path.join(stage_dir, lib_dir_runtime), + os.path.join(prev_dir, lib_dir_runtime), + ])) + + ####################################################### + # PROBLEM!!!: + # Binaries and libraries produced during runtimes make use of the newly built Clang compiler which is not + # rpath-wrapped. This causes the executable to be produced without rpath (if required) and with + # runpath set to $ORIGIN. This causes 2 problems: + # - Binaries produced for the runtimes will fail the sanity check + # - Runtimes libraries that link to libLLVM.so like 'libomptarget.so' need LD_LIBRARY_PATH to work. + # This is because even if an executable compiled with the new llvm has rpath pointing to $EBROOTLLVM/lib, + # it will not be resolved with the executable's rpath, but the library's runpath + # (rpath is ignored if runpath is set). + # Even if libLLVM.so is a direct dependency of the executable, it needs to be resolved both for the + # executable and the library. + # + # Here we create a mock binary for the current stage by copying the previous one, rpath-wrapping it and + # and than pass the rpath-wrapped binary to the runtimes build as the compiler. + ################################################# + bin_dir_new = os.path.join(stage_dir, 'bin') + with _wrap_env(bin_dir_new, lib_path): + if build_option('rpath'): + prev_clang = os.path.join(bin_dir, 'clang') + prev_clangxx = os.path.join(bin_dir, 'clang++') + nxt_clang = os.path.join(bin_dir_new, 'clang') + nxt_clangxx = os.path.join(bin_dir_new, 'clang++') + copy_file(prev_clang, nxt_clang) + copy_file(prev_clangxx, nxt_clangxx) + + tmp_toolchain = Clang(name='Clang', version='1') + # Don't need stage dir here as LD_LIBRARY_PATH is set during build, this is only needed for + # installed binaries with rpath + lib_dirs = [os.path.join(self.installdir, x) for x in SEARCH_PATH_LIB_DIRS + [lib_dir_runtime]] + tmp_toolchain.prepare_rpath_wrappers(rpath_include_dirs=lib_dirs) + remove_file(nxt_clang) + remove_file(nxt_clangxx) + msg = "Prepared MOCK rpath wrappers needed to rpath-wrap also the new compilers produced " + msg += "by the project build and than used for the runtimes build" + self.log.info(msg) + clang_mock = which('clang') + clangxx_mock = which('clang++') + + clang_mock_wrapper_dir = os.path.dirname(clang_mock) + + symlink(os.path.join(stage_dir, 'opt'), os.path.join(clang_mock_wrapper_dir, 'opt')) + + self.runtimes_cmake_args['CMAKE_C_COMPILER'] = [clang_mock] + self.runtimes_cmake_args['CMAKE_CXX_COMPILER'] = [clangxx_mock] + + # Needed for passing the variables to the build command + with _wrap_env(bin_dir, lib_path): + # If building with rpath, create RPATH wrappers for the Clang compilers for stage 2 and 3 + if build_option('rpath'): + # !!! Should be replaced with ClangFlang (or correct naming) toolchain once available + # as this will only create rpath wrappers for Clang and not Flang + my_toolchain = Clang(name='Clang', version='1') + my_toolchain.prepare_rpath_wrappers( + rpath_include_dirs=[ + os.path.join(self.installdir, 'lib'), + os.path.join(self.installdir, 'lib64'), + os.path.join(self.installdir, lib_dir_runtime), + ] + ) + self.log.info("Prepared rpath wrappers") + + # add symlink for 'opt' to wrapper dir, since Clang expects it in the same directory + # see https://github.com/easybuilders/easybuild-easyblocks/issues/3075 + clang_wrapper_dir = os.path.dirname(which('clang')) + symlink(os.path.join(prev_dir, 'opt'), os.path.join(clang_wrapper_dir, 'opt')) + + # RPATH wrappers add -Wl,rpath arguments to all command lines, including when it is just compiling + # Clang by default warns about that, and then some configure tests use -Werror which turns those + # warnings into errors. As a result, those configure tests fail, even though the compiler supports the + # requested functionality (e.g. the test that checks if -fPIC is supported would fail, and it compiles + # without resulting in relocation errors). + # See https://github.com/easybuilders/easybuild-easyblocks/pull/2799#issuecomment-1270621100 + # Here, we add -Wno-unused-command-line-argument to CXXFLAGS to avoid these warnings alltogether + cflags = os.getenv('CFLAGS', '') + cxxflags = os.getenv('CXXFLAGS', '') + setvar('CFLAGS', "%s %s" % (cflags, '-Wno-unused-command-line-argument')) + setvar('CXXFLAGS', "%s %s" % (cxxflags, '-Wno-unused-command-line-argument')) + + if self.full_llvm: + # See https://github.com/llvm/llvm-project/issues/111667 + to_add = '--unwindlib=none' + # for flags in ['CMAKE_C_FLAGS', 'CMAKE_CXX_FLAGS']: + for flags in ['CMAKE_EXE_LINKER_FLAGS']: + ptr = self.runtimes_cmake_args[flags] + if to_add not in ptr: + ptr.append(to_add) + + self._add_cmake_runtime_args() + + # determine full path to clang/clang++ (which may be wrapper scripts in case of RPATH linking) + clang = which('clang') + clangxx = which('clang++') + + self._cmakeopts['CMAKE_C_COMPILER'] = clang + self._cmakeopts['CMAKE_CXX_COMPILER'] = clangxx + self._cmakeopts['CMAKE_ASM_COMPILER'] = clang + self._cmakeopts['CMAKE_ASM_COMPILER_ID'] = 'Clang' + + # Also runs of the intermediate step compilers should be made aware of the GCC installation + if LooseVersion(self.version) >= LooseVersion('19'): + self._set_gcc_prefix() + self._create_compiler_config_file(self.cfg_compilers, self.gcc_prefix, prev_dir) + + self.add_cmake_opts() + + change_dir(stage_dir) + self.log.debug("Configuring %s", stage_dir) + cmd = ' '.join(['cmake', self.cfg['configopts'], os.path.join(self.llvm_src_dir, 'llvm')]) + run_shell_cmd(cmd) + + self.log.debug("Building %s", stage_dir) + cmd = f"make {self.make_parallel_opts} VERBOSE=1" + run_shell_cmd(cmd) + + change_dir(curdir) + + def build_step(self, *args, **kwargs): + """Build LLVM, and optionally build it using itself.""" + if self.cfg['bootstrap']: + self.log.info("Building stage 1") + print_msg("Building stage 1/3") + else: + self.log.info("Building LLVM") + print_msg("Building stage 1/1") + + change_dir(self.llvm_obj_dir_stage1) + super(EB_LLVM, self).build_step(*args, **kwargs) + + if self.cfg['bootstrap']: + self.log.info("Building stage 2") + print_msg("Building stage 2/3") + self.configure_step2() + self.build_with_prev_stage(self.llvm_obj_dir_stage1, self.llvm_obj_dir_stage2) + + self.log.info("Building stage 3") + print_msg("Building stage 3/3") + self.configure_step3() + self.build_with_prev_stage(self.llvm_obj_dir_stage2, self.llvm_obj_dir_stage3) + + def _para_test_step(self, parallel=1): + """Run test suite with the specified number of parallel jobs for make.""" + basedir = self.final_dir + + # From grep -E "^[A-Z]+: " LOG_FILE | cut -d: -f1 | sort | uniq + OUTCOMES_LOG = [ + 'FAIL', + 'TIMEOUT', + ] + # OUTCOMES_OK = [ + # 'PASS', + # 'UNSUPPORTED', + # 'XFAIL', + # ] + + change_dir(basedir) + lib_path = '' + if self.cfg['build_runtimes']: + lib_dir_runtime = self.get_runtime_lib_path(basedir, fail_ok=False) + lib_path = os.path.join(basedir, lib_dir_runtime) + + # When rpath is enabled, the easybuild rpath wrapper will be used for compiling the tests + # A combination of -Werror and the wrapper translating LD_LIBRARY_PATH to -Wl,... flags will results in failing + # tests due to -Wunused-command-line-argument + # This has shown to be a problem in builds for 18.1.8, but seems it was not necessary for LLVM >= 19 + # needs more digging into the CMake logic + old_cflags = os.getenv('CFLAGS', '') + old_cxxflags = os.getenv('CXXFLAGS', '') + # TODO: Find a better way to either force the test to use the non wrapped compiler or to pass the flags + if build_option('rpath'): + setvar('CFLAGS', "%s %s" % (old_cflags, '-Wno-unused-command-line-argument')) + setvar('CXXFLAGS', "%s %s" % (old_cxxflags, '-Wno-unused-command-line-argument')) + with _wrap_env(os.path.join(basedir, 'bin'), lib_path): + cmd = f"make -j {parallel} check-all" + res = run_shell_cmd(cmd, fail_on_error=False) + out = res.output + self.log.debug(out) + + # Reset the CFLAGS and CXXFLAGS + setvar('CFLAGS', old_cflags) + setvar('CXXFLAGS', old_cxxflags) + + ignore_patterns = self.cfg['test_suite_ignore_patterns'] or [] + ignored_pattern_matches = 0 + failed_pattern_matches = 0 + if ignore_patterns: + for line in out.splitlines(): + if any(line.startswith(f'{x}: ') for x in OUTCOMES_LOG): + if any(patt in line for patt in ignore_patterns): + self.log.info("Ignoring test failure: %s", line) + ignored_pattern_matches += 1 + failed_pattern_matches += 1 + + rgx_failed = re.compile(r'^ +Failed +: +([0-9]+)', flags=re.MULTILINE) + mch = rgx_failed.search(out) + if mch is None: + rgx_passed = re.compile(r'^ +Passed +: +([0-9]+)', flags=re.MULTILINE) + mch = rgx_passed.search(out) + if mch is None: + self.log.warning("Failed to extract number of failed/passed test results from output") + num_failed = None else: - raise EasyBuildError("Failed to find unpacked CMake modules directory at %s", cmake_modules_path) - - if LooseVersion(self.version) >= LooseVersion('16.0'): - # make sure that third-party modules are available in build directory, - # by moving the extracted folder to the expected location - third_party_modules_path = os.path.join(self.builddir, 'third-party-%s.src' % self.version) - if os.path.exists(third_party_modules_path): - move_file(third_party_modules_path, os.path.join(self.builddir, 'third-party')) + num_failed = 0 + else: + num_failed = int(mch.group(1)) + + if num_failed is not None: + num_timed_out = 0 + rgx_timed_out = re.compile(r'^ +Timed Out +: +([0-9]+)', flags=re.MULTILINE) + mch = rgx_timed_out.search(out) + if mch is not None: + num_timed_out = int(mch.group(1)) + self.log.info("Tests timed out: %s", num_timed_out) + num_failed += num_timed_out + + if num_failed != failed_pattern_matches: + msg = f"Number of failed tests ({num_failed}) does not match " + msg += f"number identified va line-by-line pattern matching: {failed_pattern_matches}" + self.log.warning(msg) + + if ignored_pattern_matches: + self.log.info("Ignored %s failed tests due to ignore patterns", ignored_pattern_matches) + num_failed -= ignored_pattern_matches + + return num_failed + + def test_step(self): + """Run tests on final stage (unless disabled).""" + if not self.cfg['skip_all_tests']: + # Also runs of test suite compilers should be made aware of the GCC installation + if LooseVersion(self.version) >= LooseVersion('19'): + self._set_gcc_prefix() + self._create_compiler_config_file(self.cfg_compilers, self.gcc_prefix, self.final_dir) + max_failed = self.cfg['test_suite_max_failed'] + num_failed = self._para_test_step(parallel=1) + if num_failed is None: + raise EasyBuildError("Failed to extract test results from output") + + if num_failed > max_failed: + raise EasyBuildError(f"Too many failed tests: {num_failed} ({max_failed} allowed)") + elif num_failed: + self.log.info(f"Test suite completed with {num_failed} failed tests ({max_failed} allowed)") + else: + self.log.info(f"Test suite completed, no failed tests ({max_failed} allowed)") + + def install_step(self): + """Install stage 1 or 3 (if bootstrap) binaries.""" + basedir = self.final_dir + change_dir(basedir) + + if self.cfg['build_runtimes']: + orig_ld_library_path = os.getenv('LD_LIBRARY_PATH') + lib_dir_runtime = self.get_runtime_lib_path(basedir, fail_ok=False) + + lib_path = ':'.join([ + os.path.join(basedir, lib_dir_runtime), + orig_ld_library_path + ]) + + self.cfg.update('preinstallopts', f'LD_LIBRARY_PATH={lib_path}') + + super(EB_LLVM, self).install_step() + + def post_processing_step(self): + """Install python bindings.""" + super(EB_LLVM, self).post_processing_step() + + # copy Python bindings here in post-install step so that it is not done more than once in multi_deps context + if self.cfg['python_bindings']: + python_bindings_source_dir = os.path.join(self.llvm_src_dir, 'clang', 'bindings', 'python') + python_bindins_target_dir = os.path.join(self.installdir, 'lib', 'python') + copy_dir(python_bindings_source_dir, python_bindins_target_dir, dirs_exist_ok=True) + + python_bindings_source_dir = os.path.join(self.llvm_src_dir, 'mlir', 'python') + copy_dir(python_bindings_source_dir, python_bindins_target_dir, dirs_exist_ok=True) + + if LooseVersion(self.version) >= LooseVersion('19'): + # For GCC aware installation create config files in order to point to the correct GCC installation + # Required as GCC_INSTALL_PREFIX was removed (see https://github.com/llvm/llvm-project/pull/87360) + self._set_gcc_prefix() + self._create_compiler_config_file(self.cfg_compilers, self.gcc_prefix, self.installdir) + + # This is needed as some older build system will select a different naming scheme for the library leading to + # The correct target <__config_site> and libclang_rt.builtins.a not being found + # An example is building BOOST + resdir_version = self.version.split('.')[0] + clang_lib = os.path.join(self.installdir, 'lib', 'clang', resdir_version, 'lib') + + for orig, other in self.symlink_lst: + for dirname in ['include', 'lib', clang_lib]: + src = os.path.join(self.installdir, dirname, orig) + dst = os.path.join(self.installdir, dirname, other) + if os.path.exists(src) and not os.path.exists(dst): + msg = f"Creating symlink for {src} to {dst} for better compatibility with expected --target" + self.log.info(msg) + symlink(src, dst) + + def get_runtime_lib_path(self, base_dir, fail_ok=True): + """Return the path to the runtime libraries.""" + arch = get_cpu_architecture() + glob_pattern = os.path.join(base_dir, 'lib', '%s-*' % arch) + matches = glob.glob(glob_pattern) + if matches: + directory = os.path.basename(matches[0]) + res = os.path.join("lib", directory) + else: + if not fail_ok: + raise EasyBuildError("Could not find runtime library directory") + print_warning("Could not find runtime library directory") + res = 'lib' + + return res + + def banned_linked_shared_libs(self): + """Return a list of shared libraries that should not be linked against.""" + res = [] + if self.full_llvm: + self.log.info("Checking that no GCC shared libraries are linked against") + res += ['libstdc++', 'libgcc_s', 'libicuuc'] + if not self.build_shared: + # Libraries should be linked statically + self.log.info("Checking that no shared libraries are linked against in static build") + res += ['libc++', 'libc++abi', 'libunwind'] + return res + + @staticmethod + def _sanity_check_gcc_prefix(compilers, gcc_prefix, installdir): + """Check if the GCC prefix of the compiler is correct""" + rgx = re.compile('Selected GCC installation: (.*)') + for comp in compilers: + cmd = "%s -v" % os.path.join(installdir, 'bin', comp) + res = run_shell_cmd(cmd, fail_on_error=False) + out = res.output + mch = rgx.search(out) + if mch is None: + raise EasyBuildError("Failed to extract GCC installation path from output of '%s': %s", cmd, out) + check_prefix = mch.group(1) + if check_prefix != gcc_prefix: + error_msg = "GCC installation path '{check_prefix}' does not match expected path '{gcc_prefix}'" + raise EasyBuildError(error_msg) + + def sanity_check_step(self, custom_paths=None, custom_commands=None, extension=False, extra_modules=None): + """Perform sanity checks on the installed LLVM.""" + lib_dir_runtime = None + if self.cfg['build_runtimes']: + lib_dir_runtime = self.get_runtime_lib_path(self.installdir, fail_ok=False) + shlib_ext = '.' + get_shared_lib_ext() + + resdir_version = self.version.split('.')[0] + + # Detect OpenMP support for CPU architecture + arch = get_cpu_architecture() + # Check architecture explicitly since Clang uses potentially different names + if arch == X86_64: + arch = 'x86_64' + elif arch == POWER: + arch = 'ppc64' + elif arch == AARCH64: + arch = 'aarch64' + + check_files = [] + check_bin_files = [] + check_lib_files = [] + check_librt_files = [] + check_inc_files = [] + check_dirs = ['include/llvm', 'include/llvm-c', 'lib/cmake/llvm'] + custom_commands = [ + "llvm-ar --help", + "llvm-ranlib --help", + "llvm-nm --help", + "llvm-objdump --help", + ] + gcc_prefix_compilers = [] + if self.build_shared: + check_lib_files += ['libLLVM.so'] + + if 'clang' in self.final_projects: + check_bin_files += [ + 'clang', 'clang++', 'clang-cpp', 'clang-cl', 'clang-repl', 'hmaptool', 'amdgpu-arch', 'nvptx-arch', + 'intercept-build', 'scan-build', 'scan-build-py', 'scan-view', 'analyze-build', 'c-index-test', + 'clang-tblgen', + ] + check_lib_files += [ + 'libclang.so', 'libclang-cpp.so', 'libclangAST.a', 'libclangCrossTU.a', 'libclangFrontend.a', + 'libclangInterpreter.a', 'libclangParse.a', 'libclangTooling.a' + ] + check_dirs += [ + 'lib/cmake/clang', 'include/clang' + ] + custom_commands += ['llvm-config --cxxflags', 'clang --help', 'clang++ --help'] + gcc_prefix_compilers += ['clang', 'clang++'] + + if 'clang-tools-extra' in self.final_projects: + # clang-pseudo removed with LLVM 20 + check_bin_files += [ + 'clangd', 'clang-tidy', 'clang-include-fixer', 'clang-query', 'clang-move', + 'clang-reorder-fields', 'clang-include-cleaner', 'clang-apply-replacements', + 'clang-change-namespace', 'pp-trace', 'modularize' + ] + check_lib_files += [ + 'libclangTidy.a', 'libclangQuery.a', 'libclangIncludeFixer.a', 'libclangIncludeCleaner.a', + ] + check_dirs += ['include/clang-tidy'] + if 'flang' in self.final_projects: + if LooseVersion(self.version) < LooseVersion('19'): + check_bin_files += ['bbc', 'flang-new', 'flang-to-external-fc', 'f18-parse-demo', 'fir-opt', 'tco'] else: - raise EasyBuildError("Failed to find unpacked 'third-party' modules directory at %s", - third_party_modules_path) + check_bin_files += ['bbc', 'flang-new', 'f18-parse-demo', 'fir-opt', 'tco'] + check_lib_files += [ + 'libFortranRuntime.a', 'libFortranSemantics.a', 'libFortranLower.a', 'libFortranParser.a', + 'libFIRCodeGen.a', 'libflangFrontend.a', 'libFortranCommon.a', 'libFortranDecimal.a', + 'libHLFIRDialect.a' + ] + check_dirs += ['lib/cmake/flang', 'include/flang'] + custom_commands += ['bbc --help', 'mlir-tblgen --help', 'flang-new --help'] + gcc_prefix_compilers += ['flang-new'] + + if 'lld' in self.final_projects: + check_bin_files += ['ld.lld', 'lld', 'lld-link', 'wasm-ld'] + check_lib_files += [ + 'liblldCOFF.a', 'liblldCommon.a', 'liblldELF.a', 'liblldMachO.a', 'liblldMinGW.a', 'liblldWasm.a' + ] + check_dirs += ['lib/cmake/lld', 'include/lld'] + if 'lldb' in self.final_projects: + check_bin_files += ['lldb'] + if self.build_shared: + check_lib_files += ['liblldb.so'] + check_dirs += ['include/lldb'] + if 'mlir' in self.final_projects: + check_bin_files += ['mlir-opt', 'tblgen-to-irdl', 'mlir-pdll'] + check_lib_files += [ + 'libMLIRIR.a', 'libmlir_async_runtime.so', 'libmlir_arm_runner_utils.so', 'libmlir_c_runner_utils.so', + 'libmlir_float16_utils.so' + ] + check_dirs += ['lib/cmake/mlir', 'include/mlir', 'include/mlir-c'] + if 'libunwind' in self.final_runtimes: + check_librt_files += ['libunwind.a'] + if self.build_shared: + check_librt_files += ['libunwind.so'] + check_inc_files += ['unwind.h', 'libunwind.h', 'mach-o/compact_unwind_encoding.h'] + if 'libcxx' in self.final_runtimes: + check_librt_files += ['libc++.a'] + if self.build_shared: + check_librt_files += ['libc++.so'] + check_dirs += ['include/c++/v1'] + if 'libcxxabi' in self.final_runtimes: + check_librt_files += ['libc++abi.a'] + if self.build_shared: + check_librt_files += ['libc++abi.so'] + + if 'polly' in self.final_projects: + check_lib_files += ['libPolly.a', 'libPollyISL.a'] + if self.build_shared: + check_lib_files += ['libPolly.so'] + check_dirs += ['lib/cmake/polly', 'include/polly'] + custom_commands += [ + ' | '.join([ + 'echo \'int main(int argc, char **argv) { return 0; }\'', + 'clang -xc -O3 -mllvm -polly -' + ]) + ' && ./a.out && rm -f a.out' + ] + if 'bolt' in self.final_projects: + check_bin_files += ['llvm-bolt', 'llvm-boltdiff', 'llvm-bolt-heatmap'] + check_lib_files += ['libbolt_rt_instr.a'] + custom_commands += ['llvm-bolt --help'] + if 'openmp' in self.final_projects: + omp_lib_files = [] + omp_lib_files += ['libomp.so', 'libompd.so'] + if self.cfg['build_openmp_offload']: + # Judging from the build process/logs of LLVM 19, the omptarget plugins (rtl..so) are now built + # as static libraries and linked into the libomptarget.so shared library + omp_lib_files += ['libomptarget.so'] + if LooseVersion(self.version) < LooseVersion('19'): + omp_lib_files += ['libomptarget.rtl.%s.so' % arch] + if 'NVPTX' in self.cfg['build_targets']: + if LooseVersion(self.version) < LooseVersion('19'): + omp_lib_files += ['libomptarget.rtl.cuda.so'] + omp_lib_files += ['libomptarget-nvptx-sm_%s.bc' % cc for cc in self.cuda_cc] + if 'AMDGPU' in self.cfg['build_targets']: + if LooseVersion(self.version) < LooseVersion('19'): + omp_lib_files += ['libomptarget.rtl.amdgpu.so'] + omp_lib_files += ['llibomptarget-amdgpu-%s.bc' % gfx for gfx in self.amd_gfx] + + if LooseVersion(self.version) < LooseVersion('19'): + # Before LLVM 19, omp related libraries are installed under 'ROOT/lib'' + check_lib_files += omp_lib_files + else: + # Starting from LLVM 19, omp related libraries are installed the runtime library directory + check_librt_files += omp_lib_files + check_bin_files += ['llvm-omp-kernel-replay', 'llvm-omp-device-info'] + + if self.cfg['build_openmp_tools']: + check_files += [os.path.join('lib', 'clang', resdir_version, 'include', 'ompt.h')] + if LooseVersion(self.version) < LooseVersion('19'): + check_lib_files += ['libarcher.so'] + elif LooseVersion(self.version) >= LooseVersion('19'): + check_librt_files += ['libarcher.so'] + if self.cfg['python_bindings']: + custom_commands += ["python -c 'import clang'"] + custom_commands += ["python -c 'import mlir'"] + + for libso in filter(lambda x: x.endswith('.so'), check_lib_files): + libext = libso.replace('.so', shlib_ext) + if libext not in check_lib_files: + check_lib_files.append(libext) + check_lib_files.remove(libso) + + check_files += [os.path.join('bin', x) for x in check_bin_files] + check_files += [os.path.join('lib', x) for x in check_lib_files] + check_files += [os.path.join(lib_dir_runtime, x) for x in check_librt_files] + check_files += [os.path.join('include', x) for x in check_inc_files] + + custom_paths = { + 'files': check_files, + 'dirs': check_dirs, + } + + self._set_gcc_prefix() + if lib_dir_runtime: + # Required for 'clang -v' to work if linked to LLVM runtimes + with _wrap_env(ld_path=os.path.join(self.installdir, lib_dir_runtime)): + self._sanity_check_gcc_prefix(gcc_prefix_compilers, self.gcc_prefix, self.installdir) + else: + self._sanity_check_gcc_prefix(gcc_prefix_compilers, self.gcc_prefix, self.installdir) + + return super(EB_LLVM, self).sanity_check_step(custom_paths=custom_paths, custom_commands=custom_commands) + + def make_module_step(self, *args, **kwargs): + """ + Clang can find its own headers and libraries but the shared libraries need to be in $LD_LIBRARY_PATH + """ + mod_env_headers = self.module_load_environment.alias_vars(MODULE_LOAD_ENV_HEADERS) + for disallowed_var in mod_env_headers: + self.module_load_environment.remove(disallowed_var) + self.log.debug(f"Purposely not updating ${disallowed_var} in {self.name} module file") + + lib_dirs = SEARCH_PATH_LIB_DIRS[:] + if self.cfg['build_runtimes']: + runtime_lib_path = self.get_runtime_lib_path(self.installdir, fail_ok=False) + lib_dirs.append(runtime_lib_path) + + self.log.debug(f"List of subdirectories for libraries to add to $LD_LIBRARY_PATH + $LIBRARY_PATH: {lib_dirs}") + self.module_load_environment.LD_LIBRARY_PATH = lib_dirs + self.module_load_environment.LIBRARY_PATH = lib_dirs + + return super().make_module_step(*args, **kwargs) - super(EB_LLVM, self).configure_step() + def make_module_extra(self): + """Custom variables for Clang module.""" + txt = super(EB_LLVM, self).make_module_extra() + # we set the symbolizer path so that asan/tsan give meanfull output by default + asan_symbolizer_path = os.path.join(self.installdir, 'bin', 'llvm-symbolizer') + txt += self.module_generator.set_environment('ASAN_SYMBOLIZER_PATH', asan_symbolizer_path) + if self.cfg['python_bindings']: + txt += self.module_generator.prepend_paths('PYTHONPATH', os.path.join('lib', 'python')) + return txt diff --git a/easybuild/easyblocks/l/lua.py b/easybuild/easyblocks/l/lua.py index a31ec8e1073..f857f8acb78 100644 --- a/easybuild/easyblocks/l/lua.py +++ b/easybuild/easyblocks/l/lua.py @@ -1,5 +1,5 @@ ## -# Copyright 2018-2024 Ghent University +# Copyright 2018-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/m/mamba.py b/easybuild/easyblocks/m/mamba.py index 232011e137d..3e7bda8c5a2 100644 --- a/easybuild/easyblocks/m/mamba.py +++ b/easybuild/easyblocks/m/mamba.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -31,7 +31,7 @@ import os -from easybuild.easyblocks.a.anaconda import EB_Anaconda +from easybuild.easyblocks.anaconda import EB_Anaconda class EB_Mamba(EB_Anaconda): diff --git a/easybuild/easyblocks/m/maple.py b/easybuild/easyblocks/m/maple.py index e10ab4d2181..6bcb41d2c6e 100644 --- a/easybuild/easyblocks/m/maple.py +++ b/easybuild/easyblocks/m/maple.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -37,7 +37,7 @@ from easybuild.easyblocks.generic.binary import Binary from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.run import run_cmd_qa +from easybuild.tools.run import run_shell_cmd class EB_Maple(Binary): @@ -55,39 +55,37 @@ def install_step(self): else: raise EasyBuildError("Could not locate installer in %s", self.builddir) - qa = { - 'PRESS TO CONTINUE:': '', - "Press [Enter] to continue:": '', - 'DO YOU ACCEPT THE TERMS OF THIS LICENSE AGREEMENT? (Y/N):': 'Y', - "Do you accept this license? [y/n]:": 'y', - 'ENTER AN ABSOLUTE PATH, OR PRESS TO ACCEPT THE DEFAULT :': self.installdir, - 'IS THIS CORRECT? (Y/N):': 'Y', - 'Language Selection\n\nPlease select the installation language\n[1] English - English\n[2] Japanese - \n' - 'Please choose an option [1] : ': '1', - 'Do you wish to have a shortcut installed on your desktop? ->1- Yes 2- No ENTER THE NUMBER ' + - 'FOR YOUR CHOICE, OR PRESS TO ACCEPT THE DEFAULT::': '2', - "Do you wish to have a shortcut installed on your desktop? [Y/n]:": 'n', - '->1- Single User License 2- Network License ENTER THE NUMBER FOR YOUR CHOICE, ' + - 'OR PRESS TO ACCEPT THE DEFAULT::': '2', - 'PRESS TO EXIT THE INSTALLER:': '', - 'License server (DEFAULT: ):': self.cfg['license_server'], - "License server []:": self.cfg['license_server'], - 'Port number (optional) (DEFAULT: ):': self.cfg['license_server_port'] or '', - '->1- Configure toolbox for Matlab 2- Do not configure at this time ENTER THE NUMBER FOR YOUR CHOICE, ' + - 'OR PRESS TO ACCEPT THE DEFAULT::': '2', - "MATLAB Configuration [y/N]:": 'n', - "Check for updates now [Y/n]:": 'n', - "Use proxy server when checking for updates [y/N]:": 'n', - "Downloads & Service Packs. [Y/n]:": 'n', - } - std_qa = { - r"Choose Install Folder \[.*\]:": self.installdir, - r"\[2\] Network License.*\nPlease choose an option \[.\] :": '2', - r"\[1\] Single Server.*\n.*\nPlease choose an option \[.\] :": '1', - r"Port number \[[0-9]+\]:": self.cfg['license_server_port'] or '', - r"Enable periodic checking for Maple .* updates after installation \[Y/n\]:": 'n', - r'Pre-Installation Summary[\s\S]*': '', - } + qa = [ + (r'PRESS TO CONTINUE:', ''), + (r"Press \[Enter\] to continue:", ''), + (r'DO YOU ACCEPT THE TERMS OF THIS LICENSE AGREEMENT\? \(Y/N\):', 'Y'), + (r"Do you accept this license\? \[y/n\]:", 'y'), + (r'ENTER AN ABSOLUTE PATH, OR PRESS TO ACCEPT THE DEFAULT :', self.installdir), + (r'IS THIS CORRECT\? \(Y/N\):', 'Y'), + (r'Language Selection\n\nPlease select the installation language\n\[1\] English - English\n' + r'\[2\] Japanese.*\nPlease choose an option \[1\] : ', '1'), + (r'Do you wish to have a shortcut installed on your desktop\? ->1- Yes 2- No ENTER THE NUMBER ' + + r'FOR YOUR CHOICE, OR PRESS TO ACCEPT THE DEFAULT::', '2'), + (r"Do you wish to have a shortcut installed on your desktop\?[\s\n]*\[Y/n\]:", 'n'), + (r'->1- Single User License 2- Network License ENTER THE NUMBER FOR YOUR CHOICE, ' + + 'OR PRESS TO ACCEPT THE DEFAULT::', '2'), + (r'PRESS TO EXIT THE INSTALLER:', ''), + (r'License server \(DEFAULT: \):', self.cfg['license_server']), + (r"License server \[\]:", self.cfg['license_server']), + (r'Port number \(optional\) \(DEFAULT: \):', self.cfg['license_server_port'] or ''), + (r'->1- Configure toolbox for Matlab 2- Do not configure at this time ENTER THE NUMBER FOR YOUR CHOICE, ' + + r'OR PRESS TO ACCEPT THE DEFAULT::', '2'), + (r"MATLAB Configuration \[y/N\]:", 'n'), + (r"Check for updates now \[Y/n\]:", 'n'), + (r"Use proxy server when checking for updates \[y/N\]:", 'n'), + (r"Downloads & Service Packs. \[Y/n\]:", 'n'), + (r"Choose Install Folder \[.*\]:", self.installdir), + (r"\[2\] Network License.*\nPlease choose an option \[.\] :", '2'), + (r"\[1\] Single Server.*\n.*\nPlease choose an option \[.\] :", '1'), + (r"Port number \[[0-9]+\]:", self.cfg['license_server_port'] or ''), + (r"Enable periodic checking for Maple .* updates after installation \[Y/n\]:", 'n'), + (r'Pre-Installation Summary[\s\S]*', ''), + ] no_qa = [ 'Graphical installers are not supported by the VM. The console mode will be used instead...', @@ -98,20 +96,19 @@ def install_step(self): r'\[[-|#|]*', ] - run_cmd_qa(cmd, qa, std_qa=std_qa, no_qa=no_qa, log_all=True, simple=True, maxhits=150) + run_shell_cmd(cmd, qa_patterns=qa, qa_wait_patterns=no_qa, qa_timeout=150) upgrade_installers = glob.glob(os.path.join(self.builddir, 'Maple*Upgrade*')) if upgrade_installers: if len(upgrade_installers) == 1: cmd = upgrade_installers[0] - qa = { - "Press [Enter] to continue:": '', - "Do you accept this license? [y/n]:": 'y', - } - std_qa = { - r"Please specify the path to your existing Maple .* Installation.\s*\n\s*\[.*\]:": self.installdir, - } - run_cmd_qa(cmd, qa, std_qa=std_qa, log_all=True, simple=True) + qa = [ + (r"Press \[Enter\] to continue:", ''), + (r"Do you accept this license\? \[y/n\]:", 'y'), + (r"Please specify the path to your existing Maple .* Installation.\s*\n\s*\[.*\]:", + self.installdir), + ] + run_shell_cmd(cmd, qa_patterns=qa) else: raise EasyBuildError("Found multiple upgrade installers: %s", ', '.join(upgrade_installers)) else: diff --git a/easybuild/easyblocks/m/mathematica.py b/easybuild/easyblocks/m/mathematica.py index 4372c06f2d4..85b13b0f926 100644 --- a/easybuild/easyblocks/m/mathematica.py +++ b/easybuild/easyblocks/m/mathematica.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2024 Ghent University +# Copyright 2013-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -35,7 +35,7 @@ from easybuild.easyblocks.generic.binary import Binary from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.run import run_cmd_qa +from easybuild.tools.run import run_shell_cmd class EB_Mathematica(Binary): @@ -49,6 +49,13 @@ def extra_options(): } return Binary.extra_options(extra_vars) + def __init__(self, *args, **kwargs): + """Easyblock constructor.""" + super(EB_Mathematica, self).__init__(*args, **kwargs) + + # custom paths in module load environment + self.module_load_environment.PATH = ['bin', 'Executables'] + def configure_step(self): """No configuration for Mathematica.""" # ensure a license server is specified @@ -76,16 +83,17 @@ def install_step(self): cmd = self.cfg['preinstallopts'] + './' + install_script shortver = '.'.join(self.version.split('.')[:2]) qa_install_path = os.path.join('/usr', 'local', 'Wolfram', self.name, shortver) - qa = { - r"Enter the installation directory, or press ENTER to select %s: >" % qa_install_path: self.installdir, - r"Create directory (y/n)? >": 'y', - r"Should the installer attempt to make this change (y/n)? >": 'n', - r"or press ENTER to select /usr/local/bin: >": os.path.join(self.installdir, "bin"), - } + qa = [ + (r"Enter the installation directory, or press ENTER to select[\s\n]*%s:[\s\n]*>" % qa_install_path, + self.installdir), + (r"Create directory \(y/n\)\?[\s\n]*>", 'y'), + (r"Should the installer attempt to make this change \(y/n\)\?[\s\n]*>", 'n'), + (r"or press ENTER to select[\s\n]*/usr/local/bin:[\s\n]*>", os.path.join(self.installdir, "bin")), + ] no_qa = [ r"Now installing.*\n\n.*\[.*\].*", ] - run_cmd_qa(cmd, qa, no_qa=no_qa, log_all=True, simple=True, maxhits=200) + run_shell_cmd(cmd, qa_patterns=qa, qa_wait_patterns=no_qa, qa_timeout=200) else: raise EasyBuildError("Failed to isolate install script using '%s': %s", install_script_glob, matches) @@ -108,24 +116,24 @@ def install_step(self): if orig_display is not None: os.environ['DISPLAY'] = orig_display - def post_install_step(self): + def post_processing_step(self): """Activate installation by using activation key, if provided.""" if self.cfg['activation_key']: # activation key is printed by using '$ActivationKey' in Mathematica, so no reason to keep it 'secret' self.log.info("Activating installation using provided activation key '%s'." % self.cfg['activation_key']) - qa = { - r"(enter return to skip Web Activation):": self.cfg['activation_key'], - r"In[1]:= ": 'Quit[]', - } - noqa = [ + qa = [ + (r"\(enter return to skip Web Activation\):", self.cfg['activation_key']), + (r"In\[1\]:= ", 'Quit[]'), + ] + no_qa = [ '^%s %s .*' % (self.name, self.version), '^Copyright.*', ] - run_cmd_qa(os.path.join(self.installdir, 'bin', 'math'), qa, no_qa=noqa) + run_shell_cmd(os.path.join(self.installdir, 'bin', 'math'), qa_patterns=qa, qa_wait_patterns=no_qa) else: self.log.info("No activation key provided, so skipping activation of the installation.") - super(EB_Mathematica, self).post_install_step() + super(EB_Mathematica, self).post_processing_step() def sanity_check_step(self): """Custom sanity check for Mathematica.""" @@ -141,12 +149,3 @@ def sanity_check_step(self): custom_commands = ['mathematica --version'] super(EB_Mathematica, self).sanity_check_step(custom_paths=custom_paths, custom_commands=custom_commands) - - def make_module_req_guess(self): - """Add both 'bin' and 'Executables' directories to PATH.""" - - guesses = super(EB_Mathematica, self).make_module_req_guess() - - guesses.update({'PATH': ['bin', 'Executables']}) - - return guesses diff --git a/easybuild/easyblocks/m/matlab.py b/easybuild/easyblocks/m/matlab.py index 56475545cdf..3097cdc03ac 100644 --- a/easybuild/easyblocks/m/matlab.py +++ b/easybuild/easyblocks/m/matlab.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -43,8 +43,7 @@ from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import adjust_permissions, change_dir, copy_file, read_file, write_file -from easybuild.tools.py2vs3 import string_type -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd class EB_MATLAB(PackedBinary): @@ -167,7 +166,7 @@ def install_step(self): except KeyError: raise EasyBuildError("The MATLAB install key is not set. This can be set either with the environment " "variable EB_MATLAB_KEY or by the easyconfig variable 'key'.") - if isinstance(keys, string_type): + if isinstance(keys, str): keys = keys.split(',') # Compile the installation key regex outside of the loop @@ -186,7 +185,7 @@ def install_step(self): except IOError as err: raise EasyBuildError("Failed to update config file %s: %s", self.configfile, err) - (out, _) = run_cmd(cmd, log_all=True, simple=False) + res = run_shell_cmd(cmd) # check installer output for known signs of trouble patterns = [ @@ -201,13 +200,13 @@ def install_step(self): for pattern in patterns: regex = re.compile(pattern, re.I) - if regex.search(out): + if regex.search(res.output): raise EasyBuildError("Found error pattern '%s' in output of installation command '%s': %s", - regex.pattern, cmd, out) + regex.pattern, cmd, res.output) with open(self.outputfile) as f: if regex.search(f.read()): - raise EasyBuildError("Found error pattern '%s' in output file of installer", - regex.pattern) + raise EasyBuildError("Found error pattern '%s' in output file of installer at %s", + regex.pattern, self.outputfile) def sanity_check_step(self): """Custom sanity check for MATLAB.""" diff --git a/easybuild/easyblocks/m/mcr.py b/easybuild/easyblocks/m/mcr.py index 76f538dc30c..04cd4db7f29 100644 --- a/easybuild/easyblocks/m/mcr.py +++ b/easybuild/easyblocks/m/mcr.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -44,8 +44,7 @@ from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import adjust_permissions, read_file, write_file -from easybuild.tools.py2vs3 import string_type -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd class EB_MCR(PackedBinary): @@ -116,12 +115,12 @@ def install_step(self): configfile = "%s/%s" % (self.builddir, self.configfilename) cmd = "%s ./install -v -inputFile %s %s" % (self.cfg['preinstallopts'], configfile, self.cfg['installopts']) - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) def sanity_check_step(self): """Custom sanity check for MCR.""" self.set_subdir() - if not isinstance(self.subdir, string_type): + if not isinstance(self.subdir, str): raise EasyBuildError("Could not identify which subdirectory to pick: %s" % self.subdir) custom_paths = { @@ -145,7 +144,7 @@ def make_module_extra(self): self.set_subdir() # if no subdir was selected, set it to NOTFOUND # this is done to enable the use of --module-only without having an actual MCR installation - if not isinstance(self.subdir, string_type): + if not isinstance(self.subdir, str): self.subdir = 'NOTFOUND' xapplresdir = os.path.join(self.installdir, self.subdir, 'X11', 'app-defaults') diff --git a/easybuild/easyblocks/m/mesa.py b/easybuild/easyblocks/m/mesa.py index e482ac9c136..9fa4c20d27a 100644 --- a/easybuild/easyblocks/m/mesa.py +++ b/easybuild/easyblocks/m/mesa.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/m/metagenome_atlas.py b/easybuild/easyblocks/m/metagenome_atlas.py index 2391595ceec..98818882cd7 100644 --- a/easybuild/easyblocks/m/metagenome_atlas.py +++ b/easybuild/easyblocks/m/metagenome_atlas.py @@ -1,5 +1,5 @@ ## -# Copyright 2020-2024 Ghent University +# Copyright 2020-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -39,7 +39,7 @@ class EB_Metagenome_Atlas(PythonPackage): Support for building/installing Metagenome-Atlas. """ - def post_install_step(self): + def post_processing_step(self): """Create snakemake config files""" # https://metagenome-atlas.readthedocs.io/en/latest/usage/getting_started.html#set-up-of-cluster-execution diff --git a/easybuild/easyblocks/m/metalwalls.py b/easybuild/easyblocks/m/metalwalls.py index 1241249a7af..59fa7841ff4 100644 --- a/easybuild/easyblocks/m/metalwalls.py +++ b/easybuild/easyblocks/m/metalwalls.py @@ -34,7 +34,7 @@ from easybuild.tools import LooseVersion from easybuild.tools.build_log import EasyBuildError from easybuild.tools.modules import get_software_root, get_software_version -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd from easybuild.easyblocks.generic.makecp import MakeCp @@ -89,15 +89,14 @@ def configure_step(self): f90flags += ['-fallow-argument-mismatch'] # Code inside ifdef causes mismatch errors fppflags += ['-DMW_USE_PLUMED'] cmd = ['touch', 'mw2.diff'] - run_cmd(' '.join(cmd), log_all=False, log_ok=False, simple=False, regexp=False) + run_shell_cmd(' '.join(cmd), fail_on_error=False) cmd = ['plumed', 'patch', '-d mw2.diff', '--patch', '--shared', '--engine', 'mw2'] - run_cmd(' '.join(cmd), log_all=True, simple=False) + run_shell_cmd(' '.join(cmd)) else: self.log.info('PLUMED not found, excluding from test-suite') rgx = tpl_rgx % 'plumed' - cmd = ['sed', '-i', "'s/^\\( \\+\\)%s$/\\1pass # %s/'" % (rgx, rgx), 'tests/regression_tests.py'] cmd = ['sed', '-i', "'s/%s/pass/'" % rgx, 'tests/regression_tests.py'] - run_cmd(' '.join(cmd), log_all=True, simple=False) + run_shell_cmd(' '.join(cmd)) if f90wrap: if not get_software_root('mpi4py'): @@ -115,7 +114,7 @@ def configure_step(self): self.log.info('f90wrap not found, excluding python interface from test-suite') rgx = tpl_rgx % 'python_interface' cmd = ['sed', '-i', "'s/%s/pass/'" % rgx, 'tests/regression_tests.py'] - run_cmd(' '.join(cmd), log_all=True, simple=False) + run_shell_cmd(' '.join(cmd)) # Add libraries with LAPACK support lapack_shared_libs = os.getenv('LAPACK_SHARED_LIBS', None) diff --git a/easybuild/easyblocks/m/metavelvet.py b/easybuild/easyblocks/m/metavelvet.py deleted file mode 100644 index 4ffc693bb25..00000000000 --- a/easybuild/easyblocks/m/metavelvet.py +++ /dev/null @@ -1,64 +0,0 @@ -## -# This file is an EasyBuild reciPY as per https://github.com/easybuilders/easybuild -# -# Copyright:: Copyright 2012-2024 Uni.Lu/LCSB, NTUA -# Authors:: Cedric Laczny , Fotis Georgatos , Kenneth Hoste -# License:: MIT/GPL -# $Id$ -# -# This work implements a part of the HPCBIOS project and is a component of the policy: -# http://hpcbios.readthedocs.org/en/latest/HPCBIOS_2012-94.html -## -""" -EasyBuild support for building and installing MetaVelvet, implemented as an easyblock - -@author: Cedric Laczny (Uni.Lu) -@author: Fotis Georgatos (Uni.Lu) -@author: Kenneth Hoste (Ghent University) -""" - -import os -import shutil - -from easybuild.easyblocks.generic.configuremake import ConfigureMake -from easybuild.tools.build_log import EasyBuildError - - -class EB_MetaVelvet(ConfigureMake): - """ - Support for building MetaVelvet - """ - - def configure_step(self): - """ - No configure - """ - pass - - def install_step(self): - """ - Install by copying files to install dir - """ - srcdir = self.cfg['start_dir'] - destdir = os.path.join(self.installdir, 'bin') - srcfile = None - # Get executable files: - # for i in $(find . -maxdepth 1 -type f -perm +111 -print | sed -e 's/\.\///g' | awk '{print "\""$0"\""}' \ - # | grep -vE "\.sh|\.html"); do echo -ne "$i, "; done && echo - try: - os.makedirs(destdir) - for filename in ["meta-velvetg"]: - srcfile = os.path.join(srcdir, filename) - shutil.copy2(srcfile, destdir) - except OSError as err: - raise EasyBuildError("Copying %s to installation dir %s failed: %s", srcfile, destdir, err) - - def sanity_check_step(self): - """Custom sanity check for MetaVelvet.""" - - custom_paths = { - 'files': ['bin/meta-velvetg'], - 'dirs': [] - } - - super(EB_MetaVelvet, self).sanity_check_step(custom_paths=custom_paths) diff --git a/easybuild/easyblocks/m/metis.py b/easybuild/easyblocks/m/metis.py index d724630d910..c4283ca784a 100644 --- a/easybuild/easyblocks/m/metis.py +++ b/easybuild/easyblocks/m/metis.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -39,7 +39,7 @@ from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_option from easybuild.tools.filetools import apply_regex_substitutions, mkdir -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd class EB_METIS(ConfigureMake): @@ -60,7 +60,7 @@ def configure_step(self, *args, **kwargs): apply_regex_substitutions('Makefile', [(r'^(CONFIG_FLAGS\s*=\s*)', r'\1 -DCMAKE_SKIP_RPATH=ON ')]) cmd = "make %s config prefix=%s" % (self.cfg['configopts'], self.installdir) - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) if 'shared=1' in self.cfg['configopts']: self.lib_exts.append('so') diff --git a/easybuild/easyblocks/m/modeller.py b/easybuild/easyblocks/m/modeller.py deleted file mode 100755 index d3dec47929b..00000000000 --- a/easybuild/easyblocks/m/modeller.py +++ /dev/null @@ -1,103 +0,0 @@ -## -# Copyright 2014-2024 Ghent University -# -# This file is part of EasyBuild, -# originally created by the HPC team of the University of Ghent (http://ugent.be/hpc). -# -# https://github.com/easybuilders/easybuild -# -# EasyBuild is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation v2. -# -# EasyBuild is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with EasyBuild. If not, see . -# -# This work implements a part of the HPCBIOS project and is a component of the policy: -# http://hpcbios.readthedocs.org/en/latest/HPCBIOS_2012-94.html -## -""" -EasyBuild support for installing Modeller, implemented as an easyblock - -@author: Pablo Escobar Lopez (SIB - University of Basel) -""" - -import os - -from easybuild.framework.easyblock import EasyBlock -from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.run import run_cmd_qa - - -class EB_Modeller(EasyBlock): - """Support for installing Modeller.""" - - def configure_step(self): - """ Skip configuration step """ - pass - - def build_step(self): - """ Skip build step """ - pass - - def install_step(self): - """Interactive install of Modeller.""" - - if self.cfg['key'] is None: - raise EasyBuildError("Easyconfig parameter 'key' is not defined") - - cmd = "%s/Install" % self.cfg['start_dir'] - - # by default modeller tries to install to $HOME/bin/modeller9.13 - # get this path to use it in the question/answer - default_install_path = os.path.join(os.path.expanduser('~'), 'bin', 'modeller%s' % self.cfg['version']) - - qa = { - # installer will autodetect the right arch. [3] = x86_64 - 'Select the type of your computer from the list above [3]:': '', - "[%s]:" % default_install_path: self.installdir, - 'http://salilab.org/modeller/registration.html:': self.cfg["key"], - 'https://salilab.org/modeller/registration.html:': self.cfg["key"], - 'Press to begin the installation:': '', - 'Press to continue:': '' - } - - run_cmd_qa(cmd, qa, log_all=True, simple=True) - - def sanity_check_step(self): - """Custom sanity check for Modeller.""" - custom_paths = { - 'files': ["bin/mod%s" % self.version, "bin/modpy.sh", "bin/modslave.py"], - 'dirs': ["doc", "lib", "examples"], - } - super(EB_Modeller, self).sanity_check_step(custom_paths=custom_paths) - - def make_module_req_guess(self): - """Custom guesses for environment variables (PYTHONPATH, LD_LIBRARY_PATH) for modeller.""" - guesses = super(EB_Modeller, self).make_module_req_guess() - - libspath = os.path.join(self.installdir, 'lib') - if os.path.exists(libspath): - libdirs = os.listdir(libspath) - if len(libdirs) == 1: - libdir = libdirs[0] - else: - raise EasyBuildError("Failed to isolate a single libdir from list of 'lib' subdirectories: %s", libdirs) - - py2libdirs = [d for d in os.listdir(os.path.join(libspath, libdir)) if d.startswith('python2')] - if len(py2libdirs) >= 1: - py2libdir = py2libdirs[-1] - else: - raise EasyBuildError("Failed to isolate latest Python lib dir from list %s", py2libdirs) - - guesses.update({ - 'PYTHONPATH': [os.path.join('lib', libdir, py2libdir), "modlib"], - 'LD_LIBRARY_PATH': [os.path.join('lib', libdir)], - }) - - return guesses diff --git a/easybuild/easyblocks/m/molpro.py b/easybuild/easyblocks/m/molpro.py index e6f53ed1bc0..fbeeab78e6d 100644 --- a/easybuild/easyblocks/m/molpro.py +++ b/easybuild/easyblocks/m/molpro.py @@ -1,5 +1,5 @@ ## -# Copyright 2015-2024 Ghent University +# Copyright 2015-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -27,19 +27,20 @@ @author: Kenneth Hoste (Ghent University) """ +import glob import os import shutil import re -from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.binary import Binary from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.framework.easyblock import EasyBlock from easybuild.framework.easyconfig import CUSTOM +from easybuild.tools import LooseVersion from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_option -from easybuild.tools.filetools import apply_regex_substitutions, mkdir, read_file, symlink -from easybuild.tools.run import run_cmd, run_cmd_qa +from easybuild.tools.filetools import apply_regex_substitutions, change_dir, mkdir, read_file, symlink +from easybuild.tools.run import run_shell_cmd class EB_Molpro(ConfigureMake, Binary): @@ -67,6 +68,14 @@ def __init__(self, *args, **kwargs): self.cleanup_token_symlink = False self.license_token = os.path.join(os.path.expanduser('~'), '.molpro', 'token') + # custom paths in module load environment + # add glob patterns for all possible locations of executables, non-existent ones will be ignored + self.module_load_environment.PATH = [ + 'bin', + '*/bin', + '*/utilities', + ] + def extract_step(self): """Extract Molpro source files, or just copy in case of binary install.""" if self.cfg['precompiled_binaries']: @@ -123,7 +132,7 @@ def configure_step(self): else: self.cfg.update('configopts', '-i8') - run_cmd("./configure -batch %s" % self.cfg['configopts']) + run_shell_cmd("./configure -batch %s" % self.cfg['configopts']) cfgfile = os.path.join(self.cfg['start_dir'], 'CONFIG') cfgtxt = read_file(cfgfile) @@ -148,8 +157,8 @@ def configure_step(self): # determine MPI launcher command that can be used during build/test # obtain command with specific number of cores (required by mpi_cmd_for), then replace that number with '%n' - launcher = self.toolchain.mpi_cmd_for('%x', self.cfg['parallel']) - launcher = launcher.replace(' %s' % self.cfg['parallel'], ' %n') + launcher = self.toolchain.mpi_cmd_for('%x', self.cfg.parallel) + launcher = launcher.replace(f' {self.cfg.parallel}', ' %n') # patch CONFIG file to change LAUNCHER definition, in order to avoid having to start mpd apply_regex_substitutions(cfgfile, [(r"^(LAUNCHER\s*=\s*).*$", r"\1 %s" % launcher)]) @@ -173,11 +182,11 @@ def test_step(self): if os.path.isfile(self.license_token) and not self.cfg['precompiled_binaries']: # check 'main routes' only - run_cmd("make quicktest") + run_shell_cmd("make quicktest") if build_option('mpi_tests'): # extensive test - run_cmd("make MOLPRO_OPTIONS='-n%s' test" % self.cfg['parallel']) + run_shell_cmd(f"make MOLPRO_OPTIONS='-n{self.cfg.parallel}' test") else: self.log.info("Skipping extensive testing of Molpro since MPI testing is disabled") @@ -194,10 +203,8 @@ def install_step(self): if self.cfg['precompiled_binaries']: """Build by running the command with the inputfiles""" - try: - os.chdir(self.cfg['start_dir']) - except OSError as err: - raise EasyBuildError("Failed to move (back) to %s: %s", self.cfg['start_dir'], err) + + change_dir(self.cfg['start_dir']) for src in self.src: if LooseVersion(self.version) >= LooseVersion('2015'): @@ -207,21 +214,20 @@ def install_step(self): else: cmd = "./{0} -batch -instbin {1}/bin -instlib {1}/lib".format(src['name'], self.installdir) - # questions whose text must match exactly as asked - qa = { - "Please give your username for accessing molpro\n": '', - "Please give your password for accessing molpro\n": '', - } - # questions whose text may be matched as a regular expression - stdqa = { - r"Enter installation directory for executable files \[.*\]\n": os.path.join(self.installdir, 'bin'), - r"Enter installation directory for library files \[.*\]\n": os.path.join(self.installdir, 'lib'), - r"directory .* does not exist, try to create [Y]/n\n": '', - } - run_cmd_qa(cmd, qa=qa, std_qa=stdqa, log_all=True, simple=True) + bindir = os.path.join(self.installdir, 'bin') + libdir = os.path.join(self.installdir, 'lib') + + qa = [ + (r"Please give your username for accessing molpro\n", ''), + (r"Please give your password for accessing molpro\n", ''), + (r"Enter installation directory for executable files \[.*\]\n", bindir), + (r"Enter installation directory for library files \[.*\]\n", libdir), + (r"directory .* does not exist, try to create [Y]/n\n", ''), + ] + run_shell_cmd(cmd, qa_patterns=qa) else: if os.path.isfile(self.license_token): - run_cmd("make tuning") + run_shell_cmd("make tuning") super(EB_Molpro, self).install_step() @@ -237,17 +243,19 @@ def install_step(self): except OSError as err: raise EasyBuildError("Failed to remove %s: %s", self.license_token, err) - def make_module_req_guess(self): - """Customize $PATH guesses for Molpro module.""" - guesses = super(EB_Molpro, self).make_module_req_guess() - guesses.update({ - 'PATH': [os.path.join(os.path.basename(self.full_prefix), x) for x in ['bin', 'utilities']], - }) - return guesses - def sanity_check_step(self): """Custom sanity check for Molpro.""" prefix_subdir = os.path.basename(self.full_prefix) + if not prefix_subdir: + # we need to guess the installation prefix whenever the configure step is skipped + # there are two possibles installation types: + # - A: installation located at the top level of self.installdir + # - B: installation located inside a subdirectory with a name specific to the + # installation type and platform (e.g. molpros_2012_1_Linux_x86_64_i8) + path_to_bin = glob.glob(os.path.join(self.installdir, 'molpro*', 'bin')) + if len(path_to_bin) > 0: + prefix_subdir = os.path.relpath(os.path.dirname(path_to_bin[0]), self.installdir) + files_to_check = ['bin/molpro'] dirs_to_check = [] if LooseVersion(self.version) >= LooseVersion('2015') or not self.cfg['precompiled_binaries']: diff --git a/easybuild/easyblocks/m/mono.py b/easybuild/easyblocks/m/mono.py deleted file mode 100644 index c82f6a5303d..00000000000 --- a/easybuild/easyblocks/m/mono.py +++ /dev/null @@ -1,188 +0,0 @@ -## -# Copyright 2009-2024 Ghent University -# -# This file is part of EasyBuild, -# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), -# with support of Ghent University (http://ugent.be/hpc), -# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), -# the Hercules foundation (http://www.herculesstichting.be/in_English) -# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). -# -# https://github.com/easybuilders/easybuild -# -# EasyBuild is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation v2. -# -# EasyBuild is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with EasyBuild. If not, see . -## -""" -EasyBuild support for Mono, implemented as an easyblock - -@author: Stijn De Weirdt (Ghent University) -@author: Dries Verdegem (Ghent University) -@author: Kenneth Hoste (Ghent University) -@author: Pieter De Baets (Ghent University) -@author: Jens Timmerman (Ghent University) -""" -from easybuild.tools import LooseVersion -import os -import shutil - -from easybuild.easyblocks.generic.configuremake import ConfigureMake -from easybuild.easyblocks.generic.rpm import Rpm -from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.filetools import apply_regex_substitutions, read_file -from easybuild.tools.run import run_cmd - - -class EB_Mono(ConfigureMake, Rpm): - """Support for building/installing Mono.""" - - @staticmethod - def extra_options(extra_vars=None): - """Combine custom easyconfig parameters specific to ConfigureMake and Rpm generic easyblocks.""" - extra_vars = ConfigureMake.extra_options() - return Rpm.extra_options(extra_vars=extra_vars) - - def __init__(self, *args, **kwargs): - """Custom constructor for Mono easyblock, initialize custom class variables.""" - super(EB_Mono, self).__init__(*args, **kwargs) - self.mono_srcs = [] - self.rpms = [] - - def extract_step(self): - """Custom extract step for Mono: don't try and extract any of the provided RPMs.""" - - for src in self.src: - if src['name'].endswith('.rpm'): - self.rpms.append(src) - else: - self.mono_srcs.append(src) - - self.src = self.mono_srcs - ConfigureMake.extract_step(self) - - def configure_step(self): - """Dedicated configure step for Mono: install Mono from RPM (if provided), then run configure.""" - - # install Mono from RPMs if provided (because we need Mono to build Mono) - if self.rpms: - - # prepare path for installing RPMs in - monorpms_path = os.path.join(self.builddir, "monorpms") - try: - os.makedirs(os.path.join(monorpms_path, 'rpm')) - except OSError as err: - raise EasyBuildError("Failed to create directories for installing Mono RPMs in: %s", err) - - self.src = self.rpms - self.rebuildRPM = True - - # rebuild RPMs to make them relocatable - Rpm.configure_step(self) - - # prepare to install RPMs - self.log.debug("Initializing temporary RPM repository to install to...") - cmd = "rpm --initdb --dbpath /rpm --root %s" % monorpms_path - run_cmd(cmd, log_all=True, simple=True) - - # install RPMs one by one - for rpm in self.src: - self.log.debug("Installing RPM %s ..." % rpm['name']) - if os.path.exists(rpm['path']): - cmd = ' '.join([ - "rpm -i", - "--dbpath %(inst)s/rpm", - "--force", - "--relocate /=%(inst)s", - "--badreloc", - "--nodeps --nopost", - "%(rpm)s", - ]) % { - 'inst': monorpms_path, - 'rpm': rpm['path'], - } - run_cmd(cmd, log_all=True, simple=True) - else: - raise EasyBuildError("RPM file %s not found", rpm['path']) - - # create patched version of gmcs command - self.log.debug("Making our own copy of gmcs (one that works).") - - mygmcs_path = os.path.join(monorpms_path, 'usr', 'bin', 'mygmcs') - try: - shutil.copy(os.path.join(monorpms_path, 'usr', 'bin', 'gmcs'), mygmcs_path) - except OSError as err: - raise EasyBuildError("Failed to copy gmcs to %s: %s", mygmcs_path, err) - - rpls = [ - ("exec /usr/bin/mono", "exec %s/usr/bin/mono" % monorpms_path), - ("`/usr/bin/monodir`", "%s/usr/lib64/mono" % monorpms_path), - ] - apply_regex_substitutions(mygmcs_path, rpls) - - self.log.debug("Patched version of gmcs (%s): %s" % (mygmcs_path, read_file(mygmcs_path))) - - # initiate bootstrap: build/install Mono with installed RPMs to temporary path - tmp_mono_path = os.path.join(self.builddir, "tmp_mono") - self.log.debug("Build/install temporary Mono version in %s using installed RPMs..." % tmp_mono_path) - - par = '' - if self.cfg['parallel']: - par = "-j %s" % self.cfg['parallel'] - - config_cmd = ' '.join([ - self.cfg['preconfigopts'], - "./configure", - "--prefix=" + tmp_mono_path, - self.cfg['configopts'], - ]) - build_cmd = ' '.join([ - "%(prebuildopts)s" - "make %(par)s", - "EXTERNAL_MCS=%(path)s/usr/bin/mygmcs", - "EXTERNAL_RUNTIME=%(path)s/usr/bin/mono", - "%(buildopts)s", - ]) % { - 'prebuildopts': self.cfg['prebuildopts'], - 'par': par, - 'path': monorpms_path, - 'buildopts': self.cfg['buildopts'], - } - install_cmd = "make install" - - for cmd in [config_cmd, build_cmd, install_cmd]: - run_cmd(cmd, log_all=True, simple=True) - - more_buildopts = ' '.join([ - "EXTERNAL_MCS=%(path)s/bin/gmcs", - "EXTERNAL_RUNTIME=%(path)s/bin/mono", - ]) % {'path': tmp_mono_path} - self.cfg.update('buildopts', more_buildopts) - - self.src = self.mono_srcs - - # continue with normal configure, and subsequent make, make install - ConfigureMake.configure_step(self) - - def sanity_check_step(self): - """Custom sanity check for Mono.""" - - binaries = ['bin/mono', 'bin/xbuild'] - if LooseVersion(self.version) >= LooseVersion('2.11'): - binaries.append('bin/mcs') - else: - binaries.append('bin/gmcs') - - custom_paths = { - 'files': binaries, - 'dirs': ['include/mono-2.0/mono', 'lib'], - } - ConfigureMake.sanity_check_step(self, custom_paths=custom_paths) diff --git a/easybuild/easyblocks/m/mothur.py b/easybuild/easyblocks/m/mothur.py deleted file mode 100644 index 7e44bc7b4fe..00000000000 --- a/easybuild/easyblocks/m/mothur.py +++ /dev/null @@ -1,108 +0,0 @@ -## -# Copyright 2013-2024 Ghent University -# -# This file is part of EasyBuild, -# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), -# with support of Ghent University (http://ugent.be/hpc), -# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), -# Flemish Research Foundation (FWO) (http://www.fwo.be/en) -# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). -# -# https://github.com/easybuilders/easybuild -# -# EasyBuild is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation v2. -# -# EasyBuild is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with EasyBuild. If not, see . -## -""" -EasyBuild support for Mothur, implemented as an easyblock - -@author: Kenneth Hoste (Ghent University) -""" -import glob -import os -import shutil -from easybuild.tools import LooseVersion - -from easybuild.easyblocks.generic.configuremake import ConfigureMake -from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.modules import get_software_root - - -class EB_Mothur(ConfigureMake): - """Support for building and installing Mothur.""" - - def guess_start_dir(self): - """Set correct start directory.""" - # Mothur zip files tend to contain multiple directories next to the actual source dir (e.g. __MACOSX), - # so the default start directory guess is most likely incorrect - mothur_dirs = glob.glob(os.path.join(self.builddir, 'Mothur.*')) - if len(mothur_dirs) == 1: - self.cfg['start_dir'] = mothur_dirs[0] - elif len(os.listdir(self.builddir)) > 1: - # we only have an issue if the default guessing approach will not work - raise EasyBuildError("Failed to guess start directory from %s", mothur_dirs) - - super(EB_Mothur, self).guess_start_dir() - - def configure_step(self, cmd_prefix=''): - """Configure Mothur build by setting make options.""" - # Fortran compiler and options - self.cfg.update('buildopts', 'FORTAN_COMPILER="%s"' % os.getenv('F77')) - self.cfg.update('buildopts', 'FORTRAN_FLAGS="%s"' % os.getenv('FFLAGS')) - # enable 64-bit build - if not self.toolchain.options['32bit']: - self.cfg.update('buildopts', '64BIT_VERSION=yes') - # enable readline support - if get_software_root('libreadline') and get_software_root('ncurses'): - self.cfg.update('buildopts', 'USEREADLINE=yes') - # enable MPI support - if self.toolchain.options.get('usempi', None): - self.cfg.update('buildopts', 'USEMPI=yes CXX="%s"' % os.getenv('MPICXX')) - self.cfg.update('prebuildopts', 'CXXFLAGS="$CXXFLAGS -DMPICH_IGNORE_CXX_SEEK"') - # enable compression - if get_software_root('bzip2') or get_software_root('gzip'): - self.cfg.update('buildopts', 'USE_COMPRESSION=yes') - # Use Boost - if get_software_root('Boost'): - self.cfg.update('buildopts', 'USEBOOST=yes') - # Use HDF5 - if get_software_root('HDF5'): - self.cfg.update('buildopts', 'USEHDF5=yes') - - def install_step(self): - """ - Install by copying files to install dir - """ - srcdir = os.path.join(self.builddir, self.cfg['start_dir']) - destdir = os.path.join(self.installdir, 'bin') - srcfile = None - # After version 1.43.0 uchime binary is not included in the Mothur tarball - if LooseVersion(self.version) > LooseVersion('1.43.0'): - files_to_copy = ['mothur'] - else: - files_to_copy = ['mothur', 'uchime'] - try: - os.makedirs(destdir) - for filename in files_to_copy: - srcfile = os.path.join(srcdir, filename) - shutil.copy2(srcfile, destdir) - except OSError as err: - raise EasyBuildError("Copying %s to installation dir %s failed: %s", srcfile, destdir, err) - - def sanity_check_step(self): - """Custom sanity check for Mothur.""" - custom_paths = { - 'files': ["bin/mothur"], - 'dirs': [], - } - - super(EB_Mothur, self).sanity_check_step(custom_paths=custom_paths) diff --git a/easybuild/easyblocks/m/motioncor2.py b/easybuild/easyblocks/m/motioncor2.py index 80daae592b5..e1a287842a4 100644 --- a/easybuild/easyblocks/m/motioncor2.py +++ b/easybuild/easyblocks/m/motioncor2.py @@ -1,5 +1,5 @@ ## -# Copyright 2019-2024 Ghent University +# Copyright 2019-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/m/mpich.py b/easybuild/easyblocks/m/mpich.py index d29092e0639..edf6356d7eb 100644 --- a/easybuild/easyblocks/m/mpich.py +++ b/easybuild/easyblocks/m/mpich.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University, Forschungszentrum Juelich +# Copyright 2009-2025 Ghent University, Forschungszentrum Juelich # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -92,23 +92,29 @@ def add_mpich_configopts(self): # additional configuration options add_configopts = [] - # use POSIX threads - add_configopts.append('--with-thread-package=pthreads') - if self.cfg['debug']: # debug build, with error checking, timing and debug info # note: this will affect performance - add_configopts.append('--enable-fast=none') + if LooseVersion(self.version) < LooseVersion('4.0.0'): + add_configopts.append('--enable-fast=none') + else: + add_configopts.append('--enable-error-checking=all') + add_configopts.append('--enable-timing=runtime') + add_configopts.append('--enable-debuginfo') else: # optimized build, no error checking, timing or debug info - add_configopts.append('--enable-fast') + if LooseVersion(self.version) < LooseVersion('4.0.0'): + add_configopts.append('--enable-fast') + else: + add_configopts.append('--enable-error-checking=no') + add_configopts.append('--enable-timing=none') # enable shared libraries, using GCC and GNU ld options - add_configopts.extend(['--enable-shared', '--enable-sharedlibs=gcc']) + add_configopts.append('--enable-shared') # enable static libraries - add_configopts.extend(['--enable-static']) + add_configopts.append('--enable-static') # enable Fortran 77/90 and C++ bindings - add_configopts.extend(['--enable-f77', '--enable-fc', '--enable-cxx']) + add_configopts.extend(['--enable-fortran=all', '--enable-cxx']) self.cfg.update('configopts', ' '.join(add_configopts)) diff --git a/easybuild/easyblocks/m/mrbayes.py b/easybuild/easyblocks/m/mrbayes.py index 55841226893..accde4bd57e 100644 --- a/easybuild/easyblocks/m/mrbayes.py +++ b/easybuild/easyblocks/m/mrbayes.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -42,7 +42,7 @@ from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import copy_file, mkdir from easybuild.tools.modules import get_software_root -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd class EB_MrBayes(ConfigureMake): @@ -75,7 +75,7 @@ def configure_step(self): # run autoconf to generate configure script cmd = "autoconf" - run_cmd(cmd) + run_shell_cmd(cmd) # set config opts beagle = get_software_root('beagle-lib') diff --git a/easybuild/easyblocks/m/mrtrix.py b/easybuild/easyblocks/m/mrtrix.py index 08ca57b0134..df978077f87 100644 --- a/easybuild/easyblocks/m/mrtrix.py +++ b/easybuild/easyblocks/m/mrtrix.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -32,7 +32,7 @@ import easybuild.tools.environment as env from easybuild.framework.easyblock import EasyBlock from easybuild.tools.filetools import copy, symlink -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd from easybuild.tools.systemtools import get_shared_lib_ext @@ -47,6 +47,11 @@ def __init__(self, *args, **kwargs): self.build_in_installdir = True self.log.debug("Enabled build-in-installdir for version %s", self.version) + self.module_load_environment.PATH.append('scripts') + + if LooseVersion(self.version) >= LooseVersion('3.0'): + self.module_load_environment.PYTHONPATH = ['lib'] + def extract_step(self): """Extract MRtrix sources.""" # strip off 'mrtrix*' part to avoid having everything in a 'mrtrix*' subdirectory @@ -65,21 +70,20 @@ def configure_step(self): env.setvar('QMAKE_CXX', os.getenv('CXX')) cmd = "python configure -verbose" - run_cmd(cmd, log_all=True, simple=True, log_ok=True) + run_shell_cmd(cmd) def build_step(self): """Custom build procedure for MRtrix.""" - parallel = self.cfg['parallel'] - env.setvar('NUMBER_OF_PROCESSORS', str(parallel)) + env.setvar('NUMBER_OF_PROCESSORS', str(self.cfg.parallel)) cmd = "python build -verbose" - run_cmd(cmd, log_all=True, simple=True, log_ok=True) + run_shell_cmd(cmd) def install_step(self): """Custom install procedure for MRtrix.""" if LooseVersion(self.version) < LooseVersion('0.3'): cmd = "python build -verbose install=%s linkto=" % self.installdir - run_cmd(cmd, log_all=True, simple=True, log_ok=True) + run_shell_cmd(cmd) elif LooseVersion(self.version) >= LooseVersion('3.0'): copy(os.path.join(self.builddir, 'bin'), self.installdir) @@ -91,20 +95,6 @@ def install_step(self): # some scripts expect 'release/bin' to be there, so we put a symlink in place symlink(self.installdir, os.path.join(self.installdir, 'release')) - def make_module_req_guess(self): - """ - Return list of subdirectories to consider to update environment variables; - also consider 'scripts' subdirectory for $PATH - """ - guesses = super(EB_MRtrix, self).make_module_req_guess() - - guesses['PATH'].append('scripts') - - if LooseVersion(self.version) >= LooseVersion('3.0'): - guesses.setdefault('PYTHONPATH', []).append('lib') - - return guesses - def sanity_check_step(self): """Custom sanity check for MRtrix.""" shlib_ext = get_shared_lib_ext() diff --git a/easybuild/easyblocks/m/msm.py b/easybuild/easyblocks/m/msm.py deleted file mode 100644 index 2f31dddabe8..00000000000 --- a/easybuild/easyblocks/m/msm.py +++ /dev/null @@ -1,131 +0,0 @@ -## -# Copyright 2009-2024 Ghent University -# -# This file is part of EasyBuild, -# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), -# with support of Ghent University (http://ugent.be/hpc), -# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), -# Flemish Research Foundation (FWO) (http://www.fwo.be/en) -# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). -# -# https://github.com/easybuilders/easybuild -# -# EasyBuild is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation v2. -# -# EasyBuild is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with EasyBuild. If not, see . -## -""" -EasyBuild support for building and installing MSM, implemented as an easyblock -@author: Davide Vanzo (Vanderbilt University) -""" -import os - -import easybuild.tools.environment as env -from easybuild.easyblocks.generic.makecp import MakeCp -from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.filetools import copy_dir, change_dir, mkdir, remove_dir -from easybuild.tools.run import run_cmd -from easybuild.tools.modules import get_software_root -from easybuild.framework.easyconfig import CUSTOM - - -class EB_MSM(MakeCp): - """Support for building and installing MSM.""" - @staticmethod - def extra_options(extra_vars=None): - """Change default values of options""" - extra = MakeCp.extra_options() - # files_to_copy is not mandatory here - extra['files_to_copy'][2] = CUSTOM - return extra - - def __init__(self, *args, **kwargs): - """Initialize MSM specific variables.""" - super(EB_MSM, self).__init__(*args, **kwargs) - self.sources_root = os.path.join(self.builddir, 'MSM_HOCR-%s' % self.version) - self.binfiles = [ - 'estimate_metric_distortion', 'msm', 'msmapplywarp', - 'msm_metric_sim', 'msmresample', 'surfconvert' - ] - - def configure_step(self): - """Create directories, copy required files and set env vars.""" - - # Ensure that nothing has been left over from previous installation attempts. - # This is necessary here since directories must be created before building - # and not removed before the installation step. - remove_dir(self.installdir) - mkdir(self.installdir) - - # Create directories recursively - dirpath = self.installdir - for dirname in ['extras', 'include']: - dirpath = os.path.join(dirpath, dirname) - mkdir(dirpath) - self.log.debug("Created directory: %s" % dirpath) - - source_dir = os.path.join(self.sources_root, 'extras', 'ELC1.04', 'ELC') - dest_dir = os.path.join(self.installdir, 'extras', 'include', 'ELC') - copy_dir(source_dir, dest_dir) - - # FSL is a required dependency since it provides FastPDlib - fsl_root = get_software_root('FSL') - if not fsl_root: - raise EasyBuildError("Required FSL dependency not found") - - # Find the machine type identified by FSL - cmd = ". %s/fsl/etc/fslconf/fslmachtype.sh" % fsl_root - (out, _) = run_cmd(cmd, log_all=True, simple=False) - fslmachtype = out.strip() - self.log.debug("FSL machine type: %s" % fslmachtype) - - env.setvar('FSLDEVDIR', self.installdir) - env.setvar('FSLCONFDIR', os.path.join(fsl_root, 'fsl', 'config')) - env.setvar('FSLMACHTYPE', fslmachtype) - - def build_step(self): - """Build MSM one component at a time.""" - - components = ['newmesh', 'DiscreteOpt', 'MSMRegLib', 'MSM'] - - paracmd = '' - if self.cfg['parallel']: - paracmd = "-j %s" % self.cfg['parallel'] - - cmd = ' '.join(['make', paracmd, 'install']) - - for comp in components: - target_dir = os.path.join(self.sources_root, 'src', comp) - self.log.debug("Building %s in directory %s", comp, target_dir) - change_dir(target_dir) - run_cmd(cmd, log_all=True, simple=True, log_output=True) - - def make_installdir(self): - """Override installdir creation""" - - self.log.warning("Not removing installation directory %s" % self.installdir) - self.cfg['keeppreviousinstall'] = True - super(EB_MSM, self).make_installdir() - - def install_step(self): - """Define files to be copied at installation.""" - - self.cfg['files_to_copy'] = [([os.path.join('src', 'MSM', f) for f in self.binfiles], 'bin')] - super(EB_MSM, self).install_step() - - def sanity_check_step(self): - """Custom sanity check for MSM.""" - - custom_paths = { - 'files': [os.path.join('bin', f) for f in self.binfiles], - 'dirs': ['doc', 'include', 'lib'] - } - super(EB_MSM, self).sanity_check_step(custom_paths=custom_paths) diff --git a/easybuild/easyblocks/m/mtl4.py b/easybuild/easyblocks/m/mtl4.py deleted file mode 100644 index a77b223c4bf..00000000000 --- a/easybuild/easyblocks/m/mtl4.py +++ /dev/null @@ -1,56 +0,0 @@ -## -# Copyright 2009-2024 Ghent University -# -# This file is part of EasyBuild, -# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), -# with support of Ghent University (http://ugent.be/hpc), -# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), -# Flemish Research Foundation (FWO) (http://www.fwo.be/en) -# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). -# -# https://github.com/easybuilders/easybuild -# -# EasyBuild is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation v2. -# -# EasyBuild is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with EasyBuild. If not, see . -## -""" -EasyBuild support for MTL4, implemented as an easyblock - -@author: Kenneth Hoste (Ghent University) -""" -import os - -from easybuild.easyblocks.generic.tarball import Tarball - - -class EB_MTL4(Tarball): - """Support for installing MTL4.""" - - def sanity_check_step(self): - """Custom sanity check for MTL4.""" - - incpref = os.path.join('include', 'boost', 'numeric') - - custom_paths = { - 'files': [], - 'dirs': [os.path.join(incpref, x) for x in ["itl", "linear_algebra", "meta_math", "mtl"]], - } - - super(EB_MTL4, self).sanity_check_step(custom_paths=custom_paths) - - def make_module_req_guess(self): - """Adjust CPATH for MTL4.""" - - guesses = super(EB_MTL4, self).make_module_req_guess() - guesses.update({'CPATH': 'include'}) - - return guesses diff --git a/easybuild/easyblocks/m/mummer.py b/easybuild/easyblocks/m/mummer.py index ee259fa0fc2..11696c5c9b8 100644 --- a/easybuild/easyblocks/m/mummer.py +++ b/easybuild/easyblocks/m/mummer.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of the University of Ghent (http://ugent.be/hpc). @@ -35,7 +35,7 @@ from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.easyblocks.perl import get_major_perl_version from easybuild.tools.filetools import apply_regex_substitutions, copy_file, is_binary, mkdir, read_file -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd class EB_MUMmer(ConfigureMake): @@ -59,7 +59,7 @@ def configure_step(self): """Configure MUMmer build by running make check and setting make options.""" cmd = "%s make check %s" % (self.cfg['preconfigopts'], self.cfg['configopts']) - run_cmd(cmd, log_all=True, simple=True, log_output=True) + run_shell_cmd(cmd) self.cfg.update('buildopts', 'all') diff --git a/easybuild/easyblocks/m/mumps.py b/easybuild/easyblocks/m/mumps.py index 4cd1b71f00d..238ef160b11 100644 --- a/easybuild/easyblocks/m/mumps.py +++ b/easybuild/easyblocks/m/mumps.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/m/mutil.py b/easybuild/easyblocks/m/mutil.py deleted file mode 100644 index 7559a44ee6d..00000000000 --- a/easybuild/easyblocks/m/mutil.py +++ /dev/null @@ -1,78 +0,0 @@ -## -# Copyright 2016-2024 Ghent University -# -# This file is part of EasyBuild, -# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), -# with support of Ghent University (http://ugent.be/hpc), -# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), -# Flemish Research Foundation (FWO) (http://www.fwo.be/en) -# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). -# -# https://github.com/easybuilders/easybuild -# -# EasyBuild is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation v2. -# -# EasyBuild is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with EasyBuild. If not, see . -## -""" -EasyBuild support for mutil, implemented as an easyblock - -@author: Ward Poelmans (Ghent University) -""" -import glob -import os -import re - -from easybuild.easyblocks.generic.makecp import MakeCp -from easybuild.framework.easyconfig import CUSTOM -from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.filetools import apply_patch - - -class EB_mutil(MakeCp): - """Easyblock to build and install mutil""" - - @staticmethod - def extra_options(): - """Change default values of options""" - extra = MakeCp.extra_options() - # files_to_copy is not mandatory here - extra['files_to_copy'][2] = CUSTOM - extra['with_configure'][0] = True - return extra - - def configure_step(self): - """Apply coreutils patch from source and run configure""" - # 1.822.3 -> 8.22 - coreutils_version = re.sub(r"^\d+.(\d+)(\d\d).\d+", r"\1.\2", self.version) - coreutils_patch = "coreutils-%s.patch" % coreutils_version - patch_path = os.path.join(self.builddir, "%s-%s" % (self.name, self.version), "patch", coreutils_patch) - if os.path.isfile(patch_path): - self.log.info("coreutils patch found at %s", patch_path) - else: - raise EasyBuildError("Could not find the patch for coreutils: %s", coreutils_patch) - - coreutils_path = glob.glob(os.path.join(self.builddir, "coreutils-*")) - if not coreutils_path: - raise EasyBuildError("Could not find the coreutils directory") - - if not apply_patch(patch_path, coreutils_path[0]): - raise EasyBuildError("Applying coreutils patch %s failed", coreutils_patch) - - super(EB_mutil, self).configure_step() - - def install_step(self): - """Specify list of files to copy""" - self.cfg['files_to_copy'] = [ - ([('src/cp', 'mcp'), ('src/md5sum', 'msum')], 'bin'), - ([('man/cp.1', 'mcp.1'), ('man/md5sum.1', 'msum.1')], 'man/man1'), - ] - super(EB_mutil, self).install_step() diff --git a/easybuild/easyblocks/m/mvapich2.py b/easybuild/easyblocks/m/mvapich2.py deleted file mode 100644 index a80bd8b8d22..00000000000 --- a/easybuild/easyblocks/m/mvapich2.py +++ /dev/null @@ -1,137 +0,0 @@ -## -# Copyright 2009-2024 Ghent University, Forschungszentrum Juelich -# -# This file is part of EasyBuild, -# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), -# with support of Ghent University (http://ugent.be/hpc), -# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), -# Flemish Research Foundation (FWO) (http://www.fwo.be/en) -# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). -# -# https://github.com/easybuilders/easybuild -# -# EasyBuild is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation v2. -# -# EasyBuild is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with EasyBuild. If not, see . -## -""" -EasyBuild support for building and installing the MVAPICH2 MPI library, implemented as an easyblock - -@author: Stijn De Weirdt (Ghent University) -@author: Dries Verdegem (Ghent University) -@author: Kenneth Hoste (Ghent University) -@author: Pieter De Baets (Ghent University) -@author: Jens Timmerman (Ghent University) -@author: Damian Alvarez (Forschungszentrum Juelich) -@author: Xavier Besseron (University of Luxembourg) -""" - -from easybuild.tools import LooseVersion - -from easybuild.easyblocks.mpich import EB_MPICH -from easybuild.framework.easyconfig import CUSTOM -from easybuild.tools.build_log import EasyBuildError - - -class EB_MVAPICH2(EB_MPICH): - """ - Support for building the MVAPICH2 MPI library. - - some compiler dependent configure options - """ - - @staticmethod - def extra_options(): - """Define custom easyconfig parameters specific to MVAPICH2.""" - extra_vars = { - 'withchkpt': [False, "Enable checkpointing support (required BLCR)", CUSTOM], - 'withmpe': [False, "Build MPE routines", CUSTOM], - 'withhwloc': [True, "Enable support for using hwloc support for process binding", CUSTOM], - 'withlimic2': [False, "Enable LiMIC2 support for intra-node communication", CUSTOM], - 'rdma_type': ["gen2", "Specify the RDMA type (gen2/udapl)", CUSTOM], - 'blcr_path': [None, "Path to BLCR package", CUSTOM], - 'blcr_inc_path': [None, "Path to BLCR header files", CUSTOM], - 'blcr_lib_path': [None, "Path to BLCR library", CUSTOM], - } - return EB_MPICH.extra_options(extra_vars) - - def configure_step(self): - """Define custom configure options for MVAPICH2.""" - - # additional configuration options - add_configopts = [] - - if self.cfg['rdma_type']: - add_configopts.append('--with-rdma=%s' % self.cfg['rdma_type']) - - # enable specific support options (if desired) - if self.cfg['withmpe']: - # --enable-mpe is a configure option of MPICH itself. - # It is not available anymore in MPICH package since version 3.0, which correspond to MVAPICH2 1.9. - # MPE can be downloaded separately at http://www.mpich.org/static/mpe/downloads/ - # However, the 'withmpe' option should be maintained for backward compatibility purpose - if LooseVersion(self.version) < LooseVersion('1.9'): - add_configopts.append('--enable-mpe') - else: - raise EasyBuildError("MPI Parallel Environment (MPE) is not available anymore starting MVAPICH2 1.9") - - if self.cfg['withlimic2']: - add_configopts.append('--enable-limic2') - - if self.cfg['withchkpt']: - add_configopts.extend(['--enable-ckpt']) - - # --with-hwloc/--without-hwloc option is not available anymore MVAPICH2 >= 2.0. Starting this version, - # hwloc is apparently distributed with MVAPICH2 and always compiled with MVAPICH2, and it cannot be disabled. - # This check happens only if 'withhwloc = False' is explicitly specified in an easyconfig with MPIVACH2 >= 2.0 - if LooseVersion(self.version) >= LooseVersion('2.0'): - if self.cfg['withhwloc']: - self.log.debug("hwloc support is always enabled in MVAPICH >= 2.0, nothing to do") - else: - raise EasyBuildError("Disabling hwloc is not supported in MVAPICH >= 2.0") - else: - if self.cfg['withhwloc']: - add_configopts.append('--with-hwloc') - else: - add_configopts.append('--without-hwloc') - - # pass BLCR paths if specified - if self.cfg['blcr_path']: - add_configopts.append('--with-blcr=%s' % self.cfg['blcr_path']) - - if self.cfg['blcr_inc_path']: - add_configopts.append('--with-blcr-include=%s' % self.cfg['blcr_inc_path']) - - if self.cfg['blcr_lib_path']: - add_configopts.append('--with-blcr-libpath=%s' % self.cfg['blcr_lib_path']) - - self.cfg.update('configopts', ' '.join(add_configopts)) - - super(EB_MVAPICH2, self).configure_step() - - # make and make install are default - - def sanity_check_step(self): - """ - Custom sanity check for MVAPICH2 - """ - mv2_bins = ['bin/mpiexec.mpirun_rsh'] - if self.cfg['withchkpt']: - mv2_bins += ['bin/mv2_checkpoint'] - - custom_paths = { - 'files': mv2_bins, - } - - # cfr. http://git.mpich.org/mpich.git/blob_plain/v3.1.1:/CHANGES - # MVAPICH2 2.1 is based on MPICH 3.1.4 - use_new_libnames = LooseVersion(self.version) >= LooseVersion('2.1') - - super(EB_MVAPICH2, self).sanity_check_step(custom_paths=custom_paths, use_new_libnames=use_new_libnames) diff --git a/easybuild/easyblocks/m/mxnet.py b/easybuild/easyblocks/m/mxnet.py index 016c5ebab8e..3fe7df0e19f 100644 --- a/easybuild/easyblocks/m/mxnet.py +++ b/easybuild/easyblocks/m/mxnet.py @@ -1,5 +1,5 @@ ## -# Copyright 2018-2024 Free University of Brussels (VUB) +# Copyright 2018-2025 Free University of Brussels (VUB) # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -40,7 +40,7 @@ from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import change_dir, mkdir, remove_dir, symlink, write_file from easybuild.tools.modules import get_software_root, get_software_version -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd from easybuild.tools.systemtools import get_shared_lib_ext # the namespace file for the R extension @@ -187,9 +187,9 @@ def extensions_step(self): self.py_ext.src = os.path.join(self.mxnet_src_dir, "python") change_dir(self.py_ext.src) - self.py_ext.prerun() - self.py_ext.run(unpack_src=False) - self.py_ext.postrun() + self.py_ext.pre_install_extension() + self.py_ext.install_extension(unpack_src=False) + self.py_ext.post_install_extension() if self.cfg['install_r_ext']: # This is off by default, because it's been working in the old version of MXNet and now it's not. @@ -212,17 +212,18 @@ def install_r_ext(self): # MXNet doesn't provide a list of its R dependencies by default write_file("NAMESPACE", R_NAMESPACE) change_dir(self.mxnet_src_dir) - self.r_ext.prerun() + self.r_ext.pre_install_extension() # MXNet is just weird. To install the R extension, we have to: # - First install the extension like it is # - Let R export the extension again. By doing this, all the dependencies get # correctly filled and some mappings are done # - Reinstal the exported version - self.r_ext.run() - run_cmd("R_LIBS=%s Rscript -e \"require(mxnet); mxnet:::mxnet.export(\\\"R-package\\\")\"" % self.installdir) + self.r_ext.install_extension() + cmd = "R_LIBS=%s Rscript -e \"require(mxnet); mxnet:::mxnet.export(\\\"R-package\\\")\"" + run_shell_cmd(cmd % self.installdir) change_dir(self.r_ext.src) - self.r_ext.run() - self.r_ext.postrun() + self.r_ext.install_extension() + self.r_ext.post_install_extension() def sanity_check_step(self): """Check for main library files for MXNet""" @@ -251,12 +252,6 @@ def make_module_extra(self, *args, **kwargs): """Custom variables for MXNet module.""" txt = super(EB_MXNet, self).make_module_extra(*args, **kwargs) - for path in self.py_ext.all_pylibdirs: - fullpath = os.path.join(self.installdir, path) - # only extend $PYTHONPATH with existing, non-empty directories - if os.path.exists(fullpath) and os.listdir(fullpath): - txt += self.module_generator.prepend_paths('PYTHONPATH', path) - txt += self.module_generator.prepend_paths("R_LIBS", ['']) # prepend R_LIBS with install path return txt diff --git a/easybuild/easyblocks/m/mymedialite.py b/easybuild/easyblocks/m/mymedialite.py deleted file mode 100644 index 5aeb784b866..00000000000 --- a/easybuild/easyblocks/m/mymedialite.py +++ /dev/null @@ -1,71 +0,0 @@ -## -# Copyright 2009-2024 Ghent University -# -# This file is part of EasyBuild, -# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), -# with support of Ghent University (http://ugent.be/hpc), -# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), -# the Hercules foundation (http://www.herculesstichting.be/in_English) -# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). -# -# https://github.com/easybuilders/easybuild -# -# EasyBuild is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation v2. -# -# EasyBuild is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with EasyBuild. If not, see . -## -""" -EasyBuild support for MyMediaLite, implemented as an easyblock - -@author: Stijn De Weirdt (Ghent University) -@author: Dries Verdegem (Ghent University) -@author: Kenneth Hoste (Ghent University) -@author: Pieter De Baets (Ghent University) -@author: Jens Timmerman (Ghent University) -""" - -from easybuild.tools import LooseVersion - -from easybuild.easyblocks.generic.configuremake import ConfigureMake -from easybuild.tools.run import run_cmd - - -class EB_MyMediaLite(ConfigureMake): - """Support for building/installing MyMediaLite.""" - - def configure_step(self): - """Custom configure step for MyMediaLite, using "make CONFIGURE_OPTIONS='...' configure".""" - - if LooseVersion(self.version) < LooseVersion('3'): - cmd = "make CONFIGURE_OPTIONS='--prefix=%s' configure" % self.installdir - run_cmd(cmd, log_all=True, simple=True) - else: - self.cfg.update('installopts', "PREFIX=%s" % self.installdir) - - def build_step(self): - """Custom build step for MyMediaLite, using 'make all' in 'src' directory.""" - - cmd = "cd src && make all && cd .." - run_cmd(cmd, log_all=True, simple=True) - - def sanity_check_step(self): - """Custom sanity check for MyMediaLite.""" - - if LooseVersion(self.version) < LooseVersion('3'): - bin_files = ["bin/%s_prediction" % x for x in ['item', 'mapping_item', 'mapping_rating', 'rating']] - else: - bin_files = ["bin/item_recommendation", "bin/rating_based_ranking", "bin/rating_prediction"] - - custom_paths = { - 'files': bin_files, - 'dirs': ["lib/mymedialite"], - } - super(EB_MyMediaLite, self).sanity_check_step(custom_paths=custom_paths) diff --git a/easybuild/easyblocks/n/namd.py b/easybuild/easyblocks/n/namd.py index 2fb826b4cfb..ae89f349835 100644 --- a/easybuild/easyblocks/n/namd.py +++ b/easybuild/easyblocks/n/namd.py @@ -1,7 +1,7 @@ ## # This file is an EasyBuild reciPY as per https://github.com/easybuilders/easybuild # -# Copyright:: Copyright 2013-2024 CaSToRC, The Cyprus Institute +# Copyright:: Copyright 2013-2025 CaSToRC, The Cyprus Institute # Authors:: George Tsouloupas # License:: MIT/GPL # $Id$ @@ -26,7 +26,7 @@ from easybuild.tools.config import build_option from easybuild.tools.filetools import apply_regex_substitutions, change_dir, extract_file from easybuild.tools.modules import get_software_root, get_software_version -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd from easybuild.tools.systemtools import POWER, X86_64, get_cpu_architecture @@ -93,7 +93,13 @@ def patch_step(self, *args, **kwargs): self.charm_dir = self.charm_tarballs[0][:-4] - charm_config = os.path.join(self.charm_dir, 'src', 'scripts', 'configure') + # NAMD-3.0 depends on charm-8.0.0 that uses Automake. The 'configure' file was + # removed in favour of 'configure.ac' + configure_file_name = 'configure' + if LooseVersion(self.version) >= LooseVersion('3.0'): + configure_file_name = 'configure.ac' + + charm_config = os.path.join(self.charm_dir, 'src', 'scripts', configure_file_name) apply_regex_substitutions(charm_config, [(r'SHELL=/bin/csh', 'SHELL=$(which csh)')]) for csh_script in [os.path.join('plugins', 'import_tree'), os.path.join('psfgen', 'import_tree'), @@ -134,15 +140,18 @@ def configure_step(self): self.namd_arch = '%s-%s' % (self.cfg['namd_basearch'], namd_comp) self.log.info("Completed NAMD target architecture: %s", self.namd_arch) - cmd = "./build charm++ %(arch)s %(opts)s --with-numa -j%(parallel)s '%(cxxflags)s'" % { + build_cmd = './build' + + cmd = "%(build_cmd)s charm++ %(arch)s %(opts)s --with-numa -j%(parallel)s '%(cxxflags)s'" % { + 'build_cmd': build_cmd, 'arch': self.cfg['charm_arch'], 'cxxflags': os.environ['CXXFLAGS'] + ' -DMPICH_IGNORE_CXX_SEEK ' + self.cfg['charm_extra_cxxflags'], 'opts': self.cfg['charm_opts'], - 'parallel': self.cfg['parallel'], + 'parallel': self.cfg.parallel, } charm_subdir = '.'.join(os.path.basename(self.charm_tarballs[0]).split('.')[:-1]) self.log.debug("Building Charm++ using cmd '%s' in '%s'" % (cmd, charm_subdir)) - run_cmd(cmd, path=charm_subdir) + run_shell_cmd(cmd, work_dir=charm_subdir) # compiler (options) self.cfg.update('namd_cfg_opts', '--cc "%s" --cc-opts "%s"' % (os.environ['CC'], os.environ['CFLAGS'])) @@ -181,7 +190,7 @@ def configure_step(self): namd_charm_arch = "--charm-arch %s" % '-'.join(self.cfg['charm_arch'].strip().split()) cmd = "./config %s %s %s " % (self.namd_arch, namd_charm_arch, self.cfg["namd_cfg_opts"]) - run_cmd(cmd) + run_shell_cmd(cmd) def build_step(self): """Build NAMD for configured architecture""" @@ -208,14 +217,13 @@ def test_step(self): 'testdir': os.path.join(self.cfg['start_dir'], self.namd_arch, 'src', 'alanin'), 'testopts': self.cfg['testopts'], } - out, ec = run_cmd(cmd, simple=False) - if ec == 0: - test_ok_regex = re.compile(r"(^Program finished.$|End of program\s*$)", re.M) - if test_ok_regex.search(out): - self.log.debug("Test '%s' ran fine." % cmd) - else: - raise EasyBuildError("Test '%s' failed ('%s' not found), output: %s", - cmd, test_ok_regex.pattern, out) + res = run_shell_cmd(cmd) + test_ok_regex = re.compile(r"(^Program finished.$|End of program\s*$)", re.M) + if test_ok_regex.search(res.output): + self.log.debug("Test '%s' ran fine." % cmd) + else: + raise EasyBuildError("Test '%s' failed ('%s' not found), output: %s", cmd, test_ok_regex.pattern, + res.output) else: self.log.debug("Skipping running NAMD test case after building") diff --git a/easybuild/easyblocks/n/nccl.py b/easybuild/easyblocks/n/nccl.py index 15ce298ce0b..cf00c8eee3c 100644 --- a/easybuild/easyblocks/n/nccl.py +++ b/easybuild/easyblocks/n/nccl.py @@ -1,5 +1,5 @@ ## -# Copyright 2021-2024 Ghent University +# Copyright 2021-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/n/ncl.py b/easybuild/easyblocks/n/ncl.py index 3169ab86d12..38a7aff9da8 100644 --- a/easybuild/easyblocks/n/ncl.py +++ b/easybuild/easyblocks/n/ncl.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -40,7 +40,7 @@ from easybuild.framework.easyblock import EasyBlock from easybuild.tools.build_log import EasyBuildError from easybuild.tools.modules import get_software_root -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd class EB_NCL(EasyBlock): @@ -61,10 +61,10 @@ def configure_step(self): raise EasyBuildError("Failed to change to the 'config' dir: %s", err) cmd = "make -f Makefile.ini" - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) cmd = "./ymake -config $PWD" - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) # figure out name of config file cfg_regexp = re.compile(r'^\s*SYSTEM_INCLUDE\s*=\s*"(.*)"\s*$', re.M) @@ -215,7 +215,7 @@ def configure_step(self): # generate Makefile cmd = "./config/ymkmf" - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) def build_step(self): """Building is done in install_step.""" @@ -225,7 +225,7 @@ def install_step(self): """Build in install dir using build_step.""" cmd = "%s make Everything %s" % (self.cfg['preinstallopts'], self.cfg['installopts']) - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) def sanity_check_step(self): """ diff --git a/easybuild/easyblocks/n/ncurses.py b/easybuild/easyblocks/n/ncurses.py deleted file mode 100644 index 66176969aae..00000000000 --- a/easybuild/easyblocks/n/ncurses.py +++ /dev/null @@ -1,56 +0,0 @@ -## -# This file is an EasyBuild reciPY as per https://github.com/easybuilders/easybuild -# -# Copyright:: Copyright 2012-2024 Uni.Lu/LCSB, NTUA -# Authors:: Cedric Laczny , Fotis Georgatos , Kenneth Hoste -# License:: MIT/GPL -# $Id$ -# -# This work implements a part of the HPCBIOS project and is a component of the policy: -# http://hpcbios.readthedocs.org/en/latest/HPCBIOS_2012-90.html -## -""" -Easybuild support for building ncurses, implemented as an easyblock - -@author: Cedric Laczny (Uni.Lu) -@author: Fotis Georgatos (Uni.Lu) -@author: Kenneth Hoste (Ghent University) -""" - -import os -from easybuild.easyblocks.generic.configuremake import ConfigureMake - - -class EB_ncurses(ConfigureMake): - """ - Support for building ncurses - """ - - def configure_step(self): - """ - No configure - """ - self.cfg.update('configopts', '--with-shared --enable-overwrite') - super(EB_ncurses, self).configure_step() - - def sanity_check_step(self): - """Custom sanity check for ncurses.""" - - binaries = ["captoinfo", "clear", "infocmp", "infotocap", "ncurses5-config", "reset", "tabs", "tic", "toe", - "tput", "tset"] - libs = ['lib%s.a' % x for x in ["form", "form", "menu", "menu_g", "ncurses", "ncurses++", "ncurses_g", - "panel", "panel_g"]] - custom_paths = { - 'files': [os.path.join('bin', x) for x in binaries] + [os.path.join('lib', x) for x in libs], - 'dirs': ['include'] - } - - super(EB_ncurses, self).sanity_check_step(custom_paths=custom_paths) - - def make_module_req_guess(self): - """ - Set correct CPLUS path. - """ - guesses = super(EB_ncurses, self).make_module_req_guess() - guesses.update({'CPLUS': ['include/ncurses']}) # will only be present without --enable-overwrite - return guesses diff --git a/easybuild/easyblocks/n/nemo.py b/easybuild/easyblocks/n/nemo.py deleted file mode 100644 index f6ce8fca1d0..00000000000 --- a/easybuild/easyblocks/n/nemo.py +++ /dev/null @@ -1,119 +0,0 @@ -## -# Copyright 2015-2024 Ghent University -# -# This file is part of EasyBuild, -# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), -# with support of Ghent University (http://ugent.be/hpc), -# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), -# Flemish Research Foundation (FWO) (http://www.fwo.be/en) -# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). -# -# https://github.com/easybuilders/easybuild -# -# EasyBuild is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation v2. -# -# EasyBuild is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with EasyBuild. If not, see . -## -""" -EasyBuild support for building and installing NEMO, implemented as an easyblock - -@author: Oriol Mula-Valls (IC3) -""" -import os -import shutil - -from easybuild.framework.easyblock import EasyBlock -from easybuild.framework.easyconfig import CUSTOM, MANDATORY -from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.filetools import write_file -from easybuild.tools.modules import get_software_root -from easybuild.tools.run import run_cmd - - -class EB_NEMO(EasyBlock): - """Support for building/installing NEMO.""" - - def __init__(self, *args, **kwargs): - """Initialisation of custom class variables for NEMO.""" - super(EB_NEMO, self).__init__(*args, **kwargs) - - self.conf_name = 'EB_NEMO_CONFIG' - self.conf_arch_file = 'NEMOGCM/ARCH/arch-eb.fcm' - - @staticmethod - def extra_options(): - """Custom easyconfig parameters for NEMO.""" - extra_vars = { - 'with_components': [None, "List of components to include (e.g. TOP_SRC)", MANDATORY], - 'add_keys': [None, "Add compilation keys", CUSTOM], - 'del_keys': [None, "Delete compilation keys", CUSTOM] - } - return EasyBlock.extra_options(extra_vars) - - def configure_step(self): - """Custom configuration procedure for NEMO.""" - netcdf_fortran_root = get_software_root('netCDF-Fortran') - if not netcdf_fortran_root: - raise EasyBuildError("netCDF-Fortran is not available, but is a required dependency") - - cfg = '\n'.join([ - "%%NCDF_INC -I%s/include" % netcdf_fortran_root, - "%%NCDF_LIB -L%s/lib -lnetcdff" % netcdf_fortran_root, - "%%FC %s" % os.getenv('F90'), - "%FCFLAGS -r8 -O3 -traceback", - "%FFLAGS %FCFLAGS", - "%LD %FC", - "%LDFLAGS ", - "%FPPFLAGS -P -C", - "%AR ar", - "%ARFLAGS rs", - "%MK make", - "%USER_INC %NCDF_INC", - "%USER_LIB %NCDF_LIB" - ]) - write_file(self.conf_arch_file, cfg) - - cmd = "./makenemo -n %s -d '%s' -j0 -m eb" % (self.conf_name, ' '.join(self.cfg['with_components'])) - - if self.cfg['add_keys'] is not None: - cmd += " add_key '%s'" % ' '.join(self.cfg['add_keys']) - if self.cfg['del_keys'] is not None: - cmd += " del_key '%s'" % ' '.join(self.cfg['del_keys']) - - try: - dst = 'NEMOGCM/CONFIG' - os.chdir(dst) - self.log.debug("Changed to directory %s", dst) - except OSError as err: - raise EasyBuildError("Failed to change to directory %s: %s", dst, err) - - run_cmd(cmd, log_all=True, simple=True, log_ok=True) - - def build_step(self): - """Custom build procedure for NEMO.""" - cmd = "./makenemo -n %s -m eb" % self.conf_name - run_cmd(cmd, log_all=True, simple=True, log_ok=True) - - def install_step(self): - """Custom install procedure for NEMO.""" - binpath = os.path.join(self.cfg['start_dir'], 'NEMOGCM', 'CONFIG', self.conf_name, 'BLD/bin') - try: - shutil.copytree(binpath, os.path.join(self.installdir, 'bin')) - except OSError as err: - raise EasyBuildError("Copying %s to installation dir failed: %s", binpath, err) - - def sanity_check_step(self): - """Custom sanity check for NEMO.""" - custom_paths = { - 'files': ['bin/nemo.exe'], - 'dirs': [], - } - super(EB_NEMO, self).sanity_check_step(custom_paths=custom_paths) diff --git a/easybuild/easyblocks/n/netcdf.py b/easybuild/easyblocks/n/netcdf.py index 3d90c795404..7477e86d707 100644 --- a/easybuild/easyblocks/n/netcdf.py +++ b/easybuild/easyblocks/n/netcdf.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/n/netcdf4_python.py b/easybuild/easyblocks/n/netcdf4_python.py index c8bb096f623..6c76c7ea2b0 100644 --- a/easybuild/easyblocks/n/netcdf4_python.py +++ b/easybuild/easyblocks/n/netcdf4_python.py @@ -1,5 +1,5 @@ ## -# Copyright 2013-2024 Ghent University +# Copyright 2013-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/n/netcdf_fortran.py b/easybuild/easyblocks/n/netcdf_fortran.py index 3365cb837db..ed143afe909 100644 --- a/easybuild/easyblocks/n/netcdf_fortran.py +++ b/easybuild/easyblocks/n/netcdf_fortran.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/n/neuron.py b/easybuild/easyblocks/n/neuron.py index fb462030fc8..8c2c4c83c3b 100644 --- a/easybuild/easyblocks/n/neuron.py +++ b/easybuild/easyblocks/n/neuron.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -27,186 +27,133 @@ @author: Kenneth Hoste (Ghent University) @author: Maxime Boissonneault (Universite Laval, Compute Canada) +@author: Alex Domingo (Vrije Universiteit Brussel) """ import os import re +import tempfile -from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.easyblocks.generic.cmakemake import CMakeMake from easybuild.easyblocks.generic.pythonpackage import det_pylibdir from easybuild.framework.easyconfig import CUSTOM +from easybuild.tools import LooseVersion from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_option +from easybuild.tools.filetools import write_file from easybuild.tools.modules import get_software_root -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd from easybuild.tools.systemtools import get_shared_lib_ext -from easybuild.tools import LooseVersion - class EB_NEURON(CMakeMake): """Support for building/installing NEURON.""" - def __init__(self, *args, **kwargs): - """Initialisation of custom class variables for NEURON.""" - super(EB_NEURON, self).__init__(*args, **kwargs) - - self.hostcpu = 'UNKNOWN' - self.with_python = False - self.pylibdir = 'UNKNOWN' - @staticmethod def extra_options(): """Custom easyconfig parameters for NEURON.""" - extra_vars = { 'paranrn': [True, "Enable support for distributed simulations.", CUSTOM], } return CMakeMake.extra_options(extra_vars) - def configure_step(self): - """Custom configuration procedure for NEURON.""" - if LooseVersion(self.version) < LooseVersion('7.8.1'): - - # make sure we're using the correct configure command - # (required because custom easyconfig parameters from CMakeMake are picked up) - self.cfg['configure_cmd'] = "./configure" - - # enable support for distributed simulations if desired - if self.cfg['paranrn']: - self.cfg.update('configopts', '--with-paranrn') - - # specify path to InterViews if it is available as a dependency - interviews_root = get_software_root('InterViews') - if interviews_root: - self.cfg.update('configopts', "--with-iv=%s" % interviews_root) - else: - self.cfg.update('configopts', "--without-iv") - - # optionally enable support for Python as alternative interpreter - python_root = get_software_root('Python') - if python_root: - self.with_python = True - self.cfg.update('configopts', "--with-nrnpython=%s/bin/python" % python_root) + def __init__(self, *args, **kwargs): + """Initialisation of custom class variables for NEURON.""" + super(EB_NEURON, self).__init__(*args, **kwargs) - # determine host CPU type - cmd = "./config.guess" - (out, ec) = run_cmd(cmd, simple=False) + self.python_root = None + self.pylibdir = 'UNKNOWN' - self.hostcpu = out.split('\n')[0].split('-')[0] - self.log.debug("Determined host CPU type as %s" % self.hostcpu) + def prepare_step(self, *args, **kwargs): + """Custom prepare step with python detection""" + super(EB_NEURON, self).prepare_step(*args, **kwargs) - # determine Python lib dir - self.pylibdir = det_pylibdir() + self.python_root = get_software_root('Python') - # complete configuration with configure_method of parent - ConfigureMake.configure_step(self) + def configure_step(self): + """Custom configuration procedure for NEURON.""" + # enable support for distributed simulations if desired + if self.cfg['paranrn']: + self.cfg.update('configopts', '-DNRN_ENABLE_MPI=ON') else: - # enable support for distributed simulations if desired - if self.cfg['paranrn']: - self.cfg.update('configopts', '-DNRN_ENABLE_MPI=ON') - else: - self.cfg.update('configopts', '-DNRN_ENABLE_MPI=OFF') - - # specify path to InterViews if it is available as a dependency - interviews_root = get_software_root('InterViews') - if interviews_root: - self.cfg.update('configopts', "-DIV_DIR=%s -DNRN_ENABLE_INTERVIEWS=ON" % interviews_root) - else: - self.cfg.update('configopts', "-DNRN_ENABLE_INTERVIEWS=OFF") - - # no longer used it seems - self.hostcpu = '' - - # optionally enable support for Python as alternative interpreter - python_root = get_software_root('Python') - if python_root: - self.with_python = True - self.cfg.update('configopts', "-DNRN_ENABLE_PYTHON=ON -DPYTHON_EXECUTABLE=%s/bin/python" % python_root) - self.cfg.update('configopts', "-DNRN_ENABLE_MODULE_INSTALL=ON " - "-DNRN_MODULE_INSTALL_OPTIONS='--prefix=%s'" % self.installdir) - else: - self.cfg.update('configopts', "-DNRN_ENABLE_PYTHON=OFF") + self.cfg.update('configopts', '-DNRN_ENABLE_MPI=OFF') - # determine Python lib dir - self.pylibdir = det_pylibdir() - - # complete configuration with configure_method of parent - CMakeMake.configure_step(self) - - def install_step(self): - """Custom install procedure for NEURON.""" + # specify path to InterViews if it is available as a dependency + interviews_root = get_software_root('InterViews') + if interviews_root: + self.cfg.update('configopts', f"-DIV_DIR={interviews_root} -DNRN_ENABLE_INTERVIEWS=ON") + else: + self.cfg.update('configopts', "-DNRN_ENABLE_INTERVIEWS=OFF") + + # optionally enable support for Python as alternative interpreter + if self.python_root: + python_cfgopts = " ".join([ + "-DNRN_ENABLE_PYTHON=ON", + f"-DPYTHON_EXECUTABLE={self.python_root}/bin/python", + "-DNRN_ENABLE_MODULE_INSTALL=ON", + f"-DNRN_MODULE_INSTALL_OPTIONS='--prefix={self.installdir}'", + ]) + self.cfg.update('configopts', python_cfgopts) + else: + self.cfg.update('configopts', "-DNRN_ENABLE_PYTHON=OFF") - super(EB_NEURON, self).install_step() + # determine Python lib dir + self.pylibdir = det_pylibdir() - # with the CMakeMake, the python module is installed automatically - if LooseVersion(self.version) < LooseVersion('7.8.1'): - if self.with_python: - pypath = os.path.join('src', 'nrnpython') - try: - pwd = os.getcwd() - os.chdir(pypath) - except OSError as err: - raise EasyBuildError("Failed to change to %s: %s", pypath, err) + # complete configuration with configure_method of parent + CMakeMake.configure_step(self) - cmd = "python setup.py install --prefix=%s" % self.installdir - run_cmd(cmd, simple=True, log_all=True, log_ok=True) + def test_step(self): + """Custom tests for NEURON.""" + if build_option('mpi_tests'): + nproc = self.cfg.parallel + try: + hoc_file = os.path.join(self.cfg['start_dir'], 'src', 'parallel', 'test0.hoc') + cmd = self.toolchain.mpi_cmd_for(f"bin/nrniv -mpi {hoc_file}", nproc) + res = run_shell_cmd(cmd) + except OSError as err: + raise EasyBuildError("Failed to run parallel hello world: %s", err) - try: - os.chdir(pwd) - except OSError as err: - raise EasyBuildError("Failed to change back to %s: %s", pwd, err) + valid = True + for i in range(0, nproc): + validate_regexp = re.compile(f"I am {i:d} of {nproc:d}") + if not validate_regexp.search(res.output): + valid = False + break + if res.exit_code or not valid: + raise EasyBuildError("Validation of parallel hello world run failed.") + self.log.info("Parallel hello world OK!") + else: + self.log.info("Skipping MPI testing of NEURON since MPI testing is disabled") def sanity_check_step(self): """Custom sanity check for NEURON.""" shlib_ext = get_shared_lib_ext() - binpath = os.path.join(self.hostcpu, 'bin') - libpath = os.path.join(self.hostcpu, 'lib', 'lib%s.' + shlib_ext) - # hoc_ed is not included in the sources of 7.4. However, it is included in the binary distribution. - # Nevertheless, the binary has a date old enough (June 2014, instead of November 2015 like all the - # others) to be considered a mistake in the distribution - binaries = ["neurondemo", "nrngui", "nrniv", "nrnivmodl", "nocmodl", "modlunit", "nrnmech_makefile", - "mkthreadsafe"] - libs = ["nrniv"] - sanity_check_dirs = ['share/nrn'] - - if LooseVersion(self.version) < LooseVersion('7.4'): - binaries += ["hoc_ed"] - - if LooseVersion(self.version) < LooseVersion('7.8.1'): - binaries += ["bbswork.sh", "hel2mos1.sh", "ivoc", "memacs", "mos2nrn", "mos2nrn2.sh", "oc"] - binaries += ["nrn%s" % x for x in ["iv_makefile", "oc", "oc_makefile", "ocmodl"]] - libs += ["ivoc", "ivos", "memacs", "meschach", "neuron_gnu", "nrnmpi", "nrnoc", "nrnpython", - "oc", "ocxt", "scopmath", "sparse13", "sundials"] - sanity_check_dirs += ['include/nrn'] - # list of included binaries changed with cmake. See - # https://github.com/neuronsimulator/nrn/issues/899 - else: - binaries += ["nrnpyenv.sh", "set_nrnpyenv.sh", "sortspike"] - libs += ["rxdmath"] - sanity_check_dirs += ['include'] - if self.with_python: - sanity_check_dirs += [os.path.join("lib", "python"), - os.path.join("lib", "python%(pyshortver)s", "site-packages")] + + binaries = ["mkthreadsafe", "modlunit", "neurondemo", "nocmodl", "nrngui", "nrniv", "nrnivmodl", + "nrnmech_makefile", "nrnpyenv.sh", "set_nrnpyenv.sh", "sortspike"] + libs = ["nrniv", "rxdmath"] + sanity_check_dirs = ['include', 'share/nrn'] + + if self.python_root: + sanity_check_dirs += [os.path.join("lib", "python")] + if LooseVersion(self.version) < LooseVersion('8'): + sanity_check_dirs += [os.path.join("lib", "python%(pyshortver)s", "site-packages")] # this is relevant for installations of Python packages for multiple Python versions (via multi_deps) # (we can not pass this via custom_paths, since then the %(pyshortver)s template value will not be resolved) # ensure that we only add to paths specified in the EasyConfig - sanity_check_files = [os.path.join(binpath, x) for x in binaries] + [libpath % x for x in libs] - self.cfg['sanity_check_paths'] = { - 'files': sanity_check_files, - 'dirs': sanity_check_dirs, - } - - super(EB_NEURON, self).sanity_check_step() + sanity_check_files = [os.path.join('bin', x) for x in binaries] + sanity_check_files += [f'lib/lib{soname}.{shlib_ext}' for soname in libs] - try: - fake_mod_data = self.load_fake_module() - except EasyBuildError as err: - self.log.debug("Loading fake module failed: %s" % err) + custom_paths = { + 'files': sanity_check_files, + 'dirs': sanity_check_dirs, + } - # test NEURON demo - inp = '\n'.join([ + # run NEURON demo + demo_dir = tempfile.mkdtemp() + demo_inp_file = os.path.join(demo_dir, 'neurondemo.inp') + demo_inp_cmds = '\n'.join([ "demo(3) // load the pyramidal cell model.", "init() // initialise the model", "t // should be zero", @@ -216,57 +163,34 @@ def sanity_check_step(self): "soma.v // will print a value other than -65, indicating that the simulation was executed", "quit()", ]) - (out, ec) = run_cmd("neurondemo", simple=False, log_all=True, log_output=True, inp=inp) - - validate_regexp = re.compile(r"^\s+-65\s*\n\s+5\s*\n\s+-68.134337", re.M) - if ec or not validate_regexp.search(out): - raise EasyBuildError("Validation of NEURON demo run failed.") - else: - self.log.info("Validation of NEURON demo OK!") - - if build_option('mpi_tests'): - nproc = self.cfg['parallel'] - try: - cwd = os.getcwd() - os.chdir(os.path.join(self.cfg['start_dir'], 'src', 'parallel')) - - cmd = self.toolchain.mpi_cmd_for("nrniv -mpi test0.hoc", nproc) - (out, ec) = run_cmd(cmd, simple=False, log_all=True, log_output=True) - - os.chdir(cwd) - except OSError as err: - raise EasyBuildError("Failed to run parallel hello world: %s", err) - - valid = True - for i in range(0, nproc): - validate_regexp = re.compile("I am %d of %d" % (i, nproc)) - if not validate_regexp.search(out): - valid = False - break - if ec or not valid: - raise EasyBuildError("Validation of parallel hello world run failed.") + write_file(demo_inp_file, demo_inp_cmds) + + demo_sanity_cmd = f"neurondemo < {demo_inp_file} 2>&1" + demo_regexp_version = f'NEURON -- VERSION {self.version}' + demo_regexp_soma = '68.134337' + + custom_commands = [ + f"{demo_sanity_cmd} | grep -c '{demo_regexp_version}'", + f"{demo_sanity_cmd} | grep -c '{demo_regexp_soma}'", + ] + + if self.python_root: + custom_commands.append("python -c 'import neuron; neuron.test()'") + + super(EB_NEURON, self).sanity_check_step(custom_paths=custom_paths, custom_commands=custom_commands) + + def make_module_step(self, *args, **kwargs): + """ + Custom paths of NEURON module load environment + """ + if self.python_root: + # location of neuron python package + if LooseVersion(self.version) < LooseVersion('8'): + self.module_load_environment.PYTHONPATH = [os.path.join("lib", "python*", "site-packages")] else: - self.log.info("Parallel hello world OK!") - else: - self.log.info("Skipping MPI testing of NEURON since MPI testing is disabled") - - if self.with_python: - cmd = "python -c 'import neuron; neuron.test()'" - (out, ec) = run_cmd(cmd, simple=False, log_all=True, log_output=True) - - # cleanup - self.clean_up_fake_module(fake_mod_data) + self.module_load_environment.PYTHONPATH = [os.path.join('lib', 'python')] - def make_module_req_guess(self): - """Custom guesses for environment variables (PATH, ...) for NEURON.""" - - guesses = super(EB_NEURON, self).make_module_req_guess() - - guesses.update({ - 'PATH': [os.path.join(self.hostcpu, 'bin')], - }) - - return guesses + return super(EB_NEURON, self).make_module_step(*args, **kwargs) def make_module_extra(self): """Define extra module entries required.""" @@ -279,15 +203,8 @@ def make_module_extra(self): val = os.getenv(var) if val: txt += self.module_generator.set_environment(var, val) - self.log.debug("%s set to %s, adding it to module" % (var, val)) + self.log.debug(f"{var} set to {val}, adding it to module") else: - self.log.debug("%s not set: %s" % (var, os.environ.get(var, None))) + self.log.debug(f"{var} not set: {os.environ.get(var, None)}") - if self.with_python: - if self.cfg['multi_deps'] and 'Python' in self.cfg['multi_deps']: - txt += self.module_generator.prepend_paths('EBPYTHONPREFIXES', '') - else: - txt += self.module_generator.prepend_paths('PYTHONPATH', [self.pylibdir]) - # also adds lib/python to PYTHONPATH - txt += self.module_generator.prepend_paths('PYTHONPATH', ['lib/python']) return txt diff --git a/easybuild/easyblocks/n/nim.py b/easybuild/easyblocks/n/nim.py index d54fdde56c1..95643e02a5c 100644 --- a/easybuild/easyblocks/n/nim.py +++ b/easybuild/easyblocks/n/nim.py @@ -1,5 +1,5 @@ ## -# Copyright 2018-2024 Ghent University +# Copyright 2018-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -31,7 +31,7 @@ from easybuild.framework.easyblock import EasyBlock from easybuild.tools.filetools import copy_file, move_file -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd class EB_Nim(EasyBlock): @@ -45,22 +45,22 @@ def build_step(self): """Custom build procedure for Nim.""" # build Nim (bin/nim) - run_cmd("sh build.sh") + run_shell_cmd("sh build.sh") # build koch management tool - run_cmd("bin/nim c -d:release koch") + run_shell_cmd("bin/nim c -d:release koch") # rebuild Nim, with readline bindings - run_cmd("./koch boot -d:release -d:useLinenoise") + run_shell_cmd("./koch boot -d:release -d:useLinenoise") # build nimble/nimgrep/nimsuggest tools - run_cmd("./koch tools") + run_shell_cmd("./koch tools") def install_step(self): """Custom install procedure for Nim.""" - run_cmd("./koch geninstall") - run_cmd("sh install.sh %s" % self.installdir) + run_shell_cmd("./koch geninstall") + run_shell_cmd("sh install.sh %s" % self.installdir) # install.sh copies stuff into /nim, so move it nim_dir = os.path.join(self.installdir, 'nim') diff --git a/easybuild/easyblocks/n/nose.py b/easybuild/easyblocks/n/nose.py index d46223e7fa5..327296da3ec 100644 --- a/easybuild/easyblocks/n/nose.py +++ b/easybuild/easyblocks/n/nose.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), diff --git a/easybuild/easyblocks/n/numexpr.py b/easybuild/easyblocks/n/numexpr.py index 8d222b40dd4..0fb54500210 100644 --- a/easybuild/easyblocks/n/numexpr.py +++ b/easybuild/easyblocks/n/numexpr.py @@ -1,5 +1,5 @@ ## -# Copyright 2019-2024 Ghent University +# Copyright 2019-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -37,16 +37,6 @@ class EB_numexpr(PythonPackage): """Support for building/installing numexpr.""" - @staticmethod - def extra_options(): - """Override some custom easyconfig parameters specifically for numexpr.""" - extra_vars = PythonPackage.extra_options() - - extra_vars['download_dep_fail'][0] = True - extra_vars['use_pip'][0] = True - - return extra_vars - def __init__(self, *args, **kwargs): """Initialisation of custom class variables for numexpr.""" super(EB_numexpr, self).__init__(*args, **kwargs) diff --git a/easybuild/easyblocks/n/numpy.py b/easybuild/easyblocks/n/numpy.py index 968cefa9197..404db621e37 100644 --- a/easybuild/easyblocks/n/numpy.py +++ b/easybuild/easyblocks/n/numpy.py @@ -1,5 +1,5 @@ ## -# Copyright 2009-2024 Ghent University +# Copyright 2009-2025 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -44,7 +44,7 @@ from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import change_dir, mkdir, read_file, remove_dir from easybuild.tools.modules import get_software_root -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd from easybuild.tools import LooseVersion @@ -204,12 +204,21 @@ def get_libs_for_mkl(varname): 'includes': includes, } + if LooseVersion(self.version) < LooseVersion('1.26'): + # NumPy detects the required math by trying to link a minimal code containing a call to `log(0.)`. + # The first try is without any libraries, which works with `gcc -fno-math-errno` (our optimization default) + # because the call gets removed due to not having any effect. So it concludes that `-lm` is not required. + # This then fails to detect availability of functions such as `acosh` which do not get removed in the same + # way and so less exact replacements are used instead which e.g. fail the tests on PPC. + # This variable makes it try `-lm` first and is supported until the Meson backend is used in 1.26+. + env.setvar('MATHLIB', 'm') + super(EB_numpy, self).configure_step() if LooseVersion(self.version) < LooseVersion('1.21'): # check configuration (for debugging purposes) cmd = "%s setup.py config" % self.python_cmd - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) if LooseVersion(self.version) >= LooseVersion('1.26'): # control BLAS/LAPACK library being used @@ -267,7 +276,7 @@ def test_step(self): mkdir(pylibdir, parents=True) pythonpath = "export PYTHONPATH=%s &&" % os.pathsep.join(abs_pylibdirs + ['$PYTHONPATH']) cmd = self.compose_install_command(tmpdir, extrapath=pythonpath) - run_cmd(cmd, log_all=True, simple=True, verbose=False) + run_shell_cmd(cmd) try: pwd = os.getcwd() @@ -283,20 +292,20 @@ def test_step(self): '-s "import numpy; x = numpy.random.random((%(size)d, %(size)d))"' % {'size': size}, '"numpy.dot(x, x.T)"', ]) - (out, ec) = run_cmd(cmd, simple=False) - self.log.debug("Test output: %s" % out) + res = run_shell_cmd(cmd) + self.log.debug("Test output: %s" % res.output) # fetch result time_msec = None msec_re = re.compile(r"\d+ loops, best of \d+: (?P