From bf2392926fe6e2f3d3dd86533bbb76a690c1131c Mon Sep 17 00:00:00 2001 From: crivella Date: Thu, 14 Mar 2024 15:45:12 +0100 Subject: [PATCH 01/18] Added easyblock for QE with cmake --- .../easyblocks/q/quantumespressocmake.py | 354 ++++++++++++++++++ 1 file changed, 354 insertions(+) create mode 100644 easybuild/easyblocks/q/quantumespressocmake.py diff --git a/easybuild/easyblocks/q/quantumespressocmake.py b/easybuild/easyblocks/q/quantumespressocmake.py new file mode 100644 index 00000000000..2209afed495 --- /dev/null +++ b/easybuild/easyblocks/q/quantumespressocmake.py @@ -0,0 +1,354 @@ +## +# Copyright 2009-2023 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for Quantum ESPRESSO, implemented as an easyblock + +@author: Davide Grassano (CECAM, EPFL) +""" +import os +import re + +from easybuild.framework.easyconfig import CUSTOM +from easybuild.tools import LooseVersion +from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.modules import get_software_root +from easybuild.tools.run import run_cmd + +from easybuild.easyblocks.generic.cmakemake import CMakeMake + + +class EB_QuantumESPRESSOcmake(CMakeMake): + """Support for building and installing Quantum ESPRESSO.""" + + TEST_SUITE_DIR = "test-suite" + + @staticmethod + def extra_options(): + """Custom easyconfig parameters for Quantum ESPRESSO.""" + extra_vars = { + 'with_cuda': [False, "Enable CUDA support", CUSTOM], + 'with_scalapack': [True, "Enable ScaLAPACK support", CUSTOM], + 'with_fox': [False, "Enable FoX support", CUSTOM], + 'with_gipaw': [True, "Enable GIPAW support", CUSTOM], + 'with_d3q': [False, "Enable D3Q support", CUSTOM], + 'with_qmcpack': [False, "Enable QMCPACK support", CUSTOM], + 'test_suite_nprocs': [1, "Number of processors to use for the test suite", CUSTOM], + 'test_suite_allow_failures': [[ + 'test_qe_xclib_', # 7.3.1: https://gitlab.com/QEF/q-e/-/issues/640 + '--hp_', # 7.3.1: Broken testsuite (https://gitlab.com/QEF/q-e/-/issues/665) + '--ph_', # 7.3.1: Broken testsuite (https://gitlab.com/QEF/q-e/-/issues/665) + '--epw_', # 7.3.1: Broken testsuite (https://gitlab.com/QEF/q-e/-/issues/665) + '--tddfpt_', # 7.3.1: Broken testsuite (https://gitlab.com/QEF/q-e/-/issues/665) + ], "List of test suite targets that are allowed to fail (name can partially match)", CUSTOM], + 'test_suite_threshold': [ + 0.97, + "Threshold for test suite success rate (does count also allowed failures)", + CUSTOM + ], + 'test_suite_max_failed': [0, "Maximum number of failing tests (does not count allowed failures)", CUSTOM], + } + return CMakeMake.extra_options(extra_vars) + + def __init__(self, *args, **kwargs): + """Add extra config options specific to Quantum ESPRESSO.""" + super(EB_QuantumESPRESSOcmake, self).__init__(*args, **kwargs) + + self.install_subdir = "qe-%s" % self.version + + self.check_bins = [] + + def _add_toolchains_opts(self): + """Enable toolchain options for Quantum ESPRESSO.""" + self._add_mpi() + self._add_openmp() + self._add_cuda() + + def _add_libraries(self): + """Enable external libraries for Quantum ESPRESSO.""" + self._add_scalapack() + self._add_fox() + self._add_libxc() + self._add_elpa() + self._add_hdf5() + + def _add_plugins(self): + """Enable plugins for Quantum ESPRESSO.""" + plugins = [] + plugins += self._add_gipaw() + plugins += self._add_d3q() + plugins += self._add_qmcpack() + if plugins: + self.cfg.update('configopts', '-DQE_ENABLE_PLUGINS="%s"' % ';'.join(plugins)) + + def _add_mpi(self): + """Enable MPI for Quantum ESPRESSO.""" + if self.toolchain.options.get('usempi', False): + self.cfg.update('configopts', '-DENABLE_MPI=ON') + + def _add_openmp(self): + """Enable OpenMP for Quantum ESPRESSO.""" + if self.toolchain.options.get('openmp', False): + self.cfg.update('configopts', '-DENABLE_OPENMP=ON') + + def _add_cuda(self): + """Enable CUDA for Quantum ESPRESSO.""" + if self.cfg.get('with_cuda', False): + self.cfg.update('configopts', '-DQE_ENABLE_CUDA=ON') + self.cfg.update('configopts', '-DQE_ENABLE_OPENACC=ON') + + def _add_scalapack(self): + """Enable ScaLAPACK for Quantum ESPRESSO.""" + if self.cfg.get('with_scalapack', False): + if not self.toolchain.options.get('usempi', False): + raise EasyBuildError("ScaLAPACK support requires MPI") + self.cfg.update('configopts', '-DQE_ENABLE_SCALAPACK=ON') + + def _add_fox(self): + """Enable FoX for Quantum ESPRESSO.""" + if self.cfg.get('with_fox', False): + self.cfg.update('configopts', '-DQE_ENABLE_FOX=ON') + + def _add_hdf5(self): + """Enable HDF5 for Quantum ESPRESSO.""" + if get_software_root("HDF5"): + self.cfg.update('configopts', '-DQE_ENABLE_HDF5=ON') + + def _add_libxc(self): + """Enable LibXC for Quantum ESPRESSO.""" + if get_software_root("libxc"): + self.cfg.update('configopts', '-DQE_ENABLE_LIBXC=ON') + + def _add_elpa(self): + """Enable ELPA for Quantum ESPRESSO.""" + if get_software_root("ELPA"): + if not self.cfg.get('with_scalapack', False): + raise EasyBuildError("ELPA support requires ScaLAPACK") + if LooseVersion(self.version) == LooseVersion('7.3') and self.toolchain.options.get('openmp', False): + raise EasyBuildError("QE 7.3 with cmake does not support ELPA with OpenMP") + self.cfg.update('configopts', '-DQE_ENABLE_ELPA=ON') + + def _add_gipaw(self): + """Enable GIPAW for Quantum ESPRESSO.""" + res = [] + if self.cfg.get('with_gipaw', False): + if LooseVersion(self.version) == LooseVersion('7.3.1'): + # See issue: https://github.com/dceresoli/qe-gipaw/issues/19 + raise EasyBuildError("GIPAW will fail to compile in QE 7.3.1") + res = ['gipaw'] + self.check_bins += ['gipaw.x'] + return res + + def _add_d3q(self): + """Enable D3Q for Quantum ESPRESSO.""" + res = [] + if self.cfg.get('with_d3q', False): + if LooseVersion(self.version) > LooseVersion('7.0'): + # See issues: + # https://gitlab.com/QEF/q-e/-/issues/666 + # https://github.com/anharmonic/d3q/issues/13 + raise EasyBuildError("D3Q is not supported in QE 7.0+") + res = ['d3q'] + self.check_bins += [ + 'd3_asr3.x', 'd3_lw.x', 'd3_q2r.x', 'd3_qq2rr.x', 'd3q.x', 'd3_r2q.x', 'd3_recenter.x', + 'd3_sparse.x', 'd3_sqom.x', 'd3_tk.x' + ] + return res + + def _add_qmcpack(self): + """Enable QMCPACK for Quantum ESPRESSO.""" + res = [] + if self.cfg.get('with_qmcpack', False): + res = ['pw2qmcpack'] + self.check_bins += ['pw2qmcpack.x'] + return res + + def configure_step(self): + """Custom configuration procedure for Quantum ESPRESSO.""" + + if LooseVersion(self.version) < LooseVersion('7.3'): + raise EasyBuildError("EB QuantumEspresso cmake is implemented for versions >= 7.3") + + self._add_toolchains_opts() + self._add_libraries() + self._add_plugins() + + # Enable/configure test suite + nprocs = self.cfg.get('test_suite_nprocs', 1) + self.cfg.update('configopts', '-DQE_ENABLE_TEST=ON') + self.cfg.update('configopts', '-DTESTCODE_NPROCS=%d' % nprocs) + + # Change format of timings to seconds only (from d/h/m/s) + self.cfg.update('configopts', '-DQE_CLOCK_SECONDS=ON') + + super(EB_QuantumESPRESSOcmake, self).configure_step() + + def test_step(self): + """ + Test the compilation using Quantum ESPRESSO's test suite. + ctest -j NCONCURRENT (NCONCURRENT = max (1, PARALLEL / NPROCS)) + """ + + thr = self.cfg.get('test_suite_threshold', 0.97) + nprocs = self.cfg.get('test_suite_nprocs', 1) + concurrent = max(1, self.cfg.get('parallel', 1) // nprocs) + allow_fail = self.cfg.get('test_suite_allow_failures', []) + + cmd = ' '.join([ + 'ctest', + '-j%d' % concurrent, + '--output-on-failure', + ]) + + (out, _) = run_cmd(cmd, log_all=False, log_ok=False, simple=False, regexp=False) + + # Example output: + # 74% tests passed, 124 tests failed out of 481 + rgx = r'^ *(?P\d+)% tests passed, +(?P\d+) +tests failed out of +(?P\d+)' + mch = re.search(rgx, out, re.MULTILINE) + if not mch: + raise EasyBuildError("Failed to parse test suite output") + + perc = int(mch.group('perc')) / 100 + num_fail = int(mch.group('failed')) + total = int(mch.group('total')) + passed = total - num_fail + failures = [] # list of tests that failed, to be logged at the end + + + # Example output for reported failures: + # 635/635 Test #570: system--epw_wfpt-correctness ......................................***Failed 3.52 sec + self.log.debug("Test suite output:") + self.log.debug(out) + for line in out.splitlines(): + if '***Failed' in line: + for allowed in allow_fail: + if allowed in line: + self.log.info('Ignoring failure: %s' % line) + break + else: + failures.append(line) + self.log.warning(line) + + # Allow for flaky tests (eg too strict thresholds on results for structure relaxation) + num_fail = len(failures) + num_fail_thr = self.cfg.get('test_suite_max_failed', 0) + # perc = spass / max(stot, 1) + self.log.info("Total tests passed %d out of %d (%.2f%%)" % (passed, total, perc * 100)) + if perc < thr: + raise EasyBuildError( + "Test suite failed with less than %.2f %% (%.2f) success rate" % (thr * 100, perc * 100) + ) + if num_fail > num_fail_thr: + raise EasyBuildError( + "Test suite failed with more than %d failures %d" % (num_fail_thr, num_fail) + ) + + return out + + def sanity_check_step(self): + """Custom sanity check for Quantum ESPRESSO.""" + + targets = self.cfg['buildopts'].split() + + # Condition for all targets being build 'make' or 'make all_currents' + all_cond = len(targets) == 0 or 'all_currents' in targets + pwall_cond = 'pwall' in targets + + + # Standard binaries + if all_cond or 'cp' in targets : + self.check_bins += ["cp.x", "cppp.x", "manycp.x", "wfdd.x"] + + if all_cond or 'epw' in targets: + self.check_bins += ["epw.x"] + + if all_cond or 'gwl' in targets: + self.check_bins += [ + "abcoeff_to_eps.x", "bse_main.x", "graph.x", "gww_fit.x", "gww.x", "head.x", "memory_pw4gww.x", + "pw4gww.x", "simple_bse.x", "simple_ip.x", "simple.x" + ] + + if all_cond or 'hp' in targets: + self.check_bins += ["hp.x"] + + if all_cond or 'ld1' in targets: + self.check_bins += ["ld1.x"] + + if all_cond or pwall_cond or 'neb' in targets: + self.check_bins += ["neb.x", "path_interpolation.x"] + + if all_cond or pwall_cond or 'ph' in targets: + self.check_bins += [ + "alpha2f.x", "dynmat.x", "fd_ef.x", "fd.x", "lambda.x", "phcg.x", "postahc.x", "q2r.x", "dvscf_q2r.x", + "epa.x", "fd_ifc.x", "fqha.x", "matdyn.x", "ph.x", "q2qstar.x" + ] + + if all_cond or pwall_cond or 'pp' in targets: + self.check_bins += [ + "average.x", "dos_sp.x", "ef.x", "fermi_int_0.x", "fermi_proj.x", "fs.x", "molecularpdos.x", + "pawplot.x", "plotband.x", "plotrho.x", "ppacf.x", "pp.x", "pw2bgw.x", "pw2gt.x", "pw2wannier90.x", + "wannier_ham.x", "wfck2r.x", "bands.x", "dos.x", "epsilon.x", "fermi_int_1.x", "fermi_velocity.x", + "initial_state.x", "open_grid.x", "plan_avg.x", "plotproj.x", "pmw.x", "pprism.x", "projwfc.x", + "pw2critic.x", "pw2gw.x", "sumpdos.x", "wannier_plot.x" + ] + + if all_cond or pwall_cond or 'pw' in targets: + self.check_bins += [ + "cell2ibrav.x", "ev.x", "ibrav2cell.x", "kpoints.x", "pwi2xsf.x", "pw.x", "scan_ibrav.x" + ] + + if all_cond or pwall_cond or 'pwcond' in targets: + self.check_bins += ["pwcond.x"] + + if all_cond or 'tddfpt' in targets: + self.check_bins += ["turbo_davidson.x", "turbo_eels.x", "turbo_lanczos.x", "turbo_magnon.x", "turbo_spectrum.x"] + + if all_cond or 'upf' in targets: + self.check_bins += ["upfconv.x", "virtual_v2.x"] + + if all_cond or 'xspectra' in targets: + self.check_bins += ["molecularnexafs.x", "spectra_correction.x", "xspectra.x"] + + # Plugins binaries + if (all_cond and self.cfg.get('with_gipaw', False)) or 'gipaw' in targets: + self.check_bins += ["gipaw.x"] + + if (all_cond and self.cfg.get('d3q', False)) or 'd3q' in targets: + self.check_bins += [ + 'd3_asr3.x', 'd3_lw.x', 'd3_q2r.x', 'd3_qq2rr.x', 'd3q.x', 'd3_r2q.x', 'd3_recenter.x', + 'd3_sparse.x', 'd3_sqom.x', 'd3_tk.x' + ] + + if (all_cond and self.cfg.get('with_qmcpack', False)) or 'pw2qmcpack' in targets: + self.check_bins += ['pw2qmcpack.x'] + + self.log.debug("Checking for binaries: %s" % ', '.join(self.check_bins)) + + custom_paths = { + 'files': [os.path.join('bin', x) for x in self.check_bins], + 'dirs': [] + } + + super(EB_QuantumESPRESSOcmake, self).sanity_check_step(custom_paths=custom_paths) From 28bbe87685e94782bd3eb8e62e16b72263e34c95 Mon Sep 17 00:00:00 2001 From: crivella Date: Thu, 14 Mar 2024 15:46:13 +0100 Subject: [PATCH 02/18] Plugin binaries already added in the _add_XXX methods --- easybuild/easyblocks/q/quantumespressocmake.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/easybuild/easyblocks/q/quantumespressocmake.py b/easybuild/easyblocks/q/quantumespressocmake.py index 2209afed495..d78180be835 100644 --- a/easybuild/easyblocks/q/quantumespressocmake.py +++ b/easybuild/easyblocks/q/quantumespressocmake.py @@ -331,21 +331,6 @@ def sanity_check_step(self): if all_cond or 'xspectra' in targets: self.check_bins += ["molecularnexafs.x", "spectra_correction.x", "xspectra.x"] - # Plugins binaries - if (all_cond and self.cfg.get('with_gipaw', False)) or 'gipaw' in targets: - self.check_bins += ["gipaw.x"] - - if (all_cond and self.cfg.get('d3q', False)) or 'd3q' in targets: - self.check_bins += [ - 'd3_asr3.x', 'd3_lw.x', 'd3_q2r.x', 'd3_qq2rr.x', 'd3q.x', 'd3_r2q.x', 'd3_recenter.x', - 'd3_sparse.x', 'd3_sqom.x', 'd3_tk.x' - ] - - if (all_cond and self.cfg.get('with_qmcpack', False)) or 'pw2qmcpack' in targets: - self.check_bins += ['pw2qmcpack.x'] - - self.log.debug("Checking for binaries: %s" % ', '.join(self.check_bins)) - custom_paths = { 'files': [os.path.join('bin', x) for x in self.check_bins], 'dirs': [] From 2a814076b3be4ecc8a157c1ab4193c60ea5b331c Mon Sep 17 00:00:00 2001 From: crivella Date: Fri, 15 Mar 2024 14:13:22 +0100 Subject: [PATCH 03/18] Added fix for intel toolchain --- easybuild/easyblocks/q/quantumespressocmake.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/easybuild/easyblocks/q/quantumespressocmake.py b/easybuild/easyblocks/q/quantumespressocmake.py index d78180be835..265bce17ac4 100644 --- a/easybuild/easyblocks/q/quantumespressocmake.py +++ b/easybuild/easyblocks/q/quantumespressocmake.py @@ -30,6 +30,7 @@ import os import re +import easybuild.tools.toolchain as toolchain from easybuild.framework.easyconfig import CUSTOM from easybuild.tools import LooseVersion from easybuild.tools.build_log import EasyBuildError @@ -189,6 +190,8 @@ def configure_step(self): if LooseVersion(self.version) < LooseVersion('7.3'): raise EasyBuildError("EB QuantumEspresso cmake is implemented for versions >= 7.3") + + comp_fam = self.toolchain.comp_family() self._add_toolchains_opts() self._add_libraries() @@ -202,6 +205,13 @@ def configure_step(self): # Change format of timings to seconds only (from d/h/m/s) self.cfg.update('configopts', '-DQE_CLOCK_SECONDS=ON') + # Needed to avoid a DSO missing from command line linking error with the Intel toolchain + # https://gitlab.com/QEF/q-e/-/issues/667 + if self.cfg.get('build_shared_libs', False) and comp_fam == toolchain.INTELCOMP: + ldflags = os.environ.get('LDFLAGS', '') + ldflags += ' -Wl,--copy-dt-needed-entries ' + os.environ['LDFLAGS'] = ldflags + super(EB_QuantumESPRESSOcmake, self).configure_step() def test_step(self): @@ -323,7 +333,9 @@ def sanity_check_step(self): self.check_bins += ["pwcond.x"] if all_cond or 'tddfpt' in targets: - self.check_bins += ["turbo_davidson.x", "turbo_eels.x", "turbo_lanczos.x", "turbo_magnon.x", "turbo_spectrum.x"] + self.check_bins += [ + "turbo_davidson.x", "turbo_eels.x", "turbo_lanczos.x", "turbo_magnon.x", "turbo_spectrum.x" + ] if all_cond or 'upf' in targets: self.check_bins += ["upfconv.x", "virtual_v2.x"] From 1337fcc368326d9cfa4adc4505aa8330ca7e41eb Mon Sep 17 00:00:00 2001 From: crivella Date: Fri, 15 Mar 2024 15:39:40 +0100 Subject: [PATCH 04/18] pre-commit hooks --- .../easyblocks/q/quantumespressocmake.py | 96 +++++++++---------- 1 file changed, 48 insertions(+), 48 deletions(-) diff --git a/easybuild/easyblocks/q/quantumespressocmake.py b/easybuild/easyblocks/q/quantumespressocmake.py index 265bce17ac4..4e027b63a61 100644 --- a/easybuild/easyblocks/q/quantumespressocmake.py +++ b/easybuild/easyblocks/q/quantumespressocmake.py @@ -43,32 +43,32 @@ class EB_QuantumESPRESSOcmake(CMakeMake): """Support for building and installing Quantum ESPRESSO.""" - TEST_SUITE_DIR = "test-suite" + TEST_SUITE_DIR = 'test-suite' @staticmethod def extra_options(): """Custom easyconfig parameters for Quantum ESPRESSO.""" extra_vars = { - 'with_cuda': [False, "Enable CUDA support", CUSTOM], - 'with_scalapack': [True, "Enable ScaLAPACK support", CUSTOM], - 'with_fox': [False, "Enable FoX support", CUSTOM], - 'with_gipaw': [True, "Enable GIPAW support", CUSTOM], - 'with_d3q': [False, "Enable D3Q support", CUSTOM], - 'with_qmcpack': [False, "Enable QMCPACK support", CUSTOM], - 'test_suite_nprocs': [1, "Number of processors to use for the test suite", CUSTOM], + 'with_cuda': [False, 'Enable CUDA support', CUSTOM], + 'with_scalapack': [True, 'Enable ScaLAPACK support', CUSTOM], + 'with_fox': [False, 'Enable FoX support', CUSTOM], + 'with_gipaw': [True, 'Enable GIPAW support', CUSTOM], + 'with_d3q': [False, 'Enable D3Q support', CUSTOM], + 'with_qmcpack': [False, 'Enable QMCPACK support', CUSTOM], + 'test_suite_nprocs': [1, 'Number of processors to use for the test suite', CUSTOM], 'test_suite_allow_failures': [[ 'test_qe_xclib_', # 7.3.1: https://gitlab.com/QEF/q-e/-/issues/640 '--hp_', # 7.3.1: Broken testsuite (https://gitlab.com/QEF/q-e/-/issues/665) '--ph_', # 7.3.1: Broken testsuite (https://gitlab.com/QEF/q-e/-/issues/665) '--epw_', # 7.3.1: Broken testsuite (https://gitlab.com/QEF/q-e/-/issues/665) '--tddfpt_', # 7.3.1: Broken testsuite (https://gitlab.com/QEF/q-e/-/issues/665) - ], "List of test suite targets that are allowed to fail (name can partially match)", CUSTOM], + ], 'List of test suite targets that are allowed to fail (name can partially match)', CUSTOM], 'test_suite_threshold': [ 0.97, - "Threshold for test suite success rate (does count also allowed failures)", + 'Threshold for test suite success rate (does count also allowed failures)', CUSTOM ], - 'test_suite_max_failed': [0, "Maximum number of failing tests (does not count allowed failures)", CUSTOM], + 'test_suite_max_failed': [0, 'Maximum number of failing tests (does not count allowed failures)', CUSTOM], } return CMakeMake.extra_options(extra_vars) @@ -76,7 +76,7 @@ def __init__(self, *args, **kwargs): """Add extra config options specific to Quantum ESPRESSO.""" super(EB_QuantumESPRESSOcmake, self).__init__(*args, **kwargs) - self.install_subdir = "qe-%s" % self.version + self.install_subdir = 'qe-%s' % self.version self.check_bins = [] @@ -123,7 +123,7 @@ def _add_scalapack(self): """Enable ScaLAPACK for Quantum ESPRESSO.""" if self.cfg.get('with_scalapack', False): if not self.toolchain.options.get('usempi', False): - raise EasyBuildError("ScaLAPACK support requires MPI") + raise EasyBuildError('ScaLAPACK support requires MPI') self.cfg.update('configopts', '-DQE_ENABLE_SCALAPACK=ON') def _add_fox(self): @@ -133,21 +133,21 @@ def _add_fox(self): def _add_hdf5(self): """Enable HDF5 for Quantum ESPRESSO.""" - if get_software_root("HDF5"): + if get_software_root('HDF5'): self.cfg.update('configopts', '-DQE_ENABLE_HDF5=ON') def _add_libxc(self): """Enable LibXC for Quantum ESPRESSO.""" - if get_software_root("libxc"): + if get_software_root('libxc'): self.cfg.update('configopts', '-DQE_ENABLE_LIBXC=ON') def _add_elpa(self): """Enable ELPA for Quantum ESPRESSO.""" - if get_software_root("ELPA"): + if get_software_root('ELPA'): if not self.cfg.get('with_scalapack', False): - raise EasyBuildError("ELPA support requires ScaLAPACK") + raise EasyBuildError('ELPA support requires ScaLAPACK') if LooseVersion(self.version) == LooseVersion('7.3') and self.toolchain.options.get('openmp', False): - raise EasyBuildError("QE 7.3 with cmake does not support ELPA with OpenMP") + raise EasyBuildError('QE 7.3 with cmake does not support ELPA with OpenMP') self.cfg.update('configopts', '-DQE_ENABLE_ELPA=ON') def _add_gipaw(self): @@ -156,7 +156,7 @@ def _add_gipaw(self): if self.cfg.get('with_gipaw', False): if LooseVersion(self.version) == LooseVersion('7.3.1'): # See issue: https://github.com/dceresoli/qe-gipaw/issues/19 - raise EasyBuildError("GIPAW will fail to compile in QE 7.3.1") + raise EasyBuildError('GIPAW will fail to compile in QE 7.3.1') res = ['gipaw'] self.check_bins += ['gipaw.x'] return res @@ -169,7 +169,7 @@ def _add_d3q(self): # See issues: # https://gitlab.com/QEF/q-e/-/issues/666 # https://github.com/anharmonic/d3q/issues/13 - raise EasyBuildError("D3Q is not supported in QE 7.0+") + raise EasyBuildError('D3Q is not supported in QE 7.0+') res = ['d3q'] self.check_bins += [ 'd3_asr3.x', 'd3_lw.x', 'd3_q2r.x', 'd3_qq2rr.x', 'd3q.x', 'd3_r2q.x', 'd3_recenter.x', @@ -189,8 +189,8 @@ def configure_step(self): """Custom configuration procedure for Quantum ESPRESSO.""" if LooseVersion(self.version) < LooseVersion('7.3'): - raise EasyBuildError("EB QuantumEspresso cmake is implemented for versions >= 7.3") - + raise EasyBuildError('EB QuantumEspresso cmake is implemented for versions >= 7.3') + comp_fam = self.toolchain.comp_family() self._add_toolchains_opts() @@ -211,7 +211,7 @@ def configure_step(self): ldflags = os.environ.get('LDFLAGS', '') ldflags += ' -Wl,--copy-dt-needed-entries ' os.environ['LDFLAGS'] = ldflags - + super(EB_QuantumESPRESSOcmake, self).configure_step() def test_step(self): @@ -238,8 +238,8 @@ def test_step(self): rgx = r'^ *(?P\d+)% tests passed, +(?P\d+) +tests failed out of +(?P\d+)' mch = re.search(rgx, out, re.MULTILINE) if not mch: - raise EasyBuildError("Failed to parse test suite output") - + raise EasyBuildError('Failed to parse test suite output') + perc = int(mch.group('perc')) / 100 num_fail = int(mch.group('failed')) total = int(mch.group('total')) @@ -249,7 +249,7 @@ def test_step(self): # Example output for reported failures: # 635/635 Test #570: system--epw_wfpt-correctness ......................................***Failed 3.52 sec - self.log.debug("Test suite output:") + self.log.debug('Test suite output:') self.log.debug(out) for line in out.splitlines(): if '***Failed' in line: @@ -265,14 +265,14 @@ def test_step(self): num_fail = len(failures) num_fail_thr = self.cfg.get('test_suite_max_failed', 0) # perc = spass / max(stot, 1) - self.log.info("Total tests passed %d out of %d (%.2f%%)" % (passed, total, perc * 100)) + self.log.info('Total tests passed %d out of %d (%.2f%%)' % (passed, total, perc * 100)) if perc < thr: raise EasyBuildError( - "Test suite failed with less than %.2f %% (%.2f) success rate" % (thr * 100, perc * 100) + 'Test suite failed with less than %.2f %% (%.2f) success rate' % (thr * 100, perc * 100) ) if num_fail > num_fail_thr: raise EasyBuildError( - "Test suite failed with more than %d failures %d" % (num_fail_thr, num_fail) + 'Test suite failed with more than %d failures %d' % (num_fail_thr, num_fail) ) return out @@ -289,59 +289,59 @@ def sanity_check_step(self): # Standard binaries if all_cond or 'cp' in targets : - self.check_bins += ["cp.x", "cppp.x", "manycp.x", "wfdd.x"] + self.check_bins += ['cp.x', 'cppp.x', 'manycp.x', 'wfdd.x'] if all_cond or 'epw' in targets: - self.check_bins += ["epw.x"] + self.check_bins += ['epw.x'] if all_cond or 'gwl' in targets: self.check_bins += [ - "abcoeff_to_eps.x", "bse_main.x", "graph.x", "gww_fit.x", "gww.x", "head.x", "memory_pw4gww.x", - "pw4gww.x", "simple_bse.x", "simple_ip.x", "simple.x" + 'abcoeff_to_eps.x', 'bse_main.x', 'graph.x', 'gww_fit.x', 'gww.x', 'head.x', 'memory_pw4gww.x', + 'pw4gww.x', 'simple_bse.x', 'simple_ip.x', 'simple.x' ] if all_cond or 'hp' in targets: - self.check_bins += ["hp.x"] + self.check_bins += ['hp.x'] if all_cond or 'ld1' in targets: - self.check_bins += ["ld1.x"] + self.check_bins += ['ld1.x'] if all_cond or pwall_cond or 'neb' in targets: - self.check_bins += ["neb.x", "path_interpolation.x"] + self.check_bins += ['neb.x', 'path_interpolation.x'] if all_cond or pwall_cond or 'ph' in targets: self.check_bins += [ - "alpha2f.x", "dynmat.x", "fd_ef.x", "fd.x", "lambda.x", "phcg.x", "postahc.x", "q2r.x", "dvscf_q2r.x", - "epa.x", "fd_ifc.x", "fqha.x", "matdyn.x", "ph.x", "q2qstar.x" + 'alpha2f.x', 'dynmat.x', 'fd_ef.x', 'fd.x', 'lambda.x', 'phcg.x', 'postahc.x', 'q2r.x', 'dvscf_q2r.x', + 'epa.x', 'fd_ifc.x', 'fqha.x', 'matdyn.x', 'ph.x', 'q2qstar.x' ] if all_cond or pwall_cond or 'pp' in targets: self.check_bins += [ - "average.x", "dos_sp.x", "ef.x", "fermi_int_0.x", "fermi_proj.x", "fs.x", "molecularpdos.x", - "pawplot.x", "plotband.x", "plotrho.x", "ppacf.x", "pp.x", "pw2bgw.x", "pw2gt.x", "pw2wannier90.x", - "wannier_ham.x", "wfck2r.x", "bands.x", "dos.x", "epsilon.x", "fermi_int_1.x", "fermi_velocity.x", - "initial_state.x", "open_grid.x", "plan_avg.x", "plotproj.x", "pmw.x", "pprism.x", "projwfc.x", - "pw2critic.x", "pw2gw.x", "sumpdos.x", "wannier_plot.x" + 'average.x', 'dos_sp.x', 'ef.x', 'fermi_int_0.x', 'fermi_proj.x', 'fs.x', 'molecularpdos.x', + 'pawplot.x', 'plotband.x', 'plotrho.x', 'ppacf.x', 'pp.x', 'pw2bgw.x', 'pw2gt.x', 'pw2wannier90.x', + 'wannier_ham.x', 'wfck2r.x', 'bands.x', 'dos.x', 'epsilon.x', 'fermi_int_1.x', 'fermi_velocity.x', + 'initial_state.x', 'open_grid.x', 'plan_avg.x', 'plotproj.x', 'pmw.x', 'pprism.x', 'projwfc.x', + 'pw2critic.x', 'pw2gw.x', 'sumpdos.x', 'wannier_plot.x' ] if all_cond or pwall_cond or 'pw' in targets: self.check_bins += [ - "cell2ibrav.x", "ev.x", "ibrav2cell.x", "kpoints.x", "pwi2xsf.x", "pw.x", "scan_ibrav.x" + 'cell2ibrav.x', 'ev.x', 'ibrav2cell.x', 'kpoints.x', 'pwi2xsf.x', 'pw.x', 'scan_ibrav.x' ] if all_cond or pwall_cond or 'pwcond' in targets: - self.check_bins += ["pwcond.x"] + self.check_bins += ['pwcond.x'] if all_cond or 'tddfpt' in targets: self.check_bins += [ - "turbo_davidson.x", "turbo_eels.x", "turbo_lanczos.x", "turbo_magnon.x", "turbo_spectrum.x" + 'turbo_davidson.x', 'turbo_eels.x', 'turbo_lanczos.x', 'turbo_magnon.x', 'turbo_spectrum.x' ] if all_cond or 'upf' in targets: - self.check_bins += ["upfconv.x", "virtual_v2.x"] + self.check_bins += ['upfconv.x', 'virtual_v2.x'] if all_cond or 'xspectra' in targets: - self.check_bins += ["molecularnexafs.x", "spectra_correction.x", "xspectra.x"] + self.check_bins += ['molecularnexafs.x', 'spectra_correction.x', 'xspectra.x'] custom_paths = { 'files': [os.path.join('bin', x) for x in self.check_bins], From 4621fb738570e90db26d768fb95429376b9e3c85 Mon Sep 17 00:00:00 2001 From: crivella Date: Mon, 18 Mar 2024 15:38:06 +0100 Subject: [PATCH 05/18] Linting + moved ignored tests to config files --- .../easyblocks/q/quantumespressocmake.py | 28 ++++++++----------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/easybuild/easyblocks/q/quantumespressocmake.py b/easybuild/easyblocks/q/quantumespressocmake.py index 4e027b63a61..dce75d3b506 100644 --- a/easybuild/easyblocks/q/quantumespressocmake.py +++ b/easybuild/easyblocks/q/quantumespressocmake.py @@ -30,7 +30,6 @@ import os import re -import easybuild.tools.toolchain as toolchain from easybuild.framework.easyconfig import CUSTOM from easybuild.tools import LooseVersion from easybuild.tools.build_log import EasyBuildError @@ -56,13 +55,11 @@ def extra_options(): 'with_d3q': [False, 'Enable D3Q support', CUSTOM], 'with_qmcpack': [False, 'Enable QMCPACK support', CUSTOM], 'test_suite_nprocs': [1, 'Number of processors to use for the test suite', CUSTOM], - 'test_suite_allow_failures': [[ - 'test_qe_xclib_', # 7.3.1: https://gitlab.com/QEF/q-e/-/issues/640 - '--hp_', # 7.3.1: Broken testsuite (https://gitlab.com/QEF/q-e/-/issues/665) - '--ph_', # 7.3.1: Broken testsuite (https://gitlab.com/QEF/q-e/-/issues/665) - '--epw_', # 7.3.1: Broken testsuite (https://gitlab.com/QEF/q-e/-/issues/665) - '--tddfpt_', # 7.3.1: Broken testsuite (https://gitlab.com/QEF/q-e/-/issues/665) - ], 'List of test suite targets that are allowed to fail (name can partially match)', CUSTOM], + 'test_suite_allow_failures': [ + [], + 'List of test suite targets that are allowed to fail (name can partially match)', + CUSTOM + ], 'test_suite_threshold': [ 0.97, 'Threshold for test suite success rate (does count also allowed failures)', @@ -165,15 +162,16 @@ def _add_d3q(self): """Enable D3Q for Quantum ESPRESSO.""" res = [] if self.cfg.get('with_d3q', False): - if LooseVersion(self.version) > LooseVersion('7.0'): + if LooseVersion(self.version) <= LooseVersion('7.3.1'): # See issues: # https://gitlab.com/QEF/q-e/-/issues/666 # https://github.com/anharmonic/d3q/issues/13 - raise EasyBuildError('D3Q is not supported in QE 7.0+') + raise EasyBuildError('D3Q compilation will fail for QE 7.3 and 7.3.1') res = ['d3q'] self.check_bins += [ - 'd3_asr3.x', 'd3_lw.x', 'd3_q2r.x', 'd3_qq2rr.x', 'd3q.x', 'd3_r2q.x', 'd3_recenter.x', - 'd3_sparse.x', 'd3_sqom.x', 'd3_tk.x' + 'd3_asr3.x', 'd3_db.x', 'd3_import_shengbte.x', 'd3_interpolate2.x', 'd3_lw.x', 'd3_q2r.x', + 'd3_qha.x','d3_qq2rr.x', 'd3q.x', 'd3_r2q.x', 'd3_recenter.x', 'd3_rmzeu.x', 'd3_sparse.x', + 'd3_sqom.x', 'd3_tk.x', ] return res @@ -191,8 +189,6 @@ def configure_step(self): if LooseVersion(self.version) < LooseVersion('7.3'): raise EasyBuildError('EB QuantumEspresso cmake is implemented for versions >= 7.3') - comp_fam = self.toolchain.comp_family() - self._add_toolchains_opts() self._add_libraries() self._add_plugins() @@ -205,9 +201,9 @@ def configure_step(self): # Change format of timings to seconds only (from d/h/m/s) self.cfg.update('configopts', '-DQE_CLOCK_SECONDS=ON') - # Needed to avoid a DSO missing from command line linking error with the Intel toolchain + # Needed to avoid a DSO missing from command line linking error # https://gitlab.com/QEF/q-e/-/issues/667 - if self.cfg.get('build_shared_libs', False) and comp_fam == toolchain.INTELCOMP: + if self.cfg.get('build_shared_libs', False) : ldflags = os.environ.get('LDFLAGS', '') ldflags += ' -Wl,--copy-dt-needed-entries ' os.environ['LDFLAGS'] = ldflags From ee3695fafd3273327d43a535afb8ab04f6da5918 Mon Sep 17 00:00:00 2001 From: crivella Date: Mon, 18 Mar 2024 15:45:30 +0100 Subject: [PATCH 06/18] Reuse message --- .../easyblocks/q/quantumespressocmake.py | 117 +++++++++--------- 1 file changed, 57 insertions(+), 60 deletions(-) diff --git a/easybuild/easyblocks/q/quantumespressocmake.py b/easybuild/easyblocks/q/quantumespressocmake.py index dce75d3b506..b9f003b37ab 100644 --- a/easybuild/easyblocks/q/quantumespressocmake.py +++ b/easybuild/easyblocks/q/quantumespressocmake.py @@ -79,115 +79,112 @@ def __init__(self, *args, **kwargs): def _add_toolchains_opts(self): """Enable toolchain options for Quantum ESPRESSO.""" - self._add_mpi() - self._add_openmp() - self._add_cuda() + if self.toolchain.options.get('usempi', False): + self._add_mpi() + if self.toolchain.options.get('openmp', False): + self._add_openmp() + if self.cfg.get('with_cuda', False): + self._add_cuda() def _add_libraries(self): """Enable external libraries for Quantum ESPRESSO.""" - self._add_scalapack() - self._add_fox() - self._add_libxc() - self._add_elpa() - self._add_hdf5() + if self.cfg.get('with_scalapack', False): + self._add_scalapack() + if self.cfg.get('with_fox', False): + self._add_fox() + if get_software_root('libxc'): + self._add_libxc() + if get_software_root('ELPA'): + self._add_elpa() + if get_software_root('HDF5'): + self._add_hdf5() def _add_plugins(self): """Enable plugins for Quantum ESPRESSO.""" plugins = [] - plugins += self._add_gipaw() - plugins += self._add_d3q() - plugins += self._add_qmcpack() + if self.cfg.get('with_gipaw', False): + plugins += self._add_gipaw() + if self.cfg.get('with_d3q', False): + plugins += self._add_d3q() + if self.cfg.get('with_qmcpack', False): + plugins += self._add_qmcpack() if plugins: self.cfg.update('configopts', '-DQE_ENABLE_PLUGINS="%s"' % ';'.join(plugins)) def _add_mpi(self): """Enable MPI for Quantum ESPRESSO.""" - if self.toolchain.options.get('usempi', False): - self.cfg.update('configopts', '-DENABLE_MPI=ON') + self.cfg.update('configopts', '-DENABLE_MPI=ON') def _add_openmp(self): """Enable OpenMP for Quantum ESPRESSO.""" - if self.toolchain.options.get('openmp', False): - self.cfg.update('configopts', '-DENABLE_OPENMP=ON') + self.cfg.update('configopts', '-DENABLE_OPENMP=ON') def _add_cuda(self): """Enable CUDA for Quantum ESPRESSO.""" - if self.cfg.get('with_cuda', False): - self.cfg.update('configopts', '-DQE_ENABLE_CUDA=ON') - self.cfg.update('configopts', '-DQE_ENABLE_OPENACC=ON') + self.cfg.update('configopts', '-DQE_ENABLE_CUDA=ON') + self.cfg.update('configopts', '-DQE_ENABLE_OPENACC=ON') def _add_scalapack(self): """Enable ScaLAPACK for Quantum ESPRESSO.""" - if self.cfg.get('with_scalapack', False): - if not self.toolchain.options.get('usempi', False): - raise EasyBuildError('ScaLAPACK support requires MPI') - self.cfg.update('configopts', '-DQE_ENABLE_SCALAPACK=ON') + if not self.toolchain.options.get('usempi', False): + raise EasyBuildError('ScaLAPACK support requires MPI') + self.cfg.update('configopts', '-DQE_ENABLE_SCALAPACK=ON') def _add_fox(self): """Enable FoX for Quantum ESPRESSO.""" - if self.cfg.get('with_fox', False): - self.cfg.update('configopts', '-DQE_ENABLE_FOX=ON') + self.cfg.update('configopts', '-DQE_ENABLE_FOX=ON') def _add_hdf5(self): """Enable HDF5 for Quantum ESPRESSO.""" - if get_software_root('HDF5'): - self.cfg.update('configopts', '-DQE_ENABLE_HDF5=ON') + self.cfg.update('configopts', '-DQE_ENABLE_HDF5=ON') def _add_libxc(self): """Enable LibXC for Quantum ESPRESSO.""" - if get_software_root('libxc'): - self.cfg.update('configopts', '-DQE_ENABLE_LIBXC=ON') + self.cfg.update('configopts', '-DQE_ENABLE_LIBXC=ON') def _add_elpa(self): """Enable ELPA for Quantum ESPRESSO.""" - if get_software_root('ELPA'): - if not self.cfg.get('with_scalapack', False): - raise EasyBuildError('ELPA support requires ScaLAPACK') - if LooseVersion(self.version) == LooseVersion('7.3') and self.toolchain.options.get('openmp', False): - raise EasyBuildError('QE 7.3 with cmake does not support ELPA with OpenMP') - self.cfg.update('configopts', '-DQE_ENABLE_ELPA=ON') + if not self.cfg.get('with_scalapack', False): + raise EasyBuildError('ELPA support requires ScaLAPACK') + if LooseVersion(self.version) == LooseVersion('7.3') and self.toolchain.options.get('openmp', False): + raise EasyBuildError('QE 7.3 with cmake does not support ELPA with OpenMP') + self.cfg.update('configopts', '-DQE_ENABLE_ELPA=ON') def _add_gipaw(self): """Enable GIPAW for Quantum ESPRESSO.""" - res = [] - if self.cfg.get('with_gipaw', False): - if LooseVersion(self.version) == LooseVersion('7.3.1'): - # See issue: https://github.com/dceresoli/qe-gipaw/issues/19 - raise EasyBuildError('GIPAW will fail to compile in QE 7.3.1') - res = ['gipaw'] - self.check_bins += ['gipaw.x'] + if LooseVersion(self.version) == LooseVersion('7.3.1'): + # See issue: https://github.com/dceresoli/qe-gipaw/issues/19 + raise EasyBuildError('GIPAW will fail to compile in QE 7.3.1') + res = ['gipaw'] + self.check_bins += ['gipaw.x'] return res def _add_d3q(self): """Enable D3Q for Quantum ESPRESSO.""" - res = [] - if self.cfg.get('with_d3q', False): - if LooseVersion(self.version) <= LooseVersion('7.3.1'): - # See issues: - # https://gitlab.com/QEF/q-e/-/issues/666 - # https://github.com/anharmonic/d3q/issues/13 - raise EasyBuildError('D3Q compilation will fail for QE 7.3 and 7.3.1') - res = ['d3q'] - self.check_bins += [ - 'd3_asr3.x', 'd3_db.x', 'd3_import_shengbte.x', 'd3_interpolate2.x', 'd3_lw.x', 'd3_q2r.x', - 'd3_qha.x','d3_qq2rr.x', 'd3q.x', 'd3_r2q.x', 'd3_recenter.x', 'd3_rmzeu.x', 'd3_sparse.x', - 'd3_sqom.x', 'd3_tk.x', - ] + if LooseVersion(self.version) <= LooseVersion('7.3.1'): + # See issues: + # https://gitlab.com/QEF/q-e/-/issues/666 + # https://github.com/anharmonic/d3q/issues/13 + raise EasyBuildError('D3Q compilation will fail for QE 7.3 and 7.3.1') + res = ['d3q'] + self.check_bins += [ + 'd3_asr3.x', 'd3_db.x', 'd3_import_shengbte.x', 'd3_interpolate2.x', 'd3_lw.x', 'd3_q2r.x', + 'd3_qha.x','d3_qq2rr.x', 'd3q.x', 'd3_r2q.x', 'd3_recenter.x', 'd3_rmzeu.x', 'd3_sparse.x', + 'd3_sqom.x', 'd3_tk.x', + ] return res def _add_qmcpack(self): """Enable QMCPACK for Quantum ESPRESSO.""" - res = [] - if self.cfg.get('with_qmcpack', False): - res = ['pw2qmcpack'] - self.check_bins += ['pw2qmcpack.x'] + res = ['pw2qmcpack'] + self.check_bins += ['pw2qmcpack.x'] return res def configure_step(self): """Custom configuration procedure for Quantum ESPRESSO.""" if LooseVersion(self.version) < LooseVersion('7.3'): - raise EasyBuildError('EB QuantumEspresso cmake is implemented for versions >= 7.3') + raise EasyBuildError('EB QuantumEspresso with cmake is implemented for versions >= 7.3') self._add_toolchains_opts() self._add_libraries() @@ -268,7 +265,7 @@ def test_step(self): ) if num_fail > num_fail_thr: raise EasyBuildError( - 'Test suite failed with more than %d failures %d' % (num_fail_thr, num_fail) + 'Test suite failed with %d failures (%d failures permitted)' % (num_fail, num_fail_thr) ) return out From f4b87a696f038dbaf4ad596a149b10d9be906381 Mon Sep 17 00:00:00 2001 From: crivella Date: Mon, 18 Mar 2024 15:56:59 +0100 Subject: [PATCH 07/18] Linting --- easybuild/easyblocks/q/quantumespressocmake.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/easybuild/easyblocks/q/quantumespressocmake.py b/easybuild/easyblocks/q/quantumespressocmake.py index b9f003b37ab..bb4fd95949f 100644 --- a/easybuild/easyblocks/q/quantumespressocmake.py +++ b/easybuild/easyblocks/q/quantumespressocmake.py @@ -169,7 +169,7 @@ def _add_d3q(self): res = ['d3q'] self.check_bins += [ 'd3_asr3.x', 'd3_db.x', 'd3_import_shengbte.x', 'd3_interpolate2.x', 'd3_lw.x', 'd3_q2r.x', - 'd3_qha.x','d3_qq2rr.x', 'd3q.x', 'd3_r2q.x', 'd3_recenter.x', 'd3_rmzeu.x', 'd3_sparse.x', + 'd3_qha.x', 'd3_qq2rr.x', 'd3q.x', 'd3_r2q.x', 'd3_recenter.x', 'd3_rmzeu.x', 'd3_sparse.x', 'd3_sqom.x', 'd3_tk.x', ] return res @@ -200,7 +200,7 @@ def configure_step(self): # Needed to avoid a DSO missing from command line linking error # https://gitlab.com/QEF/q-e/-/issues/667 - if self.cfg.get('build_shared_libs', False) : + if self.cfg.get('build_shared_libs', False): ldflags = os.environ.get('LDFLAGS', '') ldflags += ' -Wl,--copy-dt-needed-entries ' os.environ['LDFLAGS'] = ldflags @@ -239,7 +239,6 @@ def test_step(self): passed = total - num_fail failures = [] # list of tests that failed, to be logged at the end - # Example output for reported failures: # 635/635 Test #570: system--epw_wfpt-correctness ......................................***Failed 3.52 sec self.log.debug('Test suite output:') @@ -279,9 +278,8 @@ def sanity_check_step(self): all_cond = len(targets) == 0 or 'all_currents' in targets pwall_cond = 'pwall' in targets - # Standard binaries - if all_cond or 'cp' in targets : + if all_cond or 'cp' in targets: self.check_bins += ['cp.x', 'cppp.x', 'manycp.x', 'wfdd.x'] if all_cond or 'epw' in targets: From ac204f4f1cd4f080ccca811666b717bfa0bbc989 Mon Sep 17 00:00:00 2001 From: crivella Date: Thu, 21 Mar 2024 12:49:38 +0100 Subject: [PATCH 08/18] Using `env.setvar` instead of `os.environ` --- easybuild/easyblocks/q/quantumespressocmake.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/q/quantumespressocmake.py b/easybuild/easyblocks/q/quantumespressocmake.py index bb4fd95949f..fa6977141ec 100644 --- a/easybuild/easyblocks/q/quantumespressocmake.py +++ b/easybuild/easyblocks/q/quantumespressocmake.py @@ -30,6 +30,7 @@ import os import re +import easybuild.tools.environment as env from easybuild.framework.easyconfig import CUSTOM from easybuild.tools import LooseVersion from easybuild.tools.build_log import EasyBuildError @@ -201,9 +202,9 @@ def configure_step(self): # Needed to avoid a DSO missing from command line linking error # https://gitlab.com/QEF/q-e/-/issues/667 if self.cfg.get('build_shared_libs', False): - ldflags = os.environ.get('LDFLAGS', '') + ldflags = os.getenv('LDFLAGS', '') ldflags += ' -Wl,--copy-dt-needed-entries ' - os.environ['LDFLAGS'] = ldflags + env.setvar('LDFLAGS', ldflags) super(EB_QuantumESPRESSOcmake, self).configure_step() From b3759f5458f20959afb0d6d48bc4995462c52589 Mon Sep 17 00:00:00 2001 From: crivella Date: Fri, 22 Mar 2024 15:50:33 +0100 Subject: [PATCH 09/18] Added possibility to install with pre-downloaded submodules --- .../easyblocks/q/quantumespressocmake.py | 47 +++++++++++++++++-- 1 file changed, 44 insertions(+), 3 deletions(-) diff --git a/easybuild/easyblocks/q/quantumespressocmake.py b/easybuild/easyblocks/q/quantumespressocmake.py index fa6977141ec..2e845ded3d5 100644 --- a/easybuild/easyblocks/q/quantumespressocmake.py +++ b/easybuild/easyblocks/q/quantumespressocmake.py @@ -29,6 +29,7 @@ """ import os import re +import shutil import easybuild.tools.environment as env from easybuild.framework.easyconfig import CUSTOM @@ -44,6 +45,16 @@ class EB_QuantumESPRESSOcmake(CMakeMake): """Support for building and installing Quantum ESPRESSO.""" TEST_SUITE_DIR = 'test-suite' + SUBMODULES = [ + 'lapack', + 'mbd', + 'devxlib', + 'fox', + 'd3q', + 'qe-gipaw', + 'pw2qmcpack', + 'wannier90' + ] @staticmethod def extra_options(): @@ -166,7 +177,10 @@ def _add_d3q(self): # See issues: # https://gitlab.com/QEF/q-e/-/issues/666 # https://github.com/anharmonic/d3q/issues/13 - raise EasyBuildError('D3Q compilation will fail for QE 7.3 and 7.3.1') + if not os.path.exists(os.path.join(self.builddir, self.install_subdir, 'external', 'd3q', '.git')): + raise EasyBuildError( + 'D3Q compilation will fail for QE 7.3 and 7.3.1 without submodule downloaded from easyconfig.' + ) res = ['d3q'] self.check_bins += [ 'd3_asr3.x', 'd3_db.x', 'd3_import_shengbte.x', 'd3_interpolate2.x', 'd3_lw.x', 'd3_q2r.x', @@ -181,12 +195,36 @@ def _add_qmcpack(self): self.check_bins += ['pw2qmcpack.x'] return res + def _copy_submodule_dirs(self): + """Copy submodule dirs downloaded by EB into XXX/external""" + for submod in self.SUBMODULES: + src = os.path.join(self.builddir, submod) + dst = os.path.join(self.builddir, self.install_subdir, 'external', submod) + if os.path.exists(src): + self.log.info('Copying submodule %s into %s' % (submod, dst)) + # Remove empty directories and replace them with the downloaded submodule + if os.path.exists(dst): + shutil.rmtree(dst) + shutil.move(src, dst) + else: + self.log.warning('Submodule %s not found at %s' % (submod, src)) + + # Trick QE to think that the submodule is already installed in case `keep_git_dir` is not used in + # the easyconfig file + gitf = os.path.join(dst, '.git') + if not os.path.exists(gitf): + os.mkdir(gitf) + def configure_step(self): """Custom configuration procedure for Quantum ESPRESSO.""" if LooseVersion(self.version) < LooseVersion('7.3'): raise EasyBuildError('EB QuantumEspresso with cmake is implemented for versions >= 7.3') + # Needs to be before other functions that could check existance of .git for submodules to + # make compatibility checks + self._copy_submodule_dirs() + self._add_toolchains_opts() self._add_libraries() self._add_plugins() @@ -257,15 +295,18 @@ def test_step(self): # Allow for flaky tests (eg too strict thresholds on results for structure relaxation) num_fail = len(failures) num_fail_thr = self.cfg.get('test_suite_max_failed', 0) - # perc = spass / max(stot, 1) self.log.info('Total tests passed %d out of %d (%.2f%%)' % (passed, total, perc * 100)) + if failures: + self.log.warning('The following tests failed (and are not ignored):') + for failure in failures: + self.log.warning('| ' + failure) if perc < thr: raise EasyBuildError( 'Test suite failed with less than %.2f %% (%.2f) success rate' % (thr * 100, perc * 100) ) if num_fail > num_fail_thr: raise EasyBuildError( - 'Test suite failed with %d failures (%d failures permitted)' % (num_fail, num_fail_thr) + 'Test suite failed with %d non-ignored failures (%d failures permitted)' % (num_fail, num_fail_thr) ) return out From a91fabe30b722eb705d820ee3442bc2cfa3cfffb Mon Sep 17 00:00:00 2001 From: crivella Date: Fri, 22 Mar 2024 18:35:23 +0100 Subject: [PATCH 10/18] Fixed flags --- easybuild/easyblocks/q/quantumespressocmake.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/q/quantumespressocmake.py b/easybuild/easyblocks/q/quantumespressocmake.py index 2e845ded3d5..c3899aaaf83 100644 --- a/easybuild/easyblocks/q/quantumespressocmake.py +++ b/easybuild/easyblocks/q/quantumespressocmake.py @@ -125,11 +125,11 @@ def _add_plugins(self): def _add_mpi(self): """Enable MPI for Quantum ESPRESSO.""" - self.cfg.update('configopts', '-DENABLE_MPI=ON') + self.cfg.update('configopts', '-DQE_ENABLE_MPI=ON') def _add_openmp(self): """Enable OpenMP for Quantum ESPRESSO.""" - self.cfg.update('configopts', '-DENABLE_OPENMP=ON') + self.cfg.update('configopts', '-DQE_ENABLE_OPENMP=ON') def _add_cuda(self): """Enable CUDA for Quantum ESPRESSO.""" From 7433a5e0abd5edacea855f577a8c4a3948d15971 Mon Sep 17 00:00:00 2001 From: crivella Date: Mon, 25 Mar 2024 11:24:05 +0100 Subject: [PATCH 11/18] Ensure that options can also be disabled (plus error in compilation of d3q without mpi) --- .../easyblocks/q/quantumespressocmake.py | 81 ++++++++++++------- 1 file changed, 50 insertions(+), 31 deletions(-) diff --git a/easybuild/easyblocks/q/quantumespressocmake.py b/easybuild/easyblocks/q/quantumespressocmake.py index c3899aaaf83..14c606572db 100644 --- a/easybuild/easyblocks/q/quantumespressocmake.py +++ b/easybuild/easyblocks/q/quantumespressocmake.py @@ -91,25 +91,17 @@ def __init__(self, *args, **kwargs): def _add_toolchains_opts(self): """Enable toolchain options for Quantum ESPRESSO.""" - if self.toolchain.options.get('usempi', False): - self._add_mpi() - if self.toolchain.options.get('openmp', False): - self._add_openmp() - if self.cfg.get('with_cuda', False): - self._add_cuda() + self._add_mpi() + self._add_openmp() + self._add_cuda() def _add_libraries(self): """Enable external libraries for Quantum ESPRESSO.""" - if self.cfg.get('with_scalapack', False): - self._add_scalapack() - if self.cfg.get('with_fox', False): - self._add_fox() - if get_software_root('libxc'): - self._add_libxc() - if get_software_root('ELPA'): - self._add_elpa() - if get_software_root('HDF5'): - self._add_hdf5() + self._add_scalapack() + self._add_fox() + self._add_hdf5() + self._add_libxc() + self._add_elpa() def _add_plugins(self): """Enable plugins for Quantum ESPRESSO.""" @@ -125,42 +117,67 @@ def _add_plugins(self): def _add_mpi(self): """Enable MPI for Quantum ESPRESSO.""" - self.cfg.update('configopts', '-DQE_ENABLE_MPI=ON') + if self.toolchain.options.get('usempi', False): + self.cfg.update('configopts', '-DQE_ENABLE_MPI=ON') + else: + self.cfg.update('configopts', '-DQE_ENABLE_MPI=OFF') def _add_openmp(self): """Enable OpenMP for Quantum ESPRESSO.""" - self.cfg.update('configopts', '-DQE_ENABLE_OPENMP=ON') + if self.toolchain.options.get('openmp', False): + self.cfg.update('configopts', '-DQE_ENABLE_OPENMP=ON') + else: + self.cfg.update('configopts', '-DQE_ENABLE_OPENMP=OFF') def _add_cuda(self): """Enable CUDA for Quantum ESPRESSO.""" - self.cfg.update('configopts', '-DQE_ENABLE_CUDA=ON') - self.cfg.update('configopts', '-DQE_ENABLE_OPENACC=ON') + if self.cfg.get('with_cuda', False): + self.cfg.update('configopts', '-DQE_ENABLE_CUDA=ON') + self.cfg.update('configopts', '-DQE_ENABLE_OPENACC=ON') + else: + self.cfg.update('configopts', '-DQE_ENABLE_CUDA=OFF') + self.cfg.update('configopts', '-DQE_ENABLE_OPENACC=OFF') def _add_scalapack(self): """Enable ScaLAPACK for Quantum ESPRESSO.""" - if not self.toolchain.options.get('usempi', False): - raise EasyBuildError('ScaLAPACK support requires MPI') - self.cfg.update('configopts', '-DQE_ENABLE_SCALAPACK=ON') + if self.cfg.get('with_scalapack', False): + if not self.toolchain.options.get('usempi', False): + raise EasyBuildError('ScaLAPACK support requires MPI') + self.cfg.update('configopts', '-DQE_ENABLE_SCALAPACK=ON') + else: + self.cfg.update('configopts', '-DQE_ENABLE_SCALAPACK=OFF') def _add_fox(self): """Enable FoX for Quantum ESPRESSO.""" - self.cfg.update('configopts', '-DQE_ENABLE_FOX=ON') + if self.cfg.get('with_fox', False): + self.cfg.update('configopts', '-DQE_ENABLE_FOX=ON') + else: + self.cfg.update('configopts', '-DQE_ENABLE_FOX=OFF') def _add_hdf5(self): """Enable HDF5 for Quantum ESPRESSO.""" - self.cfg.update('configopts', '-DQE_ENABLE_HDF5=ON') + if get_software_root('HDF5'): + self.cfg.update('configopts', '-DQE_ENABLE_HDF5=ON') + else: + self.cfg.update('configopts', '-DQE_ENABLE_HDF5=OFF') def _add_libxc(self): """Enable LibXC for Quantum ESPRESSO.""" - self.cfg.update('configopts', '-DQE_ENABLE_LIBXC=ON') + if get_software_root('libxc'): + self.cfg.update('configopts', '-DQE_ENABLE_LIBXC=ON') + else: + self.cfg.update('configopts', '-DQE_ENABLE_LIBXC=OFF') def _add_elpa(self): """Enable ELPA for Quantum ESPRESSO.""" - if not self.cfg.get('with_scalapack', False): - raise EasyBuildError('ELPA support requires ScaLAPACK') - if LooseVersion(self.version) == LooseVersion('7.3') and self.toolchain.options.get('openmp', False): - raise EasyBuildError('QE 7.3 with cmake does not support ELPA with OpenMP') - self.cfg.update('configopts', '-DQE_ENABLE_ELPA=ON') + if get_software_root('ELPA'): + if not self.cfg.get('with_scalapack', False): + raise EasyBuildError('ELPA support requires ScaLAPACK') + if LooseVersion(self.version) == LooseVersion('7.3') and self.toolchain.options.get('openmp', False): + raise EasyBuildError('QE 7.3 with cmake does not support ELPA with OpenMP') + self.cfg.update('configopts', '-DQE_ENABLE_ELPA=ON') + else: + self.cfg.update('configopts', '-DQE_ENABLE_ELPA=OFF') def _add_gipaw(self): """Enable GIPAW for Quantum ESPRESSO.""" @@ -181,6 +198,8 @@ def _add_d3q(self): raise EasyBuildError( 'D3Q compilation will fail for QE 7.3 and 7.3.1 without submodule downloaded from easyconfig.' ) + if not self.toolchain.options.get('usempi', False): + raise EasyBuildError('D3Q support requires MPI enabled') res = ['d3q'] self.check_bins += [ 'd3_asr3.x', 'd3_db.x', 'd3_import_shengbte.x', 'd3_interpolate2.x', 'd3_lw.x', 'd3_q2r.x', From e14ec142323960619c2d5ef245a5284a8f35fa50 Mon Sep 17 00:00:00 2001 From: Davide Grassano <34096612+Crivella@users.noreply.github.com> Date: Fri, 5 Apr 2024 10:32:30 +0200 Subject: [PATCH 12/18] Update easybuild/easyblocks/q/quantumespressocmake.py Co-authored-by: ocaisa --- easybuild/easyblocks/q/quantumespressocmake.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/q/quantumespressocmake.py b/easybuild/easyblocks/q/quantumespressocmake.py index 14c606572db..5b0cffc36c7 100644 --- a/easybuild/easyblocks/q/quantumespressocmake.py +++ b/easybuild/easyblocks/q/quantumespressocmake.py @@ -196,7 +196,7 @@ def _add_d3q(self): # https://github.com/anharmonic/d3q/issues/13 if not os.path.exists(os.path.join(self.builddir, self.install_subdir, 'external', 'd3q', '.git')): raise EasyBuildError( - 'D3Q compilation will fail for QE 7.3 and 7.3.1 without submodule downloaded from easyconfig.' + 'D3Q compilation will fail for QE 7.3 and 7.3.1 without submodule downloaded via sources in easyconfig.' ) if not self.toolchain.options.get('usempi', False): raise EasyBuildError('D3Q support requires MPI enabled') From 311a819eb2f34fbd472060a4df4a73b34c0abb97 Mon Sep 17 00:00:00 2001 From: crivella Date: Fri, 5 Apr 2024 10:47:45 +0200 Subject: [PATCH 13/18] Linting and version check --- easybuild/easyblocks/q/quantumespressocmake.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/easybuild/easyblocks/q/quantumespressocmake.py b/easybuild/easyblocks/q/quantumespressocmake.py index 5b0cffc36c7..d4bb9796820 100644 --- a/easybuild/easyblocks/q/quantumespressocmake.py +++ b/easybuild/easyblocks/q/quantumespressocmake.py @@ -196,7 +196,8 @@ def _add_d3q(self): # https://github.com/anharmonic/d3q/issues/13 if not os.path.exists(os.path.join(self.builddir, self.install_subdir, 'external', 'd3q', '.git')): raise EasyBuildError( - 'D3Q compilation will fail for QE 7.3 and 7.3.1 without submodule downloaded via sources in easyconfig.' + 'D3Q compilation will fail for QE 7.3 and 7.3.1 without submodule downloaded via' + + 'sources in easyconfig.' ) if not self.toolchain.options.get('usempi', False): raise EasyBuildError('D3Q support requires MPI enabled') @@ -256,12 +257,13 @@ def configure_step(self): # Change format of timings to seconds only (from d/h/m/s) self.cfg.update('configopts', '-DQE_CLOCK_SECONDS=ON') - # Needed to avoid a DSO missing from command line linking error - # https://gitlab.com/QEF/q-e/-/issues/667 - if self.cfg.get('build_shared_libs', False): - ldflags = os.getenv('LDFLAGS', '') - ldflags += ' -Wl,--copy-dt-needed-entries ' - env.setvar('LDFLAGS', ldflags) + if LooseVersion(self.version) <= LooseVersion('7.3.1'): + # Needed to avoid a `DSO missing from command line` linking error + # https://gitlab.com/QEF/q-e/-/issues/667 + if self.cfg.get('build_shared_libs', False): + ldflags = os.getenv('LDFLAGS', '') + ldflags += ' -Wl,--copy-dt-needed-entries ' + env.setvar('LDFLAGS', ldflags) super(EB_QuantumESPRESSOcmake, self).configure_step() From c9c8bc23943bab590d21e5ec3bdfbb88d3139841 Mon Sep 17 00:00:00 2001 From: crivella Date: Fri, 5 Apr 2024 11:45:32 +0200 Subject: [PATCH 14/18] Fixed logic --- easybuild/easyblocks/q/quantumespressocmake.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/easybuild/easyblocks/q/quantumespressocmake.py b/easybuild/easyblocks/q/quantumespressocmake.py index d4bb9796820..48914d36f4e 100644 --- a/easybuild/easyblocks/q/quantumespressocmake.py +++ b/easybuild/easyblocks/q/quantumespressocmake.py @@ -220,20 +220,22 @@ def _copy_submodule_dirs(self): for submod in self.SUBMODULES: src = os.path.join(self.builddir, submod) dst = os.path.join(self.builddir, self.install_subdir, 'external', submod) + if os.path.exists(src): self.log.info('Copying submodule %s into %s' % (submod, dst)) # Remove empty directories and replace them with the downloaded submodule if os.path.exists(dst): shutil.rmtree(dst) shutil.move(src, dst) + + # Trick QE to think that the submodule is already installed in case `keep_git_dir` is not used in + # the easyconfig file + gitf = os.path.join(dst, '.git') + if not os.path.exists(gitf): + os.mkdir(gitf) else: self.log.warning('Submodule %s not found at %s' % (submod, src)) - # Trick QE to think that the submodule is already installed in case `keep_git_dir` is not used in - # the easyconfig file - gitf = os.path.join(dst, '.git') - if not os.path.exists(gitf): - os.mkdir(gitf) def configure_step(self): """Custom configuration procedure for Quantum ESPRESSO.""" From 0e0f27cd5b5f68f44b775fe94dc9aec3e03a6c58 Mon Sep 17 00:00:00 2001 From: crivella Date: Fri, 5 Apr 2024 11:46:42 +0200 Subject: [PATCH 15/18] Lint --- easybuild/easyblocks/q/quantumespressocmake.py | 1 - 1 file changed, 1 deletion(-) diff --git a/easybuild/easyblocks/q/quantumespressocmake.py b/easybuild/easyblocks/q/quantumespressocmake.py index 48914d36f4e..74554223ff8 100644 --- a/easybuild/easyblocks/q/quantumespressocmake.py +++ b/easybuild/easyblocks/q/quantumespressocmake.py @@ -236,7 +236,6 @@ def _copy_submodule_dirs(self): else: self.log.warning('Submodule %s not found at %s' % (submod, src)) - def configure_step(self): """Custom configuration procedure for Quantum ESPRESSO.""" From 53b54bfb08f896058a87b0cc54f1f924c5916593 Mon Sep 17 00:00:00 2001 From: crivella Date: Fri, 5 Apr 2024 15:21:39 +0200 Subject: [PATCH 16/18] Getting `nprocs` for test only in one place --- easybuild/easyblocks/q/quantumespressocmake.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/easybuild/easyblocks/q/quantumespressocmake.py b/easybuild/easyblocks/q/quantumespressocmake.py index 74554223ff8..70a5df031bc 100644 --- a/easybuild/easyblocks/q/quantumespressocmake.py +++ b/easybuild/easyblocks/q/quantumespressocmake.py @@ -251,9 +251,9 @@ def configure_step(self): self._add_plugins() # Enable/configure test suite - nprocs = self.cfg.get('test_suite_nprocs', 1) + self._test_nprocs = self.cfg.get('test_suite_nprocs', 1) self.cfg.update('configopts', '-DQE_ENABLE_TEST=ON') - self.cfg.update('configopts', '-DTESTCODE_NPROCS=%d' % nprocs) + self.cfg.update('configopts', '-DTESTCODE_NPROCS=%d' % self._test_nprocs) # Change format of timings to seconds only (from d/h/m/s) self.cfg.update('configopts', '-DQE_CLOCK_SECONDS=ON') @@ -275,8 +275,7 @@ def test_step(self): """ thr = self.cfg.get('test_suite_threshold', 0.97) - nprocs = self.cfg.get('test_suite_nprocs', 1) - concurrent = max(1, self.cfg.get('parallel', 1) // nprocs) + concurrent = max(1, self.cfg.get('parallel', 1) // self._test_nprocs) allow_fail = self.cfg.get('test_suite_allow_failures', []) cmd = ' '.join([ From edcfec4334f7029112213a05784e75f9b644c3f8 Mon Sep 17 00:00:00 2001 From: crivella Date: Fri, 5 Apr 2024 15:22:35 +0200 Subject: [PATCH 17/18] Check for QE < 7.3.1 on configmake easyblock --- easybuild/easyblocks/q/quantumespresso.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/easybuild/easyblocks/q/quantumespresso.py b/easybuild/easyblocks/q/quantumespresso.py index 563890072a6..dad09fb4ea1 100644 --- a/easybuild/easyblocks/q/quantumespresso.py +++ b/easybuild/easyblocks/q/quantumespresso.py @@ -367,6 +367,12 @@ def _adjust_compiler_flags(self, comp_fam): def configure_step(self): """Custom configuration procedure for Quantum ESPRESSO.""" + if LooseVersion(self.version) < LooseVersion("7.3.1"): + raise EasyBuildError( + "QuantumESPRESSO 7.3.1 and later are not supported with the this easyblock (ConfigureMake), " + + "use the EB_QuantumESPRESSOcmake (CMakeMake) easyblock instead." + ) + # compose list of DFLAGS (flag, value, keep_stuff) # for guidelines, see include/defs.h.README in sources self.dflags = [] From a1cbcad0fdd0d0868ba80e77e904d637e29e6d05 Mon Sep 17 00:00:00 2001 From: Davide Grassano <34096612+Crivella@users.noreply.github.com> Date: Fri, 5 Apr 2024 15:28:09 +0200 Subject: [PATCH 18/18] Update easybuild/easyblocks/q/quantumespresso.py Co-authored-by: ocaisa --- easybuild/easyblocks/q/quantumespresso.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/q/quantumespresso.py b/easybuild/easyblocks/q/quantumespresso.py index dad09fb4ea1..7d86dfabeb8 100644 --- a/easybuild/easyblocks/q/quantumespresso.py +++ b/easybuild/easyblocks/q/quantumespresso.py @@ -367,7 +367,7 @@ def _adjust_compiler_flags(self, comp_fam): def configure_step(self): """Custom configuration procedure for Quantum ESPRESSO.""" - if LooseVersion(self.version) < LooseVersion("7.3.1"): + if LooseVersion(self.version) >= LooseVersion("7.3.1"): raise EasyBuildError( "QuantumESPRESSO 7.3.1 and later are not supported with the this easyblock (ConfigureMake), " + "use the EB_QuantumESPRESSOcmake (CMakeMake) easyblock instead."