Skip to content

update custom easyblock for CP2K >= v2024 #3433

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
beeebiii wants to merge 5 commits into
base: develop
Choose a base branch
from
Open

update custom easyblock for CP2K >= v2024 #3433

Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
8dd4818
This is the cp2k.py(Easyblock) that i have written being based on (2918)
beeebiii Sep 4, 2024
384d739
The final changes done in easyblock for (CP2K+GPU)
beeebiii Sep 10, 2024
cb6269a
The changes(after hound correction) done in easyblock for (CP2K+GPU)
beeebiii Sep 10, 2024
b9b16ab
The test fail created by Hound resolved
beeebiii Sep 12, 2024
85b9fe7
The unused flags is removed to solve the python-linting error
beeebiii Sep 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
173 changes: 115 additions & 58 deletions easybuild/easyblocks/c/cp2k.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@
@author: Damian Alvarez (Forschungszentrum Juelich GmbH)
@author: Alan O'Cais (Forschungszentrum Juelich GmbH)
@author: Balazs Hajgato (Free University Brussels (VUB))
@author: O. Baris Malcioglu (Middle East Technical University)
@author: Bibek Chapagain(Barcelona Supercomputing Center)
"""

import fileinput
Expand Down Expand Up @@ -80,6 +82,7 @@ def __init__(self, *args, **kwargs):

# used for both libsmm and libxsmm
self.libsmm = ''
self.libsmm_path = ''
self.modincpath = ''
self.openmp = ''

Expand All @@ -103,14 +106,15 @@ def extra_options():
'plumed': [None, "Enable PLUMED support", CUSTOM],
'type': ['popt', "Type of build ('popt' or 'psmp')", CUSTOM],
'typeopt': [True, "Enable optimization", CUSTOM],
'gpuver': ['H100', "CUDA gpu version", CUSTOM],
}
return EasyBlock.extra_options(extra_vars)

def _generate_makefile(self, options):
"""Generate Makefile based on options dictionary and optional make instructions"""

text = "# Makefile generated by CP2K easyblock in EasyBuild\n"
for key, value in sorted(options.items()):
for key, value in options.items():
text += "%s = %s\n" % (key, value)
return text + self.make_instructions

Expand All @@ -119,11 +123,16 @@ def configure_step(self):
- build Libint wrapper
- generate Makefile
"""
cp2k_version = LooseVersion(self.version)

known_types = ['popt', 'psmp']
if self.cfg['type'] not in known_types:
raise EasyBuildError("Unknown build type specified: '%s', known types are %s",
self.cfg['type'], known_types)
if cp2k_version >= LooseVersion('2024') and self.cfg['type'] == 'popt':
self.cfg['type'] = 'psmp'
setvar('OMP_NUM_THREADS', '1')
self.log.debug('In 2023 version popt is psmp with OMP_NUM_THREADS set to 1')

# correct start dir, if needed
# recent CP2K versions have a 'cp2k' dir in the unpacked 'cp2k' dir
Expand Down Expand Up @@ -154,13 +163,15 @@ def configure_step(self):
if libxsmm:
self.cfg.update('extradflags', '-D__LIBXSMM')
self.libsmm = '-lxsmm -lxsmmf'
self.libsmm_path = libxsmm
self.log.debug('Using libxsmm %s' % libxsmm)
elif libsmm:
libsmms = glob.glob(os.path.join(libsmm, 'lib', 'libsmm_*nn.a'))
dfs = [os.path.basename(os.path.splitext(x)[0]).replace('lib', '-D__HAS_') for x in libsmms]
moredflags = ' ' + ' '.join(dfs)
self.cfg.update('extradflags', moredflags)
self.libsmm = ' '.join(libsmms)
self.libsmm_path = libsmm
self.log.debug('Using libsmm %s (extradflags %s)' % (self.libsmm, moredflags))

# obtain list of modinc's to use
Expand Down Expand Up @@ -219,22 +230,39 @@ def configure_step(self):
if elpa:
options['LIBS'] += ' -lelpa'
elpa_inc_dir = os.path.join(elpa, 'include', 'elpa-%s' % get_software_version('ELPA'), 'modules')
options['FCFLAGSOPT'] += ' -I%s ' % elpa_inc_dir
if cp2k_version >= LooseVersion('2024'):
options['INCS'] += ' -I%s ' % elpa_inc_dir
else:
options['FCFLAGSOPT'] += ' -I%s ' % elpa_inc_dir
if LooseVersion(self.version) >= LooseVersion('6.1'):
elpa_ver = ''.join(get_software_version('ELPA').split('.')[:2])
options['DFLAGS'] += ' -D__ELPA=%s' % elpa_ver
elpa_inc_dir = os.path.join(elpa, 'include', 'elpa-%s' % get_software_version('ELPA'), 'elpa')
options['FCFLAGSOPT'] += ' -I%s ' % elpa_inc_dir
if cp2k_version >= LooseVersion('2024'):
options['INCS'] += ' -I%s ' % elpa_inc_dir
else:
options['FCFLAGSOPT'] += ' -I%s ' % elpa_inc_dir
else:
options['DFLAGS'] += ' -D__ELPA3'

# CUDA
cuda = get_software_root('CUDA')
if cuda:
options['DFLAGS'] += ' -D__ACC -D__DBCSR_ACC'
options['LIBS'] += ' -lcudart -lcublas -lcufft -lrt'
options['NVCC'] = ' nvcc'

if cp2k_version >= LooseVersion('2024'):
options['DFLAGS'] += ' -D__OFFLOAD_CUDA -D__DBCSR_ACC '
options['LIBS'] += ' -lcufft -lcudart -lnvrtc -lcuda -lcublas'
options['OFFLOAD_CC'] = 'nvcc'
Copy link
Copy Markdown
Contributor

@gkaf89 gkaf89 Nov 20, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The OFFLOAD_CC variable is used in the makefile of CP2K to build files with OpenACC offloading, even in versions older than 2024. For instance 2023.1 fails to build with CUDA support with the old easyblock.

Maybe enable the modification for some versions earlier than 2024 if they are still relevant?

Copy link
Copy Markdown
Author

@beeebiii beeebiii Nov 21, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah it can surely be modified, but i have not tested them with old versions, when i test them will definitely modify if required. Thanks!!

options['OFFLOAD_FLAGS'] = "-O3 -g -w --std=c++11 $(DFLAGS) -Xcompiler='-fopenmp -Wall -Wextra -Werror'"
options['OFFLOAD_TARGET'] = 'cuda'
options['GPUVER'] = self.cfg['gpuver']
options['CXX'] = 'mpicxx'
options['CXXFLAGS'] = '-O3 -fopenmp -g -w --std=c++14 -fPIC $(DFLAGS) $(INCS)'
else:
options['DFLAGS'] += ' -D__ACC -D__DBCSR_ACC'
options['LIBS'] += ' -lcudart -lcublas -lcufft -lrt'
options['NVCC'] = ' nvcc'
# reset typearch
self.typearch = "Linux-x86-64-%s-cuda" % self.toolchain.name
# avoid group nesting
options['LIBS'] = options['LIBS'].replace('-Wl,--start-group', '').replace('-Wl,--end-group', '')

Expand Down Expand Up @@ -299,10 +327,12 @@ def prepmodinc(self):
def configure_common(self):
"""Common configuration for all toolchains"""

cp2k_version = LooseVersion(self.version)

# openmp introduces 2 major differences
# -automatic is default: -noautomatic -auto-scalar
# some mem-bandwidth optimisation
if self.cfg['type'] == 'psmp':
if self.cfg['type'] == 'psmp' or cp2k_version >= LooseVersion('2024'):
self.openmp = self.toolchain.get_flag('openmp')

# determine which opt flags to use
Expand Down Expand Up @@ -337,33 +367,32 @@ def configure_common(self):
if not mpi2:
raise EasyBuildError("CP2K needs MPI-2, no known MPI-2 supporting library loaded?")

cppflags = os.getenv('CPPFLAGS')
ldflags = os.getenv('LDFLAGS')
cflags = os.getenv('CFLAGS')
fflags = os.getenv('FFLAGS')
fflags_lowopt = re.sub('-O[0-9]', '-O1', fflags)

options = {
'CC': os.getenv('MPICC'),
'CPP': '',
'FC': '%s %s' % (os.getenv('MPIF90'), self.openmp),
'LD': '%s %s' % (os.getenv('MPIF90'), self.openmp),
'AR': 'ar -r',
'CPPFLAGS': '',

'FC': '%s' % (os.getenv('MPIF90')),
'LD': '%s' % (os.getenv('MPIF90')),
'FPIC': self.fpic,
'DFLAGS': ' -D__parallel -D__BLACS -D__SCALAPACK -D__FFTSG %s' % self.cfg['extradflags'],
'INCS': '',
'CFLAGS': '-O3 -fopenmp -ftree-vectorize -march=native -fno-math-errno -fopenmp -std=c11 $(FPIC) $(DEBUG) '
'$(INCS) $(DFLAGS) %s' %
self.cfg['extracflags'],
'DEBUG': self.debug,

'FCFLAGS': '$(FCFLAGS%s)' % optflags,
'FREE': '',
'FCFLAGS': '$(FCFLAGS%s) $(INCS)' % optflags,
'FCFLAGS2': '$(FCFLAGS%s)' % regflags,

'CFLAGS': ' %s %s %s $(FPIC) $(DEBUG) %s ' % (cflags, cppflags, ldflags, self.cfg['extracflags']),
'DFLAGS': ' -D__parallel -D__BLACS -D__SCALAPACK -D__FFTSG %s' % self.cfg['extradflags'],

'LIBS': os.getenv('LIBS', ''),

'FCFLAGSNOOPT': '$(DFLAGS) $(CFLAGS) -O0 $(FREE) $(FPIC) $(DEBUG)',
'FCFLAGSOPT': '%s $(FREE) $(SAFE) $(FPIC) $(DEBUG)' % fflags,
'FCFLAGSOPT2': '%s $(FREE) $(SAFE) $(FPIC) $(DEBUG)' % fflags_lowopt,

'LDFLAGS': '$(FCFLAGS) %s ' % ldflags,
'LIBS': os.getenv('LIBS', ''),

}

libint = get_software_root('LibInt')
Expand Down Expand Up @@ -410,7 +439,10 @@ def configure_common(self):
options['LIBS'] += ' %s -lstdc++ %s' % (libint_libs, libint_wrapper)

# add Libint include dir to $FCFLAGS
options['FCFLAGS'] += ' -I' + os.path.join(libint, 'include')
if cp2k_version >= LooseVersion('2024'):
options['INCS'] += ' -I' + os.path.join(libint, 'include')
else:
options['FCFLAGS'] += ' -I' + os.path.join(libint, 'include')

else:
# throw a warning, since CP2K without Libint doesn't make much sense
Expand All @@ -436,20 +468,14 @@ def configure_common(self):
options['LIBS'] += ' -L%s/lib -lxcf90 -lxc' % libxc
else:
options['LIBS'] += ' -L%s/lib -lxc' % libxc

if cp2k_version >= LooseVersion('2024'):
options['INCS'] += ' -I%s/include ' % libxc

self.log.info("Using Libxc-%s" % cur_libxc_version)
else:
self.log.info("libxc module not loaded, so building without libxc support")

libvori = get_software_root('libvori')
if libvori:
if LooseVersion(self.version) >= LooseVersion('8.1'):
options['LIBS'] += ' -lvori'
options['DFLAGS'] += ' -D__LIBVORI'
else:
raise EasyBuildError("This version of CP2K does not suppport libvori")
else:
self.log.info("libvori module not loaded, so building without support for Voronoi integration")

return options

def configure_intel_based(self):
Expand Down Expand Up @@ -498,7 +524,7 @@ def configure_intel_based(self):
# Required due to memory leak that occurs if high optimizations are used (from CP2K 7.1 intel-popt-makefile)
if ifortver >= LooseVersion("2018.5"):
self.make_instructions += "mp2_optimize_ri_basis.o: mp2_optimize_ri_basis.F\n" \
"\t$(FC) -c $(subst O2,O0,$(FCFLAGSOPT)) $<\n"
"\t$(FC) -c $(subst O2,O0,$(FCFLAGSOPT)) $<\n"
self.log.info("Optimization level of mp2_optimize_ri_basis.F was decreased to '-O0'")

# RHEL8 intel/2020a lots of CPASSERT failed (due to high optimization in cholesky decomposition)
Expand Down Expand Up @@ -543,21 +569,29 @@ def configure_intel_based(self):

def configure_GCC_based(self):
"""Configure for GCC based toolchains"""
options = self.configure_common()
cp2k_version = LooseVersion(self.version)

options.update({
# need this to prevent "Unterminated character constant beginning" errors
'FREE': '-ffree-form -ffree-line-length-none',
options = self.configure_common()
if cp2k_version >= LooseVersion('2024'):
options.update({
# need this to prevent "Unterminated character constant beginning" errors
'FREE': '-ffree-form -ffree-line-length-none -std=f2008',
})
options[
'FCFLAGSOPT'] = '-O3 -ftree-vectorize -march=native -fno-math-errno -fopenmp -fPIC $(FREE) $(DFLAGS)'

'LDFLAGS': '$(FCFLAGS)',
'OBJECTS_ARCHITECTURE': 'machine_gfortran.o',
})
else:
options.update({
# need this to prevent "Unterminated character constant beginning" errors
'FREE': '-ffree-form -ffree-line-length-none ',
'LDFLAGS': '$(FCFLAGS) ',
'OBJECTS_ARCHITECTURE': 'machine_gfortran.o',
})
options['FCFLAGSOPT'] += ' $(DFLAGS) $(CFLAGS) -fmax-stack-var-size=32768'
options['FCFLAGSOPT2'] += ' $(DFLAGS) $(CFLAGS)'

options['DFLAGS'] += ' -D__GFORTRAN'

options['FCFLAGSOPT'] += ' $(DFLAGS) $(CFLAGS) -fmax-stack-var-size=32768'
options['FCFLAGSOPT2'] += ' $(DFLAGS) $(CFLAGS)'

gcc_version = get_software_version('GCCcore') or get_software_version('GCC')
if LooseVersion(gcc_version) >= LooseVersion('10.0') and LooseVersion(self.version) <= LooseVersion('7.1'):
# -fallow-argument-mismatch is required for CP2K 7.1 (and older) when compiling with GCC 10.x & more recent,
Expand All @@ -572,6 +606,8 @@ def configure_GCC_based(self):
def configure_ACML(self, options):
"""Configure for AMD Math Core Library (ACML)"""

cp2k_version = LooseVersion(self.version)

openmp_suffix = ''
if self.openmp:
openmp_suffix = '_mp'
Expand All @@ -583,17 +619,28 @@ def configure_ACML(self, options):
blas = os.getenv('LIBBLAS', '')
blas = blas.replace('gfortran64', 'gfortran64%s' % openmp_suffix)
options['LIBS'] += ' %s %s %s' % (self.libsmm, os.getenv('LIBSCALAPACK', ''), blas)
if cp2k_version >= LooseVersion('2024'):
options['LDFLAGS'] += ' -L%s/lib ' % self.libsmm_path
options['INCS'] += ' -I%s/include ' % self.libsmm_path

return options

def configure_BLAS_lib(self, options):
"""Configure for BLAS library."""

cp2k_version = LooseVersion(self.version)

options['LIBS'] += ' %s %s' % (self.libsmm, os.getenv('LIBBLAS', ''))
if cp2k_version >= LooseVersion('2024'):
options['LDFLAGS'] += ' -L%s/lib ' % self.libsmm_path
options['INCS'] += ' -I%s/include ' % self.libsmm_path
return options

def configure_MKL(self, options):
"""Configure for Intel Math Kernel Library (MKL)"""

cp2k_version = LooseVersion(self.version)

options['INTEL_INC'] = '$(MKLROOT)/include'
options['DFLAGS'] += ' -D__FFTW3'

Expand Down Expand Up @@ -624,24 +671,34 @@ def configure_MKL(self, options):
options['INTEL_INCF'] = '$(INTEL_INC)/fftw'
options['LIBS'] = '%s %s' % (os.getenv('LIBFFT', ''), options['LIBS'])

if cp2k_version >= LooseVersion('2024'):
options['LDFLAGS'] += ' %s ' % self.libsmm

return options

def configure_FFTW3(self, options):
"""Configure for FFTW3"""

options.update({
'FFTW_INC': os.getenv('FFT_INC_DIR', ''), # GCC
'FFTW3INC': os.getenv('FFT_INC_DIR', ''), # Intel
'FFTW3LIB': os.getenv('FFT_LIB_DIR', ''), # Intel
})
cp2k_version = LooseVersion(self.version)

options['DFLAGS'] += ' -D__FFTW3'
if self.cfg['type'] == 'psmp':
if cp2k_version >= LooseVersion('2024'):
libfft = os.getenv('LIBFFT_MT', '')
options['LIBS'] += ' -lfftw3_omp -lfftw3'
options['LDFLAGS'] += ' -L%s ' % os.getenv('FFT_LIB_DIR', '')
options['INCS'] += ' -I%s ' % os.getenv('FFT_INC_DIR', '')
else:
libfft = os.getenv('LIBFFT', '')
options['LIBS'] += ' -L%s %s' % (os.getenv('FFT_LIB_DIR', '.'), libfft)
options.update({
'FFTW_INC': os.getenv('FFT_INC_DIR', ''), # GCC
'FFTW3INC': os.getenv('FFT_INC_DIR', ''), # Intel
'FFTW3LIB': os.getenv('FFT_LIB_DIR', ''), # Intel
})

if self.cfg['type'] == 'psmp':
libfft = os.getenv('LIBFFT_MT', '')
else:
libfft = os.getenv('LIBFFT', '')
options['LIBS'] += ' -L%s %s' % (os.getenv('FFT_LIB_DIR', '.'), libfft)
options['DFLAGS'] += ' -D__FFTW3'
return options

def configure_LAPACK(self, options):
Expand Down Expand Up @@ -729,15 +786,15 @@ def test_step(self):
# location of do_regtest script
cfg_fn = 'cp2k_regtest.cfg'

regtest_script = os.path.join(self.cfg['start_dir'], 'tools', 'regtesting', 'do_regtest')
regtest_cmd = [regtest_script, '-nobuild', '-config', cfg_fn]
regtest_script = os.path.join(self.cfg['start_dir'], 'tests', 'do_regtest.py')
regtest_cmd = [regtest_script, self.typearch, self.cfg['type']]
if LooseVersion(self.version) < LooseVersion('7.1'):
# -nosvn option was removed in CP2K 7.1
regtest_cmd.insert(1, '-nosvn')

# older version of CP2K
if not os.path.exists(regtest_script):
regtest_script = os.path.join(self.cfg['start_dir'], 'tools', 'do_regtest')
regtest_script = os.path.join(self.cfg['start_dir'], 'tests', 'do_regtest.py')
regtest_cmd = [regtest_script, '-nocvs', '-quick', '-nocompile', '-config', cfg_fn]

regtest_cmd = ' '.join(regtest_cmd)
Expand Down Expand Up @@ -781,7 +838,7 @@ def test_step(self):
'cp2k_dir': os.path.basename(os.path.normpath(self.cfg['start_dir'])),
'maxtasks': self.cfg['maxtasks'],
'mpicmd_prefix': self.toolchain.mpi_cmd_for('', test_core_cnt),
}
}

write_file(cfg_fn, cfg_txt)
self.log.debug("Contents of %s: %s" % (cfg_fn, cfg_txt))
Expand Down