Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

required changes for toolchain which replaces toolkit + cleanups/enhancements #7

Merged
merged 21 commits into from
Nov 5, 2012
Merged
Show file tree
Hide file tree
Changes from 17 commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
7f8c6a9
stop using toolkit, use toolchain instead
boegel Oct 24, 2012
13abdf7
stop using toolchain.opts, use toolchain.options instead
boegel Oct 24, 2012
ce36418
stop misuing mpi_family to determine MPI lib dir, use MPI_LIB_DIR ins…
boegel Oct 24, 2012
8c0f7f6
fix debug message when clearing LDFLAGS
boegel Oct 26, 2012
b1bb9a7
fix specifying libs and includes for numpy
boegel Oct 26, 2012
bac94a0
fix use of get_openmp_flag and mpi_type
boegel Oct 26, 2012
3247b58
fix specifying library/include paths, just strip off -L/-I prefixes
boegel Oct 27, 2012
297f083
make sure old install dir is removed before configuring
boegel Oct 28, 2012
70fa993
use toolchain.QLOGICMPI instead of QLOGIC
boegel Oct 28, 2012
826a4af
fix use of license_server that was renamed from licenseServer
boegel Oct 28, 2012
530904a
fix sanity check for METIS
boegel Oct 28, 2012
b32d80e
fix typo in help for shared_libs easyconfig parameters
boegel Oct 30, 2012
080cf32
fix checking of MPI family for IntelMPI
boegel Oct 30, 2012
c915ff1
fix use of INTELMPI constant, rename INTEL constant to INTELCOMP
boegel Oct 31, 2012
bbe1307
add comments to make PyDev happy about assignment to __path__
boegel Nov 3, 2012
701745f
fix rename of make_install into install_step (how did we miss that?)
boegel Nov 3, 2012
fd60bf6
add #@UndefinedVariable comments to make PyDev happy where toolchain …
boegel Nov 3, 2012
5139f84
use dontcreate parameter for make_installdir (depends on https://gith…
boegel Nov 4, 2012
927efd7
Merge branch 'toolchain_176' of github.com:boegel/easybuild-easyblock…
boegel Nov 4, 2012
9867a85
refactor out preprocessor flag in a variable, and append it to cflags…
boegel Nov 4, 2012
953e13b
use toolchain.get_variable method to obtain list of library and inclu…
boegel Nov 5, 2012
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion easybuild/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,4 +29,4 @@
from pkgutil import extend_path

# we're not the only ones in this namespace
__path__ = extend_path(__path__, __name__)
__path__ = extend_path(__path__, __name__) #@ReservedAssignment
2 changes: 1 addition & 1 deletion easybuild/easyblocks/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,6 @@ def get_git_revision():
__path__.extend([os.path.join(__path__[0], subdir) for subdir in subdirs])
# And let python know this is not the only place to look for them, so we can have multiple
# easybuild/easyblock paths in your python search path, next to the official easyblocks distribution
__path__ = extend_path(__path__, __name__)
__path__ = extend_path(__path__, __name__) #@ReservedAssignment

del subdir, subdirs, l, git_rev
2 changes: 1 addition & 1 deletion easybuild/easyblocks/a/atlas.py
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ def configure_step(self):
" required to build ATLAS with a full LAPACK library.")

# enable building of shared libraries (requires -fPIC)
if self.cfg['sharedlibs'] or self.toolchain.opts['pic']:
if self.cfg['sharedlibs'] or self.toolchain.options['pic']:
self.log.debug("Enabling -fPIC because we're building shared ATLAS libs, or just because.")
self.cfg.update('configopts', '-Fa alg -fPIC')

Expand Down
29 changes: 4 additions & 25 deletions easybuild/easyblocks/b/blacs.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,10 +35,8 @@
import os
import shutil

import easybuild.tools.toolkit as toolchain
from easybuild.easyblocks.generic.configuremake import ConfigureMake
from easybuild.tools.filetools import run_cmd
from easybuild.tools.modules import get_software_root


# also used by ScaLAPACK
Expand Down Expand Up @@ -82,31 +80,13 @@ def configure_step(self):
def build_step(self):
"""Build BLACS using build_step, after figuring out the make options based on the heuristic tools available."""

# determine MPI base dir and lib
known_mpis = {
toolchain.OPENMPI: "-L$(MPILIBdir) -lmpi_f77",
toolchain.MVAPICH2: "$(MPILIBdir)/libmpich.a $(MPILIBdir)/libfmpich.a " + \
"$(MPILIBdir)/libmpl.a -lpthread"
}

mpi_type = self.toolchain.mpi_type()

base, mpilib = None, None
if mpi_type in known_mpis.keys():
base = get_software_root(mpi_type)
mpilib = known_mpis[mpi_type]

else:
self.log.error("Unknown MPI lib %s used (known MPI libs: %s)" % (mpi_type, known_mpis.keys()))

opts = {
'mpicc': "%s %s" % (os.getenv('MPICC'), os.getenv('CFLAGS')),
'mpif77': "%s %s" % (os.getenv('MPIF77'), os.getenv('FFLAGS')),
'f77': os.getenv('F77'),
'cc': os.getenv('CC'),
'builddir': os.getcwd(),
'base': base,
'mpilib': mpilib
'mpidir': os.path.dirname(os.getenv('MPI_LIB_DIR')),
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

at some point we will have to specify what the best way is to do this: through environment or through toolchain instance

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

with best i meant "recommended"

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I feel environment variables are easier, no API for people to get familiar with, they just need to know which environment variable to use. It also allows us to make changes to toolchain, if needed.

The biggest downside is that something else may be setting the environment variables as well, outside of EasyBuild, I guess.

Is there currently an easy way to access stuff like CC, MPICC and things like CFLAGS and MPI_LIB_DIR via toolchain?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i don't understand the question. str(self.toolchain.variables['CC']) ? you could write a tiny function (get_variable) to do just that.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I didn't realize it was that simple. :) But it does make a lot of sense in retrospect.

Having a get_variable function makes sense, it hides the guts of toolchain, which is a good thing (because then we can change it without affecting outsiders).

So... Should we change this everywhere? That's going to be a fun exercise... Hard to test if we got them all too.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ask feedback and make the preferred method known. actual work can be done later and in seperate issue.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

See easybuilders/easybuild-framework#298 for a discussion on this.

}

# determine interface and transcomm settings
Expand All @@ -118,8 +98,8 @@ def build_step(self):

# need to build
cmd = "make"
cmd += " CC='%(mpicc)s' F77='%(mpif77)s -I$(MPIINCdir)' MPIdir=%(base)s" \
" MPILIB='%(mpilib)s' BTOPdir=%(builddir)s INTERFACE=NONE" % opts
cmd += " CC='%(mpicc)s' F77='%(mpif77)s' MPIdir=%(mpidir)s" \
" MPILIB='' BTOPdir=%(builddir)s INTERFACE=NONE" % opts

# determine interface using xintface
run_cmd("%s xintface" % cmd, log_all=True, simple=True)
Expand Down Expand Up @@ -169,11 +149,10 @@ def build_step(self):
opts.update({
'comm': comm,
'int': interface,
'base': base
})

add_makeopts = ' MPICC="%(mpicc)s" MPIF77="%(mpif77)s" %(comm)s ' % opts
add_makeopts += ' INTERFACE=%(int)s MPIdir=%(base)s BTOPdir=%(builddir)s mpi ' % opts
add_makeopts += ' INTERFACE=%(int)s MPIdir=%(mpidir)s BTOPdir=%(builddir)s mpi ' % opts

self.cfg.update('makeopts', add_makeopts)

Expand Down
8 changes: 4 additions & 4 deletions easybuild/easyblocks/b/boost.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
import os
import shutil

import easybuild.tools.toolkit as toolchain
import easybuild.tools.toolchain as toolchain
from easybuild.framework.easyblock import EasyBlock
from easybuild.framework.easyconfig import CUSTOM
from easybuild.tools.filetools import run_cmd
Expand All @@ -59,7 +59,7 @@ def configure_step(self):
"""Configure Boost build using custom tools"""

# mpi sanity check
if self.cfg['boost_mpi'] and not self.toolchain.opts['usempi']:
if self.cfg['boost_mpi'] and not self.toolchain.options['usempi']:
self.log.error("When enabling building boost_mpi, also enable the 'usempi' toolchain option.")

# create build directory (Boost doesn't like being built in source dir)
Expand All @@ -72,7 +72,7 @@ def configure_step(self):

# generate config depending on compiler used
toolset = None
if self.toolchain.comp_family() == toolchain.INTEL:
if self.toolchain.comp_family() == toolchain.INTELCOMP:
toolset = 'intel-linux'
elif self.toolchain.comp_family() == toolchain.GCC:
toolset = 'gcc'
Expand All @@ -84,7 +84,7 @@ def configure_step(self):

if self.cfg['boost_mpi']:

self.toolchain.opts['usempi'] = True
self.toolchain.options['usempi'] = True
# configure the boost mpi module
# http://www.boost.org/doc/libs/1_47_0/doc/html/mpi/getting_started.html
# let Boost.Build know to look here for the config file
Expand Down
10 changes: 5 additions & 5 deletions easybuild/easyblocks/c/cp2k.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@
import sys
from distutils.version import LooseVersion

import easybuild.tools.toolkit as toolchain
import easybuild.tools.toolchain as toolchain
from easybuild.framework.easyblock import EasyBlock
from easybuild.framework.easyconfig import CUSTOM
from easybuild.tools.filetools import run_cmd
Expand Down Expand Up @@ -108,10 +108,10 @@ def configure_step(self):
# set compilers options according to toolchain config
# full debug: -g -traceback -check all -fp-stack-check
# -g links to mpi debug libs
if self.toolchain.opts['debug']:
if self.toolchain.options['debug']:
self.debug = '-g'
self.log.info("Debug build")
if self.toolchain.opts['pic']:
if self.toolchain.options['pic']:
self.fpic = "-fPIC"
self.log.info("Using fPIC")

Expand Down Expand Up @@ -143,7 +143,7 @@ def configure_step(self):

# compiler toolchain specific configuration
comp_fam = self.toolchain.comp_family()
if comp_fam == toolchain.INTEL:
if comp_fam == toolchain.INTELCOMP:
options = self.configure_intel_based()
elif comp_fam == toolchain.GCC:
options = self.configure_GCC_based()
Expand Down Expand Up @@ -240,7 +240,7 @@ def configure_common(self):
# -automatic is default: -noautomatic -auto-scalar
# some mem-bandwidth optimisation
if self.cfg['type'] == 'psmp':
self.openmp = self.toolchain.get_openmp_flag()
self.openmp = self.toolchain.get_flag('openmp')

# determine which opt flags to use
if self.cfg['typeopt']:
Expand Down
6 changes: 3 additions & 3 deletions easybuild/easyblocks/d/dolfin.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@
import tempfile

import easybuild.tools.environment as env
import easybuild.tools.toolkit as toolchain
import easybuild.tools.toolchain as toolchain
from easybuild.easyblocks.generic.cmakepythonpackage import CMakePythonPackage
from easybuild.tools.modules import get_software_root, get_software_version

Expand All @@ -53,7 +53,7 @@ def configure_step(self):
fflags = os.getenv('FFLAGS')

# fix for "SEEK_SET is #defined but must not be for the C++ binding of MPI. Include mpi.h before stdio.h"
if self.toolchain.mpi_type() in [toolchain.INTEL, toolchain.MPICH2]:
if self.toolchain.mpi_family() in [toolchain.INTELMPI, toolchain.MPICH2, toolchain.MVAPICH2]:
cflags += " -DMPICH_IGNORE_CXX_SEEK"
cxxflags += " -DMPICH_IGNORE_CXX_SEEK"
fflags += " -DMPICH_IGNORE_CXX_SEEK"
Expand Down Expand Up @@ -97,7 +97,7 @@ def configure_step(self):
self.cfg.update('configopts', '-DZLIB_LIBRARY=%s' % os.path.join(depsdict['zlib'], "lib", "libz.a"))

# set correct openmp options
openmp = self.toolchain.get_openmp_flag()
openmp = self.toolchain.get_flag('openmp')
self.cfg.update('configopts', ' -DOpenMP_CXX_FLAGS="%s"' % openmp)
self.cfg.update('configopts', ' -DOpenMP_C_FLAGS="%s"' % openmp)

Expand Down
4 changes: 2 additions & 2 deletions easybuild/easyblocks/generic/packedbinary.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ def extract_step(self):
"""Unpack the source"""
EasyBlock.extract_step(self)

def make_install(self):
def install_step(self):
"""Copy all unpacked source directories to install directory, one-by-one."""
try:
os.chdir(self.builddir)
Expand All @@ -50,7 +50,7 @@ def make_install(self):
if os.path.isdir(srcpath):
# copy files to install dir via Binary
self.cfg['start_dir'] = src
Binary.make_install(self)
Binary.install_step(self)
except OSError, err:
self.log.error("Failed to copy unpacked sources to install directory: %s" % err)

4 changes: 2 additions & 2 deletions easybuild/easyblocks/h/hdf5.py
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ def configure_step(self):
self.cfg.update('configopts', "--enable-cxx --enable-fortran %s" % fcomp)

# MPI and C++ support enabled requires --enable-unsupported, because this is untested by HDF5
if self.toolchain.opts['usempi']:
if self.toolchain.options['usempi']:
self.cfg.update('configopts', "--enable-unsupported")

# make options
Expand All @@ -72,7 +72,7 @@ def sanity_check_step(self):
Custom sanity check for HDF5
"""

if self.toolchain.opts['usempi']:
if self.toolchain.options['usempi']:
extra_binaries = ["bin/%s" % x for x in ["h5perf", "h5pcc", "h5pfc", "ph5diff"]]
else:
extra_binaries = ["bin/%s" % x for x in ["h5cc", "h5fc"]]
Expand Down
2 changes: 1 addition & 1 deletion easybuild/easyblocks/i/imkl.py
Original file line number Diff line number Diff line change
Expand Up @@ -180,7 +180,7 @@ def post_install_step(self):
if i in lis3:
# use INSTALL_DIR and SPEC_OPT
extramakeopts = ''
if get_software_root('MPICH2'): # can't use toolchain.mpi_type, because of dummy toolchain
if get_software_root('MPICH2'): # can't use toolchain.mpi_family, because of dummy toolchain
extramakeopts = 'mpi=mpich2'
cmd = "make -f makefile libintel64 %s" % extramakeopts

Expand Down
8 changes: 4 additions & 4 deletions easybuild/easyblocks/l/lapack.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@
import os
import shutil

import easybuild.tools.toolkit as toolchain
import easybuild.tools.toolchain as toolchain
from easybuild.easyblocks.generic.configuremake import ConfigureMake
from easybuild.framework.easyconfig import CUSTOM
from easybuild.tools.filetools import run_cmd
Expand Down Expand Up @@ -88,9 +88,9 @@ def configure_step(self):
"""

# copy build_step.inc file from examples
if self.toolchain.comp_family() == toolchain.GCC:
if self.toolchain.comp_family() == toolchain.GCC: #@UndefinedVariable
makeinc = 'gfortran'
elif self.toolchain.comp_family() == toolchain.INTEL:
elif self.toolchain.comp_family() == toolchain.INTELCOMP: #@UndefinedVariable
makeinc = 'ifort'
else:
self.log.error("Don't know which build_step.inc file to pick, unknown compiler being used...")
Expand All @@ -111,7 +111,7 @@ def configure_step(self):

# set optimization flags
fpic = ''
if self.toolchain.opts['pic']:
if self.toolchain.options['pic']:
fpic = '-fPIC'
self.cfg.update('makeopts', 'OPTS="$FFLAGS -m64" NOOPT="%s -m64 -O0"' % fpic)

Expand Down
4 changes: 2 additions & 2 deletions easybuild/easyblocks/l/libsmm.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@
import shutil
from distutils.version import LooseVersion

import easybuild.tools.toolkit as toolchain
import easybuild.tools.toolchain as toolchain
from easybuild.framework.easyblock import EasyBlock
from easybuild.framework.easyconfig import CUSTOM
from easybuild.tools.filetools import run_cmd
Expand Down Expand Up @@ -143,7 +143,7 @@ def build_step(self):
"""

# only GCC is supported for now
if self.toolchain.comp_family() == toolchain.GCC:
if self.toolchain.comp_family() == toolchain.GCC: #@UndefinedVariable
hostcompile = os.getenv('F90')

# optimizations
Expand Down
2 changes: 1 addition & 1 deletion easybuild/easyblocks/m/maple.py
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ def install_step(self):
'Do you wish to have a shortcut installed on your desktop? ->1- Yes 2- No ENTER THE NUMBER FOR YOUR CHOICE, OR PRESS <ENTER> TO ACCEPT THE DEFAULT::': '2',
'->1- Single User License 2- Network License ENTER THE NUMBER FOR YOUR CHOICE, OR PRESS <ENTER> TO ACCEPT THE DEFAULT::': '2',
'PRESS <ENTER> TO EXIT THE INSTALLER:': '',
'License server (DEFAULT: ):': self.cfg['licenseServer'],
'License server (DEFAULT: ):': self.cfg['license_server'],
'Port number (optional) (DEFAULT: ):': '',
'->1- Configure toolbox for Matlab 2- Do not configure at this time ENTER THE NUMBER FOR YOUR CHOICE, OR PRESS <ENTER> TO ACCEPT THE DEFAULT::': '2'
}
Expand Down
22 changes: 10 additions & 12 deletions easybuild/easyblocks/m/metis.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ def build_step(self):

self.cfg.update('makeopts', 'LIBDIR=""')

if self.toolchain.opts['pic']:
if self.toolchain.options['pic']:
self.cfg.update('makeopts', 'CC="$CC -fPIC"')

super(EB_METIS, self).build_step()
Expand Down Expand Up @@ -109,19 +109,17 @@ def install_step(self):
def sanity_check_step(self):
"""Custom sanity check for METIS (more extensive for recent version (>= v5))"""

if not self.cfg['sanityCheckPaths']:
binfiles = []
if LooseVersion(self.version) > LooseVersion("5"):
binfiles += ["cmpfillin", "gpmetis", "graphchk", "m2gmetis", "mpmetis", "ndmetis"]

binfiles = []
if LooseVersion(self.version) > LooseVersion("5"):
binfiles += ["cmpfillin", "gpmetis", "graphchk", "m2gmetis", "mpmetis", "ndmetis"]

incfiles = ["metis.h"]
if LooseVersion(self.version) < LooseVersion("5"):
incfiles += ["defs.h", "macros.h", "proto.h", "rename.h", "struct.h"]
incfiles = ["metis.h"]
if LooseVersion(self.version) < LooseVersion("5"):
incfiles += ["defs.h", "macros.h", "proto.h", "rename.h", "struct.h"]

dirs = []
if LooseVersion(self.version) < LooseVersion("5"):
dirs += ["Lib"]
dirs = []
if LooseVersion(self.version) < LooseVersion("5"):
dirs += ["Lib"]

custom_paths = {
'files': ['bin/%s' % x for x in binfiles] + ['include/%s' % x for x in incfiles] +
Expand Down
2 changes: 1 addition & 1 deletion easybuild/easyblocks/m/mrbayes.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ def configure_step(self):
else:
self.log.error("BEAGLE module not loaded?")

if self.toolchain.opts['usempi']:
if self.toolchain.options['usempi']:
self.cfg.update('configopts', '--enable-mpi')

# configure
Expand Down
6 changes: 3 additions & 3 deletions easybuild/easyblocks/n/netcdf.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@
from distutils.version import LooseVersion

import easybuild.tools.environment as env
import easybuild.tools.toolkit as toolchain
import easybuild.tools.toolchain as toolchain
from easybuild.easyblocks.generic.configuremake import ConfigureMake
from easybuild.tools.modules import get_software_root, get_software_version

Expand All @@ -47,7 +47,7 @@ def configure_step(self):

self.cfg.update('configopts', "--enable-shared")

if self.toolchain.opts['pic']:
if self.toolchain.options['pic']:
self.cfg.update('configopts', '--with-pic')

self.cfg.update('configopts', 'FCFLAGS="%s" CC="%s" FC="%s"' % (os.getenv('FFLAGS'),
Expand All @@ -56,7 +56,7 @@ def configure_step(self):
))

# add -DgFortran to CPPFLAGS when building with GCC
if self.toolchain.comp_family() == toolchain.GCC:
if self.toolchain.comp_family() == toolchain.GCC: #@UndefinedVariable
self.cfg.update('configopts', 'CPPFLAGS="%s -DgFortran"' % os.getenv('CPPFLAGS'))

super(EB_netCDF, self).configure_step()
Expand Down
6 changes: 3 additions & 3 deletions easybuild/easyblocks/n/netcdf_fortran.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@
import os

import easybuild.tools.environment as env
import easybuild.tools.toolkit as toolchain
import easybuild.tools.toolchain as toolchain
from easybuild.easyblocks.generic.configuremake import ConfigureMake


Expand All @@ -43,13 +43,13 @@ class EB_netCDF_minus_Fortran(ConfigureMake):
def configure_step(self):
"""Configure build: set config options and configure"""

if self.toolchain.opts['pic']:
if self.toolchain.options['pic']:
self.cfg.update('configopts', "--with-pic")

self.cfg.update('configopts', 'FCFLAGS="%s" FC="%s"' % (os.getenv('FFLAGS'), os.getenv('F90')))

# add -DgFortran to CPPFLAGS when building with GCC
if self.toolchain.comp_family() == toolchain.GCC:
if self.toolchain.comp_family() == toolchain.GCC: #@UndefinedVariable
env.setvar('CPPFLAGS', "%s -DgFortran" % os.getenv('CPPFLAGS'))

super(EB_netCDF_minus_Fortran, self).configure_step()
Expand Down
Loading