Merge branch 'develop' into allinea-templates+system.config-handling

.travis.yml

@@ -2,16 +2,20 @@ language: python
 python: 2.6
 env:
   matrix:
-    - ENV_MOD_VERSION=3.2.10
-    - LMOD_VERSION=5.6.3 EASYBUILD_MODULES_TOOL=Lmod
-    - LMOD_VERSION=6.3.1 EASYBUILD_MODULES_TOOL=Lmod EASYBUILD_MODULE_SYNTAX=Lua
+    - ENV_MOD_VERSION=3.2.10 EASYBUILD_MODULES_TOOL=EnvironmentModulesC EASYBUILD_MODULE_SYNTAX=Tcl
+    - LMOD_VERSION=5.8
+    - LMOD_VERSION=5.8 EASYBUILD_MODULE_SYNTAX=Lua
+    - LMOD_VERSION=6.6.3
+    - LMOD_VERSION=6.6.3 EASYBUILD_MODULE_SYNTAX=Lua
+    - LMOD_VERSION=7.0
+    - LMOD_VERSION=7.0 EASYBUILD_MODULE_SYNTAX=Lua
 matrix:
   # mark build as finished as soon as job has failed
   fast_finish: true
   include:
     # also test default configuration with Python 2.7
     - python: 2.7
-      env: ENV_MOD_VERSION=3.2.10
+      env: LMOD_VERSION=5.8
 addons:
   apt:
     packages:

RELEASE_NOTES

@@ -3,7 +3,45 @@ For more detailed information, please see the git log.
 
 These release notes can also be consulted at http://easybuild.readthedocs.org/en/latest/Release_notes.html.
 
-The latest version of easybuild-easyblocks provides 180 software-specific easyblocks and 29 generic easyblocks.
+The latest version of easybuild-easyblocks provides 179 software-specific easyblocks and 30 generic easyblocks.
+
+v3.0.0 (November 16th 2016)
+---------------------------
+
+feature release
+- backwards incompatible changes:
+  - remove deprecated GenomeAnalysisTK/GATK easyblock (#1001)
+  - remove deprecated 'get_netcdf_module_set_cmds' function from netCDF easyblock (#1015)
+  - remove deprecated 'get_blas_lib' function from LAPACK easyblock (#1016)
+  - remove QLogicMPI easyblock (#1023)
+- new easyblock for installing Anaconda (#950)
+- add generic easyblock for Conda installations (#950)
+- various enhancements, including:
+  - enable use of GCCcore as toolchain for Clang, fail if no GCC prefix is found (#1002)
+  - also build Boost MPI library in parallel (#1005, #1038)
+  - enhance g2clib easyblock to allow to install 1.6.0 or higher (#1006)
+  - update QuantumESPRESSO easyblock to support packaging changes in 6.0 (#1007)
+  - add support to Scons generic easyblock to provide argument to specify installation prefix (#1008)
+  - update IntelBase and imkl easyblocks to handle the 2017 versions of compilers/imkl (#1012)
+  - leverage toolchain.linalg functionality in ScaLAPACK easyblock, use copy_file (#1014)
+  - allow netCDF-C++4 to be used with ESMF (#1019)
+  - update Advisor easyblock to support latest versions (#1021)
+  - update CBLAS easyblock to build with foss toolchain (#1024)
+  - update Gurobi easyblock to use copy_file (#1028)
+  - add support for giving /lib preference over /lib64 & co in GCC installation (#1030, #1035)
+  - enable installation of libiberty by default for binutils (#1030)
+  - avoid CMake fiddling with the RPATHs injected by EasyBuild via --rpath in CMakeMake and METIS easyblocks (#1031, #1034)
+  - simplify scipy sanity check to make it more robust w.r.t. version updates (#1037)
+- various bug fixes, including:
+  - make sure 'None' doesn't appear in modules generated with --module-only (#998)
+  - fix ATLAS easyblock for non-x86 systems (#1003)
+  - fix 'usempi' and 'with_mpi' usage to allow for a serial build of Amber 16 (#1013)
+  - add both lib/python2.7/site-packages/{,wx-3.0-gtk2} to $PYTHONPATH for wxPython (#1018)
+  - only hard inject RPATH for /usr/lib* directories when building binutils with dummy toolchain (#1026)
+  - make HDF5 easyblock handle --filter-deps correctly (#1027)
+  - update Travis config w.r.t. changes framework config defaults and required Lmod version (#1029)
+  - be more patient when running Mathematica Q&A installer (#1036)
+
 
 v2.9.0 (September 23rd 2016)
 ----------------------------
@@ -803,7 +841,7 @@ v0.8 (June 29th 2012)
 ---------------------
 
 - added support for building/installing 17 additional software packages:
-    - BEAGLE, Doxygen, g2clib, g2lib, HDF, HDF5, JasPer, libpng, Maple, 
+    - BEAGLE, Doxygen, g2clib, g2lib, HDF, HDF5, JasPer, libpng, Maple,
       MrBayes, NCL, netCDF, netCDF-Fortran, Szip, WPS, WRF, zlib
 - the build procedure for WRF and WPS includes running the tests available for these packages
 - various bug fixes and enhancements:

easybuild/easyblocks/__init__.py

@@ -44,7 +44,7 @@
 # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like
 #   UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0'
 # This causes problems further up the dependency chain...
-VERSION = LooseVersion('3.0.0.dev0')
+VERSION = LooseVersion('3.1.0.dev0')
 UNKNOWN = 'UNKNOWN'
 
 

easybuild/easyblocks/a/advisor.py

@@ -26,21 +26,32 @@
 EasyBuild support for installing the Intel Advisor XE, implemented as an easyblock
 
 @author: Lumir Jasiok (IT4Innovations)
+@author: Damian Alvarez (Forschungszentrum Juelich GmbH)
 """
 
+from distutils.version import LooseVersion
+
 from easybuild.easyblocks.generic.intelbase import IntelBase
 
 class EB_Advisor(IntelBase):
     """
     Support for installing Intel Advisor XE
     """
 
+    def __init__(self, *args, **kwargs):
+        """Constructor, initialize class variables."""
+        super(EB_Advisor, self).__init__(*args, **kwargs)
+        if LooseVersion(self.version) < LooseVersion('2017'):
+            self.base_path = 'advisor_xe'
+        else:
+            self.base_path = 'advisor'
+
     def sanity_check_step(self):
         """Custom sanity check paths for Advisor"""
 
         custom_paths = {
             'files': [],
-            'dirs': ['advisor_xe/bin64', 'advisor_xe/lib64']
+            'dirs': ['%s/bin64' % self.base_path, '%s/lib64' % self.base_path]
         }
 
         super(EB_Advisor, self).sanity_check_step(custom_paths=custom_paths)
@@ -51,15 +62,15 @@ def make_module_req_guess(self):
         """
         guesses = super(EB_Advisor, self).make_module_req_guess()
 
-        lib_path = 'advisor_xe/lib64'
-        include_path = 'advisor_xe/include'
+        lib_path = '%s/lib64' % self.base_path
+        include_path = '%s/include' % self.base_path
 
         guesses.update({
             'CPATH': [include_path],
             'INCLUDE': [include_path],
             'LD_LIBRARY_PATH': [lib_path],
             'LIBRARY_PATH': [lib_path],
-            'PATH': ['advisor_xe/bin64'],
+            'PATH': ['%s/bin64' % self.base_path],
         })
 
         return guesses

easybuild/easyblocks/a/anaconda.py

@@ -0,0 +1,76 @@
+##
+# Copyright 2009-2016 Ghent University
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# https://github.com/hpcugent/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild.  If not, see <http://www.gnu.org/licenses/>.
+##
+"""
+EasyBuild support for building and installing Anaconda, implemented as an easyblock
+
+@author: Jillian Rowe (New York University Abu Dhabi)
+@author: Kenneth Hoste (HPC-UGent)
+"""
+
+import os
+import stat
+
+import easybuild.tools.environment as env
+from easybuild.easyblocks.generic.binary import Binary
+from easybuild.framework.easyconfig import CUSTOM
+from easybuild.tools.build_log import EasyBuildError
+from easybuild.tools.filetools import adjust_permissions, rmtree2
+from easybuild.tools.run import run_cmd
+
+
+class EB_Anaconda(Binary):
+    """Support for building/installing Anaconda."""
+
+    def install_step(self):
+        """Copy all files in build directory to the install directory"""
+
+        rmtree2(self.installdir)
+        install_script = self.src[0]['name']
+
+        adjust_permissions(os.path.join(self.builddir, install_script), stat.S_IRUSR|stat.S_IXUSR)
+
+        cmd = "%s ./%s -p %s -b -f" % (self.cfg['preinstallopts'], install_script, self.installdir)
+        self.log.info("Installing %s using command '%s'..." % (self.name, cmd))
+        run_cmd(cmd, log_all=True, simple=True)
+
+    def make_module_req_guess(self):
+        """
+        A dictionary of possible directories to look for.
+        """
+        return {
+            'MANPATH': ['man', os.path.join('share', 'man')],
+            'PATH': ['bin', 'sbin'],
+            'PKG_CONFIG_PATH': [os.path.join(x, 'pkgconfig') for x in ['lib', 'lib32', 'lib64', 'share']],
+        }
+
+    def sanity_check_step(self):
+        """
+        Custom sanity check for Anaconda
+        """
+        custom_paths = {
+            'files': [os.path.join('bin', x) for x in ['2to3', 'conda', 'ipython', 'pydoc', 'python', 'sqlite3']],
+            'dirs': ['bin', 'etc', 'lib', 'pkgs'],
+        }
+        super(EB_Anaconda, self).sanity_check_step(custom_paths=custom_paths)

easybuild/easyblocks/b/binutils.py

@@ -33,35 +33,49 @@
 from distutils.version import LooseVersion
 
 from easybuild.easyblocks.generic.configuremake import ConfigureMake
+from easybuild.framework.easyconfig import CUSTOM
 from easybuild.tools.build_log import EasyBuildError
+from easybuild.tools.filetools import apply_regex_substitutions, copy_file
 from easybuild.tools.modules import get_software_libdir, get_software_root
-from easybuild.tools.filetools import apply_regex_substitutions
 from easybuild.tools.run import run_cmd
 from easybuild.tools.systemtools import get_shared_lib_ext
+from easybuild.tools.toolchain import DUMMY_TOOLCHAIN_NAME
 
 
 class EB_binutils(ConfigureMake):
     """Support for building/installing binutils."""
 
+    @staticmethod
+    def extra_options(extra_vars=None):
+        """Extra easyconfig parameters specific to the binutils easyblock."""
+        extra_vars = ConfigureMake.extra_options(extra_vars=extra_vars)
+        extra_vars.update({
+            'install_libiberty': [True, "Also install libiberty (implies building with -fPIC)", CUSTOM],
+        })
+        return extra_vars
+
     def configure_step(self):
         """Custom configuration procedure for binutils: statically link to zlib, configure options."""
 
-        # determine list of 'lib' directories to use rpath for;
-        # this should 'harden' the resulting binutils to bootstrap GCC (no trouble when other libstdc++ is build etc)
-        libdirs = []
-        for libdir in ['/usr/lib', '/usr/lib64', '/usr/lib/x86_64-linux-gnu/']:
-            # also consider /lib, /lib64
-            alt_libdir = libdir.replace('usr/', '')
+        libs = ''
 
-            if os.path.exists(libdir):
-                libdirs.append(libdir)
-                if os.path.exists(alt_libdir) and not os.path.samefile(libdir, alt_libdir):
-                    libdirs.append(alt_libdir)
+        if self.toolchain.name == DUMMY_TOOLCHAIN_NAME:
+            # determine list of 'lib' directories to use rpath for;
+            # this should 'harden' the resulting binutils to bootstrap GCC (no trouble when other libstdc++ is build etc)
+            libdirs = []
+            for libdir in ['/usr/lib', '/usr/lib64', '/usr/lib/x86_64-linux-gnu/']:
+                # also consider /lib, /lib64
+                alt_libdir = libdir.replace('usr/', '')
+
+                if os.path.exists(libdir):
+                    libdirs.append(libdir)
+                    if os.path.exists(alt_libdir) and not os.path.samefile(libdir, alt_libdir):
+                        libdirs.append(alt_libdir)
 
-            elif os.path.exists(alt_libdir):
-                libdirs.append(alt_libdir)
+                elif os.path.exists(alt_libdir):
+                    libdirs.append(alt_libdir)
 
-        libs = ' '.join('-Wl,-rpath=%s' % libdir for libdir in libdirs)
+            libs += ' '.join('-Wl,-rpath=%s' % libdir for libdir in libdirs)
 
         # statically link to zlib if it is a (build) dependency
         zlibroot = get_software_root('zlib')
@@ -99,6 +113,34 @@ def configure_step(self):
         # complete configuration with configure_method of parent
         super(EB_binutils, self).configure_step()
 
+        if self.cfg['install_libiberty']:
+            cflags = os.getenv('CFLAGS')
+            if cflags:
+                self.cfg.update('buildopts', 'CFLAGS="$CFLAGS -fPIC"')
+            else:
+                # if $CFLAGS is not defined, make sure we retain "-g -O2",
+                # since not specifying any optimization level implies -O0...
+                self.cfg.update('buildopts', 'CFLAGS="-g -O2 -fPIC"')
+
+    def install_step(self):
+        """Install using 'make install', also install libiberty if desired."""
+        super(EB_binutils, self).install_step()
+
+        # only install libiberty files if if they're not there yet;
+        # libiberty.a is installed by default for old binutils versions
+        if self.cfg['install_libiberty']:
+            if not os.path.exists(os.path.join(self.installdir, 'include', 'libiberty.h')):
+                copy_file(os.path.join(self.cfg['start_dir'], 'include', 'libiberty.h'),
+                          os.path.join(self.installdir, 'include', 'libiberty.h'))
+
+            if not glob.glob(os.path.join(self.installdir, 'lib*', 'libiberty.a')):
+                copy_file(os.path.join(self.cfg['start_dir'], 'libiberty', 'libiberty.a'),
+                          os.path.join(self.installdir, 'lib', 'libiberty.a'))
+
+            if not os.path.exists(os.path.join(self.installdir, 'info', 'libiberty.texi')):
+                copy_file(os.path.join(self.cfg['start_dir'], 'libiberty', 'libiberty.texi'),
+                          os.path.join(self.installdir, 'info', 'libiberty.texi'))
+
     def sanity_check_step(self):
         """Custom sanity check for binutils."""
 
@@ -122,6 +164,12 @@ def sanity_check_step(self):
             'dirs': [],
         }
 
+        if self.cfg['install_libiberty']:
+            custom_paths['files'].extend([
+                (os.path.join('lib', 'libiberty.a'), os.path.join('lib64', 'libiberty.a')),
+                os.path.join('include', 'libiberty.h'),
+            ])
+
         # if zlib is listed as a dependency, it should get linked in statically
         if get_software_root('zlib'):
             for binary in binaries:

easybuild/easyblocks/b/boost.py

@@ -178,7 +178,7 @@ def build_step(self):
 
             # build mpi lib first
             # let bjam know about the user-config.jam file we created in the configure step
-            run_cmd("./bjam %s" % bjammpioptions, log_all=True, simple=True)
+            run_cmd("./bjam %s %s" % (bjammpioptions, paracmd), log_all=True, simple=True)
 
             # boost.mpi was built, let's 'install' it now
             run_cmd("./bjam %s  install %s" % (bjammpioptions, paracmd), log_all=True, simple=True)

easybuild/easyblocks/e/esmf.py

@@ -91,7 +91,11 @@ def configure_step(self):
             if netcdfcxx:
                 netcdf_libs = ["-L%s/lib" % netcdfcxx] + netcdf_libs + ["-lnetcdf_c++"]
             else:
-                netcdf_libs.append('-lnetcdf_c++')
+                netcdfcxx = get_software_root('netCDF-C++4')
+                if netcdfcxx:
+                    netcdf_libs = ["-L%s/lib" % netcdfcxx] + netcdf_libs + ["-lnetcdf_c++4"]
+                else:
+                    netcdf_libs.append('-lnetcdf_c++')
 
             env.setvar('ESMF_NETCDF_LIBS', ' '.join(netcdf_libs))
 

easybuild/easyblocks/g/g2clib.py

@@ -40,6 +40,8 @@
 from easybuild.tools.build_log import EasyBuildError
 from easybuild.tools.modules import get_software_root
 
+from distutils.version import LooseVersion
+
 
 class EB_g2clib(ConfigureMake):
     """Support for building g2clib GRIB2 C library."""
@@ -68,10 +70,20 @@ def install_step(self):
             # copy library
             targetdir = os.path.join(self.installdir, "lib")
             os.mkdir(targetdir)
-            fn = "libgrib2c.a"
+
+            # not documented AFAIK, but real
+            if LooseVersion(self.version) < LooseVersion('1.6.0'):
+                fn = "libgrib2c.a"
+            else:
+                fn = "libg2c_v%s.a" % self.version
+
             shutil.copyfile(os.path.join(self.cfg['start_dir'], fn),
                             os.path.join(targetdir, fn))
 
+            # make link so other software still finds libgrib2c.a
+            if LooseVersion(self.version) >= LooseVersion('1.6.0'):
+                os.symlink("libg2c_v%s.a" % self.version, os.path.join(targetdir,"libgrib2c.a"))
+
             # copy header files
             targetdir = os.path.join(self.installdir, "include")
             os.mkdir(targetdir)