312 regtest online option

easybuild/build.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 ##
-# Copyright 2009-2012 Stijn De Weirdt, Dries Verdegem, Kenneth Hoste, Pieter De Baets, Jens Timmerman
+# Copyright 2009-2012 Stijn De Weirdt, Dries Verdegem, Kenneth Hoste, Pieter De Baets, Jens Timmerman, Toon Willems
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of the University of Ghent (http://ugent.be/hpc).
@@ -85,6 +85,7 @@ def add_build_options(parser):
     parser.add_option("-d", "--debug" , action="store_true", help="log debug messages")
     parser.add_option("-v", "--version", action="store_true", help="show version")
     parser.add_option("--regtest", action="store_true", help="enable regression test mode")
+    parser.add_option("--regtest-online", action="store_true", help="enable online regression test mode")
 
 def main():
     """
@@ -182,7 +183,7 @@ def main():
             error("Can't find path %s" % path)
 
         try:
-            packages.extend(findEasyconfigs(path, log, blocks))
+            packages.extend(findEasyconfigs(path, log, blocks, options.regtest_online))
         except IOError, err:
             log.error("Processing easyconfigs in path %s failed: %s" % (path, err))
 
@@ -247,12 +248,12 @@ def error(message, exitCode=1, optparser=None):
         optparser.print_help()
     sys.exit(exitCode)
 
-def findEasyconfigs(path, log, onlyBlocks=None):
+def findEasyconfigs(path, log, onlyBlocks=None, regtest_online=False):
     """
     Find .eb easyconfig files in path and process them
     """
     if os.path.isfile(path):
-        return processEasyconfig(path, log, onlyBlocks)
+        return processEasyconfig(path, log, onlyBlocks, regtest_online)
 
     ## Walk through the start directory, retain all files that end in .eb
     files = []
@@ -268,10 +269,10 @@ def findEasyconfigs(path, log, onlyBlocks=None):
 
     packages = []
     for filename in files:
-        packages.extend(processEasyconfig(filename, log, onlyBlocks))
+        packages.extend(processEasyconfig(filename, log, onlyBlocks, regtest_online))
     return packages
 
-def processEasyconfig(path, log, onlyBlocks=None):
+def processEasyconfig(path, log, onlyBlocks=None, regtest_online=False):
     """
     Process easyconfig, returning some information for each block
     """
@@ -285,7 +286,7 @@ def processEasyconfig(path, log, onlyBlocks=None):
 
         try:
             app = Application(debug=LOGDEBUG)
-            app.process_ebfile(spec)
+            app.process_ebfile(spec, regtest_online)
         except EasyBuildError, err:
             msg = "Failed to process easyconfig %s:\n%s" % (spec, err.msg)
             log.exception(msg)
@@ -576,7 +577,7 @@ def build(module, options, log, origEnviron, exitOnFailure=True):
     # timing info
     starttime = time.time()
     try:
-        result = app.autobuild(spec, runTests=not options.skip_tests)
+        result = app.autobuild(spec, runTests=not options.skip_tests, regtest_online=options.regtest_online)
     except EasyBuildError, err:
         lastn = 300
         errormsg = "autoBuild Failed (last %d chars): %s" % (lastn, err.msg[-lastn:])
@@ -671,7 +672,7 @@ def build(module, options, log, origEnviron, exitOnFailure=True):
     ## Check for errors
     if exitCode > 0 or filetools.errorsFoundInLog > 0:
         print_msg("\nWARNING: Build exited with exit code %d. %d possible error(s) were detected in the " \
-                  "build logs, please verify the build.\n" % (exitCode, filetools.errorsFoundInLog), 
+                  "build logs, please verify the build.\n" % (exitCode, filetools.errorsFoundInLog),
                   log)
 
     if app.postmsg:

easybuild/easybuild_config.py

@@ -1,5 +1,5 @@
 ##
-# Copyright 2009-2012 Stijn De Weirdt, Dries Verdegem, Kenneth Hoste, Pieter De Baets, Jens Timmerman
+# Copyright 2009-2012 Stijn De Weirdt, Dries Verdegem, Kenneth Hoste, Pieter De Baets, Jens Timmerman, Toon Willems
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of the University of Ghent (http://ugent.be/hpc).

easybuild/framework/application.py

@@ -1,5 +1,5 @@
 ##
-# Copyright 2009-2012 Stijn De Weirdt, Dries Verdegem, Kenneth Hoste, Pieter De Baets, Jens Timmerman
+# Copyright 2009-2012 Stijn De Weirdt, Dries Verdegem, Kenneth Hoste, Pieter De Baets, Jens Timmerman, Toon Willems
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of the University of Ghent (http://ugent.be/hpc).
@@ -23,6 +23,7 @@
 import glob
 import grp #@UnresolvedImport
 import os
+import re
 import shutil
 import time
 import urllib
@@ -153,11 +154,11 @@ def __init__(self, name=None, version=None, newBuild=True, debug=False):
         # mandatory config entries
         self.mandatory = ['name', 'version', 'homepage', 'description', 'toolkit']
 
-    def autobuild(self, ebfile, runTests):
+    def autobuild(self, ebfile, runTests, regtest_online):
         """
         Build the software package described by cfg.
         """
-        self.process_ebfile(ebfile)
+        self.process_ebfile(ebfile, regtest_online)
 
         if self.getcfg('stop') and self.getcfg('stop') == 'cfg':
             return True
@@ -349,9 +350,9 @@ def parse_dependency(self, dep):
 
         return result
 
-    ## process EasyBuild spec file 
+    ## process EasyBuild spec file
 
-    def process_ebfile(self, fn):
+    def process_ebfile(self, fn, regtest_online=False):
         """
         Read file fn, eval and add info
         - assume certain predefined variable names
@@ -397,16 +398,7 @@ def process_ebfile(self, fn):
                 self.setcfg(k, locs[k])
                 self.log.info("Using cfg option %s: value %s" % (k, self.getcfg(k)))
 
-        for k in self.mandatory:
-            if not k in locs:
-                self.log.error("No cfg option %s provided" % k)
-
-        if self.getcfg('stop') and not (self.getcfg('stop') in self.validstops):
-            self.log.error("Stop provided %s is not valid: %s" % (self.cfg['stop'], self.validstops))
-
-        if not (self.getcfg('moduleclass') in self.validmoduleclasses):
-            self.log.error("Moduleclass provided %s is not valid: %s" % (self.cfg['moduleclass'], self.validmoduleclasses))
-
+        # NOTE: this option ('cfg') cannot be provided on the commandline because it will not yet be set by Easybuild
         if self.getcfg('stop') == 'cfg':
             self.log.info("Stopping in parsing cfg")
             return
@@ -428,8 +420,6 @@ def process_ebfile(self, fn):
             self.log.debug("toolkit: %s" % self.getcfg('toolkit'))
             tk = self.getcfg('toolkit')
             self.settoolkit(tk['name'], tk['version'])
-        else:
-            self.log.error('no toolkit defined')
 
         if self.getcfg('toolkitopts'):
             self.tk.setOptions(self.getcfg('toolkitopts'))
@@ -447,6 +437,29 @@ def process_ebfile(self, fn):
         self.setparallelism()
 
         self.make_installversion()
+        self.verify_config(regtest_online)
+
+    def verify_config(self, regtest_online=False):
+        """
+        verify the config settings
+        """
+        for k in self.mandatory:
+            if not self.getcfg(k):
+                self.log.error("No cfg option %s provided" % k)
+
+        if self.getcfg('stop') and not (self.getcfg('stop') in self.validstops):
+            self.log.error("Stop provided %s is not valid: %s" % (self.cfg['stop'], self.validstops))
+
+        if not (self.getcfg('moduleclass') in self.validmoduleclasses):
+            self.log.error("Moduleclass provided %s is not valid: %s" % (self.cfg['moduleclass'], self.validmoduleclasses))
+
+        if not self.getcfg('toolkit'):
+            self.log.error('no toolkit defined')
+
+        if regtest_online and not self.verify_homepage():
+            self.log.error("Homepage (%s) does not seem to contain anything relevant to %s" % (self.getcfg("homepage"),
+                                                                                               self.name()))
+
 
     def getcfg(self, key):
         """
@@ -499,7 +512,7 @@ def check_osdeps(self, osdeps):
         else:
             self.log.error("One or more OS dependencies were not found: %s" % not_found)
 
-    ## BUILD 
+    ## BUILD
 
     def ready2build(self):
         """
@@ -674,7 +687,7 @@ def download(filename, url, path):
             if foundfile:
                 return foundfile
             else:
-                # try and download source files from specified source URLs 
+                # try and download source files from specified source URLs
                 sourceURLs = self.getcfg('sourceURLs')
                 targetdir = candidate_filepaths[0]
                 if not os.path.isdir(targetdir):
@@ -720,6 +733,36 @@ def download(filename, url, path):
 
                 self.log.error("Couldn't find file %s anywhere, and downloading it didn't work either...\nPaths attempted (in order): %s " % (filename, ', '.join(failedpaths)))
 
+
+    def verify_homepage(self):
+        """
+        Download homepage, verify if the name of the software is mentioned
+        """
+        homepage = self.getcfg("homepage")
+
+        try:
+            page = urllib.urlopen(homepage)
+        except IOError:
+            self.log.error("Homepage (%s) is unavailable." % homepage)
+            return False
+
+        regex = re.compile(self.name(), re.I)
+
+        # if url contains software name and is available we are satisfied
+        if regex.search(homepage):
+            return True
+
+        # Perform a lowercase compare against the entire contents of the html page
+        # (does not care about html)
+        for line in page:
+            if regex.search(line):
+                return True
+
+        return False
+
+
+
+
     def apply_patch(self, beginpath=None):
         """
         Apply the patches
@@ -877,8 +920,8 @@ def postproc(self):
     def cleanup(self):
         """
         Cleanup leftover mess: remove/clean build directory
-        
-        except when we're building in the installation directory, 
+
+        except when we're building in the installation directory,
         otherwise we remove the installation
         """
         if not self.build_in_installdir:
@@ -891,7 +934,7 @@ def cleanup(self):
     def sanitycheck(self):
         """
         Do a sanity check on the installation
-        - if *any* of the files/subdirectories in the installation directory listed 
+        - if *any* of the files/subdirectories in the installation directory listed
           in sanityCheckPaths are non-existent (or empty), the sanity check fails
         """
         # prepare sanity check paths
@@ -924,7 +967,7 @@ def sanitycheck(self):
                 self.log.debug("Sanity check: found file %s in %s" % (f, self.installdir))
 
         if self.sanityCheckOK:
-            # check if directories exist, and whether they are non-empty     
+            # check if directories exist, and whether they are non-empty
             for d in self.sanityCheckPaths['dirs']:
                 p = os.path.join(self.installdir, d)
                 if not os.path.isdir(p) or not os.listdir(p):
@@ -1010,7 +1053,7 @@ def make_builddir(self):
         """
         if not self.build_in_installdir:
             # make a unique build dir
-            ## if a tookitversion starts with a -, remove the - so prevent a -- in the path name 
+            ## if a tookitversion starts with a -, remove the - so prevent a -- in the path name
             tkversion = self.tk.version
             if tkversion.startswith('-'):
                 tkversion = tkversion[1:]
@@ -1228,7 +1271,7 @@ def make_module_extra_packages(self):
     def packages(self):
         """
         After make install, run this.
-        - only if variable len(pkglist) > 0 
+        - only if variable len(pkglist) > 0
         - optionally: load module that was just created using temp module file
         - find source for packages, in pkgs
         - run extraPackages
@@ -1281,7 +1324,7 @@ def find_package_patches(self, pkgName):
 
     def find_package_sources(self):
         """
-        Find source file for packages. 
+        Find source file for packages.
         """
         pkgSources = []
         for pkg in self.getcfg('pkglist'):
@@ -1384,7 +1427,7 @@ def filter_packages(self):
         """
         Called when self.skip is True
         - use this to detect existing packages and to remove them from self.pkgs
-        - based on initial R version 
+        - based on initial R version
         """
         cmdtmpl = self.getcfg('pkgfilter')[0]
         cmdinputtmpl = self.getcfg('pkgfilter')[1]
@@ -1517,7 +1560,7 @@ def get_instance(easyblock, log, name=None):
     """
     Get instance for a particular application class (or Application)
     """
-    #TODO: create proper factory for this, as explained here 
+    #TODO: create proper factory for this, as explained here
     #http://stackoverflow.com/questions/456672/class-factory-in-python
     try:
         if not easyblock: