Merge pull request #367 from alisw/next

Merge next into master in preparation of 1.4.0.rc1

Author: ktf

.travis.yml

@@ -1,5 +1,9 @@
 sudo: required
 dist: trusty
+python:
+  - "2.6"
+  - "2.7"
+  - "3.5"
 services:
   - docker
 cache:
@@ -10,6 +14,7 @@ before_install: |
   cd ..
   git clone -b IB/v5-06/next https://github.com/alisw/alidist
 install: |
+    pip install --user pyyaml
     pip install --user codecov
     pip install --user coverage
 script: |
@@ -36,6 +41,9 @@ script: |
   time coverage run -a alibuild/aliBuild -c alibuild/tests/testdist build broken4 --no-system --disable GCC-Toolchain 2>&1 | grep "Malformed header"
   time coverage run -a alibuild/aliBuild -c alibuild/tests/testdist build broken5 --no-system --disable GCC-Toolchain 2>&1 | grep "Missing package"
   time coverage run -a alibuild/aliBuild -c alibuild/tests/testdist build broken6 --no-system --disable GCC-Toolchain 2>&1 | grep scanning
+  pushd alibuild
+    time coverage run -a tests/test_parseRecipe.py
+  popd
   time coverage run -a alibuild/aliBuild build zlib --no-system --disable GCC-Toolchain
   alibuild/alienv q
   alibuild/alienv setenv zlib/latest -c bash -c '[[ $LD_LIBRARY_PATH == */zlib/* ]]'

README.rst

@@ -16,3 +16,9 @@ Instant gratification with::
 Full documentation at:
 
 https://alisw.github.io/alibuild
+
+Pre-requisites
+==============
+
+If you are using aliBuild directly from git clone, you should make sure
+you have `pyyaml` and `argparse` in your python distribution.

aliBuild

@@ -12,7 +12,7 @@ def format(s, **kwds):
   return s % kwds
 
 def execute(command):
-  popen = Popen(command, shell=(isinstance(command, basestring)), stdout=PIPE, stderr=STDOUT)
+  popen = Popen(command, shell=(isinstance(command, str)), stdout=PIPE, stderr=STDOUT)
   linesIterator = iter(popen.stdout.readline, "")
   for line in linesIterator:
     debug(line.strip("\n"))  # yield line
@@ -60,7 +60,7 @@ def deps(recipesDir, topPackage, outFile, buildRequires, transitiveRed, disable)
 
   with NamedTemporaryFile(delete=False) as fp:
     fp.write("digraph {\n")
-    for p,deps in dot.iteritems():
+    for p,deps in list(dot.items()):
       if selected and not p in selected: continue
       fp.write("  \"%s\";\n" % p)
       for d in deps:

aliDeps

@@ -4,11 +4,15 @@ import os, re
 from os.path import exists, abspath, expanduser
 from glob import glob
 import yaml
-from commands import getstatusoutput
+try:
+  from commands import getstatusoutput
+except ImportError:
+  from subprocess import getstatusoutput
 import logging
 from alibuild_helpers.log import debug, error, banner, info, success, warning
 from alibuild_helpers.log import logger_handler, logger, LogFormatter, ProgressPrint
-from alibuild_helpers.utilities import getPackageList, format, detectArch
+from alibuild_helpers.utilities import getPackageList, format, detectArch, parseDefaults, readDefaults
+from alibuild_helpers.utilities import dockerStatusOutput
 import subprocess
 
 def execute(command):
@@ -27,12 +31,12 @@ def prunePaths(workDir):
     workDirEscaped = re.escape("%s" % workDir) + "[^:]*:?"
     os.environ[x] = re.sub(workDirEscaped, "", os.environ[x])
 
-def checkPreferSystem(spec, cmd, homebrew_replacement):
+def checkPreferSystem(spec, cmd, homebrew_replacement, dockerImage):
     if cmd == "false":
       debug("Package %s can only be managed via alibuild." % spec["package"])
       return (1, "")
     cmd = homebrew_replacement + cmd
-    err, out = execute(cmd)
+    err, out = dockerStatusOutput(cmd, dockerImage=dockerImage, executor=execute)
     if not err:
       success("Package %s will be picked up from the system." % spec["package"])
       for x in out.split("\n"):
@@ -49,12 +53,12 @@ def checkPreferSystem(spec, cmd, homebrew_replacement):
                    error="\n".join(["%s: %s" % (spec["package"],x) for x in out.split("\n")])))
     return (err, "")
 
-def checkRequirements(spec, cmd, homebrew_replacement):
+def checkRequirements(spec, cmd, homebrew_replacement, dockerImage):
     if cmd == "false":
       debug("Package %s is not a system requirement." % spec["package"])
       return (0, "")
     cmd = homebrew_replacement + cmd
-    err, out = execute(cmd)
+    err, out = dockerStatusOutput(cmd, dockerImage=dockerImage, executor=execute)
     if not err:
       success("Required package %s will be picked up from the system." % spec["package"])
       debug(cmd)
@@ -92,16 +96,29 @@ if __name__ == "__main__":
   parser.add_argument("-a", "--architecture", help="force architecture",
                       dest="architecture", default=detectArch())
   parser.add_argument("-c", "--config", help="path to alidist",
-                      dest="config", default="alidist")
+                      dest="configDir", default="alidist")
   parser.add_argument("-w", "--work-dir", help="path to work dir",
                       dest="workDir", default="workDir")
   parser.add_argument("-d", "--debug", help="Show also successful tests.",
                       dest="debug", action="store_true", default=False)
+  parser.add_argument("--defaults", default="release",
+                      dest="defaults", help="Specify default to use")
+  parser.add_argument("--disable", dest="disable", default=[],
+                      metavar="PACKAGE", action="append",
+                      help="Do not build PACKAGE and all its (unique) dependencies.")
+  parser.add_argument("--always-prefer-system", dest="preferSystem", default=False,
+                      action="store_true", help="Always use system packages when compatible")
+  parser.add_argument("--no-system", dest="noSystem", default=False,
+                      action="store_true", help="Never use system packages")
   parser.add_argument("packages", nargs="+", help="Package to test",
                       default=[])
+  parser.add_argument("--docker", dest="docker", action="store_true", default=False)
+  parser.add_argument("--docker-image", dest="dockerImage",
+                      help="Image to use in case you build with docker (implies --docker-image)")
+
   args = parser.parse_args()
-  if not exists(args.config):
-    parser.error("Wrong path to alidist specified: %s" % args.config)
+  if not exists(args.configDir):
+    parser.error("Wrong path to alidist specified: %s" % args.configDir)
 
   prunePaths(abspath(args.workDir))
 
@@ -117,6 +134,11 @@ if __name__ == "__main__":
   if err:
     homebrew_replacement = "brew() { true; }; "
 
+
+  dockerImage = args.dockerImage if "dockerImage" in args else ""
+  if args.docker and not dockerImage:
+    dockerImage = "alisw/%s-builder" % args.architecture.split("_")[0]
+
   logger.setLevel(logging.BANNER)
   if args.debug:
     logger.setLevel(logging.DEBUG)
@@ -125,29 +147,36 @@ if __name__ == "__main__":
   packages = []
   exitcode = 0
   for p in args.packages:
-    path = "%s/%s.sh" % (args.config, p.lower())
+    path = "%s/%s.sh" % (args.configDir, p.lower())
     if not exists(path):
       error("Cannot find recipe %s for package %s." % (path, p))
       exitcode = 1
       continue
     packages.append(p)
-
   systemInfo()
 
   specs = {}
   def unreachable():
     assert(False)
+  defaultsReader = lambda : readDefaults(args.configDir, args.defaults, parser.error)
+  (err, overrides, taps) = parseDefaults(args.disable, defaultsReader, info)
+  if err:
+    error(err)
+    exit(1)
   (fromSystem, own, failed) = getPackageList(packages=packages,
                                          specs=specs,
-                                         configDir=args.config,
-                                         preferSystem=False,
-                                         noSystem=False,
+                                         configDir=args.configDir,
+                                         preferSystem=args.preferSystem,
+                                         noSystem=args.noSystem,
                                          architecture=args.architecture,
-                                         disable=[],
-                                         defaults="release",
+                                         disable=args.disable,
+                                         defaults=args.defaults,
                                          dieOnError=lambda x, y : unreachable,
-                                         performPreferCheck=lambda pkg, cmd : checkPreferSystem(pkg, cmd, homebrew_replacement),
-                                         performRequirementCheck=lambda pkg, cmd : checkRequirements(pkg, cmd, homebrew_replacement))
+                                         performPreferCheck=lambda pkg, cmd : checkPreferSystem(pkg, cmd, homebrew_replacement, dockerImage),
+                                         performRequirementCheck=lambda pkg, cmd : checkRequirements(pkg, cmd, homebrew_replacement, dockerImage),
+                                         overrides=overrides,
+                                         taps=taps,
+                                         log=info)
 
   if fromSystem:
     banner("The following packages will be picked up from the system:\n\n- " +

aliDoctor

@@ -1,4 +1,15 @@
 # Dummy file to package build_template.sh
-import utilities as utilities
-import analytics as analytics
-import log as log
+import sys
+
+# absolute import compatibility check
+if sys.version_info < (3, 0):
+    import utilities
+    import analytics
+    import log
+else:
+    from . import utilities
+    from . import analytics
+    from . import log
+
+# remove non-project imports from this module
+del sys

alibuild_helpers/__init__.py

@@ -1,7 +1,11 @@
 #!/usr/bin/env python
 import os, subprocess, sys
-from commands import getstatusoutput
-from urllib import urlopen
+try:
+  from commands import getstatusoutput
+  from urllib import urlopen
+except ImportError:
+  from subprocess import getstatusoutput
+  from urllib.request import urlopen
 from alibuild_helpers.log import debug, error, banner, info, logger_handler
 from os.path import exists, expanduser
 from os import unlink
@@ -21,7 +25,10 @@ def askForAnalytics():
   banner("In order to improve user experience, aliBuild would like to gather "
          "analytics about your builds.\nYou can find all the details at:\n\n"
          "  https://github.com/alisw/alibuild/blob/master/ANALYTICS.md\n")
-  a = raw_input("Is that ok for you [YES/no]? ")
+  # raw_input and input are different between python 2 and 3
+  try: input = raw_input
+  except NameError: pass
+  a = input("Is that ok for you [YES/no]? ")
   if a.strip() and a.strip().lower().startswith("n"):
     debug("User requsted disabling analytics.")
     return disable_analytics()

alibuild_helpers/analytics.py

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Build script for %(pkgname)s -- automatically generated
+# Automatically generated build script
 
 set -e
 set +h
@@ -9,29 +9,34 @@ export WORK_DIR="${WORK_DIR_OVERRIDE:-%(workDir)s}"
 # From our dependencies
 %(dependencies)s
 
-export CAN_DELETE="%(can_delete)s"
-export PKGNAME="%(pkgname)s"
-export PKGHASH="%(hash)s"
-export PKGVERSION="%(version)s"
-export PKGREVISION="%(revision)s"
-export REQUIRES="%(requires)s"
-export BUILD_REQUIRES="%(build_requires)s"
-export RUNTIME_REQUIRES="%(runtime_requires)s"
-export DEVEL_PREFIX="%(develPrefix)s"
-DEVEL_HASH="%(develHash)s"
-DEPS_HASH="%(depsHash)s"
+# The following environment variables are setup by
+# the aliBuild script itself
+#
+# - ARCHITECTURE
+# - BUILD_REQUIRES
+# - CACHED_TARBALL
+# - CAN_DELETE
+# - COMMIT_HASH
+# - DEPS_HASH
+# - DEVEL_HASH
+# - DEVEL_PREFIX
+# - GIT_TAG
+# - INCREMENTAL_BUILD_HASH
+# - JOBS
+# - PKGHASH
+# - PKGNAME
+# - PKGREVISION
+# - PKGVERSION
+# - REQUIRES
+# - RUNTIME_REQUIRES
+# - WRITE_REPO
 
 export PKG_NAME="$PKGNAME"
 export PKG_VERSION="$PKGVERSION"
 export PKG_BUILDNUM="$PKGREVISION"
 
-export ARCHITECTURE="%(architecture)s"
 export SOURCE0="${SOURCE0_DIR_OVERRIDE:-%(sourceDir)s}%(sourceName)s"
-export GIT_TAG="%(tag)s"
-export JOBS=${JOBS-%(jobs)s}
 export PKGPATH=${ARCHITECTURE}/${PKGNAME}/${PKGVERSION}-${PKGREVISION}
-%(incremental_hash)s
-INCREMENTAL_BUILD_HASH=${INCREMENTAL_BUILD_HASH:-0}
 mkdir -p "$WORK_DIR/BUILD" "$WORK_DIR/SOURCES" "$WORK_DIR/TARS" \
          "$WORK_DIR/SPECS" "$WORK_DIR/INSTALLROOT"
 export BUILDROOT="$WORK_DIR/BUILD/$PKGHASH"
@@ -44,15 +49,15 @@ if [ "${SOURCE0:0:1}" == "/" ]; then
 else
   export INSTALLROOT="$WORK_DIR/INSTALLROOT/$PKGHASH/$PKGPATH"
 fi
-export SOURCEDIR="$WORK_DIR/SOURCES/$PKGNAME/$PKGVERSION/%(commit_hash)s"
+export SOURCEDIR="$WORK_DIR/SOURCES/$PKGNAME/$PKGVERSION/$COMMIT_HASH"
 export BUILDDIR="$BUILDROOT/$PKGNAME"
 
 SHORT_TAG=${GIT_TAG:0:10}
 mkdir -p $(dirname $SOURCEDIR)
-if [[ %(commit_hash)s != ${GIT_TAG} && "${SHORT_TAG:-0}" != %(commit_hash)s ]]; then
+if [[ ${COMMIT_HASH} != ${GIT_TAG} && "${SHORT_TAG:-0}" != ${COMMIT_HASH} ]]; then
   GIT_TAG_DIR=${GIT_TAG:-0}
   GIT_TAG_DIR=${GIT_TAG_DIR//\//_}
-  ln -snf %(commit_hash)s "$WORK_DIR/SOURCES/$PKGNAME/$PKGVERSION/${GIT_TAG_DIR}"
+  ln -snf ${COMMIT_HASH} "$WORK_DIR/SOURCES/$PKGNAME/$PKGVERSION/${GIT_TAG_DIR}"
 fi
 rm -fr "$WORK_DIR/INSTALLROOT/$PKGHASH"
 # We remove the build directory only if we are not in incremental mode.
@@ -75,7 +80,7 @@ if [[ ! "$SOURCE0" == '' && "${SOURCE0:0:1}" != "/" && ! -d "$SOURCEDIR" ]]; the
   git clone ${GIT_REFERENCE:+--reference $GIT_REFERENCE} "$SOURCE0" "$SOURCEDIR"
   cd $SOURCEDIR
   git checkout "${GIT_TAG}"
-  git remote set-url --push origin %(write_repo)s
+  git remote set-url --push origin $WRITE_REPO
 elif [[ ! "$SOURCE0" == '' && "${SOURCE0:0:1}" == "/" ]]; then
   ln -snf $SOURCE0 $SOURCEDIR
 fi
@@ -84,7 +89,6 @@ mkdir -p "$SOURCEDIR"
 cd "$BUILDDIR"
 
 # Actual build script, as defined in the recipe
-CACHED_TARBALL=%(cachedTarball)s
 
 # This actually does the build, taking in to account shortcuts like
 # having a pre build tarball or having an incremental recipe (in the
@@ -112,7 +116,7 @@ else
   # files.
   rm -rf "$BUILDROOT/log"
   mkdir -p $WORK_DIR/TMP/$PKGHASH
-  %(gzip)s -dc $CACHED_TARBALL | tar -C $WORK_DIR/TMP/$PKGHASH -x
+  $MY_GZIP -dc $CACHED_TARBALL | tar -C $WORK_DIR/TMP/$PKGHASH -x
   mkdir -p $(dirname $INSTALLROOT)
   rm -rf $INSTALLROOT
   mv $WORK_DIR/TMP/$PKGHASH/$ARCHITECTURE/$PKGNAME/$PKGVERSION-* $INSTALLROOT
@@ -137,7 +141,7 @@ export ${BIGPKGNAME}_ROOT=$INSTALLROOT
 export ${BIGPKGNAME}_VERSION=$PKGVERSION
 export ${BIGPKGNAME}_REVISION=$PKGREVISION
 export ${BIGPKGNAME}_HASH=$PKGHASH
-export ${BIGPKGNAME}_COMMIT=%(commit_hash)s
+export ${BIGPKGNAME}_COMMIT=${COMMIT_HASH}
 EOF
 
 # Environment
@@ -188,9 +192,12 @@ cd "$WORK_DIR"
 tar xzf "$WORK_DIR/TARS/$HASH_PATH/$PACKAGE_WITH_REV"
 [ "X$CAN_DELETE" = X1 ] && rm "$WORK_DIR/TARS/$HASH_PATH/$PACKAGE_WITH_REV"
 bash -ex "$ARCHITECTURE/$PKGNAME/$PKGVERSION-$PKGREVISION/relocate-me.sh"
+# Last package built gets a "latest" mark.
 ln -snf $PKGVERSION-$PKGREVISION $ARCHITECTURE/$PKGNAME/latest
-if [[ $DEVEL_PREFIX ]]; then
-  ln -snf $PKGVERSION-$PKGREVISION $ARCHITECTURE/$PKGNAME/latest-$DEVEL_PREFIX
+
+# Latest package built for a given devel prefix gets latest-$BUILD_FAMILY
+if [[ $BUILD_FAMILY ]]; then
+  ln -snf $PKGVERSION-$PKGREVISION $ARCHITECTURE/$PKGNAME/latest-$BUILD_FAMILY
 fi
 
 # Mark the build as successful with a placeholder. Allows running incremental

alibuild_helpers/build_template.sh

@@ -23,7 +23,7 @@ def __init__(self, host, port):
       self.client.send({'host': self.currentHost, 'state': 'ok', 'service': "alibuild started"})
       self.enabled = True
       info("Sending log data to %s:%s" % (host, port))
-    except Exception, e:
+    except Exception as e:
       info("RIEMANN_HOST %s:%s specified, however there was a problem initialising:"  % (host, port))
       info(e)