Only clone reference repos by default

Force fetch updates with --fetch-repos or -u. This new sensible default
enables aliBuild to be used with no or limited network connectivity.

Author: dennisklein

aliBuild

@@ -75,9 +75,7 @@ def doMain(args, parser):
 
   # Setup build environment.
   if args.action == "init":
-    doInit(setdir=args.develPrefix, configDir=args.configDir,
-           pkgname=args.pkgname, referenceSources=args.referenceSources,
-           dist=args.dist, defaults=args.defaults, dryRun=args.dryRun)
+    doInit(args)
     exit(0)
 
   if args.action == "build":

alibuild_helpers/args.py

@@ -108,6 +108,8 @@ def doParseArgs(star):
                       dest="autoCleanup", action="store_false", default=True)
   build_parser.add_argument("--devel-prefix", "-z", nargs="?", help="Version name to use for development packages. Defaults to branch name.",
                       dest="develPrefix", default=argparse.SUPPRESS)
+  build_parser.add_argument("--fetch-repos", "-u", dest="fetchRepos", default=False,
+                            action="store_true", help="Fetch repository updates")
 
   group = build_parser.add_mutually_exclusive_group()
   group.add_argument("--always-prefer-system", dest="preferSystem", default=False,
@@ -151,6 +153,8 @@ def doParseArgs(star):
                             metavar="FILE", help="Specify which defaults to use")
   init_parser.add_argument("--chdir", "-C", help="Change to the specified directory first",
                            metavar="DIR", dest="chdir", default=DEFAULT_CHDIR)
+  init_parser.add_argument("--fetch-repos", "-u", dest="fetchRepos", default=False,
+                           action="store_true", help="Fetch repository updates")
 
   # Options for the version subcommand
   version_parser.add_argument("--architecture", "-a", dest="architecture",

alibuild_helpers/build.py

@@ -22,7 +22,6 @@
 from alibuild_helpers.log import logger_handler, LogFormatter, ProgressPrint, riemannStream
 from datetime import datetime
 from glob import glob
-from multiprocessing.pool import ThreadPool
 try:
   from collections import OrderedDict
 except ImportError:
@@ -344,33 +343,22 @@ def doBuild(args, parser):
                   "  git pull --rebase\n",
                   pwd=os.getcwd(), star=star()))
 
-  # Prefetch git heads in parallel
-  debug("Prefetching git heads in parallel:")
+  # Clone/update repos
+  for p in [p for p in buildOrder if "source" in specs[p]]:
+    if not args.fetchRepos:
+      specs[p]["reference"] = join(abspath(args.referenceSources), p.lower())
+    if args.fetchRepos or not exists(specs[p]["reference"]):
+      updateReferenceRepos(args.referenceSources, p, specs[p])
 
-  fetchers = list(filter(lambda p: "source" in specs[p], buildOrder))
-  poolConfig = {"processes": min(max(len(fetchers), 1), 16)}
-  pool = ThreadPool(**poolConfig)
-  debug("Created thread pool with config: %s" % poolConfig)
-
-  git_commands = {} # store futures
-  for p in fetchers:
-    # Replace source with local one if we are in development mode.
+  # Retrieve git heads
+  for p in [p for p in buildOrder if "reference" in specs[p]]:
+    cmd = "git ls-remote --heads %s" % specs[p]["reference"]
     if specs[p]["package"] in develPkgs:
       specs[p]["source"] = join(os.getcwd(), specs[p]["package"])
-
-    cmd = "git ls-remote --heads %s" % specs[p]["source"]
-    debug("Enqueued '%s'" % cmd)
-    git_commands[p] = pool.apply_async(getstatusoutput, (cmd,))
-
-  results = {}
-  for p, future in git_commands.items():
-    results[p] = future.get()
-
-  for p, result in results.items():
-    dieOnError(result[0], "Unable to fetch from %s" % specs[p]["source"])
-    specs[p]["git_heads"] = result[1].split("\n")
-
-  debug("Finished prefetching git heads")
+      cmd = "git ls-remote --heads %s" % specs[p]["source"]
+    res, output = getStatusOutputBash(cmd)
+    dieOnError(res, "Error on '%s': %s" % (cmd, output))
+    specs[p]["git_heads"] = output.split("\n")
 
   # Resolve the tag to the actual commit ref, so that
   for p in buildOrder:
@@ -771,9 +759,6 @@ def doBuild(args, parser):
             "Found tarball in %s" % spec["cachedTarball"] or
             "No cache tarballs found")
 
-    # FIXME: Why doing it here? This should really be done before anything else.
-    updateReferenceRepos(args.referenceSources, p, spec)
-
     # Generate the part which sources the environment for all the dependencies.
     # Notice that we guarantee that a dependency is always sourced before the
     # parts depending on it, but we do not guaranteed anything for the order in

alibuild_helpers/init.py

@@ -5,7 +5,7 @@
 from alibuild_helpers.log import dieOnError
 from alibuild_helpers.workarea import updateReferenceRepos
 
-from os.path import basename, join
+from os.path import abspath, basename, join
 import os.path as path
 import os
 try:
@@ -17,83 +17,89 @@ def parsePackagesDefinition(pkgname):
   return [ dict(zip(["name","ver"], y.split("@")[0:2]))
            for y in [ x+"@" for x in list(filter(lambda y: y, pkgname.split(","))) ] ]
 
-def doInit(setdir, configDir, pkgname, referenceSources, dist, defaults, dryRun):
-  assert(pkgname != None)
-  assert(type(dist) == dict)
-  assert(sorted(dist.keys()) == ["repo", "ver"])
-  pkgs = parsePackagesDefinition(pkgname)
+def doInit(args):
+  assert(args.pkgname != None)
+  assert(type(args.dist) == dict)
+  assert(sorted(args.dist.keys()) == ["repo", "ver"])
+  pkgs = parsePackagesDefinition(args.pkgname)
   assert(type(pkgs) == list)
-  if dryRun:
+  if args.dryRun:
     info("This will initialise local checkouts for %s\n"
          "--dry-run / -n specified. Doing nothing." % ",".join(x["name"] for x in pkgs))
     exit(0)
   try:
-    path.exists(setdir) or os.mkdir(setdir)
-    path.exists(referenceSources) or os.makedirs(referenceSources)
+    path.exists(args.develPrefix) or os.mkdir(args.develPrefix)
+    path.exists(args.referenceSources) or os.makedirs(args.referenceSources)
   except OSError as e:
     error(str(e))
     exit(1)
 
   # Fetch recipes first if necessary
-  if path.exists(configDir):
-    warning("using existing recipes from %s" % configDir)
+  if path.exists(args.configDir):
+    warning("using existing recipes from %s" % args.configDir)
   else:
     cmd = format("git clone %(repo)s%(branch)s %(cd)s",
-                 repo=dist["repo"] if ":" in dist["repo"] else "https://github.com/%s" % dist["repo"],
-                 branch=" -b "+dist["ver"] if dist["ver"] else "",
-                 cd=configDir)
+                 repo=args.dist["repo"] if ":" in args.dist["repo"] else "https://github.com/%s" % args.dist["repo"],
+                 branch=" -b "+args.dist["ver"] if args.dist["ver"] else "",
+                 cd=args.configDir)
     debug(cmd)
     err = execute(cmd)
     dieOnError(err!=0, "cannot clone recipes")
 
   # Use standard functions supporting overrides and taps. Ignore all disables
   # and system packages as they are irrelevant in this context
   specs = {}
-  defaultsReader = lambda: readDefaults(configDir, defaults, error)
+  defaultsReader = lambda: readDefaults(args.configDir, args.defaults, error)
   (err, overrides, taps) = parseDefaults([], defaultsReader, debug)
   (_,_,_,validDefaults) = getPackageList(packages=[ p["name"] for p in pkgs ],
                                          specs=specs,
-                                         configDir=configDir,
+                                         configDir=args.configDir,
                                          preferSystem=False,
                                          noSystem=True,
                                          architecture="",
                                          disable=[],
-                                         defaults=defaults,
+                                         defaults=args.defaults,
                                          dieOnError=lambda *x, **y: None,
                                          performPreferCheck=lambda *x, **y: (1, ""),
                                          performRequirementCheck=lambda *x, **y: (0, ""),
-                                         performValidateDefaults=lambda spec : validateDefaults(spec, defaults),
+                                         performValidateDefaults=lambda spec : validateDefaults(spec, args.defaults),
                                          overrides=overrides,
                                          taps=taps,
                                          log=debug)
-  dieOnError(validDefaults and defaults not in validDefaults,
-             "Specified default `%s' is not compatible with the packages you want to build.\n" % defaults +
+  dieOnError(validDefaults and args.defaults not in validDefaults,
+             "Specified default `%s' is not compatible with the packages you want to build.\n" % args.defaults +
              "Valid defaults:\n\n- " +
              "\n- ".join(sorted(validDefaults)))
 
   for p in pkgs:
     spec = specs.get(p["name"])
     dieOnError(spec is None, "cannot find recipe for package %s" % p["name"])
-    dest = join(setdir, spec["package"])
+    dest = join(args.develPrefix, spec["package"])
     writeRepo = spec.get("write_repo", spec.get("source"))
     dieOnError(not writeRepo, "package %s has no source field and cannot be developed" % spec["package"])
     if path.exists(dest):
       warning("not cloning %s since it already exists" % spec["package"])
       continue
     p["ver"] = p["ver"] if p["ver"] else spec.get("tag", spec["version"])
     debug("cloning %s%s for development" % (spec["package"], " version "+p["ver"] if p["ver"] else ""))
-    updateReferenceRepos(referenceSources, spec["package"], spec)
+
+    if not args.fetchRepos:
+      spec["reference"] = path.join(abspath(args.referenceSources), spec["package"].lower())
+
+    if args.fetchRepos or not path.exists(spec["reference"]):
+      updateReferenceRepos(args.referenceSources, spec["package"], spec)
+
     cmd = format("git clone %(readRepo)s%(branch)s --reference %(refSource)s %(cd)s && " +
                  "cd %(cd)s && git remote set-url --push origin %(writeRepo)s",
                  readRepo=spec["source"],
                  writeRepo=writeRepo,
                  branch=" -b "+p["ver"] if p["ver"] else "",
-                 refSource=join(referenceSources, spec["package"].lower()),
+                 refSource=join(args.referenceSources, spec["package"].lower()),
                  cd=dest)
     debug(cmd)
     err = execute(cmd)
     dieOnError(err!=0, "cannot clone %s%s" %
                        (spec["package"], " version "+p["ver"] if p["ver"] else ""))
   banner(format("Development directory %(d)s created%(pkgs)s",
          pkgs=" for "+", ".join([ x["name"].lower() for x in pkgs ]) if pkgs else "",
-         d=setdir))
+         d=args.develPrefix))

docs/user.markdown

@@ -17,7 +17,8 @@ For a quick start introduction, please look [here](./quick.html).
                     [--always-prefer-system] [--no-system]
                     [--force-unknown-architecture] [--insecure]
                     [--aggressive-cleanup] [--debug] [--no-auto-cleanup]
-                    [--devel-prefix [DEVELPREFIX]] [--dist DIST] [--dry-run]
+                    [--devel-prefix [DEVELPREFIX]] [--dist DIST]
+                    [--dry-run] [--fetch-repos|-u]
                     {init,build,clean} [pkgname]
 
     positional arguments:
@@ -62,6 +63,7 @@ For a quick start introduction, please look [here](./quick.html).
                             recipes set ([user/repo@]branch)
       --dry-run, -n         Prints what would happen, without actually doing the
                             build.
+      --fetch-repos, -u     Fetch repository updates
 
 
 ## Speedup build process by using a build store

tests/test_build.py

@@ -87,9 +87,13 @@ def dummy_getstatusoutput(x):
       "GIT_DIR=/alidist/.git git rev-parse HEAD": (0, "6cec7b7b3769826219dfa85e5daa6de6522229a0"),
       'pip --disable-pip-version-check show alibuild | grep -e "^Version:" | sed -e \'s/.* //\'': (0, "v1.5.0"),
       'which pigz': (1, ""),
-      'tar --ignore-failed-read -cvvf /dev/null /dev/zero': (0, ""),
-      'git ls-remote --heads https://github.com/root-mirror/root': (0, "87b87c4322d2a3fad315c919cb2e2dd73f2154dc\trefs/heads/master\nf7b336611753f1f4aaa94222b0d620748ae230c0\trefs/heads/v6-08-00-patches"),
-      'git ls-remote --heads https://github.com/star-externals/zlib': (0, "8822efa61f2a385e0bc83ca5819d608111b2168a\trefs/heads/master")
+      'tar --ignore-failed-read -cvvf /dev/null /dev/zero': (0, "")
+    }[x]
+
+def dummy_getStatusOutputBash(x):
+  return {
+      'git ls-remote --heads /sw/MIRROR/root': (0, "87b87c4322d2a3fad315c919cb2e2dd73f2154dc\trefs/heads/master\nf7b336611753f1f4aaa94222b0d620748ae230c0\trefs/heads/v6-08-00-patches"),
+      'git ls-remote --heads /sw/MIRROR/zlib': (0, "8822efa61f2a385e0bc83ca5819d608111b2168a\trefs/heads/master")
     }[x]
 
 TIMES_ASKED = {}
@@ -127,7 +131,17 @@ def dummy_readlink(x):
 def dummy_exists(x):
   if x.endswith("alibuild_helpers/.git"):
     return False
-  return {"/alidist": True, "/sw/SPECS": False, "/alidist/.git": True, "alibuild_helpers/.git": False}[x]
+  return {
+    "/alidist": True,
+    "/sw/SPECS": False,
+    "/alidist/.git": True,
+    "alibuild_helpers/.git": False,
+    "/sw/MIRROR/root": True,
+    "/sw/MIRROR/zlib": False}[x]
+
+def dummy_reference(referenceSources, p, spec):
+  spec["reference"] = os.path.join(os.path.abspath(referenceSources), p.lower())
+
 
 # A few errors we should handle, together with the expected result
 class BuildTestCase(unittest.TestCase):
@@ -147,7 +161,8 @@ class BuildTestCase(unittest.TestCase):
   @patch("alibuild_helpers.build.glob")
   @patch("alibuild_helpers.build.readlink")
   @patch("alibuild_helpers.build.banner")
-  def test_coverDoBuild(self, mock_banner, mock_readlink, mock_glob, mock_shutil, mock_open,  mock_utilities_open, mock_reference, mock_debug, mock_makedirs, mock_read_defaults, mock_die, mock_sys, mock_exists, mock_getstatusoutput, mock_execute, mock_urlopen):
+  @patch("alibuild_helpers.build.getStatusOutputBash")
+  def test_coverDoBuild(self, mock_getStatusOutputBash, mock_banner, mock_readlink, mock_glob, mock_shutil, mock_open,  mock_utilities_open, mock_reference, mock_debug, mock_makedirs, mock_read_defaults, mock_die, mock_sys, mock_exists, mock_getstatusoutput, mock_execute, mock_urlopen):
     mock_readlink.side_effect = dummy_readlink
     mock_glob.side_effect = lambda x : {"*": ["zlib"],
         "/sw/TARS/osx_x86-64/defaults-release/defaults-release-v1-*.osx_x86-64.tar.gz": ["/sw/TARS/osx_x86-64/defaults-release/defaults-release-v1-1.osx_x86-64.tar.gz"],
@@ -168,6 +183,7 @@ def test_coverDoBuild(self, mock_banner, mock_readlink, mock_glob, mock_shutil,
     mock_execute.side_effect = dummy_execute
     mock_exists.side_effect = dummy_exists
     mock_getstatusoutput.side_effect = dummy_getstatusoutput
+    mock_getStatusOutputBash.side_effect = dummy_getStatusOutputBash
     mock_parser = MagicMock()
     mock_read_defaults.return_value = (OrderedDict({"package": "defaults-release", "disable": []}), "")
     args = Namespace(
@@ -189,12 +205,21 @@ def test_coverDoBuild(self, mock_banner, mock_readlink, mock_glob, mock_shutil,
       aggressiveCleanup=False,
       environment={},
       autoCleanup=False,
-      noDevel=[]
+      noDevel=[],
+      fetchRepos=False
     )
+    mock_reference.side_effect = dummy_reference
     mock_sys.version_info = sys.version_info
     fmt, msg, code = doBuild(args, mock_parser)
+    self.assertEqual(mock_reference.call_count, 1, "Only one repo should have been fetched")
+
+    # Force fetching repos
+    mock_reference.reset_mock()
+    args.fetchRepos = True
+    fmt, msg, code = doBuild(args, mock_parser)
     mock_glob.assert_called_with("/sw/TARS/osx_x86-64/ROOT/ROOT-v6-08-30-*.osx_x86-64.tar.gz")
     self.assertEqual(msg, "Everything done")
+    self.assertEqual(mock_reference.call_count, 2, "Both repos should have been fetched")
 
   @patch("alibuild_helpers.build.urlopen")
   @patch("alibuild_helpers.build.execute")