Updates required for creating binary (build) caches

lib/spack/spack/binary_distribution.py

@@ -1146,6 +1146,7 @@ def _build_tarball(
     allow_root=False,
     key=None,
     regenerate_index=False,
+    skip_on_error=False,
 ):
     """
     Build a tarball from given spec and put it into the directory structure
@@ -1221,15 +1222,23 @@ def _build_tarball(
             shutil.rmtree(workdir)
             shutil.rmtree(tarfile_dir)
             shutil.rmtree(tmpdir)
-            tty.die(e)
+            if skip_on_error:
+                tty.warn('Error while creating buildcache for "{0}", skip: {1}'.format(spec, e))
+                return
+            else:
+                tty.die(e)
     else:
         try:
             check_package_relocatable(workdir, spec, allow_root)
         except Exception as e:
             shutil.rmtree(workdir)
             shutil.rmtree(tarfile_dir)
             shutil.rmtree(tmpdir)
-            tty.die(e)
+            if skip_on_error:
+                tty.warn('Error while creating buildcache for "{0}", skip: {1}'.format(spec, e))
+                return
+            else:
+                tty.die(e)
 
     # create gzip compressed tarball of the install prefix
     # On AMD Ryzen 3700X and an SSD disk, we have the following on compression speed:
@@ -1353,6 +1362,7 @@ def push(specs, push_url, specs_kwargs=None, **kwargs):
     # TODO: This seems to be an easy target for task
     # TODO: distribution using a parallel pool
     for node in nodes:
+        tty.msg('Creating buildcache for "{0}"'.format(node.format()))
         try:
             _build_tarball(node, push_url, **kwargs)
         except NoOverwriteException as e:
@@ -1609,7 +1619,7 @@ def dedupe_hardlinks_if_necessary(root, buildinfo):
         buildinfo[key] = new_list
 
 
-def relocate_package(spec, allow_root):
+def relocate_package(spec):
     """
     Relocate the given package
     """
@@ -1883,7 +1893,7 @@ def extract_tarball(spec, download_result, allow_root=False, unsigned=False, for
     os.remove(specfile_path)
 
     try:
-        relocate_package(spec, allow_root)
+        relocate_package(spec)
     except Exception as e:
         shutil.rmtree(spec.prefix)
         raise e

lib/spack/spack/cmd/buildcache.py

@@ -111,6 +111,17 @@ def setup_parser(subparser):
             " or only the dependencies"
         ),
     )
+    create.add_argument(
+        "--skip-on-error",
+        action="store_true",
+        default=False,
+        help=(
+            "Ignore errors when creating buildcaches."
+            " This option is useful when creating buildcaches"
+            " for multiple specs at once (e.g. all specs in"
+            " an environment)"
+        ),
+    )
     arguments.add_common_arguments(create, ["specs"])
     create.set_defaults(func=create_fn)
 
@@ -399,6 +410,7 @@ def create_fn(args):
         "unsigned": args.unsigned,
         "allow_root": args.allow_root,
         "regenerate_index": args.rebuild_index,
+        "skip_on_error": args.skip_on_error,
     }
     bindist.push(matches, push_url, specs_kwargs, **kwargs)
 

share/spack/spack-completion.bash

@@ -498,7 +498,7 @@ _spack_buildcache() {
 _spack_buildcache_create() {
     if $list_options
     then
-        SPACK_COMPREPLY="-h --help -r --rel -f --force -u --unsigned -a --allow-root -k --key -d --directory -m --mirror-name --mirror-url --rebuild-index --spec-file --only"
+        SPACK_COMPREPLY="-h --help -r --rel -f --force -u --unsigned -a --allow-root -k --key -d --directory -m --mirror-name --mirror-url --rebuild-index --spec-file --only --skip-on-error"
     else
         _all_packages
     fi

var/spack/repos/builtin/packages/ecflow/package.py

@@ -85,3 +85,11 @@ def cmake_args(self):
             self.define("Python3_EXECUTABLE", self.spec["python"].package.command),
             self.define("BOOST_ROOT", self.spec["boost"].prefix),
         ]
+
+    # A recursive link in the ecflow source code causes the binary cache
+    # creation to fail. This file is only in the install tree if the
+    # --source option is set when installing the package, but force_remove
+    # acts like "rm -f" and won't abort if the file doesn't exist.
+    @run_after("install")
+    def remove_recursive_symlink_in_source_code(self):
+        force_remove(join_path(self.prefix, "share/ecflow/src/cereal/cereal"))