Download: New --use_singularity_cache option

nf_core/__main__.py

@@ -202,23 +202,30 @@ def launch(pipeline, id, revision, command_only, params_in, params_out, save_all
 @nf_core_cli.command(help_priority=3)
 @click.argument("pipeline", required=True, metavar="<pipeline name>")
 @click.option("-r", "--release", type=str, help="Pipeline release")
-@click.option("-s", "--singularity", is_flag=True, default=False, help="Download singularity containers")
+@click.option("-s", "--singularity", is_flag=True, default=False, help="Download singularity images")
 @click.option("-o", "--outdir", type=str, help="Output directory")
 @click.option(
     "-c",
     "--compress",
     type=click.Choice(["tar.gz", "tar.bz2", "zip", "none"]),
     default="tar.gz",
-    help="Compression type",
+    help="Archive compression type",
 )
 @click.option("-f", "--force", is_flag=True, default=False, help="Overwrite existing files")
-@click.option("-p", "--parallel_downloads", type=int, default=4, help="Number of parallel container downloads")
-def download(pipeline, release, singularity, outdir, compress, force, parallel_downloads):
+@click.option(
+    "-c",
+    "--use_singularity_cache",
+    is_flag=True,
+    default=False,
+    help="Don't copy images to the output directory and don't configure singularity.cacheDir",
+)
+@click.option("-p", "--parallel_downloads", type=int, default=4, help="Number of parallel image downloads")
+def download(pipeline, release, singularity, outdir, compress, force, use_singularity_cache, parallel_downloads):
     """
-    Download a pipeline, configs and singularity container.
+    Download a pipeline, nf-core/configs and pipeline singularity images.
 
-    Collects all workflow files and shared configs from nf-core/configs.
-    Configures the downloaded workflow to use the relative path to the configs.
+    Collects all files in a single archive and configures the downloaded
+    workflow to use relative paths to the configs and singularity images.
     """
     dl = nf_core.download.DownloadWorkflow(pipeline, release, singularity, outdir, compress, force, parallel_downloads)
     dl.download_workflow()

nf_core/download.py

@@ -73,6 +73,7 @@ def __init__(
         outdir=None,
         compress_type="tar.gz",
         force=False,
+        use_singularity_cache=False,
         parallel_downloads=4,
     ):
         self.pipeline = pipeline
@@ -84,6 +85,7 @@ def __init__(
         if self.compress_type == "none":
             self.compress_type = None
         self.force = force
+        self.use_singularity_cache = use_singularity_cache
         self.parallel_downloads = parallel_downloads
 
         self.wf_name = None
@@ -339,7 +341,8 @@ def get_singularity_images(self):
         if len(self.containers) == 0:
             log.info("No container names found in workflow")
         else:
-            os.mkdir(os.path.join(self.outdir, "singularity-images"))
+            if not self.use_singularity_cache:
+                os.mkdir(os.path.join(self.outdir, "singularity-images"))
             if not os.environ.get("NXF_SINGULARITY_CACHEDIR"):
                 log.info(
                     "[magenta]Tip: Set env var $NXF_SINGULARITY_CACHEDIR to use a central cache for container downloads"
@@ -401,8 +404,10 @@ def get_singularity_images(self):
                     try:
                         # Iterate over each threaded download, waiting for them to finish
                         for future in concurrent.futures.as_completed(future_downloads):
-                            if future.exception():
-                                raise future.exception()
+                            try:
+                                future.result()
+                            except Exception:
+                                raise
                             else:
                                 progress.update(task, advance=1)
 
@@ -464,6 +469,12 @@ def singularity_image_filenames(self, container):
         cache_path = None
         if os.environ.get("NXF_SINGULARITY_CACHEDIR"):
             cache_path = os.path.join(os.environ["NXF_SINGULARITY_CACHEDIR"], out_name)
+            # Use only the cache - set this as the main output path
+            if self.use_singularity_cache:
+                out_path = cache_path
+                cache_path = None
+        elif self.use_singularity_cache:
+            raise FileNotFoundError("'--use_singularity_cache' specified but no $NXF_SINGULARITY_CACHEDIR set!")
 
         return (out_path, cache_path)
 
@@ -487,6 +498,14 @@ def singularity_download_image(self, container, out_path, cache_path, progress):
             progress (Progress): Rich progress bar instance to add tasks to.
         """
         log.debug(f"Downloading Singularity image: '{container}'")
+
+        # Check that download directories exist
+        if not os.path.isdir(os.path.dirname(out_path)):
+            raise FileNotFoundError("Output directory not found: '{}'".format(os.path.dirname(out_path)))
+        if cache_path and not os.path.isdir(os.path.dirname(cache_path)):
+            raise FileNotFoundError("Output directory not found: '{}'".format(os.path.dirname(cache_path)))
+
+        # Set output path to save file to
         output_path = cache_path or out_path
         output_path_tmp = None
 
@@ -534,9 +553,11 @@ def singularity_download_image(self, container, out_path, cache_path, progress):
             for t in progress.task_ids:
                 progress.remove_task(t)
             # Try to delete the incomplete download
-            log.warning(f"Deleting incompleted singularity image download:\n'{output_path_tmp}'")
-            os.remove(output_path_tmp)
-            os.remove(output_path)
+            log.debug(f"Deleting incompleted singularity image download:\n'{output_path_tmp}'")
+            if output_path_tmp and os.path.exists(output_path_tmp):
+                os.remove(output_path_tmp)
+            if output_path and os.path.exists(output_path):
+                os.remove(output_path)
             # Re-raise the caught exception
             raise