Migrate component create to nf test

nf_core/components/create.py

@@ -10,11 +10,12 @@
 import os
 import re
 import subprocess
+from pathlib import Path
+from typing import Optional
 
 import jinja2
 import questionary
 import rich
-import yaml
 from packaging.version import parse as parse_version
 
 import nf_core
@@ -27,16 +28,16 @@
 class ComponentCreate(ComponentCommand):
     def __init__(
         self,
-        component_type,
-        directory=".",
-        component="",
-        author=None,
-        process_label=None,
-        has_meta=None,
-        force=False,
-        conda_name=None,
-        conda_version=None,
-        empty_template=False,
+        component_type: str,
+        directory: str = ".",
+        component: str = "",
+        author: Optional[str] = None,
+        process_label: Optional[str] = None,
+        has_meta: Optional[str] = None,
+        force: bool = False,
+        conda_name: Optional[str] = None,
+        conda_version: Optional[str] = None,
+        empty_template: bool = False,
     ):
         super().__init__(component_type, directory)
         self.directory = directory
@@ -71,23 +72,23 @@ def create(self):
 
         If <directory> is a pipeline, this function creates a file called:
         '<directory>/modules/local/tool.nf'
-          OR
+            OR
         '<directory>/modules/local/tool_subtool.nf'
-          OR for subworkflows
+            OR for subworkflows
         '<directory>/subworkflows/local/subworkflow_name.nf'
 
         If <directory> is a clone of nf-core/modules, it creates or modifies the following files:
 
         For modules:
 
+        ```tree
         modules/modules/nf-core/tool/subtool/
-            * main.nf
-            * meta.yml
-        modules/tests/modules/nf-core/tool/subtool/
-            * main.nf
-            * test.yml
-            * nextflow.config
-        tests/config/pytest_modules.yml
+        ├── main.nf
+        ├── meta.yml
+        └── tests
+            ├── main.nf.test
+            └── tags.yml
+        ```
 
         The function will attempt to automatically find a Bioconda package called <component>
         and matching Docker / Singularity images from BioContainers.
@@ -147,32 +148,23 @@ def create(self):
         # Create component template with jinja2
         self._render_template()
 
-        if self.repo_type == "modules":
-            # Add entry to pytest_modules.yml
-            try:
-                with open(os.path.join(self.directory, "tests", "config", "pytest_modules.yml"), "r") as fh:
-                    pytest_modules_yml = yaml.safe_load(fh)
-                if self.subtool:
-                    pytest_modules_yml[self.component_name] = [
-                        f"modules/{self.org}/{self.component}/{self.subtool}/**",
-                        f"tests/modules/{self.org}/{self.component}/{self.subtool}/**",
-                    ]
+        # generate nf-tests
+        nf_core.utils.run_cmd("nf-test", f"generate process {self.file_paths['main.nf']}")
+        # move generated main.nf.test file into test directory
+
+        Path("main.nf.test").rename(self.file_paths[os.path.join(self.component_type, "test", "main.nf.test")])
+
+        # inside main.nf.test replace the path to the main.nf file with the relative path for the script
+        with open(self.file_paths["tests/main.nf.test"], "r") as f:
+            lines = f.readlines()
+        with open(self.file_paths["tests/main.nf.test"], "w") as f:
+            for line in lines:
+                if line.startswith("script"):
+                    f.write(f"script ../main.nf\n")
                 else:
-                    pytest_modules_yml[
-                        ("" if self.component_type == "modules" else self.component_type + "/") + self.component_name
-                    ] = [
-                        f"{self.component_type}/{self.org}/{self.component}/**",
-                        f"tests/{self.component_type}/{self.org}/{self.component}/**",
-                    ]
-                pytest_modules_yml = dict(sorted(pytest_modules_yml.items()))
-                with open(os.path.join(self.directory, "tests", "config", "pytest_modules.yml"), "w") as fh:
-                    yaml.dump(pytest_modules_yml, fh, sort_keys=True, Dumper=nf_core.utils.custom_yaml_dumper())
-            except FileNotFoundError:
-                raise UserWarning("Could not open 'tests/config/pytest_modules.yml' file!")
+                    f.write(line)
 
         new_files = list(self.file_paths.values())
-        if self.repo_type == "modules":
-            new_files.append(os.path.join(self.directory, "tests", "config", "pytest_modules.yml"))
         log.info("Created / edited following files:\n  " + "\n  ".join(new_files))
 
     def _get_bioconda_tool(self):
@@ -362,17 +354,12 @@ def _get_component_dirs(self):
             file_paths[os.path.join(self.component_type, "main.nf")] = component_file
 
         if self.repo_type == "modules":
-            software_dir = os.path.join(self.directory, self.component_type, self.org, self.component_dir)
-            test_dir = os.path.join(self.directory, "tests", self.component_type, self.org, self.component_dir)
+            component_dir = os.path.join(self.directory, self.component_type, self.org, self.component_dir)
 
             # Check if module/subworkflow directories exist already
-            if os.path.exists(software_dir) and not self.force_overwrite:
+            if os.path.exists(component_dir) and not self.force_overwrite:
                 raise UserWarning(
-                    f"{self.component_type[:-1]} directory exists: '{software_dir}'. Use '--force' to overwrite"
-                )
-            if os.path.exists(test_dir) and not self.force_overwrite:
-                raise UserWarning(
-                    f"{self.component_type[:-1]} test directory exists: '{test_dir}'. Use '--force' to overwrite"
+                    f"{self.component_type[:-1]} directory exists: '{component_dir}'. Use '--force' to overwrite"
                 )
 
             if self.component_type == "modules":
@@ -403,11 +390,14 @@ def _get_component_dirs(self):
 
             # Set file paths
             # For modules - can be tool/ or tool/subtool/ so can't do in template directory structure
-            file_paths[os.path.join(self.component_type, "main.nf")] = os.path.join(software_dir, "main.nf")
-            file_paths[os.path.join(self.component_type, "meta.yml")] = os.path.join(software_dir, "meta.yml")
-            file_paths[os.path.join("tests", "main.nf")] = os.path.join(test_dir, "main.nf")
-            file_paths[os.path.join("tests", "test.yml")] = os.path.join(test_dir, "test.yml")
-            file_paths[os.path.join("tests", "nextflow.config")] = os.path.join(test_dir, "nextflow.config")
+            file_paths[os.path.join(self.component_type, "main.nf")] = os.path.join(component_dir, "main.nf")
+            file_paths[os.path.join(self.component_type, "meta.yml")] = os.path.join(component_dir, "meta.yml")
+            file_paths[os.path.join(self.component_type, "test", "main.nf.test")] = os.path.join(
+                component_dir, "test", "main.nf.test"
+            )
+            file_paths[os.path.join(self.component_type, "test", "tags.yml")] = os.path.join(
+                component_dir, "test", "tags.yml"
+            )
 
         return file_paths
 

nf_core/list.py

@@ -64,10 +64,10 @@ def get_local_wf(workflow, revision=None):
 
     # Wasn't local, fetch it
     log.info(f"Downloading workflow: {workflow} ({revision})")
-    pull_cmd = f"nextflow pull {workflow}"
+    pull_cmd = f"pull {workflow}"
     if revision is not None:
         pull_cmd += f" -r {revision}"
-    nf_core.utils.nextflow_cmd(pull_cmd)
+    nf_core.utils.run_cmd("nextflow", pull_cmd)
     local_wf = LocalWorkflow(workflow)
     local_wf.get_local_nf_workflow_details()
     return local_wf.local_path
@@ -128,7 +128,7 @@ def get_local_nf_workflows(self):
         # Fetch details about local cached pipelines with `nextflow list`
         else:
             log.debug("Getting list of local nextflow workflows")
-            nflist_raw = nf_core.utils.nextflow_cmd("nextflow list")
+            nflist_raw = nf_core.utils.run_cmd("nextflow", "list")
             for wf_name in nflist_raw.splitlines():
                 if not str(wf_name).startswith("nf-core/"):
                     self.local_unmatched.append(wf_name)
@@ -342,7 +342,7 @@ def get_local_nf_workflow_details(self):
 
             # Use `nextflow info` to get more details about the workflow
             else:
-                nfinfo_raw = str(nf_core.utils.nextflow_cmd(f"nextflow info -d {self.full_name}"))
+                nfinfo_raw = str(nf_core.utils.run_cmd("nextflow", f"info -d {self.full_name}"))
                 re_patterns = {"repository": r"repository\s*: (.*)", "local_path": r"local path\s*: (.*)"}
                 for key, pattern in re_patterns.items():
                     m = re.search(pattern, nfinfo_raw)

nf_core/module-template/modules/tests/tags.yml

@@ -0,0 +1,2 @@
+{ { component_name_underscore } }:
+  - "modules/{{ org }}/{{ component_dir }}/**"

nf_core/utils.py

@@ -17,6 +17,7 @@
 import sys
 import time
 from pathlib import Path
+from typing import ByteString, Optional, Union
 
 import git
 import prompt_toolkit
@@ -269,7 +270,7 @@ def fetch_wf_config(wf_path, cache_config=True):
     log.debug("No config cache found")
 
     # Call `nextflow config`
-    nfconfig_raw = nextflow_cmd(f"nextflow config -flat {wf_path}")
+    nfconfig_raw = nextflow_cmd("nextflow", f"config -flat {wf_path}")
     for l in nfconfig_raw.splitlines():
         ul = l.decode("utf-8")
         try:
@@ -303,17 +304,20 @@ def fetch_wf_config(wf_path, cache_config=True):
     return config
 
 
-def nextflow_cmd(cmd):
-    """Run a Nextflow command and capture the output. Handle errors nicely"""
+def run_cmd(executable: str, cmd: str) -> Union[Optional[ByteString], str]:
+    """Run a specified command and capture the output. Handle errors nicely."""
+    full_cmd = f"{executable} {cmd}"
     try:
-        nf_proc = subprocess.run(shlex.split(cmd), stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True)
-        return nf_proc.stdout
+        proc = subprocess.run(shlex.split(full_cmd), stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True)
+        return proc.stdout
     except OSError as e:
         if e.errno == errno.ENOENT:
-            raise AssertionError("It looks like Nextflow is not installed. It is required for most nf-core functions.")
+            raise AssertionError(
+                f"It looks like {executable} is not installed. Please ensure it is available in your PATH."
+            )
     except subprocess.CalledProcessError as e:
         raise AssertionError(
-            f"Command '{cmd}' returned non-zero error code '{e.returncode}':\n[red]> {e.stderr.decode()}{e.stdout.decode()}"
+            f"Command '{full_cmd}' returned non-zero error code '{e.returncode}':\n[red]> {e.stderr.decode()}{e.stdout.decode()}"
         )
 
 

tests/test_download.py

@@ -16,7 +16,7 @@
 import nf_core.utils
 from nf_core.download import ContainerError, DownloadWorkflow, WorkflowRepo
 from nf_core.synced_repo import SyncedRepo
-from nf_core.utils import NFCORE_CACHE_DIR, NFCORE_DIR, nextflow_cmd
+from nf_core.utils import NFCORE_CACHE_DIR, NFCORE_DIR, run_cmd
 
 from .utils import with_temporary_file, with_temporary_folder
 
@@ -156,8 +156,8 @@ def test_find_container_images_config_basic(self, tmp_path, mock_fetch_wf_config
     @mock.patch("nf_core.utils.fetch_wf_config")
     def test__find_container_images_config_nextflow(self, tmp_path, mock_fetch_wf_config):
         download_obj = DownloadWorkflow(pipeline="dummy", outdir=tmp_path)
-        nfconfig_raw = nextflow_cmd(
-            f"nextflow config -flat {Path(__file__).resolve().parent / 'data/mock_config_containers'}"
+        nfconfig_raw = run_cmd(
+            "nextflow", f"config -flat {Path(__file__).resolve().parent / 'data/mock_config_containers'}"
         )
         config = {}
         for l in nfconfig_raw.splitlines():