add --migrate-pytest option

CHANGELOG.md

@@ -20,11 +20,13 @@
 - Added stub test creation to `create_test_yml` ([#2476](https://github.com/nf-core/tools/pull/2476))
 - Replace ModulePatch by ComponentPatch ([#2482](https://github.com/nf-core/tools/pull/2482))
 - Fixed `nf-core modules lint` to work with new module structure for nf-test ([#2494](https://github.com/nf-core/tools/pull/2494))
+- Add option `--migrate-pytest` to create a module with nf-test taking into account an existing module
 
 ### Subworkflows
 
 - Added stub test creation to `create_test_yml` ([#2476](https://github.com/nf-core/tools/pull/2476))
 - Fixed `nf-core subworkflows lint` to work with new module structure for nf-test ([#2494](https://github.com/nf-core/tools/pull/2494))
+- Add option `--migrate-pytest` to create a subworkflow with nf-test taking into account an existing subworkflow
 
 ### General
 

nf_core/__main__.py

@@ -815,8 +815,20 @@ def remove(ctx, dir, tool):
     default=False,
     help="Create a module from the template without TODOs or examples",
 )
+@click.option("--migrate-pytest", is_flag=True, default=False, help="Migrate a module with pytest tests to nf-test")
 def create_module(
-    ctx, tool, dir, author, label, meta, no_meta, force, conda_name, conda_package_version, empty_template
+    ctx,
+    tool,
+    dir,
+    author,
+    label,
+    meta,
+    no_meta,
+    force,
+    conda_name,
+    conda_package_version,
+    empty_template,
+    migrate_pytest,
 ):
     """
     Create a new DSL2 module from the nf-core template.
@@ -841,7 +853,7 @@ def create_module(
     # Run function
     try:
         module_create = ModuleCreate(
-            dir, tool, author, label, has_meta, force, conda_name, conda_package_version, empty_template
+            dir, tool, author, label, has_meta, force, conda_name, conda_package_version, empty_template, migrate_pytest
         )
         module_create.create()
     except UserWarning as e:
@@ -1035,7 +1047,8 @@ def bump_versions(ctx, tool, dir, all, show_all):
 @click.option("-d", "--dir", type=click.Path(exists=True), default=".", metavar="<directory>")
 @click.option("-a", "--author", type=str, metavar="<author>", help="Module author's GitHub username prefixed with '@'")
 @click.option("-f", "--force", is_flag=True, default=False, help="Overwrite any files if they already exist")
-def create_subworkflow(ctx, subworkflow, dir, author, force):
+@click.option("--migrate-pytest", is_flag=True, default=False, help="Migrate a module with pytest tests to nf-test")
+def create_subworkflow(ctx, subworkflow, dir, author, force, migrate_pytest):
     """
     Create a new subworkflow from the nf-core template.
 
@@ -1049,7 +1062,7 @@ def create_subworkflow(ctx, subworkflow, dir, author, force):
 
     # Run function
     try:
-        subworkflow_create = SubworkflowCreate(dir, subworkflow, author, force)
+        subworkflow_create = SubworkflowCreate(dir, subworkflow, author, force, migrate_pytest)
         subworkflow_create.create()
     except UserWarning as e:
         log.critical(e)

nf_core/components/create.py

@@ -9,6 +9,7 @@
 import logging
 import os
 import re
+import shutil
 import subprocess
 from pathlib import Path
 from typing import Dict, Optional
@@ -38,6 +39,7 @@ def __init__(
         conda_name: Optional[str] = None,
         conda_version: Optional[str] = None,
         empty_template: bool = False,
+        migrate_pytest: bool = False,
     ):
         super().__init__(component_type, directory)
         self.directory = directory
@@ -58,6 +60,7 @@ def __init__(
         self.docker_container = None
         self.file_paths: Dict[str, str] = {}
         self.not_empty_template = not empty_template
+        self.migrate_pytest = migrate_pytest
 
     def create(self):
         """
@@ -136,20 +139,36 @@ def create(self):
         # Check existence of directories early for fast-fail
         self.file_paths = self._get_component_dirs()
 
-        if self.component_type == "modules":
-            # Try to find a bioconda package for 'component'
-            self._get_bioconda_tool()
+        if self.migrate_pytest:
+            # Rename the component directory to old
+            component_old = self.component_dir + "_old"
+            component_old_path = Path(self.directory, self.component_type, self.org, component_old)
+            Path(self.directory, self.component_type, self.org, self.component_dir).rename(component_old_path)
+        else:
+            if self.component_type == "modules":
+                # Try to find a bioconda package for 'component'
+                self._get_bioconda_tool()
 
-        # Prompt for GitHub username
-        self._get_username()
+            # Prompt for GitHub username
+            self._get_username()
 
-        if self.component_type == "modules":
-            self._get_module_structure_components()
+            if self.component_type == "modules":
+                self._get_module_structure_components()
 
         # Create component template with jinja2
         self._render_template()
         log.info(f"Created component template: '{self.component_name}'")
 
+        if self.migrate_pytest:
+            self._copy_old_files(component_old_path)
+            log.info("Migrate pytest tests: Copied original module files to new module")
+            try:
+                self._update_nftest_file()
+                log.info("Migrate pytest tests: Updated `main.nf.test` with contents of pytest")
+            except Exception as e:
+                log.info(f"Could not update `main.nf.test` file: {e}")
+            shutil.rmtree(component_old_path)
+
         new_files = list(self.file_paths.values())
         log.info("Created following files:\n  " + "\n  ".join(new_files))
 
@@ -348,7 +367,7 @@ def _get_component_dirs(self):
             component_dir = os.path.join(self.directory, self.component_type, self.org, self.component_dir)
 
             # Check if module/subworkflow directories exist already
-            if os.path.exists(component_dir) and not self.force_overwrite:
+            if os.path.exists(component_dir) and not self.force_overwrite and not self.migrate_pytest:
                 raise UserWarning(
                     f"{self.component_type[:-1]} directory exists: '{component_dir}'. Use '--force' to overwrite"
                 )
@@ -358,7 +377,7 @@ def _get_component_dirs(self):
                 parent_tool_main_nf = os.path.join(
                     self.directory, self.component_type, self.org, self.component, "main.nf"
                 )
-                if self.subtool and os.path.exists(parent_tool_main_nf):
+                if self.subtool and os.path.exists(parent_tool_main_nf) and not self.migrate_pytest:
                     raise UserWarning(
                         f"Module '{parent_tool_main_nf}' exists already, cannot make subtool '{self.component_name}'"
                     )
@@ -367,7 +386,7 @@ def _get_component_dirs(self):
                 tool_glob = glob.glob(
                     f"{os.path.join(self.directory, self.component_type, self.org, self.component)}/*/main.nf"
                 )
-                if not self.subtool and tool_glob:
+                if not self.subtool and tool_glob and not self.migrate_pytest:
                     raise UserWarning(
                         f"Module subtool '{tool_glob[0]}' exists already, cannot make tool '{self.component_name}'"
                     )
@@ -411,3 +430,131 @@ def _get_username(self):
                 f"[violet]GitHub Username:[/]{' (@author)' if author_default is None else ''}",
                 default=author_default,
             )
+
+    def _copy_old_files(self, component_old_path):
+        """Copy files from old module to new module"""
+        log.debug("Copying original main.nf file")
+        shutil.copyfile(component_old_path / "main.nf", self.file_paths[self.component_type + "/main.nf"])
+        log.debug("Copying original meta.yml file")
+        shutil.copyfile(component_old_path / "meta.yml", self.file_paths[self.component_type + "/meta.yml"])
+        if self.component_type == "modules":
+            log.debug("Copying original environment.yml file")
+            shutil.copyfile(
+                component_old_path / "environment.yml", self.file_paths[self.component_type + "/environment.yml"]
+            )
+        # Create a nextflow.config file if it contains information other than publishDir
+        pytest_dir = Path(self.directory, "tests", self.component_type, self.org, self.component_dir)
+        nextflow_config = pytest_dir / "nextflow.config"
+        if nextflow_config.is_file():
+            with open(nextflow_config, "r") as fh:
+                config_lines = ""
+                for line in fh:
+                    if "publishDir" not in line:
+                        config_lines += line
+            if len(config_lines) > 0:
+                log.debug("Copying nextflow.config file from pytest tests")
+                with open(
+                    Path(self.directory, self.component_type, self.org, self.component_dir, "tests", "nextflow.config"),
+                    "w+",
+                ) as ofh:
+                    ofh.write(config_lines)
+
+    def _collect_pytest_tests(self):
+        pytest_dir = Path(self.directory, "tests", self.component_type, self.org, self.component_dir)
+        tests = []
+        name = None
+        input = None
+        in_input = False
+        input_number = 0
+        number_of_inputs = []
+        with open(pytest_dir / "main.nf") as fh:
+            for line in fh:
+                if line.strip().startswith("workflow"):
+                    # One test
+                    if name and input:
+                        tests.append((name, input))
+                        name = None
+                        input = None
+                    name = line.split()[1]
+                elif line.strip().startswith("input"):
+                    # First input
+                    input = [line.split("=")[1]]
+                    in_input = True
+                    number_of_inputs.append(1)
+                elif "=" in line and "nextflow.enable.dsl" not in line:
+                    # We need another input
+                    input_number += 1
+                    in_input = True
+                    input.append(line.split("=")[1])
+                    number_of_inputs[input_number] += 1
+                elif in_input:
+                    # Retrieve all lines of an input
+                    if self.component_dir.replace("/", "_").upper() in line:
+                        in_input = False
+                        continue
+                    input[input_number] += line
+
+        if name and input:
+            tests.append((name, input))
+
+        if max(number_of_inputs) > 1:
+            # Check that all tests have the same number of inputs
+            for test in tests:
+                if len(test[1]) < max(number_of_inputs):
+                    for i in range(max(number_of_inputs) - len(test[1])):
+                        test[1].append(" []")
+
+        return tests
+
+    def _create_new_test(self, name, inputs):
+        input_string = ""
+        for i, input in enumerate(inputs):
+            input_string += f"                input[{i}] ={input}"
+        with open(Path("../module-template/tests/main.nf.test"), "r") as fh:
+            test = fh.readlines()[15:48]
+        test[1] = f'    test("{name}") {{'
+        test[12:20] = f"                {input_string}"
+        return "".join(test)
+
+    def _update_nftest_file(self):
+        """Update the nftest file with the pytest tests"""
+        test_script = self.file_paths[os.path.join(self.component_type, "tests", "main.nf.test")]
+        nextflow_config = Path(
+            self.directory, self.component_type, self.org, self.component_dir, "tests", "nextflow.config"
+        )
+        pytest_tests = self._collect_pytest_tests()
+        subtool_lines = 0
+        meta_lines = 0
+
+        # Update test script
+        with open(test_script, "r") as fh:
+            main_nf_test = fh.readlines()
+
+        if nextflow_config.is_file():
+            # Add nextflow config
+            main_nf_test[5] += '    config "./nextflow.config"\n'
+        # Update test name
+        if self.subtool:
+            # Add 1 to the line index as we have the subtool tag
+            subtool_lines += 1
+        main_nf_test[13 + subtool_lines] = f'    test("{pytest_tests[0][0]}") {{\n'
+        # Update input
+        if self.has_meta:
+            meta_lines += 4
+        main_nf_test[
+            25 + subtool_lines : 26 + subtool_lines + meta_lines
+        ] = f"                input[0] ={pytest_tests[0][1][0]}"
+        # Add more inputs if we have more than one
+        if len(pytest_tests[0][1]) > 1:
+            input_number = 1
+            for input in pytest_tests[0][1][1:]:
+                main_nf_test[25 + subtool_lines] += f"                input[{input_number}] ={input}"
+                input_number += 1
+        # Add more tests if we have more than one
+        if len(pytest_tests) > 1:
+            for t in pytest_tests[1:]:
+                name, inputs = t
+                main_nf_test[38 + subtool_lines] += self._create_new_test(name, inputs)
+
+        with open(test_script, "w") as fh:
+            fh.write(main_nf_test)

nf_core/modules/create.py

@@ -17,6 +17,7 @@ def __init__(
         conda_name=None,
         conda_version=None,
         empty_template=False,
+        migrate_pytest=False,
     ):
         super().__init__(
             "modules",
@@ -29,4 +30,5 @@ def __init__(
             conda_name,
             conda_version,
             empty_template,
+            migrate_pytest,
         )

nf_core/subworkflows/create.py

@@ -12,11 +12,13 @@ def __init__(
         component="",
         author=None,
         force=False,
+        migrate_pytest=False,
     ):
         super().__init__(
             "subworkflows",
             pipeline_dir,
             component,
             author,
             force=force,
+            migrate_pytest=migrate_pytest,
         )