From ca138458e7ff19b0679ecdebb48eaff5634ff7ea Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 27 Nov 2023 17:10:34 +0100 Subject: [PATCH 1/8] add --migrate-pytest option --- CHANGELOG.md | 2 + nf_core/__main__.py | 21 +++- nf_core/components/create.py | 196 +++++++++++++++++++++++++++++++-- nf_core/modules/create.py | 2 + nf_core/subworkflows/create.py | 2 + 5 files changed, 209 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f64047db4b..c38a182345 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,11 +19,13 @@ - Added stub test creation to `create_test_yml` ([#2476](https://github.com/nf-core/tools/pull/2476)) - Replace ModulePatch by ComponentPatch ([#2482](https://github.com/nf-core/tools/pull/2482)) - Fixed `nf-core modules lint` to work with new module structure for nf-test ([#2494](https://github.com/nf-core/tools/pull/2494)) +- Add option `--migrate-pytest` to create a module with nf-test taking into account an existing module ### Subworkflows - Added stub test creation to `create_test_yml` ([#2476](https://github.com/nf-core/tools/pull/2476)) - Fixed `nf-core subworkflows lint` to work with new module structure for nf-test ([#2494](https://github.com/nf-core/tools/pull/2494)) +- Add option `--migrate-pytest` to create a subworkflow with nf-test taking into account an existing subworkflow ### General diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 232f1bf116..6b8befb062 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -815,8 +815,20 @@ def remove(ctx, dir, tool): default=False, help="Create a module from the template without TODOs or examples", ) +@click.option("--migrate-pytest", is_flag=True, default=False, help="Migrate a module with pytest tests to nf-test") def create_module( - ctx, tool, dir, author, label, meta, no_meta, force, conda_name, conda_package_version, empty_template + ctx, + tool, + dir, + author, + label, + meta, + no_meta, + force, + conda_name, + conda_package_version, + empty_template, + migrate_pytest, ): """ Create a new DSL2 module from the nf-core template. @@ -841,7 +853,7 @@ def create_module( # Run function try: module_create = ModuleCreate( - dir, tool, author, label, has_meta, force, conda_name, conda_package_version, empty_template + dir, tool, author, label, has_meta, force, conda_name, conda_package_version, empty_template, migrate_pytest ) module_create.create() except UserWarning as e: @@ -1035,7 +1047,8 @@ def bump_versions(ctx, tool, dir, all, show_all): @click.option("-d", "--dir", type=click.Path(exists=True), default=".", metavar="") @click.option("-a", "--author", type=str, metavar="", help="Module author's GitHub username prefixed with '@'") @click.option("-f", "--force", is_flag=True, default=False, help="Overwrite any files if they already exist") -def create_subworkflow(ctx, subworkflow, dir, author, force): +@click.option("--migrate-pytest", is_flag=True, default=False, help="Migrate a module with pytest tests to nf-test") +def create_subworkflow(ctx, subworkflow, dir, author, force, migrate_pytest): """ Create a new subworkflow from the nf-core template. @@ -1049,7 +1062,7 @@ def create_subworkflow(ctx, subworkflow, dir, author, force): # Run function try: - subworkflow_create = SubworkflowCreate(dir, subworkflow, author, force) + subworkflow_create = SubworkflowCreate(dir, subworkflow, author, force, migrate_pytest) subworkflow_create.create() except UserWarning as e: log.critical(e) diff --git a/nf_core/components/create.py b/nf_core/components/create.py index 54fade29a2..b13fc81454 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -9,6 +9,7 @@ import logging import os import re +import shutil import subprocess from pathlib import Path from typing import Dict, Optional @@ -38,6 +39,7 @@ def __init__( conda_name: Optional[str] = None, conda_version: Optional[str] = None, empty_template: bool = False, + migrate_pytest: bool = False, ): super().__init__(component_type, directory) self.directory = directory @@ -58,6 +60,7 @@ def __init__( self.docker_container = None self.file_paths: Dict[str, str] = {} self.not_empty_template = not empty_template + self.migrate_pytest = migrate_pytest def create(self): """ @@ -136,20 +139,36 @@ def create(self): # Check existence of directories early for fast-fail self.file_paths = self._get_component_dirs() - if self.component_type == "modules": - # Try to find a bioconda package for 'component' - self._get_bioconda_tool() + if self.migrate_pytest: + # Rename the component directory to old + component_old = self.component_dir + "_old" + component_old_path = Path(self.directory, self.component_type, self.org, component_old) + Path(self.directory, self.component_type, self.org, self.component_dir).rename(component_old_path) + else: + if self.component_type == "modules": + # Try to find a bioconda package for 'component' + self._get_bioconda_tool() - # Prompt for GitHub username - self._get_username() + # Prompt for GitHub username + self._get_username() - if self.component_type == "modules": - self._get_module_structure_components() + if self.component_type == "modules": + self._get_module_structure_components() # Create component template with jinja2 self._render_template() log.info(f"Created component template: '{self.component_name}'") + if self.migrate_pytest: + self._copy_old_files(component_old_path) + log.info("Migrate pytest tests: Copied original module files to new module") + try: + self._update_nftest_file() + log.info("Migrate pytest tests: Updated `main.nf.test` with contents of pytest") + except Exception as e: + log.info(f"Could not update `main.nf.test` file: {e}") + shutil.rmtree(component_old_path) + new_files = list(self.file_paths.values()) log.info("Created following files:\n " + "\n ".join(new_files)) @@ -348,7 +367,7 @@ def _get_component_dirs(self): component_dir = os.path.join(self.directory, self.component_type, self.org, self.component_dir) # Check if module/subworkflow directories exist already - if os.path.exists(component_dir) and not self.force_overwrite: + if os.path.exists(component_dir) and not self.force_overwrite and not self.migrate_pytest: raise UserWarning( f"{self.component_type[:-1]} directory exists: '{component_dir}'. Use '--force' to overwrite" ) @@ -358,7 +377,7 @@ def _get_component_dirs(self): parent_tool_main_nf = os.path.join( self.directory, self.component_type, self.org, self.component, "main.nf" ) - if self.subtool and os.path.exists(parent_tool_main_nf): + if self.subtool and os.path.exists(parent_tool_main_nf) and not self.migrate_pytest: raise UserWarning( f"Module '{parent_tool_main_nf}' exists already, cannot make subtool '{self.component_name}'" ) @@ -367,7 +386,7 @@ def _get_component_dirs(self): tool_glob = glob.glob( f"{os.path.join(self.directory, self.component_type, self.org, self.component)}/*/main.nf" ) - if not self.subtool and tool_glob: + if not self.subtool and tool_glob and not self.migrate_pytest: raise UserWarning( f"Module subtool '{tool_glob[0]}' exists already, cannot make tool '{self.component_name}'" ) @@ -411,3 +430,160 @@ def _get_username(self): f"[violet]GitHub Username:[/]{' (@author)' if author_default is None else ''}", default=author_default, ) + + def _copy_old_files(self, component_old_path): + """Copy files from old module to new module""" + log.debug("Copying original main.nf file") + shutil.copyfile(component_old_path / "main.nf", self.file_paths[self.component_type + "/main.nf"]) + log.debug("Copying original meta.yml file") + shutil.copyfile(component_old_path / "meta.yml", self.file_paths[self.component_type + "/meta.yml"]) + if self.component_type == "modules": + log.debug("Copying original environment.yml file") + shutil.copyfile( + component_old_path / "environment.yml", self.file_paths[self.component_type + "/environment.yml"] + ) + # Create a nextflow.config file if it contains information other than publishDir + pytest_dir = Path(self.directory, "tests", self.component_type, self.org, self.component_dir) + nextflow_config = pytest_dir / "nextflow.config" + if nextflow_config.is_file(): + with open(nextflow_config, "r") as fh: + config_lines = "" + for line in fh: + if "publishDir" not in line: + config_lines += line + if len(config_lines) > 0: + log.debug("Copying nextflow.config file from pytest tests") + with open( + Path(self.directory, self.component_type, self.org, self.component_dir, "tests", "nextflow.config"), + "w+", + ) as ofh: + ofh.write(config_lines) + + def _collect_pytest_tests(self): + pytest_dir = Path(self.directory, "tests", self.component_type, self.org, self.component_dir) + tests = [] + name = None + input = None + in_input = False + input_number = 0 + number_of_inputs = [] + with open(pytest_dir / "main.nf") as fh: + for line in fh: + if line.strip().startswith("workflow"): + # One test + if name and input: + tests.append((name, input)) + name = None + input = None + name = line.split()[1] + elif line.strip().startswith("input"): + # First input + input = [line.split("=")[1]] + in_input = True + number_of_inputs.append(1) + elif "=" in line and "nextflow.enable.dsl" not in line: + # We need another input + input_number += 1 + in_input = True + input.append(line.split("=")[1]) + number_of_inputs[input_number] += 1 + elif in_input: + # Retrieve all lines of an input + if self.component_dir.replace("/", "_").upper() in line: + in_input = False + continue + input[input_number] += line + + if name and input: + tests.append((name, input)) + + if max(number_of_inputs) > 1: + # Check that all tests have the same number of inputs + for test in tests: + if len(test[1]) < max(number_of_inputs): + for i in range(max(number_of_inputs) - len(test[1])): + test[1].append(" []") + + return tests + + def _create_new_test(self, name, inputs): + input_string = "" + for i, input in enumerate(inputs): + input_string += f" input[{i}] ={input}" + test = f""" + test("{name}") {{ + when {{ + process {{ + \""" + {input_string} + \""" + }} + }} + + then {{ + assertAll( + {{ assert process.success }}, + {{ assert snapshot(process.out.versions).match("versions") }} + {{ assert snapshot(process.out).match() }} + ) + }} + }} + """ + return test + + def _update_nftest_file(self): + """Update the nftest file with the pytest tests""" + test_script = self.file_paths[os.path.join(self.component_type, "tests", "main.nf.test")] + nextflow_config = Path( + self.directory, self.component_type, self.org, self.component_dir, "tests", "nextflow.config" + ) + pytest_tests = self._collect_pytest_tests() + in_input = False + # Update test script + new_lines = "" + with open(test_script, "r") as fh: + for line in fh: + if line.strip().startswith("script"): + new_lines += line + if nextflow_config.is_file(): + # Add nextflow config + new_lines += ' config "./nextflow.config"\n' + continue + elif line.strip().startswith("test"): + # Update test name + test_name = line.split('"') + test_name[1] = pytest_tests[0][0] + new_lines += '"'.join(test_name) + continue + elif line.strip().startswith("input"): + # Update input + in_input = True + new_lines += f" input[0] ={pytest_tests[0][1][0]}" + if len(pytest_tests[0][1]) > 1: + # Add more inputs + input_number = 1 + for input in pytest_tests[0][1][1:]: + new_lines += f" input[{input_number}] ={input}" + input_number += 1 + continue + elif in_input: + # While we are inside the process script defining inputs + if line.strip() == '"""': + in_input = False + new_lines += line + continue + elif line == " }\n": + # If we finished adding a test, check if there are more tests to add + new_lines += line + if len(pytest_tests) > 1: + for t in pytest_tests[1:]: + name, inputs = t + new_lines += self._create_new_test(name, inputs) + continue + elif "assert snapshot(process.out.versions).match(" in line: + new_lines += line + new_lines += " { assert snapshot(process.out).match() }\n" + continue + new_lines += line + with open(test_script, "w") as fh: + fh.write(new_lines) diff --git a/nf_core/modules/create.py b/nf_core/modules/create.py index b5368130ce..a5e0795a9f 100644 --- a/nf_core/modules/create.py +++ b/nf_core/modules/create.py @@ -17,6 +17,7 @@ def __init__( conda_name=None, conda_version=None, empty_template=False, + migrate_pytest=False, ): super().__init__( "modules", @@ -29,4 +30,5 @@ def __init__( conda_name, conda_version, empty_template, + migrate_pytest, ) diff --git a/nf_core/subworkflows/create.py b/nf_core/subworkflows/create.py index 963076455e..93e9f271be 100644 --- a/nf_core/subworkflows/create.py +++ b/nf_core/subworkflows/create.py @@ -12,6 +12,7 @@ def __init__( component="", author=None, force=False, + migrate_pytest=False, ): super().__init__( "subworkflows", @@ -19,4 +20,5 @@ def __init__( component, author, force=force, + migrate_pytest=migrate_pytest, ) From 24b0b881558abcb40c4d9acdf18e11f0814c3027 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 27 Nov 2023 17:10:34 +0100 Subject: [PATCH 2/8] add --migrate-pytest option --- CHANGELOG.md | 2 + nf_core/__main__.py | 21 +++- nf_core/components/create.py | 196 +++++++++++++++++++++++++++++++-- nf_core/modules/create.py | 2 + nf_core/subworkflows/create.py | 2 + 5 files changed, 209 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ae69fc7ab..a68fcf810c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,11 +20,13 @@ - Added stub test creation to `create_test_yml` ([#2476](https://github.com/nf-core/tools/pull/2476)) - Replace ModulePatch by ComponentPatch ([#2482](https://github.com/nf-core/tools/pull/2482)) - Fixed `nf-core modules lint` to work with new module structure for nf-test ([#2494](https://github.com/nf-core/tools/pull/2494)) +- Add option `--migrate-pytest` to create a module with nf-test taking into account an existing module ### Subworkflows - Added stub test creation to `create_test_yml` ([#2476](https://github.com/nf-core/tools/pull/2476)) - Fixed `nf-core subworkflows lint` to work with new module structure for nf-test ([#2494](https://github.com/nf-core/tools/pull/2494)) +- Add option `--migrate-pytest` to create a subworkflow with nf-test taking into account an existing subworkflow ### General diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 232f1bf116..6b8befb062 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -815,8 +815,20 @@ def remove(ctx, dir, tool): default=False, help="Create a module from the template without TODOs or examples", ) +@click.option("--migrate-pytest", is_flag=True, default=False, help="Migrate a module with pytest tests to nf-test") def create_module( - ctx, tool, dir, author, label, meta, no_meta, force, conda_name, conda_package_version, empty_template + ctx, + tool, + dir, + author, + label, + meta, + no_meta, + force, + conda_name, + conda_package_version, + empty_template, + migrate_pytest, ): """ Create a new DSL2 module from the nf-core template. @@ -841,7 +853,7 @@ def create_module( # Run function try: module_create = ModuleCreate( - dir, tool, author, label, has_meta, force, conda_name, conda_package_version, empty_template + dir, tool, author, label, has_meta, force, conda_name, conda_package_version, empty_template, migrate_pytest ) module_create.create() except UserWarning as e: @@ -1035,7 +1047,8 @@ def bump_versions(ctx, tool, dir, all, show_all): @click.option("-d", "--dir", type=click.Path(exists=True), default=".", metavar="") @click.option("-a", "--author", type=str, metavar="", help="Module author's GitHub username prefixed with '@'") @click.option("-f", "--force", is_flag=True, default=False, help="Overwrite any files if they already exist") -def create_subworkflow(ctx, subworkflow, dir, author, force): +@click.option("--migrate-pytest", is_flag=True, default=False, help="Migrate a module with pytest tests to nf-test") +def create_subworkflow(ctx, subworkflow, dir, author, force, migrate_pytest): """ Create a new subworkflow from the nf-core template. @@ -1049,7 +1062,7 @@ def create_subworkflow(ctx, subworkflow, dir, author, force): # Run function try: - subworkflow_create = SubworkflowCreate(dir, subworkflow, author, force) + subworkflow_create = SubworkflowCreate(dir, subworkflow, author, force, migrate_pytest) subworkflow_create.create() except UserWarning as e: log.critical(e) diff --git a/nf_core/components/create.py b/nf_core/components/create.py index 54fade29a2..b13fc81454 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -9,6 +9,7 @@ import logging import os import re +import shutil import subprocess from pathlib import Path from typing import Dict, Optional @@ -38,6 +39,7 @@ def __init__( conda_name: Optional[str] = None, conda_version: Optional[str] = None, empty_template: bool = False, + migrate_pytest: bool = False, ): super().__init__(component_type, directory) self.directory = directory @@ -58,6 +60,7 @@ def __init__( self.docker_container = None self.file_paths: Dict[str, str] = {} self.not_empty_template = not empty_template + self.migrate_pytest = migrate_pytest def create(self): """ @@ -136,20 +139,36 @@ def create(self): # Check existence of directories early for fast-fail self.file_paths = self._get_component_dirs() - if self.component_type == "modules": - # Try to find a bioconda package for 'component' - self._get_bioconda_tool() + if self.migrate_pytest: + # Rename the component directory to old + component_old = self.component_dir + "_old" + component_old_path = Path(self.directory, self.component_type, self.org, component_old) + Path(self.directory, self.component_type, self.org, self.component_dir).rename(component_old_path) + else: + if self.component_type == "modules": + # Try to find a bioconda package for 'component' + self._get_bioconda_tool() - # Prompt for GitHub username - self._get_username() + # Prompt for GitHub username + self._get_username() - if self.component_type == "modules": - self._get_module_structure_components() + if self.component_type == "modules": + self._get_module_structure_components() # Create component template with jinja2 self._render_template() log.info(f"Created component template: '{self.component_name}'") + if self.migrate_pytest: + self._copy_old_files(component_old_path) + log.info("Migrate pytest tests: Copied original module files to new module") + try: + self._update_nftest_file() + log.info("Migrate pytest tests: Updated `main.nf.test` with contents of pytest") + except Exception as e: + log.info(f"Could not update `main.nf.test` file: {e}") + shutil.rmtree(component_old_path) + new_files = list(self.file_paths.values()) log.info("Created following files:\n " + "\n ".join(new_files)) @@ -348,7 +367,7 @@ def _get_component_dirs(self): component_dir = os.path.join(self.directory, self.component_type, self.org, self.component_dir) # Check if module/subworkflow directories exist already - if os.path.exists(component_dir) and not self.force_overwrite: + if os.path.exists(component_dir) and not self.force_overwrite and not self.migrate_pytest: raise UserWarning( f"{self.component_type[:-1]} directory exists: '{component_dir}'. Use '--force' to overwrite" ) @@ -358,7 +377,7 @@ def _get_component_dirs(self): parent_tool_main_nf = os.path.join( self.directory, self.component_type, self.org, self.component, "main.nf" ) - if self.subtool and os.path.exists(parent_tool_main_nf): + if self.subtool and os.path.exists(parent_tool_main_nf) and not self.migrate_pytest: raise UserWarning( f"Module '{parent_tool_main_nf}' exists already, cannot make subtool '{self.component_name}'" ) @@ -367,7 +386,7 @@ def _get_component_dirs(self): tool_glob = glob.glob( f"{os.path.join(self.directory, self.component_type, self.org, self.component)}/*/main.nf" ) - if not self.subtool and tool_glob: + if not self.subtool and tool_glob and not self.migrate_pytest: raise UserWarning( f"Module subtool '{tool_glob[0]}' exists already, cannot make tool '{self.component_name}'" ) @@ -411,3 +430,160 @@ def _get_username(self): f"[violet]GitHub Username:[/]{' (@author)' if author_default is None else ''}", default=author_default, ) + + def _copy_old_files(self, component_old_path): + """Copy files from old module to new module""" + log.debug("Copying original main.nf file") + shutil.copyfile(component_old_path / "main.nf", self.file_paths[self.component_type + "/main.nf"]) + log.debug("Copying original meta.yml file") + shutil.copyfile(component_old_path / "meta.yml", self.file_paths[self.component_type + "/meta.yml"]) + if self.component_type == "modules": + log.debug("Copying original environment.yml file") + shutil.copyfile( + component_old_path / "environment.yml", self.file_paths[self.component_type + "/environment.yml"] + ) + # Create a nextflow.config file if it contains information other than publishDir + pytest_dir = Path(self.directory, "tests", self.component_type, self.org, self.component_dir) + nextflow_config = pytest_dir / "nextflow.config" + if nextflow_config.is_file(): + with open(nextflow_config, "r") as fh: + config_lines = "" + for line in fh: + if "publishDir" not in line: + config_lines += line + if len(config_lines) > 0: + log.debug("Copying nextflow.config file from pytest tests") + with open( + Path(self.directory, self.component_type, self.org, self.component_dir, "tests", "nextflow.config"), + "w+", + ) as ofh: + ofh.write(config_lines) + + def _collect_pytest_tests(self): + pytest_dir = Path(self.directory, "tests", self.component_type, self.org, self.component_dir) + tests = [] + name = None + input = None + in_input = False + input_number = 0 + number_of_inputs = [] + with open(pytest_dir / "main.nf") as fh: + for line in fh: + if line.strip().startswith("workflow"): + # One test + if name and input: + tests.append((name, input)) + name = None + input = None + name = line.split()[1] + elif line.strip().startswith("input"): + # First input + input = [line.split("=")[1]] + in_input = True + number_of_inputs.append(1) + elif "=" in line and "nextflow.enable.dsl" not in line: + # We need another input + input_number += 1 + in_input = True + input.append(line.split("=")[1]) + number_of_inputs[input_number] += 1 + elif in_input: + # Retrieve all lines of an input + if self.component_dir.replace("/", "_").upper() in line: + in_input = False + continue + input[input_number] += line + + if name and input: + tests.append((name, input)) + + if max(number_of_inputs) > 1: + # Check that all tests have the same number of inputs + for test in tests: + if len(test[1]) < max(number_of_inputs): + for i in range(max(number_of_inputs) - len(test[1])): + test[1].append(" []") + + return tests + + def _create_new_test(self, name, inputs): + input_string = "" + for i, input in enumerate(inputs): + input_string += f" input[{i}] ={input}" + test = f""" + test("{name}") {{ + when {{ + process {{ + \""" + {input_string} + \""" + }} + }} + + then {{ + assertAll( + {{ assert process.success }}, + {{ assert snapshot(process.out.versions).match("versions") }} + {{ assert snapshot(process.out).match() }} + ) + }} + }} + """ + return test + + def _update_nftest_file(self): + """Update the nftest file with the pytest tests""" + test_script = self.file_paths[os.path.join(self.component_type, "tests", "main.nf.test")] + nextflow_config = Path( + self.directory, self.component_type, self.org, self.component_dir, "tests", "nextflow.config" + ) + pytest_tests = self._collect_pytest_tests() + in_input = False + # Update test script + new_lines = "" + with open(test_script, "r") as fh: + for line in fh: + if line.strip().startswith("script"): + new_lines += line + if nextflow_config.is_file(): + # Add nextflow config + new_lines += ' config "./nextflow.config"\n' + continue + elif line.strip().startswith("test"): + # Update test name + test_name = line.split('"') + test_name[1] = pytest_tests[0][0] + new_lines += '"'.join(test_name) + continue + elif line.strip().startswith("input"): + # Update input + in_input = True + new_lines += f" input[0] ={pytest_tests[0][1][0]}" + if len(pytest_tests[0][1]) > 1: + # Add more inputs + input_number = 1 + for input in pytest_tests[0][1][1:]: + new_lines += f" input[{input_number}] ={input}" + input_number += 1 + continue + elif in_input: + # While we are inside the process script defining inputs + if line.strip() == '"""': + in_input = False + new_lines += line + continue + elif line == " }\n": + # If we finished adding a test, check if there are more tests to add + new_lines += line + if len(pytest_tests) > 1: + for t in pytest_tests[1:]: + name, inputs = t + new_lines += self._create_new_test(name, inputs) + continue + elif "assert snapshot(process.out.versions).match(" in line: + new_lines += line + new_lines += " { assert snapshot(process.out).match() }\n" + continue + new_lines += line + with open(test_script, "w") as fh: + fh.write(new_lines) diff --git a/nf_core/modules/create.py b/nf_core/modules/create.py index b5368130ce..a5e0795a9f 100644 --- a/nf_core/modules/create.py +++ b/nf_core/modules/create.py @@ -17,6 +17,7 @@ def __init__( conda_name=None, conda_version=None, empty_template=False, + migrate_pytest=False, ): super().__init__( "modules", @@ -29,4 +30,5 @@ def __init__( conda_name, conda_version, empty_template, + migrate_pytest, ) diff --git a/nf_core/subworkflows/create.py b/nf_core/subworkflows/create.py index 963076455e..93e9f271be 100644 --- a/nf_core/subworkflows/create.py +++ b/nf_core/subworkflows/create.py @@ -12,6 +12,7 @@ def __init__( component="", author=None, force=False, + migrate_pytest=False, ): super().__init__( "subworkflows", @@ -19,4 +20,5 @@ def __init__( component, author, force=force, + migrate_pytest=migrate_pytest, ) From f7bc3f7025015ce4ff63cdaa9db2caf70010dcaf Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 29 Nov 2023 13:38:30 +0100 Subject: [PATCH 3/8] improve code thanks to @mashehu suggestions --- nf_core/components/create.py | 103 +++++++++++++---------------------- 1 file changed, 37 insertions(+), 66 deletions(-) diff --git a/nf_core/components/create.py b/nf_core/components/create.py index b13fc81454..1a8c7ce2a9 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -510,26 +510,11 @@ def _create_new_test(self, name, inputs): input_string = "" for i, input in enumerate(inputs): input_string += f" input[{i}] ={input}" - test = f""" - test("{name}") {{ - when {{ - process {{ - \""" - {input_string} - \""" - }} - }} - - then {{ - assertAll( - {{ assert process.success }}, - {{ assert snapshot(process.out.versions).match("versions") }} - {{ assert snapshot(process.out).match() }} - ) - }} - }} - """ - return test + with open(Path("../module-template/tests/main.nf.test"), "r") as fh: + test = fh.readlines()[15:48] + test[1] = f' test("{name}") {{' + test[12:20] = f" {input_string}" + return "".join(test) def _update_nftest_file(self): """Update the nftest file with the pytest tests""" @@ -538,52 +523,38 @@ def _update_nftest_file(self): self.directory, self.component_type, self.org, self.component_dir, "tests", "nextflow.config" ) pytest_tests = self._collect_pytest_tests() - in_input = False + subtool_lines = 0 + meta_lines = 0 + # Update test script - new_lines = "" with open(test_script, "r") as fh: - for line in fh: - if line.strip().startswith("script"): - new_lines += line - if nextflow_config.is_file(): - # Add nextflow config - new_lines += ' config "./nextflow.config"\n' - continue - elif line.strip().startswith("test"): - # Update test name - test_name = line.split('"') - test_name[1] = pytest_tests[0][0] - new_lines += '"'.join(test_name) - continue - elif line.strip().startswith("input"): - # Update input - in_input = True - new_lines += f" input[0] ={pytest_tests[0][1][0]}" - if len(pytest_tests[0][1]) > 1: - # Add more inputs - input_number = 1 - for input in pytest_tests[0][1][1:]: - new_lines += f" input[{input_number}] ={input}" - input_number += 1 - continue - elif in_input: - # While we are inside the process script defining inputs - if line.strip() == '"""': - in_input = False - new_lines += line - continue - elif line == " }\n": - # If we finished adding a test, check if there are more tests to add - new_lines += line - if len(pytest_tests) > 1: - for t in pytest_tests[1:]: - name, inputs = t - new_lines += self._create_new_test(name, inputs) - continue - elif "assert snapshot(process.out.versions).match(" in line: - new_lines += line - new_lines += " { assert snapshot(process.out).match() }\n" - continue - new_lines += line + main_nf_test = fh.readlines() + + if nextflow_config.is_file(): + # Add nextflow config + main_nf_test[5] += ' config "./nextflow.config"\n' + # Update test name + if self.subtool: + # Add 1 to the line index as we have the subtool tag + subtool_lines += 1 + main_nf_test[13 + subtool_lines] = f' test("{pytest_tests[0][0]}") {{\n' + # Update input + if self.has_meta: + meta_lines += 4 + main_nf_test[ + 25 + subtool_lines : 26 + subtool_lines + meta_lines + ] = f" input[0] ={pytest_tests[0][1][0]}" + # Add more inputs if we have more than one + if len(pytest_tests[0][1]) > 1: + input_number = 1 + for input in pytest_tests[0][1][1:]: + main_nf_test[25 + subtool_lines] += f" input[{input_number}] ={input}" + input_number += 1 + # Add more tests if we have more than one + if len(pytest_tests) > 1: + for t in pytest_tests[1:]: + name, inputs = t + main_nf_test[38 + subtool_lines] += self._create_new_test(name, inputs) + with open(test_script, "w") as fh: - fh.write(new_lines) + fh.write(main_nf_test) From 0f6e37ed9c9740e68190559fecaa55974c34fdf8 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 30 Nov 2023 13:52:22 +0100 Subject: [PATCH 4/8] don't modify main.nf.test, remove pytest folder, and add tests --- nf_core/components/create.py | 123 ++++++----------------------------- tests/modules/create.py | 60 ++++++++++++++++- tests/test_modules.py | 2 + 3 files changed, 81 insertions(+), 104 deletions(-) diff --git a/nf_core/components/create.py b/nf_core/components/create.py index 1a8c7ce2a9..db5807021d 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -162,12 +162,8 @@ def create(self): if self.migrate_pytest: self._copy_old_files(component_old_path) log.info("Migrate pytest tests: Copied original module files to new module") - try: - self._update_nftest_file() - log.info("Migrate pytest tests: Updated `main.nf.test` with contents of pytest") - except Exception as e: - log.info(f"Could not update `main.nf.test` file: {e}") shutil.rmtree(component_old_path) + self._print_and_delete_pytest_files() new_files = list(self.file_paths.values()) log.info("Created following files:\n " + "\n ".join(new_files)) @@ -459,102 +455,23 @@ def _copy_old_files(self, component_old_path): ) as ofh: ofh.write(config_lines) - def _collect_pytest_tests(self): + def _print_and_delete_pytest_files(self): + """Prompt if pytest files should be deleted and printed to stdout""" pytest_dir = Path(self.directory, "tests", self.component_type, self.org, self.component_dir) - tests = [] - name = None - input = None - in_input = False - input_number = 0 - number_of_inputs = [] - with open(pytest_dir / "main.nf") as fh: - for line in fh: - if line.strip().startswith("workflow"): - # One test - if name and input: - tests.append((name, input)) - name = None - input = None - name = line.split()[1] - elif line.strip().startswith("input"): - # First input - input = [line.split("=")[1]] - in_input = True - number_of_inputs.append(1) - elif "=" in line and "nextflow.enable.dsl" not in line: - # We need another input - input_number += 1 - in_input = True - input.append(line.split("=")[1]) - number_of_inputs[input_number] += 1 - elif in_input: - # Retrieve all lines of an input - if self.component_dir.replace("/", "_").upper() in line: - in_input = False - continue - input[input_number] += line - - if name and input: - tests.append((name, input)) - - if max(number_of_inputs) > 1: - # Check that all tests have the same number of inputs - for test in tests: - if len(test[1]) < max(number_of_inputs): - for i in range(max(number_of_inputs) - len(test[1])): - test[1].append(" []") - - return tests - - def _create_new_test(self, name, inputs): - input_string = "" - for i, input in enumerate(inputs): - input_string += f" input[{i}] ={input}" - with open(Path("../module-template/tests/main.nf.test"), "r") as fh: - test = fh.readlines()[15:48] - test[1] = f' test("{name}") {{' - test[12:20] = f" {input_string}" - return "".join(test) - - def _update_nftest_file(self): - """Update the nftest file with the pytest tests""" - test_script = self.file_paths[os.path.join(self.component_type, "tests", "main.nf.test")] - nextflow_config = Path( - self.directory, self.component_type, self.org, self.component_dir, "tests", "nextflow.config" - ) - pytest_tests = self._collect_pytest_tests() - subtool_lines = 0 - meta_lines = 0 - - # Update test script - with open(test_script, "r") as fh: - main_nf_test = fh.readlines() - - if nextflow_config.is_file(): - # Add nextflow config - main_nf_test[5] += ' config "./nextflow.config"\n' - # Update test name - if self.subtool: - # Add 1 to the line index as we have the subtool tag - subtool_lines += 1 - main_nf_test[13 + subtool_lines] = f' test("{pytest_tests[0][0]}") {{\n' - # Update input - if self.has_meta: - meta_lines += 4 - main_nf_test[ - 25 + subtool_lines : 26 + subtool_lines + meta_lines - ] = f" input[0] ={pytest_tests[0][1][0]}" - # Add more inputs if we have more than one - if len(pytest_tests[0][1]) > 1: - input_number = 1 - for input in pytest_tests[0][1][1:]: - main_nf_test[25 + subtool_lines] += f" input[{input_number}] ={input}" - input_number += 1 - # Add more tests if we have more than one - if len(pytest_tests) > 1: - for t in pytest_tests[1:]: - name, inputs = t - main_nf_test[38 + subtool_lines] += self._create_new_test(name, inputs) - - with open(test_script, "w") as fh: - fh.write(main_nf_test) + if rich.prompt.Confirm.ask( + "[violet]Do you want to delete pytest files?[/]\nPytest file 'main.nf' will be printed to standard output to allow copying the tests manually to 'main.nf.test'.", + default=False, + ): + with open(pytest_dir / "main.nf", "r") as fh: + log.info(fh.read()) + shutil.rmtree(pytest_dir) + log.info( + f"[yellow]Please copy the pytest tests to nf-test 'main.nf.test'.[/]" + f"You can find more information about nf-test [link=https://nf-co.re/docs/contributing/modules#migrating-from-pytest-to-nf-test]at the nf-core web[/link]. " + ) + else: + log.info( + f"[yellow]Please copy the pytest tests to nf-test 'main.nf.test'.[/]" + f"You can find more information about nf-test [link=https://nf-co.re/docs/contributing/modules#migrating-from-pytest-to-nf-test]at the nf-core web[/link]. " + f"Once done, make sure to delete the module pytest files to avoid linting errors: {pytest_dir}" + ) diff --git a/tests/modules/create.py b/tests/modules/create.py index 7cfba484a1..062761d059 100644 --- a/tests/modules/create.py +++ b/tests/modules/create.py @@ -1,11 +1,21 @@ +import filecmp import os +import shutil +from pathlib import Path +from unittest import mock import pytest import requests_cache import responses +from git.repo import Repo import nf_core.modules -from tests.utils import mock_anaconda_api_calls, mock_biocontainers_api_calls +from tests.utils import ( + GITLAB_SUBWORKFLOWS_ORG_PATH_BRANCH, + GITLAB_URL, + mock_anaconda_api_calls, + mock_biocontainers_api_calls, +) def test_modules_create_succeed(self): @@ -65,3 +75,51 @@ def test_modules_create_nfcore_modules_subtool(self): assert os.path.exists( os.path.join(self.nfcore_modules, "modules", "nf-core", "star", "index", "tests", "main.nf.test") ) + + +@mock.patch("rich.prompt.Confirm.ask") +def test_modules_migrate(self, mock_rich_ask): + """Create a module with the --migrate-pytest option to convert pytest to nf-test""" + pytest_dir = Path(self.nfcore_modules, "tests", "modules", "nf-core", "samtools", "sort") + + # Clone modules repo with pytests + shutil.rmtree(self.nfcore_modules) + Repo.clone_from(GITLAB_URL, self.nfcore_modules, branch=GITLAB_SUBWORKFLOWS_ORG_PATH_BRANCH) + old_main_nf = Path(self.nfcore_modules, "modules", "nf-core", "samtools", "sort", "main.nf") + old_meta_yml = Path(self.nfcore_modules, "modules", "nf-core", "samtools", "sort", "meta.yml") + + # Create a module with --migrate-pytest + mock_rich_ask.return_value = True + module_create = nf_core.modules.ModuleCreate(self.nfcore_modules, "samtools/sort", migrate_pytest=True) + module_create.create() + + new_main_nf = Path(self.nfcore_modules, "modules", "nf-core", "samtools", "sort", "main.nf") + new_meta_yml = Path(self.nfcore_modules, "modules", "nf-core", "samtools", "sort", "meta.yml") + nextflow_config = Path(self.nfcore_modules, "modules", "nf-core", "samtools", "sort", "tests", "nextflow.config") + + # Check that old files have been copied to the new module + assert filecmp.cmp(old_main_nf, new_main_nf, shallow=True) + assert filecmp.cmp(old_meta_yml, new_meta_yml, shallow=True) + assert nextflow_config.is_file() + + # Check that pytest folder is deleted + assert not pytest_dir.is_dir() + + +@mock.patch("rich.prompt.Confirm.ask") +def test_modules_migrate_no_delete(self, mock_rich_ask): + """Create a module with the --migrate-pytest option to convert pytest to nf-test. + Test that pytest directory is not deleted.""" + pytest_dir = Path(self.nfcore_modules, "tests", "modules", "nf-core", "samtools", "sort") + + # Clone modules repo with pytests + shutil.rmtree(self.nfcore_modules) + Repo.clone_from(GITLAB_URL, self.nfcore_modules, branch=GITLAB_SUBWORKFLOWS_ORG_PATH_BRANCH) + + # Create a module with --migrate-pytest + mock_rich_ask.return_value = False + module_create = nf_core.modules.ModuleCreate(self.nfcore_modules, "samtools/sort", migrate_pytest=True) + module_create.create() + + # Check that pytest folder is not deleted + assert pytest_dir.is_dir() diff --git a/tests/test_modules.py b/tests/test_modules.py index 21a28d5e46..e2e024a751 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -152,6 +152,8 @@ def test_modulesrepo_class(self): test_modules_create_nfcore_modules, test_modules_create_nfcore_modules_subtool, test_modules_create_succeed, + test_modules_migrate, + test_modules_migrate_no_delete, ) from .modules.info import ( # type: ignore[misc] test_modules_info_in_modules_repo, From 80ad88da5e74f6543066a26352288a70bbe5541c Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 30 Nov 2023 14:11:38 +0100 Subject: [PATCH 5/8] add subworkflows migrate-pytest tests --- nf_core/components/create.py | 8 ++--- tests/subworkflows/create.py | 60 ++++++++++++++++++++++++++++++++++++ tests/test_subworkflows.py | 2 ++ 3 files changed, 66 insertions(+), 4 deletions(-) diff --git a/nf_core/components/create.py b/nf_core/components/create.py index db5807021d..70fcfb21a0 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -466,12 +466,12 @@ def _print_and_delete_pytest_files(self): log.info(fh.read()) shutil.rmtree(pytest_dir) log.info( - f"[yellow]Please copy the pytest tests to nf-test 'main.nf.test'.[/]" - f"You can find more information about nf-test [link=https://nf-co.re/docs/contributing/modules#migrating-from-pytest-to-nf-test]at the nf-core web[/link]. " + "[yellow]Please copy the pytest tests to nf-test 'main.nf.test'.[/]\n" + "You can find more information about nf-test [link=https://nf-co.re/docs/contributing/modules#migrating-from-pytest-to-nf-test]at the nf-core web[/link]. " ) else: log.info( - f"[yellow]Please copy the pytest tests to nf-test 'main.nf.test'.[/]" - f"You can find more information about nf-test [link=https://nf-co.re/docs/contributing/modules#migrating-from-pytest-to-nf-test]at the nf-core web[/link]. " + "[yellow]Please copy the pytest tests to nf-test 'main.nf.test'.[/]\n" + "You can find more information about nf-test [link=https://nf-co.re/docs/contributing/modules#migrating-from-pytest-to-nf-test]at the nf-core web[/link].\n" f"Once done, make sure to delete the module pytest files to avoid linting errors: {pytest_dir}" ) diff --git a/tests/subworkflows/create.py b/tests/subworkflows/create.py index 94c2a66331..88f98b9453 100644 --- a/tests/subworkflows/create.py +++ b/tests/subworkflows/create.py @@ -1,8 +1,14 @@ +import filecmp import os +import shutil +from pathlib import Path +from unittest import mock import pytest +from git.repo import Repo import nf_core.subworkflows +from tests.utils import GITLAB_SUBWORKFLOWS_ORG_PATH_BRANCH, GITLAB_URL def test_subworkflows_create_succeed(self): @@ -35,3 +41,57 @@ def test_subworkflows_create_nfcore_modules(self): assert os.path.exists( os.path.join(self.nfcore_modules, "subworkflows", "nf-core", "test_subworkflow", "tests", "main.nf.test") ) + + +@mock.patch("rich.prompt.Confirm.ask") +def test_subworkflows_migrate(self, mock_rich_ask): + """Create a subworkflow with the --migrate-pytest option to convert pytest to nf-test""" + pytest_dir = Path(self.nfcore_modules, "tests", "subworkflows", "nf-core", "bam_stats_samtools") + + # Clone modules repo with pytests + shutil.rmtree(self.nfcore_modules) + Repo.clone_from(GITLAB_URL, self.nfcore_modules, branch=GITLAB_SUBWORKFLOWS_ORG_PATH_BRANCH) + old_main_nf = Path(self.nfcore_modules, "subworkflows", "nf-core", "bam_stats_samtools", "main.nf") + old_meta_yml = Path(self.nfcore_modules, "subworkflows", "nf-core", "bam_stats_samtools", "meta.yml") + + # Create a module with --migrate-pytest + mock_rich_ask.return_value = True + module_create = nf_core.subworkflows.SubworkflowCreate( + self.nfcore_modules, "bam_stats_samtools", migrate_pytest=True + ) + module_create.create() + + new_main_nf = Path(self.nfcore_modules, "subworkflows", "nf-core", "bam_stats_samtools", "main.nf") + new_meta_yml = Path(self.nfcore_modules, "subworkflows", "nf-core", "bam_stats_samtools", "meta.yml") + nextflow_config = Path( + self.nfcore_modules, "subworkflows", "nf-core", "bam_stats_samtools", "tests", "nextflow.config" + ) + + # Check that old files have been copied to the new module + assert filecmp.cmp(old_main_nf, new_main_nf, shallow=True) + assert filecmp.cmp(old_meta_yml, new_meta_yml, shallow=True) + assert nextflow_config.is_file() + + # Check that pytest folder is deleted + assert not pytest_dir.is_dir() + + +@mock.patch("rich.prompt.Confirm.ask") +def test_subworkflows_migrate_no_delete(self, mock_rich_ask): + """Create a subworkflow with the --migrate-pytest option to convert pytest to nf-test. + Test that pytest directory is not deleted.""" + pytest_dir = Path(self.nfcore_modules, "tests", "subworkflows", "nf-core", "bam_stats_samtools") + + # Clone modules repo with pytests + shutil.rmtree(self.nfcore_modules) + Repo.clone_from(GITLAB_URL, self.nfcore_modules, branch=GITLAB_SUBWORKFLOWS_ORG_PATH_BRANCH) + + # Create a module with --migrate-pytest + mock_rich_ask.return_value = False + module_create = nf_core.subworkflows.SubworkflowCreate( + self.nfcore_modules, "bam_stats_samtools", migrate_pytest=True + ) + module_create.create() + + # Check that pytest folder is not deleted + assert pytest_dir.is_dir() diff --git a/tests/test_subworkflows.py b/tests/test_subworkflows.py index 33cad81e3d..638a4bedb1 100644 --- a/tests/test_subworkflows.py +++ b/tests/test_subworkflows.py @@ -95,6 +95,8 @@ def tearDown(self): test_subworkflows_create_fail_exists, test_subworkflows_create_nfcore_modules, test_subworkflows_create_succeed, + test_subworkflows_migrate, + test_subworkflows_migrate_no_delete, ) from .subworkflows.info import ( # type: ignore[misc] test_subworkflows_info_in_modules_repo, From d03b7ae235fd4b7de55b6c26bb57489a9b041399 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 30 Nov 2023 14:55:56 +0100 Subject: [PATCH 6/8] delete component from pytest_modules.yml --- nf_core/components/create.py | 11 +++++++++++ tests/modules/create.py | 11 +++++++++++ tests/subworkflows/create.py | 11 +++++++++++ 3 files changed, 33 insertions(+) diff --git a/nf_core/components/create.py b/nf_core/components/create.py index 70fcfb21a0..b85acdc57f 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -17,11 +17,13 @@ import jinja2 import questionary import rich +import yaml from packaging.version import parse as parse_version import nf_core import nf_core.utils from nf_core.components.components_command import ComponentCommand +from nf_core.lint_utils import run_prettier_on_file log = logging.getLogger(__name__) @@ -475,3 +477,12 @@ def _print_and_delete_pytest_files(self): "You can find more information about nf-test [link=https://nf-co.re/docs/contributing/modules#migrating-from-pytest-to-nf-test]at the nf-core web[/link].\n" f"Once done, make sure to delete the module pytest files to avoid linting errors: {pytest_dir}" ) + # Delete tags from pytest_modules.yml + modules_yml = Path(self.directory, "tests", "config", "pytest_modules.yml") + with open(modules_yml, "r") as fh: + yml_file = yaml.safe_load(fh) + yml_key = self.component_dir if self.component_type == "modules" else f"subworkflows/{self.component_dir}" + del yml_file[yml_key] + with open(modules_yml, "w") as fh: + yaml.dump(yml_file, fh) + run_prettier_on_file(modules_yml) diff --git a/tests/modules/create.py b/tests/modules/create.py index 062761d059..9c855a11e6 100644 --- a/tests/modules/create.py +++ b/tests/modules/create.py @@ -7,6 +7,7 @@ import pytest import requests_cache import responses +import yaml from git.repo import Repo import nf_core.modules @@ -105,6 +106,11 @@ def test_modules_migrate(self, mock_rich_ask): # Check that pytest folder is deleted assert not pytest_dir.is_dir() + # Check that pytest_modules.yml is updated + with open(Path(self.nfcore_modules, "tests", "config", "pytest_modules.yml")) as fh: + modules_yml = yaml.safe_load(fh) + assert "samtools/sort" not in modules_yml.keys() + @mock.patch("rich.prompt.Confirm.ask") def test_modules_migrate_no_delete(self, mock_rich_ask): @@ -123,3 +129,8 @@ def test_modules_migrate_no_delete(self, mock_rich_ask): # Check that pytest folder is not deleted assert pytest_dir.is_dir() + + # Check that pytest_modules.yml is updated + with open(Path(self.nfcore_modules, "tests", "config", "pytest_modules.yml")) as fh: + modules_yml = yaml.safe_load(fh) + assert "samtools/sort" not in modules_yml.keys() diff --git a/tests/subworkflows/create.py b/tests/subworkflows/create.py index 88f98b9453..2855e68e2c 100644 --- a/tests/subworkflows/create.py +++ b/tests/subworkflows/create.py @@ -5,6 +5,7 @@ from unittest import mock import pytest +import yaml from git.repo import Repo import nf_core.subworkflows @@ -75,6 +76,11 @@ def test_subworkflows_migrate(self, mock_rich_ask): # Check that pytest folder is deleted assert not pytest_dir.is_dir() + # Check that pytest_modules.yml is updated + with open(Path(self.nfcore_modules, "tests", "config", "pytest_modules.yml")) as fh: + modules_yml = yaml.safe_load(fh) + assert "subworkflows/bam_stats_samtools" not in modules_yml.keys() + @mock.patch("rich.prompt.Confirm.ask") def test_subworkflows_migrate_no_delete(self, mock_rich_ask): @@ -95,3 +101,8 @@ def test_subworkflows_migrate_no_delete(self, mock_rich_ask): # Check that pytest folder is not deleted assert pytest_dir.is_dir() + + # Check that pytest_modules.yml is updated + with open(Path(self.nfcore_modules, "tests", "config", "pytest_modules.yml")) as fh: + modules_yml = yaml.safe_load(fh) + assert "subworkflows/bam_stats_samtools" not in modules_yml.keys() From 9fb0059c4c5b4f2b4429c20c1c8f7d7d7a6a2b76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Fri, 1 Dec 2023 11:24:08 +0100 Subject: [PATCH 7/8] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matthias Hörtenhuber --- CHANGELOG.md | 4 ++-- nf_core/components/create.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a467c75023..cb96e5633d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,13 +20,13 @@ - Added stub test creation to `create_test_yml` ([#2476](https://github.com/nf-core/tools/pull/2476)) - Replace ModulePatch by ComponentPatch ([#2482](https://github.com/nf-core/tools/pull/2482)) - Fixed `nf-core modules lint` to work with new module structure for nf-test ([#2494](https://github.com/nf-core/tools/pull/2494)) -- Add option `--migrate-pytest` to create a module with nf-test taking into account an existing module +- Add option `--migrate-pytest` to create a module with nf-test taking into account an existing module ([#2549](https://github.com/nf-core/tools/pull/2549)) ### Subworkflows - Added stub test creation to `create_test_yml` ([#2476](https://github.com/nf-core/tools/pull/2476)) - Fixed `nf-core subworkflows lint` to work with new module structure for nf-test ([#2494](https://github.com/nf-core/tools/pull/2494)) -- Add option `--migrate-pytest` to create a subworkflow with nf-test taking into account an existing subworkflow +- Add option `--migrate-pytest` to create a subworkflow with nf-test taking into account an existing subworkflow ([#2549](https://github.com/nf-core/tools/pull/2549)) ### General diff --git a/nf_core/components/create.py b/nf_core/components/create.py index b85acdc57f..5134dd199f 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -461,19 +461,19 @@ def _print_and_delete_pytest_files(self): """Prompt if pytest files should be deleted and printed to stdout""" pytest_dir = Path(self.directory, "tests", self.component_type, self.org, self.component_dir) if rich.prompt.Confirm.ask( - "[violet]Do you want to delete pytest files?[/]\nPytest file 'main.nf' will be printed to standard output to allow copying the tests manually to 'main.nf.test'.", + "[violet]Do you want to delete the pytest files?[/]\nPytest file 'main.nf' will be printed to standard output to allow migrating the tests manually to 'main.nf.test'.", default=False, ): with open(pytest_dir / "main.nf", "r") as fh: log.info(fh.read()) shutil.rmtree(pytest_dir) log.info( - "[yellow]Please copy the pytest tests to nf-test 'main.nf.test'.[/]\n" + "[yellow]Please convert the pytest tests to nf-test in 'main.nf.test'.[/]\n" "You can find more information about nf-test [link=https://nf-co.re/docs/contributing/modules#migrating-from-pytest-to-nf-test]at the nf-core web[/link]. " ) else: log.info( - "[yellow]Please copy the pytest tests to nf-test 'main.nf.test'.[/]\n" + "[yellow]Please migrate the pytest tests to nf-test in 'main.nf.test'.[/]\n" "You can find more information about nf-test [link=https://nf-co.re/docs/contributing/modules#migrating-from-pytest-to-nf-test]at the nf-core web[/link].\n" f"Once done, make sure to delete the module pytest files to avoid linting errors: {pytest_dir}" ) From 257e9e14eafb43ced403b96486f97ef4647c702d Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 4 Dec 2023 14:36:43 +0100 Subject: [PATCH 8/8] apply suggestions from code review and fix tests --- nf_core/components/create.py | 9 ++++++--- tests/modules/create.py | 19 ++++++++++++------- tests/subworkflows/create.py | 27 +++++++++++++++------------ 3 files changed, 33 insertions(+), 22 deletions(-) diff --git a/nf_core/components/create.py b/nf_core/components/create.py index 5134dd199f..2105104922 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -143,9 +143,12 @@ def create(self): if self.migrate_pytest: # Rename the component directory to old - component_old = self.component_dir + "_old" - component_old_path = Path(self.directory, self.component_type, self.org, component_old) - Path(self.directory, self.component_type, self.org, self.component_dir).rename(component_old_path) + component_old_dir = self.component_dir + "_old" + component_parent_path = Path(self.directory, self.component_type, self.org) + component_old_path = component_parent_path / component_old_dir + component_path = component_parent_path / self.component_dir + + component_path.rename(component_old_path) else: if self.component_type == "modules": # Try to find a bioconda package for 'component' diff --git a/tests/modules/create.py b/tests/modules/create.py index 9c855a11e6..74e5ec3896 100644 --- a/tests/modules/create.py +++ b/tests/modules/create.py @@ -82,25 +82,30 @@ def test_modules_create_nfcore_modules_subtool(self): def test_modules_migrate(self, mock_rich_ask): """Create a module with the --migrate-pytest option to convert pytest to nf-test""" pytest_dir = Path(self.nfcore_modules, "tests", "modules", "nf-core", "samtools", "sort") + module_dir = Path(self.nfcore_modules, "modules", "nf-core", "samtools", "sort") # Clone modules repo with pytests shutil.rmtree(self.nfcore_modules) Repo.clone_from(GITLAB_URL, self.nfcore_modules, branch=GITLAB_SUBWORKFLOWS_ORG_PATH_BRANCH) - old_main_nf = Path(self.nfcore_modules, "modules", "nf-core", "samtools", "sort", "main.nf") - old_meta_yml = Path(self.nfcore_modules, "modules", "nf-core", "samtools", "sort", "meta.yml") + with open(module_dir / "main.nf", "r") as fh: + old_main_nf = fh.read() + with open(module_dir / "meta.yml", "r") as fh: + old_meta_yml = fh.read() # Create a module with --migrate-pytest mock_rich_ask.return_value = True module_create = nf_core.modules.ModuleCreate(self.nfcore_modules, "samtools/sort", migrate_pytest=True) module_create.create() - new_main_nf = Path(self.nfcore_modules, "modules", "nf-core", "samtools", "sort", "main.nf") - new_meta_yml = Path(self.nfcore_modules, "modules", "nf-core", "samtools", "sort", "meta.yml") - nextflow_config = Path(self.nfcore_modules, "modules", "nf-core", "samtools", "sort", "tests", "nextflow.config") + with open(module_dir / "main.nf", "r") as fh: + new_main_nf = fh.read() + with open(module_dir / "meta.yml", "r") as fh: + new_meta_yml = fh.read() + nextflow_config = module_dir / "tests" / "nextflow.config" # Check that old files have been copied to the new module - assert filecmp.cmp(old_main_nf, new_main_nf, shallow=True) - assert filecmp.cmp(old_meta_yml, new_meta_yml, shallow=True) + assert old_main_nf == new_main_nf + assert old_meta_yml == new_meta_yml assert nextflow_config.is_file() # Check that pytest folder is deleted diff --git a/tests/subworkflows/create.py b/tests/subworkflows/create.py index 2855e68e2c..fc628df34f 100644 --- a/tests/subworkflows/create.py +++ b/tests/subworkflows/create.py @@ -48,29 +48,32 @@ def test_subworkflows_create_nfcore_modules(self): def test_subworkflows_migrate(self, mock_rich_ask): """Create a subworkflow with the --migrate-pytest option to convert pytest to nf-test""" pytest_dir = Path(self.nfcore_modules, "tests", "subworkflows", "nf-core", "bam_stats_samtools") + subworkflow_dir = Path(self.nfcore_modules, "subworkflows", "nf-core", "bam_stats_samtools") # Clone modules repo with pytests shutil.rmtree(self.nfcore_modules) Repo.clone_from(GITLAB_URL, self.nfcore_modules, branch=GITLAB_SUBWORKFLOWS_ORG_PATH_BRANCH) - old_main_nf = Path(self.nfcore_modules, "subworkflows", "nf-core", "bam_stats_samtools", "main.nf") - old_meta_yml = Path(self.nfcore_modules, "subworkflows", "nf-core", "bam_stats_samtools", "meta.yml") + with open(subworkflow_dir / "main.nf", "r") as fh: + old_main_nf = fh.read() + with open(subworkflow_dir / "meta.yml", "r") as fh: + old_meta_yml = fh.read() - # Create a module with --migrate-pytest + # Create a subworkflow with --migrate-pytest mock_rich_ask.return_value = True - module_create = nf_core.subworkflows.SubworkflowCreate( + subworkflow_create = nf_core.subworkflows.SubworkflowCreate( self.nfcore_modules, "bam_stats_samtools", migrate_pytest=True ) - module_create.create() + subworkflow_create.create() - new_main_nf = Path(self.nfcore_modules, "subworkflows", "nf-core", "bam_stats_samtools", "main.nf") - new_meta_yml = Path(self.nfcore_modules, "subworkflows", "nf-core", "bam_stats_samtools", "meta.yml") - nextflow_config = Path( - self.nfcore_modules, "subworkflows", "nf-core", "bam_stats_samtools", "tests", "nextflow.config" - ) + with open(subworkflow_dir / "main.nf", "r") as fh: + new_main_nf = fh.read() + with open(subworkflow_dir / "meta.yml", "r") as fh: + new_meta_yml = fh.read() + nextflow_config = subworkflow_dir / "tests" / "nextflow.config" # Check that old files have been copied to the new module - assert filecmp.cmp(old_main_nf, new_main_nf, shallow=True) - assert filecmp.cmp(old_meta_yml, new_meta_yml, shallow=True) + assert old_main_nf == new_main_nf + assert old_meta_yml == new_meta_yml assert nextflow_config.is_file() # Check that pytest folder is deleted