Merge pull request #1588 from mirpedrol/update_modules

Add option to update modules with linting

CHANGELOG.md

@@ -9,6 +9,7 @@
 ### Linting
 
 - Check that the `.prettierignore` file exists and that starts with the same content.
+- Add isort configuration and GitHub workflow ([#1538](https://github.com/nf-core/tools/pull/1538))
 
 ### General
 
@@ -18,12 +19,9 @@
 
 ### Modules
 
+- Add `--fix-version` flag to `nf-core modules lint` command to update modules to the latest version ([#1588](https://github.com/nf-core/tools/pull/1588))
 - Fix a bug in the regex extracting the version from biocontainers URLs [#1598](https://github.com/nf-core/tools/pull/1598)
 
-### Linting
-
-- Add isort configuration and GitHub workflow ([#1538](https://github.com/nf-core/tools/pull/1538))
-
 ## [v2.4.1 - Cobolt Koala Patch](https://github.com/nf-core/tools/releases/tag/2.4) - [2022-05-16]
 
 - Patch release to try to fix the template sync ([#1585](https://github.com/nf-core/tools/pull/1585))

nf_core/__main__.py

@@ -596,7 +596,8 @@ def create_test_yml(ctx, tool, run_tests, output, force, no_prompts):
 @click.option("-a", "--all", is_flag=True, help="Run on all modules")
 @click.option("--local", is_flag=True, help="Run additional lint tests for local modules")
 @click.option("--passed", is_flag=True, help="Show passed tests")
-def lint(ctx, tool, dir, key, all, local, passed):
+@click.option("--fix-version", is_flag=True, help="Fix the module version if a newer version is available")
+def lint(ctx, tool, dir, key, all, local, passed, fix_version):
     """
     Lint one or more modules in a directory.
 
@@ -609,7 +610,15 @@ def lint(ctx, tool, dir, key, all, local, passed):
     try:
         module_lint = nf_core.modules.ModuleLint(dir=dir)
         module_lint.modules_repo = ctx.obj["modules_repo_obj"]
-        module_lint.lint(module=tool, key=key, all_modules=all, print_results=True, local=local, show_passed=passed)
+        module_lint.lint(
+            module=tool,
+            key=key,
+            all_modules=all,
+            print_results=True,
+            local=local,
+            show_passed=passed,
+            fix_version=fix_version,
+        )
         if len(module_lint.failed) > 0:
             sys.exit(1)
     except nf_core.modules.lint.ModuleLintException as e:

nf_core/modules/lint/__init__.py

@@ -101,7 +101,16 @@ def __init__(self, dir):
     def _get_all_lint_tests():
         return ["main_nf", "meta_yml", "module_todos", "module_deprecations"]
 
-    def lint(self, module=None, key=(), all_modules=False, print_results=True, show_passed=False, local=False):
+    def lint(
+        self,
+        module=None,
+        key=(),
+        all_modules=False,
+        print_results=True,
+        show_passed=False,
+        local=False,
+        fix_version=False,
+    ):
         """
         Lint all or one specific module
 
@@ -118,6 +127,7 @@ def lint(self, module=None, key=(), all_modules=False, print_results=True, show_
         :param module:          A specific module to lint
         :param print_results:   Whether to print the linting results
         :param show_passed:     Whether passed tests should be shown as well
+        :param fix_version:     Update the module version if a newer version is available
 
         :returns:               A ModuleLint object containing information of
                                 the passed, warned and failed tests
@@ -174,11 +184,11 @@ def lint(self, module=None, key=(), all_modules=False, print_results=True, show_
 
         # Lint local modules
         if local and len(local_modules) > 0:
-            self.lint_modules(local_modules, local=True)
+            self.lint_modules(local_modules, local=True, fix_version=fix_version)
 
         # Lint nf-core modules
         if len(nfcore_modules) > 0:
-            self.lint_modules(nfcore_modules, local=False)
+            self.lint_modules(nfcore_modules, local=False, fix_version=fix_version)
 
         if print_results:
             self._print_results(show_passed=show_passed)
@@ -282,19 +292,21 @@ def get_installed_modules(self):
 
         return local_modules, nfcore_modules
 
-    def lint_modules(self, modules, local=False):
+    def lint_modules(self, modules, local=False, fix_version=False):
         """
         Lint a list of modules
 
         Args:
             modules ([NFCoreModule]): A list of module objects
             local (boolean): Whether the list consist of local or nf-core modules
+            fix_version (boolean): Fix the module version if a newer version is available
         """
         progress_bar = rich.progress.Progress(
             "[bold blue]{task.description}",
             rich.progress.BarColumn(bar_width=None),
             "[magenta]{task.completed} of {task.total}[reset] » [bold yellow]{task.fields[test_name]}",
             transient=True,
+            console=console,
         )
         with progress_bar:
             lint_progress = progress_bar.add_task(
@@ -305,9 +317,9 @@ def lint_modules(self, modules, local=False):
 
             for mod in modules:
                 progress_bar.update(lint_progress, advance=1, test_name=mod.module_name)
-                self.lint_module(mod, local=local)
+                self.lint_module(mod, progress_bar, local=local, fix_version=fix_version)
 
-    def lint_module(self, mod, local=False):
+    def lint_module(self, mod, progress_bar, local=False, fix_version=False):
         """
         Perform linting on one module
 
@@ -326,14 +338,17 @@ def lint_module(self, mod, local=False):
 
         # Only check the main script in case of a local module
         if local:
-            self.main_nf(mod)
+            self.main_nf(mod, fix_version, progress_bar)
             self.passed += [LintResult(mod, *m) for m in mod.passed]
             self.warned += [LintResult(mod, *m) for m in (mod.warned + mod.failed)]
 
         # Otherwise run all the lint tests
         else:
             for test_name in self.lint_tests:
-                getattr(self, test_name)(mod)
+                if test_name == "main_nf":
+                    getattr(self, test_name)(mod, fix_version, progress_bar)
+                else:
+                    getattr(self, test_name)(mod)
 
             self.passed += [LintResult(mod, *m) for m in mod.passed]
             self.warned += [LintResult(mod, *m) for m in mod.warned]

nf_core/modules/lint/main_nf.py

@@ -3,12 +3,19 @@
 Lint the main.nf file of a module
 """
 
+import logging
 import re
 
+import requests
+from galaxy.tool_util.deps.mulled.util import build_target
+
 import nf_core
+import nf_core.modules.module_utils
+
+log = logging.getLogger(__name__)
 
 
-def main_nf(module_lint_object, module):
+def main_nf(module_lint_object, module, fix_version, progress_bar):
     """
     Lint a ``main.nf`` module file
 
@@ -100,7 +107,7 @@ def main_nf(module_lint_object, module):
         module.passed.append(("main_nf_script_outputs", "Process 'output' block found", module.main_nf))
 
     # Check the process definitions
-    if check_process_section(module, process_lines):
+    if check_process_section(module, process_lines, fix_version, progress_bar):
         module.passed.append(("main_nf_container", "Container versions match", module.main_nf))
     else:
         module.warned.append(("main_nf_container", "Container versions do not match", module.main_nf))
@@ -190,7 +197,7 @@ def check_when_section(self, lines):
         self.passed.append(("when_condition", "when: condition is unchanged", self.main_nf))
 
 
-def check_process_section(self, lines):
+def check_process_section(self, lines, fix_version, progress_bar):
     """
     Lint the section of a module between the process definition
     and the 'input:' definition
@@ -236,14 +243,14 @@ def check_process_section(self, lines):
         self.warned.append(("process_standard_label", "Process label unspecified", self.main_nf))
 
     for l in lines:
-        if re.search("bioconda::", l):
+        if _container_type(l) == "bioconda":
             bioconda_packages = [b for b in l.split() if "bioconda::" in b]
         l = l.strip(" '\"")
-        if l.startswith("https://containers") or l.startswith("https://depot"):
+        if _container_type(l) == "singularity":
             # e.g. "https://containers.biocontainers.pro/s3/SingImgsRepo/biocontainers/v1.2.0_cv1/biocontainers_v1.2.0_cv1.img' :" -> v1.2.0_cv1
             # e.g. "https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0' :" -> 0.11.9--0
             singularity_tag = re.search(r"(?:/)?(?:biocontainers_)?(?::)?([A-Za-z\d\-_.]+?)(?:\.img)?['\"]", l).group(1)
-        if l.startswith("biocontainers/") or l.startswith("quay.io/"):
+        if _container_type(l) == "docker":
             # e.g. "quay.io/biocontainers/krona:2.7.1--pl526_5' }" -> 2.7.1--pl526_5
             # e.g. "biocontainers/biocontainers:v1.2.0_cv1' }" -> v1.2.0_cv1
             docker_tag = re.search(r"(?:[/])?(?::)?([A-Za-z\d\-_.]+)['\"]", l).group(1)
@@ -272,9 +279,30 @@ def check_process_section(self, lines):
             last_ver = response.get("latest_version")
             if last_ver is not None and last_ver != bioconda_version:
                 package, ver = bp.split("=", 1)
-                self.warned.append(
-                    ("bioconda_latest", f"Conda update: {package} `{ver}` -> `{last_ver}`", self.main_nf)
-                )
+                # If a new version is available and fix is True, update the version
+                if fix_version:
+                    if _fix_module_version(self, bioconda_version, last_ver, singularity_tag, response):
+                        progress_bar.print(f"[blue]INFO[/blue]\t Updating package '{package}' {ver} -> {last_ver}")
+                        log.debug(f"Updating package {package} {ver} -> {last_ver}")
+                        self.passed.append(
+                            (
+                                "bioconda_latest",
+                                f"Conda package has been updated to the latest available: `{bp}`",
+                                self.main_nf,
+                            )
+                        )
+                    else:
+                        progress_bar.print(
+                            f"[blue]INFO[/blue]\t Tried to update package. Unable to update package '{package}' {ver} -> {last_ver}"
+                        )
+                        log.debug(f"Unable to update package {package} {ver} -> {last_ver}")
+                        self.warned.append(
+                            ("bioconda_latest", f"Conda update: {package} `{ver}` -> `{last_ver}`", self.main_nf)
+                        )
+                else:
+                    self.warned.append(
+                        ("bioconda_latest", f"Conda update: {package} `{ver}` -> `{last_ver}`", self.main_nf)
+                    )
             else:
                 self.passed.append(("bioconda_latest", f"Conda package is the latest available: `{bp}`", self.main_nf))
 
@@ -341,3 +369,70 @@ def _is_empty(self, line):
     if line.strip().replace(" ", "") == "":
         empty = True
     return empty
+
+
+def _fix_module_version(self, current_version, latest_version, singularity_tag, response):
+    """Updates the module version
+
+    Changes the bioconda current version by the latest version.
+    Obtains the latest build from bioconda response
+    Checks that the new URLs for docker and singularity with the tag [version]--[build] are valid
+    Changes the docker and singularity URLs
+    """
+    # Get latest build
+    build = _get_build(response)
+
+    with open(self.main_nf, "r") as source:
+        lines = source.readlines()
+
+    # Check if the new version + build exist and replace
+    new_lines = []
+    for line in lines:
+        l = line.strip(" '\"")
+        build_type = _container_type(l)
+        if build_type == "bioconda":
+            new_lines.append(re.sub(rf"{current_version}", f"{latest_version}", line))
+        elif build_type == "singularity" or build_type == "docker":
+            # Check that the new url is valid
+            new_url = re.search(
+                "(?:')(.+)(?:')", re.sub(rf"{singularity_tag}", f"{latest_version}--{build}", line)
+            ).group(1)
+            response_new_container = requests.get(
+                "https://" + new_url if not new_url.startswith("https://") else new_url, stream=True
+            )
+            log.debug(
+                f"Connected to URL: {'https://' + new_url if not new_url.startswith('https://') else new_url}, status_code: {response_new_container.status_code}"
+            )
+            if response_new_container.status_code != 200:
+                return False
+            new_lines.append(re.sub(rf"{singularity_tag}", f"{latest_version}--{build}", line))
+        else:
+            new_lines.append(line)
+
+    # Replace outdated versions by the latest one
+    with open(self.main_nf, "w") as source:
+        for line in new_lines:
+            source.write(line)
+
+    return True
+
+
+def _get_build(response):
+    """Get the latest build of the container version"""
+    build_times = []
+    latest_v = response.get("latest_version")
+    files = response.get("files")
+    for f in files:
+        if f.get("version") == latest_v:
+            build_times.append((f.get("upload_time"), f.get("attrs").get("build")))
+    return sorted(build_times, key=lambda tup: tup[0], reverse=True)[0][1]
+
+
+def _container_type(line):
+    """Returns the container type of a build."""
+    if re.search("bioconda::", line):
+        return "bioconda"
+    if line.startswith("https://containers") or line.startswith("https://depot"):
+        return "singularity"
+    if line.startswith("biocontainers/") or line.startswith("quay.io/"):
+        return "docker"