Lint for version captures in modules

nf_core/modules/lint/module_tests.py

@@ -15,6 +15,35 @@
 log = logging.getLogger(__name__)
 
 
+def _contains_version_hash(test_content):
+    """
+    Check if test content contains version information in hash format.
+
+    Uses precise regex patterns to detect version hash formats while avoiding
+    false positives from similar strings.
+
+    Args:
+        test_content: Content of a single test from snapshot
+
+    Returns:
+        bool: True if hash format detected, False otherwise
+    """
+    # More precise regex patterns with proper boundaries
+    version_hash_patterns = [
+        r"\bversions\.yml:md5,[a-f0-9]{32}\b",  # Exact MD5 format (32 hex chars)
+        r"\bversions\.yml:sha[0-9]*,[a-f0-9]+\b",  # SHA format with variable length
+    ]
+
+    # Convert to string only once and search efficiently
+    content_str = str(test_content)
+
+    for pattern in version_hash_patterns:
+        if re.search(pattern, content_str):
+            return True
+
+    return False
+
+
 def module_tests(_, module: NFCoreComponent, allow_missing: bool = False):
     """
     Lint the tests of a module in ``nf-core/modules``
@@ -162,22 +191,43 @@ def module_tests(_, module: NFCoreComponent, allow_missing: bool = False):
                                             snap_file,
                                         )
                                     )
-                            if "versions" in str(snap_content[test_name]) or "versions" in str(snap_content.keys()):
-                                module.passed.append(
-                                    (
-                                        "test_snap_versions",
-                                        "versions found in snapshot file",
-                                        snap_file,
+                                if "versions" in str(snap_content[test_name]) or "versions" in str(snap_content.keys()):
+                                    module.passed.append(
+                                        (
+                                            "test_snap_versions",
+                                            "versions found in snapshot file",
+                                            snap_file,
+                                        )
                                     )
-                                )
-                            else:
-                                module.failed.append(
-                                    (
-                                        "test_snap_versions",
-                                        "versions not found in snapshot file",
-                                        snap_file,
+                                    # Check if version content is actual content vs MD5/SHA hash
+                                    # Related to: https://github.com/nf-core/modules/issues/6505
+                                    # Ensures version snapshots contain actual content instead of hash values
+                                    if _contains_version_hash(snap_content[test_name]):
+                                        # Invalid - contains hash format
+                                        module.failed.append(
+                                            (
+                                                "test_snap_version_content",
+                                                "Version information should contain actual YAML content (e.g., {'tool': {'version': '1.0'}}), not hash format like 'versions.yml:md5,hash'",
+                                                snap_file,
+                                            )
+                                        )
+                                    else:
+                                        # Valid - either contains actual content or no version hash detected
+                                        module.passed.append(
+                                            (
+                                                "test_snap_version_content",
+                                                "version information contains actual content instead of hash",
+                                                snap_file,
+                                            )
+                                        )
+                                else:
+                                    module.failed.append(
+                                        (
+                                            "test_snap_versions",
+                                            "versions not found in snapshot file",
+                                            snap_file,
+                                        )
                                     )
-                                )
                         except json.decoder.JSONDecodeError as e:
                             module.failed.append(
                                 (

pyproject.toml

@@ -3,7 +3,11 @@ build-backend = "setuptools.build_meta"
 requires = ["setuptools>=40.6.0", "wheel"]
 
 [tool.pytest.ini_options]
-markers = ["datafiles: load datafiles", "integration"]
+markers = [
+    "datafiles: load datafiles",
+    "integration",
+    "issue: mark test with related issue URL"
+]
 testpaths = ["tests"]
 python_files = ["test_*.py"]
 asyncio_mode = "auto"

tests/modules/lint/test_module_tests.py

@@ -1,6 +1,7 @@
 import json
 from pathlib import Path
 
+import pytest
 from git.repo import Repo
 
 import nf_core.modules.lint
@@ -134,12 +135,20 @@ def test_nftest_failing_linting(self):
         module_lint = nf_core.modules.lint.ModuleLint(directory=self.nfcore_modules)
         module_lint.lint(print_results=False, module="kallisto/quant")
 
-        assert len(module_lint.failed) == 2, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}"
+        assert len(module_lint.failed) == 4, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}"
         assert len(module_lint.passed) >= 0
         assert len(module_lint.warned) >= 0
-        assert module_lint.failed[0].lint_test == "meta_yml_valid"
-        assert module_lint.failed[1].lint_test == "test_main_tags"
-        assert "kallisto/index" in module_lint.failed[1].message
+
+        # Check for expected failure types
+        failed_tests = [x.lint_test for x in module_lint.failed]
+        assert "meta_yml_valid" in failed_tests
+        assert "test_main_tags" in failed_tests
+        assert failed_tests.count("test_snap_version_content") == 2  # Should appear twice for the two version entries
+
+        # Check test_main_tags failure contains the expected message
+        main_tags_failures = [x for x in module_lint.failed if x.lint_test == "test_main_tags"]
+        assert len(main_tags_failures) == 1
+        assert "kallisto/index" in main_tags_failures[0].message
 
     def test_modules_absent_version(self):
         """Test linting a nf-test module if the versions is absent in the snapshot file `"""
@@ -213,3 +222,174 @@ def test_modules_empty_file_in_stub_snapshot(self):
         # reset the file
         with open(snap_file, "w") as fh:
             fh.write(content)
+
+    @pytest.mark.issue("https://github.com/nf-core/modules/issues/6505")
+    def test_modules_version_snapshot_content_md5_hash(self):
+        """Test linting a nf-test module with version information as MD5 hash instead of actual content, which should fail.
+
+        Related to: https://github.com/nf-core/modules/issues/6505
+        Fixed in: https://github.com/nf-core/tools/pull/3676
+        """
+        snap_file = self.bpipe_test_module_path / "tests" / "main.nf.test.snap"
+        snap = json.load(snap_file.open())
+        content = snap_file.read_text()
+
+        # Add a version entry with MD5 hash format (the old way that should be flagged)
+        snap["my test"]["content"][0]["versions"] = "versions.yml:md5,949da9c6297b613b50e24c421576f3f1"
+
+        with open(snap_file, "w") as fh:
+            json.dump(snap, fh)
+
+        module_lint = nf_core.modules.lint.ModuleLint(directory=self.nfcore_modules)
+        module_lint.lint(print_results=False, module="bpipe/test")
+
+        # Should fail because version is using MD5 hash instead of actual content
+        # Filter for only our specific test
+        version_content_failures = [x for x in module_lint.failed if x.lint_test == "test_snap_version_content"]
+        assert len(version_content_failures) == 1, (
+            f"Expected 1 test_snap_version_content failure, got {len(version_content_failures)}"
+        )
+        assert version_content_failures[0].lint_test == "test_snap_version_content"
+
+        # reset the file
+        with open(snap_file, "w") as fh:
+            fh.write(content)
+
+    @pytest.mark.issue("https://github.com/nf-core/modules/issues/6505")
+    def test_modules_version_snapshot_content_valid(self):
+        """Test linting a nf-test module with version information as actual content, which should pass.
+
+        Related to: https://github.com/nf-core/modules/issues/6505
+        Fixed in: https://github.com/nf-core/tools/pull/3676
+        """
+        snap_file = self.bpipe_test_module_path / "tests" / "main.nf.test.snap"
+        snap = json.load(snap_file.open())
+        content = snap_file.read_text()
+
+        # Add a version entry with actual content (the new way that should pass)
+        snap["my test"]["content"][0]["versions"] = {"ALE": {"ale": "20180904"}}
+
+        with open(snap_file, "w") as fh:
+            json.dump(snap, fh)
+
+        module_lint = nf_core.modules.lint.ModuleLint(directory=self.nfcore_modules)
+        module_lint.lint(print_results=False, module="bpipe/test")
+
+        # Should pass because version contains actual content
+        # Filter for only our specific test
+        version_content_failures = [x for x in module_lint.failed if x.lint_test == "test_snap_version_content"]
+        assert len(version_content_failures) == 0, (
+            f"Expected 0 test_snap_version_content failures, got {len(version_content_failures)}"
+        )
+
+        # Check for test_snap_version_content in passed tests
+        version_content_passed = [
+            x
+            for x in module_lint.passed
+            if (hasattr(x, "lint_test") and x.lint_test == "test_snap_version_content")
+            or (isinstance(x, tuple) and len(x) > 0 and x[0] == "test_snap_version_content")
+        ]
+        assert len(version_content_passed) > 0, "test_snap_version_content not found in passed tests"
+
+        # reset the file
+        with open(snap_file, "w") as fh:
+            fh.write(content)
+
+    @pytest.mark.issue("https://github.com/nf-core/modules/issues/6505")
+    def test_modules_version_snapshot_content_sha_hash(self):
+        """Test linting a nf-test module with version information as SHA hash, which should fail.
+
+        Related to: https://github.com/nf-core/modules/issues/6505
+        Fixed in: https://github.com/nf-core/tools/pull/3676
+        """
+        snap_file = self.bpipe_test_module_path / "tests" / "main.nf.test.snap"
+        snap = json.load(snap_file.open())
+        content = snap_file.read_text()
+
+        # Add a version entry with SHA hash format (should be flagged)
+        snap["my test"]["content"][0]["versions"] = (
+            "versions.yml:sha256,e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+        )
+
+        with open(snap_file, "w") as fh:
+            json.dump(snap, fh)
+
+        module_lint = nf_core.modules.lint.ModuleLint(directory=self.nfcore_modules)
+        module_lint.lint(print_results=False, module="bpipe/test")
+
+        # Should fail because version is using SHA hash instead of actual content
+        version_content_failures = [x for x in module_lint.failed if x.lint_test == "test_snap_version_content"]
+        assert len(version_content_failures) == 1, (
+            f"Expected 1 test_snap_version_content failure, got {len(version_content_failures)}"
+        )
+
+        # reset the file
+        with open(snap_file, "w") as fh:
+            fh.write(content)
+
+    @pytest.mark.issue("https://github.com/nf-core/modules/issues/6505")
+    def test_modules_version_snapshot_content_mixed_scenario(self):
+        """Test linting with mixed version content - some valid, some hash format.
+
+        Related to: https://github.com/nf-core/modules/issues/6505
+        Fixed in: https://github.com/nf-core/tools/pull/3676
+        """
+        snap_file = self.bpipe_test_module_path / "tests" / "main.nf.test.snap"
+        snap = json.load(snap_file.open())
+        content = snap_file.read_text()
+
+        # Create a scenario with multiple tests - one with hash, one with valid content
+        snap["test_with_hash"] = {"content": [{"versions": "versions.yml:md5,949da9c6297b613b50e24c421576f3f1"}]}
+        snap["test_with_valid_content"] = {"content": [{"versions": {"BPIPE": {"bpipe": "0.9.11"}}}]}
+
+        with open(snap_file, "w") as fh:
+            json.dump(snap, fh)
+
+        module_lint = nf_core.modules.lint.ModuleLint(directory=self.nfcore_modules)
+        module_lint.lint(print_results=False, module="bpipe/test")
+
+        # Should have failure for the hash test
+        version_content_failures = [x for x in module_lint.failed if x.lint_test == "test_snap_version_content"]
+        assert len(version_content_failures) >= 1, "Expected at least 1 failure for hash format"
+
+        # Should have pass for the valid content test
+        version_content_passed = [
+            x
+            for x in module_lint.passed
+            if (hasattr(x, "lint_test") and x.lint_test == "test_snap_version_content")
+            or (isinstance(x, tuple) and len(x) > 0 and x[0] == "test_snap_version_content")
+        ]
+        assert len(version_content_passed) >= 1, "Expected at least 1 pass for valid content"
+
+        # reset the file
+        with open(snap_file, "w") as fh:
+            fh.write(content)
+
+    @pytest.mark.issue("https://github.com/nf-core/modules/issues/6505")
+    def test_modules_version_snapshot_no_version_content(self):
+        """Test linting when no version information is present - should not trigger version content check.
+
+        Related to: https://github.com/nf-core/modules/issues/6505
+        Fixed in: https://github.com/nf-core/tools/pull/3676
+        """
+        snap_file = self.bpipe_test_module_path / "tests" / "main.nf.test.snap"
+        snap = json.load(snap_file.open())
+        content = snap_file.read_text()
+
+        # Remove version information entirely
+        if "content" in snap["my test"] and snap["my test"]["content"]:
+            snap["my test"]["content"][0].pop("versions", None)
+
+        with open(snap_file, "w") as fh:
+            json.dump(snap, fh)
+
+        module_lint = nf_core.modules.lint.ModuleLint(directory=self.nfcore_modules)
+        module_lint.lint(print_results=False, module="bpipe/test")
+
+        # Should not have version content check failures when no version data present
+        version_content_failures = [x for x in module_lint.failed if x.lint_test == "test_snap_version_content"]
+        assert len(version_content_failures) == 0, "Should not have version content failures when no versions present"
+
+        # reset the file
+        with open(snap_file, "w") as fh:
+            fh.write(content)