Create lockfile for Cargo package if missing

easybuild/easyblocks/generic/cargo.py

@@ -44,14 +44,17 @@
 import easybuild.tools.tomllib as tomllib
 from easybuild.framework.easyconfig import CUSTOM
 from easybuild.framework.extensioneasyblock import ExtensionEasyBlock
+from easybuild.tools import LooseVersion
 from easybuild.tools.build_log import EasyBuildError, print_warning
 from easybuild.tools.config import build_option
 from easybuild.tools.filetools import CHECKSUM_TYPE_SHA256, compute_checksum, copy_dir, dump_toml, extract_file, mkdir
 from easybuild.tools.filetools import read_file, remove_dir, write_file, which
+from easybuild.tools.modules import get_software_version
 from easybuild.tools.run import run_shell_cmd
 from easybuild.tools.toolchain.compiler import OPTARCH_GENERIC
 
 CRATESIO_SOURCE = "https://crates.io/api/v1/crates"
+CRATES_REGISTRY_URL = 'registry+https://github.com/rust-lang/crates.io-index'
 
 CONFIG_TOML_SOURCE_VENDOR = """
 [source.vendored-sources]
@@ -77,6 +80,14 @@
 replace-with = "vendored-sources"
 """
 
+CONFIG_LOCK_SOURCE = """
+[[package]]
+name = "{name}"
+version = "{version}"
+source = "{source}"
+# checksum intentionally not set
+"""
+
 CARGO_CHECKSUM_JSON = '{{"files": {{}}, "package": "{checksum}"}}'
 
 
@@ -214,30 +225,38 @@ def __init__(self, *args, **kwargs):
         self.cargo_home = os.path.join(self.builddir, '.cargo')
         self.set_cargo_vars()
 
-        # Populate sources from "crates" list of tuples
-        sources = []
-        for crate_info in self.crates:
-            if len(crate_info) == 2:
-                sources.append({
-                    'download_filename': self.crate_download_filename(*crate_info),
-                    'filename': self.crate_src_filename(*crate_info),
-                    'source_urls': [CRATESIO_SOURCE],
-                    'alt_location': 'crates.io',
-                })
-            else:
-                crate, version, repo, rev = crate_info
-                url, repo_name = repo.rsplit('/', maxsplit=1)
-                if repo_name.endswith('.git'):
-                    repo_name = repo_name[:-4]
-                sources.append({
-                    'git_config': {'url': url, 'repo_name': repo_name, 'commit': rev},
-                    'filename': self.crate_src_filename(crate, version, rev=rev),
-                })
-
         # copy EasyConfig instance before we make changes to it
         self.cfg = self.cfg.copy()
 
-        self.cfg.update('sources', sources)
+        if self.is_extension:
+            self.cfg['crates'] = self.options.get('crates', [])  # Don't inherit crates from parent
+            # The (regular) extract step for extensions is not run so our handling of crates as (multiple) sources
+            # cannot be used for extensions.
+            if self.crates:
+                raise EasyBuildError(f"Extension '{self.name}' cannot have crates. "
+                                     "You can add them to the top-level config parameters when using e.g."
+                                     "the Cargo or CargoPythonBundle easyblock.")
+        else:
+            # Populate sources from "crates" list of tuples
+            sources = []
+            for crate_info in self.crates:
+                if len(crate_info) == 2:
+                    sources.append({
+                        'download_filename': self.crate_download_filename(*crate_info),
+                        'filename': self.crate_src_filename(*crate_info),
+                        'source_urls': [CRATESIO_SOURCE],
+                        'alt_location': 'crates.io',
+                    })
+                else:
+                    crate, version, repo, rev = crate_info
+                    url, repo_name = repo.rsplit('/', maxsplit=1)
+                    if repo_name.endswith('.git'):
+                        repo_name = repo_name[:-4]
+                    sources.append({
+                        'git_config': {'url': url, 'repo_name': repo_name, 'commit': rev},
+                        'filename': self.crate_src_filename(crate, version, rev=rev),
+                    })
+            self.cfg.update('sources', sources)
 
     def set_cargo_vars(self):
         """Set environment variables for Rust compilation and Cargo"""
@@ -473,8 +492,48 @@ def prepare_step(self, *args, **kwargs):
         self.set_cargo_vars()
 
     def configure_step(self):
-        """Empty configuration step."""
-        pass
+        """Create lockfile if it doesn't exist"""
+        cargo_lock = 'Cargo.lock'
+        # Crates should be specified in the main easyconfig for extensions
+        if self.is_extension:
+            crates = self.master.crates
+        else:
+            crates = self.crates
+        if crates and os.path.exists('Cargo.toml') and not os.path.exists(cargo_lock):
+            locate_project_output = run_shell_cmd('cargo -q locate-project --message-format=plain --workspace').output
+            # Usually it is the only line, but there might be some prior messages like warnings
+            # Find path right path by going backwards through each line
+            try:
+                root_toml = next(p for p in locate_project_output.splitlines()[::-1] if os.path.exists(p))
+            except StopIteration:
+                self.log.warning("Failed to find project Cargo.toml. Skipping lockfile check & creation.")
+            else:
+                cargo_lock_path = os.path.join(os.path.dirname(root_toml), cargo_lock)
+                if not os.path.exists(cargo_lock_path):
+                    rust_version = LooseVersion(get_software_version('Rust'))
+                    # File format version, the latest supported is used for forward compatibility
+                    # Versions 1 and 2 were internal only, 2 being autodetected
+                    # 1.47 introduced the version marker as version 3
+                    # 1.78 marked version 4 as stable
+                    if rust_version < '1.47':
+                        version = None
+                    elif rust_version < '1.78':
+                        version = 3
+                    else:
+                        version = 4
+                    # Use vendored crates to ensure those versions are used
+                    self.log.info(f"No {cargo_lock} file found, creating one at {cargo_lock_path}")
+                    content = f'version = {version}\n' if version is not None else ''
+                    for crate_info in crates:
+                        if len(crate_info) == 2:
+                            name, version = crate_info
+                            source = CRATES_REGISTRY_URL
+                        else:
+                            name, version, repo, rev = crate_info
+                            source = f'git+{repo}?rev={rev}#{rev}'
+
+                        content += CONFIG_LOCK_SOURCE.format(name=name, version=version, source=source)
+                    write_file(cargo_lock_path, content)
 
     @property
     def profile(self):
@@ -564,7 +623,7 @@ def generate_crate_list(sourcedir):
         if name == app_name:
             app_in_cratesio = True  # exclude app itself, needs to be first in crates list or taken from pypi
         else:
-            if source_url == 'registry+https://github.com/rust-lang/crates.io-index':
+            if source_url == CRATES_REGISTRY_URL:
                 crates.append((name, version))
             else:
                 # Lock file has revision and branch in the url

easybuild/easyblocks/generic/cargopythonbundle.py

@@ -45,13 +45,15 @@ def extra_options(extra_vars=None):
         """Define extra easyconfig parameters specific to Cargo"""
         extra_vars = PythonBundle.extra_options(extra_vars)
         extra_vars = Cargo.extra_options(extra_vars)  # not all extra options here will used here
+        extra_vars['default_easyblock'][0] = 'CargoPythonPackage'
 
         return extra_vars
 
     def __init__(self, *args, **kwargs):
         """Constructor for CargoPythonBundle easyblock."""
         self.check_for_sources = False  # make Bundle allow sources (as crates are treated as sources)
         super().__init__(*args, **kwargs)
+        self.cfg['exts_defaultclass'] = 'CargoPythonPackage'
 
         # Cargo inherits from ExtensionEasyBlock, thus EB treats the software itself as an extension
         # Setting modulename to False to ensure that sanity checks are performed on the extensions only

easybuild/easyblocks/generic/cargopythonpackage.py

@@ -50,3 +50,8 @@ def extra_options(extra_vars=None):
     def extract_step(self):
         """Specifically use the overloaded variant from Cargo as is populates vendored sources with checksums."""
         return Cargo.extract_step(self)
+
+    def configure_step(self):
+        """Run configure for Cargo and PythonPackage"""
+        Cargo.configure_step(self)
+        PythonPackage.configure_step(self)