diff --git a/easybuild/easyblocks/generic/cargo.py b/easybuild/easyblocks/generic/cargo.py index 2808408628a..2fddd2743df 100755 --- a/easybuild/easyblocks/generic/cargo.py +++ b/easybuild/easyblocks/generic/cargo.py @@ -44,14 +44,17 @@ import easybuild.tools.tomllib as tomllib from easybuild.framework.easyconfig import CUSTOM from easybuild.framework.extensioneasyblock import ExtensionEasyBlock +from easybuild.tools import LooseVersion from easybuild.tools.build_log import EasyBuildError, print_warning from easybuild.tools.config import build_option from easybuild.tools.filetools import CHECKSUM_TYPE_SHA256, compute_checksum, copy_dir, dump_toml, extract_file, mkdir from easybuild.tools.filetools import read_file, remove_dir, write_file, which +from easybuild.tools.modules import get_software_version from easybuild.tools.run import run_shell_cmd from easybuild.tools.toolchain.compiler import OPTARCH_GENERIC CRATESIO_SOURCE = "https://crates.io/api/v1/crates" +CRATES_REGISTRY_URL = 'registry+https://github.com/rust-lang/crates.io-index' CONFIG_TOML_SOURCE_VENDOR = """ [source.vendored-sources] @@ -77,6 +80,14 @@ replace-with = "vendored-sources" """ +CONFIG_LOCK_SOURCE = """ +[[package]] +name = "{name}" +version = "{version}" +source = "{source}" +# checksum intentionally not set +""" + CARGO_CHECKSUM_JSON = '{{"files": {{}}, "package": "{checksum}"}}' @@ -214,30 +225,38 @@ def __init__(self, *args, **kwargs): self.cargo_home = os.path.join(self.builddir, '.cargo') self.set_cargo_vars() - # Populate sources from "crates" list of tuples - sources = [] - for crate_info in self.crates: - if len(crate_info) == 2: - sources.append({ - 'download_filename': self.crate_download_filename(*crate_info), - 'filename': self.crate_src_filename(*crate_info), - 'source_urls': [CRATESIO_SOURCE], - 'alt_location': 'crates.io', - }) - else: - crate, version, repo, rev = crate_info - url, repo_name = repo.rsplit('/', maxsplit=1) - if repo_name.endswith('.git'): - repo_name = repo_name[:-4] - sources.append({ - 'git_config': {'url': url, 'repo_name': repo_name, 'commit': rev}, - 'filename': self.crate_src_filename(crate, version, rev=rev), - }) - # copy EasyConfig instance before we make changes to it self.cfg = self.cfg.copy() - self.cfg.update('sources', sources) + if self.is_extension: + self.cfg['crates'] = self.options.get('crates', []) # Don't inherit crates from parent + # The (regular) extract step for extensions is not run so our handling of crates as (multiple) sources + # cannot be used for extensions. + if self.crates: + raise EasyBuildError(f"Extension '{self.name}' cannot have crates. " + "You can add them to the top-level config parameters when using e.g." + "the Cargo or CargoPythonBundle easyblock.") + else: + # Populate sources from "crates" list of tuples + sources = [] + for crate_info in self.crates: + if len(crate_info) == 2: + sources.append({ + 'download_filename': self.crate_download_filename(*crate_info), + 'filename': self.crate_src_filename(*crate_info), + 'source_urls': [CRATESIO_SOURCE], + 'alt_location': 'crates.io', + }) + else: + crate, version, repo, rev = crate_info + url, repo_name = repo.rsplit('/', maxsplit=1) + if repo_name.endswith('.git'): + repo_name = repo_name[:-4] + sources.append({ + 'git_config': {'url': url, 'repo_name': repo_name, 'commit': rev}, + 'filename': self.crate_src_filename(crate, version, rev=rev), + }) + self.cfg.update('sources', sources) def set_cargo_vars(self): """Set environment variables for Rust compilation and Cargo""" @@ -473,8 +492,48 @@ def prepare_step(self, *args, **kwargs): self.set_cargo_vars() def configure_step(self): - """Empty configuration step.""" - pass + """Create lockfile if it doesn't exist""" + cargo_lock = 'Cargo.lock' + # Crates should be specified in the main easyconfig for extensions + if self.is_extension: + crates = self.master.crates + else: + crates = self.crates + if crates and os.path.exists('Cargo.toml') and not os.path.exists(cargo_lock): + locate_project_output = run_shell_cmd('cargo -q locate-project --message-format=plain --workspace').output + # Usually it is the only line, but there might be some prior messages like warnings + # Find path right path by going backwards through each line + try: + root_toml = next(p for p in locate_project_output.splitlines()[::-1] if os.path.exists(p)) + except StopIteration: + self.log.warning("Failed to find project Cargo.toml. Skipping lockfile check & creation.") + else: + cargo_lock_path = os.path.join(os.path.dirname(root_toml), cargo_lock) + if not os.path.exists(cargo_lock_path): + rust_version = LooseVersion(get_software_version('Rust')) + # File format version, the latest supported is used for forward compatibility + # Versions 1 and 2 were internal only, 2 being autodetected + # 1.47 introduced the version marker as version 3 + # 1.78 marked version 4 as stable + if rust_version < '1.47': + version = None + elif rust_version < '1.78': + version = 3 + else: + version = 4 + # Use vendored crates to ensure those versions are used + self.log.info(f"No {cargo_lock} file found, creating one at {cargo_lock_path}") + content = f'version = {version}\n' if version is not None else '' + for crate_info in crates: + if len(crate_info) == 2: + name, version = crate_info + source = CRATES_REGISTRY_URL + else: + name, version, repo, rev = crate_info + source = f'git+{repo}?rev={rev}#{rev}' + + content += CONFIG_LOCK_SOURCE.format(name=name, version=version, source=source) + write_file(cargo_lock_path, content) @property def profile(self): @@ -564,7 +623,7 @@ def generate_crate_list(sourcedir): if name == app_name: app_in_cratesio = True # exclude app itself, needs to be first in crates list or taken from pypi else: - if source_url == 'registry+https://github.com/rust-lang/crates.io-index': + if source_url == CRATES_REGISTRY_URL: crates.append((name, version)) else: # Lock file has revision and branch in the url diff --git a/easybuild/easyblocks/generic/cargopythonbundle.py b/easybuild/easyblocks/generic/cargopythonbundle.py index 137589400b2..fb9878766d9 100644 --- a/easybuild/easyblocks/generic/cargopythonbundle.py +++ b/easybuild/easyblocks/generic/cargopythonbundle.py @@ -45,6 +45,7 @@ def extra_options(extra_vars=None): """Define extra easyconfig parameters specific to Cargo""" extra_vars = PythonBundle.extra_options(extra_vars) extra_vars = Cargo.extra_options(extra_vars) # not all extra options here will used here + extra_vars['default_easyblock'][0] = 'CargoPythonPackage' return extra_vars @@ -52,6 +53,7 @@ def __init__(self, *args, **kwargs): """Constructor for CargoPythonBundle easyblock.""" self.check_for_sources = False # make Bundle allow sources (as crates are treated as sources) super().__init__(*args, **kwargs) + self.cfg['exts_defaultclass'] = 'CargoPythonPackage' # Cargo inherits from ExtensionEasyBlock, thus EB treats the software itself as an extension # Setting modulename to False to ensure that sanity checks are performed on the extensions only diff --git a/easybuild/easyblocks/generic/cargopythonpackage.py b/easybuild/easyblocks/generic/cargopythonpackage.py index 5a073333384..6580c78f9b1 100644 --- a/easybuild/easyblocks/generic/cargopythonpackage.py +++ b/easybuild/easyblocks/generic/cargopythonpackage.py @@ -50,3 +50,8 @@ def extra_options(extra_vars=None): def extract_step(self): """Specifically use the overloaded variant from Cargo as is populates vendored sources with checksums.""" return Cargo.extract_step(self) + + def configure_step(self): + """Run configure for Cargo and PythonPackage""" + Cargo.configure_step(self) + PythonPackage.configure_step(self)