From 495802d229967771df5b64a2f79b88a0eaf00edb Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 18 Jan 2017 09:59:11 -0500 Subject: [PATCH] Implement dependency mapping for environment modules, galaxy packages, and Conda. --- lib/galaxy/tools/deps/resolvers/__init__.py | 69 +++++++++++++++ lib/galaxy/tools/deps/resolvers/conda.py | 11 ++- .../tools/deps/resolvers/galaxy_packages.py | 11 ++- lib/galaxy/tools/deps/resolvers/modules.py | 11 ++- test/unit/tools/test_tool_deps.py | 84 +++++++++++++++++-- 5 files changed, 172 insertions(+), 14 deletions(-) diff --git a/lib/galaxy/tools/deps/resolvers/__init__.py b/lib/galaxy/tools/deps/resolvers/__init__.py index 3c277abd8cd6..abc3ea503210 100644 --- a/lib/galaxy/tools/deps/resolvers/__init__.py +++ b/lib/galaxy/tools/deps/resolvers/__init__.py @@ -5,6 +5,9 @@ abstractproperty, ) +import yaml + +from galaxy.util import listify from galaxy.util.dictifiable import Dictifiable from ..requirements import ToolRequirement @@ -65,6 +68,72 @@ def _to_requirement(self, name, version=None): return ToolRequirement(name=name, type="package", version=version) +class MappableDependencyResolver: + """Mix this into a ``DependencyResolver`` to allow mapping files. + + Mapping files allow adapting generic requirements to specific local implementations. + """ + + def _setup_mapping(self, dependency_manager, **kwds): + mapping_files = dependency_manager.get_resolver_option(self, "mapping_file", explicit_resolver_options=kwds) + mappings = [] + if mapping_files: + mapping_files = listify(mapping_files) + for mapping_file in mapping_files: + mappings.extend(MappableDependencyResolver._mapping_file_to_list(mapping_file)) + self._mappings = mappings + + @staticmethod + def _mapping_file_to_list(mapping_file): + with open(mapping_file, "r") as f: + raw_mapping = yaml.load(f) + return map(RequirementMapping.from_dict, raw_mapping) + + def _expand_mappings(self, requirement): + for mapping in self._mappings: + if requirement.name == mapping.from_name: + if mapping.from_version is not None and mapping.from_version != requirement.version: + continue + + requirement = requirement.copy() + requirement.name = mapping.to_name + if mapping.to_version is not None: + requirement.version = mapping.to_version + + break + + return requirement + + +class RequirementMapping(object): + + def __init__(self, from_name, from_version, to_name, to_version): + self.from_name = from_name + self.from_version = from_version + self.to_name = to_name + self.to_version = to_version + + @staticmethod + def from_dict(raw_mapping): + from_raw = raw_mapping.get("from") + if isinstance(from_raw, dict): + from_name = from_raw.get("name") + from_version = str(from_raw.get("version")) + else: + from_name = from_raw + from_version = None + + to_raw = raw_mapping.get("to") + if isinstance(to_raw, dict): + to_name = to_raw.get("name", from_name) + to_version = str(to_raw.get("version")) + else: + to_name = to_raw + to_version = None + + return RequirementMapping(from_name, from_version, to_name, to_version) + + class SpecificationAwareDependencyResolver: """Mix this into a :class:`DependencyResolver` to implement URI specification matching. diff --git a/lib/galaxy/tools/deps/resolvers/conda.py b/lib/galaxy/tools/deps/resolvers/conda.py index ff78d8a14767..aefb460261e2 100644 --- a/lib/galaxy/tools/deps/resolvers/conda.py +++ b/lib/galaxy/tools/deps/resolvers/conda.py @@ -29,6 +29,7 @@ DependencyResolver, InstallableDependencyResolver, ListableDependencyResolver, + MappableDependencyResolver, MultipleDependencyResolver, NullDependency, SpecificationPatternDependencyResolver, @@ -42,7 +43,7 @@ log = logging.getLogger(__name__) -class CondaDependencyResolver(DependencyResolver, MultipleDependencyResolver, ListableDependencyResolver, InstallableDependencyResolver, SpecificationPatternDependencyResolver): +class CondaDependencyResolver(DependencyResolver, MultipleDependencyResolver, ListableDependencyResolver, InstallableDependencyResolver, SpecificationPatternDependencyResolver, MappableDependencyResolver): dict_collection_visible_keys = DependencyResolver.dict_collection_visible_keys + ['conda_prefix', 'versionless', 'ensure_channels', 'auto_install'] resolver_type = "conda" config_options = { @@ -57,6 +58,7 @@ class CondaDependencyResolver(DependencyResolver, MultipleDependencyResolver, Li _specification_pattern = re.compile(r"https\:\/\/anaconda.org\/\w+\/\w+") def __init__(self, dependency_manager, **kwds): + self._setup_mapping(dependency_manager, **kwds) self.versionless = _string_as_bool(kwds.get('versionless', 'false')) self.dependency_manager = dependency_manager @@ -146,7 +148,7 @@ def resolve_all(self, requirements, **kwds): conda_targets = [] for requirement in requirements: - requirement = self._expand_specs(requirement) + requirement = self._expand_requirement(requirement) version = requirement.version if self.versionless: @@ -186,7 +188,7 @@ def merged_environment_name(self, conda_targets): return conda_targets[0].install_environment def resolve(self, requirement, **kwds): - requirement = self._expand_specs(requirement) + requirement = self._expand_requirement(requirement) name, version, type = requirement.name, requirement.version, requirement.type # Check for conda just not being there, this way we can enable @@ -236,6 +238,9 @@ def resolve(self, requirement, **kwds): preserve_python_environment=preserve_python_environment, ) + def _expand_requirement(self, requirement): + return self._expand_specs(self._expand_mappings(requirement)) + def list_dependencies(self): for install_target in installed_conda_targets(self.conda_context): name = install_target.package diff --git a/lib/galaxy/tools/deps/resolvers/galaxy_packages.py b/lib/galaxy/tools/deps/resolvers/galaxy_packages.py index bbfb3e401688..8e1a11450300 100644 --- a/lib/galaxy/tools/deps/resolvers/galaxy_packages.py +++ b/lib/galaxy/tools/deps/resolvers/galaxy_packages.py @@ -16,6 +16,7 @@ Dependency, DependencyResolver, ListableDependencyResolver, + MappableDependencyResolver, NullDependency, ) @@ -103,9 +104,17 @@ def _galaxy_package_dep( self, path, version, name, exact ): return NullDependency(version=version, name=name) -class GalaxyPackageDependencyResolver(BaseGalaxyPackageDependencyResolver, ListableDependencyResolver): +class GalaxyPackageDependencyResolver(BaseGalaxyPackageDependencyResolver, ListableDependencyResolver, MappableDependencyResolver): resolver_type = "galaxy_packages" + def __init__(self, dependency_manager, **kwds): + super(GalaxyPackageDependencyResolver, self).__init__(dependency_manager, **kwds) + self._setup_mapping(dependency_manager, **kwds) + + def resolve(self, requirement, **kwds): + requirement = self._expand_mappings(requirement) + return super(GalaxyPackageDependencyResolver, self).resolve(requirement, **kwds) + def list_dependencies(self): base_path = self.base_path for package_name in listdir(base_path): diff --git a/lib/galaxy/tools/deps/resolvers/modules.py b/lib/galaxy/tools/deps/resolvers/modules.py index cb40ed9f4bc0..0cbbc5764f8d 100644 --- a/lib/galaxy/tools/deps/resolvers/modules.py +++ b/lib/galaxy/tools/deps/resolvers/modules.py @@ -13,7 +13,12 @@ from six import StringIO -from ..resolvers import Dependency, DependencyResolver, NullDependency +from ..resolvers import ( + Dependency, + DependencyResolver, + MappableDependencyResolver, + NullDependency, +) log = logging.getLogger( __name__ ) @@ -24,11 +29,12 @@ UNKNOWN_FIND_BY_MESSAGE = "ModuleDependencyResolver does not know how to find modules by [%s], find_by should be one of %s" -class ModuleDependencyResolver(DependencyResolver): +class ModuleDependencyResolver(DependencyResolver, MappableDependencyResolver): dict_collection_visible_keys = DependencyResolver.dict_collection_visible_keys + ['base_path', 'modulepath'] resolver_type = "modules" def __init__(self, dependency_manager, **kwds): + self._setup_mapping(dependency_manager, **kwds) self.versionless = _string_as_bool(kwds.get('versionless', 'false')) find_by = kwds.get('find_by', 'avail') prefetch = _string_as_bool(kwds.get('prefetch', DEFAULT_MODULE_PREFETCH)) @@ -52,6 +58,7 @@ def __default_modulespath(self): return module_path def resolve(self, requirement, **kwds): + requirement = self._expand_mappings(requirement) name, version, type = requirement.name, requirement.version, requirement.type if type != "package": diff --git a/test/unit/tools/test_tool_deps.py b/test/unit/tools/test_tool_deps.py index 8b35be2d887b..c2d674be020b 100644 --- a/test/unit/tools/test_tool_deps.py +++ b/test/unit/tools/test_tool_deps.py @@ -115,13 +115,7 @@ def __build_ts_test_package(base_path, script_contents=''): def test_module_dependency_resolver(): with __test_base_path() as temp_directory: - module_script = os.path.join(temp_directory, "modulecmd") - __write_script(module_script, '''#!/bin/sh -cat %s/example_output 1>&2; -''' % temp_directory) - with open(os.path.join(temp_directory, "example_output"), "w") as f: - # Subset of module avail from MSI cluster. - f.write(''' + module_script = _setup_module_command(temp_directory, ''' -------------------------- /soft/modules/modulefiles --------------------------- JAGS/3.2.0-gcc45 JAGS/3.3.0-gcc4.7.2 @@ -141,7 +135,7 @@ def test_module_dependency_resolver(): advisor/2013/update2 intel/11.1.080 mkl/10.2.5.035 advisor/2013/update3 intel/12.0 mkl/10.2.7.041 ''') - resolver = ModuleDependencyResolver(None, modulecmd=module_script) + resolver = ModuleDependencyResolver(_SimpleDependencyManager(), modulecmd=module_script) module = resolver.resolve( ToolRequirement( name="R", version=None, type="package" ) ) assert module.module_name == "R" assert module.module_version is None @@ -154,6 +148,74 @@ def test_module_dependency_resolver(): assert isinstance(module, NullDependency) +def test_module_resolver_with_mapping(): + with __test_base_path() as temp_directory: + module_script = _setup_module_command(temp_directory, ''' +-------------------------- /soft/modules/modulefiles --------------------------- +blast/2.24 +''') + mapping_file = os.path.join(temp_directory, "mapping") + with open(mapping_file, "w") as f: + f.write(''' +- from: blast+ + to: blast +''') + + resolver = ModuleDependencyResolver(_SimpleDependencyManager(), modulecmd=module_script, mapping_file=mapping_file) + module = resolver.resolve( ToolRequirement( name="blast+", version="2.24", type="package" ) ) + assert module.module_name == "blast" + assert module.module_version == "2.24", module.module_version + + +def test_module_resolver_with_mapping_versions(): + with __test_base_path() as temp_directory: + module_script = _setup_module_command(temp_directory, ''' +-------------------------- /soft/modules/modulefiles --------------------------- +blast/2.22.0-mpi +blast/2.23 +blast/2.24.0-mpi +''') + mapping_file = os.path.join(temp_directory, "mapping") + with open(mapping_file, "w") as f: + f.write(''' +- from: + name: blast+ + version: 2.24 + to: + name: blast + version: 2.24.0-mpi +- from: + name: blast + version: 2.22 + to: + version: 2.22.0-mpi +''') + + resolver = ModuleDependencyResolver(_SimpleDependencyManager(), modulecmd=module_script, mapping_file=mapping_file) + module = resolver.resolve( ToolRequirement( name="blast+", version="2.24", type="package" ) ) + assert module.module_name == "blast" + assert module.module_version == "2.24.0-mpi", module.module_version + + resolver = ModuleDependencyResolver(_SimpleDependencyManager(), modulecmd=module_script, mapping_file=mapping_file) + module = resolver.resolve( ToolRequirement( name="blast+", version="2.23", type="package" ) ) + assert isinstance(module, NullDependency) + + module = resolver.resolve( ToolRequirement( name="blast", version="2.22", type="package" ) ) + assert module.module_name == "blast" + assert module.module_version == "2.22.0-mpi", module.module_version + + +def _setup_module_command(temp_directory, contents): + module_script = os.path.join(temp_directory, "modulecmd") + __write_script(module_script, '''#!/bin/sh +cat %s/example_output 1>&2; +''' % temp_directory) + with open(os.path.join(temp_directory, "example_output"), "w") as f: + # Subset of module avail from MSI cluster. + f.write(contents) + return module_script + + def test_module_dependency(): with __test_base_path() as temp_directory: # Create mock modulecmd script that just exports a variable @@ -386,3 +448,9 @@ def __dependency_manager(xml_content): f.flush() dm = DependencyManager( default_base_path=base_path, conf_file=f.name ) yield dm + + +class _SimpleDependencyManager(object): + + def get_resolver_option(self, resolver, key, explicit_resolver_options={}): + return explicit_resolver_options.get(key)